Skip to content

Commit

Permalink
Merge pull request #55 from terminal-fi/cacheunipools
Browse files Browse the repository at this point in the history 
uniswap: add caching of pool data and initialized caches for faster startup
  • Loading branch information
@zviadm
zviadm authored Jul 20, 2023
2 parents 2ab7545 + d287b4e commit 6dcdc8d
Show file tree
Hide file tree
Showing 9 changed files with 334 additions and 196 deletions.
163 changes: 125 additions & 38 deletions src/registries/uniswapv2.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import path from "path"
import fs from "fs"
import BigNumber from "bignumber.js"
import Web3 from "web3"

Expand All @@ -8,6 +10,34 @@ import { PairUniswapV2 } from "../pairs/uniswapv2"
import { Registry } from "../registry"
import { initPairsAndFilterByWhitelist } from "../utils"
import { fastConcurrentMap } from "../utils/async"
import { fetchEvents } from "../utils/events"

import * as mainnetUbeswapCachedData from "../../tools/caches/swappa.uniswapv2.42220.0x62d5b84bE28a183aBB507E125B384122D2C25fAE.pairs.json"
import * as mainnetSushiSwapCachedData from "../../tools/caches/swappa.uniswapv2.42220.0xc35DADB65012eC5796536bD9864eD8773aBc74C4.pairs.json"
import * as mainnetCeloDexCachedData from "../../tools/caches/swappa.uniswapv2.42220.0x31bD38d982ccDf3C2D95aF45a3456d319f0Ee1b6.pairs.json"

interface UniV2Pair {
token0: string
token1: string
pair: string
}

interface CachedData {
blockN: number
pairs: UniV2Pair[]
}

const cachedDataGlobal: {[key: number]: {[key: string]: CachedData | undefined} | undefined} = {
42220: {
"0x62d5b84bE28a183aBB507E125B384122D2C25fAE": mainnetUbeswapCachedData,
"0xc35DADB65012eC5796536bD9864eD8773aBc74C4": mainnetSushiSwapCachedData,
"0x31bD38d982ccDf3C2D95aF45a3456d319f0Ee1b6": mainnetCeloDexCachedData,
}
}

const cacheDataFileName = (chainId: number, factoryAddr: Address) => {
return path.join("/tmp", `swappa.uniswapv2.${chainId}.${factoryAddr}.pairs.json`)
}

export class RegistryUniswapV2 extends Registry {
private factory: IUniswapV2Factory
Expand All @@ -19,8 +49,8 @@ export class RegistryUniswapV2 extends Registry {
private factoryAddr: Address,
private opts?: {
fixedFee?: BigNumber,
fetchUsing?: "PairEvents" | "RegistryHelper" | "TokenList"
registryHelperAddr?: Address
fetchUsingTokenList?: boolean,
},
) {
super(name)
Expand All @@ -30,51 +60,108 @@ export class RegistryUniswapV2 extends Registry {
}
}

findPairs = async (tokenWhitelist: Address[]): Promise<Pair[]> => {
findPairs = async (tokenWhitelist: Address[]): Promise<Pair[]> => {
const chainId = await this.web3.eth.getChainId()
let pairsFetched
if (this.helper) {
// registry helper contract is available for fast discovery of pairs
const nPairs = Number.parseInt(await this.factory.methods.allPairsLength().call())
const limit = 100
pairsFetched = []
for (let offset = 0; offset < nPairs; offset += limit) {
const result = await this.helper.methods.findPairs(this.factoryAddr, offset, limit).call()
for (let pairInfo of result) {
if (tokenWhitelist.indexOf(pairInfo.token0) === -1 && tokenWhitelist.indexOf(pairInfo.token1) === -1) {
continue
const fetchUsing = this.opts?.fetchUsing || "PairEvents"
switch (fetchUsing) {
case "RegistryHelper": {
// registry helper contract is available for fast discovery of pairs
const nPairs = Number.parseInt(await this.factory.methods.allPairsLength().call())
const limit = 100
const pairsFetched = []
for (let offset = 0; offset < nPairs; offset += limit) {
const result = await this.helper!.methods.findPairs(this.factoryAddr, offset, limit).call()
for (let pairInfo of result) {
if (tokenWhitelist.indexOf(pairInfo.token0) === -1 && tokenWhitelist.indexOf(pairInfo.token1) === -1) {
continue
}
pairsFetched.push(new PairUniswapV2(chainId, this.web3, pairInfo.pair, this.opts?.fixedFee))
}
pairsFetched.push(new PairUniswapV2(chainId, this.web3, pairInfo.pair, this.opts?.fixedFee))
}
return initPairsAndFilterByWhitelist(pairsFetched, tokenWhitelist)
}
} else if (this.opts?.fetchUsingTokenList) {
const pairsToFetch: {tokenA: Address, tokenB: Address}[] = []
for (let i = 0; i < tokenWhitelist.length - 1; i += 1) {
for (let j = i + 1; j < tokenWhitelist.length; j += 1) {
pairsToFetch.push({tokenA: tokenWhitelist[i], tokenB: tokenWhitelist[j]})
case "TokenList": {
const pairsToFetch: {tokenA: Address, tokenB: Address}[] = []
for (let i = 0; i < tokenWhitelist.length - 1; i += 1) {
for (let j = i + 1; j < tokenWhitelist.length; j += 1) {
pairsToFetch.push({tokenA: tokenWhitelist[i], tokenB: tokenWhitelist[j]})
}
}
const pairsFetched = await fastConcurrentMap(
10,
pairsToFetch,
async (toFetch) => {
const pairAddr = await this.factory.methods.getPair(toFetch.tokenA, toFetch.tokenB).call()
if (pairAddr === "0x0000000000000000000000000000000000000000") {
return null
}
return new PairUniswapV2(chainId, this.web3, pairAddr, this.opts?.fixedFee)
})
return initPairsAndFilterByWhitelist(pairsFetched.filter((p) => p !== null) as Pair[], tokenWhitelist)
}
pairsFetched = await fastConcurrentMap(
10,
pairsToFetch,
async (toFetch) => {
const pairAddr = await this.factory.methods.getPair(toFetch.tokenA, toFetch.tokenB).call()
if (pairAddr === "0x0000000000000000000000000000000000000000") {
return null
case "PairEvents": {
let cachedData = cachedDataGlobal[chainId]?.[this.factory.options.address]
const cachedDataFile = cacheDataFileName(chainId, this.factory.options.address)
try {
if (fs.existsSync(cachedDataFile)) {
const cachedDataFromFile: CachedData = JSON.parse(fs.readFileSync(cachedDataFile).toString())
if (!cachedData || cachedDataFromFile.blockN > cachedData.blockN) {
cachedData = cachedDataFromFile
}
}
return new PairUniswapV2(chainId, this.web3, pairAddr, this.opts?.fixedFee)
})
} else {
const nPairs = Number.parseInt(await this.factory.methods.allPairsLength().call())
pairsFetched = await fastConcurrentMap(
10,
[...Array(nPairs).keys()],
async (idx) => {
const pairAddr = await this.factory.methods.allPairs(idx).call()
return new PairUniswapV2(chainId, this.web3, pairAddr, this.opts?.fixedFee)
} catch (e) {
console.warn(`UniV2Registry: Error while trying to read cache file: ${cachedDataFile}: ${e}`)
}

let pairs: UniV2Pair[] = []
let fromBlock = 0
if (cachedData) {
pairs.push(...cachedData.pairs)
fromBlock = cachedData.blockN + 1
}
const pairAddrSet = new Set<string>()
pairs = pairs.filter((p) => {
if (pairAddrSet.has(p.pair)) {
return false
}
pairAddrSet.add(p.pair)
return true
})
const endBlock = await this.web3.eth.getBlockNumber()
await fetchEvents(
async (fromBlock, toBlock) => {
const events = await this.factory.getPastEvents("PairCreated", {fromBlock, toBlock})
return events.map(
(v) => ({
token0: v.returnValues.token0 as string,
token1: v.returnValues.token1 as string,
pair: v.returnValues.pair as string,
} as UniV2Pair)
)
},
fromBlock,
endBlock,
(fromBlock: number, toBlock: number, batch: UniV2Pair[]) => {
batch = batch.filter((p) => {
if (pairAddrSet.has(p.pair)) {
return false
}
pairAddrSet.add(p.pair)
return true
})
pairs.push(...batch)
const toCache: CachedData = {
blockN: toBlock,
pairs: pairs,
}
fs.writeFileSync(cachedDataFile, JSON.stringify(toCache))
},
"UNIV2REGISTRY",
)
pairs = pairs.filter((p) => tokenWhitelist.indexOf(p.token0) >= 0 && tokenWhitelist.indexOf(p.token1) >= 0);
const pairsFetched = pairs.map((p) => new PairUniswapV2(chainId, this.web3, p.pair))
return initPairsAndFilterByWhitelist(pairsFetched, tokenWhitelist)
}
}
const pairs = pairsFetched.filter((p) => p !== null) as Pair[]
return initPairsAndFilterByWhitelist(pairs, tokenWhitelist)
}
}
Loading

0 comments on commit 6dcdc8d

Please sign in to comment.