From 0b283aad9cf7346483cb1f419f98f480e1b1ffb6 Mon Sep 17 00:00:00 2001 From: Dong-Ha Kim Date: Thu, 12 Sep 2024 01:06:55 +0700 Subject: [PATCH 1/6] feat: cron job cache relayer balances --- api/_cache.ts | 19 ++++++++++- api/_utils.ts | 37 ++++++++------------ api/cron-cache-balances.ts | 69 ++++++++++++++++++++++++++++++++++++++ vercel.json | 6 ++++ 4 files changed, 107 insertions(+), 24 deletions(-) create mode 100644 api/cron-cache-balances.ts diff --git a/api/_cache.ts b/api/_cache.ts index b170042e3..5f6391034 100644 --- a/api/_cache.ts +++ b/api/_cache.ts @@ -66,7 +66,7 @@ export async function getCachedValue( ttl: number, fetcher: () => Promise, parser?: (value: T) => T -) { +): Promise { const cachedValue = await redisCache.get(key); if (cachedValue) { return parser ? parser(cachedValue) : cachedValue; @@ -76,3 +76,20 @@ export async function getCachedValue( await redisCache.set(key, value, ttl); return value; } + +export function makeCacheGetterAndSetter( + key: string, + ttl: number, + fetcher: () => Promise, + parser?: (value: T) => T +) { + return { + get: async () => { + return getCachedValue(key, ttl, fetcher, parser); + }, + set: async () => { + const value = await fetcher(); + await redisCache.set(key, value, ttl); + }, + }; +} diff --git a/api/_utils.ts b/api/_utils.ts index 18c868466..58dcac2b5 100644 --- a/api/_utils.ts +++ b/api/_utils.ts @@ -54,7 +54,11 @@ import { relayerFeeCapitalCostConfig, } from "./_constants"; import { PoolStateOfUser, PoolStateResult } from "./_types"; -import { buildInternalCacheKey, getCachedValue } from "./_cache"; +import { + buildInternalCacheKey, + getCachedValue, + makeCacheGetterAndSetter, +} from "./_cache"; type LoggingUtility = sdk.relayFeeCalculator.Logger; type RpcProviderName = keyof typeof rpcProvidersJson.providers.urls; @@ -1085,7 +1089,11 @@ export const getCachedTokenBalance = async ( account: string, token: string ): Promise => { - const balance = await getCachedLatestBalance(Number(chainId), token, account); + const balance = await latestBalanceCache( + Number(chainId), + token, + account + ).get(); return balance; }; @@ -1896,34 +1904,17 @@ export function getCachedLatestBlock(chainId: number) { ); } -export function getCachedGasPrice(chainId: number) { - const ttlPerChain = { - default: 5, - [CHAIN_IDs.ARBITRUM]: 2, - }; - - return getCachedValue( - buildInternalCacheKey("gasPrice", chainId), - ttlPerChain[chainId] || ttlPerChain.default, - async () => { - const gasPrice = await getProvider(chainId).getGasPrice(); - return gasPrice.mul(2); - }, - (bnFromCache) => BigNumber.from(bnFromCache) - ); -} - -export function getCachedLatestBalance( +export function latestBalanceCache( chainId: number, tokenAddress: string, address: string ) { const ttlPerChain = { - default: 30, - [CHAIN_IDs.MAINNET]: 30, + default: 60, + [CHAIN_IDs.MAINNET]: 60, }; - return getCachedValue( + return makeCacheGetterAndSetter( buildInternalCacheKey("latestBalance", tokenAddress, chainId, address), ttlPerChain[chainId] || ttlPerChain.default, () => getBalance(chainId, address, tokenAddress), diff --git a/api/cron-cache-balances.ts b/api/cron-cache-balances.ts new file mode 100644 index 000000000..a195b7527 --- /dev/null +++ b/api/cron-cache-balances.ts @@ -0,0 +1,69 @@ +import { VercelResponse } from "@vercel/node"; +import { ethers } from "ethers"; +import { TypedVercelRequest } from "./_types"; + +import { + HUB_POOL_CHAIN_ID, + getLogger, + handleErrorCondition, + latestBalanceCache, +} from "./_utils"; + +import mainnetChains from "../src/data/chains_1.json"; + +const handler = async ( + _: TypedVercelRequest>, + response: VercelResponse +) => { + const logger = getLogger(); + logger.debug({ + at: "CronCacheBalances", + message: "Starting cron job...", + }); + try { + const { + REACT_APP_FULL_RELAYERS, // These are relayers running a full auto-rebalancing strategy. + REACT_APP_TRANSFER_RESTRICTED_RELAYERS, // These are relayers whose funds stay put. + } = process.env; + + const fullRelayers = !REACT_APP_FULL_RELAYERS + ? [] + : (JSON.parse(REACT_APP_FULL_RELAYERS) as string[]).map((relayer) => { + return ethers.utils.getAddress(relayer); + }); + const transferRestrictedRelayers = !REACT_APP_TRANSFER_RESTRICTED_RELAYERS + ? [] + : (JSON.parse(REACT_APP_TRANSFER_RESTRICTED_RELAYERS) as string[]).map( + (relayer) => { + return ethers.utils.getAddress(relayer); + } + ); + + // Skip cron job on testnet + if (HUB_POOL_CHAIN_ID !== 1) { + return; + } + + for (const chain of mainnetChains) { + for (const token of chain.inputTokens) { + const setTokenBalance = async (relayer: string) => { + await latestBalanceCache(chain.chainId, relayer, token.address).set(); + }; + await Promise.all([ + Promise.all(fullRelayers.map(setTokenBalance)), + Promise.all(transferRestrictedRelayers.map(setTokenBalance)), + ]); + } + } + + logger.debug({ + at: "CronCacheBalances", + message: "Finished", + }); + response.status(200); + } catch (error: unknown) { + return handleErrorCondition("cron-cache-balances", response, logger, error); + } +}; + +export default handler; diff --git a/vercel.json b/vercel.json index 4c51df1e3..264a5b0b1 100644 --- a/vercel.json +++ b/vercel.json @@ -1,5 +1,11 @@ { "devCommand": "yarn dev", + "crons": [ + { + "path": "/api/cron-cache-balances", + "schedule": "* * * * *" + } + ], "rewrites": [ { "source": "/api/deposit/status", From bbc2ea8aa17b8fa66e3eafcf8f6599b170e6b796 Mon Sep 17 00:00:00 2001 From: Dong-Ha Kim Date: Thu, 12 Sep 2024 01:55:07 +0700 Subject: [PATCH 2/6] fixup --- api/account-balance.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/account-balance.ts b/api/account-balance.ts index a1738f5cb..0ed3eade8 100644 --- a/api/account-balance.ts +++ b/api/account-balance.ts @@ -2,7 +2,7 @@ import { VercelResponse } from "@vercel/node"; import { assert, Infer, type, string } from "superstruct"; import { TypedVercelRequest } from "./_types"; import { - getCachedLatestBalance, + latestBalanceCache, getLogger, handleErrorCondition, validAddress, @@ -31,11 +31,11 @@ const handler = async ( let { token, account, chainId } = query; - const balance = await getCachedLatestBalance( + const balance = await latestBalanceCache( Number(chainId), token, account - ); + ).get(); const result = { balance: balance.toString(), account: account, From b997ae20588ae1b81a62c90a11f3eed09b1494bb Mon Sep 17 00:00:00 2001 From: Dong-Ha Kim Date: Thu, 12 Sep 2024 14:45:15 +0700 Subject: [PATCH 3/6] secure cron job --- api/cron-cache-balances.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/cron-cache-balances.ts b/api/cron-cache-balances.ts index a195b7527..f3f7ef377 100644 --- a/api/cron-cache-balances.ts +++ b/api/cron-cache-balances.ts @@ -12,7 +12,7 @@ import { import mainnetChains from "../src/data/chains_1.json"; const handler = async ( - _: TypedVercelRequest>, + request: TypedVercelRequest>, response: VercelResponse ) => { const logger = getLogger(); @@ -21,6 +21,14 @@ const handler = async ( message: "Starting cron job...", }); try { + const authHeader = (request.headers as any)?.get("authorization"); + if ( + !process.env.CRON_SECRET || + authHeader !== `Bearer ${process.env.CRON_SECRET}` + ) { + return response.status(401).json({ success: false }); + } + const { REACT_APP_FULL_RELAYERS, // These are relayers running a full auto-rebalancing strategy. REACT_APP_TRANSFER_RESTRICTED_RELAYERS, // These are relayers whose funds stay put. From d62a6f9a9cf063f3a2d4d415a35f5e6ba88b1c5e Mon Sep 17 00:00:00 2001 From: Dong-Ha Kim Date: Thu, 12 Sep 2024 15:03:39 +0700 Subject: [PATCH 4/6] fixup --- api/cron-cache-balances.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/cron-cache-balances.ts b/api/cron-cache-balances.ts index f3f7ef377..d5475aed2 100644 --- a/api/cron-cache-balances.ts +++ b/api/cron-cache-balances.ts @@ -21,7 +21,7 @@ const handler = async ( message: "Starting cron job...", }); try { - const authHeader = (request.headers as any)?.get("authorization"); + const authHeader = request.headers?.["authorization"]; if ( !process.env.CRON_SECRET || authHeader !== `Bearer ${process.env.CRON_SECRET}` From 1539f8b42ada56c27caeb097b2ddb4ce36ed5b12 Mon Sep 17 00:00:00 2001 From: Dong-Ha Kim Date: Thu, 12 Sep 2024 15:57:23 +0700 Subject: [PATCH 5/6] use batch account balance in cron --- api/_cache.ts | 6 ++++-- api/_utils.ts | 21 +++++++++++---------- api/account-balance.ts | 10 +++++----- api/cron-cache-balances.ts | 38 ++++++++++++++++++++++++++++---------- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/api/_cache.ts b/api/_cache.ts index 5f6391034..6f40d8d11 100644 --- a/api/_cache.ts +++ b/api/_cache.ts @@ -87,8 +87,10 @@ export function makeCacheGetterAndSetter( get: async () => { return getCachedValue(key, ttl, fetcher, parser); }, - set: async () => { - const value = await fetcher(); + set: async (value?: T) => { + if (!value) { + value = await fetcher(); + } await redisCache.set(key, value, ttl); }, }; diff --git a/api/_utils.ts b/api/_utils.ts index 58dcac2b5..4924a4194 100644 --- a/api/_utils.ts +++ b/api/_utils.ts @@ -1089,11 +1089,11 @@ export const getCachedTokenBalance = async ( account: string, token: string ): Promise => { - const balance = await latestBalanceCache( - Number(chainId), - token, - account - ).get(); + const balance = await latestBalanceCache({ + chainId: Number(chainId), + tokenAddress: token, + address: account, + }).get(); return balance; }; @@ -1904,11 +1904,12 @@ export function getCachedLatestBlock(chainId: number) { ); } -export function latestBalanceCache( - chainId: number, - tokenAddress: string, - address: string -) { +export function latestBalanceCache(params: { + chainId: number; + tokenAddress: string; + address: string; +}) { + const { chainId, tokenAddress, address } = params; const ttlPerChain = { default: 60, [CHAIN_IDs.MAINNET]: 60, diff --git a/api/account-balance.ts b/api/account-balance.ts index 0ed3eade8..071a2bc7e 100644 --- a/api/account-balance.ts +++ b/api/account-balance.ts @@ -31,11 +31,11 @@ const handler = async ( let { token, account, chainId } = query; - const balance = await latestBalanceCache( - Number(chainId), - token, - account - ).get(); + const balance = await latestBalanceCache({ + chainId: Number(chainId), + tokenAddress: token, + address: account, + }).get(); const result = { balance: balance.toString(), account: account, diff --git a/api/cron-cache-balances.ts b/api/cron-cache-balances.ts index d5475aed2..773388a52 100644 --- a/api/cron-cache-balances.ts +++ b/api/cron-cache-balances.ts @@ -1,9 +1,10 @@ import { VercelResponse } from "@vercel/node"; -import { ethers } from "ethers"; +import { BigNumber, ethers } from "ethers"; import { TypedVercelRequest } from "./_types"; import { HUB_POOL_CHAIN_ID, + getCachedTokenBalances, getLogger, handleErrorCondition, latestBalanceCache, @@ -52,16 +53,32 @@ const handler = async ( return; } + const allRelayers = [...fullRelayers, ...transferRestrictedRelayers]; for (const chain of mainnetChains) { - for (const token of chain.inputTokens) { - const setTokenBalance = async (relayer: string) => { - await latestBalanceCache(chain.chainId, relayer, token.address).set(); - }; - await Promise.all([ - Promise.all(fullRelayers.map(setTokenBalance)), - Promise.all(transferRestrictedRelayers.map(setTokenBalance)), - ]); - } + const batchResult = await getCachedTokenBalances( + chain.chainId, + allRelayers, + [ + ethers.constants.AddressZero, + ...chain.outputTokens.map((token) => token.address), + ] + ); + + await Promise.all( + allRelayers.map((relayer) => { + return Promise.all( + chain.outputTokens.map((token) => { + return latestBalanceCache({ + chainId: chain.chainId, + address: relayer, + tokenAddress: token.address, + }).set( + BigNumber.from(batchResult.balances[relayer][token.address]) + ); + }) + ); + }) + ); } logger.debug({ @@ -69,6 +86,7 @@ const handler = async ( message: "Finished", }); response.status(200); + response.send("OK"); } catch (error: unknown) { return handleErrorCondition("cron-cache-balances", response, logger, error); } From ec7b2cabff5e751ee3a48d6e7f45f6cc6debd8d8 Mon Sep 17 00:00:00 2001 From: Dong-Ha Kim Date: Thu, 12 Sep 2024 16:56:25 +0700 Subject: [PATCH 6/6] use rpc call directly --- api/cron-cache-balances.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/cron-cache-balances.ts b/api/cron-cache-balances.ts index 773388a52..d3bd1bb3f 100644 --- a/api/cron-cache-balances.ts +++ b/api/cron-cache-balances.ts @@ -4,7 +4,7 @@ import { TypedVercelRequest } from "./_types"; import { HUB_POOL_CHAIN_ID, - getCachedTokenBalances, + getBatchBalanceViaMulticall3, getLogger, handleErrorCondition, latestBalanceCache, @@ -55,7 +55,7 @@ const handler = async ( const allRelayers = [...fullRelayers, ...transferRestrictedRelayers]; for (const chain of mainnetChains) { - const batchResult = await getCachedTokenBalances( + const batchResult = await getBatchBalanceViaMulticall3( chain.chainId, allRelayers, [