From 19313c5eba3c248256e126160d48e818ad665d19 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:44:16 -0800 Subject: [PATCH] [IND-511] e2e test for limit order matching (#901) --- e2e-testing/README.md | 30 ++ e2e-testing/__tests__/helpers/constants.ts | 56 ++++ e2e-testing/__tests__/helpers/types.ts | 9 + e2e-testing/__tests__/helpers/utils.ts | 37 ++ e2e-testing/__tests__/orders.test.ts | 315 ++++++++++++++++++ e2e-testing/__tests__/transfers.test.ts | 85 +++-- e2e-testing/clear-all-kafa-topics.sh | 17 + e2e-testing/package.json | 3 + e2e-testing/pnpm-lock.yaml | 19 ++ .../__tests__/stores/fill-table.test.ts | 44 +++ .../__tests__/stores/order-table.test.ts | 18 + indexer/packages/postgres/src/db/helpers.ts | 16 +- .../postgres/src/stores/fill-table.ts | 16 + .../postgres/src/stores/order-table.ts | 19 ++ .../postgres/src/types/query-types.ts | 2 + .../__tests__/lib/candles-generator.test.ts | 13 +- .../ender/src/lib/candles-generator.ts | 3 +- protocol/Makefile | 5 +- 18 files changed, 653 insertions(+), 54 deletions(-) create mode 100644 e2e-testing/__tests__/helpers/types.ts create mode 100644 e2e-testing/__tests__/orders.test.ts create mode 100755 e2e-testing/clear-all-kafa-topics.sh diff --git a/e2e-testing/README.md b/e2e-testing/README.md index b3eeffd029..3aab0154f4 100644 --- a/e2e-testing/README.md +++ b/e2e-testing/README.md @@ -16,3 +16,33 @@ In another terminal, run ``` pnpm build && pnpm test ``` + +#### Quickest way to reset the network/clear all Indexer data sources without rebuilding from scratch + +Go to Docker Desktop + +Stop all containers + +Delete all dydxprotocold* containers + +Reset the protocol by doing the following: +``` +cd ../protocol +make reset-chain +``` + +Delete the postgres container. + +Restart the Kafka container. + +Clear all Kafka topics: +``` +docker cp remove-all-kafka-msgs.sh :/opt/kafka +docker exec -it /bin/bash +./remove-all-kafka-msgs.sh +``` + +Restart all containers: +``` +docker compose -f docker-compose-e2e-test.yml up +``` diff --git a/e2e-testing/__tests__/helpers/constants.ts b/e2e-testing/__tests__/helpers/constants.ts index 37b2d4f20c..6f77771115 100644 --- a/e2e-testing/__tests__/helpers/constants.ts +++ b/e2e-testing/__tests__/helpers/constants.ts @@ -1,2 +1,58 @@ +import { + IPlaceOrder, Order_Side, Order_TimeInForce, OrderFlags, +} from '@dydxprotocol/v4-client-js'; +import Long from 'long'; + +import { OrderDetails } from './types'; + export const DYDX_LOCAL_ADDRESS = 'dydx199tqg4wdlnu4qjlxchpd7seg454937hjrknju4'; export const DYDX_LOCAL_MNEMONIC = 'merge panther lobster crazy road hollow amused security before critic about cliff exhibit cause coyote talent happy where lion river tobacco option coconut small'; +export const DYDX_LOCAL_ADDRESS_2 = 'dydx10fx7sy6ywd5senxae9dwytf8jxek3t2gcen2vs'; +export const DYDX_LOCAL_MNEMONIC_2 = 'color habit donor nurse dinosaur stable wonder process post perfect raven gold census inside worth inquiry mammal panic olive toss shadow strong name drum'; + +export const MNEMONIC_TO_ADDRESS: Record = { + [DYDX_LOCAL_MNEMONIC]: DYDX_LOCAL_ADDRESS, + [DYDX_LOCAL_MNEMONIC_2]: DYDX_LOCAL_ADDRESS_2, +}; + +export const ADDRESS_TO_MNEMONIC: Record = { + [DYDX_LOCAL_ADDRESS]: DYDX_LOCAL_MNEMONIC, + [DYDX_LOCAL_ADDRESS_2]: DYDX_LOCAL_MNEMONIC_2, +}; + +export const PERPETUAL_PAIR_BTC_USD: number = 0; +export const quantums: Long = new Long(1_000_000_000); +export const subticks: Long = new Long(1_000_000_000); + +export const defaultOrder: IPlaceOrder = { + clientId: 0, + orderFlags: OrderFlags.SHORT_TERM, + clobPairId: PERPETUAL_PAIR_BTC_USD, + side: Order_Side.SIDE_BUY, + quantums, + subticks, + timeInForce: Order_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + reduceOnly: false, + clientMetadata: 0, +}; + +export const orderDetails: OrderDetails[] = [ + { + mnemonic: DYDX_LOCAL_MNEMONIC, + timeInForce: 0, + orderFlags: 64, + side: 1, + clobPairId: PERPETUAL_PAIR_BTC_USD, + quantums: 10000000, + subticks: 5000000000, + }, + { + mnemonic: DYDX_LOCAL_MNEMONIC_2, + timeInForce: 0, + orderFlags: 64, + side: 2, + clobPairId: PERPETUAL_PAIR_BTC_USD, + quantums: 5000000, + subticks: 5000000000, + }, +]; diff --git a/e2e-testing/__tests__/helpers/types.ts b/e2e-testing/__tests__/helpers/types.ts new file mode 100644 index 0000000000..44e1b49664 --- /dev/null +++ b/e2e-testing/__tests__/helpers/types.ts @@ -0,0 +1,9 @@ +export type OrderDetails = { + mnemonic: string; + timeInForce: number; + orderFlags: number; + side: number; + clobPairId: number; + quantums: number; + subticks: number; +}; diff --git a/e2e-testing/__tests__/helpers/utils.ts b/e2e-testing/__tests__/helpers/utils.ts index c5529d7041..6bd83059bb 100644 --- a/e2e-testing/__tests__/helpers/utils.ts +++ b/e2e-testing/__tests__/helpers/utils.ts @@ -1,3 +1,40 @@ +import { IPlaceOrder, Network, SocketClient } from '@dydxprotocol/v4-client-js'; +import Long from 'long'; + +import { defaultOrder } from './constants'; +import { OrderDetails } from './types'; + export async function sleep(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } + +export function createModifiedOrder( + order: OrderDetails, +): IPlaceOrder { + const modifiedOrder: IPlaceOrder = defaultOrder; + modifiedOrder.clientId = Math.floor(Math.random() * 1000000000); + modifiedOrder.goodTilBlock = 0; + modifiedOrder.clobPairId = order.clobPairId; + modifiedOrder.timeInForce = order.timeInForce; + modifiedOrder.reduceOnly = false; + modifiedOrder.orderFlags = order.orderFlags; + modifiedOrder.side = order.side; + modifiedOrder.quantums = Long.fromNumber(order.quantums); + modifiedOrder.subticks = Long.fromNumber(order.subticks); + return modifiedOrder; +} + +export function connectAndValidateSocketClient(validateMessage: Function): void { + const mySocket = new SocketClient( + Network.local().indexerConfig, + () => {}, + () => {}, + (message) => { + if (typeof message.data === 'string') { + const data = JSON.parse(message.data as string); + validateMessage(data, mySocket); + } + }, + ); + mySocket.connect(); +} diff --git a/e2e-testing/__tests__/orders.test.ts b/e2e-testing/__tests__/orders.test.ts new file mode 100644 index 0000000000..64f72c78a9 --- /dev/null +++ b/e2e-testing/__tests__/orders.test.ts @@ -0,0 +1,315 @@ +import { + BECH32_PREFIX, + HeightResponse, + IndexerClient, + IPlaceOrder, + LocalWallet, + Network, + SocketClient, + SubaccountInfo, + ValidatorClient, +} from '@dydxprotocol/v4-client-js'; +import { + DYDX_LOCAL_ADDRESS, + DYDX_LOCAL_ADDRESS_2, + DYDX_LOCAL_MNEMONIC, + DYDX_LOCAL_MNEMONIC_2, + orderDetails, + PERPETUAL_PAIR_BTC_USD, +} from './helpers/constants'; +import { DateTime } from 'luxon'; +import * as utils from './helpers/utils'; +import { connectAndValidateSocketClient, createModifiedOrder } from './helpers/utils'; +import { + CandleResolution, + FillTable, + FillType, + helpers, + IsoString, + Liquidity, + OrderSide, + OrderTable, + SubaccountTable, +} from '@dydxprotocol-indexer/postgres'; + +async function placeOrder( + mnemonic: string, + order: IPlaceOrder, +): Promise { + const wallet = await LocalWallet.fromMnemonic(mnemonic, BECH32_PREFIX); + const client = await ValidatorClient.connect(Network.local().validatorConfig); + + const subaccount = new SubaccountInfo(wallet, 0); + const modifiedOrder: IPlaceOrder = order; + if (order.orderFlags !== 0) { + // cancel the order 30 seconds from now + modifiedOrder.goodTilBlock = 0; + const now = new Date(); + const millisecondsPerSecond = 1000; + const interval = 30 * millisecondsPerSecond; + const future = new Date(now.valueOf() + interval); + modifiedOrder.goodTilBlockTime = Math.round(future.getTime() / 1000); + } else { + modifiedOrder.goodTilBlockTime = 0; + } + + await client.post.placeOrderObject( + subaccount, + modifiedOrder, + ); +} + +describe('orders', () => { + it('test orders', async () => { + const indexerClient = new IndexerClient(Network.local().indexerConfig); + const heightResp: HeightResponse = await indexerClient.utility.getHeight(); + const height: number = heightResp.height; + connectAndValidateSocketClient(validateOrders); + + // place all orders + for (const order of orderDetails) { + const modifiedOrder: IPlaceOrder = createModifiedOrder(order); + + await placeOrder(order.mnemonic, modifiedOrder); + } + const candleStart: IsoString = helpers.calculateNormalizedCandleStartTime( + DateTime.utc(), + CandleResolution.ONE_MINUTE, + ).toISO(); + + await utils.sleep(10000); // wait 10s for orders to be placed & matched + const [wallet, wallet2] = await Promise.all([ + LocalWallet.fromMnemonic(DYDX_LOCAL_MNEMONIC, BECH32_PREFIX), + LocalWallet.fromMnemonic(DYDX_LOCAL_MNEMONIC_2, BECH32_PREFIX), + ]); + + const subaccountId = SubaccountTable.uuid(wallet.address!, 0); + const subaccountId2 = SubaccountTable.uuid(wallet2.address!, 0); + const [makerOrders, takerOrders] = await Promise.all([ + OrderTable.findBySubaccountIdAndClobPairAfterHeight( + subaccountId, + PERPETUAL_PAIR_BTC_USD.toString(), + height, + ), + OrderTable.findBySubaccountIdAndClobPairAfterHeight( + subaccountId2, + PERPETUAL_PAIR_BTC_USD.toString(), + height, + ), + ]); + expect(makerOrders).toHaveLength(1); + expect(takerOrders).toHaveLength(1); + + const [makerFills, takerFills] = await Promise.all([ + FillTable.findAll( + { + subaccountId: [subaccountId], + createdOnOrAfterHeight: height.toString(), + }, + [], + {}, + ), + FillTable.findAll( + { + subaccountId: [subaccountId2], + createdOnOrAfterHeight: height.toString(), + }, + [], + {}, + ), + ]); + + expect(makerFills.length).toEqual(1); + expect(makerFills[0]).toEqual(expect.objectContaining({ + subaccountId, + side: OrderSide.BUY, + liquidity: Liquidity.MAKER, + type: FillType.LIMIT, + clobPairId: '0', + orderId: makerOrders[0].id, + size: '0.0005', + price: '50000', + quoteAmount: '25', + clientMetadata: '0', + fee: '-0.00275', + })); + + expect(takerFills.length).toEqual(1); + expect(takerFills[0]).toEqual(expect.objectContaining({ + subaccountId: subaccountId2, + side: OrderSide.SELL, + liquidity: Liquidity.TAKER, + type: FillType.LIMIT, + clobPairId: '0', + orderId: takerOrders[0].id, + size: '0.0005', + price: '50000', + quoteAmount: '25', + clientMetadata: '0', + fee: '0.0125', + })); + + // Check API /v4/orders endpoint + const [ordersResponse, ordersResponse2] = await Promise.all([ + indexerClient.account.getSubaccountOrders(DYDX_LOCAL_ADDRESS, + 0, + undefined, + undefined, + undefined, + undefined, + undefined, + 10, + undefined, + undefined, + true), + indexerClient.account.getSubaccountOrders(DYDX_LOCAL_ADDRESS_2, + 0, + undefined, + undefined, + undefined, + undefined, + undefined, + 10, + undefined, + undefined, + true), + ]); + expect(ordersResponse[0]).toEqual( + expect.objectContaining({ + subaccountId: SubaccountTable.uuid(DYDX_LOCAL_ADDRESS, 0), + clobPairId: '0', + side: 'BUY', + size: '0.001', + totalFilled: '0.0005', + price: '50000', + type: 'LIMIT', + timeInForce: 'GTT', + reduceOnly: false, + orderFlags: '64', + postOnly: false, + ticker: 'BTC-USD', + }), + ); + expect(ordersResponse2[0]).toEqual( + expect.objectContaining({ + subaccountId: SubaccountTable.uuid(DYDX_LOCAL_ADDRESS_2, 0), + clobPairId: '0', + side: 'SELL', + size: '0.0005', + totalFilled: '0.0005', + price: '50000', + type: 'LIMIT', + status: 'FILLED', + timeInForce: 'GTT', + reduceOnly: false, + orderFlags: '64', + postOnly: false, + ticker: 'BTC-USD', + }), + ); + + // Check API /v4/perpetualPositions endpoint + const [response, response2] = await Promise.all([ + indexerClient.account.getSubaccountPerpetualPositions(DYDX_LOCAL_ADDRESS, 0), + indexerClient.account.getSubaccountPerpetualPositions(DYDX_LOCAL_ADDRESS_2, 0), + ]); + expect(response.positions.length).toEqual(1); + expect(response.positions[0]).toEqual( + expect.objectContaining({ + market: 'BTC-USD', + status: 'OPEN', + side: 'LONG', + entryPrice: '50000', + }), + ); + expect(response2.positions.length).toEqual(1); + expect(response2.positions[0]).toEqual( + expect.objectContaining({ + market: 'BTC-USD', + status: 'OPEN', + side: 'SHORT', + entryPrice: '50000', + }), + ); + + // Check API /v4/orderbooks endpoint + const orderbooksResponse = await indexerClient.markets.getPerpetualMarketOrderbook('BTC-USD'); + expect(orderbooksResponse).toEqual( + expect.objectContaining({ + bids: [ + { + price: '50000', + size: '0.0005', + }, + ], + asks: [], + }), + ); + + // Check API /v4/candles endpoint + const candlesResponse = await indexerClient.markets.getPerpetualMarketCandles( + 'BTC-USD', + CandleResolution.ONE_MINUTE, + undefined, + undefined, + 1, + ); + expect(candlesResponse.candles[0]).toEqual( + expect.objectContaining({ + startedAt: candleStart, + ticker: 'BTC-USD', + resolution: '1MIN', + low: '50000', + high: '50000', + open: '50000', + close: '50000', + baseTokenVolume: '0.0005', + usdVolume: '25', + trades: 1, + }), + ); + }); + + function validateOrders(data: any, socketClient: SocketClient): void { + if (data.type === 'connected') { + socketClient.subscribeToSubaccount(DYDX_LOCAL_ADDRESS, 0); + } else if (data.type === 'subscribed') { + expect(data.channel).toEqual('v4_subaccounts'); + expect(data.id).toEqual(`${DYDX_LOCAL_ADDRESS}/0`); + expect(data.contents.subaccount).toEqual( + expect.objectContaining({ + address: DYDX_LOCAL_ADDRESS, + subaccountNumber: 0, + }), + ); + } else if (data.type === 'channel_data' && data.contents.perpetualPositions) { + expect(data.contents.perpetualPositions[0]).toEqual( + expect.objectContaining({ + address: DYDX_LOCAL_ADDRESS, + subaccountNumber: 0, + market: 'BTC-USD', + side: 'LONG', + status: 'OPEN', + netFunding: '0', + exitPrice: null, + }), + ); + } else if (data.type === 'channel_data' && data.contents.fills) { + expect(data.contents.fills[0]).toEqual( + expect.objectContaining({ + fee: '-0.00275', + side: 'BUY', + size: '0.0005', + type: 'LIMIT', + price: '50000', + liquidity: 'MAKER', + clobPairId: '0', + quoteAmount: '25', + subaccountId: SubaccountTable.uuid(DYDX_LOCAL_ADDRESS, 0), + clientMetadata: '0', + ticker: 'BTC-USD', + }), + ); + } + } +}); diff --git a/e2e-testing/__tests__/transfers.test.ts b/e2e-testing/__tests__/transfers.test.ts index 8c5ce65f51..898c77ed02 100644 --- a/e2e-testing/__tests__/transfers.test.ts +++ b/e2e-testing/__tests__/transfers.test.ts @@ -1,6 +1,7 @@ import Long from 'long'; import { BECH32_PREFIX, + HeightResponse, IndexerClient, LocalWallet, Network, @@ -16,15 +17,19 @@ import { TransferTable, } from '@dydxprotocol-indexer/postgres'; import * as utils from './helpers/utils'; +import Big from 'big.js'; import { DYDX_LOCAL_ADDRESS, DYDX_LOCAL_MNEMONIC } from './helpers/constants'; +import { connectAndValidateSocketClient } from './helpers/utils'; describe('transfers', () => { it('test deposit', async () => { - connectAndValidateSocketClient(); + connectAndValidateSocketClient(validateTransfers); const wallet = await LocalWallet.fromMnemonic(DYDX_LOCAL_MNEMONIC, BECH32_PREFIX); const validatorClient = await ValidatorClient.connect(Network.local().validatorConfig); const indexerClient = new IndexerClient(Network.local().indexerConfig); + const heightResp: HeightResponse = await indexerClient.utility.getHeight(); + const height: number = heightResp.height; const subaccount = new SubaccountInfo(wallet, 0); @@ -45,12 +50,15 @@ describe('transfers', () => { ); // TODO(IND-547): investigate deterministically advancing network height - await utils.sleep(5000); // wait 5s for deposit to complete + await utils.sleep(10000); // wait 10s for deposit to complete const defaultSubaccountId: string = SubaccountTable.uuid(wallet.address!, 0); // Check DB const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId( - { subaccountId: [defaultSubaccountId] }, + { + subaccountId: [defaultSubaccountId], + createdAfterHeight: height.toString(), + }, [], { orderBy: [[TransferColumns.id, Ordering.ASC]], }); @@ -90,49 +98,36 @@ describe('transfers', () => { expect(assetPosResp).not.toBeNull(); const usdcPositionSizeAfter = assetPosResp.positions[0].size; // expect usdcPositionSizeAfter to be usdcPositionSizeBefore + 10 - expect(usdcPositionSizeAfter).toEqual((parseInt(usdcPositionSizeBefore, 10) + 10).toString()); + expect(usdcPositionSizeAfter).toEqual(new Big(usdcPositionSizeBefore).plus(10).toString()); }); - function connectAndValidateSocketClient(): void { - const mySocket = new SocketClient( - Network.local().indexerConfig, - () => { - }, - () => { - }, - (message) => { - if (typeof message.data === 'string') { - const data = JSON.parse(message.data as string); - if (data.type === 'connected') { - mySocket.subscribeToSubaccount(DYDX_LOCAL_ADDRESS, 0); - } else if (data.type === 'subscribed') { - expect(data.channel).toEqual('v4_subaccounts'); - expect(data.id).toEqual(`${DYDX_LOCAL_ADDRESS}/0`); - expect(data.contents.subaccount).toEqual( - expect.objectContaining({ - address: DYDX_LOCAL_ADDRESS, - subaccountNumber: 0, - }), - ); - } else if (data.type === 'channel_data' && data.contents.transfers) { - expect(data.contents.transfers).toEqual( - expect.objectContaining({ - sender: { - address: DYDX_LOCAL_ADDRESS, - }, - recipient: { - address: DYDX_LOCAL_ADDRESS, - subaccountNumber: 0, - }, - size: '10', - symbol: 'USDC', - type: 'DEPOSIT', - }), - ); - } - } - }, - ); - mySocket.connect(); + function validateTransfers(data: any, socketClient: SocketClient): void { + if (data.type === 'connected') { + socketClient.subscribeToSubaccount(DYDX_LOCAL_ADDRESS, 0); + } else if (data.type === 'subscribed') { + expect(data.channel).toEqual('v4_subaccounts'); + expect(data.id).toEqual(`${DYDX_LOCAL_ADDRESS}/0`); + expect(data.contents.subaccount).toEqual( + expect.objectContaining({ + address: DYDX_LOCAL_ADDRESS, + subaccountNumber: 0, + }), + ); + } else if (data.type === 'channel_data' && data.contents.transfers) { + expect(data.contents.transfers).toEqual( + expect.objectContaining({ + sender: { + address: DYDX_LOCAL_ADDRESS, + }, + recipient: { + address: DYDX_LOCAL_ADDRESS, + subaccountNumber: 0, + }, + size: '10', + symbol: 'USDC', + type: 'DEPOSIT', + }), + ); + } } }); diff --git a/e2e-testing/clear-all-kafa-topics.sh b/e2e-testing/clear-all-kafa-topics.sh new file mode 100755 index 0000000000..aea3cde28b --- /dev/null +++ b/e2e-testing/clear-all-kafa-topics.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Kafka Zookeeper address +ZOOKEEPER="localhost:2181" + +# Topics to process +declare -a topics=("to-ender" "to-vulcan" "to-websockets-orderbooks" "to-websockets-subaccounts" "to-websockets-trades" "to-websockets-markets" "to-websockets-candles") + +for topic in "${topics[@]}" +do + echo "Deleting topic: $topic" + kafka-topics.sh --zookeeper $ZOOKEEPER --delete --topic $topic || { echo "Failed to delete topic: $topic"; exit 1; } + echo "Creating topic: $topic" + kafka-topics.sh --create --zookeeper $ZOOKEEPER --replication-factor 1 --partitions 1 --topic $topic || { echo "Failed to create topic: $topic"; exit 1; } +done + +echo "Topic processing completed." diff --git a/e2e-testing/package.json b/e2e-testing/package.json index 4199f0ded0..a8555832e9 100644 --- a/e2e-testing/package.json +++ b/e2e-testing/package.json @@ -27,12 +27,15 @@ "kafkajs": "^2.1.0", "lodash": "^4.17.21", "long": "5.2.3", + "luxon": "3.0.1", "yargs": "^13.3.0" }, "devDependencies": { "@dydxprotocol-indexer/dev": "file:../indexer/packages/dev", + "@types/big.js": "^6.1.5", "@types/jest": "^28.1.4", "@types/long": "4.0.2", + "@types/luxon": "3.0.0", "@types/node": "^18.0.3", "@types/ws": "8.5.4", "@types/yargs": "^16.0.0", diff --git a/e2e-testing/pnpm-lock.yaml b/e2e-testing/pnpm-lock.yaml index c61aa332e2..d89e81ce10 100644 --- a/e2e-testing/pnpm-lock.yaml +++ b/e2e-testing/pnpm-lock.yaml @@ -8,8 +8,10 @@ specifiers: '@dydxprotocol-indexer/v4-proto-parser': file:../indexer/packages/v4-proto-parser '@dydxprotocol-indexer/v4-protos': file:../indexer/packages/v4-protos '@dydxprotocol/v4-client-js': ^1.0.12 + '@types/big.js': ^6.1.5 '@types/jest': ^28.1.4 '@types/long': 4.0.2 + '@types/luxon': 3.0.0 '@types/node': ^18.0.3 '@types/ws': 8.5.4 '@types/yargs': ^16.0.0 @@ -28,6 +30,7 @@ specifiers: kafkajs: ^2.1.0 lodash: ^4.17.21 long: 5.2.3 + luxon: 3.0.1 ts-node: ^10.8.2 tsconfig-paths: ^4.0.0 typescript: 4.7.4 @@ -45,12 +48,15 @@ dependencies: kafkajs: 2.2.4 lodash: 4.17.21 long: 5.2.3 + luxon: 3.0.1 yargs: 13.3.2 devDependencies: '@dydxprotocol-indexer/dev': link:../indexer/packages/dev + '@types/big.js': 6.2.2 '@types/jest': 28.1.8 '@types/long': 4.0.2 + '@types/luxon': 3.0.0 '@types/node': 18.19.3 '@types/ws': 8.5.4 '@types/yargs': 16.0.9 @@ -1080,6 +1086,10 @@ packages: '@babel/types': 7.23.6 dev: true + /@types/big.js/6.2.2: + resolution: {integrity: sha512-e2cOW9YlVzFY2iScnGBBkplKsrn2CsObHQ2Hiw4V1sSyiGbgWL8IyqE3zFi1Pt5o1pdAtYkDAIsF3KKUPjdzaA==} + dev: true + /@types/eslint-visitor-keys/1.0.0: resolution: {integrity: sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==} dev: true @@ -1124,6 +1134,10 @@ packages: /@types/long/4.0.2: resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} + /@types/luxon/3.0.0: + resolution: {integrity: sha512-Lx+EZoJxUKw4dp8uei9XiUVNlgkYmax5+ovqt6Xf3LzJOnWhlfJw/jLBmqfGVwOP/pDr4HT8bI1WtxK0IChMLw==} + dev: true + /@types/node/18.15.13: resolution: {integrity: sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==} dev: false @@ -3706,6 +3720,11 @@ packages: yallist: 4.0.0 dev: true + /luxon/3.0.1: + resolution: {integrity: sha512-hF3kv0e5gwHQZKz4wtm4c+inDtyc7elkanAsBq+fundaCdUBNJB1dHEGUZIM6SfSBUlbVFduPwEtNjFK8wLtcw==} + engines: {node: '>=12'} + dev: false + /make-dir/4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} diff --git a/indexer/packages/postgres/__tests__/stores/fill-table.test.ts b/indexer/packages/postgres/__tests__/stores/fill-table.test.ts index 7cdec9dac4..d31ed9e177 100644 --- a/indexer/packages/postgres/__tests__/stores/fill-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/fill-table.test.ts @@ -166,6 +166,50 @@ describe('Fill store', () => { checkLengthAndContains(fills, expectedLength, expectedFill); }); + it.each([ + [1, 1, defaultFill], + [0, 1, defaultFill], + [-1, 0, undefined], + ])('Successfuly finds Fill with createdOnOrAfter, delta %d seconds', async ( + deltaSeconds: number, + expectedLength: number, + expectedFill?: FillCreateObject, + ) => { + await FillTable.create(defaultFill); + + const fills: FillFromDatabase[] = await FillTable.findAll( + { + createdOnOrAfter: createdDateTime.minus({ seconds: deltaSeconds }).toISO(), + }, + [], + { readReplica: true }, + ); + + checkLengthAndContains(fills, expectedLength, expectedFill); + }); + + it.each([ + [1, 1, defaultFill], + [0, 1, defaultFill], + [-1, 0, undefined], + ])('Successfuly finds Fill with createdOnOrAfterHeight, delta %d blocks', async ( + deltaBlocks: number, + expectedLength: number, + expectedFill?: FillCreateObject, + ) => { + await FillTable.create(defaultFill); + + const fills: FillFromDatabase[] = await FillTable.findAll( + { + createdOnOrAfterHeight: Big(createdHeight).minus(deltaBlocks).toFixed(), + }, + [], + { readReplica: true }, + ); + + checkLengthAndContains(fills, expectedLength, expectedFill); + }); + it('Successfully finds a Fill', async () => { await FillTable.create(defaultFill); diff --git a/indexer/packages/postgres/__tests__/stores/order-table.test.ts b/indexer/packages/postgres/__tests__/stores/order-table.test.ts index 1bb4fb2833..1cd9e26a60 100644 --- a/indexer/packages/postgres/__tests__/stores/order-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/order-table.test.ts @@ -148,6 +148,24 @@ describe('Order store', () => { expect(orders).toHaveLength(2); }); + it('Successfully finds all orders by subaccount/clob pair id after height', async () => { + await Promise.all([ + OrderTable.create(defaultOrder), + OrderTable.create({ + ...defaultOrder, + clientId: '2', + createdAtHeight: '5', + }), + ]); + const orders: OrderFromDatabase[] = await OrderTable.findBySubaccountIdAndClobPairAfterHeight( + defaultOrder.subaccountId, + defaultOrder.clobPairId, + 4, + ); + + expect(orders).toHaveLength(1); + }); + it.each([ [ 'goodTilBlockBeforeOrAt', diff --git a/indexer/packages/postgres/src/db/helpers.ts b/indexer/packages/postgres/src/db/helpers.ts index 96af31df77..6fb1a2f3f0 100644 --- a/indexer/packages/postgres/src/db/helpers.ts +++ b/indexer/packages/postgres/src/db/helpers.ts @@ -1,8 +1,10 @@ import { logger } from '@dydxprotocol-indexer/base'; import Big from 'big.js'; +import { DateTime } from 'luxon'; -import { ONE_MILLION } from '../constants'; +import { NUM_SECONDS_IN_CANDLE_RESOLUTIONS, ONE_MILLION } from '../constants'; import { + CandleResolution, FundingIndexMap, MarketsMap, PerpetualMarketFromDatabase, @@ -129,3 +131,15 @@ export function getTransferType( } throw new Error(`Transfer ${transfer.id} does not involve subaccount ${subaccountId}`); } + +export function calculateNormalizedCandleStartTime( + time: DateTime, + resolution: CandleResolution, +): DateTime { + const epochSeconds: number = Math.floor(time.toUTC().toSeconds()); + const normalizedTimeSeconds: number = epochSeconds - ( + epochSeconds % NUM_SECONDS_IN_CANDLE_RESOLUTIONS[resolution] + ); + + return DateTime.fromSeconds(normalizedTimeSeconds).toUTC(); +} diff --git a/indexer/packages/postgres/src/stores/fill-table.ts b/indexer/packages/postgres/src/stores/fill-table.ts index daf40ff1b2..b39483d99c 100644 --- a/indexer/packages/postgres/src/stores/fill-table.ts +++ b/indexer/packages/postgres/src/stores/fill-table.ts @@ -45,6 +45,8 @@ export async function findAll( transactionHash, createdBeforeOrAtHeight, createdBeforeOrAt, + createdOnOrAfterHeight, + createdOnOrAfter, clientMetadata, fee, }: FillQueryConfig, @@ -64,6 +66,8 @@ export async function findAll( transactionHash, createdBeforeOrAtHeight, createdBeforeOrAt, + createdOnOrAfterHeight, + createdOnOrAfter, clientMetadata, } as QueryConfig, requiredFields, @@ -118,6 +122,18 @@ export async function findAll( baseQuery = baseQuery.where(FillColumns.createdAt, '<=', createdBeforeOrAt); } + if (createdOnOrAfterHeight !== undefined) { + baseQuery = baseQuery.where( + FillColumns.createdAtHeight, + '>=', + createdOnOrAfterHeight, + ); + } + + if (createdOnOrAfter !== undefined) { + baseQuery = baseQuery.where(FillColumns.createdAt, '>=', createdOnOrAfter); + } + if (clientMetadata !== undefined) { baseQuery = baseQuery.where(FillColumns.clientMetadata, clientMetadata); } diff --git a/indexer/packages/postgres/src/stores/order-table.ts b/indexer/packages/postgres/src/stores/order-table.ts index 8e93ce61f4..0cfe50979b 100644 --- a/indexer/packages/postgres/src/stores/order-table.ts +++ b/indexer/packages/postgres/src/stores/order-table.ts @@ -267,6 +267,25 @@ export async function findBySubaccountIdAndClobPair( return orders; } +export async function findBySubaccountIdAndClobPairAfterHeight( + subaccountId: string, + clobPairId: string, + height: number, + options: Options = {}, +): Promise { + const baseQuery: QueryBuilder = setupBaseQuery( + OrderModel, + options, + ); + + const orders: OrderFromDatabase[] = await baseQuery + .where(OrderColumns.subaccountId, subaccountId) + .where(OrderColumns.clobPairId, clobPairId) + .where(OrderColumns.createdAtHeight, '>=', height) + .returning('*'); + return orders; +} + export async function upsert( orderToUpsert: OrderCreateObject, options: Options = { txId: undefined }, diff --git a/indexer/packages/postgres/src/types/query-types.ts b/indexer/packages/postgres/src/types/query-types.ts index eb9a2427bc..14bb205ed8 100644 --- a/indexer/packages/postgres/src/types/query-types.ts +++ b/indexer/packages/postgres/src/types/query-types.ts @@ -140,6 +140,8 @@ export interface FillQueryConfig extends QueryConfig { [QueryableField.TRANSACTION_HASH]?: string; [QueryableField.CREATED_BEFORE_OR_AT_HEIGHT]?: string; [QueryableField.CREATED_BEFORE_OR_AT]?: string; + [QueryableField.CREATED_ON_OR_AFTER_HEIGHT]?: string; + [QueryableField.CREATED_ON_OR_AFTER]?: string; [QueryableField.CLIENT_METADATA]?: string; [QueryableField.FEE]?: string; } diff --git a/indexer/services/ender/__tests__/lib/candles-generator.test.ts b/indexer/services/ender/__tests__/lib/candles-generator.test.ts index 590cebf106..6723dc4d56 100644 --- a/indexer/services/ender/__tests__/lib/candles-generator.test.ts +++ b/indexer/services/ender/__tests__/lib/candles-generator.test.ts @@ -17,6 +17,7 @@ import { testConstants, testMocks, Transaction, + helpers, } from '@dydxprotocol-indexer/postgres'; import { CandleMessage, CandleMessage_Resolution } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -71,11 +72,11 @@ describe('candleHelper', () => { trades: 2, startingOpenInterest: '0', }; - const startedAt: IsoString = CandlesGenerator.calculateNormalizedCandleStartTime( + const startedAt: IsoString = helpers.calculateNormalizedCandleStartTime( testConstants.createdDateTime, CandleResolution.ONE_MINUTE, ).toISO(); - const previousStartedAt: IsoString = CandlesGenerator.calculateNormalizedCandleStartTime( + const previousStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( testConstants.createdDateTime.minus({ minutes: 1 }), CandleResolution.ONE_MINUTE, ).toISO(); @@ -111,7 +112,7 @@ describe('candleHelper', () => { const expectedCandles: CandleFromDatabase[] = _.map( Object.values(CandleResolution), (resolution: CandleResolution) => { - const currentStartedAt: IsoString = CandlesGenerator.calculateNormalizedCandleStartTime( + const currentStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( testConstants.createdDateTime, resolution, ).toISO(); @@ -151,7 +152,7 @@ describe('candleHelper', () => { const expectedCandles: CandleFromDatabase[] = _.map( Object.values(CandleResolution), (resolution: CandleResolution) => { - const currentStartedAt: IsoString = CandlesGenerator.calculateNormalizedCandleStartTime( + const currentStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( testConstants.createdDateTime, resolution, ).toISO(); @@ -181,7 +182,7 @@ describe('candleHelper', () => { const usdVolume: string = Big(existingPrice).times(baseTokenVolume).toString(); await Promise.all( _.map(Object.values(CandleResolution), (resolution: CandleResolution) => { - const currentStartedAt: IsoString = CandlesGenerator.calculateNormalizedCandleStartTime( + const currentStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( testConstants.createdDateTime, resolution, ).toISO(); @@ -215,7 +216,7 @@ describe('candleHelper', () => { const expectedCandles: CandleFromDatabase[] = _.map( Object.values(CandleResolution), (resolution: CandleResolution) => { - const currentStartedAt: IsoString = CandlesGenerator.calculateNormalizedCandleStartTime( + const currentStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( testConstants.createdDateTime, resolution, ).toISO(); diff --git a/indexer/services/ender/src/lib/candles-generator.ts b/indexer/services/ender/src/lib/candles-generator.ts index caaa98f9e5..209557fad5 100644 --- a/indexer/services/ender/src/lib/candles-generator.ts +++ b/indexer/services/ender/src/lib/candles-generator.ts @@ -18,6 +18,7 @@ import { PerpetualPositionTable, TradeContent, TradeMessageContents, + helpers, } from '@dydxprotocol-indexer/postgres'; import { CandleMessage } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -283,7 +284,7 @@ export class CandlesGenerator { return _.some( Object.values(CandleResolution), (resolution: CandleResolution) => { - const startedAt: DateTime = CandlesGenerator.calculateNormalizedCandleStartTime( + const startedAt: DateTime = helpers.calculateNormalizedCandleStartTime( this.blockTimestamp, resolution, ); diff --git a/protocol/Makefile b/protocol/Makefile index 86e8d1b7ca..0b86e7e259 100644 --- a/protocol/Makefile +++ b/protocol/Makefile @@ -347,7 +347,7 @@ localnet-build: localnet-build-amd64: DOCKER_BUILDKIT=1 docker build --build-arg VERSION=$(VERSION) --build-arg COMMIT=$(COMMIT) -f Dockerfile . -t dydxprotocol-base --no-cache --platform linux/amd64 -localnet-init: +reset-chain: rm -rf localnet # Creates the directories which store the chain data for each node. mkdir -p localnet/dydxprotocol0 @@ -360,6 +360,9 @@ localnet-init: echo '{"height": "0", "round": 0, "step": 0 }' > localnet/dydxprotocol1/priv_validator_state.json echo '{"height": "0", "round": 0, "step": 0 }' > localnet/dydxprotocol2/priv_validator_state.json echo '{"height": "0", "round": 0, "step": 0 }' > localnet/dydxprotocol3/priv_validator_state.json + +localnet-init: + make reset-chain # Build the base image. make localnet-build