From b1567908ba0f1db580709a8674363882d018f260 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Wed, 29 Nov 2023 12:59:12 -0800 Subject: [PATCH] [IND-481] Use a single SQL function to process a block. --- .../kafka/src/batch-kafka-producer.ts | 2 +- .../postgres/src/helpers/stores-helpers.ts | 27 +- .../postgres/src/types/utility-types.ts | 4 + .../__tests__/handlers/asset-handler.test.ts | 1 + .../handlers/funding-handler.test.ts | 1 + .../handlers/liquidity-tier-handler.test.ts | 1 + .../markets/market-create-handler.test.ts | 14 +- .../markets/market-modify-handler.test.ts | 15 +- .../market-price-update-handler.test.ts | 14 +- .../order-fills/deleveraging-handler.test.ts | 1 + .../order-fills/liquidation-handler.test.ts | 1 + .../order-fills/order-handler.test.ts | 1 + .../handlers/perpetual-market-handler.test.ts | 1 + ...onditional-order-placement-handler.test.ts | 1 + ...onditional-order-triggered-handler.test.ts | 1 + .../stateful-order-placement-handler.test.ts | 1 + .../stateful-order-removal-handler.test.ts | 1 + .../subaccount-update-handler.test.ts | 1 + .../handlers/transfer-handler.test.ts | 1 + .../handlers/update-clob-pair-handler.test.ts | 1 + .../handlers/update-perpetual-handler.test.ts | 1 + .../__tests__/lib/batched-handler.test.ts | 1 + .../__tests__/lib/block-processor.test.ts | 14 +- .../ender/__tests__/lib/on-message.test.ts | 315 ++++-------------- .../ender/__tests__/lib/sync-handlers.test.ts | 167 ++-------- .../ender/__tests__/scripts/scripts.test.ts | 2 +- .../validators/asset-validator.test.ts | 1 + .../validators/deleveraging-validator.test.ts | 2 + .../validators/funding-validator.test.ts | 2 + .../liquidity-tier-validator.test.ts | 3 + .../validators/market-validator.test.ts | 2 + .../validators/order-fill-validator.test.ts | 2 + .../perpetual-market-validator.test.ts | 3 + .../stateful-order-validator.test.ts | 2 + .../subaccount-update-validator.test.ts | 2 + .../validators/transfer-validator.test.ts | 2 + .../update-clob-pair-validator.test.ts | 2 + .../update-perpetual-validator.test.ts | 2 + .../abstract-stateful-order-handler.ts | 46 --- .../ender/src/handlers/asset-handler.ts | 22 +- .../ender/src/handlers/funding-handler.ts | 33 +- .../services/ender/src/handlers/handler.ts | 21 +- .../src/handlers/liquidity-tier-handler.ts | 22 +- .../handlers/markets/market-create-handler.ts | 33 +- .../handlers/markets/market-modify-handler.ts | 38 +-- .../markets/market-price-update-handler.ts | 37 +- .../order-fills/deleveraging-handler.ts | 36 +- .../order-fills/liquidation-handler.ts | 48 +-- .../src/handlers/order-fills/order-handler.ts | 64 +--- .../src/handlers/perpetual-market-handler.ts | 22 +- .../conditional-order-placement-handler.ts | 17 +- .../conditional-order-triggered-handler.ts | 21 +- .../stateful-order-placement-handler.ts | 5 +- .../stateful-order-removal-handler.ts | 4 +- .../src/handlers/subaccount-update-handler.ts | 28 +- .../ender/src/handlers/transfer-handler.ts | 33 +- .../src/handlers/update-clob-pair-handler.ts | 22 +- .../src/handlers/update-perpetual-handler.ts | 22 +- .../helpers/postgres/postgres-functions.ts | 3 + .../ender/src/lib/batched-handlers.ts | 11 +- .../services/ender/src/lib/block-processor.ts | 91 ++++- indexer/services/ender/src/lib/helper.ts | 13 + .../services/ender/src/lib/kafka-publisher.ts | 1 + indexer/services/ender/src/lib/on-message.ts | 38 --- .../services/ender/src/lib/sync-handlers.ts | 8 +- indexer/services/ender/src/lib/types.ts | 13 + .../src/scripts/dydx_block_processor.sql | 39 +++ .../dydx_block_processor_batched_handlers.sql | 69 ++++ .../dydx_block_processor_sync_handlers.sql | 63 ++++ ...eate_initial_rows_for_tendermint_block.sql | 16 +- .../scripts/dydx_create_tendermint_event.sql | 4 +- .../src/scripts/dydx_create_transaction.sql | 6 +- .../scripts/dydx_stateful_order_handler.sql | 2 +- .../dydx_uuid_from_transaction_parts.sql | 2 +- .../ender/src/validators/asset-validator.ts | 1 + .../src/validators/deleveraging-validator.ts | 1 + .../ender/src/validators/funding-validator.ts | 1 + .../validators/liquidity-tier-validator.ts | 1 + .../ender/src/validators/market-validator.ts | 1 + .../src/validators/order-fill-validator.ts | 32 +- .../validators/perpetual-market-validator.ts | 1 + .../validators/stateful-order-validator.ts | 1 + .../validators/subaccount-update-validator.ts | 1 + .../src/validators/transfer-validator.ts | 1 + .../validators/update-clob-pair-validator.ts | 1 + .../validators/update-perpetual-validator.ts | 1 + .../ender/src/validators/validator.ts | 17 +- 87 files changed, 636 insertions(+), 992 deletions(-) create mode 100644 indexer/services/ender/src/scripts/dydx_block_processor.sql create mode 100644 indexer/services/ender/src/scripts/dydx_block_processor_batched_handlers.sql create mode 100644 indexer/services/ender/src/scripts/dydx_block_processor_sync_handlers.sql diff --git a/indexer/packages/kafka/src/batch-kafka-producer.ts b/indexer/packages/kafka/src/batch-kafka-producer.ts index 7e70ff4b05c..c4546db041a 100644 --- a/indexer/packages/kafka/src/batch-kafka-producer.ts +++ b/indexer/packages/kafka/src/batch-kafka-producer.ts @@ -28,7 +28,7 @@ export class BatchKafkaProducer { constructor( topic: KafkaTopics, producer: Producer, - maxBatchSizeBytes: number = config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES, + maxBatchSizeBytes: number, ) { this.maxBatchSizeBytes = maxBatchSizeBytes; this.producer = producer; diff --git a/indexer/packages/postgres/src/helpers/stores-helpers.ts b/indexer/packages/postgres/src/helpers/stores-helpers.ts index 7fd583d843a..94f1b9f7542 100644 --- a/indexer/packages/postgres/src/helpers/stores-helpers.ts +++ b/indexer/packages/postgres/src/helpers/stores-helpers.ts @@ -1,4 +1,4 @@ -import Knex from 'knex'; +import Knex, { RawBinding } from 'knex'; import _ from 'lodash'; import { Model, ModelClass, QueryBuilderType } from 'objection'; @@ -64,25 +64,18 @@ export async function rawQuery( options: Options, // eslint-disable-next-line @typescript-eslint/no-explicit-any ): Promise> { - if (options.readReplica) { - if (options.txId) { - return knexReadReplica.getConnection().raw(queryString).transacting( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - >Transaction.get(options.txId), - ); - } else { - return knexReadReplica.getConnection().raw(queryString); - } - } else { - if (options.txId) { - return knexPrimary.raw(queryString).transacting( + const connection = options.readReplica ? knexReadReplica.getConnection() : knexPrimary; + let queryBuilder = options.bindings === undefined ? connection.raw(queryString) : connection.raw(queryString, options.bindings); + if (options.txId) { + queryBuilder = queryBuilder.transacting( // eslint-disable-next-line @typescript-eslint/no-explicit-any >Transaction.get(options.txId), - ); - } else { - return knexPrimary.raw(queryString); - } + ); + } + if (options.sqlOptions) { + queryBuilder = queryBuilder.options(options.sqlOptions); } + return queryBuilder; } /* ------- Bulk Helpers ------- */ diff --git a/indexer/packages/postgres/src/types/utility-types.ts b/indexer/packages/postgres/src/types/utility-types.ts index b22cd58d7f9..0210a8f79dd 100644 --- a/indexer/packages/postgres/src/types/utility-types.ts +++ b/indexer/packages/postgres/src/types/utility-types.ts @@ -1,4 +1,6 @@ /* ------- UTILITY TYPES ------- */ +import { RawBinding } from "knex"; + export type IsoString = string; export type RegexPattern = string; @@ -17,6 +19,8 @@ export interface Options { orderBy?: [string, Ordering][]; readReplica?: boolean, random?: boolean; + bindings?: readonly RawBinding[]; + sqlOptions?: Readonly<{ [key: string]: any }>; } export enum Ordering { diff --git a/indexer/services/ender/__tests__/handlers/asset-handler.test.ts b/indexer/services/ender/__tests__/handlers/asset-handler.test.ts index 75f210031d9..1982e07c087 100644 --- a/indexer/services/ender/__tests__/handlers/asset-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/asset-handler.test.ts @@ -85,6 +85,7 @@ describe('assetHandler', () => { const handler: AssetCreationHandler = new AssetCreationHandler( block, + 0, indexerTendermintEvent, 0, defaultAssetCreateEvent, diff --git a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts index 8a5bb2883a5..f676a88eaa7 100644 --- a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts @@ -89,6 +89,7 @@ describe('fundingHandler', () => { const handler: FundingHandler = new FundingHandler( block, + 0, indexerTendermintEvent, 0, defaultFundingUpdateSampleEvent, diff --git a/indexer/services/ender/__tests__/handlers/liquidity-tier-handler.test.ts b/indexer/services/ender/__tests__/handlers/liquidity-tier-handler.test.ts index ff86a71d69e..a9353d03c0c 100644 --- a/indexer/services/ender/__tests__/handlers/liquidity-tier-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/liquidity-tier-handler.test.ts @@ -90,6 +90,7 @@ describe('liquidityTierHandler', () => { const handler: LiquidityTierHandler = new LiquidityTierHandler( block, + 0, indexerTendermintEvent, 0, defaultLiquidityTierUpsertEvent, diff --git a/indexer/services/ender/__tests__/handlers/markets/market-create-handler.test.ts b/indexer/services/ender/__tests__/handlers/markets/market-create-handler.test.ts index 839626cdd5e..5ce1a88233d 100644 --- a/indexer/services/ender/__tests__/handlers/markets/market-create-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/markets/market-create-handler.test.ts @@ -1,4 +1,4 @@ -import { logger, ParseMessageError } from '@dydxprotocol-indexer/base'; +import { logger } from '@dydxprotocol-indexer/base'; import { dbHelpers, MarketFromDatabase, MarketTable, testMocks, } from '@dydxprotocol-indexer/postgres'; @@ -46,7 +46,6 @@ describe('marketCreateHandler', () => { }); const loggerCrit = jest.spyOn(logger, 'crit'); - const loggerError = jest.spyOn(logger, 'error'); const producerSendMock: jest.SpyInstance = jest.spyOn(producer, 'send'); describe('getParallelizationIds', () => { @@ -75,6 +74,7 @@ describe('marketCreateHandler', () => { const handler: MarketCreateHandler = new MarketCreateHandler( block, + 0, indexerTendermintEvent, 0, marketEvent, @@ -128,7 +128,7 @@ describe('marketCreateHandler', () => { txHash: defaultTxHash, }); await expect(onMessage(kafkaMessage)).rejects.toThrowError( - new ParseMessageError('Market in MarketCreate already exists'), + 'Market in MarketCreate already exists', ); // Check that market in database is the old market. @@ -137,13 +137,9 @@ describe('marketCreateHandler', () => { ) as MarketFromDatabase; expect(market.minPriceChangePpm).toEqual(50); - expect(loggerError).toHaveBeenCalledWith(expect.objectContaining({ - at: 'MarketCreateHandler#logAndThrowParseMessageError', - message: 'Market in MarketCreate already exists', - })); expect(loggerCrit).toHaveBeenCalledWith(expect.objectContaining({ - at: 'onMessage#onMessage', - message: 'Error: Unable to parse message, this must be due to a bug in V4 node', + at: expect.stringContaining('PL/pgSQL function dydx_market_create_handler('), + message: expect.stringContaining('Market in MarketCreate already exists'), })); expect(producerSendMock.mock.calls.length).toEqual(0); }); diff --git a/indexer/services/ender/__tests__/handlers/markets/market-modify-handler.test.ts b/indexer/services/ender/__tests__/handlers/markets/market-modify-handler.test.ts index c80d4a1a7f3..cf2a2d1c58e 100644 --- a/indexer/services/ender/__tests__/handlers/markets/market-modify-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/markets/market-modify-handler.test.ts @@ -1,4 +1,4 @@ -import { logger, ParseMessageError } from '@dydxprotocol-indexer/base'; +import { logger } from '@dydxprotocol-indexer/base'; import { dbHelpers, MarketFromDatabase, MarketTable, testMocks, } from '@dydxprotocol-indexer/postgres'; @@ -40,7 +40,6 @@ describe('marketModifyHandler', () => { }); const loggerCrit = jest.spyOn(logger, 'crit'); - const loggerError = jest.spyOn(logger, 'error'); const producerSendMock: jest.SpyInstance = jest.spyOn(producer, 'send'); describe('getParallelizationIds', () => { @@ -69,6 +68,7 @@ describe('marketModifyHandler', () => { const handler: MarketModifyHandler = new MarketModifyHandler( block, + 0, indexerTendermintEvent, 0, marketEvent, @@ -115,16 +115,11 @@ describe('marketModifyHandler', () => { }); await expect(onMessage(kafkaMessage)).rejects.toThrowError( - new ParseMessageError('Market in MarketModify doesn\'t exist'), + 'Market in MarketModify doesn\'t exist', ); - - expect(loggerError).toHaveBeenCalledWith(expect.objectContaining({ - at: 'MarketModifyHandler#logAndThrowParseMessageError', - message: 'Market in MarketModify doesn\'t exist', - })); expect(loggerCrit).toHaveBeenCalledWith(expect.objectContaining({ - at: 'onMessage#onMessage', - message: 'Error: Unable to parse message, this must be due to a bug in V4 node', + at: expect.stringContaining('PL/pgSQL function dydx_market_modify_handler('), + message: expect.stringContaining('Market in MarketModify doesn\'t exist'), })); expect(producerSendMock.mock.calls.length).toEqual(0); }); diff --git a/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts b/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts index 77eb916dbe2..6c858450327 100644 --- a/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts @@ -1,4 +1,4 @@ -import { logger, ParseMessageError } from '@dydxprotocol-indexer/base'; +import { logger } from '@dydxprotocol-indexer/base'; import { dbHelpers, MarketFromDatabase, @@ -54,7 +54,6 @@ describe('marketPriceUpdateHandler', () => { jest.resetAllMocks(); }); const loggerCrit = jest.spyOn(logger, 'crit'); - const loggerError = jest.spyOn(logger, 'error'); const producerSendMock: jest.SpyInstance = jest.spyOn(producer, 'send'); describe('getParallelizationIds', () => { @@ -83,6 +82,7 @@ describe('marketPriceUpdateHandler', () => { const handler: MarketPriceUpdateHandler = new MarketPriceUpdateHandler( block, + 0, indexerTendermintEvent, 0, marketEvent, @@ -111,16 +111,12 @@ describe('marketPriceUpdateHandler', () => { }); await expect(onMessage(kafkaMessage)).rejects.toThrowError( - new ParseMessageError('MarketPriceUpdateEvent contains a non-existent market id'), + 'MarketPriceUpdateEvent contains a non-existent market id', ); - expect(loggerError).toHaveBeenCalledWith(expect.objectContaining({ - at: 'MarketPriceUpdateHandler#logAndThrowParseMessageError', - message: 'MarketPriceUpdateEvent contains a non-existent market id', - })); expect(loggerCrit).toHaveBeenCalledWith(expect.objectContaining({ - at: 'onMessage#onMessage', - message: 'Error: Unable to parse message, this must be due to a bug in V4 node', + at: expect.stringContaining('PL/pgSQL function dydx_market_price_update_handler('), + message: expect.stringContaining('MarketPriceUpdateEvent contains a non-existent market id'), })); expect(producerSendMock.mock.calls.length).toEqual(0); }); diff --git a/indexer/services/ender/__tests__/handlers/order-fills/deleveraging-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/deleveraging-handler.test.ts index fbe69502926..8c06ba595b5 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/deleveraging-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/deleveraging-handler.test.ts @@ -143,6 +143,7 @@ describe('DeleveragingHandler', () => { const handler: DeleveragingHandler = new DeleveragingHandler( block, + 0, indexerTendermintEvent, 0, defaultDeleveragingEvent, diff --git a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts index ee139e53479..1b7f3ddb0f7 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts @@ -178,6 +178,7 @@ describe('LiquidationHandler', () => { const handler: LiquidationHandler = new LiquidationHandler( block, + 0, indexerTendermintEvent, 0, { diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index 9e4867b1d1b..46b005c8412 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -180,6 +180,7 @@ describe('OrderHandler', () => { const handler: OrderHandler = new OrderHandler( block, + 0, indexerTendermintEvent, 0, { diff --git a/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts b/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts index 336ced9392a..50cc4e46818 100644 --- a/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts @@ -90,6 +90,7 @@ describe('perpetualMarketHandler', () => { const handler: PerpetualMarketCreationHandler = new PerpetualMarketCreationHandler( block, + 0, indexerTendermintEvent, 0, defaultPerpetualMarketCreateEvent, diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts index 3f7c2d2fe1d..0f3b55b0bb9 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts @@ -108,6 +108,7 @@ describe('conditionalOrderPlacementHandler', () => { const handler: ConditionalOrderPlacementHandler = new ConditionalOrderPlacementHandler( block, + 0, indexerTendermintEvent, 0, defaultStatefulOrderEvent, diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts index fa234c9a38a..e9a02dc1649 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts @@ -94,6 +94,7 @@ describe('conditionalOrderTriggeredHandler', () => { const handler: ConditionalOrderTriggeredHandler = new ConditionalOrderTriggeredHandler( block, + 0, indexerTendermintEvent, 0, defaultStatefulOrderEvent, diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts index ae5a38d00c7..dd0ca482c46 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts @@ -119,6 +119,7 @@ describe('statefulOrderPlacementHandler', () => { const handler: StatefulOrderPlacementHandler = new StatefulOrderPlacementHandler( block, + 0, indexerTendermintEvent, 0, statefulOrderEvent, diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts index e860242e384..68aa0b226b1 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts @@ -88,6 +88,7 @@ describe('statefulOrderRemovalHandler', () => { const handler: StatefulOrderRemovalHandler = new StatefulOrderRemovalHandler( block, + 0, indexerTendermintEvent, 0, defaultStatefulOrderEvent, diff --git a/indexer/services/ender/__tests__/handlers/subaccount-update-handler.test.ts b/indexer/services/ender/__tests__/handlers/subaccount-update-handler.test.ts index a94556b1956..6c41322434d 100644 --- a/indexer/services/ender/__tests__/handlers/subaccount-update-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/subaccount-update-handler.test.ts @@ -127,6 +127,7 @@ describe('subaccountUpdateHandler', () => { const handler: SubaccountUpdateHandler = new SubaccountUpdateHandler( block, + 0, indexerTendermintEvent, 0, defaultEmptySubaccountUpdate, diff --git a/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts b/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts index b99fe937b9e..ed22d563842 100644 --- a/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts @@ -120,6 +120,7 @@ describe('transferHandler', () => { const handler: TransferHandler = new TransferHandler( block, + 0, indexerTendermintEvent, 0, defaultTransferEvent, diff --git a/indexer/services/ender/__tests__/handlers/update-clob-pair-handler.test.ts b/indexer/services/ender/__tests__/handlers/update-clob-pair-handler.test.ts index 0c4f307fff9..f66c1d68c58 100644 --- a/indexer/services/ender/__tests__/handlers/update-clob-pair-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/update-clob-pair-handler.test.ts @@ -78,6 +78,7 @@ describe('update-clob-pair-handler', () => { const handler: UpdateClobPairHandler = new UpdateClobPairHandler( block, + 0, indexerTendermintEvent, 0, defaultUpdateClobPairEvent, diff --git a/indexer/services/ender/__tests__/handlers/update-perpetual-handler.test.ts b/indexer/services/ender/__tests__/handlers/update-perpetual-handler.test.ts index 6dba0779ba1..f6d67cfcdca 100644 --- a/indexer/services/ender/__tests__/handlers/update-perpetual-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/update-perpetual-handler.test.ts @@ -77,6 +77,7 @@ describe('update-perpetual-handler', () => { const handler: UpdatePerpetualHandler = new UpdatePerpetualHandler( block, + 0, indexerTendermintEvent, 0, defaultUpdatePerpetualEvent, diff --git a/indexer/services/ender/__tests__/lib/batched-handler.test.ts b/indexer/services/ender/__tests__/lib/batched-handler.test.ts index 27f148eb954..dce139356cc 100644 --- a/indexer/services/ender/__tests__/lib/batched-handler.test.ts +++ b/indexer/services/ender/__tests__/lib/batched-handler.test.ts @@ -48,6 +48,7 @@ function generateFakeHandler(parallelizationIds: string[]): FakeHandler { const handler: FakeHandler = new FakeHandler( block, + 0, defaultEvent, fakeTxId, {}, diff --git a/indexer/services/ender/__tests__/lib/block-processor.test.ts b/indexer/services/ender/__tests__/lib/block-processor.test.ts index a5217a49c3f..d0ea660cdbd 100644 --- a/indexer/services/ender/__tests__/lib/block-processor.test.ts +++ b/indexer/services/ender/__tests__/lib/block-processor.test.ts @@ -23,6 +23,7 @@ import { BlockProcessor } from '../../src/lib/block-processor'; import { BatchedHandlers } from '../../src/lib/batched-handlers'; import { SyncHandlers } from '../../src/lib/sync-handlers'; import { mock, MockProxy } from 'jest-mock-extended'; +import { createPostgresFunctions } from '../../src/helpers/postgres/postgres-functions'; describe('block-processor', () => { let batchedHandlers: MockProxy; @@ -36,6 +37,7 @@ describe('block-processor', () => { beforeAll(async () => { await dbHelpers.migrate(); + await createPostgresFunctions(); }); afterEach(async () => { @@ -90,18 +92,18 @@ describe('block-processor', () => { transactionIndex0, eventIndex0, ), - createIndexerTendermintEvent( - DydxIndexerSubtypes.ASSET, - defaultAssetEventBinary, - transactionIndex0, - eventIndex1, - ), createIndexerTendermintEvent( DydxIndexerSubtypes.MARKET, defaultMarketEventBinary, transactionIndex1, eventIndex0, ), + createIndexerTendermintEvent( + DydxIndexerSubtypes.ASSET, + defaultAssetEventBinary, + transactionIndex0, + eventIndex1, + ), ]; it('batched handlers called before sync handlers for normal blocks', async () => { diff --git a/indexer/services/ender/__tests__/lib/on-message.test.ts b/indexer/services/ender/__tests__/lib/on-message.test.ts index 5a8e287969e..a447e905426 100644 --- a/indexer/services/ender/__tests__/lib/on-message.test.ts +++ b/indexer/services/ender/__tests__/lib/on-message.test.ts @@ -1,6 +1,7 @@ import { DateTime } from 'luxon'; import { assetRefresher, + AssetTable, BlockFromDatabase, BlockTable, dbHelpers, @@ -17,7 +18,6 @@ import { TransactionTable, } from '@dydxprotocol-indexer/postgres'; import { - DeleveragingEventV1, FundingEventV1, IndexerTendermintBlock, IndexerTendermintEvent, @@ -30,16 +30,14 @@ import { import { createIndexerTendermintBlock, createIndexerTendermintEvent } from '../helpers/indexer-proto-helpers'; import { onMessage } from '../../src/lib/on-message'; import { KafkaMessage } from 'kafkajs'; -import { createKafkaMessage, KafkaTopics, producer } from '@dydxprotocol-indexer/kafka'; +import { + createKafkaMessage, KafkaTopics, producer, +} from '@dydxprotocol-indexer/kafka'; import { MILLIS_IN_NANOS, SECONDS_IN_MILLIS } from '../../src/constants'; import { ConsolidatedKafkaEvent, DydxIndexerSubtypes } from '../../src/lib/types'; -import { SubaccountUpdateHandler } from '../../src/handlers/subaccount-update-handler'; import config from '../../src/config'; import { logger, stats } from '@dydxprotocol-indexer/base'; -import { TransferHandler } from '../../src/handlers/transfer-handler'; -import { FundingHandler } from '../../src/handlers/funding-handler'; import { - defaultDeleveragingEvent, defaultFundingUpdateSampleEvent, defaultHeight, defaultMarketModify, @@ -47,47 +45,14 @@ import { defaultSubaccountMessage, } from '../helpers/constants'; import { updateBlockCache } from '../../src/caches/block-cache'; -import { MarketModifyHandler } from '../../src/handlers/markets/market-modify-handler'; import Long from 'long'; import { createPostgresFunctions } from '../../src/helpers/postgres/postgres-functions'; -import { DeleveragingHandler } from '../../src/handlers/order-fills/deleveraging-handler'; - -jest.mock('../../src/handlers/subaccount-update-handler'); -jest.mock('../../src/handlers/transfer-handler'); -jest.mock('../../src/handlers/funding-handler'); -jest.mock('../../src/handlers/order-fills/deleveraging-handler'); -jest.mock('../../src/handlers/markets/market-modify-handler'); describe('on-message', () => { let producerSendMock: jest.SpyInstance; const loggerError = jest.spyOn(logger, 'error'); beforeEach(() => { - (SubaccountUpdateHandler as jest.Mock).mockReturnValue({ - handle: () => [], - validate: () => null, - getParallelizationIds: () => [], - }); - (TransferHandler as jest.Mock).mockReturnValue({ - handle: () => [], - validate: () => null, - getParallelizationIds: () => [], - }); - (MarketModifyHandler as jest.Mock).mockReturnValue({ - handle: () => [], - validate: () => null, - getParallelizationIds: () => [], - }); - (FundingHandler as jest.Mock).mockReturnValue({ - handle: () => [], - validate: () => null, - getParallelizationIds: () => [], - }); - (DeleveragingHandler as jest.Mock).mockReturnValue({ - handle: () => [], - validate: () => null, - getParallelizationIds: () => [], - }); producerSendMock = jest.spyOn(producer, 'send'); producerSendMock.mockImplementation(() => { }); @@ -161,10 +126,6 @@ describe('on-message', () => { defaultMarketModify, ).finish()); - const defaultDeleveragingEventBinary: Uint8Array = Uint8Array.from(DeleveragingEventV1.encode( - defaultDeleveragingEvent, - ).finish()); - it('successfully processes block with transaction event', async () => { const transactionIndex: number = 0; const eventIndex: number = 0; @@ -193,14 +154,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultSubaccountUpdateEvent, - ); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -209,16 +162,7 @@ describe('on-message', () => { expect.any(Number), 1, { success: 'true' }); }); - it.each([ - [ - 'via knex', - false, - ], - [ - 'via SQL function', - true, - ], - ])('successfully processes block with transaction event with unset version', async () => { + it('successfully processes block with transaction event with unset version', async () => { const transactionIndex: number = 0; const eventIndex: number = 0; const events: IndexerTendermintEvent[] = [ @@ -247,14 +191,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultSubaccountUpdateEvent, - ); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -264,6 +200,10 @@ describe('on-message', () => { }); it('successfully processes block with transfer event', async () => { + await Promise.all([ + MarketTable.create(testConstants.defaultMarket), + AssetTable.create(testConstants.defaultAsset), + ]); const transactionIndex: number = 0; const eventIndex: number = 0; const events: IndexerTendermintEvent[] = [ @@ -291,14 +231,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((TransferHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((TransferHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultTransferEvent, - ); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -348,71 +280,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((FundingHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((FundingHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultFundingUpdateSampleEvent, - ); - expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); - expect(stats.timing).toHaveBeenCalledWith( - 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); - expect(stats.gauge).toHaveBeenCalledWith('ender.processing_block_height', expect.any(Number)); - expect(stats.timing).toHaveBeenCalledWith('ender.processed_block.timing', - expect.any(Number), 1, { success: 'true' }); - }); - - it('successfully processes block with deleveraging event', async () => { - await Promise.all([ - MarketTable.create(testConstants.defaultMarket), - MarketTable.create(testConstants.defaultMarket2), - ]); - await Promise.all([ - LiquidityTiersTable.create(testConstants.defaultLiquidityTier), - LiquidityTiersTable.create(testConstants.defaultLiquidityTier2), - ]); - await Promise.all([ - PerpetualMarketTable.create(testConstants.defaultPerpetualMarket), - PerpetualMarketTable.create(testConstants.defaultPerpetualMarket2), - ]); - await perpetualMarketRefresher.updatePerpetualMarkets(); - - const transactionIndex: number = 0; - const eventIndex: number = 0; - const events: IndexerTendermintEvent[] = [ - createIndexerTendermintEvent( - DydxIndexerSubtypes.DELEVERAGING, - defaultDeleveragingEventBinary, - transactionIndex, - eventIndex, - ), - ]; - - const block: IndexerTendermintBlock = createIndexerTendermintBlock( - defaultHeight, - defaultTime, - events, - [defaultTxHash], - ); - const binaryBlock: Uint8Array = Uint8Array.from(IndexerTendermintBlock.encode(block).finish()); - const kafkaMessage: KafkaMessage = createKafkaMessage(Buffer.from(binaryBlock)); - - await onMessage(kafkaMessage); - await Promise.all([ - expectTendermintEvent(defaultHeight.toString(), transactionIndex, eventIndex), - expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), - ]); - - expect((DeleveragingHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((DeleveragingHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultDeleveragingEvent, - ); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -448,6 +315,10 @@ describe('on-message', () => { }); it('skips over unknown events while processing', async () => { + await Promise.all([ + MarketTable.create(testConstants.defaultMarket), + AssetTable.create(testConstants.defaultAsset), + ]); const transactionIndex: number = 0; const eventIndex: number = 0; const eventIndex1: number = 1; @@ -482,14 +353,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((TransferHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((TransferHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultTransferEvent, - ); expect(loggerError).toHaveBeenCalledWith(expect.objectContaining({ at: 'helpers#indexerTendermintEventToEventWithType', message: 'Unable to parse event subtype: unknown', @@ -500,6 +363,9 @@ describe('on-message', () => { }); it('successfully processes block with market event', async () => { + await Promise.all([ + MarketTable.create(testConstants.defaultMarket), + ]); const transactionIndex: number = 0; const eventIndex: number = 0; const events: IndexerTendermintEvent[] = [ @@ -527,14 +393,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((MarketModifyHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((MarketModifyHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultMarketModify, - ); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -544,13 +402,26 @@ describe('on-message', () => { }); it('successfully processes block with block event', async () => { - // -1 so that createIndexerTendermintEvent creates a block event + await Promise.all([ + MarketTable.create(testConstants.defaultMarket), + MarketTable.create(testConstants.defaultMarket2), + ]); + await Promise.all([ + LiquidityTiersTable.create(testConstants.defaultLiquidityTier), + LiquidityTiersTable.create(testConstants.defaultLiquidityTier2), + ]); + await Promise.all([ + PerpetualMarketTable.create(testConstants.defaultPerpetualMarket), + PerpetualMarketTable.create(testConstants.defaultPerpetualMarket2), + ]); + await perpetualMarketRefresher.updatePerpetualMarkets(); + const transactionIndex: number = -1; const eventIndex: number = 0; const events: IndexerTendermintEvent[] = [ createIndexerTendermintEvent( - DydxIndexerSubtypes.SUBACCOUNT_UPDATE, - defaultSubaccountUpdateEventBinary, + DydxIndexerSubtypes.FUNDING, + defaultFundingEventBinary, transactionIndex, eventIndex, ), @@ -571,12 +442,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - const transactions: TransactionFromDatabase[] = await - TransactionTable.findAll({}, [], { readReplica: true }); - - expect(transactions.length).toEqual(0); - - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenCalledTimes(0); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -586,20 +451,35 @@ describe('on-message', () => { }); it('successfully processes block with transaction event and block event', async () => { - const transactionIndex: number = 0; - const eventIndex: number = 0; + await Promise.all([ + MarketTable.create(testConstants.defaultMarket), + MarketTable.create(testConstants.defaultMarket2), + ]); + await Promise.all([ + LiquidityTiersTable.create(testConstants.defaultLiquidityTier), + LiquidityTiersTable.create(testConstants.defaultLiquidityTier2), + ]); + await Promise.all([ + PerpetualMarketTable.create(testConstants.defaultPerpetualMarket), + PerpetualMarketTable.create(testConstants.defaultPerpetualMarket2), + ]); + await perpetualMarketRefresher.updatePerpetualMarkets(); const blockTransactionIndex: number = -1; + const transactionIndex: number = 0; + const eventIndex: number = 0; const events: IndexerTendermintEvent[] = [ + // MARKET is a transaction event. createIndexerTendermintEvent( - DydxIndexerSubtypes.SUBACCOUNT_UPDATE, - defaultSubaccountUpdateEventBinary, + DydxIndexerSubtypes.MARKET, + defaultMarketEventBinary, transactionIndex, eventIndex, ), + // FUNDING is a block event. createIndexerTendermintEvent( - DydxIndexerSubtypes.SUBACCOUNT_UPDATE, - defaultSubaccountUpdateEventBinary, + DydxIndexerSubtypes.FUNDING, + defaultFundingEventBinary, blockTransactionIndex, eventIndex, ), @@ -621,21 +501,6 @@ describe('on-message', () => { expectTransactionWithHash([defaultTxHash]), expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenCalledTimes(1); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultSubaccountUpdateEvent, - ); - expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); - expect(stats.timing).toHaveBeenCalledWith( - 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); - expect(stats.gauge).toHaveBeenCalledWith('ender.processing_block_height', expect.any(Number)); - expect(stats.timing).toHaveBeenCalledWith('ender.processed_block.timing', - expect.any(Number), 1, { success: 'true' }); }); it('successfully processes block with multiple transactions', async () => { @@ -686,28 +551,6 @@ describe('on-message', () => { expectBlock(defaultHeight.toString(), defaultDateTime.toISO()), ]); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenCalledTimes(3); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenNthCalledWith( - 1, - block, - events[0], - expect.any(Number), - defaultSubaccountUpdateEvent, - ); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenNthCalledWith( - 2, - block, - events[1], - expect.any(Number), - defaultSubaccountUpdateEvent, - ); - expect((SubaccountUpdateHandler as jest.Mock)).toHaveBeenNthCalledWith( - 3, - block, - events[2], - expect.any(Number), - defaultSubaccountUpdateEvent, - ); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); expect(stats.timing).toHaveBeenCalledWith( 'ender.message_time_in_queue', expect.any(Number), 1, { topic: KafkaTopics.TO_ENDER }); @@ -715,14 +558,17 @@ describe('on-message', () => { expect(stats.timing).toHaveBeenCalledWith('ender.processed_block.timing', expect.any(Number), 1, { success: 'true' }); }); - it('successfully batches up kafka messages', async () => { + await Promise.all([ + MarketTable.create(testConstants.defaultMarket), + AssetTable.create(testConstants.defaultAsset), + ]); const transactionIndex: number = 0; const eventIndex: number = 0; const events: IndexerTendermintEvent[] = [ createIndexerTendermintEvent( - DydxIndexerSubtypes.SUBACCOUNT_UPDATE, - defaultSubaccountUpdateEventBinary, + DydxIndexerSubtypes.TRANSFER, + defaultTransferEventBinary, transactionIndex, eventIndex, ), @@ -737,26 +583,7 @@ describe('on-message', () => { const binaryBlock: Uint8Array = Uint8Array.from(IndexerTendermintBlock.encode(block).finish()); const kafkaMessage: KafkaMessage = createKafkaMessage(Buffer.from(binaryBlock)); - // Mock the return of kafka messages that in total have size > max message size in bytes - const kafkaMessages: ConsolidatedKafkaEvent[] = []; - let totalSizeBytes: number = 0; - const subaccountByteChange: number = Buffer.from( - Uint8Array.from(SubaccountMessage.encode(defaultSubaccountMessage).finish()), - ).byteLength; - while (totalSizeBytes <= config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES) { - kafkaMessages.push({ - topic: KafkaTopics.TO_WEBSOCKETS_SUBACCOUNTS, - message: defaultSubaccountMessage, - }); - totalSizeBytes += subaccountByteChange; - } - - (SubaccountUpdateHandler as jest.Mock).mockReturnValue({ - handle: () => kafkaMessages, - validate: () => null, - getParallelizationIds: () => [], - }); - + config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES = 1; await onMessage(kafkaMessage); await Promise.all([ expectTendermintEvent(defaultHeight.toString(), transactionIndex, eventIndex), @@ -765,9 +592,9 @@ describe('on-message', () => { ]); expect(producerSendMock).toHaveBeenCalledTimes(2); - // First message sent should contain all but 1 of the message buffers - expect(producerSendMock.mock.calls[0][0].messages).toHaveLength(kafkaMessages.length - 1); - // Second message sent should contain just a single message buffer + // First message batch sent should contain the first message + expect(producerSendMock.mock.calls[0][0].messages).toHaveLength(1); + // Second message batch should contain the second message expect(producerSendMock.mock.calls[1][0].messages).toHaveLength(1); expect(stats.increment).toHaveBeenCalledWith('ender.received_kafka_message', 1); @@ -810,12 +637,6 @@ describe('on-message', () => { totalSizeBytes += subaccountByteChange; } - (SubaccountUpdateHandler as jest.Mock).mockReturnValue({ - handle: () => kafkaMessages, - validate: () => null, - getParallelizationIds: () => [], - }); - config.SEND_WEBSOCKET_MESSAGES = false; await onMessage(kafkaMessage); await Promise.all([ @@ -892,16 +713,6 @@ describe('on-message', () => { // Initialize assetRefresher await assetRefresher.updateAssets(); await perpetualMarketRefresher.updatePerpetualMarkets(); - (SubaccountUpdateHandler as jest.Mock).mockReturnValue({ - handle: () => { - // clear cache so we can confirm that the cache is updated after the error - assetRefresher.clear(); - perpetualMarketRefresher.clear(); - throw new Error(); - }, - validate: () => null, - getParallelizationIds: () => [], - }); await onMessage(kafkaMessage); expect(assetRefresher.getAssetsMap()).not.toEqual({}); diff --git a/indexer/services/ender/__tests__/lib/sync-handlers.test.ts b/indexer/services/ender/__tests__/lib/sync-handlers.test.ts index f68f6a0f97d..a3ebc6761c5 100644 --- a/indexer/services/ender/__tests__/lib/sync-handlers.test.ts +++ b/indexer/services/ender/__tests__/lib/sync-handlers.test.ts @@ -1,155 +1,42 @@ import { SyncHandlers } from '../../src/lib/sync-handlers'; -import { - AssetCreateEventV1, - IndexerTendermintBlock, - IndexerTendermintEvent, - MarketEventV1, -} from '@dydxprotocol-indexer/v4-protos'; -import { createIndexerTendermintBlock, createIndexerTendermintEvent } from '../helpers/indexer-proto-helpers'; -import { - defaultAssetCreateEvent, - defaultDateTime, - defaultMarketCreate, - defaultTime, - defaultTxHash, -} from '../helpers/constants'; import { DydxIndexerSubtypes } from '../../src/lib/types'; -import { AssetCreationHandler } from '../../src/handlers/asset-handler'; -import { MarketCreateHandler } from '../../src/handlers/markets/market-create-handler'; -import { - AssetColumns, - AssetFromDatabase, - AssetTable, - BlockTable, - dbHelpers, - MarketColumns, - MarketFromDatabase, - MarketTable, - Ordering, - TendermintEventTable, - Transaction, -} from '@dydxprotocol-indexer/postgres'; import { KafkaPublisher } from '../../src/lib/kafka-publisher'; -import { createPostgresFunctions } from '../../src/helpers/postgres/postgres-functions'; - -const defaultMarketEventBinary: Uint8Array = Uint8Array.from(MarketEventV1.encode( - defaultMarketCreate, -).finish()); - -const defaultAssetEventBinary: Uint8Array = Uint8Array.from(AssetCreateEventV1.encode( - defaultAssetCreateEvent, -).finish()); +import * as pg from 'pg'; +import { mock, MockProxy } from 'jest-mock-extended'; +import { Handler } from '../../src/handlers/handler'; describe('syncHandler', () => { - const defaultTransactionIndex: number = 0; - const defaultMarketEvent: IndexerTendermintEvent = createIndexerTendermintEvent( - DydxIndexerSubtypes.MARKET, - defaultMarketEventBinary, - defaultTransactionIndex, - 0, - ); - - const defaultAssetEvent: IndexerTendermintEvent = createIndexerTendermintEvent( - DydxIndexerSubtypes.ASSET, - defaultAssetEventBinary, - defaultTransactionIndex, - 1, - ); - - const block: IndexerTendermintBlock = createIndexerTendermintBlock( - 1, - defaultTime, - [defaultMarketEvent, defaultAssetEvent], - [defaultTxHash], - ); - describe('addHandler/process', () => { - beforeAll(async () => { - await createPostgresFunctions(); - }); - - beforeEach(async () => { - await BlockTable.create({ - blockHeight: '1', - time: defaultDateTime.toISO(), - }); - await Promise.all([ - TendermintEventTable.create({ - blockHeight: '1', - transactionIndex: defaultTransactionIndex, - eventIndex: 0, - }), - TendermintEventTable.create({ - blockHeight: '1', - transactionIndex: defaultTransactionIndex, - eventIndex: 1, - }), - ]); - }); - - afterEach(async () => { - await dbHelpers.clearData(); - }); - - afterAll(async () => { - await dbHelpers.teardown(); - }); - it('successfully adds handler', async () => { - const synchHandlers: SyncHandlers = new SyncHandlers(); - const txId: number = await Transaction.start(); - const assetHandler = new AssetCreationHandler( - block, - defaultAssetEvent, - txId, - defaultAssetCreateEvent, - ); + const firstHandler: MockProxy> = mock>(); + firstHandler.blockEventIndex = 0; + const secondHandler: MockProxy> = mock>(); + secondHandler.blockEventIndex = 1; + const handlerNotInvoked: MockProxy> = mock>(); + handlerNotInvoked.blockEventIndex = 2; + const syncHandlers: SyncHandlers = new SyncHandlers(); - const marketHandler = new MarketCreateHandler( - block, - defaultMarketEvent, - txId, - defaultMarketCreate, - ); // handlers are processed in the order in which they are received. - synchHandlers.addHandler(DydxIndexerSubtypes.MARKET, marketHandler); - synchHandlers.addHandler(DydxIndexerSubtypes.ASSET, assetHandler); + syncHandlers.addHandler(DydxIndexerSubtypes.MARKET, firstHandler); + syncHandlers.addHandler(DydxIndexerSubtypes.ASSET, secondHandler); // should be ignored, because transfers are not handled by syncHandlers - synchHandlers.addHandler(DydxIndexerSubtypes.TRANSFER, assetHandler); - - const assets: AssetFromDatabase[] = await AssetTable.findAll( - {}, - [], { - orderBy: [[AssetColumns.id, Ordering.ASC]], - }); + syncHandlers.addHandler(DydxIndexerSubtypes.TRANSFER, handlerNotInvoked); - expect(assets.length).toEqual(0); - const markets: MarketFromDatabase[] = await MarketTable.findAll( - {}, - [], { - orderBy: [[MarketColumns.id, Ordering.ASC]], - }); - - expect(markets.length).toEqual(0); + const resultRow: pg.QueryResultRow = [ + 'forFirstHandler', + 'forSecondHandler', + 'forNotInvokedHandler', + ]; const kafkaPublisher: KafkaPublisher = new KafkaPublisher(); - await synchHandlers.process(kafkaPublisher); - await Transaction.commit(txId); - - // check that assets/markets are populated - const newAssets: AssetFromDatabase[] = await AssetTable.findAll( - {}, - [], { - orderBy: [[AssetColumns.id, Ordering.ASC]], - }); - - expect(newAssets.length).toEqual(1); - const newMarkets: MarketFromDatabase[] = await MarketTable.findAll( - {}, - [], { - orderBy: [[MarketColumns.id, Ordering.ASC]], - }); - - expect(newMarkets.length).toEqual(1); + await syncHandlers.process(kafkaPublisher, resultRow); + + expect(firstHandler.handle).toHaveBeenCalledWith('forFirstHandler'); + expect(firstHandler.handle).toHaveBeenCalledTimes(1); + expect(secondHandler.handle).toHaveBeenCalledWith('forSecondHandler'); + expect(secondHandler.handle).toHaveBeenCalledTimes(1); + expect(firstHandler.handle.mock.invocationCallOrder[0]).toBeLessThan( + secondHandler.handle.mock.invocationCallOrder[0]); + expect(handlerNotInvoked.handle).toHaveBeenCalledTimes(0); }); }); }); diff --git a/indexer/services/ender/__tests__/scripts/scripts.test.ts b/indexer/services/ender/__tests__/scripts/scripts.test.ts index 37957a64054..d365dcbf172 100644 --- a/indexer/services/ender/__tests__/scripts/scripts.test.ts +++ b/indexer/services/ender/__tests__/scripts/scripts.test.ts @@ -527,7 +527,7 @@ describe('SQL Function Tests', () => { const txHashes = [defaultTxHash, defaultTxHash2]; const dateTimeIso = '2020-01-01T00:00:00.000Z'; await getSingleRawQueryResultRow( - `SELECT dydx_create_initial_rows_for_tendermint_block('${blockHeight}'::text, '${dateTimeIso}'::text, ARRAY['${txHashes.join("','")}']::text[], ARRAY['${events.map((event) => JSON.stringify(event)).join("','")}']::jsonb[])`, + `SELECT dydx_create_initial_rows_for_tendermint_block(${blockHeight}, '${dateTimeIso}', '${JSON.stringify(txHashes)}', '${JSON.stringify(events)}')`, ); // Validate blocks table const blocks = await BlockTable.findAll({}, [], { readReplica: true }); diff --git a/indexer/services/ender/__tests__/validators/asset-validator.test.ts b/indexer/services/ender/__tests__/validators/asset-validator.test.ts index c093150713a..20ebf739452 100644 --- a/indexer/services/ender/__tests__/validators/asset-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/asset-validator.test.ts @@ -28,6 +28,7 @@ describe('asset-validator', () => { const validator: AssetValidator = new AssetValidator( defaultAssetCreateEvent, createBlock(defaultAssetCreateEvent), + 0, ); validator.validate(); diff --git a/indexer/services/ender/__tests__/validators/deleveraging-validator.test.ts b/indexer/services/ender/__tests__/validators/deleveraging-validator.test.ts index 93dc4347d63..5529a45aca3 100644 --- a/indexer/services/ender/__tests__/validators/deleveraging-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/deleveraging-validator.test.ts @@ -22,6 +22,7 @@ describe('deleveraging-validator', () => { const validator: DeleveragingValidator = new DeleveragingValidator( defaultDeleveragingEvent, createBlock(defaultDeleveragingEvent), + 0, ); validator.validate(); @@ -49,6 +50,7 @@ describe('deleveraging-validator', () => { const validator: DeleveragingValidator = new DeleveragingValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(message)); diff --git a/indexer/services/ender/__tests__/validators/funding-validator.test.ts b/indexer/services/ender/__tests__/validators/funding-validator.test.ts index 907e8140b6f..3d91fbe009d 100644 --- a/indexer/services/ender/__tests__/validators/funding-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/funding-validator.test.ts @@ -42,6 +42,7 @@ describe('funding-validator', () => { const validator: FundingValidator = new FundingValidator( event, createBlock(event), + 0, ); validator.validate(); @@ -83,6 +84,7 @@ describe('funding-validator', () => { const validator: FundingValidator = new FundingValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(message)); diff --git a/indexer/services/ender/__tests__/validators/liquidity-tier-validator.test.ts b/indexer/services/ender/__tests__/validators/liquidity-tier-validator.test.ts index 9c7443b617d..c1af377c40c 100644 --- a/indexer/services/ender/__tests__/validators/liquidity-tier-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/liquidity-tier-validator.test.ts @@ -33,6 +33,7 @@ describe('liquidity-tier-validator', () => { const validator: LiquidityTierValidator = new LiquidityTierValidator( defaultLiquidityTierUpsertEvent, createBlock(defaultLiquidityTierUpsertEvent), + 0, ); validator.validate(); @@ -60,6 +61,7 @@ describe('liquidity-tier-validator', () => { const validator: LiquidityTierValidator = new LiquidityTierValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(expectedMessage)); }); @@ -89,6 +91,7 @@ describe('liquidity-tier-validator', () => { const validator: LiquidityTierValidator = new LiquidityTierValidator( event, createBlock(event), + 0, ); validator.validate(); expect(loggerError).toHaveBeenCalledWith(expect.objectContaining({ diff --git a/indexer/services/ender/__tests__/validators/market-validator.test.ts b/indexer/services/ender/__tests__/validators/market-validator.test.ts index 0f9abaa4436..4742b95abed 100644 --- a/indexer/services/ender/__tests__/validators/market-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/market-validator.test.ts @@ -37,6 +37,7 @@ describe('market-validator', () => { const validator: MarketValidator = new MarketValidator( event, createBlock(event), + 0, ); validator.validate(); @@ -153,6 +154,7 @@ describe('market-validator', () => { const validator: MarketValidator = new MarketValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(message)); diff --git a/indexer/services/ender/__tests__/validators/order-fill-validator.test.ts b/indexer/services/ender/__tests__/validators/order-fill-validator.test.ts index 46e52c4e198..3cb81896b3f 100644 --- a/indexer/services/ender/__tests__/validators/order-fill-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/order-fill-validator.test.ts @@ -46,6 +46,7 @@ describe('order-fill-validator', () => { const validator: OrderFillValidator = new OrderFillValidator( event, createBlock(event), + 0, ); validator.validate(); @@ -173,6 +174,7 @@ describe('order-fill-validator', () => { const validator: OrderFillValidator = new OrderFillValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(message)); diff --git a/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts b/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts index 8f6a569c439..95848824d2c 100644 --- a/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts @@ -32,6 +32,7 @@ describe('perpetual-market-validator', () => { const validator: PerpetualMarketValidator = new PerpetualMarketValidator( defaultPerpetualMarketCreateEvent, createBlock(defaultPerpetualMarketCreateEvent), + 0, ); validator.validate(); @@ -43,6 +44,7 @@ describe('perpetual-market-validator', () => { const validator: PerpetualMarketValidator = new PerpetualMarketValidator( defaultPerpetualMarketCreateEvent, createBlock(defaultPerpetualMarketCreateEvent), + 0, ); const message: string = 'PerpetualMarketCreateEvent id already exists'; expect(() => validator.validate()).toThrow(new ParseMessageError(message)); @@ -77,6 +79,7 @@ describe('perpetual-market-validator', () => { const validator: PerpetualMarketValidator = new PerpetualMarketValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(expectedMessage)); }); diff --git a/indexer/services/ender/__tests__/validators/stateful-order-validator.test.ts b/indexer/services/ender/__tests__/validators/stateful-order-validator.test.ts index 38092de371f..b6d074706fd 100644 --- a/indexer/services/ender/__tests__/validators/stateful-order-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/stateful-order-validator.test.ts @@ -46,6 +46,7 @@ describe('stateful-order-validator', () => { const validator: StatefulOrderValidator = new StatefulOrderValidator( event, createBlock(event), + 0, ); validator.validate(); @@ -346,6 +347,7 @@ describe('stateful-order-validator', () => { const validator: StatefulOrderValidator = new StatefulOrderValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(message)); diff --git a/indexer/services/ender/__tests__/validators/subaccount-update-validator.test.ts b/indexer/services/ender/__tests__/validators/subaccount-update-validator.test.ts index a851a96bb1f..bac021ef474 100644 --- a/indexer/services/ender/__tests__/validators/subaccount-update-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/subaccount-update-validator.test.ts @@ -27,6 +27,7 @@ describe('subaccount-update-validator', () => { const validator: SubaccountUpdateValidator = new SubaccountUpdateValidator( defaultEmptySubaccountUpdateEvent, createBlock(defaultEmptySubaccountUpdateEvent), + 0, ); validator.validate(); @@ -41,6 +42,7 @@ describe('subaccount-update-validator', () => { const validator: SubaccountUpdateValidator = new SubaccountUpdateValidator( invalidEvent, createBlock(invalidEvent), + 0, ); const message: string = 'SubaccountUpdateEvent must contain a subaccountId'; diff --git a/indexer/services/ender/__tests__/validators/transfer-validator.test.ts b/indexer/services/ender/__tests__/validators/transfer-validator.test.ts index 4d5085ba374..ca271982818 100644 --- a/indexer/services/ender/__tests__/validators/transfer-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/transfer-validator.test.ts @@ -31,6 +31,7 @@ describe('transfer-validator', () => { const validator: TransferValidator = new TransferValidator( defaultTransferEvent, createBlock(defaultTransferEvent), + 0, ); validator.validate(); @@ -93,6 +94,7 @@ describe('transfer-validator', () => { const validator: TransferValidator = new TransferValidator( event, createBlock(event), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError(message)); diff --git a/indexer/services/ender/__tests__/validators/update-clob-pair-validator.test.ts b/indexer/services/ender/__tests__/validators/update-clob-pair-validator.test.ts index 4fe4fe46e40..7b2b841f441 100644 --- a/indexer/services/ender/__tests__/validators/update-clob-pair-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/update-clob-pair-validator.test.ts @@ -47,6 +47,7 @@ describe('update-clob-pair-validator', () => { const validator: UpdateClobPairValidator = new UpdateClobPairValidator( defaultUpdateClobPairEvent, createBlock(defaultUpdateClobPairEvent), + 0, ); validator.validate(); @@ -60,6 +61,7 @@ describe('update-clob-pair-validator', () => { clobPairId: 20, }, createBlock(defaultUpdateClobPairEvent), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError( diff --git a/indexer/services/ender/__tests__/validators/update-perpetual-validator.test.ts b/indexer/services/ender/__tests__/validators/update-perpetual-validator.test.ts index c7ca30bb808..8ba71284942 100644 --- a/indexer/services/ender/__tests__/validators/update-perpetual-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/update-perpetual-validator.test.ts @@ -47,6 +47,7 @@ describe('update-perpetual-validator', () => { const validator: UpdatePerpetualValidator = new UpdatePerpetualValidator( defaultUpdatePerpetualEvent, createBlock(defaultUpdatePerpetualEvent), + 0, ); validator.validate(); @@ -60,6 +61,7 @@ describe('update-perpetual-validator', () => { id: 20, }, createBlock(defaultUpdatePerpetualEvent), + 0, ); expect(() => validator.validate()).toThrow(new ParseMessageError( diff --git a/indexer/services/ender/src/handlers/abstract-stateful-order-handler.ts b/indexer/services/ender/src/handlers/abstract-stateful-order-handler.ts index e2dfb35283f..7d275d57431 100644 --- a/indexer/services/ender/src/handlers/abstract-stateful-order-handler.ts +++ b/indexer/services/ender/src/handlers/abstract-stateful-order-handler.ts @@ -1,18 +1,3 @@ -import { logger } from '@dydxprotocol-indexer/base'; -import { - OrderFromDatabase, - PerpetualMarketFromDatabase, - storeHelpers, - OrderModel, - PerpetualMarketModel, - SubaccountFromDatabase, -} from '@dydxprotocol-indexer/postgres'; -import SubaccountModel from '@dydxprotocol-indexer/postgres/build/src/models/subaccount-model'; -import { - StatefulOrderEventV1, -} from '@dydxprotocol-indexer/v4-protos'; -import * as pg from 'pg'; - import { STATEFUL_ORDER_ORDER_FILL_EVENT_TYPE } from '../constants'; import { Handler } from './handler'; @@ -25,35 +10,4 @@ export abstract class AbstractStatefulOrderHandler extends Handler { `${STATEFUL_ORDER_ORDER_FILL_EVENT_TYPE}_${orderId}`, ]; } - - protected async handleEventViaSqlFunction(): - Promise<[OrderFromDatabase, - PerpetualMarketFromDatabase, - SubaccountFromDatabase | undefined]> { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_stateful_order_handler( - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(StatefulOrderEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'AbstractStatefulOrderHandler#handleEventViaSqlFunction', - message: 'Failed to handle StatefulOrderEventV1', - error, - }); - throw error; - }); - - return [ - OrderModel.fromJson(result.rows[0].result.order) as OrderFromDatabase, - PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase, - result.rows[0].result.subaccount - ? SubaccountModel.fromJson(result.rows[0].result.subaccount) as SubaccountFromDatabase - : undefined, - ]; - } } diff --git a/indexer/services/ender/src/handlers/asset-handler.ts b/indexer/services/ender/src/handlers/asset-handler.ts index 70c488e9d5c..b505f6af5ca 100644 --- a/indexer/services/ender/src/handlers/asset-handler.ts +++ b/indexer/services/ender/src/handlers/asset-handler.ts @@ -1,9 +1,7 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { AssetFromDatabase, AssetModel, assetRefresher, - storeHelpers, } from '@dydxprotocol-indexer/postgres'; import { AssetCreateEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; @@ -19,25 +17,9 @@ export class AssetCreationHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_asset_create_handler( - '${JSON.stringify(AssetCreateEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'AssetCreationHandler#internalHandle', - message: 'Failed to handle AssetCreateEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const asset: AssetFromDatabase = AssetModel.fromJson( - result.rows[0].result.asset) as AssetFromDatabase; + resultRow.asset) as AssetFromDatabase; assetRefresher.addAsset(asset); return []; } diff --git a/indexer/services/ender/src/handlers/funding-handler.ts b/indexer/services/ender/src/handlers/funding-handler.ts index 454c91223c2..772ae562a01 100644 --- a/indexer/services/ender/src/handlers/funding-handler.ts +++ b/indexer/services/ender/src/handlers/funding-handler.ts @@ -4,12 +4,10 @@ import { PerpetualMarketFromDatabase, TendermintEventTable, protocolTranslations, - storeHelpers, PerpetualMarketModel, } from '@dydxprotocol-indexer/postgres'; import { NextFundingCache } from '@dydxprotocol-indexer/redis'; import { - FundingEventV1, FundingEventV1_Type, FundingUpdateV1, } from '@dydxprotocol-indexer/v4-protos'; @@ -49,33 +47,10 @@ export class FundingHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const transactionIndex: number = indexerTendermintEventToTransactionIndex( - this.indexerTendermintEvent, - ); - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_funding_handler( - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(FundingEventV1.decode(eventDataBinary))}', - ${this.indexerTendermintEvent.eventIndex}, - ${transactionIndex} - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'FundingHandler#internalHandle', - message: 'Failed to handle FundingEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const perpetualMarkets: Map = new Map(); - for (const [key, perpetualMarket] of Object.entries(result.rows[0].result.perpetual_markets)) { + for (const [key, perpetualMarket] of Object.entries(resultRow.perpetual_markets)) { perpetualMarkets.set( key, PerpetualMarketModel.fromJson(perpetualMarket as object) as PerpetualMarketFromDatabase, @@ -86,10 +61,10 @@ export class FundingHandler extends Handler { for (let i: number = 0; i < this.event.updates.length; i++) { const update: FundingUpdateV1 = this.event.updates[i]; - if (result.rows[0].result.errors[i] != null) { + if (resultRow.errors[i] != null) { logger.error({ at: 'FundingHandler#handleFundingSample', - message: result.rows[0].result.errors[i], + message: resultRow.errors[i], update, }); continue; diff --git a/indexer/services/ender/src/handlers/handler.ts b/indexer/services/ender/src/handlers/handler.ts index 95fa0a54a7a..d7a40a9fa93 100644 --- a/indexer/services/ender/src/handlers/handler.ts +++ b/indexer/services/ender/src/handlers/handler.ts @@ -19,6 +19,7 @@ import { SubaccountId, } from '@dydxprotocol-indexer/v4-protos'; import { DateTime } from 'luxon'; +import * as pg from 'pg'; import config from '../config'; import { indexerTendermintEventToTransactionIndex } from '../lib/helper'; @@ -28,6 +29,7 @@ import { export type HandlerInitializer = new ( block: IndexerTendermintBlock, + blockEventIndex: number, indexerTendermintEvent: IndexerTendermintEvent, txId: number, event: EventMessage, @@ -43,16 +45,19 @@ export abstract class Handler { indexerTendermintEvent: IndexerTendermintEvent; timestamp: DateTime; txId: number; + blockEventIndex: number; event: T; abstract eventType: string; constructor( block: IndexerTendermintBlock, + blockEventIndex: number, indexerTendermintEvent: IndexerTendermintEvent, txId: number, event: T, ) { this.block = block; + this.blockEventIndex = blockEventIndex; this.indexerTendermintEvent = indexerTendermintEvent; this.timestamp = DateTime.fromJSDate(block.time!); this.txId = txId; @@ -66,19 +71,21 @@ export abstract class Handler { public abstract getParallelizationIds(): string[]; /** - * Processes the event and updates Postgres in the transaction specified by - * txId provided in the constructor, then returns all consolidated kafka events to be - * written to Kafka. + * Performs side effects based upon the results returned from the SQL based handler + * implementations and then returns all consolidated Kafka events to be written to Kafka. */ - public abstract internalHandle(): Promise; + public abstract internalHandle(resultRow: pg.QueryResultRow): Promise; /** - * Handle the event and export timing stats + * Performs side effects based upon the results returned from the SQL based handler + * implementations and then returns all consolidated Kafka events to be written to Kafka. + * + * Wraps internalHandle with timing information. */ - public async handle(): Promise { + public async handle(resultRow: pg.QueryResultRow): Promise { const start: number = Date.now(); try { - return await this.internalHandle(); + return await this.internalHandle(resultRow); } finally { stats.timing( `${config.SERVICE_NAME}.handle_event.timing`, diff --git a/indexer/services/ender/src/handlers/liquidity-tier-handler.ts b/indexer/services/ender/src/handlers/liquidity-tier-handler.ts index 1e77d46b2b7..2c89d9f66f0 100644 --- a/indexer/services/ender/src/handlers/liquidity-tier-handler.ts +++ b/indexer/services/ender/src/handlers/liquidity-tier-handler.ts @@ -1,11 +1,9 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { LiquidityTiersFromDatabase, LiquidityTiersModel, PerpetualMarketFromDatabase, liquidityTierRefresher, perpetualMarketRefresher, - storeHelpers, } from '@dydxprotocol-indexer/postgres'; import { LiquidityTierUpsertEventV1 } from '@dydxprotocol-indexer/v4-protos'; import _ from 'lodash'; @@ -23,25 +21,9 @@ export class LiquidityTierHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_liquidity_tier_handler( - '${JSON.stringify(LiquidityTierUpsertEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'LiquidityTierHandler#internalHandle', - message: 'Failed to handle LiquidityTierUpsertEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const liquidityTier: LiquidityTiersFromDatabase = LiquidityTiersModel.fromJson( - result.rows[0].result.liquidity_tier) as LiquidityTiersFromDatabase; + resultRow.liquidity_tier) as LiquidityTiersFromDatabase; liquidityTierRefresher.upsertLiquidityTier(liquidityTier); return this.generateWebsocketEventsForLiquidityTier(liquidityTier); } diff --git a/indexer/services/ender/src/handlers/markets/market-create-handler.ts b/indexer/services/ender/src/handlers/markets/market-create-handler.ts index 77be14834ea..6f539e3665a 100644 --- a/indexer/services/ender/src/handlers/markets/market-create-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-create-handler.ts @@ -1,10 +1,8 @@ import { logger } from '@dydxprotocol-indexer/base'; -import { - storeHelpers, -} from '@dydxprotocol-indexer/postgres'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; +import * as pg from 'pg'; -import { ConsolidatedKafkaEvent, MarketCreateEventMessage } from '../../lib/types'; +import { ConsolidatedKafkaEvent } from '../../lib/types'; import { Handler } from '../handler'; export class MarketCreateHandler extends Handler { @@ -15,37 +13,14 @@ export class MarketCreateHandler extends Handler { return [`${this.eventType}_${this.event.marketId}`]; } - public async internalHandle(): Promise { + // eslint-disable-next-line @typescript-eslint/require-await + public async internalHandle(_: pg.QueryResultRow): Promise { logger.info({ at: 'MarketCreateHandler#handle', message: 'Received MarketEvent with MarketCreate.', event: this.event, }); - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - await storeHelpers.rawQuery( - `SELECT dydx_market_create_handler( - '${JSON.stringify(MarketEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'MarketCreateHandler#internalHandle', - message: 'Failed to handle MarketEventV1', - error, - }); - - if (error.message.includes('Market in MarketCreate already exists')) { - const marketCreate: MarketCreateEventMessage = this.event as MarketCreateEventMessage; - this.logAndThrowParseMessageError( - 'Market in MarketCreate already exists', - { marketCreate }, - ); - } - - throw error; - }); - return []; } } diff --git a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts index 33f5c963c3a..eeee9188e3c 100644 --- a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts @@ -1,8 +1,8 @@ import { logger } from '@dydxprotocol-indexer/base'; -import { storeHelpers } from '@dydxprotocol-indexer/postgres'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; +import * as pg from 'pg'; -import { ConsolidatedKafkaEvent, MarketModifyEventMessage } from '../../lib/types'; +import { ConsolidatedKafkaEvent } from '../../lib/types'; import { Handler } from '../handler'; export class MarketModifyHandler extends Handler { @@ -13,42 +13,14 @@ export class MarketModifyHandler extends Handler { return [`${this.eventType}_${this.event.marketId}`]; } - public async internalHandle(): Promise { + // eslint-disable-next-line @typescript-eslint/require-await + public async internalHandle(_: pg.QueryResultRow): Promise { logger.info({ at: 'MarketModifyHandler#handle', - message: 'Received MarketEvent with MarketCreate.', + message: 'Received MarketEvent with MarketModify.', event: this.event, }); - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - await storeHelpers.rawQuery( - `SELECT dydx_market_modify_handler( - '${JSON.stringify(MarketEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'MarketModifyHandler#internalHandle', - message: 'Failed to handle MarketEventV1', - error, - }); - - const castedMarketModifyMessage: - MarketModifyEventMessage = this.event as MarketModifyEventMessage; - - if (error.message.includes('Market in MarketModify doesn\'t exist')) { - this.logAndThrowParseMessageError( - 'Market in MarketModify doesn\'t exist', - { castedMarketModifyMessage }, - ); - } - - this.logAndThrowParseMessageError( - 'Failed to update market in markets table', - { castedMarketModifyMessage }, - ); - }); - return []; } } diff --git a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts index 4e3ddfd9dd1..56c9978fd9a 100644 --- a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts @@ -3,7 +3,7 @@ import { MarketFromDatabase, OraclePriceFromDatabase, OraclePriceModel, - MarketMessageContents, storeHelpers, MarketModel, + MarketMessageContents, MarketModel, } from '@dydxprotocol-indexer/postgres'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; @@ -11,7 +11,6 @@ import * as pg from 'pg'; import { generateOraclePriceContents } from '../../helpers/kafka-helper'; import { ConsolidatedKafkaEvent, - MarketPriceUpdateEventMessage, } from '../../lib/types'; import { Handler } from '../handler'; @@ -23,44 +22,18 @@ export class MarketPriceUpdateHandler extends Handler { return [`${this.eventType}_${this.event.marketId}`]; } - public async internalHandle(): Promise { + // eslint-disable-next-line @typescript-eslint/require-await + public async internalHandle(resultRow: pg.QueryResultRow): Promise { logger.info({ at: 'MarketPriceUpdateHandler#handle', message: 'Received MarketEvent with MarketPriceUpdate.', event: this.event, }); - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_market_price_update_handler( - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(MarketEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'MarketPriceUpdateHandler#internalHandle', - message: 'Failed to handle MarketEventV1', - error, - }); - - if (error.message.includes('MarketPriceUpdateEvent contains a non-existent market id')) { - const castedMarketPriceUpdateMessage: - MarketPriceUpdateEventMessage = this.event as MarketPriceUpdateEventMessage; - this.logAndThrowParseMessageError( - 'MarketPriceUpdateEvent contains a non-existent market id', - { castedMarketPriceUpdateMessage }, - ); - } - - throw error; - }); - const market: MarketFromDatabase = MarketModel.fromJson( - result.rows[0].result.market) as MarketFromDatabase; + resultRow.market) as MarketFromDatabase; const oraclePrice: OraclePriceFromDatabase = OraclePriceModel.fromJson( - result.rows[0].result.oracle_price) as OraclePriceFromDatabase; + resultRow.oracle_price) as OraclePriceFromDatabase; return [ this.generateKafkaEvent( diff --git a/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts b/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts index 2c8dbd2dc10..712779a6a74 100644 --- a/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts @@ -7,14 +7,12 @@ import { perpetualMarketRefresher, PerpetualPositionFromDatabase, PerpetualPositionModel, - storeHelpers, SubaccountTable, } from '@dydxprotocol-indexer/postgres'; import { DeleveragingEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; import { SUBACCOUNT_ORDER_FILL_EVENT_TYPE } from '../../constants'; -import { indexerTendermintEventToTransactionIndex } from '../../lib/helper'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { AbstractOrderFillHandler } from './abstract-order-fill-handler'; @@ -48,41 +46,19 @@ export class DeleveragingHandler extends AbstractOrderFillHandler { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const transactionIndex: number = indexerTendermintEventToTransactionIndex( - this.indexerTendermintEvent, - ); - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_deleveraging_handler( - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(DeleveragingEventV1.decode(eventDataBinary))}', - ${this.indexerTendermintEvent.eventIndex}, - ${transactionIndex}, - '${this.block.txHashes[transactionIndex]}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'DeleveragingHandler#internalHandle', - message: 'Failed to handle DeleveragingEventV1', - error, - }); - throw error; - }); + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const liquidatedFill: FillFromDatabase = FillModel.fromJson( - result.rows[0].result.liquidated_fill) as FillFromDatabase; + resultRow.liquidated_fill) as FillFromDatabase; const offsettingFill: FillFromDatabase = FillModel.fromJson( - result.rows[0].result.offsetting_fill) as FillFromDatabase; + resultRow.offsetting_fill) as FillFromDatabase; const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase; + resultRow.perpetual_market) as PerpetualMarketFromDatabase; const liquidatedPerpetualPosition: PerpetualPositionFromDatabase = PerpetualPositionModel.fromJson( - result.rows[0].result.liquidated_perpetual_position) as PerpetualPositionFromDatabase; + resultRow.liquidated_perpetual_position) as PerpetualPositionFromDatabase; const offsettingPerpetualPosition: PerpetualPositionFromDatabase = PerpetualPositionModel.fromJson( - result.rows[0].result.offsetting_perpetual_position) as PerpetualPositionFromDatabase; + resultRow.offsetting_perpetual_position) as PerpetualPositionFromDatabase; const kafkaEvents: ConsolidatedKafkaEvent[] = [ this.generateConsolidatedKafkaEvent( this.event.liquidated!, diff --git a/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts b/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts index 83bf565f6f7..4b9b7efb99a 100644 --- a/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts @@ -1,4 +1,3 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { FillFromDatabase, FillModel, @@ -10,15 +9,13 @@ import { PerpetualMarketModel, PerpetualPositionFromDatabase, PerpetualPositionModel, - storeHelpers, SubaccountTable, - USDC_ASSET_ID, - OrderStatus, FillType, + OrderStatus, } from '@dydxprotocol-indexer/postgres'; import { StateFilledQuantumsCache } from '@dydxprotocol-indexer/redis'; import { isStatefulOrder } from '@dydxprotocol-indexer/v4-proto-parser'; import { - LiquidationOrderV1, IndexerOrderId, OrderFillEventV1, + LiquidationOrderV1, IndexerOrderId, } from '@dydxprotocol-indexer/v4-protos'; import Long from 'long'; import * as pg from 'pg'; @@ -29,7 +26,6 @@ import { redisClient } from '../../helpers/redis/redis-controller'; import { orderFillWithLiquidityToOrderFillEventWithLiquidation, } from '../../helpers/translation-helper'; -import { indexerTendermintEventToTransactionIndex } from '../../lib/helper'; import { OrderFillWithLiquidity } from '../../lib/translated-types'; import { ConsolidatedKafkaEvent, @@ -84,56 +80,26 @@ export class LiquidationHandler extends AbstractOrderFillHandler { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const transactionIndex: number = indexerTendermintEventToTransactionIndex( - this.indexerTendermintEvent, - ); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const castedLiquidationFillEventMessage: OrderFillEventWithLiquidation = orderFillWithLiquidityToOrderFillEventWithLiquidation( this.event, ); const field: string = this.event.liquidity === Liquidity.MAKER ? 'makerOrder' : 'liquidationOrder'; - const fillType: string = this.event.liquidity === Liquidity.MAKER - ? FillType.LIQUIDATION : FillType.LIQUIDATED; - - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_liquidation_fill_handler_per_order( - '${field}', - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(OrderFillEventV1.decode(eventDataBinary))}', - ${this.indexerTendermintEvent.eventIndex}, - ${transactionIndex}, - '${this.block.txHashes[transactionIndex]}', - '${this.event.liquidity}', - '${fillType}', - '${USDC_ASSET_ID}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'liquidationHandler#internalHandle', - message: 'Failed to handle OrderFillEventV1', - error, - }); - throw error; - }); const fill: FillFromDatabase = FillModel.fromJson( - result.rows[0].result.fill) as FillFromDatabase; + resultRow[field].fill) as FillFromDatabase; const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase; + resultRow[field].perpetual_market) as PerpetualMarketFromDatabase; const position: PerpetualPositionFromDatabase = PerpetualPositionModel.fromJson( - result.rows[0].result.perpetual_position) as PerpetualPositionFromDatabase; + resultRow[field].perpetual_position) as PerpetualPositionFromDatabase; if (this.event.liquidity === Liquidity.MAKER) { // Must be done in this order, because fills refer to an order // We do not create a taker order for liquidations. const makerOrder: OrderFromDatabase = OrderModel.fromJson( - result.rows[0].result.order) as OrderFromDatabase; + resultRow[field].order) as OrderFromDatabase; // Update the cache tracking the state-filled amount per order for use in vulcan await StateFilledQuantumsCache.updateStateFilledQuantums( diff --git a/indexer/services/ender/src/handlers/order-fills/order-handler.ts b/indexer/services/ender/src/handlers/order-fills/order-handler.ts index c3be2a5b41f..5b0e6198bd8 100644 --- a/indexer/services/ender/src/handlers/order-fills/order-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/order-handler.ts @@ -1,4 +1,3 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { FillFromDatabase, FillModel, @@ -10,15 +9,13 @@ import { PerpetualMarketModel, PerpetualPositionFromDatabase, PerpetualPositionModel, - storeHelpers, SubaccountTable, - USDC_ASSET_ID, OrderStatus, } from '@dydxprotocol-indexer/postgres'; -import { CanceledOrderStatus, CanceledOrdersCache, StateFilledQuantumsCache } from '@dydxprotocol-indexer/redis'; +import { StateFilledQuantumsCache } from '@dydxprotocol-indexer/redis'; import { isStatefulOrder } from '@dydxprotocol-indexer/v4-proto-parser'; import { - OrderFillEventV1, IndexerOrderId, IndexerSubaccountId, IndexerOrder, + IndexerOrderId, IndexerSubaccountId, IndexerOrder, } from '@dydxprotocol-indexer/v4-protos'; import Long from 'long'; import * as pg from 'pg'; @@ -27,7 +24,6 @@ import { STATEFUL_ORDER_ORDER_FILL_EVENT_TYPE, SUBACCOUNT_ORDER_FILL_EVENT_TYPE import { convertPerpetualPosition } from '../../helpers/kafka-helper'; import { redisClient } from '../../helpers/redis/redis-controller'; import { orderFillWithLiquidityToOrderFillEventWithOrder } from '../../helpers/translation-helper'; -import { indexerTendermintEventToTransactionIndex } from '../../lib/helper'; import { OrderFillWithLiquidity } from '../../lib/translated-types'; import { ConsolidatedKafkaEvent, OrderFillEventWithOrder } from '../../lib/types'; import { AbstractOrderFillHandler } from './abstract-order-fill-handler'; @@ -58,17 +54,7 @@ export class OrderHandler extends AbstractOrderFillHandler { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const transactionIndex: number = indexerTendermintEventToTransactionIndex( - this.indexerTendermintEvent, - ); + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const kafkaEvents: ConsolidatedKafkaEvent[] = []; const castedOrderFillEventMessage: @@ -78,44 +64,14 @@ export class OrderHandler extends AbstractOrderFillHandler { - logger.error({ - at: 'orderHandler#internalHandle', - message: 'Failed to handle OrderFillEventV1', - error, - }); - throw error; - }); const order: OrderFromDatabase = OrderModel.fromJson( - result.rows[0].result.order) as OrderFromDatabase; + resultRow[field].order) as OrderFromDatabase; const fill: FillFromDatabase = FillModel.fromJson( - result.rows[0].result.fill) as FillFromDatabase; + resultRow[field].fill) as FillFromDatabase; const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase; + resultRow[field].perpetual_market) as PerpetualMarketFromDatabase; const position: PerpetualPositionFromDatabase = PerpetualPositionModel.fromJson( - result.rows[0].result.perpetual_position) as PerpetualPositionFromDatabase; + resultRow[field].perpetual_position) as PerpetualPositionFromDatabase; let subaccountId: IndexerSubaccountId; if (this.event.liquidity === Liquidity.MAKER) { @@ -162,4 +118,10 @@ export class OrderHandler extends AbstractOrderFillHandler { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_perpetual_market_handler( - '${JSON.stringify(PerpetualMarketCreateEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'PerpetualMarketCreationHandler#internalHandle', - message: 'Failed to handle PerpetualMarketCreateEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase; + resultRow.perpetual_market) as PerpetualMarketFromDatabase; perpetualMarketRefresher.upsertPerpetualMarket(perpetualMarket); return [ diff --git a/indexer/services/ender/src/handlers/stateful-order/conditional-order-placement-handler.ts b/indexer/services/ender/src/handlers/stateful-order/conditional-order-placement-handler.ts index dadc145cd9f..83d09718840 100644 --- a/indexer/services/ender/src/handlers/stateful-order/conditional-order-placement-handler.ts +++ b/indexer/services/ender/src/handlers/stateful-order/conditional-order-placement-handler.ts @@ -1,14 +1,14 @@ import { - OrderFromDatabase, + OrderFromDatabase, OrderModel, OrderTable, - PerpetualMarketFromDatabase, - SubaccountFromDatabase, + PerpetualMarketFromDatabase, PerpetualMarketModel, SubaccountMessageContents, } from '@dydxprotocol-indexer/postgres'; import { IndexerSubaccountId, StatefulOrderEventV1, } from '@dydxprotocol-indexer/v4-protos'; +import * as pg from 'pg'; import { generateOrderSubaccountMessage } from '../../helpers/kafka-helper'; import { ConsolidatedKafkaEvent } from '../../lib/types'; @@ -26,15 +26,14 @@ export class ConditionalOrderPlacementHandler extends } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const result: - [OrderFromDatabase, - PerpetualMarketFromDatabase, - SubaccountFromDatabase | undefined] = await this.handleEventViaSqlFunction(); + public async internalHandle(resultRow: pg.QueryResultRow): Promise { + const order: OrderFromDatabase = OrderModel.fromJson(resultRow.order) as OrderFromDatabase; + const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( + resultRow.perpetual_market) as PerpetualMarketFromDatabase; const subaccountId: IndexerSubaccountId = this.event.conditionalOrderPlacement!.order!.orderId!.subaccountId!; - return this.createKafkaEvents(subaccountId, result[0], result[1]); + return this.createKafkaEvents(subaccountId, order, perpetualMarket); } private createKafkaEvents( diff --git a/indexer/services/ender/src/handlers/stateful-order/conditional-order-triggered-handler.ts b/indexer/services/ender/src/handlers/stateful-order/conditional-order-triggered-handler.ts index 99a3521a08f..0cab1392da7 100644 --- a/indexer/services/ender/src/handlers/stateful-order/conditional-order-triggered-handler.ts +++ b/indexer/services/ender/src/handlers/stateful-order/conditional-order-triggered-handler.ts @@ -3,8 +3,9 @@ import { OrderTable, PerpetualMarketFromDatabase, orderTranslations, - SubaccountFromDatabase, + SubaccountFromDatabase, OrderModel, PerpetualMarketModel, } from '@dydxprotocol-indexer/postgres'; +import SubaccountModel from '@dydxprotocol-indexer/postgres/build/src/models/subaccount-model'; import { getOrderIdHash } from '@dydxprotocol-indexer/v4-proto-parser'; import { IndexerOrder, @@ -12,6 +13,7 @@ import { OrderPlaceV1_OrderPlacementStatus, StatefulOrderEventV1, } from '@dydxprotocol-indexer/v4-protos'; +import * as pg from 'pg'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { AbstractStatefulOrderHandler } from '../abstract-stateful-order-handler'; @@ -28,15 +30,16 @@ export class ConditionalOrderTriggeredHandler extends } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const result: - [OrderFromDatabase, - PerpetualMarketFromDatabase, - SubaccountFromDatabase | undefined] = await this.handleEventViaSqlFunction(); + public async internalHandle(resultRow: pg.QueryResultRow): Promise { + const order: OrderFromDatabase = OrderModel.fromJson(resultRow.order) as OrderFromDatabase; + const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( + resultRow.perpetual_market) as PerpetualMarketFromDatabase; + const subaccount: SubaccountFromDatabase = SubaccountModel.fromJson( + resultRow.subaccount) as SubaccountFromDatabase; - const order: IndexerOrder = orderTranslations.convertToIndexerOrderWithSubaccount( - result[0], result[1], result[2]!); - return this.createKafkaEvents(order); + const indexerOrder: IndexerOrder = orderTranslations.convertToIndexerOrderWithSubaccount( + order, perpetualMarket, subaccount); + return this.createKafkaEvents(indexerOrder); } private createKafkaEvents(order: IndexerOrder): ConsolidatedKafkaEvent[] { diff --git a/indexer/services/ender/src/handlers/stateful-order/stateful-order-placement-handler.ts b/indexer/services/ender/src/handlers/stateful-order/stateful-order-placement-handler.ts index aea4b59152b..a2ea67f5788 100644 --- a/indexer/services/ender/src/handlers/stateful-order/stateful-order-placement-handler.ts +++ b/indexer/services/ender/src/handlers/stateful-order/stateful-order-placement-handler.ts @@ -8,6 +8,7 @@ import { IndexerOrder, StatefulOrderEventV1, } from '@dydxprotocol-indexer/v4-protos'; +import * as pg from 'pg'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { AbstractStatefulOrderHandler } from '../abstract-stateful-order-handler'; @@ -30,9 +31,7 @@ export class StatefulOrderPlacementHandler extends } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - await this.handleEventViaSqlFunction(); - + public async internalHandle(_: pg.QueryResultRow): Promise { let order: IndexerOrder; // TODO(IND-334): Remove after deprecating StatefulOrderPlacementEvent if (this.event.orderPlace !== undefined) { diff --git a/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts b/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts index 69a11bddedc..5a162c850a5 100644 --- a/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts +++ b/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts @@ -8,6 +8,7 @@ import { OrderRemoveV1_OrderRemovalStatus, StatefulOrderEventV1, } from '@dydxprotocol-indexer/v4-protos'; +import * as pg from 'pg'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { AbstractStatefulOrderHandler } from '../abstract-stateful-order-handler'; @@ -23,9 +24,8 @@ export class StatefulOrderRemovalHandler extends } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { + public async internalHandle(_: pg.QueryResultRow): Promise { const orderIdProto: IndexerOrderId = this.event.orderRemoval!.removedOrderId!; - await this.handleEventViaSqlFunction(); return this.createKafkaEvents(orderIdProto); } diff --git a/indexer/services/ender/src/handlers/subaccount-update-handler.ts b/indexer/services/ender/src/handlers/subaccount-update-handler.ts index 43fac7a6d96..44443799cc7 100644 --- a/indexer/services/ender/src/handlers/subaccount-update-handler.ts +++ b/indexer/services/ender/src/handlers/subaccount-update-handler.ts @@ -1,4 +1,3 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { AssetPositionFromDatabase, AssetPositionModel, @@ -11,7 +10,6 @@ import { perpetualMarketRefresher, PerpetualMarketsMap, PerpetualPositionModel, - storeHelpers, SubaccountMessageContents, SubaccountTable, UpdatedPerpetualPositionSubaccountKafkaObject, @@ -21,7 +19,6 @@ import * as pg from 'pg'; import { SUBACCOUNT_ORDER_FILL_EVENT_TYPE } from '../constants'; import { addPositionsToContents, annotateWithPnl } from '../helpers/kafka-helper'; -import { indexerTendermintEventToTransactionIndex } from '../lib/helper'; import { SubaccountUpdate } from '../lib/translated-types'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -39,33 +36,14 @@ export class SubaccountUpdateHandler extends Handler { ]; } - public async internalHandle(): Promise { - const transactionIndex: number = indexerTendermintEventToTransactionIndex( - this.indexerTendermintEvent, - ); - - const result: pg.QueryResult = await storeHelpers.rawQuery(`SELECT dydx_subaccount_update_handler( - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(this.event)}', - ${this.indexerTendermintEvent.eventIndex}, - ${transactionIndex}) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'subaccountUpdateHandler#internalHandle', - message: 'Failed to handle SubaccountUpdateEventV1', - error, - }); - throw error; - }); + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const updateObjects: UpdatedPerpetualPositionSubaccountKafkaObject[] = _.map( - result.rows[0].result.perpetual_positions, + resultRow.perpetual_positions, (value) => PerpetualPositionModel.fromJson( value) as UpdatedPerpetualPositionSubaccountKafkaObject, ); const updatedAssetPositions: AssetPositionFromDatabase[] = _.map( - result.rows[0].result.asset_positions, + resultRow.asset_positions, (value) => AssetPositionModel.fromJson(value) as AssetPositionFromDatabase, ); const markets: MarketFromDatabase[] = await MarketTable.findAll( diff --git a/indexer/services/ender/src/handlers/transfer-handler.ts b/indexer/services/ender/src/handlers/transfer-handler.ts index 975a7eb9cb6..dcd24ad38f5 100644 --- a/indexer/services/ender/src/handlers/transfer-handler.ts +++ b/indexer/services/ender/src/handlers/transfer-handler.ts @@ -1,8 +1,6 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { AssetFromDatabase, AssetModel, - storeHelpers, SubaccountMessageContents, TransferFromDatabase, TransferModel, @@ -11,7 +9,6 @@ import { TransferEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; import { generateTransferContents } from '../helpers/kafka-helper'; -import { indexerTendermintEventToTransactionIndex } from '../lib/helper'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -24,35 +21,11 @@ export class TransferHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const transactionIndex: number = indexerTendermintEventToTransactionIndex( - this.indexerTendermintEvent, - ); - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_transfer_handler( - ${this.block.height}, - '${this.block.time?.toISOString()}', - '${JSON.stringify(TransferEventV1.decode(eventDataBinary))}', - ${this.indexerTendermintEvent.eventIndex}, - ${transactionIndex}, - '${this.block.txHashes[transactionIndex]}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'TransferHandler#internalHandle', - message: 'Failed to handle TransferEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const asset: AssetFromDatabase = AssetModel.fromJson( - result.rows[0].result.asset) as AssetFromDatabase; + resultRow.asset) as AssetFromDatabase; const transfer: TransferFromDatabase = TransferModel.fromJson( - result.rows[0].result.transfer) as TransferFromDatabase; + resultRow.transfer) as TransferFromDatabase; return this.generateKafkaEvents( transfer, asset, diff --git a/indexer/services/ender/src/handlers/update-clob-pair-handler.ts b/indexer/services/ender/src/handlers/update-clob-pair-handler.ts index 18217fc78fe..08408352275 100644 --- a/indexer/services/ender/src/handlers/update-clob-pair-handler.ts +++ b/indexer/services/ender/src/handlers/update-clob-pair-handler.ts @@ -1,9 +1,7 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { PerpetualMarketFromDatabase, PerpetualMarketModel, perpetualMarketRefresher, - storeHelpers, } from '@dydxprotocol-indexer/postgres'; import { UpdateClobPairEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; @@ -20,25 +18,9 @@ export class UpdateClobPairHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_update_clob_pair_handler( - '${JSON.stringify(UpdateClobPairEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'UpdateClobPairHandler#internalHandle', - message: 'Failed to handle UpdateClobPairEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase; + resultRow.perpetual_market) as PerpetualMarketFromDatabase; perpetualMarketRefresher.upsertPerpetualMarket(perpetualMarket); diff --git a/indexer/services/ender/src/handlers/update-perpetual-handler.ts b/indexer/services/ender/src/handlers/update-perpetual-handler.ts index 5e7df335597..c3a9175b216 100644 --- a/indexer/services/ender/src/handlers/update-perpetual-handler.ts +++ b/indexer/services/ender/src/handlers/update-perpetual-handler.ts @@ -1,8 +1,6 @@ -import { logger } from '@dydxprotocol-indexer/base'; import { PerpetualMarketFromDatabase, perpetualMarketRefresher, - storeHelpers, PerpetualMarketModel, } from '@dydxprotocol-indexer/postgres'; import { UpdatePerpetualEventV1 } from '@dydxprotocol-indexer/v4-protos'; @@ -20,25 +18,9 @@ export class UpdatePerpetualHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(): Promise { - const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes; - const result: pg.QueryResult = await storeHelpers.rawQuery( - `SELECT dydx_update_perpetual_handler( - '${JSON.stringify(UpdatePerpetualEventV1.decode(eventDataBinary))}' - ) AS result;`, - { txId: this.txId }, - ).catch((error: Error) => { - logger.error({ - at: 'UpdatePerpetualHandler#internalHandle', - message: 'Failed to handle UpdatePerpetualEventV1', - error, - }); - - throw error; - }); - + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( - result.rows[0].result.perpetual_market) as PerpetualMarketFromDatabase; + resultRow.perpetual_market) as PerpetualMarketFromDatabase; await perpetualMarketRefresher.upsertPerpetualMarket(perpetualMarket); diff --git a/indexer/services/ender/src/helpers/postgres/postgres-functions.ts b/indexer/services/ender/src/helpers/postgres/postgres-functions.ts index fe5abaa1ce0..41a6838f155 100644 --- a/indexer/services/ender/src/helpers/postgres/postgres-functions.ts +++ b/indexer/services/ender/src/helpers/postgres/postgres-functions.ts @@ -30,6 +30,9 @@ const scripts: string[] = [ 'create_extension_pg_stat_statements.sql', 'create_extension_uuid_ossp.sql', 'dydx_asset_create_handler.sql', + 'dydx_block_processor.sql', + 'dydx_block_processor_batched_handlers.sql', + 'dydx_block_processor_sync_handlers.sql', 'dydx_clob_pair_status_to_market_status.sql', 'dydx_deleveraging_handler.sql', 'dydx_market_create_handler.sql', diff --git a/indexer/services/ender/src/lib/batched-handlers.ts b/indexer/services/ender/src/lib/batched-handlers.ts index e5e33bf63ae..88e174e64c5 100644 --- a/indexer/services/ender/src/lib/batched-handlers.ts +++ b/indexer/services/ender/src/lib/batched-handlers.ts @@ -1,5 +1,6 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; import _ from 'lodash'; +import * as pg from 'pg'; import config from '../config'; import { Handler } from '../handlers/handler'; @@ -9,9 +10,11 @@ import { ConsolidatedKafkaEvent, EventMessage } from './types'; // type alias for an array of handlers, handlers in a `HandlerBatch` can be processed in parallel. type HandlerBatch = Handler[]; +// TODO(IND-514): Remove the batch and sync handlers completely by moving all redis updates into +// a pipeline similar to how we return kafka events and then batch and emit them. export class BatchedHandlers { // An array of `HandlerBatch`s. Handlers in a `HandlerBatch` can be processed in parallel, and - // `HandlerBatch`s are processed in a sequential order following the order in `bartchedHandlers`. + // `HandlerBatch`s are processed in a sequential order following the order in `batchedHandlers`. batchedHandlers: HandlerBatch[]; // An array of sets of parallization ids. Each array of ids is the parallelization ids for the // corresponding array of handlers in this.batchedHandlers as well as all the parallelization ids @@ -71,7 +74,9 @@ export class BatchedHandlers { * and ensuring that handlers with overlapping parallelization ids are not processed in parallel. * Adds events to the kafkaPublisher. */ - public async process(kafkaPublisher: KafkaPublisher): Promise { + public async process( + kafkaPublisher: KafkaPublisher, + resultRow: pg.QueryResultRow): Promise { for (let batchIndex = 0; batchIndex < this.batchedHandlers.length; batchIndex++) { const start: number = Date.now(); const handlerCountMapping: { [key: string]: number } = {}; @@ -87,7 +92,7 @@ export class BatchedHandlers { Date.now() - this.initializationTime, { eventType: handler.eventType }, ); - return handler.handle(); + return handler.handle(resultRow[handler.blockEventIndex]); }), ); diff --git a/indexer/services/ender/src/lib/block-processor.ts b/indexer/services/ender/src/lib/block-processor.ts index c284966c15c..27f0878023c 100644 --- a/indexer/services/ender/src/lib/block-processor.ts +++ b/indexer/services/ender/src/lib/block-processor.ts @@ -1,8 +1,17 @@ /* eslint-disable max-len */ -import { logger } from '@dydxprotocol-indexer/base'; -import { IndexerTendermintBlock, IndexerTendermintEvent } from '@dydxprotocol-indexer/v4-protos'; +import { logger, stats, STATS_NO_SAMPLING } from '@dydxprotocol-indexer/base'; +import { + storeHelpers, +} from '@dydxprotocol-indexer/postgres'; +import { + IndexerTendermintBlock, + IndexerTendermintEvent, +} from '@dydxprotocol-indexer/v4-protos'; import _ from 'lodash'; +import * as pg from 'pg'; +import { DatabaseError } from 'pg'; +import config from '../config'; import { Handler } from '../handlers/handler'; import { AssetValidator } from '../validators/asset-validator'; import { DeleveragingValidator } from '../validators/deleveraging-validator'; @@ -50,8 +59,19 @@ function serializeSubtypeAndVersion( return `${subtype}-${version}`; } +type DecodedIndexerTendermintBlock = Omit & { + events: DecodedIndexerTendermintEvent[]; +}; + +type DecodedIndexerTendermintEvent = Omit & { + /** Decoded tendermint event. */ + dataBytes: object; +}; + export class BlockProcessor { block: IndexerTendermintBlock; + sqlEvents: Promise[]; + sqlBlock: DecodedIndexerTendermintBlock; txId: number; batchedHandlers: BatchedHandlers; syncHandlers: SyncHandlers; @@ -62,6 +82,11 @@ export class BlockProcessor { ) { this.block = block; this.txId = txId; + this.sqlBlock = { + ...this.block, + events: new Array(this.block.events.length), + }; + this.sqlEvents = new Array(this.block.events.length); this.batchedHandlers = new BatchedHandlers(); this.syncHandlers = new SyncHandlers(); } @@ -81,6 +106,7 @@ export class BlockProcessor { * @returns the kafka publisher which contains all the events to be published to the kafka */ public async process(): Promise { + const groupedEvents: GroupedEvents = this.groupEvents(); this.validateAndOrganizeEvents(groupedEvents); return this.processEvents(); @@ -101,22 +127,24 @@ export class BlockProcessor { groupedEvents.transactionEvents.push([]); } - _.forEach(this.block.events, (event: IndexerTendermintEvent) => { + for (let i: number = 0; i < this.block.events.length; i++) { + const event: IndexerTendermintEvent = this.block.events[i]; const transactionIndex: number = indexerTendermintEventToTransactionIndex(event); const eventProtoWithType: EventProtoWithTypeAndVersion | undefined = indexerTendermintEventToEventProtoWithType( + i, event, ); if (eventProtoWithType === undefined) { - return; + continue; } if (transactionIndex === -1) { groupedEvents.blockEvents.push(eventProtoWithType); - return; + continue; } groupedEvents.transactionEvents[transactionIndex].push(eventProtoWithType); - }); + } return groupedEvents; } @@ -168,9 +196,11 @@ export class BlockProcessor { const validator: Validator = new Initializer( eventProto.eventProto, this.block, + eventProto.blockEventIndex, ); validator.validate(); + this.sqlEvents[eventProto.blockEventIndex] = validator.getEventForBlockProcessor(); const handlers: Handler[] = validator.createHandlers( eventProto.indexerTendermintEvent, this.txId, @@ -187,14 +217,55 @@ export class BlockProcessor { private async processEvents(): Promise { const kafkaPublisher: KafkaPublisher = new KafkaPublisher(); + + await Promise.all(this.sqlEvents).then((values) => { + for (let i: number = 0; i < this.block.events.length; i++) { + const event: IndexerTendermintEvent = this.block.events[i]; + this.sqlBlock.events[i] = { + ...event, + dataBytes: values[i], + }; + } + }); + + const start: number = Date.now(); + let success = false; + let resultRow: pg.QueryResultRow; + try { + const result: pg.QueryResult = await storeHelpers.rawQuery( + 'SELECT dydx_block_processor(?) AS result;', + { + txId: this.txId, + bindings: [JSON.stringify(this.sqlBlock)], + sqlOptions: { name: 'dydx_block_processor' }, + }, + ).catch((error: DatabaseError) => { + logger.crit({ + at: `BlockProcessor#processEvents\n${error.where}`, + message: error.message, + error, + }); + throw error; + }); + resultRow = result.rows[0].result; + success = true; + } finally { + stats.timing( + `${config.SERVICE_NAME}.processed_block_sql.timing`, + Date.now() - start, + STATS_NO_SAMPLING, + { success: success.toString() }, + ); + } + // in genesis, handle sync events first, then batched events. // in other blocks, handle batched events first, then sync events. if (this.block.height === 0) { - await this.syncHandlers.process(kafkaPublisher); - await this.batchedHandlers.process(kafkaPublisher); + await this.syncHandlers.process(kafkaPublisher, resultRow); + await this.batchedHandlers.process(kafkaPublisher, resultRow); } else { - await this.batchedHandlers.process(kafkaPublisher); - await this.syncHandlers.process(kafkaPublisher); + await this.batchedHandlers.process(kafkaPublisher, resultRow); + await this.syncHandlers.process(kafkaPublisher, resultRow); } return kafkaPublisher; } diff --git a/indexer/services/ender/src/lib/helper.ts b/indexer/services/ender/src/lib/helper.ts index d1ee2f9a9c7..c6682b9f60c 100644 --- a/indexer/services/ender/src/lib/helper.ts +++ b/indexer/services/ender/src/lib/helper.ts @@ -91,6 +91,7 @@ export function dateToDateTime( * @returns */ export function indexerTendermintEventToEventProtoWithType( + blockEventIndex: number, event: IndexerTendermintEvent, ): EventProtoWithTypeAndVersion | undefined { const eventDataBinary: Uint8Array = event.dataBytes; @@ -103,6 +104,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: OrderFillEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.SUBACCOUNT_UPDATE.toString()): { @@ -111,6 +113,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: SubaccountUpdateEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.TRANSFER.toString()): { @@ -119,6 +122,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: TransferEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.MARKET.toString()): { @@ -127,6 +131,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: MarketEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.STATEFUL_ORDER.toString()): { @@ -135,6 +140,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: StatefulOrderEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.FUNDING.toString()): { @@ -143,6 +149,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: FundingEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.ASSET.toString()): { @@ -151,6 +158,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: AssetCreateEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.PERPETUAL_MARKET.toString()): { @@ -159,6 +167,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: PerpetualMarketCreateEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.LIQUIDITY_TIER.toString()): { @@ -167,6 +176,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: LiquidityTierUpsertEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.UPDATE_PERPETUAL.toString()): { @@ -175,6 +185,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: UpdatePerpetualEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.UPDATE_CLOB_PAIR.toString()): { @@ -183,6 +194,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: UpdateClobPairEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } case (DydxIndexerSubtypes.DELEVERAGING.toString()): { @@ -191,6 +203,7 @@ export function indexerTendermintEventToEventProtoWithType( eventProto: DeleveragingEventV1.decode(eventDataBinary), indexerTendermintEvent: event, version, + blockEventIndex, }; } default: { diff --git a/indexer/services/ender/src/lib/kafka-publisher.ts b/indexer/services/ender/src/lib/kafka-publisher.ts index 36aa302d15e..e1d7b018f4f 100644 --- a/indexer/services/ender/src/lib/kafka-publisher.ts +++ b/indexer/services/ender/src/lib/kafka-publisher.ts @@ -186,6 +186,7 @@ export class KafkaPublisher { const batchProducer: BatchKafkaProducer = new BatchKafkaProducer( topicKafkaMessages.topic, producer, + config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES, ); for (const message of topicKafkaMessages.messages) { batchProducer.addMessageAndMaybeFlush(message); diff --git a/indexer/services/ender/src/lib/on-message.ts b/indexer/services/ender/src/lib/on-message.ts index 1c53defe559..5819d169686 100644 --- a/indexer/services/ender/src/lib/on-message.ts +++ b/indexer/services/ender/src/lib/on-message.ts @@ -4,14 +4,12 @@ import { ParseMessageError, wrapBackgroundTask, STATS_NO_SAMPLING, - runFuncWithTimingStat, } from '@dydxprotocol-indexer/base'; import { KafkaTopics } from '@dydxprotocol-indexer/kafka'; import { Transaction, IsolationLevel, CandleFromDatabase, - storeHelpers, } from '@dydxprotocol-indexer/postgres'; import { IndexerTendermintBlock, @@ -68,15 +66,6 @@ export async function onMessage(message: KafkaMessage): Promise { try { validateIndexerTendermintBlock(indexerTendermintBlock); - await runFuncWithTimingStat( - createInitialRows( - blockHeight, - txId, - indexerTendermintBlock, - ), - {}, - 'create_initial_rows', - ); const blockProcessor: BlockProcessor = new BlockProcessor( indexerTendermintBlock, txId, @@ -214,30 +203,3 @@ function validateIndexerTendermintBlock( ); } } - -async function createInitialRows( - blockHeight: string, - txId: number, - block: IndexerTendermintBlock, -): Promise { - const txHashesString = block.txHashes.length > 0 ? `ARRAY['${block.txHashes.join("','")}']::text[]` : 'null'; - const eventsString = block.events.length > 0 ? `ARRAY['${block.events.map((event) => JSON.stringify(event)).join("','")}']::jsonb[]` : 'null'; - - const queryString: string = `SELECT dydx_create_initial_rows_for_tendermint_block( - '${blockHeight}'::text, - '${block.time!.toISOString()}'::text, - ${txHashesString}, - ${eventsString} - ) AS result;`; - await storeHelpers.rawQuery( - queryString, - { txId }, - ).catch((error: Error) => { - logger.error({ - at: 'on-message#createInitialRowsViaSqlFunction', - message: 'Failed to create initial rows', - error, - }); - throw error; - }); -} diff --git a/indexer/services/ender/src/lib/sync-handlers.ts b/indexer/services/ender/src/lib/sync-handlers.ts index 3562fe45445..1359973a45b 100644 --- a/indexer/services/ender/src/lib/sync-handlers.ts +++ b/indexer/services/ender/src/lib/sync-handlers.ts @@ -1,5 +1,6 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; import _ from 'lodash'; +import * as pg from 'pg'; import config from '../config'; import { Handler } from '../handlers/handler'; @@ -24,6 +25,8 @@ export const SYNCHRONOUS_SUBTYPES: DydxIndexerSubtypes[] = [ * After genesis block, these events should be handled after events in BatchedHandlers. * It is used for processing asset and market events. */ +// TODO(IND-514): Remove the batch and sync handlers completely by moving all redis updates into +// a pipeline similar to how we return kafka events and then batch and emit them. export class SyncHandlers { handlerBatch: HandlerBatch; initializationTime: number; @@ -60,7 +63,7 @@ export class SyncHandlers { * Adds events to the kafkaPublisher. */ public async process( - kafkaPublisher: KafkaPublisher, + kafkaPublisher: KafkaPublisher, resultRow: pg.QueryResultRow, ): Promise { const start: number = Date.now(); const handlerCountMapping: { [key: string]: number } = {}; @@ -71,7 +74,8 @@ export class SyncHandlers { handlerCountMapping[handlerName] = 0; } handlerCountMapping[handlerName] += 1; - const events: ConsolidatedKafkaEvent[] = await handler.handle(); + const events: ConsolidatedKafkaEvent[] = await handler.handle( + resultRow[handler.blockEventIndex]); consolidatedKafkaEventGroup.push(events); } diff --git a/indexer/services/ender/src/lib/types.ts b/indexer/services/ender/src/lib/types.ts index 5984f8fa610..969d24b38b6 100644 --- a/indexer/services/ender/src/lib/types.ts +++ b/indexer/services/ender/src/lib/types.ts @@ -60,66 +60,79 @@ export type EventProtoWithTypeAndVersion = { eventProto: EventMessage, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } & ({ type: DydxIndexerSubtypes.ORDER_FILL, eventProto: OrderFillEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.SUBACCOUNT_UPDATE, eventProto: SubaccountUpdateEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.TRANSFER, eventProto: TransferEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.MARKET, eventProto: MarketEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.STATEFUL_ORDER, eventProto: StatefulOrderEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.FUNDING, eventProto: FundingEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.ASSET, eventProto: AssetCreateEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.PERPETUAL_MARKET, eventProto: PerpetualMarketCreateEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.LIQUIDITY_TIER, eventProto: LiquidityTierUpsertEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.UPDATE_PERPETUAL, eventProto: UpdatePerpetualEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.UPDATE_CLOB_PAIR, eventProto: UpdateClobPairEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, } | { type: DydxIndexerSubtypes.DELEVERAGING, eventProto: DeleveragingEventV1, indexerTendermintEvent: IndexerTendermintEvent, version: number, + blockEventIndex: number, }); // Events grouped into events block events and events for each transactionIndex diff --git a/indexer/services/ender/src/scripts/dydx_block_processor.sql b/indexer/services/ender/src/scripts/dydx_block_processor.sql new file mode 100644 index 00000000000..7571d1abf64 --- /dev/null +++ b/indexer/services/ender/src/scripts/dydx_block_processor.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE FUNCTION dydx_block_processor(block jsonb) RETURNS jsonb AS $$ +/** + Processes an entire block by creating the initial tendermint rows for the block and then processes each event + individually through their respective handlers. + + Parameters: + - block: A 'DecodedIndexerTendermintBlock' converted to JSON format. Conversion to JSON is expected to be done by JSON.stringify. + + Returns: + An array containing the results for each event or NULL if this event is not handled by this block processor. + See each individual handler function for a description of the the inputs and outputs. + + (Note that no text should exist before the function declaration to ensure that exception line numbers are correct.) +*/ +DECLARE + block_height int = (block->'height')::int; + block_time timestamp = (block->>'time')::timestamp; + rval jsonb[]; + rval_to_merge jsonb[]; +BEGIN + PERFORM dydx_create_initial_rows_for_tendermint_block(block_height, block_time, block->'txHashes', block->'events'); + + /** In genesis, handle sync events first, then batched events. In other blocks, handle batched events first, then sync events. */ + IF NOT block_height = 0 THEN + rval = dydx_block_processor_batched_handlers(block); + rval_to_merge = dydx_block_processor_sync_handlers(block); + ELSE + rval = dydx_block_processor_sync_handlers(block); + rval_to_merge = dydx_block_processor_batched_handlers(block); + END IF; + + /** Note that arrays are 1-indexed in PostgreSQL and empty arrays return NULL for array_length. */ + FOR i in 1..coalesce(array_length(rval, 1), 0) LOOP + rval[i] = coalesce(rval[i], rval_to_merge[i]); + END LOOP; + + RETURN to_jsonb(rval); +END; +$$ LANGUAGE plpgsql; diff --git a/indexer/services/ender/src/scripts/dydx_block_processor_batched_handlers.sql b/indexer/services/ender/src/scripts/dydx_block_processor_batched_handlers.sql new file mode 100644 index 00000000000..5f3f2aa320f --- /dev/null +++ b/indexer/services/ender/src/scripts/dydx_block_processor_batched_handlers.sql @@ -0,0 +1,69 @@ +CREATE OR REPLACE FUNCTION dydx_block_processor_batched_handlers(block jsonb) RETURNS jsonb[] AS $$ +/** + Processes each event that should be handled by the batched handler. This includes all supported non synchronous types + (https://github.com/dydxprotocol/v4-chain/blob/b5d4e8a7c5cc48c460731b21c47f22eabef8b2b7/indexer/services/ender/src/lib/sync-handlers.ts#L11). + + Parameters: + - block: A 'DecodedIndexerTendermintBlock' converted to JSON format. Conversion to JSON is expected to be done by JSON.stringify. + + Returns: + An array containing the results for each event or NULL if this event is not handled by this block processor. + See each individual handler function for a description of the the inputs and outputs. + + (Note that no text should exist before the function declaration to ensure that exception line numbers are correct.) + + TODO(IND-514): Remove the batch and sync handlers completely by moving all redis updates into + a pipeline similar to how we return kafka events and then batch and emit them. +*/ +DECLARE + USDC_ASSET_ID constant text = '0'; + + block_height int = (block->'height')::int; + block_time timestamp = (block->>'time')::timestamp; + event_ jsonb; + rval jsonb[]; + event_index int; + transaction_index int; + event_data jsonb; +BEGIN + rval = array_fill(NULL::jsonb, ARRAY[coalesce(jsonb_array_length(block->'events'), 0)]::integer[]); + + /** Note that arrays are 1-indexed in PostgreSQL and empty arrays return NULL for array_length. */ + FOR i in 1..coalesce(array_length(rval, 1), 0) LOOP + event_ = jsonb_array_element(block->'events', i-1); + transaction_index = dydx_tendermint_event_to_transaction_index(event_); + event_index = (event_->'eventIndex')::int; + event_data = event_->'dataBytes'; + CASE event_->'subtype' + WHEN '"order_fill"'::jsonb THEN + IF event_data->'order' IS NOT NULL THEN + rval[i] = jsonb_build_object( + 'makerOrder', + dydx_order_fill_handler_per_order('makerOrder', block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index), 'MAKER', 'LIMIT', USDC_ASSET_ID, event_data->>'makerCanceledOrderStatus'), + 'order', + dydx_order_fill_handler_per_order('order', block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index), 'TAKER', 'LIMIT', USDC_ASSET_ID, event_data->>'takerCanceledOrderStatus')); + ELSE + rval[i] = jsonb_build_object( + 'makerOrder', + dydx_liquidation_fill_handler_per_order('makerOrder', block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index), 'MAKER', 'LIQUIDATION', USDC_ASSET_ID), + 'liquidationOrder', + dydx_liquidation_fill_handler_per_order('liquidationOrder', block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index), 'TAKER', 'LIQUIDATED', USDC_ASSET_ID)); + END IF; + WHEN '"subaccount_update"'::jsonb THEN + rval[i] = dydx_subaccount_update_handler(block_height, block_time, event_data, event_index, transaction_index); + WHEN '"transfer"'::jsonb THEN + rval[i] = dydx_transfer_handler(block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index)); + WHEN '"stateful_order"'::jsonb THEN + rval[i] = dydx_stateful_order_handler(block_height, block_time, event_data); + WHEN '"funding_values"'::jsonb THEN + rval[i] = dydx_funding_handler(block_height, block_time, event_data, event_index, transaction_index); + WHEN '"deleveraging"'::jsonb THEN + rval[i] = dydx_deleveraging_handler(block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index)); + ELSE + NULL; + END CASE; + END LOOP; + + RETURN rval; +END; +$$ LANGUAGE plpgsql; diff --git a/indexer/services/ender/src/scripts/dydx_block_processor_sync_handlers.sql b/indexer/services/ender/src/scripts/dydx_block_processor_sync_handlers.sql new file mode 100644 index 00000000000..cbc2e5904ed --- /dev/null +++ b/indexer/services/ender/src/scripts/dydx_block_processor_sync_handlers.sql @@ -0,0 +1,63 @@ +CREATE OR REPLACE FUNCTION dydx_block_processor_sync_handlers(block jsonb) RETURNS jsonb[] AS $$ +/** + Processes each event that should be handled by the batched handler. This includes all synchronous types + (https://github.com/dydxprotocol/v4-chain/blob/b5d4e8a7c5cc48c460731b21c47f22eabef8b2b7/indexer/services/ender/src/lib/sync-handlers.ts#L11). + + Parameters: + - block: A 'DecodedIndexerTendermintBlock' converted to JSON format. Conversion to JSON is expected to be done by JSON.stringify. + + Returns: + An array containing the results for each event or NULL if this event is not handled by this block processor. + See each individual handler function for a description of the the inputs and outputs. + + (Note that no text should exist before the function declaration to ensure that exception line numbers are correct.) + + TODO(IND-514): Remove the batch and sync handlers completely by moving all redis updates into + a pipeline similar to how we return kafka events and then batch and emit them. +*/ +DECLARE + block_height int = (block->'height')::int; + block_time timestamp = (block->>'time')::timestamp; + event_ jsonb; + rval jsonb[]; + event_index int; + transaction_index int; + event_data jsonb; +BEGIN + rval = array_fill(NULL::jsonb, ARRAY[coalesce(jsonb_array_length(block->'events'), 0)]::integer[]); + + /** Note that arrays are 1-indexed in PostgreSQL and empty arrays return NULL for array_length. */ + FOR i in 1..coalesce(array_length(rval, 1), 0) LOOP + event_ = jsonb_array_element(block->'events', i-1); + transaction_index = dydx_tendermint_event_to_transaction_index(event_); + event_index = (event_->'eventIndex')::int; + event_data = event_->'dataBytes'; + CASE event_->'subtype' + WHEN '"market"'::jsonb THEN + IF event_data->'priceUpdate' IS NOT NULL THEN + rval[i] = dydx_market_price_update_handler(block_height, block_time, event_data); + ELSIF event_data->'marketCreate' IS NOT NULL THEN + rval[i] = dydx_market_create_handler(event_data); + ELSIF event_data->'marketModify' IS NOT NULL THEN + rval[i] = dydx_market_modify_handler(event_data); + ELSE + RAISE EXCEPTION 'Unknown market event %', event_; + END IF; + WHEN '"asset"'::jsonb THEN + rval[i] = dydx_asset_create_handler(event_data); + WHEN '"perpetual_market"'::jsonb THEN + rval[i] = dydx_perpetual_market_handler(event_data); + WHEN '"liquidity_tier"'::jsonb THEN + rval[i] = dydx_liquidity_tier_handler(event_data); + WHEN '"update_perpetual"'::jsonb THEN + rval[i] = dydx_update_perpetual_handler(event_data); + WHEN '"update_clob_pair"'::jsonb THEN + rval[i] = dydx_update_clob_pair_handler(event_data); + ELSE + NULL; + END CASE; + END LOOP; + + RETURN rval; +END; +$$ LANGUAGE plpgsql; diff --git a/indexer/services/ender/src/scripts/dydx_create_initial_rows_for_tendermint_block.sql b/indexer/services/ender/src/scripts/dydx_create_initial_rows_for_tendermint_block.sql index 11fd91ba08f..58e717d8874 100644 --- a/indexer/services/ender/src/scripts/dydx_create_initial_rows_for_tendermint_block.sql +++ b/indexer/services/ender/src/scripts/dydx_create_initial_rows_for_tendermint_block.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION dydx_create_initial_rows_for_tendermint_block( - block_height text, block_time text, tx_hashes text[], events jsonb[]) RETURNS void AS $$ + block_height int, block_time timestamp, tx_hashes jsonb, events jsonb) RETURNS void AS $$ /** Parameters: - block_height: the height of the block being processed. @@ -12,19 +12,19 @@ CREATE OR REPLACE FUNCTION dydx_create_initial_rows_for_tendermint_block( */ BEGIN -- Create block. - INSERT INTO blocks ("blockHeight", "time") VALUES (block_height::bigint, block_time::timestamp); + INSERT INTO blocks ("blockHeight", "time") VALUES (block_height, block_time); -- Create transactions. - IF tx_hashes IS NOT NULL AND array_length(tx_hashes, 1) > 0 THEN - FOR i IN 1..array_length(tx_hashes, 1) LOOP - PERFORM dydx_create_transaction(tx_hashes[i], block_height, i); + IF tx_hashes IS NOT NULL AND jsonb_array_length(tx_hashes) > 0 THEN + FOR i IN 0..jsonb_array_length(tx_hashes)-1 LOOP + PERFORM dydx_create_transaction(jsonb_array_element_text(tx_hashes, i), block_height, i); END LOOP; END IF; -- Create tendermint events. - IF events IS NOT NULL AND array_length(events, 1) > 0 THEN - FOR i IN 1..array_length(events, 1) LOOP - PERFORM dydx_create_tendermint_event(events[i], block_height); + IF events IS NOT NULL AND jsonb_array_length(events) > 0 THEN + FOR i IN 0..jsonb_array_length(events)-1 LOOP + PERFORM dydx_create_tendermint_event(jsonb_array_element(events, i), block_height); END LOOP; END IF; END; diff --git a/indexer/services/ender/src/scripts/dydx_create_tendermint_event.sql b/indexer/services/ender/src/scripts/dydx_create_tendermint_event.sql index b720be1057e..d5c6742cf79 100644 --- a/indexer/services/ender/src/scripts/dydx_create_tendermint_event.sql +++ b/indexer/services/ender/src/scripts/dydx_create_tendermint_event.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION dydx_create_tendermint_event( - event jsonb, block_height text + event jsonb, block_height int ) RETURNS jsonb AS $$ /** Parameters: @@ -18,7 +18,7 @@ BEGIN event_id := dydx_event_id_from_parts(CAST(block_height AS int), transaction_idx, CAST(event->>'eventIndex' AS int)); INSERT INTO tendermint_events ("id", "blockHeight", "transactionIndex", "eventIndex") - VALUES (event_id, block_height::bigint, transaction_idx, CAST(event->>'eventIndex' AS int)) + VALUES (event_id, block_height, transaction_idx, (event->'eventIndex')::int) RETURNING to_jsonb(tendermint_events.*) INTO inserted_event; RETURN inserted_event; diff --git a/indexer/services/ender/src/scripts/dydx_create_transaction.sql b/indexer/services/ender/src/scripts/dydx_create_transaction.sql index 989667d746e..4723d69c235 100644 --- a/indexer/services/ender/src/scripts/dydx_create_transaction.sql +++ b/indexer/services/ender/src/scripts/dydx_create_transaction.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION dydx_create_transaction( - transaction_hash text, block_height text, transaction_index int + transaction_hash text, block_height int, transaction_index int ) RETURNS jsonb AS $$ /** Parameters: @@ -14,8 +14,8 @@ DECLARE inserted_transaction jsonb; BEGIN INSERT INTO transactions ("blockHeight", "transactionIndex", "transactionHash", "id") - VALUES (block_height::bigint, transaction_index, transaction_hash, - dydx_uuid_from_transaction_parts(block_height, transaction_index::text)) + VALUES (block_height, transaction_index, transaction_hash, + dydx_uuid_from_transaction_parts(block_height, transaction_index)) RETURNING to_jsonb(transactions.*) INTO inserted_transaction; RETURN inserted_transaction; diff --git a/indexer/services/ender/src/scripts/dydx_stateful_order_handler.sql b/indexer/services/ender/src/scripts/dydx_stateful_order_handler.sql index b8a4b108a8d..e4962c07a33 100644 --- a/indexer/services/ender/src/scripts/dydx_stateful_order_handler.sql +++ b/indexer/services/ender/src/scripts/dydx_stateful_order_handler.sql @@ -24,7 +24,7 @@ DECLARE BEGIN /** TODO(IND-334): Remove after deprecating StatefulOrderPlacementEvent. */ IF event_data->'orderPlace' IS NOT NULL OR event_data->'longTermOrderPlacement' IS NOT NULL OR event_data->'conditionalOrderPlacement' IS NOT NULL THEN - order_ = COALESCE(event_data->'orderPlace'->'order', event_data->'longTermOrderPlacement'->'order', event_data->'conditionalOrderPlacement'->'order'); + order_ = coalesce(event_data->'orderPlace'->'order', event_data->'longTermOrderPlacement'->'order', event_data->'conditionalOrderPlacement'->'order'); clob_pair_id = (order_->'orderId'->'clobPairId')::bigint; perpetual_market_record = dydx_get_perpetual_market_for_clob_pair(clob_pair_id); diff --git a/indexer/services/ender/src/scripts/dydx_uuid_from_transaction_parts.sql b/indexer/services/ender/src/scripts/dydx_uuid_from_transaction_parts.sql index cdbd16d71e0..8bd3438e597 100644 --- a/indexer/services/ender/src/scripts/dydx_uuid_from_transaction_parts.sql +++ b/indexer/services/ender/src/scripts/dydx_uuid_from_transaction_parts.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION dydx_uuid_from_transaction_parts(block_height text, transaction_index text) RETURNS uuid AS $$ +CREATE OR REPLACE FUNCTION dydx_uuid_from_transaction_parts(block_height int, transaction_index int) RETURNS uuid AS $$ /** Returns a UUID using the parts of a transaction. diff --git a/indexer/services/ender/src/validators/asset-validator.ts b/indexer/services/ender/src/validators/asset-validator.ts index 7bcc2422fc8..0bf140e322a 100644 --- a/indexer/services/ender/src/validators/asset-validator.ts +++ b/indexer/services/ender/src/validators/asset-validator.ts @@ -13,6 +13,7 @@ export class AssetValidator extends Validator { ): Handler[] { const handler: Handler = new AssetCreationHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/deleveraging-validator.ts b/indexer/services/ender/src/validators/deleveraging-validator.ts index c722b68159a..7a755579775 100644 --- a/indexer/services/ender/src/validators/deleveraging-validator.ts +++ b/indexer/services/ender/src/validators/deleveraging-validator.ts @@ -29,6 +29,7 @@ export class DeleveragingValidator extends Validator { return [ new DeleveragingHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/funding-validator.ts b/indexer/services/ender/src/validators/funding-validator.ts index 0429aa52045..894a69bf418 100644 --- a/indexer/services/ender/src/validators/funding-validator.ts +++ b/indexer/services/ender/src/validators/funding-validator.ts @@ -42,6 +42,7 @@ export class FundingValidator extends Validator { ): Handler[] { const handler: Handler = new FundingHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event as FundingEventMessage, diff --git a/indexer/services/ender/src/validators/liquidity-tier-validator.ts b/indexer/services/ender/src/validators/liquidity-tier-validator.ts index 5db8d771fc3..3c6417e0f97 100644 --- a/indexer/services/ender/src/validators/liquidity-tier-validator.ts +++ b/indexer/services/ender/src/validators/liquidity-tier-validator.ts @@ -47,6 +47,7 @@ export class LiquidityTierValidator extends Validator[] { const handler: Handler = new LiquidityTierHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/market-validator.ts b/indexer/services/ender/src/validators/market-validator.ts index 665aa58921a..fef4dbd7877 100644 --- a/indexer/services/ender/src/validators/market-validator.ts +++ b/indexer/services/ender/src/validators/market-validator.ts @@ -116,6 +116,7 @@ export class MarketValidator extends Validator { // @ts-ignore const handler: Handler = new Initializer( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/order-fill-validator.ts b/indexer/services/ender/src/validators/order-fill-validator.ts index cc4aabbbeb1..ac6bcaab721 100644 --- a/indexer/services/ender/src/validators/order-fill-validator.ts +++ b/indexer/services/ender/src/validators/order-fill-validator.ts @@ -1,4 +1,8 @@ -import { Liquidity } from '@dydxprotocol-indexer/postgres'; +import { + Liquidity, + OrderTable, +} from '@dydxprotocol-indexer/postgres'; +import { CanceledOrdersCache } from '@dydxprotocol-indexer/redis'; import { IndexerTendermintEvent, LiquidationOrderV1, @@ -9,6 +13,7 @@ import _ from 'lodash'; import { Handler, HandlerInitializer } from '../handlers/handler'; import { LiquidationHandler } from '../handlers/order-fills/liquidation-handler'; import { OrderHandler } from '../handlers/order-fills/order-handler'; +import { redisClient } from '../helpers/redis/redis-controller'; import { orderFillEventV1ToOrderFill } from '../helpers/translation-helper'; import { OrderFillWithLiquidity } from '../lib/translated-types'; import { OrderFillEventWithLiquidity } from '../lib/types'; @@ -32,6 +37,30 @@ export class OrderFillValidator extends Validator { } } + public async getEventForBlockProcessor(): Promise { + if (this.event.order) { + return Promise.all([ + CanceledOrdersCache.getOrderCanceledStatus( + OrderTable.orderIdToUuid(this.event.makerOrder!.orderId!), + redisClient, + ), + CanceledOrdersCache.getOrderCanceledStatus( + OrderTable.orderIdToUuid(this.event.order.orderId!), + redisClient, + ), + ], + ).then((canceledOrderStatuses) => { + return { + makerCanceledOrderStatus: canceledOrderStatuses[0], + takerCanceledOrderstatus: canceledOrderStatuses[1], + ...this.event, + }; + }); + } + + return this.event; + } + private validateOrder( order: IndexerOrder, liquidity: Liquidity, @@ -82,6 +111,7 @@ export class OrderFillValidator extends Validator { (orderFillEventWithLiquidity: OrderFillEventWithLiquidity) => { return new Initializer( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, orderFillEventV1ToOrderFill(orderFillEventWithLiquidity), diff --git a/indexer/services/ender/src/validators/perpetual-market-validator.ts b/indexer/services/ender/src/validators/perpetual-market-validator.ts index 2ea504c35c6..93531c835b1 100644 --- a/indexer/services/ender/src/validators/perpetual-market-validator.ts +++ b/indexer/services/ender/src/validators/perpetual-market-validator.ts @@ -41,6 +41,7 @@ export class PerpetualMarketValidator extends Validator[] { const handler: Handler = new PerpetualMarketCreationHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/stateful-order-validator.ts b/indexer/services/ender/src/validators/stateful-order-validator.ts index e2e6bd5cd35..066b7fbb20c 100644 --- a/indexer/services/ender/src/validators/stateful-order-validator.ts +++ b/indexer/services/ender/src/validators/stateful-order-validator.ts @@ -223,6 +223,7 @@ export class StatefulOrderValidator extends Validator { // @ts-ignore const handler: Handler = new Initializer( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/subaccount-update-validator.ts b/indexer/services/ender/src/validators/subaccount-update-validator.ts index 7c1de9fdad3..a747a437160 100644 --- a/indexer/services/ender/src/validators/subaccount-update-validator.ts +++ b/indexer/services/ender/src/validators/subaccount-update-validator.ts @@ -23,6 +23,7 @@ export class SubaccountUpdateValidator extends Validator { return [ new TransferHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/update-clob-pair-validator.ts b/indexer/services/ender/src/validators/update-clob-pair-validator.ts index 07bd892a822..315075da3f9 100644 --- a/indexer/services/ender/src/validators/update-clob-pair-validator.ts +++ b/indexer/services/ender/src/validators/update-clob-pair-validator.ts @@ -23,6 +23,7 @@ export class UpdateClobPairValidator extends Validator { ): Handler[] { const handler: Handler = new UpdateClobPairHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/update-perpetual-validator.ts b/indexer/services/ender/src/validators/update-perpetual-validator.ts index 2f2f5586e66..b0511d7e486 100644 --- a/indexer/services/ender/src/validators/update-perpetual-validator.ts +++ b/indexer/services/ender/src/validators/update-perpetual-validator.ts @@ -21,6 +21,7 @@ export class UpdatePerpetualValidator extends Validator ): Handler[] { const handler: Handler = new UpdatePerpetualHandler( this.block, + this.blockEventIndex, indexerTendermintEvent, txId, this.event, diff --git a/indexer/services/ender/src/validators/validator.ts b/indexer/services/ender/src/validators/validator.ts index 7808fa5ae3d..796546acfb8 100644 --- a/indexer/services/ender/src/validators/validator.ts +++ b/indexer/services/ender/src/validators/validator.ts @@ -7,14 +7,17 @@ import { EventMessage } from '../lib/types'; export type ValidatorInitializer = new ( event: EventMessage, block: IndexerTendermintBlock, + eventBlockIndex: number, ) => Validator; -export abstract class Validator { +export abstract class Validator { event: T; block: IndexerTendermintBlock; + blockEventIndex: number; - constructor(event: T, block: IndexerTendermintBlock) { + constructor(event: T, block: IndexerTendermintBlock, blockEventIndex: number) { this.event = event; + this.blockEventIndex = blockEventIndex; this.block = block; } @@ -23,6 +26,16 @@ export abstract class Validator { */ public abstract validate(): void; + /** + * Returns the decoded event and any additional details the SQL function needs to process. + */ + // TODO(IND-513): Convert handlers to have a 1-1 relationship with each validator by merging + // the order fill handlers together into a single handler (and the respective SQL function) + // and push this method down into the handler itself. + public getEventForBlockProcessor(): Promise { + return Promise.resolve(this.event); + } + protected logAndThrowParseMessageError( message: string, // eslint-disable-next-line @typescript-eslint/no-explicit-any