From 56b7cce80a3c8ebd91e9b2adc451cbb186055c39 Mon Sep 17 00:00:00 2001 From: shrenujb <98204323+shrenujb@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:13:45 -0400 Subject: [PATCH] [TRA-108] Update indexer to ingest and return market type for perpetuals (#1266) Signed-off-by: Shrenuj Bansal --- .../postgres/__tests__/helpers/constants.ts | 6 +++ ...44524_add_perpetual_markets_market_type.ts | 13 ++++++ .../src/models/perpetual-market-model.ts | 7 +++- .../postgres/src/types/db-model-types.ts | 3 +- .../src/types/perpetual-market-types.ts | 6 +++ .../src/types/websocket-message-types.ts | 3 +- .../request-transformer.test.ts | 1 + .../comlink/public/api-documentation.md | 41 ++++++++++++++++--- indexer/services/comlink/public/swagger.json | 13 +++++- .../request-helpers/request-transformer.ts | 1 + indexer/services/comlink/src/types.ts | 2 + .../helpers/indexer-proto-helpers.ts | 19 +++++++++ .../validators/asset-validator.test.ts | 11 +++++ .../helpers/postgres/postgres-functions.ts | 1 + .../dydx_perpetual_market_handler.sql | 1 + ...l_market_type_to_perpetual_market_type.sql | 11 +++++ 16 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240327144524_add_perpetual_markets_market_type.ts create mode 100644 indexer/services/ender/src/scripts/helpers/dydx_protocol_market_type_to_perpetual_market_type.sql diff --git a/indexer/packages/postgres/__tests__/helpers/constants.ts b/indexer/packages/postgres/__tests__/helpers/constants.ts index 26f714041b..f6fe55c99d 100644 --- a/indexer/packages/postgres/__tests__/helpers/constants.ts +++ b/indexer/packages/postgres/__tests__/helpers/constants.ts @@ -41,6 +41,7 @@ import { OrderType, PerpetualMarketCreateObject, PerpetualMarketStatus, + PerpetualMarketType, PerpetualPositionCreateObject, PerpetualPositionStatus, PnlTicksCreateObject, @@ -208,6 +209,7 @@ export const defaultPerpetualMarket: PerpetualMarketCreateObject = { subticksPerTick: 100, stepBaseQuantums: 10, liquidityTierId: 0, + marketType: PerpetualMarketType.CROSS, }; export const defaultPerpetualMarket2: PerpetualMarketCreateObject = { id: '1', @@ -225,6 +227,7 @@ export const defaultPerpetualMarket2: PerpetualMarketCreateObject = { subticksPerTick: 10, stepBaseQuantums: 1, liquidityTierId: 0, + marketType: PerpetualMarketType.CROSS, }; export const defaultPerpetualMarket3: PerpetualMarketCreateObject = { id: '2', @@ -242,6 +245,7 @@ export const defaultPerpetualMarket3: PerpetualMarketCreateObject = { subticksPerTick: 10, stepBaseQuantums: 1, liquidityTierId: 0, + marketType: PerpetualMarketType.CROSS, }; export const isolatedPerpetualMarket: PerpetualMarketCreateObject = { @@ -260,6 +264,7 @@ export const isolatedPerpetualMarket: PerpetualMarketCreateObject = { subticksPerTick: 10, stepBaseQuantums: 1, liquidityTierId: 0, + marketType: PerpetualMarketType.ISOLATED, }; export const isolatedPerpetualMarket2: PerpetualMarketCreateObject = { @@ -278,6 +283,7 @@ export const isolatedPerpetualMarket2: PerpetualMarketCreateObject = { subticksPerTick: 10, stepBaseQuantums: 1, liquidityTierId: 0, + marketType: PerpetualMarketType.ISOLATED, }; // ============== Orders ============== diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240327144524_add_perpetual_markets_market_type.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240327144524_add_perpetual_markets_market_type.ts new file mode 100644 index 0000000000..b850da4b16 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240327144524_add_perpetual_markets_market_type.ts @@ -0,0 +1,13 @@ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + return knex.schema.table('perpetual_markets', (table) => { + table.enum('marketType', ['CROSS', 'ISOLATED']).notNullable().defaultTo('CROSS'); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.table('perpetual_markets', (table) => { + table.dropColumn('marketType'); + }); +} diff --git a/indexer/packages/postgres/src/models/perpetual-market-model.ts b/indexer/packages/postgres/src/models/perpetual-market-model.ts index 124383d11b..7bf743496e 100644 --- a/indexer/packages/postgres/src/models/perpetual-market-model.ts +++ b/indexer/packages/postgres/src/models/perpetual-market-model.ts @@ -8,7 +8,7 @@ import { NumericPattern, } from '../lib/validators'; import { - PerpetualMarketStatus, + PerpetualMarketStatus, PerpetualMarketType, } from '../types'; export default class PerpetualMarketModel extends Model { @@ -66,6 +66,7 @@ export default class PerpetualMarketModel extends Model { 'subticksPerTick', 'stepBaseQuantums', 'liquidityTierId', + 'marketType', ], properties: { id: { type: 'string', pattern: IntegerPattern }, @@ -83,6 +84,7 @@ export default class PerpetualMarketModel extends Model { subticksPerTick: { type: 'integer' }, stepBaseQuantums: { type: 'integer' }, liquidityTierId: { type: 'integer' }, + marketType: { type: 'string' }, }, }; } @@ -110,6 +112,7 @@ export default class PerpetualMarketModel extends Model { subticksPerTick: 'integer', stepBaseQuantums: 'integer', liquidityTierId: 'integer', + marketType: 'string', }; } @@ -142,4 +145,6 @@ export default class PerpetualMarketModel extends Model { stepBaseQuantums!: number; liquidityTierId!: number; + + marketType!: PerpetualMarketType; } diff --git a/indexer/packages/postgres/src/types/db-model-types.ts b/indexer/packages/postgres/src/types/db-model-types.ts index f6e9a40c52..b617a07557 100644 --- a/indexer/packages/postgres/src/types/db-model-types.ts +++ b/indexer/packages/postgres/src/types/db-model-types.ts @@ -7,7 +7,7 @@ import { FillType, Liquidity } from './fill-types'; import { OrderSide, OrderStatus, OrderType, TimeInForce, } from './order-types'; -import { PerpetualMarketStatus } from './perpetual-market-types'; +import { PerpetualMarketStatus, PerpetualMarketType } from './perpetual-market-types'; import { PerpetualPositionStatus } from './perpetual-position-types'; import { PositionSide } from './position-types'; import { TradingRewardAggregationPeriod } from './trading-reward-aggregation-types'; @@ -91,6 +91,7 @@ export interface PerpetualMarketFromDatabase { subticksPerTick: number; stepBaseQuantums: number; liquidityTierId: number; + marketType: PerpetualMarketType; } export interface FillFromDatabase { diff --git a/indexer/packages/postgres/src/types/perpetual-market-types.ts b/indexer/packages/postgres/src/types/perpetual-market-types.ts index 13318b3c68..6956960bed 100644 --- a/indexer/packages/postgres/src/types/perpetual-market-types.ts +++ b/indexer/packages/postgres/src/types/perpetual-market-types.ts @@ -16,6 +16,7 @@ export interface PerpetualMarketCreateObject { subticksPerTick: number; stepBaseQuantums: number; liquidityTierId: number; + marketType: PerpetualMarketType; } export interface PerpetualMarketUpdateObject { @@ -62,3 +63,8 @@ export enum PerpetualMarketStatus { INITIALIZING = 'INITIALIZING', FINAL_SETTLEMENT = 'FINAL_SETTLEMENT', } + +export enum PerpetualMarketType { + CROSS = 'CROSS', + ISOLATED = 'ISOLATED', +} diff --git a/indexer/packages/postgres/src/types/websocket-message-types.ts b/indexer/packages/postgres/src/types/websocket-message-types.ts index 8e68b8f3eb..0ee86d509b 100644 --- a/indexer/packages/postgres/src/types/websocket-message-types.ts +++ b/indexer/packages/postgres/src/types/websocket-message-types.ts @@ -5,7 +5,7 @@ import { OrderStatus, OrderType, } from './order-types'; -import { PerpetualMarketStatus } from './perpetual-market-types'; +import { PerpetualMarketStatus, PerpetualMarketType } from './perpetual-market-types'; import { PerpetualPositionStatus } from './perpetual-position-types'; import { PositionSide } from './position-types'; import { TradeType } from './trade-types'; @@ -212,6 +212,7 @@ export interface TradingPerpetualMarketMessage { atomicResolution?: number; subticksPerTick?: number; stepBaseQuantums?: number; + marketType?: PerpetualMarketType; // Fields that are likely to change priceChange24H?: string; diff --git a/indexer/services/comlink/__tests__/lib/request-helpers/request-transformer.test.ts b/indexer/services/comlink/__tests__/lib/request-helpers/request-transformer.test.ts index 07ebb2d930..e6450a90d1 100644 --- a/indexer/services/comlink/__tests__/lib/request-helpers/request-transformer.test.ts +++ b/indexer/services/comlink/__tests__/lib/request-helpers/request-transformer.test.ts @@ -77,6 +77,7 @@ describe('request-transformer', () => { stepSize: Big(10).pow(-9).toFixed(), // 10 * 1e-10 = 1e-9 stepBaseQuantums: perpetualMarket.stepBaseQuantums, subticksPerTick: perpetualMarket.subticksPerTick, + marketType: perpetualMarket.marketType, }, ); }); diff --git a/indexer/services/comlink/public/api-documentation.md b/indexer/services/comlink/public/api-documentation.md index f3dd667640..6a68e0106c 100644 --- a/indexer/services/comlink/public/api-documentation.md +++ b/indexer/services/comlink/public/api-documentation.md @@ -1720,7 +1720,8 @@ fetch('https://dydx-testnet.imperator.co/v4/perpetualMarkets', "tickSize": "string", "stepSize": "string", "stepBaseQuantums": 0, - "subticksPerTick": 0 + "subticksPerTick": 0, + "marketType": "CROSS" }, "property2": { "clobPairId": "string", @@ -1739,7 +1740,8 @@ fetch('https://dydx-testnet.imperator.co/v4/perpetualMarkets', "tickSize": "string", "stepSize": "string", "stepBaseQuantums": 0, - "subticksPerTick": 0 + "subticksPerTick": 0, + "marketType": "CROSS" } } } @@ -3719,6 +3721,31 @@ or |*anonymous*|INITIALIZING| |*anonymous*|FINAL_SETTLEMENT| +## PerpetualMarketType + + + + + + +```json +"CROSS" + +``` + +### Properties + +|Name|Type|Required|Restrictions|Description| +|---|---|---|---|---| +|*anonymous*|string|false|none|none| + +#### Enumerated Values + +|Property|Value| +|---|---| +|*anonymous*|CROSS| +|*anonymous*|ISOLATED| + ## PerpetualMarketResponseObject @@ -3744,7 +3771,8 @@ or "tickSize": "string", "stepSize": "string", "stepBaseQuantums": 0, - "subticksPerTick": 0 + "subticksPerTick": 0, + "marketType": "CROSS" } ``` @@ -3770,6 +3798,7 @@ or |stepSize|string|true|none|none| |stepBaseQuantums|number(double)|true|none|none| |subticksPerTick|number(double)|true|none|none| +|marketType|[PerpetualMarketType](#schemaperpetualmarkettype)|true|none|none| ## PerpetualMarketResponse @@ -3798,7 +3827,8 @@ or "tickSize": "string", "stepSize": "string", "stepBaseQuantums": 0, - "subticksPerTick": 0 + "subticksPerTick": 0, + "marketType": "CROSS" }, "property2": { "clobPairId": "string", @@ -3817,7 +3847,8 @@ or "tickSize": "string", "stepSize": "string", "stepBaseQuantums": 0, - "subticksPerTick": 0 + "subticksPerTick": 0, + "marketType": "CROSS" } } } diff --git a/indexer/services/comlink/public/swagger.json b/indexer/services/comlink/public/swagger.json index cf3507be2c..b5967352a3 100644 --- a/indexer/services/comlink/public/swagger.json +++ b/indexer/services/comlink/public/swagger.json @@ -874,6 +874,13 @@ ], "type": "string" }, + "PerpetualMarketType": { + "enum": [ + "CROSS", + "ISOLATED" + ], + "type": "string" + }, "PerpetualMarketResponseObject": { "properties": { "clobPairId": { @@ -931,6 +938,9 @@ "subticksPerTick": { "type": "number", "format": "double" + }, + "marketType": { + "$ref": "#/components/schemas/PerpetualMarketType" } }, "required": [ @@ -950,7 +960,8 @@ "tickSize", "stepSize", "stepBaseQuantums", - "subticksPerTick" + "subticksPerTick", + "marketType" ], "type": "object", "additionalProperties": false diff --git a/indexer/services/comlink/src/request-helpers/request-transformer.ts b/indexer/services/comlink/src/request-helpers/request-transformer.ts index 3eea8dc703..fc58a7d340 100644 --- a/indexer/services/comlink/src/request-helpers/request-transformer.ts +++ b/indexer/services/comlink/src/request-helpers/request-transformer.ts @@ -354,6 +354,7 @@ export function perpetualMarketToResponseObject( stepSize: protocolTranslations.getStepSize(perpetualMarket), stepBaseQuantums: perpetualMarket.stepBaseQuantums, subticksPerTick: perpetualMarket.subticksPerTick, + marketType: perpetualMarket.marketType, }; } diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 83afcd18a7..a4d68eca90 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -15,6 +15,7 @@ import { OrderStatus, OrderType, PerpetualMarketStatus, + PerpetualMarketType, PerpetualPositionFromDatabase, PerpetualPositionStatus, PositionSide, @@ -267,6 +268,7 @@ export interface PerpetualMarketResponseObject { stepSize: string; stepBaseQuantums: number; subticksPerTick: number; + marketType: PerpetualMarketType; } /* ------- ORDERBOOK TYPES ------- */ diff --git a/indexer/services/ender/__tests__/helpers/indexer-proto-helpers.ts b/indexer/services/ender/__tests__/helpers/indexer-proto-helpers.ts index 120b19593e..7b87fc196b 100644 --- a/indexer/services/ender/__tests__/helpers/indexer-proto-helpers.ts +++ b/indexer/services/ender/__tests__/helpers/indexer-proto-helpers.ts @@ -51,6 +51,9 @@ import { PerpetualMarketCreateEventV1, DeleveragingEventV1, } from '@dydxprotocol-indexer/v4-protos'; +import { + PerpetualMarketType, +} from '@dydxprotocol-indexer/v4-protos/build/codegen/dydxprotocol/indexer/protocol/v1/perpetual'; import { IHeaders, Message, ProducerRecord } from 'kafkajs'; import _ from 'lodash'; @@ -902,5 +905,21 @@ export function expectPerpetualMarket( subticksPerTick: perpetual.subticksPerTick, stepBaseQuantums: Number(perpetual.stepBaseQuantums), liquidityTierId: perpetual.liquidityTier, + marketType: eventPerpetualMarketTypeToIndexerPerpetualMarketType( + perpetual.marketType, + ), })); } + +function eventPerpetualMarketTypeToIndexerPerpetualMarketType( + perpetualMarketType: PerpetualMarketType, +): string { + switch (perpetualMarketType) { + case PerpetualMarketType.PERPETUAL_MARKET_TYPE_CROSS: + return 'CROSS'; + case PerpetualMarketType.PERPETUAL_MARKET_TYPE_ISOLATED: + return 'ISOLATED'; + default: + throw new Error(`Unknown perpetual market type: ${perpetualMarketType}`); + } +} diff --git a/indexer/services/ender/__tests__/validators/asset-validator.test.ts b/indexer/services/ender/__tests__/validators/asset-validator.test.ts index 20ebf73945..d7dd4f0ac0 100644 --- a/indexer/services/ender/__tests__/validators/asset-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/asset-validator.test.ts @@ -11,8 +11,14 @@ import { } from '../helpers/indexer-proto-helpers'; import { expectDidntLogError } from '../helpers/validator-helpers'; import { AssetValidator } from '../../src/validators/asset-validator'; +import { createPostgresFunctions } from '../../src/helpers/postgres/postgres-functions'; describe('asset-validator', () => { + beforeAll(async () => { + await dbHelpers.migrate(); + await createPostgresFunctions(); + }); + beforeEach(async () => { await testMocks.seedData(); jest.spyOn(logger, 'error'); @@ -23,6 +29,11 @@ describe('asset-validator', () => { jest.clearAllMocks(); }); + afterAll(async () => { + await dbHelpers.teardown(); + jest.resetAllMocks(); + }); + describe('validate', () => { it('does not throw error on valid asset create event', () => { const validator: AssetValidator = new AssetValidator( diff --git a/indexer/services/ender/src/helpers/postgres/postgres-functions.ts b/indexer/services/ender/src/helpers/postgres/postgres-functions.ts index 24583ed387..f8f045c1f0 100644 --- a/indexer/services/ender/src/helpers/postgres/postgres-functions.ts +++ b/indexer/services/ender/src/helpers/postgres/postgres-functions.ts @@ -86,6 +86,7 @@ const HELPER_SCRIPTS: string[] = [ 'dydx_uuid_from_trading_rewards_parts.sql', 'dydx_uuid_from_transaction_parts.sql', 'dydx_uuid_from_transfer_parts.sql', + 'dydx_protocol_market_type_to_perpetual_market_type.sql', ]; const MAIN_SCRIPTS: string[] = [ diff --git a/indexer/services/ender/src/scripts/handlers/dydx_perpetual_market_handler.sql b/indexer/services/ender/src/scripts/handlers/dydx_perpetual_market_handler.sql index ad9210b675..92fa9b104a 100644 --- a/indexer/services/ender/src/scripts/handlers/dydx_perpetual_market_handler.sql +++ b/indexer/services/ender/src/scripts/handlers/dydx_perpetual_market_handler.sql @@ -26,6 +26,7 @@ BEGIN perpetual_market_record."subticksPerTick" = (event_data->'subticksPerTick')::integer; perpetual_market_record."stepBaseQuantums" = dydx_from_jsonlib_long(event_data->'stepBaseQuantums'); perpetual_market_record."liquidityTierId" = (event_data->'liquidityTier')::integer; + perpetual_market_record."marketType" = dydx_protocol_market_type_to_perpetual_market_type(event_data->'marketType'); INSERT INTO perpetual_markets VALUES (perpetual_market_record.*) RETURNING * INTO perpetual_market_record; diff --git a/indexer/services/ender/src/scripts/helpers/dydx_protocol_market_type_to_perpetual_market_type.sql b/indexer/services/ender/src/scripts/helpers/dydx_protocol_market_type_to_perpetual_market_type.sql new file mode 100644 index 0000000000..2214b68655 --- /dev/null +++ b/indexer/services/ender/src/scripts/helpers/dydx_protocol_market_type_to_perpetual_market_type.sql @@ -0,0 +1,11 @@ +CREATE OR REPLACE FUNCTION dydx_protocol_market_type_to_perpetual_market_type(marketType jsonb) + RETURNS text AS $$ + +BEGIN + CASE marketType + WHEN '1'::jsonb THEN RETURN 'CROSS'; /** MARKET_TYPE_CROSS */ + WHEN '2'::jsonb THEN RETURN 'ISOLATED'; /** MARKET_TYPE_ISOLATED */ + ELSE RAISE EXCEPTION 'Invalid market type: %', marketType; + END CASE; +END; +$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; \ No newline at end of file