merge from main

indexer/packages/kafka/src/batch-kafka-producer.ts

@@ -2,7 +2,6 @@ import { logger } from '@dydxprotocol-indexer/base';
 import { Producer, RecordMetadata } from 'kafkajs';
 import _ from 'lodash';
 
-import config from './config';
 import { KafkaTopics } from './types';
 
 /**
@@ -28,7 +27,10 @@ export class BatchKafkaProducer {
   constructor(
     topic: KafkaTopics,
     producer: Producer,
-    maxBatchSizeBytes: number = config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES,
+    // Note that default parameters are bound during module load time making it difficult
+    // to modify the parameter during a test so we explicitly require callers to pass in
+    // config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES.
+    maxBatchSizeBytes: number,
   ) {
     this.maxBatchSizeBytes = maxBatchSizeBytes;
     this.producer = producer;

indexer/packages/postgres/src/helpers/stores-helpers.ts

@@ -64,25 +64,19 @@ export async function rawQuery(
   options: Options,
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): Promise<Knex.Raw<any>> {
-  if (options.readReplica) {
-    if (options.txId) {
-      return knexReadReplica.getConnection().raw(queryString).transacting(
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        <Knex.Transaction<any, any>>Transaction.get(options.txId),
-      );
-    } else {
-      return knexReadReplica.getConnection().raw(queryString);
-    }
-  } else {
-    if (options.txId) {
-      return knexPrimary.raw(queryString).transacting(
+  const connection = options.readReplica ? knexReadReplica.getConnection() : knexPrimary;
+  let queryBuilder = options.bindings === undefined
+    ? connection.raw(queryString) : connection.raw(queryString, options.bindings);
+  if (options.txId) {
+    queryBuilder = queryBuilder.transacting(
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         <Knex.Transaction<any, any>>Transaction.get(options.txId),
-      );
-    } else {
-      return knexPrimary.raw(queryString);
-    }
+    );
+  }
+  if (options.sqlOptions) {
+    queryBuilder = queryBuilder.options(options.sqlOptions);
   }
+  return queryBuilder;
 }
 
 /* ------- Bulk Helpers ------- */

indexer/packages/postgres/src/types/utility-types.ts

@@ -1,4 +1,6 @@
 /* ------- UTILITY TYPES ------- */
+import { RawBinding } from 'knex';
+
 export type IsoString = string;
 
 export type RegexPattern = string;
@@ -17,6 +19,9 @@ export interface Options {
   orderBy?: [string, Ordering][];
   readReplica?: boolean,
   random?: boolean;
+  bindings?: readonly RawBinding[];
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  sqlOptions?: Readonly<{ [key: string]: any }>;
 }
 
 export enum Ordering {

indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/clob_pair.ts

@@ -37,7 +37,8 @@ export enum ClobPair_Status {
   /**
    * STATUS_FINAL_SETTLEMENT - STATUS_FINAL_SETTLEMENT represents a clob pair which is deactivated
    * and trading has ceased. All open positions will be closed by the
-   * protocol.
+   * protocol. Open stateful orders will be cancelled. Open short-term
+   * orders will be left to expire.
    */
   STATUS_FINAL_SETTLEMENT = 6,
   UNRECOGNIZED = -1,
@@ -79,7 +80,8 @@ export enum ClobPair_StatusSDKType {
   /**
    * STATUS_FINAL_SETTLEMENT - STATUS_FINAL_SETTLEMENT represents a clob pair which is deactivated
    * and trading has ceased. All open positions will be closed by the
-   * protocol.
+   * protocol. Open stateful orders will be cancelled. Open short-term
+   * orders will be left to expire.
    */
   STATUS_FINAL_SETTLEMENT = 6,
   UNRECOGNIZED = -1,

indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/matches.ts

@@ -128,6 +128,14 @@ export interface MatchPerpetualDeleveraging {
   /** An ordered list of fills created by this liquidation. */
 
   fills: MatchPerpetualDeleveraging_Fill[];
+  /**
+   * Flag denoting whether the deleveraging operation was for the purpose
+   * of final settlement. Final settlement matches are at the oracle price,
+   * whereas deleveraging happens at the bankruptcy price of the deleveraged
+   * subaccount.
+   */
+
+  isFinalSettlement: boolean;
 }
 /**
  * MatchPerpetualDeleveraging is an injected message used for deleveraging a
@@ -143,6 +151,14 @@ export interface MatchPerpetualDeleveragingSDKType {
   /** An ordered list of fills created by this liquidation. */
 
   fills: MatchPerpetualDeleveraging_FillSDKType[];
+  /**
+   * Flag denoting whether the deleveraging operation was for the purpose
+   * of final settlement. Final settlement matches are at the oracle price,
+   * whereas deleveraging happens at the bankruptcy price of the deleveraged
+   * subaccount.
+   */
+
+  is_final_settlement: boolean;
 }
 /** Fill represents a fill between the liquidated and offsetting subaccount. */
 
@@ -451,7 +467,8 @@ function createBaseMatchPerpetualDeleveraging(): MatchPerpetualDeleveraging {
   return {
     liquidated: undefined,
     perpetualId: 0,
-    fills: []
+    fills: [],
+    isFinalSettlement: false
   };
 }
 
@@ -469,6 +486,10 @@ export const MatchPerpetualDeleveraging = {
       MatchPerpetualDeleveraging_Fill.encode(v!, writer.uint32(26).fork()).ldelim();
     }
 
+    if (message.isFinalSettlement === true) {
+      writer.uint32(32).bool(message.isFinalSettlement);
+    }
+
     return writer;
   },
 
@@ -493,6 +514,10 @@ export const MatchPerpetualDeleveraging = {
           message.fills.push(MatchPerpetualDeleveraging_Fill.decode(reader, reader.uint32()));
           break;
 
+        case 4:
+          message.isFinalSettlement = reader.bool();
+          break;
+
         default:
           reader.skipType(tag & 7);
           break;
@@ -507,6 +532,7 @@ export const MatchPerpetualDeleveraging = {
     message.liquidated = object.liquidated !== undefined && object.liquidated !== null ? SubaccountId.fromPartial(object.liquidated) : undefined;
     message.perpetualId = object.perpetualId ?? 0;
     message.fills = object.fills?.map(e => MatchPerpetualDeleveraging_Fill.fromPartial(e)) || [];
+    message.isFinalSettlement = object.isFinalSettlement ?? false;
     return message;
   }
 

indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/mev.ts

@@ -94,6 +94,8 @@ export interface ValidatorMevMatchesSDKType {
 export interface MevNodeToNodeMetrics {
   validatorMevMatches?: ValidatorMevMatches;
   clobMidPrices: ClobMidPrice[];
+  bpMevMatches?: ValidatorMevMatches;
+  proposalReceiveTime: Long;
 }
 /**
  * MevNodeToNodeMetrics is a data structure for encapsulating all MEV node <>
@@ -103,6 +105,8 @@ export interface MevNodeToNodeMetrics {
 export interface MevNodeToNodeMetricsSDKType {
   validator_mev_matches?: ValidatorMevMatchesSDKType;
   clob_mid_prices: ClobMidPriceSDKType[];
+  bp_mev_matches?: ValidatorMevMatchesSDKType;
+  proposal_receive_time: Long;
 }
 
 function createBaseMEVMatch(): MEVMatch {
@@ -448,7 +452,9 @@ export const ValidatorMevMatches = {
 function createBaseMevNodeToNodeMetrics(): MevNodeToNodeMetrics {
   return {
     validatorMevMatches: undefined,
-    clobMidPrices: []
+    clobMidPrices: [],
+    bpMevMatches: undefined,
+    proposalReceiveTime: Long.UZERO
   };
 }
 
@@ -462,6 +468,14 @@ export const MevNodeToNodeMetrics = {
       ClobMidPrice.encode(v!, writer.uint32(18).fork()).ldelim();
     }
 
+    if (message.bpMevMatches !== undefined) {
+      ValidatorMevMatches.encode(message.bpMevMatches, writer.uint32(26).fork()).ldelim();
+    }
+
+    if (!message.proposalReceiveTime.isZero()) {
+      writer.uint32(32).uint64(message.proposalReceiveTime);
+    }
+
     return writer;
   },
 
@@ -482,6 +496,14 @@ export const MevNodeToNodeMetrics = {
           message.clobMidPrices.push(ClobMidPrice.decode(reader, reader.uint32()));
           break;
 
+        case 3:
+          message.bpMevMatches = ValidatorMevMatches.decode(reader, reader.uint32());
+          break;
+
+        case 4:
+          message.proposalReceiveTime = (reader.uint64() as Long);
+          break;
+
         default:
           reader.skipType(tag & 7);
           break;
@@ -495,6 +517,8 @@ export const MevNodeToNodeMetrics = {
     const message = createBaseMevNodeToNodeMetrics();
     message.validatorMevMatches = object.validatorMevMatches !== undefined && object.validatorMevMatches !== null ? ValidatorMevMatches.fromPartial(object.validatorMevMatches) : undefined;
     message.clobMidPrices = object.clobMidPrices?.map(e => ClobMidPrice.fromPartial(e)) || [];
+    message.bpMevMatches = object.bpMevMatches !== undefined && object.bpMevMatches !== null ? ValidatorMevMatches.fromPartial(object.bpMevMatches) : undefined;
+    message.proposalReceiveTime = object.proposalReceiveTime !== undefined && object.proposalReceiveTime !== null ? Long.fromValue(object.proposalReceiveTime) : Long.UZERO;
     return message;
   }
 

indexer/packages/v4-protos/src/codegen/dydxprotocol/indexer/events/events.ts

@@ -465,12 +465,20 @@ export interface DeleveragingEventV1 {
    */
 
   fillAmount: Long;
-  /** Bankruptcy price of liquidated subaccount, in USDC quote quantums. */
+  /** Fill price of deleveraging event, in USDC quote quantums. */
 
   price: Long;
   /** `true` if liquidating a short position, `false` otherwise. */
 
   isBuy: boolean;
+  /**
+   * `true` if the deleveraging event is for final settlement, indicating
+   * the match occurred at the oracle price rather than bankruptcy price.
+   * When this flag is `false`, the fill price is the bankruptcy price
+   * of the liquidated subaccount.
+   */
+
+  isFinalSettlement: boolean;
 }
 /**
  * DeleveragingEvent message contains all the information for a deleveraging
@@ -493,12 +501,20 @@ export interface DeleveragingEventV1SDKType {
    */
 
   fill_amount: Long;
-  /** Bankruptcy price of liquidated subaccount, in USDC quote quantums. */
+  /** Fill price of deleveraging event, in USDC quote quantums. */
 
   price: Long;
   /** `true` if liquidating a short position, `false` otherwise. */
 
   is_buy: boolean;
+  /**
+   * `true` if the deleveraging event is for final settlement, indicating
+   * the match occurred at the oracle price rather than bankruptcy price.
+   * When this flag is `false`, the fill price is the bankruptcy price
+   * of the liquidated subaccount.
+   */
+
+  is_final_settlement: boolean;
 }
 /**
  * LiquidationOrder represents the liquidation taker order to be included in a
@@ -1782,7 +1798,8 @@ function createBaseDeleveragingEventV1(): DeleveragingEventV1 {
     perpetualId: 0,
     fillAmount: Long.UZERO,
     price: Long.UZERO,
-    isBuy: false
+    isBuy: false,
+    isFinalSettlement: false
   };
 }
 
@@ -1812,6 +1829,10 @@ export const DeleveragingEventV1 = {
       writer.uint32(48).bool(message.isBuy);
     }
 
+    if (message.isFinalSettlement === true) {
+      writer.uint32(56).bool(message.isFinalSettlement);
+    }
+
     return writer;
   },
 
@@ -1848,6 +1869,10 @@ export const DeleveragingEventV1 = {
           message.isBuy = reader.bool();
           break;
 
+        case 7:
+          message.isFinalSettlement = reader.bool();
+          break;
+
         default:
           reader.skipType(tag & 7);
           break;
@@ -1865,6 +1890,7 @@ export const DeleveragingEventV1 = {
     message.fillAmount = object.fillAmount !== undefined && object.fillAmount !== null ? Long.fromValue(object.fillAmount) : Long.UZERO;
     message.price = object.price !== undefined && object.price !== null ? Long.fromValue(object.price) : Long.UZERO;
     message.isBuy = object.isBuy ?? false;
+    message.isFinalSettlement = object.isFinalSettlement ?? false;
     return message;
   }
 

indexer/services/bazooka/src/vulcan-helpers.ts

@@ -24,6 +24,7 @@ import { Long } from '@dydxprotocol-indexer/v4-protos/build/codegen/helpers';
 import Big from 'big.js';
 import _ from 'lodash';
 
+import config from './config';
 import { ZERO } from './constants';
 
 interface VulcanMessage {
@@ -134,6 +135,7 @@ export async function sendStatefulOrderMessages() {
     const batchProducer: BatchKafkaProducer = new BatchKafkaProducer(
       KafkaTopics.TO_VULCAN,
       producer,
+      config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES,
     );
     for (const message of messages) {
       batchProducer.addMessageAndMaybeFlush(message);

indexer/services/ender/__tests__/handlers/asset-handler.test.ts

@@ -85,6 +85,7 @@ describe('assetHandler', () => {
 
       const handler: AssetCreationHandler = new AssetCreationHandler(
         block,
+        0,
         indexerTendermintEvent,
         0,
         defaultAssetCreateEvent,

indexer/services/ender/__tests__/handlers/funding-handler.test.ts

@@ -89,6 +89,7 @@ describe('fundingHandler', () => {
 
       const handler: FundingHandler = new FundingHandler(
         block,
+        0,
         indexerTendermintEvent,
         0,
         defaultFundingUpdateSampleEvent,

indexer/services/ender/__tests__/handlers/liquidity-tier-handler.test.ts

@@ -90,6 +90,7 @@ describe('liquidityTierHandler', () => {
 
       const handler: LiquidityTierHandler = new LiquidityTierHandler(
         block,
+        0,
         indexerTendermintEvent,
         0,
         defaultLiquidityTierUpsertEvent,