Cherry-pick changes for indexer v1.0.2

indexer/packages/postgres/__tests__/helpers/constants.ts

@@ -226,6 +226,7 @@ export const defaultOrderGoodTilBlockTime: OrderCreateObject = {
   clientId: '2',
   goodTilBlock: undefined,
   goodTilBlockTime: '2023-01-22T00:00:00.000Z',
+  createdAtHeight: '1',
   orderFlags: ORDER_FLAG_LONG_TERM.toString(),
 };
 

indexer/packages/postgres/src/constants.ts

@@ -1,12 +1,17 @@
 import { CandleMessage_Resolution, ClobPairStatus } from '@dydxprotocol-indexer/v4-protos';
 
 import config from './config';
+import AssetModel from './models/asset-model';
 import AssetPositionModel from './models/asset-position-model';
 import FillModel from './models/fill-model';
+import LiquidityTiersModel from './models/liquidity-tiers-model';
+import MarketModel from './models/market-model';
+import OraclePriceModel from './models/oracle-price-model';
 import OrderModel from './models/order-model';
 import PerpetualMarketModel from './models/perpetual-market-model';
 import PerpetualPositionModel from './models/perpetual-position-model';
 import SubaccountModel from './models/subaccount-model';
+import TransferModel from './models/transfer-model';
 import {
   APITimeInForce,
   CandleResolution,
@@ -81,12 +86,17 @@ export const TIME_IN_FORCE_TO_API_TIME_IN_FORCE: Record<TimeInForce, APITimeInFo
 
 // A list of models that have sqlToJsonConversions defined.
 export const SQL_TO_JSON_DEFINED_MODELS = [
+  AssetModel,
   AssetPositionModel,
   FillModel,
+  LiquidityTiersModel,
+  MarketModel,
+  OraclePriceModel,
   OrderModel,
   PerpetualMarketModel,
   PerpetualPositionModel,
   SubaccountModel,
+  TransferModel,
 ];
 
 export type SpecifiedClobPairStatus =

indexer/packages/postgres/src/helpers/db-helpers.ts

@@ -70,7 +70,7 @@ $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;`;
       return rawQuery(sqlFn, {}).catch((error) => {
         logger.error({
           at: 'dbHelpers#createModelToJsonFunctions',
-          message: `Failed to create or replace function dydx_to_json for model ${model.tableName}.`,
+          message: `Failed to create or replace function dydx_to_jsonb for model ${model.tableName}.`,
           error,
         });
         throw error;

indexer/packages/postgres/src/index.ts

@@ -3,11 +3,16 @@ export * from './constants';
 
 export { default as Transaction } from './helpers/transaction';
 export { postgresConfigSchema } from './config';
+export { default as AssetModel } from './models/asset-model';
 export { default as AssetPositionModel } from './models/asset-position-model';
 export { default as FillModel } from './models/fill-model';
+export { default as LiquidityTiersModel } from './models/liquidity-tiers-model';
+export { default as MarketModel } from './models/market-model';
+export { default as OraclePriceModel } from './models/oracle-price-model';
 export { default as OrderModel } from './models/order-model';
 export { default as PerpetualMarketModel } from './models/perpetual-market-model';
 export { default as PerpetualPositionModel } from './models/perpetual-position-model';
+export { default as TransferModel } from './models/transfer-model';
 
 export * as AssetTable from './stores/asset-table';
 export * as AssetPositionTable from './stores/asset-position-table';

indexer/packages/postgres/src/lib/order-translations.ts

@@ -22,13 +22,11 @@ import {
  *
  * @param order
  */
-export async function convertToIndexerOrder(
+export function convertToIndexerOrderWithSubaccount(
   order: OrderFromDatabase,
   perpetualMarket: PerpetualMarketFromDatabase,
-): Promise<IndexerOrder> {
-  const subaccount: SubaccountFromDatabase | undefined = await SubaccountTable.findById(
-    order.subaccountId,
-  );
+  subaccount: SubaccountFromDatabase,
+): IndexerOrder {
   if (!OrderTable.isLongTermOrConditionalOrder(order.orderFlags)) {
     logger.error({
       at: 'protocol-translations#convertToIndexerOrder',
@@ -77,3 +75,29 @@ export async function convertToIndexerOrder(
 
   return indexerOrder;
 }
+
+/**
+ * Converts an order from the database to an IndexerOrder proto.
+ * This is used to resend open stateful orders to Vulcan during Indexer fast sync
+ * to uncross the orderbook.
+ *
+ * @param order
+ */
+export async function convertToIndexerOrder(
+  order: OrderFromDatabase,
+  perpetualMarket: PerpetualMarketFromDatabase,
+): Promise<IndexerOrder> {
+  const subaccount: SubaccountFromDatabase | undefined = await SubaccountTable.findById(
+    order.subaccountId,
+  );
+
+  if (!subaccount === undefined) {
+    logger.error({
+      at: 'protocol-translations#convertToIndexerOrder',
+      message: 'Subaccount for order not found',
+      order,
+    });
+    throw new Error(`Subaccount for order not found: ${order.subaccountId}`);
+  }
+  return convertToIndexerOrderWithSubaccount(order, perpetualMarket, subaccount!);
+}

indexer/packages/postgres/src/loops/market-refresher.ts

@@ -40,6 +40,13 @@ export async function updateMarkets(options?: Options): Promise<void> {
   stats.timing(`${config.SERVICE_NAME}.loops.update_markets`, Date.now() - startTime);
 }
 
+/**
+ * Updates the markets map with the specified market.
+ */
+export function updateMarket(market: MarketFromDatabase): void {
+  idToMarket[market.id] = market;
+}
+
 /**
  * Gets the market for a given id.
  */

indexer/packages/postgres/src/models/asset-model.ts

@@ -59,6 +59,22 @@ export default class AssetModel extends Model {
     };
   }
 
+  /**
+   * A mapping from column name to JSON conversion expected.
+   * See getSqlConversionForDydxModelTypes for valid conversions.
+   *
+   * TODO(IND-239): Ensure that jsonSchema() / sqlToJsonConversions() / model fields match.
+   */
+  static get sqlToJsonConversions() {
+    return {
+      id: 'string',
+      symbol: 'string',
+      atomicResolution: 'integer',
+      hasMarket: 'boolean',
+      marketId: 'integer',
+    };
+  }
+
   id!: string;
 
   symbol!: string;

indexer/packages/postgres/src/models/fill-model.ts

@@ -84,7 +84,7 @@ export default class FillModel extends Model {
         transactionHash: { type: 'string' },
         createdAt: { type: 'string', format: 'date-time' },
         createdAtHeight: { type: 'string', pattern: IntegerPattern },
-        clientMetadata: { type: 'string', pattern: IntegerPattern },
+        clientMetadata: { type: ['string', 'null'], pattern: IntegerPattern },
         fee: { type: 'string', pattern: NumericPattern },
       },
     };

indexer/packages/postgres/src/models/liquidity-tiers-model.ts

@@ -33,6 +33,22 @@ export default class LiquidityTiersModel extends BaseModel {
     };
   }
 
+  /**
+   * A mapping from column name to JSON conversion expected.
+   * See getSqlConversionForDydxModelTypes for valid conversions.
+   *
+   * TODO(IND-239): Ensure that jsonSchema() / sqlToJsonConversions() / model fields match.
+   */
+  static get sqlToJsonConversions() {
+    return {
+      id: 'integer',
+      name: 'string',
+      initialMarginPpm: 'string',
+      maintenanceFractionPpm: 'string',
+      basePositionNotional: 'string',
+    };
+  }
+
   id!: number;
 
   QueryBuilderType!: UpsertQueryBuilder<this>;

indexer/packages/postgres/src/models/market-model.ts

@@ -51,6 +51,22 @@ export default class MarketModel extends Model {
     };
   }
 
+  /**
+   * A mapping from column name to JSON conversion expected.
+   * See getSqlConversionForDydxModelTypes for valid conversions.
+   *
+   * TODO(IND-239): Ensure that jsonSchema() / sqlToJsonConversions() / model fields match.
+   */
+  static get sqlToJsonConversions() {
+    return {
+      id: 'integer',
+      pair: 'string',
+      exponent: 'integer',
+      minPriceChangePpm: 'integer',
+      oraclePrice: 'string',
+    };
+  }
+
   id!: number;
 
   pair!: string;

indexer/packages/postgres/src/models/oracle-price-model.ts

@@ -53,6 +53,22 @@ export default class OraclePriceModel extends Model {
     };
   }
 
+  /**
+   * A mapping from column name to JSON conversion expected.
+   * See getSqlConversionForDydxModelTypes for valid conversions.
+   *
+   * TODO(IND-239): Ensure that jsonSchema() / sqlToJsonConversions() / model fields match.
+   */
+  static get sqlToJsonConversions() {
+    return {
+      id: 'string',
+      marketId: 'integer',
+      price: 'string',
+      effectiveAt: 'date-time',
+      effectiveAtHeight: 'string',
+    };
+  }
+
   id!: string;
 
   marketId!: number;

indexer/packages/postgres/src/models/transfer-model.ts

@@ -104,6 +104,28 @@ export default class TransferModel extends Model {
     };
   }
 
+  /**
+   * A mapping from column name to JSON conversion expected.
+   * See getSqlConversionForDydxModelTypes for valid conversions.
+   *
+   * TODO(IND-239): Ensure that jsonSchema() / sqlToJsonConversions() / model fields match.
+   */
+  static get sqlToJsonConversions() {
+    return {
+      id: 'string',
+      senderSubaccountId: 'string',
+      recipientSubaccountId: 'string',
+      senderWalletAddress: 'string',
+      recipientWalletAddress: 'string',
+      assetId: 'string',
+      size: 'string',
+      eventId: 'hex-string',
+      transactionHash: 'string',
+      createdAt: 'date-time',
+      createdAtHeight: 'string',
+    };
+  }
+
   id!: string;
 
   senderSubaccountId?: string;

indexer/packages/postgres/src/stores/asset-position-table.ts

@@ -20,6 +20,7 @@ import {
 } from '../types';
 
 export function uuid(subaccountId: string, assetId: string): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${subaccountId}-${assetId}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/candle-table.ts

@@ -22,6 +22,7 @@ import {
 } from '../types';
 
 export function uuid(startedAt: IsoString, ticker: string, resolution: CandleResolution): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${startedAt}-${ticker}-${resolution}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/fill-table.ts

@@ -28,6 +28,7 @@ import {
 } from '../types';
 
 export function uuid(eventId: Buffer, liquidity: Liquidity): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${eventId.toString('hex')}-${liquidity}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/funding-index-updates-table.ts

@@ -26,6 +26,7 @@ export function uuid(
   eventId: Buffer,
   perpetualId: string,
 ): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${blockHeight}-${eventId.toString('hex')}-${perpetualId}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/oracle-price-table.ts

@@ -24,6 +24,7 @@ import {
 export function uuid(
   marketId: number, height: string,
 ): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${marketId.toString()}-${height}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/order-table.ts

@@ -26,6 +26,7 @@ export function uuid(
   clobPairId: string,
   orderFlags: string,
 ): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(
     Buffer.from(
       `${subaccountId}-${clientId}-${clobPairId}-${orderFlags}`,

indexer/packages/postgres/src/stores/perpetual-position-table.ts

@@ -45,6 +45,7 @@ const DEFAULT_SUBACCOUNT_UPDATE_DEFAULT_POSITION_FIELDS = {
 };
 
 export function uuid(subaccountId: string, openEventId: Buffer): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${subaccountId}-${openEventId.toString('hex')}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/pnl-ticks-table.ts

@@ -22,6 +22,7 @@ export function uuid(
   subaccountId: string,
   createdAt: string,
 ): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(
     Buffer.from(
       `${subaccountId}-${createdAt}`,

indexer/packages/postgres/src/stores/subaccount-table.ts

@@ -23,6 +23,7 @@ import {
 } from '../types';
 
 export function uuid(address: string, subaccountNumber: number): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${address}-${subaccountNumber}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/transaction-table.ts

@@ -20,6 +20,7 @@ import {
 } from '../types';
 
 export function uuid(blockHeight: string, transactionIndex: number): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(Buffer.from(`${blockHeight}-${transactionIndex}`, BUFFER_ENCODING_UTF_8));
 }
 

indexer/packages/postgres/src/stores/transfer-table.ts

@@ -31,6 +31,7 @@ export function uuid(
   senderWalletAddress?: string,
   recipientWalletAddress?: string,
 ): string {
+  // TODO(IND-483): Fix all uuid string substitutions to use Array.join.
   return getUuid(
     Buffer.from(
       `${senderSubaccountId}-${recipientSubaccountId}-${senderWalletAddress}-${recipientWalletAddress}-${eventId.toString('hex')}-${assetId}`,

indexer/packages/postgres/src/types/db-model-types.ts

@@ -66,6 +66,7 @@ export interface OrderFromDatabase extends IdBasedModelFromDatabase {
   updatedAtHeight: string;
   goodTilBlock?: string;
   goodTilBlockTime?: string;
+  // createdAtHeight is optional because short term orders do not have a createdAtHeight.
   createdAtHeight?: string;
   clientMetadata: string;
   triggerPrice?: string;

indexer/packages/postgres/src/types/order-types.ts

@@ -62,6 +62,7 @@ export interface OrderCreateObject {
   updatedAtHeight: string;
   goodTilBlock?: string;
   goodTilBlockTime?: string;
+  // createdAtHeight is optional because short term orders do not have a createdAtHeight.
   createdAtHeight?: string;
   clientMetadata: string;
   triggerPrice?: string,