Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
merge from main
Browse files Browse the repository at this point in the history
jakob-dydx committed Dec 5, 2023
2 parents 4050771 + 56d653a commit 45d6a49
Showing 114 changed files with 1,441 additions and 1,335 deletions.
6 changes: 4 additions & 2 deletions indexer/packages/kafka/src/batch-kafka-producer.ts
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@ import { logger } from '@dydxprotocol-indexer/base';
import { Producer, RecordMetadata } from 'kafkajs';
import _ from 'lodash';

import config from './config';
import { KafkaTopics } from './types';

/**
@@ -28,7 +27,10 @@ export class BatchKafkaProducer {
constructor(
topic: KafkaTopics,
producer: Producer,
maxBatchSizeBytes: number = config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES,
// Note that default parameters are bound during module load time making it difficult
// to modify the parameter during a test so we explicitly require callers to pass in
// config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES.
maxBatchSizeBytes: number,
) {
this.maxBatchSizeBytes = maxBatchSizeBytes;
this.producer = producer;
26 changes: 10 additions & 16 deletions indexer/packages/postgres/src/helpers/stores-helpers.ts
Original file line number Diff line number Diff line change
@@ -64,25 +64,19 @@ export async function rawQuery(
options: Options,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
): Promise<Knex.Raw<any>> {
if (options.readReplica) {
if (options.txId) {
return knexReadReplica.getConnection().raw(queryString).transacting(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
<Knex.Transaction<any, any>>Transaction.get(options.txId),
);
} else {
return knexReadReplica.getConnection().raw(queryString);
}
} else {
if (options.txId) {
return knexPrimary.raw(queryString).transacting(
const connection = options.readReplica ? knexReadReplica.getConnection() : knexPrimary;
let queryBuilder = options.bindings === undefined
? connection.raw(queryString) : connection.raw(queryString, options.bindings);
if (options.txId) {
queryBuilder = queryBuilder.transacting(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
<Knex.Transaction<any, any>>Transaction.get(options.txId),
);
} else {
return knexPrimary.raw(queryString);
}
);
}
if (options.sqlOptions) {
queryBuilder = queryBuilder.options(options.sqlOptions);
}
return queryBuilder;
}

/* ------- Bulk Helpers ------- */
5 changes: 5 additions & 0 deletions indexer/packages/postgres/src/types/utility-types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/* ------- UTILITY TYPES ------- */
import { RawBinding } from 'knex';

export type IsoString = string;

export type RegexPattern = string;
@@ -17,6 +19,9 @@ export interface Options {
orderBy?: [string, Ordering][];
readReplica?: boolean,
random?: boolean;
bindings?: readonly RawBinding[];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
sqlOptions?: Readonly<{ [key: string]: any }>;
}

export enum Ordering {
Original file line number Diff line number Diff line change
@@ -37,7 +37,8 @@ export enum ClobPair_Status {
/**
* STATUS_FINAL_SETTLEMENT - STATUS_FINAL_SETTLEMENT represents a clob pair which is deactivated
* and trading has ceased. All open positions will be closed by the
* protocol.
* protocol. Open stateful orders will be cancelled. Open short-term
* orders will be left to expire.
*/
STATUS_FINAL_SETTLEMENT = 6,
UNRECOGNIZED = -1,
@@ -79,7 +80,8 @@ export enum ClobPair_StatusSDKType {
/**
* STATUS_FINAL_SETTLEMENT - STATUS_FINAL_SETTLEMENT represents a clob pair which is deactivated
* and trading has ceased. All open positions will be closed by the
* protocol.
* protocol. Open stateful orders will be cancelled. Open short-term
* orders will be left to expire.
*/
STATUS_FINAL_SETTLEMENT = 6,
UNRECOGNIZED = -1,
Original file line number Diff line number Diff line change
@@ -128,6 +128,14 @@ export interface MatchPerpetualDeleveraging {
/** An ordered list of fills created by this liquidation. */

fills: MatchPerpetualDeleveraging_Fill[];
/**
* Flag denoting whether the deleveraging operation was for the purpose
* of final settlement. Final settlement matches are at the oracle price,
* whereas deleveraging happens at the bankruptcy price of the deleveraged
* subaccount.
*/

isFinalSettlement: boolean;
}
/**
* MatchPerpetualDeleveraging is an injected message used for deleveraging a
@@ -143,6 +151,14 @@ export interface MatchPerpetualDeleveragingSDKType {
/** An ordered list of fills created by this liquidation. */

fills: MatchPerpetualDeleveraging_FillSDKType[];
/**
* Flag denoting whether the deleveraging operation was for the purpose
* of final settlement. Final settlement matches are at the oracle price,
* whereas deleveraging happens at the bankruptcy price of the deleveraged
* subaccount.
*/

is_final_settlement: boolean;
}
/** Fill represents a fill between the liquidated and offsetting subaccount. */

@@ -451,7 +467,8 @@ function createBaseMatchPerpetualDeleveraging(): MatchPerpetualDeleveraging {
return {
liquidated: undefined,
perpetualId: 0,
fills: []
fills: [],
isFinalSettlement: false
};
}

@@ -469,6 +486,10 @@ export const MatchPerpetualDeleveraging = {
MatchPerpetualDeleveraging_Fill.encode(v!, writer.uint32(26).fork()).ldelim();
}

if (message.isFinalSettlement === true) {
writer.uint32(32).bool(message.isFinalSettlement);
}

return writer;
},

@@ -493,6 +514,10 @@ export const MatchPerpetualDeleveraging = {
message.fills.push(MatchPerpetualDeleveraging_Fill.decode(reader, reader.uint32()));
break;

case 4:
message.isFinalSettlement = reader.bool();
break;

default:
reader.skipType(tag & 7);
break;
@@ -507,6 +532,7 @@ export const MatchPerpetualDeleveraging = {
message.liquidated = object.liquidated !== undefined && object.liquidated !== null ? SubaccountId.fromPartial(object.liquidated) : undefined;
message.perpetualId = object.perpetualId ?? 0;
message.fills = object.fills?.map(e => MatchPerpetualDeleveraging_Fill.fromPartial(e)) || [];
message.isFinalSettlement = object.isFinalSettlement ?? false;
return message;
}

26 changes: 25 additions & 1 deletion indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/mev.ts
Original file line number Diff line number Diff line change
@@ -94,6 +94,8 @@ export interface ValidatorMevMatchesSDKType {
export interface MevNodeToNodeMetrics {
validatorMevMatches?: ValidatorMevMatches;
clobMidPrices: ClobMidPrice[];
bpMevMatches?: ValidatorMevMatches;
proposalReceiveTime: Long;
}
/**
* MevNodeToNodeMetrics is a data structure for encapsulating all MEV node <>
@@ -103,6 +105,8 @@ export interface MevNodeToNodeMetrics {
export interface MevNodeToNodeMetricsSDKType {
validator_mev_matches?: ValidatorMevMatchesSDKType;
clob_mid_prices: ClobMidPriceSDKType[];
bp_mev_matches?: ValidatorMevMatchesSDKType;
proposal_receive_time: Long;
}

function createBaseMEVMatch(): MEVMatch {
@@ -448,7 +452,9 @@ export const ValidatorMevMatches = {
function createBaseMevNodeToNodeMetrics(): MevNodeToNodeMetrics {
return {
validatorMevMatches: undefined,
clobMidPrices: []
clobMidPrices: [],
bpMevMatches: undefined,
proposalReceiveTime: Long.UZERO
};
}

@@ -462,6 +468,14 @@ export const MevNodeToNodeMetrics = {
ClobMidPrice.encode(v!, writer.uint32(18).fork()).ldelim();
}

if (message.bpMevMatches !== undefined) {
ValidatorMevMatches.encode(message.bpMevMatches, writer.uint32(26).fork()).ldelim();
}

if (!message.proposalReceiveTime.isZero()) {
writer.uint32(32).uint64(message.proposalReceiveTime);
}

return writer;
},

@@ -482,6 +496,14 @@ export const MevNodeToNodeMetrics = {
message.clobMidPrices.push(ClobMidPrice.decode(reader, reader.uint32()));
break;

case 3:
message.bpMevMatches = ValidatorMevMatches.decode(reader, reader.uint32());
break;

case 4:
message.proposalReceiveTime = (reader.uint64() as Long);
break;

default:
reader.skipType(tag & 7);
break;
@@ -495,6 +517,8 @@ export const MevNodeToNodeMetrics = {
const message = createBaseMevNodeToNodeMetrics();
message.validatorMevMatches = object.validatorMevMatches !== undefined && object.validatorMevMatches !== null ? ValidatorMevMatches.fromPartial(object.validatorMevMatches) : undefined;
message.clobMidPrices = object.clobMidPrices?.map(e => ClobMidPrice.fromPartial(e)) || [];
message.bpMevMatches = object.bpMevMatches !== undefined && object.bpMevMatches !== null ? ValidatorMevMatches.fromPartial(object.bpMevMatches) : undefined;
message.proposalReceiveTime = object.proposalReceiveTime !== undefined && object.proposalReceiveTime !== null ? Long.fromValue(object.proposalReceiveTime) : Long.UZERO;
return message;
}

Original file line number Diff line number Diff line change
@@ -465,12 +465,20 @@ export interface DeleveragingEventV1 {
*/

fillAmount: Long;
/** Bankruptcy price of liquidated subaccount, in USDC quote quantums. */
/** Fill price of deleveraging event, in USDC quote quantums. */

price: Long;
/** `true` if liquidating a short position, `false` otherwise. */

isBuy: boolean;
/**
* `true` if the deleveraging event is for final settlement, indicating
* the match occurred at the oracle price rather than bankruptcy price.
* When this flag is `false`, the fill price is the bankruptcy price
* of the liquidated subaccount.
*/

isFinalSettlement: boolean;
}
/**
* DeleveragingEvent message contains all the information for a deleveraging
@@ -493,12 +501,20 @@ export interface DeleveragingEventV1SDKType {
*/

fill_amount: Long;
/** Bankruptcy price of liquidated subaccount, in USDC quote quantums. */
/** Fill price of deleveraging event, in USDC quote quantums. */

price: Long;
/** `true` if liquidating a short position, `false` otherwise. */

is_buy: boolean;
/**
* `true` if the deleveraging event is for final settlement, indicating
* the match occurred at the oracle price rather than bankruptcy price.
* When this flag is `false`, the fill price is the bankruptcy price
* of the liquidated subaccount.
*/

is_final_settlement: boolean;
}
/**
* LiquidationOrder represents the liquidation taker order to be included in a
@@ -1782,7 +1798,8 @@ function createBaseDeleveragingEventV1(): DeleveragingEventV1 {
perpetualId: 0,
fillAmount: Long.UZERO,
price: Long.UZERO,
isBuy: false
isBuy: false,
isFinalSettlement: false
};
}

@@ -1812,6 +1829,10 @@ export const DeleveragingEventV1 = {
writer.uint32(48).bool(message.isBuy);
}

if (message.isFinalSettlement === true) {
writer.uint32(56).bool(message.isFinalSettlement);
}

return writer;
},

@@ -1848,6 +1869,10 @@ export const DeleveragingEventV1 = {
message.isBuy = reader.bool();
break;

case 7:
message.isFinalSettlement = reader.bool();
break;

default:
reader.skipType(tag & 7);
break;
@@ -1865,6 +1890,7 @@ export const DeleveragingEventV1 = {
message.fillAmount = object.fillAmount !== undefined && object.fillAmount !== null ? Long.fromValue(object.fillAmount) : Long.UZERO;
message.price = object.price !== undefined && object.price !== null ? Long.fromValue(object.price) : Long.UZERO;
message.isBuy = object.isBuy ?? false;
message.isFinalSettlement = object.isFinalSettlement ?? false;
return message;
}

2 changes: 2 additions & 0 deletions indexer/services/bazooka/src/vulcan-helpers.ts
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@ import { Long } from '@dydxprotocol-indexer/v4-protos/build/codegen/helpers';
import Big from 'big.js';
import _ from 'lodash';

import config from './config';
import { ZERO } from './constants';

interface VulcanMessage {
@@ -134,6 +135,7 @@ export async function sendStatefulOrderMessages() {
const batchProducer: BatchKafkaProducer = new BatchKafkaProducer(
KafkaTopics.TO_VULCAN,
producer,
config.KAFKA_MAX_BATCH_WEBSOCKET_MESSAGE_SIZE_BYTES,
);
for (const message of messages) {
batchProducer.addMessageAndMaybeFlush(message);
Original file line number Diff line number Diff line change
@@ -85,6 +85,7 @@ describe('assetHandler', () => {

const handler: AssetCreationHandler = new AssetCreationHandler(
block,
0,
indexerTendermintEvent,
0,
defaultAssetCreateEvent,
Original file line number Diff line number Diff line change
@@ -89,6 +89,7 @@ describe('fundingHandler', () => {

const handler: FundingHandler = new FundingHandler(
block,
0,
indexerTendermintEvent,
0,
defaultFundingUpdateSampleEvent,
Original file line number Diff line number Diff line change
@@ -90,6 +90,7 @@ describe('liquidityTierHandler', () => {

const handler: LiquidityTierHandler = new LiquidityTierHandler(
block,
0,
indexerTendermintEvent,
0,
defaultLiquidityTierUpsertEvent,
Loading

0 comments on commit 45d6a49

Please sign in to comment.