update add markets to send market map proposal first and validation f…

…low to use v7.x commit

.github/workflows/validate-other-market-data.yml

@@ -37,7 +37,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: 'dydxprotocol/v4-chain'
-          ref: '725ad716cbe9e9aebf2472e877fe2a71e8a63d87'
+          ref: '18c72e1b2d9bf9849c1062ac022caba4fa9ecf60'
           path: 'v4-chain'
 
       - name: Start v4 localnet
@@ -49,11 +49,6 @@ jobs:
           echo "Starting localnet..."
           DOCKER_BUILDKIT=1 make localnet-init
           DOCKER_BUILDKIT=1 make localnet-compose-upd -e RAYDIUM_URL=${{ secrets.RAYDIUM_URL }}
-
-      - name: Get diff of otherMarketData.json
-        run: |
-          git fetch origin
-          git diff remotes/origin/main -- public/configs/otherMarketData.json > otherMarketDiff.txt
       
       - name: Checkout main branch
         uses: actions/checkout@v3

scripts/markets/add-markets.ts

@@ -4,22 +4,27 @@ This script adds markets to a dYdX chain. Markets are read from public/config/ot
 Supported environments: local, dev, dev2, dev3, dev4, dev5, staging.
 
 Usage:
-  $ pnpx tsx scripts/markets/add-markets.ts <environment> <number_of_markets>
+  $ pnpx tsx scripts/markets/add-markets.ts <environment> <number_of_markets> <path_to_dydx_binary>
 Example (add 10 markets on staging):
-  $ pnpx tsx scripts/markets/add-markets.ts staging 10
+  $ pnpx tsx scripts/markets/add-markets.ts staging 10 /Users/alice/v4-chain/protocol/build/dydxprotocold
 */
 import {
   CompositeClient,
   IndexerConfig,
   LocalWallet as LocalWalletType,
   Network,
-  ValidatorConfig,
+  ValidatorConfig
 } from '@dydxprotocol/v4-client-js';
-import { PerpetualMarketType } from '@dydxprotocol/v4-proto/src/codegen/dydxprotocol/perpetuals/perpetual';
 import { readFileSync } from 'fs';
 import Long from 'long';
-
-import { Proposal, retry, sleep, voteOnProposals } from './help';
+import {
+  createAndSendMarketMapProposal,
+  PerpetualMarketType,
+  Proposal,
+  retry,
+  sleep,
+  voteOnProposals
+} from './help';
 
 const LocalWalletModule = await import(
   '@dydxprotocol/v4-client-js/src/clients/modules/local-wallet'
@@ -132,7 +137,12 @@ const ENV_CONFIG = {
   },
 };
 
-async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]): Promise<void> {
+async function addMarkets(
+  env: Env,
+  numMarkets: number,
+  proposals: Proposal[],
+  binary: string
+): Promise<void> {
   // Initialize client and wallets.
   const config = ENV_CONFIG[env];
   const indexerConfig = new IndexerConfig(config.indexerRestEndpoint, config.indexerWsEndpoint);
@@ -150,6 +160,7 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]):
     'Client Example'
   );
   const network = new Network(env, indexerConfig, validatorConfig);
+  const sleepMsBtwTxs = 3.5 * config.blockTimeSeconds * 1000;
 
   const client = await CompositeClient.connect(network);
   const wallets: LocalWalletType[] = await Promise.all(
@@ -158,16 +169,37 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]):
     })
   );
 
-  // Send proposals to add all markets (skip markets that already exist).
+  // Filter out markets that already exist on-chain.
   const allPerps = await client.validatorClient.get.getAllPerpetuals();
   const allTickers = allPerps.perpetual.map((perp) => perp.params!.ticker);
   const filteredProposals = proposals.filter(
     (proposal) => !allTickers.includes(proposal.params.ticker)
   );
 
-  console.log(`Adding ${numMarkets} new markets to ${env}...`);
+  // Add markets to market map first.
+  console.log("Submitting market map proposal...");
+  await createAndSendMarketMapProposal(
+    filteredProposals.slice(0, numMarkets),
+    config.validatorEndpoint,
+    config.chainId,
+    binary,
+  );
+  await sleep(sleepMsBtwTxs);
+  console.log("Submitted market map proposal");
+
+  // Get latest gov proposal ID.
+  const allProposalsResp = await client.validatorClient.get.getAllGovProposals();
+  let latestProposalId = allProposalsResp.proposals.reduce(
+    (max, proposal) => (proposal.id.toNumber() > max ? proposal.id.toNumber() : max),
+    0
+  );
+
+  for (const wallet of wallets) {
+    retry(() => voteOnProposals([latestProposalId], client, wallet));
+  }
+  console.log(`Voted on market map proposal with id ${latestProposalId}`);
+  await sleep(sleepMsBtwTxs);
 
-  const sleepMsBtwTxs = 3.5 * config.blockTimeSeconds * 1000;
   let numProposalsToSend = Math.min(numMarkets, filteredProposals.length);
   let numProposalsSent = 0;
   const numExistingMarkets = allPerps.perpetual.reduce(
@@ -184,8 +216,7 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]):
         break;
       }
       const proposal = proposalsToSend[j];
-      const proposalId: number = i + j + 1;
-      const marketId: number = numExistingMarkets + proposalId;
+      const marketId: number = numExistingMarkets + numProposalsSent + 1;
 
       // Send proposal.
       const exchangeConfigString = `{"exchanges":${JSON.stringify(
@@ -220,7 +251,7 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]):
       console.log(`Proposed market ${marketId} with ticker ${proposal.params.ticker}`);
 
       // Record proposed market.
-      proposalIds.push(proposalId);
+      proposalIds.push(++latestProposalId);
       numProposalsSent++;
     }
 
@@ -242,12 +273,15 @@ async function main(): Promise<void> {
   const args = process.argv.slice(2);
   const env = args[0] as Env;
   const numMarkets = parseInt(args[1], 10);
+  const binary = args[2];
 
   // Validate inputs.
   if (!Object.values(Env).includes(env)) {
     throw new Error(`Invalid environment: ${env}`);
   } else if (isNaN(numMarkets) || numMarkets <= 0) {
     throw new Error(`Invalid number of markets: ${numMarkets}`);
+  } else if (!binary) {
+    throw new Error(`dydx binary path not provided`);
   }
 
   // Read proposals.
@@ -256,7 +290,7 @@ async function main(): Promise<void> {
   );
 
   // Add markets.
-  await addMarkets(env, numMarkets, Object.values(proposals));
+  await addMarkets(env, numMarkets, Object.values(proposals), binary);
 }
 
 main()

scripts/markets/help.ts

@@ -9,8 +9,11 @@ import {
   VoteOption,
 } from '@dydxprotocol/v4-client-js';
 import { MsgVote } from '@dydxprotocol/v4-proto/src/codegen/cosmos/gov/v1/tx';
+import { spawn } from 'child_process';
+import * as fs from 'fs';
 import Long from 'long';
 
+
 const VOTE_FEE: StdFee = {
   amount: [
     {
@@ -25,6 +28,7 @@ export interface Exchange {
   exchangeName: ExchangeName;
   ticker: string;
   adjustByMarket?: string;
+  invert?: boolean;
 }
 
 export enum ExchangeName {
@@ -68,6 +72,61 @@ export interface Proposal {
   params: Params;
 }
 
+
+export enum PerpetualMarketType {
+  /** PERPETUAL_MARKET_TYPE_UNSPECIFIED - Unspecified market type. */
+  PERPETUAL_MARKET_TYPE_UNSPECIFIED = 0,
+
+  /** PERPETUAL_MARKET_TYPE_CROSS - Market type for cross margin perpetual markets. */
+  PERPETUAL_MARKET_TYPE_CROSS = 1,
+
+  /** PERPETUAL_MARKET_TYPE_ISOLATED - Market type for isolated margin perpetual markets. */
+  PERPETUAL_MARKET_TYPE_ISOLATED = 2,
+  UNRECOGNIZED = -1,
+}
+
+interface Market {
+  ticker: {
+    currency_pair: {
+      Base: string;
+      Quote: string;
+    };
+    decimals: string;
+    enabled: boolean;
+    min_provider_count: string;
+    metadata_JSON: string;
+  };
+  provider_configs: ProviderConfig[];
+}
+
+interface ProviderConfig {
+  name: string;
+  normalize_by_pair: {
+    Base: string;
+    Quote: string;
+  } | null;
+  off_chain_ticker: string;
+  invert: boolean;
+  metadata_JSON: string;
+}
+
+const exchangeNameToMarketMapProviderName: Record<ExchangeName, string> = {
+  [ExchangeName.Binance]: 'binance_ws',
+  [ExchangeName.BinanceUS]: 'binance_ws',
+  [ExchangeName.Bitfinex]: 'bitfinex_ws',
+  [ExchangeName.Bitstamp]: 'bitstamp_api',
+  [ExchangeName.Bybit]: 'bybit_ws',
+  [ExchangeName.CoinbasePro]: 'coinbase_ws',
+  [ExchangeName.CryptoCom]: 'crypto_dot_com_ws',
+  [ExchangeName.Gate]: 'gate_ws',
+  [ExchangeName.Huobi]: 'huobi_ws',
+  [ExchangeName.Kraken]: 'kraken_api',
+  [ExchangeName.Kucoin]: 'kucoin_ws',
+  [ExchangeName.Mexc]: 'mexc_ws',
+  [ExchangeName.Okx]: 'okx_ws',
+  [ExchangeName.Raydium]: 'raydium_api',
+};
+
 export async function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => {
     setTimeout(resolve, ms);
@@ -130,4 +189,126 @@ export async function voteOnProposals(
   } else {
     console.log(`Voted on proposals ${proposalIds} with wallet ${wallet.address}`);
   }
-}
+}
+
+export async function createAndSendMarketMapProposal(
+  proposals: Proposal[],
+  validatorEndpoint: string,
+  chainId: string,
+  binary: string,
+) {
+  const markets: Market[] = proposals.map((proposal) => {
+    const { ticker, priceExponent, minExchanges, exchangeConfigJson } = proposal.params;
+
+    const providerConfigs: ProviderConfig[] = exchangeConfigJson.map((config) => {
+      let normalize_by_pair: { Base: string; Quote: string } | null = null;
+
+      if (config.adjustByMarket) {
+        const [Base, Quote] = config.adjustByMarket.split('-');
+        normalize_by_pair = { Base, Quote };
+      }
+
+      return {
+        name: `${exchangeNameToMarketMapProviderName[config.exchangeName as ExchangeName]}`,
+        normalize_by_pair,
+        off_chain_ticker: config.ticker,
+        invert: config.invert || false,
+        metadata_JSON: '',
+      };
+    });
+
+    return {
+      ticker: {
+        currency_pair: {
+          Base: ticker.split('-')[0],
+          Quote: ticker.split('-')[1],
+        },
+        decimals: Math.abs(priceExponent).toString(),
+        enabled: true,
+        min_provider_count: minExchanges.toString(),
+        metadata_JSON: '',
+      },
+      provider_configs: providerConfigs,
+    };
+  });
+
+  const proposal = {
+    "title": "Add markets to market map",
+    "summary":"Add markets to market map",
+    "messages": [
+      {
+        "@type": "/slinky.marketmap.v1.MsgCreateMarkets",
+        "authority": "dydx10d07y265gmmuvt4z0w9aw880jnsr700jnmapky",
+        "create_markets": markets,
+      },
+    ],
+    "deposit":"5000000000000000000adv4tnt",
+    "expedited": true,
+  };
+
+  const proposalFile = 'marketMapProposal.json';
+  fs.writeFileSync(proposalFile, JSON.stringify(proposal, null, 2), 'utf-8');
+
+  try {
+    await execCLI(
+      binary,
+      ['keys', 'show', 'alice'],
+    )
+  } catch (error) {
+    await execCLI(
+      binary,
+      ['keys', 'add', 'tom', '--recover'],
+      'merge panther lobster crazy road hollow amused security before critic about cliff exhibit cause coyote talent happy where lion river tobacco option coconut small',
+    )
+  }
+
+  await execCLI(
+    binary,
+    [
+      '--node', validatorEndpoint,
+      'tx', 'gov', 'submit-proposal', 'marketMapProposal.json',
+      '--from', 'alice',
+      '--fees', '300000000000000000adv4tnt',
+      '--chain-id', chainId,
+      '--gas', 'auto'
+    ],
+    'y',
+  )
+
+  fs.unlinkSync(proposalFile);
+}
+
+export function execCLI(
+  command: string,
+  args?: string[],
+  input?: string,
+): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const process = spawn(command, args);
+
+    let output = '';
+
+    process.stdout.on('data', (data) => {
+      console.log(`stdout: ${data}`);
+      output += data.toString();
+    });
+
+    process.stderr.on('data', (data) => {
+      console.error(`stderr: ${data}`);
+      output += data.toString();
+    });
+
+    process.on('close', (code) => {
+      if (code === 0) {
+        resolve(output);
+      } else {
+        reject(`Process exited with code: ${code}`);
+      }
+    });
+
+    if (input) {
+      process.stdin.write(`${input}\n`);
+    }
+    process.stdin.end();
+  });
+}