feat(price_pusher): make jito bundles independent (#1565)

* Checkpoint

* bump

* Comment

* Fix

* Improvde test

apps/price_pusher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/price-pusher",
-  "version": "6.7.2",
+  "version": "6.8.0",
   "description": "Pyth Price Pusher",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",

apps/price_pusher/src/solana/solana.ts

@@ -12,6 +12,7 @@ import {
   sendTransactionsJito,
 } from "@pythnetwork/solana-utils";
 import { SearcherClient } from "jito-ts/dist/sdk/block-engine/searcher";
+import { sliceAccumulatorUpdateData } from "@pythnetwork/price-service-sdk";
 
 export class SolanaPriceListener extends ChainPriceListener {
   constructor(
@@ -107,6 +108,8 @@ export class SolanaPricePusher implements IPricePusher {
   }
 }
 
+const UPDATES_PER_JITO_BUNDLE = 7;
+
 export class SolanaPricePusherJito implements IPricePusher {
   constructor(
     private pythSolanaReceiver: PythSolanaReceiver,
@@ -121,7 +124,7 @@ export class SolanaPricePusherJito implements IPricePusher {
     priceIds: string[],
     pubTimesToPush: number[]
   ): Promise<void> {
-    let priceFeedUpdateData;
+    let priceFeedUpdateData: string[];
     try {
       priceFeedUpdateData = await this.priceServiceConnection.getLatestVaas(
         priceIds
@@ -131,47 +134,29 @@ export class SolanaPricePusherJito implements IPricePusher {
       return;
     }
 
-    const transactionBuilder = this.pythSolanaReceiver.newTransactionBuilder({
-      closeUpdateAccounts: false,
-    });
-    await transactionBuilder.addUpdatePriceFeed(
-      priceFeedUpdateData,
-      this.shardId
-    );
-    await transactionBuilder.addClosePreviousEncodedVaasInstructions();
-
-    const transactions = await transactionBuilder.buildVersionedTransactions({
-      jitoTipLamports: this.jitoTipLamports,
-      tightComputeBudget: true,
-      jitoBundleSize: this.jitoBundleSize,
-    });
-
-    const firstSignature = await sendTransactionsJito(
-      transactions.slice(0, this.jitoBundleSize),
-      this.searcherClient,
-      this.pythSolanaReceiver.wallet
-    );
+    for (let i = 0; i < priceIds.length; i += UPDATES_PER_JITO_BUNDLE) {
+      const transactionBuilder = this.pythSolanaReceiver.newTransactionBuilder({
+        closeUpdateAccounts: true,
+      });
+      await transactionBuilder.addUpdatePriceFeed(
+        priceFeedUpdateData.map((x) => {
+          return sliceAccumulatorUpdateData(
+            Buffer.from(x, "base64"),
+            i,
+            i + UPDATES_PER_JITO_BUNDLE
+          ).toString("base64");
+        }),
+        this.shardId
+      );
 
-    const blockhashResult =
-      await this.pythSolanaReceiver.connection.getLatestBlockhashAndContext({
-        commitment: "confirmed",
+      const transactions = await transactionBuilder.buildVersionedTransactions({
+        jitoTipLamports: this.jitoTipLamports,
+        tightComputeBudget: true,
+        jitoBundleSize: this.jitoBundleSize,
       });
-    await this.pythSolanaReceiver.connection.confirmTransaction(
-      {
-        signature: firstSignature,
-        blockhash: blockhashResult.value.blockhash,
-        lastValidBlockHeight: blockhashResult.value.lastValidBlockHeight,
-      },
-      "confirmed"
-    );
 
-    for (
-      let i = this.jitoBundleSize;
-      i < transactions.length;
-      i += this.jitoBundleSize
-    ) {
       await sendTransactionsJito(
-        transactions.slice(i, i + this.jitoBundleSize),
+        transactions,
         this.searcherClient,
         this.pythSolanaReceiver.wallet
       );

price_service/sdk/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/price-service-sdk",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "description": "Pyth price service SDK",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",

price_service/sdk/js/src/AccumulatorUpdateData.ts

@@ -64,6 +64,59 @@ export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage {
   };
 }
 
+/**
+ * An AccumulatorUpdateData contains a VAA and a list of updates. This function returns a new serialized AccumulatorUpdateData with only the updates in the range [start, end).
+ */
+export function sliceAccumulatorUpdateData(
+  data: Buffer,
+  start?: number,
+  end?: number
+): Buffer {
+  if (!isAccumulatorUpdateData(data)) {
+    throw new Error("Invalid accumulator message");
+  }
+  let cursor = 6;
+  const trailingPayloadSize = data.readUint8(cursor);
+  cursor += 1 + trailingPayloadSize;
+
+  // const proofType = data.readUint8(cursor);
+  cursor += 1;
+
+  const vaaSize = data.readUint16BE(cursor);
+  cursor += 2;
+  cursor += vaaSize;
+
+  const endOfVaa = cursor;
+
+  const updates = [];
+  const numUpdates = data.readUInt8(cursor);
+  cursor += 1;
+
+  for (let i = 0; i < numUpdates; i++) {
+    const updateStart = cursor;
+    const messageSize = data.readUint16BE(cursor);
+    cursor += 2;
+    cursor += messageSize;
+
+    const numProofs = data.readUInt8(cursor);
+    cursor += 1;
+    cursor += KECCAK160_HASH_SIZE * numProofs;
+
+    updates.push(data.subarray(updateStart, cursor));
+  }
+
+  if (cursor !== data.length) {
+    throw new Error("Didn't reach the end of the message");
+  }
+
+  const sliceUpdates = updates.slice(start, end);
+  return Buffer.concat([
+    data.subarray(0, endOfVaa),
+    Buffer.from([sliceUpdates.length]),
+    ...updates.slice(start, end),
+  ]);
+}
+
 export function parseAccumulatorUpdateData(
   data: Buffer
 ): AccumulatorUpdateData {

price_service/sdk/js/src/__tests__/AccumulatorUpdateData.test.ts

@@ -1,28 +1,109 @@
 import {
   parseAccumulatorUpdateData,
   parsePriceFeedMessage,
+  sliceAccumulatorUpdateData,
 } from "../AccumulatorUpdateData";
 
 // This is just a sample update data from hermes
 const TEST_ACCUMULATOR_UPDATE_DATA =
-  "UE5BVQEAAAADuAEAAAADDQILBDsleD7xENCN7O7KjXAe15OQQ8FvD0E6L+bIemppBjNEZEOF0XuxCPQ/ti4gWUplKpieCWYiibuaDvXOeSFFAQOuYekUPV6yqK2BRWo/KRmJ00SV01OJDRhNzc1YnzUi+j8dt1tNBkVY99NIs5leyixUmeq4Dn1n8y37xN4JtAacAQSxJzGeZ6tejBD1YlPBlVwaCX8crkp19R1es7rDBhX/iit2plz5grz66fPj/mpoffZqKo95Fq/0sxWHIvn4nhgXAQYLl99cpa6KlaA8q1Pj7sN6TXNrXtmTBlzRU6dZ0ptO8VKp4K3zkVqbWkB5mbHCeuYNgOGMCnVsS7Ce9J7NganNAQf0nyez/5yR/U2zu+XRbi8eNzI1yJ9Hc4lmMl8pTPPQRgrs9HyiVCliCOcHdLzLio3JoLBhmFxQ3ygYj2eB+k3UAQgHX1e/+vbCjBNnmx/UQV8m0y/wifKAMfYpK4mR8voG3wgxo5MIFUvvCZ9/Gt1GizTX5CuoQD9J4ioxjoCFghVtAQqG5lFSpVRpC0dQlMv2ju2K89Ph0tJGsX7LGRXRnh9lEkkM8W+Uxf1R50HFsZHiXU08Grz0mKRPavesrzD+1xYGAQuYL6q5SagvBS7TfZJYS4kUMw74TvMiHLWx2ps3EdEJbh3WCWGfOM3amrplQBnqctDYh3StqspyTdaU5QTxfyYvAQwNWdPBEtAR6yIHB8KYrEDGGUH91uqD768NGigW6ziLwnNw2un+gcDUiafL3pZpqC4yIDhmnEz26PmQs4cAI5nkAA3/Zl3Pt7fLG3E5xBa/lbdrBUT3J+znFExbuFZuZipvbBwnQq/yyBSXqyfuHG3GTQZ/wXBto5zUEyex9889XYzaAQ52EUUCG0X4i0nWHeAf00+s6cODkW6hanQ1MHfTdvvVMXqK9nfvicz8pBna/NVp1wiTN5zR9rWjQuAf0g0c6TRLARCPT46a0/3xER/tV7WLQ6JQUWHMbV0G6cXKmdFT0Qg3/m08Dlabic+EHW9u2ugZA5sJ/Jl4oGk/lWLJoNoxDFMZARJgsgN2RdhjvJMRmf/Kj0d5PSvI7kE+J7ShlJd5058+ETZVPR15fJpT3BWUJ8i/vdGmU90A6iGyXRmNRBFx21qqAGXeGOEAAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAACjFEsAUFVV1YAAAAAAAeUpesAACcQy5naBQ5EEmAnr2RvKBD1SUJH9zwBAFUA7w2Lb9os66QdoV1AldHaOSoNL47Qxse8D0z6yMKAtW0AAAACgAWPuQAAAAAAX+C0////+AAAAABl3hjhAAAAAGXeGOEAAAACgrM/OAAAAAAAaAvYCuE9uKnM0BlXJ1E/fLYtWytcVLhkSCzHW9p1ReYbPMt07qbcN5KykfYDlCJxjBT3UmyTbhTT30PmOLkZu9zraLg22Wysdg1W67WoQZi654djZPBpAiHRU2KQXbDSGqJcekD0W+TqKO+QagPAoXksP0iMGEYpBdVZKGhSvw0NpXv/5Qb5NHO/+dTahPFBgXJH+9geJaxYru9ZRMg5o+YjIvkwuom/2NP0mTbfp4syeDQBs+fmcdGmAjgcdF0wt1gR6kYbsWQ/CZ08";
+  "UE5BVQEAAAADuAEAAAAEDQDCQmx/SUJVpnlJ6D/HXQvOYGhjtvPa8y2xwvTduTs6oj2N406RF1OYsMmqnuqXv8f7iEimuEkvQdHhgY28eRy9AAJanMT6xkyTcnMxNt8Vgn+EgOAxCryiqLhc05Rlpcwe0S3+OmLN1ifjPuanHH3D8CkwOvDJgUbU1PLhTloGH+0oAAPYrlxLSvd8hYqfjiC7eSdbpeD7X0R2jXb+0nL7YVHrAUeu3uEnvAziRg73GOTc0G9R6UWCg+YP/zRp3krAsDIPAASBDxiDxF2HE9LCH4NeC7D3s47gZKUwl0B3ptabRZYvc0U/7Ttz2RTzl5PfAXTK60DWJnJERDlAbj8c59Jos9v4AAY8OPOzSRUyoQhYpphlBaTjO8q3Dg5Qrv5amnGDclx6VAG6vGfqErtSpsMjBZLnz8Lhxp4eJ1Ot4DI1IGmxJbRdAAes8Nc5dDCvIiTPwMpzN4ma51whWivcHq/ymviUKhg9pFibGCzRQW8NsxRDfZH2/cf2fVyC1mr7Pftv2EPBJO1uAApXWWLkjOZXKUWDiEWkWyAE14xLHCNclXDlVPehMM0huEmDgijMSUKyRPHaw/NMFTzA3OecXGskVKxmdFQcX0DCAQv5QVoq0b+Td0Cs1/TwftoUGr+R8AmdUUuwDn2oRK4I61NmRhF4mYaszUH5ERsHo4SNxTA+RbcTT5fflAC7XriVAQxGICt7NNC5EnA6+MvTsQhRgbbmr+qnBSq5VvEF65iWyFWwaeRDhjtk81u6DZkxhfS7+QzUsFFjO9sGkl1ZMv8hAA1uAeD1DRgMxbipcmjTkmI6mXMWzbyFmMAJUi+jXe7740OVQOBMEjkYHGeDXdNaKXQmRCmNy5mXRnFO1n9piFzVAA4QwHiq6D/IJveCc8+ynJsaR+PNwADmbIrdGb4Y4sMSuWC6kEp6WyKcNZizrk1ZB1Dl8jF3aiunNXtb8DjtAMTDAA9yFaEkIKOml5mSceZ0yDnkDkE53a1/0yHKG1RLAF1iPD/aToPh3U07FRcf8uVnhof0q61VkNy1Bgm5R7cJDJFoABJToX2me8ANo3nZC/NDDxCfVBZcvIfgGsqPuxFEkgFOKGAqCWnMYRzhxaqPrgg1q6nYa/8qONS7zprGCiUHoI4iAWZCZIoAAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAADW+Y9AUFVV1YAAAAAAAhmA68AACcQh+eO4lll0hkFZY214Rd4PGknF0YDAFUAyWRY05P+net6fWOgrEHiiYpnp3UNvRZmcyeeBsho3woAAAAAAEeOXAAAAAAAABge////+AAAAABmQmSKAAAAAGZCZIkAAAAAAEeS3gAAAAAAABhxChHz7A8jzwzaF8ZQL4TSYFOrMO27C2wkaI7qTgtVcAmYcC/k7aSXpmkPACMiQd+IP4agmvqvwdByAMA2cVSYxfwESuHDoqjanEewjAA6SION5ZwUkIrqTCPO+naSyR6H808OYDuzUX37m5Dc91HlPJqzeZBUg60znGDwRXLHtMte5ZKwxskxaSaMdPfK3dn+QLjw7IvRuvJNlhjDTC/KzQ3Pe7huLggEYJPpvJSw++VhJh9389orPHR1YFWlYdzY15NdQwX9gzObAFUA/2FJGpMREt3xvYFHzRtkE3X3n1glEm1mVICHRjT9Cs4AAABEq/mnjgAAAAAK+/OL////+AAAAABmQmSKAAAAAGZCZIkAAABEllmXcAAAAAAORmsgCmwQvv7XRaz2EALTUYcqq0yTDDQmryC22unSWFv2fJZ1MSkiFzk5ncckHRMfyPUbSdhSA26rcSJqnebJc6cnkSmWOgWUr1ewm4DCmcnBvdBzaQweGwv9Da04OQWF8I58YusFjTt/xajFt/SSBrSAmdcnLtMsOPGTh3HeistRvyzfTXD+qiT0KPwvwUd53dn+QLjw7IvRuvJNlhjDTC/KzQ3Pe7huLggEYJPpvJSw++VhJh9389orPHR1YFWlYdzY15NdQwX9gzObAFUA7w2Lb9os66QdoV1AldHaOSoNL47Qxse8D0z6yMKAtW0AAAADckz7IgAAAAAAl4iI////+AAAAABmQmSKAAAAAGZCZIkAAAADbhyjdAAAAAAAp3pCCgPM32dNQNYyhQutl5S290omaXtVA0QUgyoKd9L303zqKVOkRfXMQNf4p02im3SVDqEFHrvT9Dcv6ryXTbR+45EDouH3kPsTPI36oF9UCOLlPcIN790WYmTciwR/xgq4ftKmoGzXUl1bEduniNVERqzrUXF0Qi4E63HeistRvyzfTXD+qiT0KPwvwUd53dn+QLjw7IvRuvJNlhjDTC/KzQ3Pe7huLggEYJPpvJSw++VhJh9389orPHR1YFWlYdzY15NdQwX9gzOb";
+
 describe("Test parse accumulator update", () => {
   test("Happy path", async () => {
     const { vaa, updates } = parseAccumulatorUpdateData(
       Buffer.from(TEST_ACCUMULATOR_UPDATE_DATA, "base64")
     );
 
-    const priceMessage = parsePriceFeedMessage(updates[0].message);
-    expect(priceMessage.feedId.toString("hex")).toBe(
+    const priceMessages = updates.map((update) => {
+      return parsePriceFeedMessage(update.message);
+    });
+    expect(priceMessages[0].feedId.toString("hex")).toBe(
+      "c96458d393fe9deb7a7d63a0ac41e2898a67a7750dbd166673279e06c868df0a"
+    );
+    expect(priceMessages[0].price.toString()).toBe("4689500");
+    expect(priceMessages[0].confidence.toString()).toBe("6174");
+    expect(priceMessages[0].exponent).toBe(-8);
+    expect(priceMessages[0].publishTime.toString()).toBe("1715627146");
+    expect(priceMessages[0].prevPublishTime.toString()).toBe("1715627145");
+    expect(priceMessages[0].emaPrice.toString()).toBe("4690654");
+    expect(priceMessages[0].emaConf.toString()).toBe("6257");
+
+    expect(priceMessages[1].feedId.toString("hex")).toBe(
+      "ff61491a931112ddf1bd8147cd1b641375f79f5825126d665480874634fd0ace"
+    );
+    expect(priceMessages[1].price.toString()).toBe("294943041422");
+    expect(priceMessages[1].confidence.toString()).toBe("184284043");
+    expect(priceMessages[1].exponent).toBe(-8);
+    expect(priceMessages[1].publishTime.toString()).toBe("1715627146");
+    expect(priceMessages[1].prevPublishTime.toString()).toBe("1715627145");
+    expect(priceMessages[1].emaPrice.toString()).toBe("294580230000");
+    expect(priceMessages[1].emaConf.toString()).toBe("239495968");
+
+    expect(priceMessages[2].feedId.toString("hex")).toBe(
+      "ef0d8b6fda2ceba41da15d4095d1da392a0d2f8ed0c6c7bc0f4cfac8c280b56d"
+    );
+    expect(priceMessages[2].price.toString()).toBe("14802549538");
+    expect(priceMessages[2].confidence.toString()).toBe("9930888");
+    expect(priceMessages[2].exponent).toBe(-8);
+    expect(priceMessages[2].publishTime.toString()).toBe("1715627146");
+    expect(priceMessages[2].prevPublishTime.toString()).toBe("1715627145");
+    expect(priceMessages[2].emaPrice.toString()).toBe("14732272500");
+    expect(priceMessages[2].emaConf.toString()).toBe("10975810");
+  });
+
+  test("Slice accumulator update data", async () => {
+    expect(
+      parseAccumulatorUpdateData(
+        sliceAccumulatorUpdateData(
+          Buffer.from(TEST_ACCUMULATOR_UPDATE_DATA, "base64"),
+          2,
+          1
+        )
+      ).updates.length
+    ).toBe(0);
+
+    expect(
+      parseAccumulatorUpdateData(
+        sliceAccumulatorUpdateData(
+          Buffer.from(TEST_ACCUMULATOR_UPDATE_DATA, "base64"),
+          0,
+          5
+        )
+      ).updates.length
+    ).toBe(3);
+
+    const { vaa, updates } = parseAccumulatorUpdateData(
+      sliceAccumulatorUpdateData(
+        Buffer.from(TEST_ACCUMULATOR_UPDATE_DATA, "base64"),
+        1,
+        3
+      )
+    );
+
+    const priceMessages = updates.map((update) => {
+      return parsePriceFeedMessage(update.message);
+    });
+    expect(priceMessages[0].feedId.toString("hex")).toBe(
+      "ff61491a931112ddf1bd8147cd1b641375f79f5825126d665480874634fd0ace"
+    );
+    expect(priceMessages[0].price.toString()).toBe("294943041422");
+    expect(priceMessages[0].confidence.toString()).toBe("184284043");
+    expect(priceMessages[0].exponent).toBe(-8);
+    expect(priceMessages[0].publishTime.toString()).toBe("1715627146");
+    expect(priceMessages[0].prevPublishTime.toString()).toBe("1715627145");
+    expect(priceMessages[0].emaPrice.toString()).toBe("294580230000");
+    expect(priceMessages[0].emaConf.toString()).toBe("239495968");
+
+    expect(priceMessages[1].feedId.toString("hex")).toBe(
       "ef0d8b6fda2ceba41da15d4095d1da392a0d2f8ed0c6c7bc0f4cfac8c280b56d"
     );
-    expect(priceMessage.price.toString()).toBe("10737782713");
-    expect(priceMessage.confidence.toString()).toBe("6283444");
-    expect(priceMessage.exponent).toBe(-8);
-    expect(priceMessage.publishTime.toString()).toBe("1709054177");
-    expect(priceMessage.prevPublishTime.toString()).toBe("1709054177");
-    expect(priceMessage.emaPrice.toString()).toBe("10782719800");
-    expect(priceMessage.emaConf.toString()).toBe("6818776");
+    expect(priceMessages[1].price.toString()).toBe("14802549538");
+    expect(priceMessages[1].confidence.toString()).toBe("9930888");
+    expect(priceMessages[1].exponent).toBe(-8);
+    expect(priceMessages[1].publishTime.toString()).toBe("1715627146");
+    expect(priceMessages[1].prevPublishTime.toString()).toBe("1715627145");
+    expect(priceMessages[1].emaPrice.toString()).toBe("14732272500");
+    expect(priceMessages[1].emaConf.toString()).toBe("10975810");
   });
 
   test("Wrong magic number", async () => {

price_service/sdk/js/src/index.ts

@@ -11,6 +11,7 @@ export type HexString = string;
 
 export {
   isAccumulatorUpdateData,
+  sliceAccumulatorUpdateData,
   parseAccumulatorUpdateData,
   AccumulatorUpdateData,
   parsePriceFeedMessage,