Develop (#807)

* Check existing record against update window (#787) * Check to see if existing record was updated during integrity check window * Fetch existing closer to upsert * numeric for u32 (#790) * uncomment logic (#792) * Fix timestamp used for genesis end epoch info (#805) * Fix timestamp used for genesis end epoch info * Spacing fix * Bump version * bugfix (#806) * bugfix * fix * Bump cargo --------- Co-authored-by: Noah Prince <[email protected]> Co-authored-by: Noah Prince <[email protected]>
helium · Jan 31, 2025 · cc0912f · cc0912f
1 parent c27321b
commit cc0912f
Show file tree

Hide file tree

Showing 121 changed files with 1,591 additions and 1,208 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.9.23](https://github.com/helium/helium-program-library/compare/v0.9.22...v0.9.23) (2025-01-31)
+
+**Note:** Version bump only for package helium-program-library
+
+
+
+
+
 ## [0.9.22](https://github.com/helium/helium-program-library/compare/v0.9.21...v0.9.22) (2025-01-30)
 
 **Note:** Version bump only for package helium-program-library

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/lerna.json b/lerna.json
@@ -4,5 +4,5 @@
     "packages/*"
   ],
   "useWorkspaces": true,
-  "version": "0.9.22"
+  "version": "0.9.23"
 }
diff --git a/packages/account-fetch-cache-hooks/CHANGELOG.md b/packages/account-fetch-cache-hooks/CHANGELOG.md
@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.9.23](https://github.com/helium/helium-program-libary/compare/v0.9.22...v0.9.23) (2025-01-31)
+
+**Note:** Version bump only for package @helium/account-fetch-cache-hooks
+
+
+
+
+
 ## [0.9.22](https://github.com/helium/helium-program-libary/compare/v0.9.21...v0.9.22) (2025-01-30)
 
 **Note:** Version bump only for package @helium/account-fetch-cache-hooks

diff --git a/packages/account-fetch-cache-hooks/package.json b/packages/account-fetch-cache-hooks/package.json
@@ -5,7 +5,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "license": "Apache-2.0",
-  "version": "0.9.22",
+  "version": "0.9.23",
   "description": "React hooks and context for account-fetch-cache",
   "repository": {
     "type": "git",
@@ -31,7 +31,7 @@
     "prebuild": "npm run clean && npm run package"
   },
   "dependencies": {
-    "@helium/account-fetch-cache": "^0.9.22",
+    "@helium/account-fetch-cache": "^0.9.23",
     "@solana/web3.js": "^1.91.1",
     "react-async-hook": "^4.0.0"
   },

diff --git a/packages/account-fetch-cache-hooks/yarn.deploy.lock b/packages/account-fetch-cache-hooks/yarn.deploy.lock
@@ -18,7 +18,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@helium/account-fetch-cache-hooks@workspace:."
   dependencies:
-    "@helium/account-fetch-cache": ^0.9.22
+    "@helium/account-fetch-cache": ^0.9.23
     "@solana/web3.js": ^1.91.1
     git-format-staged: ^2.1.3
     react-async-hook: ^4.0.0
@@ -30,7 +30,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@helium/account-fetch-cache@^0.9.22":
+"@helium/account-fetch-cache@^0.9.23":
   version: 0.0.0-use.local
   resolution: "@helium/account-fetch-cache@workspace:packages/account-fetch-cache"
   dependencies:

diff --git a/packages/account-fetch-cache/CHANGELOG.md b/packages/account-fetch-cache/CHANGELOG.md
@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.9.23](https://github.com/helium/helium-program-library/compare/v0.9.22...v0.9.23) (2025-01-31)
+
+**Note:** Version bump only for package @helium/account-fetch-cache
+
+
+
+
+
 ## [0.9.22](https://github.com/helium/helium-program-library/compare/v0.9.21...v0.9.22) (2025-01-30)
 
 **Note:** Version bump only for package @helium/account-fetch-cache

diff --git a/packages/account-fetch-cache/package.json b/packages/account-fetch-cache/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@helium/account-fetch-cache",
-  "version": "0.9.22",
+  "version": "0.9.23",
   "description": "Solana account fetch cache to eliminate reduntant fetching, and batch fetches",
   "publishConfig": {
     "access": "public",

diff --git a/packages/account-postgres-sink-service/CHANGELOG.md b/packages/account-postgres-sink-service/CHANGELOG.md
@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.9.23](https://github.com/helium/helium-program-libary/compare/v0.9.22...v0.9.23) (2025-01-31)
+
+**Note:** Version bump only for package @helium/account-postgres-sink-service
+
+
+
+
+
 ## [0.9.22](https://github.com/helium/helium-program-libary/compare/v0.9.21...v0.9.22) (2025-01-30)
 
 **Note:** Version bump only for package @helium/account-postgres-sink-service

diff --git a/packages/account-postgres-sink-service/package.json b/packages/account-postgres-sink-service/package.json
@@ -6,7 +6,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "license": "Apache-2.0",
-  "version": "0.9.22",
+  "version": "0.9.23",
   "description": "Sync account data to postgres",
   "repository": {
     "type": "git",
@@ -37,8 +37,8 @@
     "@connectrpc/connect-node": "^1.4.0",
     "@coral-xyz/anchor": "^0.28.0",
     "@fastify/cors": "^8.1.1",
-    "@helium/account-fetch-cache": "^0.9.22",
-    "@helium/spl-utils": "^0.9.22",
+    "@helium/account-fetch-cache": "^0.9.23",
+    "@helium/spl-utils": "^0.9.23",
     "@metaplex-foundation/mpl-token-metadata": "^2.10.0",
     "@solana/web3.js": "^1.91.1",
     "@substreams/core": "^0.16.0",

diff --git a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts
@@ -13,8 +13,8 @@ const TypeMap = new Map<string, any>([
   ["u8", DataTypes.INTEGER.UNSIGNED],
   ["i16", DataTypes.INTEGER],
   ["u16", DataTypes.INTEGER.UNSIGNED],
-  ["i32", DataTypes.INTEGER],
-  ["u32", DataTypes.INTEGER.UNSIGNED],
+  ["i32", DataTypes.DECIMAL],
+  ["u32", DataTypes.DECIMAL.UNSIGNED],
   ["i64", DataTypes.DECIMAL],
   ["u64", DataTypes.DECIMAL.UNSIGNED],
   ["i128", DataTypes.DECIMAL],

diff --git a/packages/account-postgres-sink-service/src/utils/integrityCheckProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/integrityCheckProgramAccounts.ts
@@ -63,17 +63,10 @@ export const integrityCheckProgramAccounts = async ({
   }
 
   const performIntegrityCheck = async () => {
+    const startTime = new Date();
     const t = await sequelize.transaction({
       isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED,
     });
-    const txIdsByAccountId: { [key: string]: string[] } = {};
-    const corrections: {
-      type: string;
-      accountId: string;
-      txSignatures: string[];
-      currentValues: null | { [key: string]: any };
-      newValues: { [key: string]: any };
-    }[] = [];
 
     try {
       const program = new anchor.Program(idl, programId, provider);
@@ -85,6 +78,15 @@ export const integrityCheckProgramAccounts = async ({
         provider,
       });
 
+      const txIdsByAccountId: { [key: string]: string[] } = {};
+      const corrections: {
+        type: string;
+        accountId: string;
+        txSignatures: string[];
+        currentValues: null | { [key: string]: any };
+        newValues: { [key: string]: any };
+      }[] = [];
+
       if (!blockTime24HoursAgo) {
         throw new Error("Unable to get blocktime from 24 hours ago");
       }
@@ -158,62 +160,90 @@ export const integrityCheckProgramAccounts = async ({
         return acc;
       }, {} as Record<string, IInitedPlugin[]>);
 
+      const discriminatorsByType = new Map(
+        accounts.map(({ type }) => [
+          type,
+          anchor.BorshAccountsCoder.accountDiscriminator(type),
+        ])
+      );
+
       await Promise.all(
         chunks(accountInfosWithPk, 1000).map(async (chunk) => {
-          for (const c of chunk) {
-            const accName = accounts.find(({ type }) => {
-              return (
-                c.data &&
-                anchor.BorshAccountsCoder.accountDiscriminator(type).equals(
-                  c.data.subarray(0, 8)
-                )
-              );
-            })?.type;
+          const accountsByType: Record<string, typeof accountInfosWithPk> = {};
+          chunk.forEach((accountInfo) => {
+            const accName = accounts.find(
+              ({ type }) =>
+                accountInfo.data &&
+                discriminatorsByType
+                  .get(type)
+                  ?.equals(accountInfo.data.subarray(0, 8))
+            )?.type;
 
-            if (!accName) {
-              continue;
+            if (accName) {
+              accountsByType[accName] = accountsByType[accName] || [];
+              accountsByType[accName].push(accountInfo);
             }
+          });
 
-            const decodedAcc = program.coder.accounts.decode(
-              accName!,
-              c.data as Buffer
-            );
-
-            if (accName) {
+          await Promise.all(
+            Object.entries(accountsByType).map(async ([accName, accounts]) => {
               const model = sequelize.models[accName];
-              const existing = await model.findByPk(c.pubkey, {
-                transaction: t,
-              });
+              await Promise.all(
+                accounts.map(async (c) => {
+                  const decodedAcc = program.coder.accounts.decode(
+                    accName,
+                    c.data as Buffer
+                  );
 
-              let sanitized = {
-                refreshed_at: new Date().toISOString(),
-                address: c.pubkey,
-                ...sanitizeAccount(decodedAcc),
-              };
+                  let sanitized = {
+                    refreshed_at: new Date().toISOString(),
+                    address: c.pubkey,
+                    ...sanitizeAccount(decodedAcc),
+                  };
 
-              for (const plugin of pluginsByAccountType[accName]) {
-                if (plugin?.processAccount) {
-                  sanitized = await plugin.processAccount(sanitized);
-                }
-              }
+                  if (pluginsByAccountType[accName]?.length > 0) {
+                    const pluginResults = await Promise.all(
+                      pluginsByAccountType[accName].map((plugin) =>
+                        plugin?.processAccount
+                          ? plugin.processAccount(sanitized)
+                          : sanitized
+                      )
+                    );
+                    sanitized = pluginResults.reduce(
+                      (acc, curr) => ({ ...acc, ...curr }),
+                      sanitized
+                    );
+                  }
 
-              const shouldUpdate = !deepEqual(
-                _omit(sanitized, OMIT_KEYS),
-                _omit(existing?.dataValues, OMIT_KEYS)
-              );
+                  const existing = await model.findByPk(c.pubkey, {
+                    transaction: t,
+                  });
 
-              if (shouldUpdate) {
-                corrections.push({
-                  type: accName,
-                  accountId: c.pubkey,
-                  txSignatures: txIdsByAccountId[c.pubkey],
-                  currentValues: existing ? existing.dataValues : null,
-                  newValues: sanitized,
-                });
-                await model.upsert({ ...sanitized }, { transaction: t });
-              }
-            }
-          }
+                  const shouldUpdate =
+                    !deepEqual(
+                      _omit(sanitized, OMIT_KEYS),
+                      _omit(existing?.dataValues, OMIT_KEYS)
+                    ) &&
+                    !(
+                      existing?.dataValues.refreshed_at &&
+                      new Date(existing.dataValues.refreshed_at) >= startTime &&
+                      new Date(existing.dataValues.refreshed_at) <= new Date()
+                    );
+
+                  if (shouldUpdate) {
+                    corrections.push({
+                      type: accName,
+                      accountId: c.pubkey,
+                      txSignatures: txIdsByAccountId[c.pubkey],
+                      currentValues: existing ? existing.dataValues : null,
+                      newValues: sanitized,
+                    });
+                    await model.upsert({ ...sanitized }, { transaction: t });
+                  }
+                })
+              );
+            })
+          );
         })
       );
 

diff --git a/packages/account-postgres-sink-service/yarn.deploy.lock b/packages/account-postgres-sink-service/yarn.deploy.lock
@@ -179,7 +179,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@helium/account-fetch-cache@^0.9.22":
+"@helium/account-fetch-cache@^0.9.23":
   version: 0.0.0-use.local
   resolution: "@helium/account-fetch-cache@workspace:packages/account-fetch-cache"
   dependencies:
@@ -199,8 +199,8 @@ __metadata:
     "@connectrpc/connect-node": ^1.4.0
     "@coral-xyz/anchor": ^0.28.0
     "@fastify/cors": ^8.1.1
-    "@helium/account-fetch-cache": ^0.9.22
-    "@helium/spl-utils": ^0.9.22
+    "@helium/account-fetch-cache": ^0.9.23
+    "@helium/spl-utils": ^0.9.23
     "@metaplex-foundation/mpl-token-metadata": ^2.10.0
     "@solana/web3.js": ^1.91.1
     "@substreams/core": ^0.16.0
@@ -256,7 +256,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@helium/anchor-resolvers@^0.9.22":
+"@helium/anchor-resolvers@^0.9.23":
   version: 0.0.0-use.local
   resolution: "@helium/anchor-resolvers@workspace:packages/anchor-resolvers"
   dependencies:
@@ -270,14 +270,14 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@helium/spl-utils@^0.9.22":
+"@helium/spl-utils@^0.9.23":
   version: 0.0.0-use.local
   resolution: "@helium/spl-utils@workspace:packages/spl-utils"
   dependencies:
     "@coral-xyz/anchor": ^0.28.0
-    "@helium/account-fetch-cache": ^0.9.22
+    "@helium/account-fetch-cache": ^0.9.23
     "@helium/address": ^4.10.2
-    "@helium/anchor-resolvers": ^0.9.22
+    "@helium/anchor-resolvers": ^0.9.23
     "@metaplex-foundation/mpl-token-metadata": ^2.10.0
     "@solana/spl-account-compression": ^0.1.7
     "@solana/spl-token": ^0.3.8

diff --git a/packages/active-device-oracle/CHANGELOG.md b/packages/active-device-oracle/CHANGELOG.md
@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.9.23](https://github.com/helium/helium-program-libary/compare/v0.9.22...v0.9.23) (2025-01-31)
+
+**Note:** Version bump only for package @helium/active-device-oracle
+
+
+
+
+
 ## [0.9.22](https://github.com/helium/helium-program-libary/compare/v0.9.21...v0.9.22) (2025-01-30)
 
 **Note:** Version bump only for package @helium/active-device-oracle

diff --git a/packages/active-device-oracle/package.json b/packages/active-device-oracle/package.json
@@ -6,7 +6,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "license": "Apache-2.0",
-  "version": "0.9.22",
+  "version": "0.9.23",
   "description": "Oracle of helium active devices",
   "repository": {
     "type": "git",