feat: load interactions in pages

warp-contracts · Feb 28, 2024 · 82d01ca · 82d01ca
1 parent e31aa05
commit 82d01ca
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 22 deletions.
diff --git a/src/loadInteractions.js b/src/loadInteractions.js
@@ -1,9 +1,10 @@
 const { config } = require('./config');
-module.exports = async (startTimestamp, endTimestamp, whiteListedSources, blacklistedContracts, limit) => {
+module.exports = async (startTimestamp, endTimestamp, whiteListedSources, blacklistedContracts, limit, offset) => {
   const response = await postData({
     start: startTimestamp,
     end: endTimestamp,
     limit,
+    offset,
     src_ids: whiteListedSources,
     blacklisted_contracts: blacklistedContracts
   });

diff --git a/src/workers/pollGateway.js b/src/workers/pollGateway.js
@@ -7,16 +7,12 @@ const { isTxIdValid } = require('../common');
 const { partition } = require('./common');
 const { config } = require('../config');
 const { init } = require('./pollWorkerRunner');
+const {of} = require("../tools/weavedb");
 
 const logger = LoggerFactory.INST.create('syncer');
 LoggerFactory.INST.logLevel('info', 'syncer');
 
-function filterInvalidEntries(entries, responseSizeLimit) {
-  if (entries.length >= responseSizeLimit) {
-    logger.warn(`Max entries in response (${responseSizeLimit}), either reduce window size or increase interactions limit in response via the .env.POLL_RESPONSE_LENGTH_LIMIT`);
-    process.exit(0);
-  }
-
+function filterInvalidEntries(entries) {
   const validEntries = entries.filter((entry) => {
     if (!entry.contractTxId || !isTxIdValid(entry.contractTxId)) {
       logger.warn(`No valid 'contractTxId' in entry: ${JSON.stringify(entry)}`);
@@ -89,22 +85,30 @@ module.exports = async function (
         toDate: new Date(endTimestamp)
       });
 
-      let result;
+      let result = {
+        interactions: []
+      };
+      let partialResult;
+      let offset = 0;
       try {
-        result = await loadInteractions(
-          startTimestamp,
-          endTimestamp,
-          whitelistedSources,
-          blacklistedContracts,
-          config.pollResponseLengthLimit
-        );
-        if (!result) {
-          throw new Error('Result is null or undefined');
-        }
-        if (!result.interactions) {
-          throw new Error("Result does not contain 'interactions' field");
-        }
-        result.interactions = filterInvalidEntries(result.interactions, config.pollResponseLengthLimit);
+        do {
+          partialResult = await loadInteractions(
+              startTimestamp,
+              endTimestamp,
+              whitelistedSources,
+              blacklistedContracts,
+              config.pollResponseLengthLimit,
+              offset
+          );
+          if (!partialResult) {
+            throw new Error('Result is null or undefined');
+          }
+          if (!partialResult.interactions) {
+            throw new Error("Result does not contain 'interactions' field");
+          }
+          result.interactions.concat(filterInvalidEntries(partialResult.interactions));
+          offset += config.pollResponseLengthLimit;
+        } while (partialResult.interactions.length === config.pollResponseLengthLimit)
       } catch (e) {
         logger.error(
           'Error while loading interactions',