Add a script that checks consistency between different fetching methods

.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+build

Dockerfile

@@ -0,0 +1,12 @@
+FROM node:10.15-alpine
+
+RUN apk add --no-cache --virtual build-dependencies bash git python make g++ ca-certificates
+
+COPY yarn.lock package.json ./
+RUN yarn install --frozen-lockfile && yarn cache clean
+
+COPY . .
+
+RUN yarn prepack
+
+ENTRYPOINT ["bash", "-c"]

package.json

@@ -82,6 +82,7 @@
     "@truffle/contract": "^4.2.1",
     "@types/bn.js": "^4.11.6",
     "@types/chai": "^4.2.11",
+    "assert-diff": "^3.0.1",
     "axios": "^0.19.2",
     "bignumber.js": "^9.0.0",
     "chai": "^4.2.0",

scripts/verify_orderbook_consistency.js

@@ -0,0 +1,80 @@
+const BatchExchange = artifacts.require("BatchExchange")
+const BatchExchangeViewer = artifacts.require("BatchExchangeViewer")
+
+const assert = require("assert-diff")
+
+const { getOpenOrdersPaginated, getFinalizedOrdersPaginated, getOrdersPaginated } = require("../src/onchain_reading")
+
+let mostRecentBatch = 0
+
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+async function getLastBlockInBatchBefore(web3, instance, batchId) {
+  let block = await web3.eth.getBlockNumber()
+  while (parseInt(await instance.contract.methods.getCurrentBatchId().call(block)) >= batchId) {
+    block--
+  }
+  return block
+}
+
+async function getOrdersFromViewer(viewer, fn, targetBlock) {
+  let result = []
+  try {
+    for await (const page of fn(viewer.contract, 300, targetBlock)) {
+      result = result.concat(page)
+    }
+  } catch (error) {
+    throw new Error(`${fn.name} failed with ${error}`)
+  }
+  console.log(`${fn.name} returned ${result.length} orders`)
+  return result
+}
+
+async function getOrdersFromExchange(exchange, targetBlock, targetBatch) {
+  try {
+    const unfiltered = await getOrdersPaginated(exchange.contract, 250, targetBlock)
+    const result = unfiltered.filter((order) => order.validFrom <= targetBatch && order.validUntil >= targetBatch)
+    console.log(`getOrdersFromExchange returned ${result.length} orders`)
+    return result
+  } catch (error) {
+    throw new Error(`getOrdersFromExchange failed with ${error}`)
+  }
+}
+
+module.exports = async () => {
+  const instance = await BatchExchange.deployed()
+  const viewer = await BatchExchangeViewer.deployed()
+
+  for (;;) {
+    try {
+      const batchId = (await instance.getCurrentBatchId()).toNumber()
+      if (batchId > mostRecentBatch) {
+        // Wait some time to avoid reorg inconsistency
+        await sleep(30000)
+        const lastBlockInPreviousBatch = await getLastBlockInBatchBefore(web3, instance, batchId)
+        const firstBlockInNewBatch = lastBlockInPreviousBatch + 1
+        console.log(`Start verification for batch ${batchId} with blocks ${lastBlockInPreviousBatch}/${firstBlockInNewBatch}`)
+
+        const [openOrders, finalizedOrders, legacy] = (
+          await Promise.all([
+            getOrdersFromViewer(viewer, getOpenOrdersPaginated, lastBlockInPreviousBatch),
+            getOrdersFromViewer(viewer, getFinalizedOrdersPaginated, firstBlockInNewBatch),
+            getOrdersFromExchange(instance, firstBlockInNewBatch, batchId - 1),
+          ])
+        ).map((r) => JSON.stringify(r))
+
+        assert.deepEqual(openOrders, finalizedOrders, "open orders != finalized orders")
+        assert.deepEqual(openOrders, legacy, "open orders != legacy orders")
+        assert.deepEqual(finalizedOrders, legacy, "finalized orders != legacy orders")
+
+        console.log(`Verification succeeded for batch ${batchId}`)
+        mostRecentBatch = batchId
+      }
+      await sleep(10000)
+    } catch (error) {
+      console.error(error)
+    }
+  }
+}

src/onchain_reading.js

@@ -5,13 +5,39 @@ const { decodeOrdersBN } = require("./encoding")
  * @param {BatchExchangeViewer} contract to query from
  * @param {number} pageSize the number of items to fetch per page
  */
-const getOpenOrdersPaginated = async function* (contract, pageSize) {
+const getOpenOrdersPaginated = async function* (contract, pageSize, blockNumber) {
   let nextPageUser = "0x0000000000000000000000000000000000000000"
   let nextPageUserOffset = 0
   let hasNextPage = true
 
   while (hasNextPage) {
-    const page = await contract.methods.getOpenOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize).call()
+    const page = await contract.methods
+      .getOpenOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize)
+      .call({}, blockNumber)
+    const elements = decodeOrdersBN(page.elements)
+    yield elements
+
+    //Update page info
+    hasNextPage = page.hasNextPage
+    nextPageUser = page.nextPageUser
+    nextPageUserOffset = page.nextPageUserOffset
+  }
+}
+
+/**
+ * Returns an iterator yielding an item for each page of order in the orderbook that is currently being solved.
+ * @param {BatchExchangeViewer} contract to query from
+ * @param {number} pageSize the number of items to fetch per page
+ */
+const getFinalizedOrdersPaginated = async function* (contract, pageSize, blockNumber) {
+  let nextPageUser = "0x0000000000000000000000000000000000000000"
+  let nextPageUserOffset = 0
+  let hasNextPage = true
+
+  while (hasNextPage) {
+    const page = await contract.methods
+      .getFinalizedOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize)
+      .call({}, blockNumber)
     const elements = decodeOrdersBN(page.elements)
     yield elements
 
@@ -27,13 +53,15 @@ const getOpenOrdersPaginated = async function* (contract, pageSize) {
  * @param {BatchExchange} contract to query from
  * @param {number} pageSize the number of items to fetch per page
  */
-const getOrdersPaginated = async (contract, pageSize) => {
+const getOrdersPaginated = async (contract, pageSize, blockNumber) => {
   let orders = []
   let currentUser = "0x0000000000000000000000000000000000000000"
   let currentOffSet = 0
   let lastPageSize = pageSize
   while (lastPageSize == pageSize) {
-    const page = decodeOrdersBN(await contract.methods.getEncodedUsersPaginated(currentUser, currentOffSet, pageSize).call())
+    const page = decodeOrdersBN(
+      await contract.methods.getEncodedUsersPaginated(currentUser, currentOffSet, pageSize).call({}, blockNumber)
+    )
     orders = orders.concat(page)
     for (const index in page) {
       if (page[index].user != currentUser) {
@@ -49,5 +77,6 @@ const getOrdersPaginated = async (contract, pageSize) => {
 
 module.exports = {
   getOpenOrdersPaginated,
+  getFinalizedOrdersPaginated,
   getOrdersPaginated,
 }

yarn.lock

@@ -765,6 +765,14 @@ asn1@~0.2.3:
   dependencies:
     safer-buffer "~2.1.0"
 
+assert-diff@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/assert-diff/-/assert-diff-3.0.1.tgz#41720c433a2cef91a7cd4cedd2b21b4ba8719512"
+  integrity sha512-TxoFgLKQCGHNBDMEayf0YKSEf0CS3Xxmmx1RX6dsiun+YkwqO3NEoy6kpmQkrTw9e3juLbi4TUtrppUrXiYfrw==
+  dependencies:
+    assert-plus "1.0.0"
+    json-diff "0.5.4"
+
 [email protected], assert-plus@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525"
@@ -1921,6 +1929,13 @@ cli-color@^1.4.0:
     memoizee "^0.4.14"
     timers-ext "^0.1.5"
 
+cli-color@~0.1.6:
+  version "0.1.7"
+  resolved "https://registry.yarnpkg.com/cli-color/-/cli-color-0.1.7.tgz#adc3200fa471cc211b0da7f566b71e98b9d67347"
+  integrity sha1-rcMgD6RxzCEbDaf1ZrcemLnWc0c=
+  dependencies:
+    es5-ext "0.8.x"
+
 cli-cursor@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5"
@@ -2611,6 +2626,13 @@ diffie-hellman@^5.0.0:
     miller-rabin "^4.0.0"
     randombytes "^2.0.0"
 
+difflib@~0.2.1:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/difflib/-/difflib-0.2.4.tgz#b5e30361a6db023176d562892db85940a718f47e"
+  integrity sha1-teMDYabbAjF21WKJLbhZQKcY9H4=
+  dependencies:
+    heap ">= 0.2.0"
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
@@ -2656,6 +2678,13 @@ drbg.js@^1.0.1:
     create-hash "^1.1.2"
     create-hmac "^1.1.4"
 
+dreamopt@~0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/dreamopt/-/dreamopt-0.6.0.tgz#d813ccdac8d39d8ad526775514a13dda664d6b4b"
+  integrity sha1-2BPM2sjTnYrVJndVFKE92mZNa0s=
+  dependencies:
+    wordwrap ">=0.0.2"
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -2827,6 +2856,11 @@ es-to-primitive@^1.2.1:
     is-date-object "^1.0.1"
     is-symbol "^1.0.2"
 
+[email protected]:
+  version "0.8.2"
+  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.8.2.tgz#aba8d9e1943a895ac96837a62a39b3f55ecd94ab"
+  integrity sha1-q6jZ4ZQ6iVrJaDemKjmz9V7NlKs=
+
 es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46:
   version "0.10.51"
   resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.51.tgz#ed2d7d9d48a12df86e0299287e93a09ff478842f"
@@ -4447,6 +4481,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
 
+"heap@>= 0.2.0":
+  version "0.2.6"
+  resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac"
+  integrity sha1-CH4fELBGky/IWU3Z5tN4r8nR5aw=
+
 hmac-drbg@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1"
@@ -5096,6 +5135,15 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898"
   integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=
 
+[email protected]:
+  version "0.5.4"
+  resolved "https://registry.yarnpkg.com/json-diff/-/json-diff-0.5.4.tgz#7bc8198c441756632aab66c7d9189d365a7a035a"
+  integrity sha512-q5Xmx9QXNOzOzIlMoYtLrLiu4Jl/Ce2bn0CNcv54PhyH89CI4GWlGVDye8ei2Ijt9R3U+vsWPsXpLUNob8bs8Q==
+  dependencies:
+    cli-color "~0.1.6"
+    difflib "~0.2.1"
+    dreamopt "~0.6.0"
+
 json-parse-better-errors@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
@@ -9544,7 +9592,7 @@ word-wrap@~1.2.3:
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
-wordwrap@^1.0.0:
+wordwrap@>=0.0.2, wordwrap@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
   integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=