diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..b7dab5e9c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +node_modules +build \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..396a0ad26 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM node:10.15-alpine + +RUN apk add --no-cache --virtual build-dependencies bash git python make g++ ca-certificates + +COPY yarn.lock package.json ./ +RUN yarn install --frozen-lockfile && yarn cache clean + +COPY . . + +RUN yarn prepack + +ENTRYPOINT ["bash", "-c"] diff --git a/package.json b/package.json index f55fd334a..42b8e1564 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "@truffle/contract": "^4.2.1", "@types/bn.js": "^4.11.6", "@types/chai": "^4.2.11", + "assert-diff": "^3.0.1", "axios": "^0.19.2", "bignumber.js": "^9.0.0", "chai": "^4.2.0", diff --git a/scripts/verify_orderbook_consistency.js b/scripts/verify_orderbook_consistency.js new file mode 100644 index 000000000..3a54b96db --- /dev/null +++ b/scripts/verify_orderbook_consistency.js @@ -0,0 +1,80 @@ +const BatchExchange = artifacts.require("BatchExchange") +const BatchExchangeViewer = artifacts.require("BatchExchangeViewer") + +const assert = require("assert-diff") + +const { getOpenOrdersPaginated, getFinalizedOrdersPaginated, getOrdersPaginated } = require("../src/onchain_reading") + +let mostRecentBatch = 0 + +async function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +async function getLastBlockInBatchBefore(web3, instance, batchId) { + let block = await web3.eth.getBlockNumber() + while (parseInt(await instance.contract.methods.getCurrentBatchId().call(block)) >= batchId) { + block-- + } + return block +} + +async function getOrdersFromViewer(viewer, fn, targetBlock) { + let result = [] + try { + for await (const page of fn(viewer.contract, 300, targetBlock)) { + result = result.concat(page) + } + } catch (error) { + throw new Error(`${fn.name} failed with ${error}`) + } + console.log(`${fn.name} returned ${result.length} orders`) + return result +} + +async function getOrdersFromExchange(exchange, targetBlock, targetBatch) { + try { + const unfiltered = await getOrdersPaginated(exchange.contract, 250, targetBlock) + const result = unfiltered.filter((order) => order.validFrom <= targetBatch && order.validUntil >= targetBatch) + console.log(`getOrdersFromExchange returned ${result.length} orders`) + return result + } catch (error) { + throw new Error(`getOrdersFromExchange failed with ${error}`) + } +} + +module.exports = async () => { + const instance = await BatchExchange.deployed() + const viewer = await BatchExchangeViewer.deployed() + + for (;;) { + try { + const batchId = (await instance.getCurrentBatchId()).toNumber() + if (batchId > mostRecentBatch) { + // Wait some time to avoid reorg inconsistency + await sleep(30000) + const lastBlockInPreviousBatch = await getLastBlockInBatchBefore(web3, instance, batchId) + const firstBlockInNewBatch = lastBlockInPreviousBatch + 1 + console.log(`Start verification for batch ${batchId} with blocks ${lastBlockInPreviousBatch}/${firstBlockInNewBatch}`) + + const [openOrders, finalizedOrders, legacy] = ( + await Promise.all([ + getOrdersFromViewer(viewer, getOpenOrdersPaginated, lastBlockInPreviousBatch), + getOrdersFromViewer(viewer, getFinalizedOrdersPaginated, firstBlockInNewBatch), + getOrdersFromExchange(instance, firstBlockInNewBatch, batchId - 1), + ]) + ).map((r) => JSON.stringify(r)) + + assert.deepEqual(openOrders, finalizedOrders, "open orders != finalized orders") + assert.deepEqual(openOrders, legacy, "open orders != legacy orders") + assert.deepEqual(finalizedOrders, legacy, "finalized orders != legacy orders") + + console.log(`Verification succeeded for batch ${batchId}`) + mostRecentBatch = batchId + } + await sleep(10000) + } catch (error) { + console.error(error) + } + } +} diff --git a/src/onchain_reading.js b/src/onchain_reading.js index 8d8990cdf..a94e6f4e3 100644 --- a/src/onchain_reading.js +++ b/src/onchain_reading.js @@ -5,13 +5,39 @@ const { decodeOrdersBN } = require("./encoding") * @param {BatchExchangeViewer} contract to query from * @param {number} pageSize the number of items to fetch per page */ -const getOpenOrdersPaginated = async function* (contract, pageSize) { +const getOpenOrdersPaginated = async function* (contract, pageSize, blockNumber) { let nextPageUser = "0x0000000000000000000000000000000000000000" let nextPageUserOffset = 0 let hasNextPage = true while (hasNextPage) { - const page = await contract.methods.getOpenOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize).call() + const page = await contract.methods + .getOpenOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize) + .call({}, blockNumber) + const elements = decodeOrdersBN(page.elements) + yield elements + + //Update page info + hasNextPage = page.hasNextPage + nextPageUser = page.nextPageUser + nextPageUserOffset = page.nextPageUserOffset + } +} + +/** + * Returns an iterator yielding an item for each page of order in the orderbook that is currently being solved. + * @param {BatchExchangeViewer} contract to query from + * @param {number} pageSize the number of items to fetch per page + */ +const getFinalizedOrdersPaginated = async function* (contract, pageSize, blockNumber) { + let nextPageUser = "0x0000000000000000000000000000000000000000" + let nextPageUserOffset = 0 + let hasNextPage = true + + while (hasNextPage) { + const page = await contract.methods + .getFinalizedOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize) + .call({}, blockNumber) const elements = decodeOrdersBN(page.elements) yield elements @@ -27,13 +53,15 @@ const getOpenOrdersPaginated = async function* (contract, pageSize) { * @param {BatchExchange} contract to query from * @param {number} pageSize the number of items to fetch per page */ -const getOrdersPaginated = async (contract, pageSize) => { +const getOrdersPaginated = async (contract, pageSize, blockNumber) => { let orders = [] let currentUser = "0x0000000000000000000000000000000000000000" let currentOffSet = 0 let lastPageSize = pageSize while (lastPageSize == pageSize) { - const page = decodeOrdersBN(await contract.methods.getEncodedUsersPaginated(currentUser, currentOffSet, pageSize).call()) + const page = decodeOrdersBN( + await contract.methods.getEncodedUsersPaginated(currentUser, currentOffSet, pageSize).call({}, blockNumber) + ) orders = orders.concat(page) for (const index in page) { if (page[index].user != currentUser) { @@ -49,5 +77,6 @@ const getOrdersPaginated = async (contract, pageSize) => { module.exports = { getOpenOrdersPaginated, + getFinalizedOrdersPaginated, getOrdersPaginated, } diff --git a/yarn.lock b/yarn.lock index 931578c80..e27868fa3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -765,6 +765,14 @@ asn1@~0.2.3: dependencies: safer-buffer "~2.1.0" +assert-diff@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/assert-diff/-/assert-diff-3.0.1.tgz#41720c433a2cef91a7cd4cedd2b21b4ba8719512" + integrity sha512-TxoFgLKQCGHNBDMEayf0YKSEf0CS3Xxmmx1RX6dsiun+YkwqO3NEoy6kpmQkrTw9e3juLbi4TUtrppUrXiYfrw== + dependencies: + assert-plus "1.0.0" + json-diff "0.5.4" + assert-plus@1.0.0, assert-plus@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" @@ -1921,6 +1929,13 @@ cli-color@^1.4.0: memoizee "^0.4.14" timers-ext "^0.1.5" +cli-color@~0.1.6: + version "0.1.7" + resolved "https://registry.yarnpkg.com/cli-color/-/cli-color-0.1.7.tgz#adc3200fa471cc211b0da7f566b71e98b9d67347" + integrity sha1-rcMgD6RxzCEbDaf1ZrcemLnWc0c= + dependencies: + es5-ext "0.8.x" + cli-cursor@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5" @@ -2611,6 +2626,13 @@ diffie-hellman@^5.0.0: miller-rabin "^4.0.0" randombytes "^2.0.0" +difflib@~0.2.1: + version "0.2.4" + resolved "https://registry.yarnpkg.com/difflib/-/difflib-0.2.4.tgz#b5e30361a6db023176d562892db85940a718f47e" + integrity sha1-teMDYabbAjF21WKJLbhZQKcY9H4= + dependencies: + heap ">= 0.2.0" + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -2656,6 +2678,13 @@ drbg.js@^1.0.1: create-hash "^1.1.2" create-hmac "^1.1.4" +dreamopt@~0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/dreamopt/-/dreamopt-0.6.0.tgz#d813ccdac8d39d8ad526775514a13dda664d6b4b" + integrity sha1-2BPM2sjTnYrVJndVFKE92mZNa0s= + dependencies: + wordwrap ">=0.0.2" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -2827,6 +2856,11 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" +es5-ext@0.8.x: + version "0.8.2" + resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.8.2.tgz#aba8d9e1943a895ac96837a62a39b3f55ecd94ab" + integrity sha1-q6jZ4ZQ6iVrJaDemKjmz9V7NlKs= + es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46: version "0.10.51" resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.51.tgz#ed2d7d9d48a12df86e0299287e93a09ff478842f" @@ -4447,6 +4481,11 @@ he@1.2.0: resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== +"heap@>= 0.2.0": + version "0.2.6" + resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac" + integrity sha1-CH4fELBGky/IWU3Z5tN4r8nR5aw= + hmac-drbg@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1" @@ -5096,6 +5135,15 @@ json-buffer@3.0.0: resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg= +json-diff@0.5.4: + version "0.5.4" + resolved "https://registry.yarnpkg.com/json-diff/-/json-diff-0.5.4.tgz#7bc8198c441756632aab66c7d9189d365a7a035a" + integrity sha512-q5Xmx9QXNOzOzIlMoYtLrLiu4Jl/Ce2bn0CNcv54PhyH89CI4GWlGVDye8ei2Ijt9R3U+vsWPsXpLUNob8bs8Q== + dependencies: + cli-color "~0.1.6" + difflib "~0.2.1" + dreamopt "~0.6.0" + json-parse-better-errors@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -9544,7 +9592,7 @@ word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -wordwrap@^1.0.0: +wordwrap@>=0.0.2, wordwrap@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=