From bb85e0688a1411ab3f9799632512c6dfdae0dd09 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 16 Jul 2019 18:39:34 +0200 Subject: [PATCH 01/15] Add initial stateless protoype tester --- tests/StatelessRunner.js | 170 ++++++++++++++++++++++++++ tests/stateless.js | 128 +++++++++++++++++++ tests/tester.js | 258 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 556 insertions(+) create mode 100644 tests/StatelessRunner.js create mode 100644 tests/stateless.js create mode 100755 tests/tester.js diff --git a/tests/StatelessRunner.js b/tests/StatelessRunner.js new file mode 100644 index 0000000000..a6e74b4d0a --- /dev/null +++ b/tests/StatelessRunner.js @@ -0,0 +1,170 @@ +const testUtil = require('./util') +const { promisify } = require('util') +const ethUtil = require('ethereumjs-util') +const Account = require('ethereumjs-account').default +const BN = ethUtil.BN +const { getRequiredForkConfigAlias } = require('./util') +const { HookedStateManager, stateFromProofs } = require('./stateless') + +const VM = require('../dist/index.js').default +const PStateManager = require('../dist/state/promisified').default + +async function runTestCase (options, testData, t) { + let expectedPostStateRoot = testData.postStateRoot + if (expectedPostStateRoot.substr(0, 2) === '0x') { + expectedPostStateRoot = expectedPostStateRoot.substr(2) + } + + // Prepare tx and block + let tx = testUtil.makeTx(testData.transaction) + let block = testUtil.makeBlockFromEnv(testData.env) + tx._homestead = true + tx.enableHomestead = true + block.isHomestead = function () { + return true + } + if (!tx.validate()) { + return + } + + // Hooked state manager proves necessary trie nodes + // for a stateless execution + let stateManager = new HookedStateManager() + // Set up VM + let vm = new VM({ + stateManager: stateManager, + hardfork: options.forkConfig.toLowerCase() + }) + if (options.jsontrace) { + hookVM(vm, t) + } + await promisify(testUtil.setupPreConditions)(stateManager._trie, testData) + + const preStateRoot = stateManager._trie.root + stateManager.preStateRoot = preStateRoot + + try { + await vm.runTx({ tx: tx, block: block }) + } catch (err) { + await deleteCoinbase(new PStateManager(stateManager), block.header.coinbase) + } + t.equal(stateManager._trie.root.toString('hex'), expectedPostStateRoot, 'the state roots should match') + + const proofData = { + accountProofs: stateManager.proofs, + codes: stateManager.codes, + storageProofs: stateManager.storageProofs + } + // Now use proofs to construct stateManager + // and run tx again to make sure it works + stateManager = await stateFromProofs(preStateRoot, proofData) + vm = new VM({ + stateManager: stateManager, + hardfork: options.forkConfig.toLowerCase() + }) + if (options.jsontrace) { + hookVM(vm, t) + } + try { + await vm.runTx({ tx: tx, block: block }) + } catch (err) { + await deleteCoinbase(new PStateManager(stateManager), block.header.coinbase) + } + t.equal(stateManager._trie.root.toString('hex'), expectedPostStateRoot, 'the state roots should match') +} + +/* + * If tx is invalid and coinbase is empty, the test harness + * expects the coinbase account to be deleted from state. + * Without this ecmul_0-3_5616_28000_96 would fail. + */ +async function deleteCoinbase (pstate, coinbaseAddr) { + const account = await pstate.getAccount(coinbaseAddr) + if (new BN(account.balance).isZero()) { + await pstate.putAccount(coinbaseAddr, new Account()) + await pstate.cleanupTouchedAccounts() + await promisify(pstate._wrapped._cache.flush.bind(pstate._wrapped._cache))() + } +} + +function hookVM (vm, t) { + vm.on('step', function (e) { + let hexStack = [] + hexStack = e.stack.map(item => { + return '0x' + new BN(item).toString(16, 0) + }) + + var opTrace = { + 'pc': e.pc, + 'op': e.opcode.opcode, + 'gas': '0x' + e.gasLeft.toString('hex'), + 'gasCost': '0x' + e.opcode.fee.toString(16), + 'stack': hexStack, + 'depth': e.depth, + 'opName': e.opcode.name + } + + t.comment(JSON.stringify(opTrace)) + }) + vm.on('afterTx', function (results) { + let stateRoot = { + 'stateRoot': vm.stateManager._trie.root.toString('hex') + } + t.comment(JSON.stringify(stateRoot)) + }) +} + +function parseTestCases (forkConfig, testData, data, gasLimit, value) { + let testCases = [] + if (testData['post'][forkConfig]) { + testCases = testData['post'][forkConfig].map(testCase => { + let testIndexes = testCase['indexes'] + let tx = { ...testData.transaction } + if (data !== undefined && testIndexes['data'] !== data) { + return null + } + + if (value !== undefined && testIndexes['value'] !== value) { + return null + } + + if (gasLimit !== undefined && testIndexes['gas'] !== gasLimit) { + return null + } + + tx.data = testData.transaction.data[testIndexes['data']] + tx.gasLimit = testData.transaction.gasLimit[testIndexes['gas']] + tx.value = testData.transaction.value[testIndexes['value']] + return { + 'transaction': tx, + 'postStateRoot': testCase['hash'], + 'env': testData['env'], + 'pre': testData['pre'] + } + }) + } + + testCases = testCases.filter(testCase => { + return testCase != null + }) + + return testCases +} + +module.exports = async function runStateTest (options, testData, t) { + const forkConfig = getRequiredForkConfigAlias(options.forkConfig) + try { + const testCases = parseTestCases(forkConfig, testData, options.data, options.gasLimit, options.value) + if (testCases.length > 0) { + for (const testCase of testCases) { + await runTestCase(options, testCase, t) + } + } else { + t.comment(`No ${forkConfig} post state defined, skip test`) + return + } + } catch (e) { + t.fail('error running test case for fork: ' + forkConfig) + console.log('error:', e) + } +} diff --git a/tests/stateless.js b/tests/stateless.js new file mode 100644 index 0000000000..51f2377947 --- /dev/null +++ b/tests/stateless.js @@ -0,0 +1,128 @@ +const { promisify } = require('util') +const assert = require('assert') +const ethUtil = require('ethereumjs-util') +const SecureTrie = require('merkle-patricia-tree/secure') +const Account = require('ethereumjs-account').default +const StateManager = require('../dist/state/stateManager').default +const runTx = require('../dist/runTx').default + +const verifyProof = promisify(SecureTrie.verifyProof) + +class HookedStateManager extends StateManager { + constructor (opts) { + super(opts) + this.proofs = {} + this.codes = {} + this.storageProofs = {} + this.seenKeys = {} + } + + getAccount (addr, cb) { + super.getAccount(addr, (err, res) => { + if (!err && !this.seenKeys[addr.toString('hex')]) { + this.seenKeys[addr.toString('hex')] = true + + const trie = this._trie.copy() + trie.root = this.preStateRoot + trie._checkpoints = [] + + SecureTrie.prove(trie, addr, (err, proof) => { + if (err) { + return cb(err, null) + } + this.proofs[addr.toString('hex')] = proof + cb(null, res) + }) + } else { + cb(err, res) + } + }) + } + + getContractCode (addr, cb) { + super.getContractCode(addr, (err, code) => { + if (!err && !this.codes[addr.toString('hex')]) { + this.codes[addr.toString('hex')] = code + } + cb(err, code) + }) + } + + getContractStorage (addr, key, cb) { + super.getContractStorage(addr, key, (err, value) => { + const addrS = addr.toString('hex') + const keyS = key.toString('hex') + if (err || value.length === 0) { + return cb(err, value) + } + + if (this.storageProofs[addrS] && this.storageProofs[addrS][keyS]) { + return cb(err, value) + } + + this._getStorageTrie(addr, (err, trie) => { + if (err) return cb(err, null) + SecureTrie.prove(trie, key, (err, proof) => { + if (err) return cb(err, null) + if (!this.storageProofs[addrS]) { + this.storageProofs[addrS] = {} + } + this.storageProofs[addrS][keyS] = proof + }) + }) + + cb(err, value) + }) + } + + _isAccountEmpty (account) { + return account.nonce.toString('hex') === '' && + account.balance.toString('hex') === '' && + account.codeHash.toString('hex') === ethUtil.KECCAK256_NULL_S + } +} + +async function proveTx (opts) { + return runTx(opts) +} + +async function stateFromProofs (preStateRoot, data) { + const { accountProofs, codes, storageProofs } = data + const stateManager = new StateManager() + stateManager._trie.root = preStateRoot + for (const key in accountProofs) { + let keyBuf = Buffer.from(key, 'hex') + await trieFromProof(stateManager._trie, accountProofs[key]) + const accountRaw = await promisify(stateManager._trie.get.bind(stateManager._trie))(keyBuf) + const account = new Account(accountRaw) + if (!account.codeHash.equals(ethUtil.KECCAK256_NULL)) { + const code = codes[key] + assert(code) + const codeHash = await promisify(account.setCode.bind(account))(stateManager._trie, code) + assert(codeHash.equals(account.codeHash)) + } + if (!account.stateRoot.equals(ethUtil.KECCAK256_RLP)) { + for (const k in storageProofs[key]) { + const v = await verifyProof(account.stateRoot, Buffer.from(k, 'hex'), storageProofs[key][k]) + assert(v) + await trieFromProof(stateManager._trie, storageProofs[key][k]) + } + } + } + + return stateManager +} + +async function trieFromProof (trie, proofNodes) { + let opStack = proofNodes.map((nodeValue) => { + return { type: 'put', key: ethUtil.keccak256(nodeValue), value: ethUtil.toBuffer(nodeValue) } + }) + + return promisify(trie.db.batch.bind(trie.db))(opStack) +} + +module.exports = { + HookedStateManager, + proveTx, + stateFromProofs +} diff --git a/tests/tester.js b/tests/tester.js new file mode 100755 index 0000000000..4088f50175 --- /dev/null +++ b/tests/tester.js @@ -0,0 +1,258 @@ +#!/usr/bin/env node + +const argv = require('minimist')(process.argv.slice(2)) +const tape = require('tape') +const testing = require('ethereumjs-testing') +const FORK_CONFIG = argv.fork || 'Petersburg' +const { + getRequiredForkConfigAlias +} = require('./util') +// tests which should be fixed +const skipBroken = [ + 'dynamicAccountOverwriteEmpty' // temporary till fixed (2019-01-30), skipped along constantinopleFix work time constraints +] +// tests skipped due to system specifics / design considerations +const skipPermanent = [ + 'SuicidesMixingCoinbase', // sucides to the coinbase, since we run a blockLevel we create coinbase account. + 'static_SuicidesMixingCoinbase', // sucides to the coinbase, since we run a blockLevel we create coinbase account. + 'ForkUncle', // Only BlockchainTest, correct behaviour unspecified (?) + 'UncleFromSideChain' // Only BlockchainTest, same as ForkUncle, the TD is the same for two diffent branches so its not clear which one should be the finally chain +] +// tests running slow (run from time to time) +const skipSlow = [ + 'Call50000', // slow + 'Call50000_ecrec', // slow + 'Call50000_identity', // slow + 'Call50000_identity2', // slow + 'Call50000_sha256', // slow + 'Call50000_rip160', // slow + 'Call50000bytesContract50_1', // slow + 'Call50000bytesContract50_2', + 'Call1MB1024Calldepth', // slow + 'static_Call1MB1024Calldepth', // slow + 'static_Call50000', // slow + 'static_Call50000_ecrec', + 'static_Call50000_identity', + 'static_Call50000_identity2', + 'static_Call50000_rip160', + 'static_Return50000_2', + 'Callcode50000', // slow + 'Return50000', // slow + 'Return50000_2', // slow + 'static_Call50000', // slow + 'static_Call50000_ecrec', // slow + 'static_Call50000_identity', // slow + 'static_Call50000_identity2', // slow + 'static_Call50000_sha256', // slow + 'static_Call50000_rip160', // slow + 'static_Call50000bytesContract50_1', // slow + 'static_Call50000bytesContract50_2', + 'static_Call1MB1024Calldepth', // slow + 'static_Callcode50000', // slow + 'static_Return50000', // slow + 'static_Return50000_2', // slow + 'QuadraticComplexitySolidity_CallDataCopy' +] + +/* +NOTE: VM tests have been disabled since they are generated using Frontier gas costs, and ethereumjs-vm doesn't support historical fork rules + +TODO: some VM tests do not appear to be executing (don't print an "ok" statement): +... +# file: vmLogTest test: log0_emptyMem +ok 38984 valid gas usage +# file: vmLogTest test: log0_logMemStartTooHigh +# file: vmLogTest test: log0_logMemsizeTooHigh +# file: vmLogTest test: log0_logMemsizeZero +ok 38985 valid gas usage +# file: vmLogTest test: log0_nonEmptyMem +*/ + +const skipVM = [ + // slow performance tests + 'loop-mul', + 'loop-add-10M', + 'loop-divadd-10M', + 'loop-divadd-unr100-10M', + 'loop-exp-16b-100k', + 'loop-exp-1b-1M', + 'loop-exp-2b-100k', + 'loop-exp-32b-100k', + 'loop-exp-4b-100k', + 'loop-exp-8b-100k', + 'loop-exp-nop-1M', + 'loop-mulmod-2M', + // some VM tests fail because the js runner executes CALLs + // see https://github.com/ethereum/tests/wiki/VM-Tests > Since these tests are meant only as a basic test of VM operation, the CALL and CREATE instructions are not actually executed. + 'ABAcalls0', + 'ABAcallsSuicide0', + 'ABAcallsSuicide1', + 'sha3_bigSize', + 'CallRecursiveBomb0', + 'CallToNameRegistrator0', + 'CallToPrecompiledContract', + 'CallToReturn1', + 'PostToNameRegistrator0', + 'PostToReturn1', + 'callcodeToNameRegistrator0', + 'callcodeToReturn1', + 'callstatelessToNameRegistrator0', + 'callstatelessToReturn1', + 'createNameRegistrator', + 'randomTest643' // TODO fix this +] + +if (argv.r) { + randomized(argv.r, argv.v) +} else if (argv.s) { + runTests('GeneralStateTests', argv) +} else if (argv.v) { + runTests('VMTests', argv) +} else if (argv.b) { + runTests('BlockchainTests', argv) +} else if (argv.stateless) { + runTests('Stateless', argv) +} + +// randomized tests +// returns 1 if the tests fails +// returns 0 if the tests succeds +function randomized (stateTest) { + const stateRunner = require('./stateRunner.js') + let errored = false + + tape.createStream({ + objectMode: true + }).on('data', function (row) { + if (row.ok === false && !errored) { + errored = true + process.stdout.write('1') + process.exit() + } + }).on('end', function () { + process.stdout.write('0') + }) + + try { + stateTest = JSON.parse(stateTest) + } catch (e) { + console.error('invalid json') + process.exit() + } + + var keys = Object.keys(stateTest) + stateTest = stateTest[keys[0]] + + tape('', t => { + stateRunner({}, stateTest, t, t.end) + }) +} + +function getSkipTests (choices, defaultChoice) { + let skipTests = [] + if (!choices) { + choices = defaultChoice + } + choices = choices.toLowerCase() + if (choices !== 'none') { + let choicesList = choices.split(',') + let all = choicesList.includes('all') + if (all || choicesList.includes('broken')) { + skipTests = skipTests.concat(skipBroken) + } + if (all || choicesList.includes('permanent')) { + skipTests = skipTests.concat(skipPermanent) + } + if (all || choicesList.includes('slow')) { + skipTests = skipTests.concat(skipSlow) + } + } + return skipTests +} + +function runTests (name, runnerArgs, cb) { + let testGetterArgs = {} + + testGetterArgs.skipTests = getSkipTests(argv.skip, argv.runSkipped ? 'NONE' : 'ALL') + testGetterArgs.runSkipped = getSkipTests(argv.runSkipped, 'NONE') + testGetterArgs.skipVM = skipVM + testGetterArgs.forkConfig = getRequiredForkConfigAlias(FORK_CONFIG) + testGetterArgs.file = argv.file + testGetterArgs.test = argv.test + testGetterArgs.dir = argv.dir + testGetterArgs.excludeDir = argv.excludeDir + testGetterArgs.testsPath = argv.testsPath + + testGetterArgs.customStateTest = argv.customStateTest + + runnerArgs.forkConfig = FORK_CONFIG + runnerArgs.jsontrace = argv.jsontrace + runnerArgs.debug = argv.debug // for BlockchainTests + + // for GeneralStateTests + runnerArgs.data = argv.data + runnerArgs.gasLimit = argv.gas + runnerArgs.value = argv.value + + // runnerArgs.vmtrace = true; // for VMTests + + if (name === 'Stateless') { + tape(name, t => { + const stateTestRunner = require('./StatelessRunner.js') + testing.getTestsFromArgs('GeneralStateTests', async (fileName, testName, test) => { + let runSkipped = testGetterArgs.runSkipped + let inRunSkipped = runSkipped.includes(fileName) + if (runSkipped.length === 0 || inRunSkipped) { + t.comment(`file: ${fileName} test: ${testName}`) + return stateTestRunner(runnerArgs, test, t) + } + }, testGetterArgs).then(() => { + t.end() + }).catch((err) => { + console.log(err) + t.end() + }) + }) + } else if (argv.customStateTest) { + const stateTestRunner = require('./GeneralStateTestsRunner.js') + let fileName = argv.customStateTest + tape(name, t => { + testing.getTestFromSource(fileName, (err, test) => { + if (err) { + return t.fail(err) + } + + t.comment(`file: ${fileName} test: ${test.testName}`) + stateTestRunner(runnerArgs, test, t, () => { + t.end() + }) + }) + }) + } else { + tape(name, t => { + const runner = require(`./${name}Runner.js`) + testing.getTestsFromArgs(name, (fileName, testName, test) => { + return new Promise((resolve, reject) => { + if (name === 'VMTests') { + // suppress some output of VMTests + // t.comment(`file: ${fileName} test: ${testName}`) + test.fileName = fileName + test.testName = testName + runner(runnerArgs, test, t, resolve) + } else { + let runSkipped = testGetterArgs.runSkipped + let inRunSkipped = runSkipped.includes(fileName) + if (runSkipped.length === 0 || inRunSkipped) { + t.comment(`file: ${fileName} test: ${testName}`) + runner(runnerArgs, test, t, resolve) + } else { + resolve() + } + } + }).catch(err => console.log(err)) + }, testGetterArgs).then(() => { + t.end() + }) + }) + } +} From 86dfb24b2d040456c9044eb14d05344aeb3292dc Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 22 Jul 2019 11:52:19 +0200 Subject: [PATCH 02/15] Migrate stateless to ts, mv to lib --- lib/stateless.ts | 136 +++++++++++++++++++++++++++++++++ packages/vm/tsconfig.json | 3 + packages/vm/tsconfig.prod.json | 3 +- tests/StatelessRunner.js | 2 +- tests/stateless.js | 128 ------------------------------- 5 files changed, 142 insertions(+), 130 deletions(-) create mode 100644 lib/stateless.ts delete mode 100644 tests/stateless.js diff --git a/lib/stateless.ts b/lib/stateless.ts new file mode 100644 index 0000000000..cd9dd71f17 --- /dev/null +++ b/lib/stateless.ts @@ -0,0 +1,136 @@ +import { promisify } from 'util' +import * as assert from 'assert' +import * as ethUtil from 'ethereumjs-util' +import Account from 'ethereumjs-account' +import VM from './index' +import { default as StateManager, StateManagerOpts } from './state/stateManager' +import runTx from './runTx' + +const SecureTrie = require('merkle-patricia-tree/secure') +const verifyProof = promisify(SecureTrie.verifyProof) + +export class HookedStateManager extends StateManager { + proofs: Map + codes: Map + storageProofs: Map> + seenKeys: Set + preStateRoot: Buffer + + constructor (opts: StateManagerOpts) { + super(opts) + this.proofs = new Map() + this.codes = new Map() + this.storageProofs = new Map() + this.seenKeys = new Set() + this.preStateRoot = ethUtil.KECCAK256_RLP + } + + getAccount (addr: Buffer, cb: Function) { + super.getAccount(addr, (err: Error, res: Account) => { + if (!err && !this.seenKeys.has(addr.toString('hex'))) { + this.seenKeys.add(addr.toString('hex')) + + const trie = this._trie.copy() + trie.root = this.preStateRoot + trie._checkpoints = [] + + SecureTrie.prove(trie, addr, (err: Error, proof: Buffer[]) => { + if (err) { + return cb(err, null) + } + this.proofs.set(addr.toString('hex'), proof) + cb(null, res) + }) + } else { + cb(err, res) + } + }) + } + + getContractCode (addr: Buffer, cb: Function) { + super.getContractCode(addr, (err: Error, code: Buffer) => { + if (!err && !this.codes.has(addr.toString('hex'))) { + this.codes.set(addr.toString('hex'), code) + } + cb(err, code) + }) + } + + getContractStorage (addr: Buffer, key: Buffer, cb: Function) { + super.getContractStorage(addr, key, (err: Error, value: any) => { + const addrS = addr.toString('hex') + const keyS = key.toString('hex') + if (err || value.length === 0) { + return cb(err, value) + } + + if (this.storageProofs.has(addrS) && this.storageProofs.get(addrS)!.has(keyS)) { + return cb(err, value) + } + + this._getStorageTrie(addr, (err: Error, trie: any) => { + if (err) return cb(err, null) + SecureTrie.prove(trie, key, (err: Error, proof: Buffer[]) => { + if (err) return cb(err, null) + let storageMap = this.storageProofs.get(addrS) + if (storageMap === undefined) { + storageMap = new Map() + this.storageProofs.set(addrS, storageMap) + } + storageMap.set(keyS, proof) + }) + }) + + cb(err, value) + }) + } + + _isAccountEmpty (account: Account) { + return account.nonce.toString('hex') === '' && + account.balance.toString('hex') === '' && + account.codeHash.toString('hex') === ethUtil.KECCAK256_NULL_S + } +} + +export async function proveTx (vm: VM, opts: any) { + return runTx.bind(vm)(opts) +} + +export async function stateFromProofs (preStateRoot: Buffer, data: { accountProofs: Map, codes: Map, storageProofs: Map> }) { + const { accountProofs, codes, storageProofs } = data + const stateManager = new StateManager() + stateManager._trie.root = preStateRoot + for (const [key, value] of accountProofs.entries()) { + let keyBuf = Buffer.from(key, 'hex') + await trieFromProof(stateManager._trie, value) + const accountRaw = await promisify(stateManager._trie.get.bind(stateManager._trie))(keyBuf) + const account = new Account(accountRaw) + if (!account.codeHash.equals(ethUtil.KECCAK256_NULL)) { + const code = codes.get(key) + assert(code) + // There's some issue with how typescript handles promisify and cb types + // @ts-ignore + const codeHash = await promisify(account.setCode.bind(account))(stateManager._trie, code) + assert(codeHash.equals(account.codeHash)) + } + if (!account.stateRoot.equals(ethUtil.KECCAK256_RLP)) { + const storageMap = storageProofs.get(key) + assert(storageMap) + for (const [k, v] of storageMap!.entries()) { + const sp = await verifyProof(account.stateRoot, Buffer.from(k, 'hex'), v) + assert(sp) + await trieFromProof(stateManager._trie, v) + } + } + } + + return stateManager +} + +async function trieFromProof (trie: any, proofNodes: Buffer[]) { + let opStack = proofNodes.map((nodeValue) => { + return { type: 'put', key: ethUtil.keccak256(nodeValue), value: ethUtil.toBuffer(nodeValue) } + }) + + return promisify(trie.db.batch.bind(trie.db))(opStack) +} diff --git a/packages/vm/tsconfig.json b/packages/vm/tsconfig.json index 5a6b8bfc3c..bf94cf052f 100644 --- a/packages/vm/tsconfig.json +++ b/packages/vm/tsconfig.json @@ -1,4 +1,7 @@ { "extends": "@ethereumjs/config-tsc", + "compilerOptions": { + "downlevelIteration": true + }, "include": ["lib/**/*.ts"] } diff --git a/packages/vm/tsconfig.prod.json b/packages/vm/tsconfig.prod.json index fcb7eccf94..057ebb8a31 100644 --- a/packages/vm/tsconfig.prod.json +++ b/packages/vm/tsconfig.prod.json @@ -1,7 +1,8 @@ { "extends": "@ethereumjs/config-tsc", "compilerOptions": { - "outDir": "./dist" + "outDir": "./dist", + "downlevelIteration": true }, "include": ["lib/**/*.ts"] } diff --git a/tests/StatelessRunner.js b/tests/StatelessRunner.js index a6e74b4d0a..36d0a34638 100644 --- a/tests/StatelessRunner.js +++ b/tests/StatelessRunner.js @@ -4,7 +4,7 @@ const ethUtil = require('ethereumjs-util') const Account = require('ethereumjs-account').default const BN = ethUtil.BN const { getRequiredForkConfigAlias } = require('./util') -const { HookedStateManager, stateFromProofs } = require('./stateless') +const { HookedStateManager, stateFromProofs } = require('../dist/stateless') const VM = require('../dist/index.js').default const PStateManager = require('../dist/state/promisified').default diff --git a/tests/stateless.js b/tests/stateless.js deleted file mode 100644 index 51f2377947..0000000000 --- a/tests/stateless.js +++ /dev/null @@ -1,128 +0,0 @@ -const { promisify } = require('util') -const assert = require('assert') -const ethUtil = require('ethereumjs-util') -const SecureTrie = require('merkle-patricia-tree/secure') -const Account = require('ethereumjs-account').default -const StateManager = require('../dist/state/stateManager').default -const runTx = require('../dist/runTx').default - -const verifyProof = promisify(SecureTrie.verifyProof) - -class HookedStateManager extends StateManager { - constructor (opts) { - super(opts) - this.proofs = {} - this.codes = {} - this.storageProofs = {} - this.seenKeys = {} - } - - getAccount (addr, cb) { - super.getAccount(addr, (err, res) => { - if (!err && !this.seenKeys[addr.toString('hex')]) { - this.seenKeys[addr.toString('hex')] = true - - const trie = this._trie.copy() - trie.root = this.preStateRoot - trie._checkpoints = [] - - SecureTrie.prove(trie, addr, (err, proof) => { - if (err) { - return cb(err, null) - } - this.proofs[addr.toString('hex')] = proof - cb(null, res) - }) - } else { - cb(err, res) - } - }) - } - - getContractCode (addr, cb) { - super.getContractCode(addr, (err, code) => { - if (!err && !this.codes[addr.toString('hex')]) { - this.codes[addr.toString('hex')] = code - } - cb(err, code) - }) - } - - getContractStorage (addr, key, cb) { - super.getContractStorage(addr, key, (err, value) => { - const addrS = addr.toString('hex') - const keyS = key.toString('hex') - if (err || value.length === 0) { - return cb(err, value) - } - - if (this.storageProofs[addrS] && this.storageProofs[addrS][keyS]) { - return cb(err, value) - } - - this._getStorageTrie(addr, (err, trie) => { - if (err) return cb(err, null) - SecureTrie.prove(trie, key, (err, proof) => { - if (err) return cb(err, null) - if (!this.storageProofs[addrS]) { - this.storageProofs[addrS] = {} - } - this.storageProofs[addrS][keyS] = proof - }) - }) - - cb(err, value) - }) - } - - _isAccountEmpty (account) { - return account.nonce.toString('hex') === '' && - account.balance.toString('hex') === '' && - account.codeHash.toString('hex') === ethUtil.KECCAK256_NULL_S - } -} - -async function proveTx (opts) { - return runTx(opts) -} - -async function stateFromProofs (preStateRoot, data) { - const { accountProofs, codes, storageProofs } = data - const stateManager = new StateManager() - stateManager._trie.root = preStateRoot - for (const key in accountProofs) { - let keyBuf = Buffer.from(key, 'hex') - await trieFromProof(stateManager._trie, accountProofs[key]) - const accountRaw = await promisify(stateManager._trie.get.bind(stateManager._trie))(keyBuf) - const account = new Account(accountRaw) - if (!account.codeHash.equals(ethUtil.KECCAK256_NULL)) { - const code = codes[key] - assert(code) - const codeHash = await promisify(account.setCode.bind(account))(stateManager._trie, code) - assert(codeHash.equals(account.codeHash)) - } - if (!account.stateRoot.equals(ethUtil.KECCAK256_RLP)) { - for (const k in storageProofs[key]) { - const v = await verifyProof(account.stateRoot, Buffer.from(k, 'hex'), storageProofs[key][k]) - assert(v) - await trieFromProof(stateManager._trie, storageProofs[key][k]) - } - } - } - - return stateManager -} - -async function trieFromProof (trie, proofNodes) { - let opStack = proofNodes.map((nodeValue) => { - return { type: 'put', key: ethUtil.keccak256(nodeValue), value: ethUtil.toBuffer(nodeValue) } - }) - - return promisify(trie.db.batch.bind(trie.db))(opStack) -} - -module.exports = { - HookedStateManager, - proveTx, - stateFromProofs -} From a407fb8d61f27f40d1add69dfe65de2ee43be112 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 22 Jul 2019 12:00:00 +0200 Subject: [PATCH 03/15] stateless: don't throw on non-existent code in stateFromProof --- lib/stateless.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/stateless.ts b/lib/stateless.ts index cd9dd71f17..b0e919f607 100644 --- a/lib/stateless.ts +++ b/lib/stateless.ts @@ -48,6 +48,7 @@ export class HookedStateManager extends StateManager { } getContractCode (addr: Buffer, cb: Function) { + console.log('get contract code for', addr) super.getContractCode(addr, (err: Error, code: Buffer) => { if (!err && !this.codes.has(addr.toString('hex'))) { this.codes.set(addr.toString('hex'), code) @@ -107,11 +108,12 @@ export async function stateFromProofs (preStateRoot: Buffer, data: { accountProo const account = new Account(accountRaw) if (!account.codeHash.equals(ethUtil.KECCAK256_NULL)) { const code = codes.get(key) - assert(code) - // There's some issue with how typescript handles promisify and cb types - // @ts-ignore - const codeHash = await promisify(account.setCode.bind(account))(stateManager._trie, code) - assert(codeHash.equals(account.codeHash)) + if (code !== undefined) { + // There's some issue with how typescript handles promisify and cb types + // @ts-ignore + const codeHash = await promisify(account.setCode.bind(account))(stateManager._trie, code) + assert(codeHash.equals(account.codeHash)) + } } if (!account.stateRoot.equals(ethUtil.KECCAK256_RLP)) { const storageMap = storageProofs.get(key) From b5396074542d4c1bddb6b017cab8818f206739cc Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 22 Jul 2019 15:42:50 +0200 Subject: [PATCH 04/15] stateless: prove storage keys with original root, other minor fix --- lib/stateless.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/stateless.ts b/lib/stateless.ts index b0e919f607..2a435973cd 100644 --- a/lib/stateless.ts +++ b/lib/stateless.ts @@ -13,6 +13,7 @@ export class HookedStateManager extends StateManager { proofs: Map codes: Map storageProofs: Map> + storageTrieRoots: Map seenKeys: Set preStateRoot: Buffer @@ -21,6 +22,7 @@ export class HookedStateManager extends StateManager { this.proofs = new Map() this.codes = new Map() this.storageProofs = new Map() + this.storageTrieRoots = new Map() this.seenKeys = new Set() this.preStateRoot = ethUtil.KECCAK256_RLP } @@ -48,7 +50,6 @@ export class HookedStateManager extends StateManager { } getContractCode (addr: Buffer, cb: Function) { - console.log('get contract code for', addr) super.getContractCode(addr, (err: Error, code: Buffer) => { if (!err && !this.codes.has(addr.toString('hex'))) { this.codes.set(addr.toString('hex'), code) @@ -71,6 +72,17 @@ export class HookedStateManager extends StateManager { this._getStorageTrie(addr, (err: Error, trie: any) => { if (err) return cb(err, null) + + // Cache root of contract's storage root the first time + // to use the same root even after modifications + trie = trie.copy() + let trieRoot = this.storageTrieRoots.get(addrS) + if (trieRoot === undefined) { + trieRoot = trie.root + this.storageTrieRoots.set(addrS, trieRoot!) + } + trie.root = trieRoot + SecureTrie.prove(trie, key, (err: Error, proof: Buffer[]) => { if (err) return cb(err, null) let storageMap = this.storageProofs.get(addrS) @@ -117,11 +129,11 @@ export async function stateFromProofs (preStateRoot: Buffer, data: { accountProo } if (!account.stateRoot.equals(ethUtil.KECCAK256_RLP)) { const storageMap = storageProofs.get(key) - assert(storageMap) - for (const [k, v] of storageMap!.entries()) { - const sp = await verifyProof(account.stateRoot, Buffer.from(k, 'hex'), v) - assert(sp) - await trieFromProof(stateManager._trie, v) + if (storageMap) { + for (const [k, v] of storageMap!.entries()) { + const sp = await verifyProof(account.stateRoot, Buffer.from(k, 'hex'), v) + await trieFromProof(stateManager._trie, v) + } } } } From 7421331fdadfeb4dbb640cd9e56dc9871d8e7eaf Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 23 Jul 2019 12:10:46 +0200 Subject: [PATCH 05/15] stateless: use pre-tx trie to prove contract storage keys --- lib/stateless.ts | 22 ++++++++-------------- tests/StatelessRunner.js | 4 +++- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/stateless.ts b/lib/stateless.ts index 2a435973cd..d042285886 100644 --- a/lib/stateless.ts +++ b/lib/stateless.ts @@ -15,7 +15,7 @@ export class HookedStateManager extends StateManager { storageProofs: Map> storageTrieRoots: Map seenKeys: Set - preStateRoot: Buffer + origState: StateManager constructor (opts: StateManagerOpts) { super(opts) @@ -24,7 +24,7 @@ export class HookedStateManager extends StateManager { this.storageProofs = new Map() this.storageTrieRoots = new Map() this.seenKeys = new Set() - this.preStateRoot = ethUtil.KECCAK256_RLP + this.origState = new StateManager() } getAccount (addr: Buffer, cb: Function) { @@ -32,9 +32,7 @@ export class HookedStateManager extends StateManager { if (!err && !this.seenKeys.has(addr.toString('hex'))) { this.seenKeys.add(addr.toString('hex')) - const trie = this._trie.copy() - trie.root = this.preStateRoot - trie._checkpoints = [] + const trie = this.origState._trie.copy() SecureTrie.prove(trie, addr, (err: Error, proof: Buffer[]) => { if (err) { @@ -70,18 +68,14 @@ export class HookedStateManager extends StateManager { return cb(err, value) } - this._getStorageTrie(addr, (err: Error, trie: any) => { + // Use state previous to running this transaction to fetch storage trie for account + // because account could be modified during current transaction. + this.origState._getStorageTrie(addr, (err: Error, trie: any) => { if (err) return cb(err, null) - // Cache root of contract's storage root the first time - // to use the same root even after modifications - trie = trie.copy() - let trieRoot = this.storageTrieRoots.get(addrS) - if (trieRoot === undefined) { - trieRoot = trie.root - this.storageTrieRoots.set(addrS, trieRoot!) + if (trie.root.equals(ethUtil.KECCAK256_RLP)) { + return cb(null, value) } - trie.root = trieRoot SecureTrie.prove(trie, key, (err: Error, proof: Buffer[]) => { if (err) return cb(err, null) diff --git a/tests/StatelessRunner.js b/tests/StatelessRunner.js index 36d0a34638..485a91002c 100644 --- a/tests/StatelessRunner.js +++ b/tests/StatelessRunner.js @@ -2,9 +2,11 @@ const testUtil = require('./util') const { promisify } = require('util') const ethUtil = require('ethereumjs-util') const Account = require('ethereumjs-account').default +const Trie = require('merkle-patricia-tree/secure') const BN = ethUtil.BN const { getRequiredForkConfigAlias } = require('./util') const { HookedStateManager, stateFromProofs } = require('../dist/stateless') +const StateManager = require('../dist/state/stateManager').default const VM = require('../dist/index.js').default const PStateManager = require('../dist/state/promisified').default @@ -41,7 +43,7 @@ async function runTestCase (options, testData, t) { await promisify(testUtil.setupPreConditions)(stateManager._trie, testData) const preStateRoot = stateManager._trie.root - stateManager.preStateRoot = preStateRoot + stateManager.origState = new StateManager({ trie: new Trie(stateManager._trie.db._leveldb, preStateRoot) }) try { await vm.runTx({ tx: tx, block: block }) From 236b8255a28f9288706a132f8533b9ed2588c2da Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 23 Jul 2019 12:11:36 +0200 Subject: [PATCH 06/15] Fix formatting issues --- lib/stateless.ts | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/stateless.ts b/lib/stateless.ts index d042285886..4878a27b32 100644 --- a/lib/stateless.ts +++ b/lib/stateless.ts @@ -17,7 +17,7 @@ export class HookedStateManager extends StateManager { seenKeys: Set origState: StateManager - constructor (opts: StateManagerOpts) { + constructor(opts: StateManagerOpts) { super(opts) this.proofs = new Map() this.codes = new Map() @@ -27,7 +27,7 @@ export class HookedStateManager extends StateManager { this.origState = new StateManager() } - getAccount (addr: Buffer, cb: Function) { + getAccount(addr: Buffer, cb: Function) { super.getAccount(addr, (err: Error, res: Account) => { if (!err && !this.seenKeys.has(addr.toString('hex'))) { this.seenKeys.add(addr.toString('hex')) @@ -47,7 +47,7 @@ export class HookedStateManager extends StateManager { }) } - getContractCode (addr: Buffer, cb: Function) { + getContractCode(addr: Buffer, cb: Function) { super.getContractCode(addr, (err: Error, code: Buffer) => { if (!err && !this.codes.has(addr.toString('hex'))) { this.codes.set(addr.toString('hex'), code) @@ -56,7 +56,7 @@ export class HookedStateManager extends StateManager { }) } - getContractStorage (addr: Buffer, key: Buffer, cb: Function) { + getContractStorage(addr: Buffer, key: Buffer, cb: Function) { super.getContractStorage(addr, key, (err: Error, value: any) => { const addrS = addr.toString('hex') const keyS = key.toString('hex') @@ -92,18 +92,27 @@ export class HookedStateManager extends StateManager { }) } - _isAccountEmpty (account: Account) { - return account.nonce.toString('hex') === '' && + _isAccountEmpty(account: Account) { + return ( + account.nonce.toString('hex') === '' && account.balance.toString('hex') === '' && account.codeHash.toString('hex') === ethUtil.KECCAK256_NULL_S + ) } } -export async function proveTx (vm: VM, opts: any) { +export async function proveTx(vm: VM, opts: any) { return runTx.bind(vm)(opts) } -export async function stateFromProofs (preStateRoot: Buffer, data: { accountProofs: Map, codes: Map, storageProofs: Map> }) { +export async function stateFromProofs( + preStateRoot: Buffer, + data: { + accountProofs: Map + codes: Map + storageProofs: Map> + }, +) { const { accountProofs, codes, storageProofs } = data const stateManager = new StateManager() stateManager._trie.root = preStateRoot @@ -135,8 +144,8 @@ export async function stateFromProofs (preStateRoot: Buffer, data: { accountProo return stateManager } -async function trieFromProof (trie: any, proofNodes: Buffer[]) { - let opStack = proofNodes.map((nodeValue) => { +async function trieFromProof(trie: any, proofNodes: Buffer[]) { + let opStack = proofNodes.map(nodeValue => { return { type: 'put', key: ethUtil.keccak256(nodeValue), value: ethUtil.toBuffer(nodeValue) } }) From 2f50243e14953e766b274ae38001ea7d99d9e65d Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Jul 2019 14:11:33 +0200 Subject: [PATCH 07/15] stateless: fix bug in proving non-existing storage keys --- lib/stateless.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stateless.ts b/lib/stateless.ts index 4878a27b32..1f3311f943 100644 --- a/lib/stateless.ts +++ b/lib/stateless.ts @@ -60,7 +60,7 @@ export class HookedStateManager extends StateManager { super.getContractStorage(addr, key, (err: Error, value: any) => { const addrS = addr.toString('hex') const keyS = key.toString('hex') - if (err || value.length === 0) { + if (err) { return cb(err, value) } From 0ddb8458b6d78b25b80c02b375b9237dfb383de9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Jul 2019 17:08:11 +0200 Subject: [PATCH 08/15] stateless: de-duplicate proof nodes, simplify stateFromProofs --- lib/stateless.ts | 83 ++++++++-------------------------------- tests/StatelessRunner.js | 16 ++++---- 2 files changed, 23 insertions(+), 76 deletions(-) diff --git a/lib/stateless.ts b/lib/stateless.ts index 1f3311f943..ab021eeb06 100644 --- a/lib/stateless.ts +++ b/lib/stateless.ts @@ -10,21 +10,15 @@ const SecureTrie = require('merkle-patricia-tree/secure') const verifyProof = promisify(SecureTrie.verifyProof) export class HookedStateManager extends StateManager { - proofs: Map - codes: Map - storageProofs: Map> - storageTrieRoots: Map seenKeys: Set origState: StateManager + proofNodes: Map constructor(opts: StateManagerOpts) { super(opts) - this.proofs = new Map() - this.codes = new Map() - this.storageProofs = new Map() - this.storageTrieRoots = new Map() this.seenKeys = new Set() this.origState = new StateManager() + this.proofNodes = new Map() } getAccount(addr: Buffer, cb: Function) { @@ -38,7 +32,10 @@ export class HookedStateManager extends StateManager { if (err) { return cb(err, null) } - this.proofs.set(addr.toString('hex'), proof) + for (const n of proof) { + const h = ethUtil.keccak256(n) + this.proofNodes.set(h.toString('hex'), n) + } cb(null, res) }) } else { @@ -49,8 +46,9 @@ export class HookedStateManager extends StateManager { getContractCode(addr: Buffer, cb: Function) { super.getContractCode(addr, (err: Error, code: Buffer) => { - if (!err && !this.codes.has(addr.toString('hex'))) { - this.codes.set(addr.toString('hex'), code) + if (!err) { + const h = ethUtil.keccak256(code) + this.proofNodes.set(h.toString('hex'), code) } cb(err, code) }) @@ -64,10 +62,12 @@ export class HookedStateManager extends StateManager { return cb(err, value) } - if (this.storageProofs.has(addrS) && this.storageProofs.get(addrS)!.has(keyS)) { + if (this.seenKeys.has(addrS.concat(keyS))) { return cb(err, value) } + this.seenKeys.add(addrS.concat(keyS)) + // Use state previous to running this transaction to fetch storage trie for account // because account could be modified during current transaction. this.origState._getStorageTrie(addr, (err: Error, trie: any) => { @@ -79,12 +79,10 @@ export class HookedStateManager extends StateManager { SecureTrie.prove(trie, key, (err: Error, proof: Buffer[]) => { if (err) return cb(err, null) - let storageMap = this.storageProofs.get(addrS) - if (storageMap === undefined) { - storageMap = new Map() - this.storageProofs.set(addrS, storageMap) + for (const n of proof) { + const h = ethUtil.keccak256(n) + this.proofNodes.set(h.toString('hex'), n) } - storageMap.set(keyS, proof) }) }) @@ -100,54 +98,3 @@ export class HookedStateManager extends StateManager { ) } } - -export async function proveTx(vm: VM, opts: any) { - return runTx.bind(vm)(opts) -} - -export async function stateFromProofs( - preStateRoot: Buffer, - data: { - accountProofs: Map - codes: Map - storageProofs: Map> - }, -) { - const { accountProofs, codes, storageProofs } = data - const stateManager = new StateManager() - stateManager._trie.root = preStateRoot - for (const [key, value] of accountProofs.entries()) { - let keyBuf = Buffer.from(key, 'hex') - await trieFromProof(stateManager._trie, value) - const accountRaw = await promisify(stateManager._trie.get.bind(stateManager._trie))(keyBuf) - const account = new Account(accountRaw) - if (!account.codeHash.equals(ethUtil.KECCAK256_NULL)) { - const code = codes.get(key) - if (code !== undefined) { - // There's some issue with how typescript handles promisify and cb types - // @ts-ignore - const codeHash = await promisify(account.setCode.bind(account))(stateManager._trie, code) - assert(codeHash.equals(account.codeHash)) - } - } - if (!account.stateRoot.equals(ethUtil.KECCAK256_RLP)) { - const storageMap = storageProofs.get(key) - if (storageMap) { - for (const [k, v] of storageMap!.entries()) { - const sp = await verifyProof(account.stateRoot, Buffer.from(k, 'hex'), v) - await trieFromProof(stateManager._trie, v) - } - } - } - } - - return stateManager -} - -async function trieFromProof(trie: any, proofNodes: Buffer[]) { - let opStack = proofNodes.map(nodeValue => { - return { type: 'put', key: ethUtil.keccak256(nodeValue), value: ethUtil.toBuffer(nodeValue) } - }) - - return promisify(trie.db.batch.bind(trie.db))(opStack) -} diff --git a/tests/StatelessRunner.js b/tests/StatelessRunner.js index 485a91002c..e8862c05a5 100644 --- a/tests/StatelessRunner.js +++ b/tests/StatelessRunner.js @@ -5,7 +5,7 @@ const Account = require('ethereumjs-account').default const Trie = require('merkle-patricia-tree/secure') const BN = ethUtil.BN const { getRequiredForkConfigAlias } = require('./util') -const { HookedStateManager, stateFromProofs } = require('../dist/stateless') +const { HookedStateManager } = require('../dist/stateless') const StateManager = require('../dist/state/stateManager').default const VM = require('../dist/index.js').default @@ -52,14 +52,14 @@ async function runTestCase (options, testData, t) { } t.equal(stateManager._trie.root.toString('hex'), expectedPostStateRoot, 'the state roots should match') - const proofData = { - accountProofs: stateManager.proofs, - codes: stateManager.codes, - storageProofs: stateManager.storageProofs + const trie = new Trie(null, preStateRoot) + const opStack = [] + for (const [k, v] of stateManager.proofNodes) { + opStack.push({ type: 'put', key: Buffer.from(k, 'hex'), value: v }) } - // Now use proofs to construct stateManager - // and run tx again to make sure it works - stateManager = await stateFromProofs(preStateRoot, proofData) + await promisify(trie.db.batch.bind(trie.db))(opStack) + + stateManager = new StateManager({ trie: trie }) vm = new VM({ stateManager: stateManager, hardfork: options.forkConfig.toLowerCase() From b1208da555d1a08789b24122b527e981b1d0a481 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Jul 2019 18:45:33 +0200 Subject: [PATCH 09/15] stateless: all state tests pass, greatly simplified version --- lib/stateless.ts | 100 --------------------------------------- tests/StatelessRunner.js | 40 ++++++++++++---- 2 files changed, 32 insertions(+), 108 deletions(-) delete mode 100644 lib/stateless.ts diff --git a/lib/stateless.ts b/lib/stateless.ts deleted file mode 100644 index ab021eeb06..0000000000 --- a/lib/stateless.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { promisify } from 'util' -import * as assert from 'assert' -import * as ethUtil from 'ethereumjs-util' -import Account from 'ethereumjs-account' -import VM from './index' -import { default as StateManager, StateManagerOpts } from './state/stateManager' -import runTx from './runTx' - -const SecureTrie = require('merkle-patricia-tree/secure') -const verifyProof = promisify(SecureTrie.verifyProof) - -export class HookedStateManager extends StateManager { - seenKeys: Set - origState: StateManager - proofNodes: Map - - constructor(opts: StateManagerOpts) { - super(opts) - this.seenKeys = new Set() - this.origState = new StateManager() - this.proofNodes = new Map() - } - - getAccount(addr: Buffer, cb: Function) { - super.getAccount(addr, (err: Error, res: Account) => { - if (!err && !this.seenKeys.has(addr.toString('hex'))) { - this.seenKeys.add(addr.toString('hex')) - - const trie = this.origState._trie.copy() - - SecureTrie.prove(trie, addr, (err: Error, proof: Buffer[]) => { - if (err) { - return cb(err, null) - } - for (const n of proof) { - const h = ethUtil.keccak256(n) - this.proofNodes.set(h.toString('hex'), n) - } - cb(null, res) - }) - } else { - cb(err, res) - } - }) - } - - getContractCode(addr: Buffer, cb: Function) { - super.getContractCode(addr, (err: Error, code: Buffer) => { - if (!err) { - const h = ethUtil.keccak256(code) - this.proofNodes.set(h.toString('hex'), code) - } - cb(err, code) - }) - } - - getContractStorage(addr: Buffer, key: Buffer, cb: Function) { - super.getContractStorage(addr, key, (err: Error, value: any) => { - const addrS = addr.toString('hex') - const keyS = key.toString('hex') - if (err) { - return cb(err, value) - } - - if (this.seenKeys.has(addrS.concat(keyS))) { - return cb(err, value) - } - - this.seenKeys.add(addrS.concat(keyS)) - - // Use state previous to running this transaction to fetch storage trie for account - // because account could be modified during current transaction. - this.origState._getStorageTrie(addr, (err: Error, trie: any) => { - if (err) return cb(err, null) - - if (trie.root.equals(ethUtil.KECCAK256_RLP)) { - return cb(null, value) - } - - SecureTrie.prove(trie, key, (err: Error, proof: Buffer[]) => { - if (err) return cb(err, null) - for (const n of proof) { - const h = ethUtil.keccak256(n) - this.proofNodes.set(h.toString('hex'), n) - } - }) - }) - - cb(err, value) - }) - } - - _isAccountEmpty(account: Account) { - return ( - account.nonce.toString('hex') === '' && - account.balance.toString('hex') === '' && - account.codeHash.toString('hex') === ethUtil.KECCAK256_NULL_S - ) - } -} diff --git a/tests/StatelessRunner.js b/tests/StatelessRunner.js index e8862c05a5..153901f879 100644 --- a/tests/StatelessRunner.js +++ b/tests/StatelessRunner.js @@ -5,7 +5,6 @@ const Account = require('ethereumjs-account').default const Trie = require('merkle-patricia-tree/secure') const BN = ethUtil.BN const { getRequiredForkConfigAlias } = require('./util') -const { HookedStateManager } = require('../dist/stateless') const StateManager = require('../dist/state/stateManager').default const VM = require('../dist/index.js').default @@ -29,9 +28,10 @@ async function runTestCase (options, testData, t) { return } - // Hooked state manager proves necessary trie nodes - // for a stateless execution - let stateManager = new HookedStateManager() + let stateManager = new StateManager() + await promisify(testUtil.setupPreConditions)(stateManager._trie, testData) + const preStateRoot = stateManager._trie.root + // Set up VM let vm = new VM({ stateManager: stateManager, @@ -40,10 +40,33 @@ async function runTestCase (options, testData, t) { if (options.jsontrace) { hookVM(vm, t) } - await promisify(testUtil.setupPreConditions)(stateManager._trie, testData) - const preStateRoot = stateManager._trie.root - stateManager.origState = new StateManager({ trie: new Trie(stateManager._trie.db._leveldb, preStateRoot) }) + // Determine set of all node hashes in the database + // before running the tx. + const existingKeys = new Set() + const it = stateManager._trie.db._leveldb.iterator() + const next = promisify(it.next.bind(it)) + while (true) { + const key = await next() + if (!key) break + existingKeys.add(key.toString('hex')) + } + + // Hook leveldb.get and add any node that was fetched during execution + // to a bag of proof nodes, under the condition that this node existed + // before execution. + const proofNodes = new Map() + const getFunc = stateManager._trie.db._leveldb.get.bind(stateManager._trie.db._leveldb) + stateManager._trie.db._leveldb.get = (key, opts, cb) => { + getFunc(key, opts, (err, v) => { + if (!err && v) { + if (existingKeys.has(key.toString('hex'))) { + proofNodes.set(key.toString('hex'), v) + } + } + cb(err, v) + }) + } try { await vm.runTx({ tx: tx, block: block }) @@ -52,9 +75,10 @@ async function runTestCase (options, testData, t) { } t.equal(stateManager._trie.root.toString('hex'), expectedPostStateRoot, 'the state roots should match') + // Save bag of proof nodes to a new trie's underlying leveldb const trie = new Trie(null, preStateRoot) const opStack = [] - for (const [k, v] of stateManager.proofNodes) { + for (const [k, v] of proofNodes) { opStack.push({ type: 'put', key: Buffer.from(k, 'hex'), value: v }) } await promisify(trie.db.batch.bind(trie.db))(opStack) From 92bf49dc2a921913bdfc44abc74e05fbaed6c5fd Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Jul 2019 19:19:20 +0200 Subject: [PATCH 10/15] Update trie to v3.0.0, use v3 for StatelessRunner --- tests/StatelessRunner.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/StatelessRunner.js b/tests/StatelessRunner.js index 153901f879..1e0422d639 100644 --- a/tests/StatelessRunner.js +++ b/tests/StatelessRunner.js @@ -44,7 +44,7 @@ async function runTestCase (options, testData, t) { // Determine set of all node hashes in the database // before running the tx. const existingKeys = new Set() - const it = stateManager._trie.db._leveldb.iterator() + const it = stateManager._trie.db.iterator() const next = promisify(it.next.bind(it)) while (true) { const key = await next() @@ -56,8 +56,8 @@ async function runTestCase (options, testData, t) { // to a bag of proof nodes, under the condition that this node existed // before execution. const proofNodes = new Map() - const getFunc = stateManager._trie.db._leveldb.get.bind(stateManager._trie.db._leveldb) - stateManager._trie.db._leveldb.get = (key, opts, cb) => { + const getFunc = stateManager._trie.db.get.bind(stateManager._trie.db) + stateManager._trie.db.get = (key, opts, cb) => { getFunc(key, opts, (err, v) => { if (!err && v) { if (existingKeys.has(key.toString('hex'))) { From 570cba8f2689b62e5240282fc914091d7333578a Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Jul 2019 19:19:33 +0200 Subject: [PATCH 11/15] ci: add stateless test job --- .circleci/config.yml | 140 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..06075da3d2 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,140 @@ +version: 2 + +defaults: &defaults + working_directory: ~/project/ethereumjs-vm + docker: + - image: circleci/node:8-browsers +restore_node_modules: &restore_node_modules + restore_cache: + name: Restore node_modules cache + keys: + - v1-node-{{ .Branch }}-{{ checksum "package.json" }} + - v1-node-{{ checksum "package.json" }} +jobs: + install: + <<: *defaults + steps: + - checkout + - *restore_node_modules + - run: + name: npm install + command: npm install + - save_cache: + name: Save node_modules cache + key: v1-node-{{ .Branch }}-{{ checksum "package.json" }} + paths: + - node_modules/ + - persist_to_workspace: + root: ~/project + paths: + - ethereumjs-vm/ + lint: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: Lint + command: npm run lint + test_api: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testAPI + command: npm run testAPI + test_api_browser: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testAPI:browser + command: npm run testAPI:browser + test_state_byzantium: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testStateByzantium + command: npm run testStateByzantium + test_state_constantinople: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testStateConstantinople + command: npm run testStateConstantinople + test_state_petersburg: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testStatePetersburg + command: npm run testStatePetersburg + test_stateless: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testStateless + command: npm run testStateless + test_blockchain: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: testBlockchain + command: npm run testBlockchain + coveralls: + <<: *defaults + steps: + - attach_workspace: + at: ~/project + - *restore_node_modules + - run: + name: coveralls + command: npm run coveralls +workflows: + version: 2 + install-lint-test: + jobs: + - install + - lint: + requires: + - install + - test_api: + requires: + - install + - test_api_browser: + requires: + - install + - test_state_byzantium: + requires: + - install + - test_state_constantinople: + requires: + - install + - test_state_petersburg: + requires: + - install + - test_blockchain: + requires: + - install + - coveralls: + requires: + - install From 38097331cc15abc3187ab47d863d6f1b95ac5086 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 24 Jul 2019 19:40:27 +0200 Subject: [PATCH 12/15] ci: fix stateless job --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 06075da3d2..8997d6f7f5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,6 +132,9 @@ workflows: - test_state_petersburg: requires: - install + - test_stateless: + requires: + - install - test_blockchain: requires: - install From 9572f572575f9c3f33ba34b0a9a45c31f2f7e718 Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Sat, 15 Aug 2020 10:07:58 +0200 Subject: [PATCH 13/15] vm -> stateless: monorepo transition fixes, tests/tester.js integration, added GitHub actions workflow --- .github/workflows/vm-pr.yml | 27 ++ packages/vm/package.json | 1 + .../vm/tests}/StatelessRunner.js | 0 packages/vm/tests/tester.js | 22 ++ tests/tester.js | 258 ------------------ 5 files changed, 50 insertions(+), 258 deletions(-) rename {tests => packages/vm/tests}/StatelessRunner.js (100%) delete mode 100755 tests/tester.js diff --git a/.github/workflows/vm-pr.yml b/.github/workflows/vm-pr.yml index e6dd99f1ad..71b24c63cc 100644 --- a/.github/workflows/vm-pr.yml +++ b/.github/workflows/vm-pr.yml @@ -111,3 +111,30 @@ jobs: working-directory: ${{github.workspace}} - run: npm run test:blockchain -- ${{ matrix.args }} --fork=${{ matrix.fork }} + + test-vm-stateless: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-node@v1 + with: + node-version: 12.x + - uses: actions/checkout@v1 + + - name: Dependency cache + uses: actions/cache@v2 + id: cache + with: + key: VM-${{ runner.os }}-12-${{ hashFiles('**/package-lock.json') }} + path: '**/node_modules' + + # Installs root dependencies, ignoring Bootstrap All script. + # Bootstraps the current package only + - run: npm install --ignore-scripts && npm run bootstrap:vm + if: steps.cache.outputs.cache-hit != 'true' + working-directory: ${{github.workspace}} + + # Builds current package and the ones it depends from. + - run: npm run build:vm + working-directory: ${{github.workspace}} + + - run: npm run test:stateless diff --git a/packages/vm/package.json b/packages/vm/package.json index 32166468d7..f7f6670f77 100644 --- a/packages/vm/package.json +++ b/packages/vm/package.json @@ -20,6 +20,7 @@ "test:buildIntegrity": "npm run test:state -- --test='stackOverflow'", "test:blockchain": "node -r ts-node/register --stack-size=1500 ./tests/tester --blockchain", "test:blockchain:allForks": "echo 'Homestead TangerineWhistle SpuriousDragon Byzantium Constantinople Petersburg Istanbul MuirGlacier' | xargs -n1 | xargs -I v1 node -r ts-node/register --stack-size=1500 ./tests/tester --blockchain --fork=v1", + "test:stateless": "npm run build && node ./tests/tester --stateless --fork='Petersburg' --dist", "test:API": "tape -r ts-node/register --stack-size=1500 ./tests/api/**/*.js", "test:API:browser": "npm run build && karma start karma.conf.js", "test": "echo \"[INFO] Generic test cmd not used. See package.json for more specific test run cmds.\"", diff --git a/tests/StatelessRunner.js b/packages/vm/tests/StatelessRunner.js similarity index 100% rename from tests/StatelessRunner.js rename to packages/vm/tests/StatelessRunner.js diff --git a/packages/vm/tests/tester.js b/packages/vm/tests/tester.js index 364ad1e0a6..1a4bb9ae98 100755 --- a/packages/vm/tests/tester.js +++ b/packages/vm/tests/tester.js @@ -12,6 +12,8 @@ function runTests() { name = 'GeneralStateTests' } else if (argv.blockchain) { name = 'BlockchainTests' + } else if (argv.stateless) { + name = 'Stateless' } const FORK_CONFIG = (argv.fork || config.DEFAULT_FORK_CONFIG) @@ -78,6 +80,7 @@ function runTests() { console.log(`+${'-'.repeat(width)}+`) console.log() + // Run a custom state test if (argv.customStateTest) { const stateTestRunner = require('./GeneralStateTestsRunner.js') let fileName = argv.customStateTest @@ -91,6 +94,25 @@ function runTests() { t.end() }) }) + // Stateless test execution + } else if (name === 'Stateless') { + tape(name, t => { + const stateTestRunner = require('./StatelessRunner.js') + testing.getTestsFromArgs('GeneralStateTests', async (fileName, testName, test) => { + let runSkipped = testGetterArgs.runSkipped + let inRunSkipped = runSkipped.includes(fileName) + if (runSkipped.length === 0 || inRunSkipped) { + t.comment(`file: ${fileName} test: ${testName}`) + return stateTestRunner(runnerArgs, test, t) + } + }, testGetterArgs).then(() => { + t.end() + }).catch((err) => { + console.log(err) + t.end() + }) + }) + // Blockchain and State Tests } else { tape(name, t => { const runner = require(`./${name}Runner.js`) diff --git a/tests/tester.js b/tests/tester.js deleted file mode 100755 index 4088f50175..0000000000 --- a/tests/tester.js +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env node - -const argv = require('minimist')(process.argv.slice(2)) -const tape = require('tape') -const testing = require('ethereumjs-testing') -const FORK_CONFIG = argv.fork || 'Petersburg' -const { - getRequiredForkConfigAlias -} = require('./util') -// tests which should be fixed -const skipBroken = [ - 'dynamicAccountOverwriteEmpty' // temporary till fixed (2019-01-30), skipped along constantinopleFix work time constraints -] -// tests skipped due to system specifics / design considerations -const skipPermanent = [ - 'SuicidesMixingCoinbase', // sucides to the coinbase, since we run a blockLevel we create coinbase account. - 'static_SuicidesMixingCoinbase', // sucides to the coinbase, since we run a blockLevel we create coinbase account. - 'ForkUncle', // Only BlockchainTest, correct behaviour unspecified (?) - 'UncleFromSideChain' // Only BlockchainTest, same as ForkUncle, the TD is the same for two diffent branches so its not clear which one should be the finally chain -] -// tests running slow (run from time to time) -const skipSlow = [ - 'Call50000', // slow - 'Call50000_ecrec', // slow - 'Call50000_identity', // slow - 'Call50000_identity2', // slow - 'Call50000_sha256', // slow - 'Call50000_rip160', // slow - 'Call50000bytesContract50_1', // slow - 'Call50000bytesContract50_2', - 'Call1MB1024Calldepth', // slow - 'static_Call1MB1024Calldepth', // slow - 'static_Call50000', // slow - 'static_Call50000_ecrec', - 'static_Call50000_identity', - 'static_Call50000_identity2', - 'static_Call50000_rip160', - 'static_Return50000_2', - 'Callcode50000', // slow - 'Return50000', // slow - 'Return50000_2', // slow - 'static_Call50000', // slow - 'static_Call50000_ecrec', // slow - 'static_Call50000_identity', // slow - 'static_Call50000_identity2', // slow - 'static_Call50000_sha256', // slow - 'static_Call50000_rip160', // slow - 'static_Call50000bytesContract50_1', // slow - 'static_Call50000bytesContract50_2', - 'static_Call1MB1024Calldepth', // slow - 'static_Callcode50000', // slow - 'static_Return50000', // slow - 'static_Return50000_2', // slow - 'QuadraticComplexitySolidity_CallDataCopy' -] - -/* -NOTE: VM tests have been disabled since they are generated using Frontier gas costs, and ethereumjs-vm doesn't support historical fork rules - -TODO: some VM tests do not appear to be executing (don't print an "ok" statement): -... -# file: vmLogTest test: log0_emptyMem -ok 38984 valid gas usage -# file: vmLogTest test: log0_logMemStartTooHigh -# file: vmLogTest test: log0_logMemsizeTooHigh -# file: vmLogTest test: log0_logMemsizeZero -ok 38985 valid gas usage -# file: vmLogTest test: log0_nonEmptyMem -*/ - -const skipVM = [ - // slow performance tests - 'loop-mul', - 'loop-add-10M', - 'loop-divadd-10M', - 'loop-divadd-unr100-10M', - 'loop-exp-16b-100k', - 'loop-exp-1b-1M', - 'loop-exp-2b-100k', - 'loop-exp-32b-100k', - 'loop-exp-4b-100k', - 'loop-exp-8b-100k', - 'loop-exp-nop-1M', - 'loop-mulmod-2M', - // some VM tests fail because the js runner executes CALLs - // see https://github.com/ethereum/tests/wiki/VM-Tests > Since these tests are meant only as a basic test of VM operation, the CALL and CREATE instructions are not actually executed. - 'ABAcalls0', - 'ABAcallsSuicide0', - 'ABAcallsSuicide1', - 'sha3_bigSize', - 'CallRecursiveBomb0', - 'CallToNameRegistrator0', - 'CallToPrecompiledContract', - 'CallToReturn1', - 'PostToNameRegistrator0', - 'PostToReturn1', - 'callcodeToNameRegistrator0', - 'callcodeToReturn1', - 'callstatelessToNameRegistrator0', - 'callstatelessToReturn1', - 'createNameRegistrator', - 'randomTest643' // TODO fix this -] - -if (argv.r) { - randomized(argv.r, argv.v) -} else if (argv.s) { - runTests('GeneralStateTests', argv) -} else if (argv.v) { - runTests('VMTests', argv) -} else if (argv.b) { - runTests('BlockchainTests', argv) -} else if (argv.stateless) { - runTests('Stateless', argv) -} - -// randomized tests -// returns 1 if the tests fails -// returns 0 if the tests succeds -function randomized (stateTest) { - const stateRunner = require('./stateRunner.js') - let errored = false - - tape.createStream({ - objectMode: true - }).on('data', function (row) { - if (row.ok === false && !errored) { - errored = true - process.stdout.write('1') - process.exit() - } - }).on('end', function () { - process.stdout.write('0') - }) - - try { - stateTest = JSON.parse(stateTest) - } catch (e) { - console.error('invalid json') - process.exit() - } - - var keys = Object.keys(stateTest) - stateTest = stateTest[keys[0]] - - tape('', t => { - stateRunner({}, stateTest, t, t.end) - }) -} - -function getSkipTests (choices, defaultChoice) { - let skipTests = [] - if (!choices) { - choices = defaultChoice - } - choices = choices.toLowerCase() - if (choices !== 'none') { - let choicesList = choices.split(',') - let all = choicesList.includes('all') - if (all || choicesList.includes('broken')) { - skipTests = skipTests.concat(skipBroken) - } - if (all || choicesList.includes('permanent')) { - skipTests = skipTests.concat(skipPermanent) - } - if (all || choicesList.includes('slow')) { - skipTests = skipTests.concat(skipSlow) - } - } - return skipTests -} - -function runTests (name, runnerArgs, cb) { - let testGetterArgs = {} - - testGetterArgs.skipTests = getSkipTests(argv.skip, argv.runSkipped ? 'NONE' : 'ALL') - testGetterArgs.runSkipped = getSkipTests(argv.runSkipped, 'NONE') - testGetterArgs.skipVM = skipVM - testGetterArgs.forkConfig = getRequiredForkConfigAlias(FORK_CONFIG) - testGetterArgs.file = argv.file - testGetterArgs.test = argv.test - testGetterArgs.dir = argv.dir - testGetterArgs.excludeDir = argv.excludeDir - testGetterArgs.testsPath = argv.testsPath - - testGetterArgs.customStateTest = argv.customStateTest - - runnerArgs.forkConfig = FORK_CONFIG - runnerArgs.jsontrace = argv.jsontrace - runnerArgs.debug = argv.debug // for BlockchainTests - - // for GeneralStateTests - runnerArgs.data = argv.data - runnerArgs.gasLimit = argv.gas - runnerArgs.value = argv.value - - // runnerArgs.vmtrace = true; // for VMTests - - if (name === 'Stateless') { - tape(name, t => { - const stateTestRunner = require('./StatelessRunner.js') - testing.getTestsFromArgs('GeneralStateTests', async (fileName, testName, test) => { - let runSkipped = testGetterArgs.runSkipped - let inRunSkipped = runSkipped.includes(fileName) - if (runSkipped.length === 0 || inRunSkipped) { - t.comment(`file: ${fileName} test: ${testName}`) - return stateTestRunner(runnerArgs, test, t) - } - }, testGetterArgs).then(() => { - t.end() - }).catch((err) => { - console.log(err) - t.end() - }) - }) - } else if (argv.customStateTest) { - const stateTestRunner = require('./GeneralStateTestsRunner.js') - let fileName = argv.customStateTest - tape(name, t => { - testing.getTestFromSource(fileName, (err, test) => { - if (err) { - return t.fail(err) - } - - t.comment(`file: ${fileName} test: ${test.testName}`) - stateTestRunner(runnerArgs, test, t, () => { - t.end() - }) - }) - }) - } else { - tape(name, t => { - const runner = require(`./${name}Runner.js`) - testing.getTestsFromArgs(name, (fileName, testName, test) => { - return new Promise((resolve, reject) => { - if (name === 'VMTests') { - // suppress some output of VMTests - // t.comment(`file: ${fileName} test: ${testName}`) - test.fileName = fileName - test.testName = testName - runner(runnerArgs, test, t, resolve) - } else { - let runSkipped = testGetterArgs.runSkipped - let inRunSkipped = runSkipped.includes(fileName) - if (runSkipped.length === 0 || inRunSkipped) { - t.comment(`file: ${fileName} test: ${testName}`) - runner(runnerArgs, test, t, resolve) - } else { - resolve() - } - } - }).catch(err => console.log(err)) - }, testGetterArgs).then(() => { - t.end() - }) - }) - } -} From fc13bb5b9d4b32cb55ac92cfd554dbdd120aad0b Mon Sep 17 00:00:00 2001 From: holgerd77 Date: Sat, 15 Aug 2020 10:49:10 +0200 Subject: [PATCH 14/15] vm -> stateless: adopt to changed test environment, switch to promisified StateManager --- .circleci/config.yml | 143 --------------------------- packages/vm/package.json | 2 +- packages/vm/tests/StatelessRunner.js | 40 ++++---- packages/vm/tests/tester.js | 10 +- 4 files changed, 27 insertions(+), 168 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 8997d6f7f5..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,143 +0,0 @@ -version: 2 - -defaults: &defaults - working_directory: ~/project/ethereumjs-vm - docker: - - image: circleci/node:8-browsers -restore_node_modules: &restore_node_modules - restore_cache: - name: Restore node_modules cache - keys: - - v1-node-{{ .Branch }}-{{ checksum "package.json" }} - - v1-node-{{ checksum "package.json" }} -jobs: - install: - <<: *defaults - steps: - - checkout - - *restore_node_modules - - run: - name: npm install - command: npm install - - save_cache: - name: Save node_modules cache - key: v1-node-{{ .Branch }}-{{ checksum "package.json" }} - paths: - - node_modules/ - - persist_to_workspace: - root: ~/project - paths: - - ethereumjs-vm/ - lint: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: Lint - command: npm run lint - test_api: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testAPI - command: npm run testAPI - test_api_browser: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testAPI:browser - command: npm run testAPI:browser - test_state_byzantium: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testStateByzantium - command: npm run testStateByzantium - test_state_constantinople: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testStateConstantinople - command: npm run testStateConstantinople - test_state_petersburg: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testStatePetersburg - command: npm run testStatePetersburg - test_stateless: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testStateless - command: npm run testStateless - test_blockchain: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: testBlockchain - command: npm run testBlockchain - coveralls: - <<: *defaults - steps: - - attach_workspace: - at: ~/project - - *restore_node_modules - - run: - name: coveralls - command: npm run coveralls -workflows: - version: 2 - install-lint-test: - jobs: - - install - - lint: - requires: - - install - - test_api: - requires: - - install - - test_api_browser: - requires: - - install - - test_state_byzantium: - requires: - - install - - test_state_constantinople: - requires: - - install - - test_state_petersburg: - requires: - - install - - test_stateless: - requires: - - install - - test_blockchain: - requires: - - install - - coveralls: - requires: - - install diff --git a/packages/vm/package.json b/packages/vm/package.json index f7f6670f77..8db574d5fc 100644 --- a/packages/vm/package.json +++ b/packages/vm/package.json @@ -20,7 +20,7 @@ "test:buildIntegrity": "npm run test:state -- --test='stackOverflow'", "test:blockchain": "node -r ts-node/register --stack-size=1500 ./tests/tester --blockchain", "test:blockchain:allForks": "echo 'Homestead TangerineWhistle SpuriousDragon Byzantium Constantinople Petersburg Istanbul MuirGlacier' | xargs -n1 | xargs -I v1 node -r ts-node/register --stack-size=1500 ./tests/tester --blockchain --fork=v1", - "test:stateless": "npm run build && node ./tests/tester --stateless --fork='Petersburg' --dist", + "test:stateless": "npm run build && node ./tests/tester --stateless --dist", "test:API": "tape -r ts-node/register --stack-size=1500 ./tests/api/**/*.js", "test:API:browser": "npm run build && karma start karma.conf.js", "test": "echo \"[INFO] Generic test cmd not used. See package.json for more specific test run cmds.\"", diff --git a/packages/vm/tests/StatelessRunner.js b/packages/vm/tests/StatelessRunner.js index 1e0422d639..a96ec9f834 100644 --- a/packages/vm/tests/StatelessRunner.js +++ b/packages/vm/tests/StatelessRunner.js @@ -1,14 +1,11 @@ -const testUtil = require('./util') -const { promisify } = require('util') const ethUtil = require('ethereumjs-util') -const Account = require('ethereumjs-account').default -const Trie = require('merkle-patricia-tree/secure') const BN = ethUtil.BN -const { getRequiredForkConfigAlias } = require('./util') -const StateManager = require('../dist/state/stateManager').default - -const VM = require('../dist/index.js').default -const PStateManager = require('../dist/state/promisified').default +const { getRequiredForkConfigAlias, setupPreConditions, makeTx, makeBlockFromEnv } = require('./util') +const Account = require('@ethereumjs/account').default +const Trie = require('merkle-patricia-tree').SecureTrie +const { default: Common } = require('@ethereumjs/common') +const { default: VM } = require('../dist/index.js') +const { default: DefaultStateManager } = require('../dist/state/stateManager') async function runTestCase (options, testData, t) { let expectedPostStateRoot = testData.postStateRoot @@ -17,8 +14,8 @@ async function runTestCase (options, testData, t) { } // Prepare tx and block - let tx = testUtil.makeTx(testData.transaction) - let block = testUtil.makeBlockFromEnv(testData.env) + let tx = makeTx(testData.transaction) + let block = makeBlockFromEnv(testData.env) tx._homestead = true tx.enableHomestead = true block.isHomestead = function () { @@ -28,14 +25,15 @@ async function runTestCase (options, testData, t) { return } - let stateManager = new StateManager() - await promisify(testUtil.setupPreConditions)(stateManager._trie, testData) + const common = new Common('mainnet', options.forkConfigVM.toLowerCase()) + const stateManager = new DefaultStateManager({ common: common }) + await setupPreConditions(stateManager._trie, testData) const preStateRoot = stateManager._trie.root // Set up VM let vm = new VM({ stateManager: stateManager, - hardfork: options.forkConfig.toLowerCase() + common: common }) if (options.jsontrace) { hookVM(vm, t) @@ -94,7 +92,7 @@ async function runTestCase (options, testData, t) { try { await vm.runTx({ tx: tx, block: block }) } catch (err) { - await deleteCoinbase(new PStateManager(stateManager), block.header.coinbase) + await deleteCoinbase(stateManager, block.header.coinbase) } t.equal(stateManager._trie.root.toString('hex'), expectedPostStateRoot, 'the state roots should match') } @@ -104,12 +102,12 @@ async function runTestCase (options, testData, t) { * expects the coinbase account to be deleted from state. * Without this ecmul_0-3_5616_28000_96 would fail. */ -async function deleteCoinbase (pstate, coinbaseAddr) { - const account = await pstate.getAccount(coinbaseAddr) +async function deleteCoinbase (stateManager, coinbaseAddr) { + const account = await stateManager.getAccount(coinbaseAddr) if (new BN(account.balance).isZero()) { - await pstate.putAccount(coinbaseAddr, new Account()) - await pstate.cleanupTouchedAccounts() - await promisify(pstate._wrapped._cache.flush.bind(pstate._wrapped._cache))() + await stateManager.putAccount(coinbaseAddr, new Account()) + await stateManager.cleanupTouchedAccounts() + await stateManager._wrapped._cache.flush() } } @@ -178,7 +176,7 @@ function parseTestCases (forkConfig, testData, data, gasLimit, value) { } module.exports = async function runStateTest (options, testData, t) { - const forkConfig = getRequiredForkConfigAlias(options.forkConfig) + const forkConfig = getRequiredForkConfigAlias(options.forkConfigTestSuite) try { const testCases = parseTestCases(forkConfig, testData, options.data, options.gasLimit, options.value) if (testCases.length > 0) { diff --git a/packages/vm/tests/tester.js b/packages/vm/tests/tester.js index 1a4bb9ae98..dab1ac710d 100755 --- a/packages/vm/tests/tester.js +++ b/packages/vm/tests/tester.js @@ -98,12 +98,16 @@ function runTests() { } else if (name === 'Stateless') { tape(name, t => { const stateTestRunner = require('./StatelessRunner.js') - testing.getTestsFromArgs('GeneralStateTests', async (fileName, testName, test) => { + let count = 0 + testLoader.getTestsFromArgs('GeneralStateTests', async (fileName, testName, test) => { let runSkipped = testGetterArgs.runSkipped let inRunSkipped = runSkipped.includes(fileName) if (runSkipped.length === 0 || inRunSkipped) { - t.comment(`file: ${fileName} test: ${testName}`) - return stateTestRunner(runnerArgs, test, t) + count += 1 + if (count < 2) { + t.comment(`file: ${fileName} test: ${testName}`) + return stateTestRunner(runnerArgs, test, t) + } } }, testGetterArgs).then(() => { t.end() From a38352287e47986df0b66feb8cd66017d6ea0f92 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 18 Aug 2020 10:46:44 +0200 Subject: [PATCH 15/15] stateless: remove unnecessary homestead flags Co-authored-by: Ryan Ghods --- packages/vm/tests/StatelessRunner.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/vm/tests/StatelessRunner.js b/packages/vm/tests/StatelessRunner.js index a96ec9f834..feb517c981 100644 --- a/packages/vm/tests/StatelessRunner.js +++ b/packages/vm/tests/StatelessRunner.js @@ -16,11 +16,6 @@ async function runTestCase (options, testData, t) { // Prepare tx and block let tx = makeTx(testData.transaction) let block = makeBlockFromEnv(testData.env) - tx._homestead = true - tx.enableHomestead = true - block.isHomestead = function () { - return true - } if (!tx.validate()) { return }