From 00251c0802c3058065e323563683516f1de8b7de Mon Sep 17 00:00:00 2001 From: Andrei Goncharov Date: Thu, 15 Jun 2017 15:30:45 +0300 Subject: [PATCH] Stable (#181) * returning proper error message (#77) * isolate test_suspenstion.py since it has test that need to make breaking changes in test environment, what leads to failing of other tests * add missing __init_/_.py to test/replay * fix import of EndpointException * fixing imports * fix import of Looper * fix in tests * cleanup * replacing warn with warning methods * Added code that will create a local pool This code will take server permeters and do all the work to get a pool up and running, including all the stewers and their keys * fixed endpoint check * using correct format for endpoint * use test runner for tests * use test runner for tests * setting resource limit for tests * moving TempStorage to common * fix imports for logging-related things * default value of service * fixing imports and setup * fixed test runner (#81) * fix imports in add_leys.py; add function for getting seeds from file * fix imports in load_test.py * merge with master * fixing imports and names * Merge master and make updates to support changes in plenum (#83) * add storing of ATTR transaction in state tree * Create StateTreeStore class * remove nonce since it is not used on server side * change logic for enc, raw and hash fields * move putting of attrib to state tree to StateTreeStore * move hashing to function * add did parameter; implement getAttr * merge logic of enc and hash value and key creation * add blank line before class name * cann state tree store's addTxn in node * store ddo in state tree * replace raising of ValueError by logger.debug since implementation not required for now * add gteDdo method * rename _addAttrib to _addAttr and fix getting of item from raw field * use isCommitted=False in StateTreeStore * push test for attrib and ddi storing in state tree * change name of tree store to more appropriate * move instantation of StateTreeStor below super init * remove type annotation for getAttr * add getters for schema and issuer key * add building of path to schema from it's namer, version and did * add tests for schema storing in state tree * rearrange test cases * add test for pathes of issuer key and revoc key * use maxsplit=5 because time in iso format uses : too * add implementation of _makeIssuerKeyPath and _makeRevocKeyPath methods * add test for storing issuer_key in state tree * implement _addIssuerKey and getIssuerKey * add parameter checking for getSchema * create lookup method and use it instead of direct accessing state to get data from it * remove unnecessary comment and todo * add some implementation thoughts * remove functionality related to ddo and revocation * do nothing if unsupported transaction passed * remove did from addTxn param list since it can be fetched from txn * use state tree for getting issuer key * make getters return string instead of bytes * swap did and key parameters in getAttr * remove conversion of keys to string as if it is parsed json, because it is not * use state tree in processGetSchemaReq instead of graph store * add isCommited for all getters * start using state trie store in processGetAttrsReq * create storeTxn method which must be used instead of direct call of storeTxnInStateTree and storeTxnInGraph * add doc-comment storeTxn * make path type markers to have a byte type instead of str * fix imports for logging-related things * fixed test runner (#81) * fix imports * remove IDPROOF from IDENTITY_TXN_TYPES * update subclasses of plenum's Requesthandler * merge with master * changes to fix some failures happening in end-to-end test * changes for request handlers * Versioning fix (#85) * [Closes SOV-905] Big Fixed: Orientdb is not installing with the latest build 0.3.16 (#86) * changing to rlp encoding * [Closes SOV-905] Big Fixed: Orientdb is not installing with the latest build 0.3.16 (#88) * Sovrin-common update -> 0.2.25 (#89) * changing config req_handler * passing correct args * Attr schema issuerkey in state trie (#90) * add storing of ATTR transaction in state tree * Create StateTreeStore class * remove nonce since it is not used on server side * change logic for enc, raw and hash fields * move putting of attrib to state tree to StateTreeStore * move hashing to function * add did parameter; implement getAttr * merge logic of enc and hash value and key creation * add blank line before class name * cann state tree store's addTxn in node * store ddo in state tree * replace raising of ValueError by logger.debug since implementation not required for now * add gteDdo method * rename _addAttrib to _addAttr and fix getting of item from raw field * use isCommitted=False in StateTreeStore * push test for attrib and ddi storing in state tree * change name of tree store to more appropriate * move instantation of StateTreeStor below super init * remove type annotation for getAttr * add getters for schema and issuer key * add building of path to schema from it's namer, version and did * add tests for schema storing in state tree * rearrange test cases * add test for pathes of issuer key and revoc key * use maxsplit=5 because time in iso format uses : too * add implementation of _makeIssuerKeyPath and _makeRevocKeyPath methods * add test for storing issuer_key in state tree * implement _addIssuerKey and getIssuerKey * add parameter checking for getSchema * create lookup method and use it instead of direct accessing state to get data from it * remove unnecessary comment and todo * add some implementation thoughts * remove functionality related to ddo and revocation * do nothing if unsupported transaction passed * remove did from addTxn param list since it can be fetched from txn * use state tree for getting issuer key * make getters return string instead of bytes * swap did and key parameters in getAttr * remove conversion of keys to string as if it is parsed json, because it is not * use state tree in processGetSchemaReq instead of graph store * add isCommited for all getters * start using state trie store in processGetAttrsReq * create storeTxn method which must be used instead of direct call of storeTxnInStateTree and storeTxnInGraph * add doc-comment storeTxn * make path type markers to have a byte type instead of str * fix imports for logging-related things * fixed test runner (#81) * fix imports * remove IDPROOF from IDENTITY_TXN_TYPES * update subclasses of plenum's Requesthandler * fix imports of Ledger * remove unused commented method * update processGetNymReq, remove txnid from it * update processGetSchemaReq, remove etxnid from it * update processGetAttrsReq * update processGetIssuerKeyReq * make StateTreeStore store last seqno * move checking of fields presence to addTxn * use cls and self instead of full class name * make lookup reutrn (None, None) if nothing found * support getting of seqno from StateTreeStore in noe * remove storing of all types of transactions except nym in storeTxnInGraph * remove _addTxnsToGraphIfNeeded since it is not used * merage with master * temp-commit * adding load testing script * raising a different exception * adding attribute store * Stp (#91) * increment version of sovrin-common * replace addSponsoredIdentity by addTrustAnchoredIdentity, remove redundant args from get*FromInbox methods calls, add logging of error for checkig replies (#92) * updated sovrin-common-dev dep version (#93) * bump sovrin-common-dev to 0.2.28 (#94) * replacing orientdb * Up common version to 0.2.29 (#95) * up common version to 23 * up common version to 0.2.29 * sync-up with plenum slow-factor (#96) * bump sovrin-common-dev==0.2.32 * sync-up the tests with plenum test code * removing commented code and fixing bug in pool request handler * bump sovrin-client-dev==0.3.45 (#97) * added time to get pool started * [Closes SOV-910] Bug Fixed: Latest build of sovrin-node can not be installed (#99) * jenkins support for 3pc-batch (#100) * made this branch publishable to pypi * fixed sovrin-client version * Bug Fixed: Update works incorrectly after versioning fix (#103) * deleting obsolete tests and changing signature of methods * fixed sovrin-client version (#104) * Rename ISSUER_KEY to CLAIM_DEF (#98) * rename issuer key and related things to claim def * add usage of signature_type to processGetClaimDefReq * remove setting of signature type in schemaDefGvt * up common version * over-riding correct method of node * bumping dependency version * Up client version to 0.3.48 Up client version to 0.3.49 (#105) * up client version to 0.3.48 * up client version 0.3.49 * Up client version to 0.3.50 * Feature Added: Deb build * Updated sovrin deps * renaming method * refactoring * Updated sovrin-common dep * deprecating orientdb * Updated sovrin-common dep * updating TestNode class hierarchy * parameterize test timeouts (#102) * add 'repeat until fail' mode for test runner * parameterize test timeouts * fix pool upgrade tests * move load test script under scripts and remove unused imports * added missing import * moved to latest sovrin-common * remove obsolete method * remove unused method and incorrect use of cache * Add missing separator to the end of transaction_*files (#111) * add missing separator to the end of transaction_*files * up common version to 0.2.37 * up client version to 0.3.54 * up client version to 0.3.57 * jenkins shared api usage update (#116) * fixed sovrin-client version * updated to use new API from shared lib * jenkins shared api usage update (master) (#115) * +1 * +1 * updated to use new API from shared lib * removed debug line * removed debug line * 3pc batch (#117) * bump sovrin-client-dev==0.3.45 (#97) * [Closes SOV-910] Bug Fixed: Latest build of sovrin-node can not be installed (#99) * Bug Fixed: Update works incorrectly after versioning fix (#103) * Rename ISSUER_KEY to CLAIM_DEF (#98) * rename issuer key and related things to claim def * add usage of signature_type to processGetClaimDefReq * remove setting of signature type in schemaDefGvt * up common version * Up client version to 0.3.48 Up client version to 0.3.49 (#105) * up client version to 0.3.48 * up client version 0.3.49 * Up client version to 0.3.50 * Updated sovrin deps * parameterize test timeouts (#102) * add 'repeat until fail' mode for test runner * parameterize test timeouts * fix pool upgrade tests * modify DomainReqHandler to make it use ClaimDef instead of IssuerKey * use get instead of [] * use checkRejects instead of checkNacks in testNonTrustyCannotCancelUpgrade * remove deprecated method * repin sovrin-common * increment sovrin-common version * fix import * skip failing tests * repin sovrin-client * increment sovrin-client * Merge branches '3pc-batch' and 'master' of github.com:sovrin-foundation/sovrin-client into 3pc-batch # Conflicts: # setup.py * updated sovrin-common version (#119) * updated sovrin-common version * pin sovrin-common version * fix usage of signature type in claim def in domain_req_handler - it should be used as a part of key instead of value (#120) * increment sovrin-client version * fix test * fix pool_upgrade tests * up commons version to 0.2.43, clients to 0.3.61 (#121) * [Closes SOV-976] Unable to create the genesis transaction files (#122) * Up commons and client version (#123) * up client version to 0.3.63 * up common version to 0.2.45 * increment sovrin-common * Updated README.md (#125) Updated README.MD to reflect build status * increment sovrin-common version * Common test code extract (#126) * Factored out common code related to tests from sovrin-node and sovrin-client to sovrin-common. * - Removed sovrin-node fixtures that duplicate sovrin-client fixtures related to client functionality. - Removed nodeThetaAdded fixture from sovrin-client since it is unused. * Updated sovrin-common-dev and sovrin-client-dev dependencies. * Up versions of dependencies (#127) * up commons to 0.2.48 * up client to 0.3.67 * schema stores data in json form * Add support of GET_NYM for load test (#128) * change default request type to NYM * add support of GET_NYM to load test script * Common test code extract (#129) * Factored out common code related to tests from sovrin-node and sovrin-client to sovrin-common. * - Removed sovrin-node fixtures that duplicate sovrin-client fixtures related to client functionality. - Removed nodeThetaAdded fixture from sovrin-client since it is unused. * Updated sovrin-common-dev and sovrin-client-dev dependencies. * Removed testNodeClass and nodeSet fixtures (now the corresponding fixtures from sovrin-client are used instead). * Removed updatedPoolTxnData, trusteeData, trusteeWallet and trustee fixtures (now the corresponding fixtures from sovrin-client are used instead). * Removed unused imports. * - Removed userWalletB fixture (now the corresponding fixture from sovrin-client is used instead). - Corrected imports and removed unused imports. * Updated sovrin-client-dev dependency. * increment sovrin-common version * increment sovrin-common * Agent generalization (#130) * import tweek for changes in client * fix getting started 2 script issues * refactor kv storage interfaces - moved kv storage and state to state repo - fixed docs - get rid of orientdb * split long methods * - Fixed a bug in Upgrader.compareVersions method. (#131) - Enhanced testVersions test to verify the fix. * increment sovrin-common * increment sovrin-common * increment sovrin-common * move test that suspends a node to the end of suite * Updated deps (#132) * remove unused dependency * increment sovrin-common * fix a case when the value is not cached * fix guide * handling non-existent schema and claim def * move request methods to request handler * refactor and fix documentation * increment sovrin-common; fix import * fix idrCacheDb name * enable debug logging in tests; increment sovrin-common * using reverted changes in state interface * increment state and sovrin-common version * increment sovrin-common * - Removed initial pool transactions files from sovrin-node project. - Removed creation of "sovrin_config.py" in "~/.sovrin" from sovrin-node project. * - Removed initial pool transactions files from sovrin-node project. (#133) - Removed creation of "sovrin_config.py" in "~/.sovrin" from sovrin-node project. * Updated sovrin-common-dev dependency. * Updated sovrin-client-dev dependency. * sync up with plenum (#136) * fix test timeout * bump sovrin-common-dev==0.2.51 * fix tests * replace import from sovrin_node by import from sovrin_common (#137) * Updated sovrin-common-dev dependency. * Updated sovrin-client-dev dependency. * Updated sovrin-client-dev dependency. * tests pass on the minimal configuration, SOV-1026 (#140) * speed up suspension tests * make testTrusteeCannotChangeVerkey passed each run * midcommit * node adds txns to graph that it receives from catchup * bumping dependency version * update test to not use orientdb * remove duplicate test * Post merge fixes (#144) * remove development options from Jenkinsfile * fix names * up common and client versions * remove cleanup of orientdb (#145) * documentation to test and change import * remove a dependency from "eventuallyAll" in load_test.py (#143) totalTimeout can be huge for load tests but 'eventually' has an assertion which limit the timeout * Remove query time counter since it is not measured (#146) * domain ledger syncs after config ledger which syncs after pool ledger, general refactoring in state storage handling encrypted and hash attribute and update test * bump dependency version * Use Role's names instead of Role's encoded values in error messages (#147) * Use Role's names instead of Role's encoded values in error messages * bumping dependency version * Fix for connection loss and refactoring (#148) * add exception handling for case when no answers received yet * up common to 0.2.64 * up client to 0.3.100 * up client version to 0.3.101 * bumped sovrin-common and sovrin-client versions * increment sovrin-common versions * Fix usage of ChunkedFileStore (#152) * use dafultFile when creating Ledger * up common version to 0.2.67, up client version to 0.3.105 * down client version to 0.3.104 because up is not needed * replace self.config.baseDir by self.basedirpath in node to make base dir be configurable for each node (#154) * Bump dependency version * Bump dependency version * Input validation: parameterize the Request class in the Node (#153) * Input validation: parameterize the Request class in the Node * bump sovrin-common-dev==0.2.70 * bump sovrin-common-dev==0.2.71 * bump sovrin-common-dev==0.2.72 * Corrected the status comment at the disabled test testUpgradeLatestUncancelledVersion. (#158) * Feature/migrations (#159) * Updated node control tool * Updated node control tool Added migration_tool * Completed migration_tool * Updated migration tool Added test for migration tool * Custom logger was deprecated * Updated migration tool * Migration test * Updated migration README * Code review fixes * Deps update * Changes needed corresponding to plenum (#161) * some refactoring * Update Jenkinsfile * Update setup.py * Update setup.py * Update Jenkinsfile * Update setup.py * Feature/migrations (#159) * Updated node control tool * Updated node control tool Added migration_tool * Completed migration_tool * Updated migration tool Added test for migration tool * Custom logger was deprecated * Updated migration tool * Migration test * Updated migration README * Code review fixes * Deps update * bump dependency version * removing some jenkins steps * Update Jenkinsfile * ATTRIB checks, INDY-71 (#163) * Enable ATTRIB with "endpoint:None" * Remove ATTRIB checks The checks were moved to the validation logic * bump sovrin-common-dev==0.2.76 * Test migrations (#164) * Revert "Test migrations (#164)" (#165) This reverts commit f263447f03781a8857fcb768011a629ac8169c59. * Deps update * bump sovrin-common-dev==0.2.80 * up commons version to 0.2.81 * up client version to 0.3.125 * increment plenum version * removing an unused parameter and adding a test for indy-155 (#172) * Non empty verkey (#174) * update tests for non-empty verkey * bump dependency version * change idr cache to store batches in list to avoid problems with batches that result to same state roots (#175) * change idr cache to store batches in list to avoid problems with batches that result to same state roots * bump dependency version * bugfix and update test * update test * increment sovrin-common * bump sovrin-common-dev==0.2.86 * bump dependency version * set default loglevel to INFO (INDY-197) * set default loglevel to INFO (INDY-197) * repin plenum --- scripts/start_sovrin_node | 1 + scripts/upgrade_sovrin_node.bat | 2 +- setup.py | 4 +- sovrin_node/persistence/idr_cache.py | 42 ++-- sovrin_node/server/domain_req_handler.py | 4 +- sovrin_node/server/node.py | 6 +- .../test_requests_post_new_node_catchup.py | 24 +- sovrin_node/test/conftest.py | 3 +- sovrin_node/test/did/helper.py | 4 +- .../did/test_did_with_abbreviated_verkey.py | 5 +- .../test/did/test_did_with_full_verkey.py | 13 +- .../test/did/test_did_with_no_verkey.py | 9 +- sovrin_node/test/helper.py | 5 +- .../test_successive_batch_no_state_change.py | 216 ++++++++++++++++++ .../test/suspension/test_suspension.py | 20 +- sovrin_node/test/upgrade/conftest.py | 3 +- .../test/upgrade/test_migration_tool.py | 2 +- 17 files changed, 309 insertions(+), 54 deletions(-) create mode 100644 sovrin_node/test/replay/test_successive_batch_no_state_change.py diff --git a/scripts/start_sovrin_node b/scripts/start_sovrin_node index 19d082863..71788753c 100755 --- a/scripts/start_sovrin_node +++ b/scripts/start_sovrin_node @@ -35,6 +35,7 @@ if __name__ == "__main__": Logger().setupRaet(RAETVerbosity, RAETLogFile) logger = getlogger() + logger.setLevel(config.logLevel) logger.debug("You can find logs in {}".format(logFileName)) vars = [var for var in os.environ.keys() if var.startswith("SOVRIN")] diff --git a/scripts/upgrade_sovrin_node.bat b/scripts/upgrade_sovrin_node.bat index 866512e51..f6ff21d4b 100644 --- a/scripts/upgrade_sovrin_node.bat +++ b/scripts/upgrade_sovrin_node.bat @@ -14,4 +14,4 @@ pip install --upgrade --no-cache-dir sovrin-node%SOVRIN_NODE_PACKAGE_POSTFIX%==" SET RET=%ERRORLEVEL% IF NOT "%RET%"=="0" ( echo "Upgrade to version %VERS% failed %RET%" -) +) \ No newline at end of file diff --git a/setup.py b/setup.py index 771a467fc..05c937b16 100644 --- a/setup.py +++ b/setup.py @@ -73,9 +73,9 @@ def run(self): data_files=[( (BASE_DIR, ['data/nssm_original.exe']) )], - install_requires=['sovrin-common==0.2.11', 'python-dateutil'], + install_requires=['sovrin-common==0.2.12', 'python-dateutil'], setup_requires=['pytest-runner'], - tests_require=['pytest', 'sovrin-client==0.3.20'], + tests_require=['pytest', 'sovrin-client==0.3.21'], scripts=['scripts/start_sovrin_node', 'scripts/node_control_tool.py', 'scripts/migration_tool.py', diff --git a/sovrin_node/persistence/idr_cache.py b/sovrin_node/persistence/idr_cache.py index d8fa1b861..d59cef529 100644 --- a/sovrin_node/persistence/idr_cache.py +++ b/sovrin_node/persistence/idr_cache.py @@ -25,15 +25,18 @@ def __init__(self, name, keyValueStorage: KeyValueStorage): logger.debug('Initializing identity cache {}'.format(name)) self._keyValueStorage = keyValueStorage self._name = name - # OrderedDict where key is the state root after batch and value is a - # dictionary similar to cache which can be queried like the - # database, i.e `self._db`. Keys (state roots are purged) when they - # get committed or reverted. - self.unCommitted = OrderedDict() # type: Dict[bytes, OrderedDict] + # List of Tuples where first items is the state root after batch and + # second item is a dictionary similar to cache which can be queried + # like the database, i.e `self._db`. Keys (state roots are purged) + # when they get committed or reverted. + self.unCommitted = [] # type: List[Tuple[bytes, OrderedDict]] # Relevant NYMs operation done in current batch, in order self.currentBatchOps = [] # type: List[Tuple] + def __repr__(self): + return self._name + @staticmethod def encodeVerkey(verkey): if verkey is None: @@ -71,7 +74,7 @@ def get(self, idr, isCommitted=True): else: # Looking for uncommitted values, iterating over `self.unCommitted` # in reverse to get the latest value - for i, cache in reversed(self.unCommitted.items()): + for _, cache in reversed(self.unCommitted): if idr in cache: value = cache[idr] break @@ -91,18 +94,29 @@ def close(self): self._keyValueStorage.close() def currentBatchCreated(self, stateRoot): - self.unCommitted[stateRoot] = OrderedDict(self.currentBatchOps) + self.unCommitted.append((stateRoot, OrderedDict(self.currentBatchOps))) self.currentBatchOps = [] - def batchRejected(self, stateRoot=None): - if stateRoot: - self.unCommitted[stateRoot] = OrderedDict(self.currentBatchOps) - else: - self.currentBatchOps = [] + def batchRejected(self): + # Batches are always rejected from end of `self.unCommitted` + self.currentBatchOps = [] + self.unCommitted = self.unCommitted[:-1] def onBatchCommitted(self, stateRoot): - self._keyValueStorage.setBatch([(idr, val) for idr, val in self.unCommitted[stateRoot].items()]) - self.unCommitted.pop(stateRoot) + # Commit an already created batch + if self.unCommitted: + assert self.unCommitted[0][0] == stateRoot, 'The first created batch has ' \ + 'not been committed or ' \ + 'reverted and yet another ' \ + 'batch is trying to be ' \ + 'committed, {} {}'.format( + self.unCommitted[0][0], stateRoot) + self._keyValueStorage.setBatch([(idr, val) for idr, val in + self.unCommitted[0][1].items()]) + self.unCommitted = self.unCommitted[1:] + else: + logger.warning('{} is trying to commit a batch with state root {} ' + 'but no uncommitted found'.format(self, stateRoot)) def setVerkey(self, idr, verkey): # This method acts as if guardianship is being terminated. diff --git a/sovrin_node/server/domain_req_handler.py b/sovrin_node/server/domain_req_handler.py index a2bce537d..caa2a7210 100644 --- a/sovrin_node/server/domain_req_handler.py +++ b/sovrin_node/server/domain_req_handler.py @@ -39,8 +39,8 @@ def __init__(self, ledger, state, requestProcessor, idrCache, attributeStore): def onBatchCreated(self, stateRoot): self.idrCache.currentBatchCreated(stateRoot) - def onBatchRejected(self, stateRoot=None): - self.idrCache.batchRejected(stateRoot) + def onBatchRejected(self): + self.idrCache.batchRejected() def _updateStateWithSingleTxn(self, txn, isCommitted=False): typ = txn.get(TXN_TYPE) diff --git a/sovrin_node/server/node.py b/sovrin_node/server/node.py index 7b727571a..cbe43fbd0 100644 --- a/sovrin_node/server/node.py +++ b/sovrin_node/server/node.py @@ -443,8 +443,8 @@ def onBatchCreated(self, ledgerId, stateRoot): else: super().onBatchCreated(ledgerId, stateRoot) - def onBatchRejected(self, ledgerId, stateRoot=None): + def onBatchRejected(self, ledgerId): if ledgerId == CONFIG_LEDGER_ID: - self.configReqHandler.onBatchRejected(stateRoot) + self.configReqHandler.onBatchRejected() else: - super().onBatchRejected(ledgerId, stateRoot) + super().onBatchRejected(ledgerId) diff --git a/sovrin_node/test/catchup/test_requests_post_new_node_catchup.py b/sovrin_node/test/catchup/test_requests_post_new_node_catchup.py index 0e6507b59..c69bec69f 100644 --- a/sovrin_node/test/catchup/test_requests_post_new_node_catchup.py +++ b/sovrin_node/test/catchup/test_requests_post_new_node_catchup.py @@ -11,14 +11,34 @@ from sovrin_client.test.helper import getClientAddedWithRole from sovrin_node.test.conftest import nodeThetaAdded +from sovrin_node.test.did.helper import updateSovrinIdrWithVerkey from sovrin_node.test.helper import TestNode, addRawAttribute, getAttribute +@pytest.fixture(scope="module") +def some_transactions_done(looper, nodeSet, tdirWithPoolTxns, trustee, + trusteeWallet): + new_c, new_w = getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, + trustee, trusteeWallet, 'some_name', + addVerkey=False) + new_idr = new_w.defaultId + updateSovrinIdrWithVerkey(looper, trusteeWallet, trustee, + new_idr, new_w.getVerkey(new_idr)) + # TODO: Since empty verkey and absence of verkey are stored in the ledger + # in the same manner, this fails during catchup since the nodes that + # processed the transaction saw verkey as `''` but while deserialising the + # ledger they cannot differentiate between None and empty string. + # updateSovrinIdrWithVerkey(looper, new_w, new_c, + # new_idr, '') + + def test_new_node_catchup_update_projection(looper, tdirWithPoolTxns, tdirWithDomainTxnsUpdated, nodeSet, tconf, trustee, trusteeWallet, - allPluginsPath): + allPluginsPath, + some_transactions_done + ): """ A node which receives txns from catchup updates both ledger and projection 4 nodes start up and some txns happen, after txns are done, new node joins @@ -85,7 +105,7 @@ def check_sizes(nodes): trust_anchors.append(getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, trustee, trusteeWallet, - 'TA'+str(i), TRUST_ANCHOR, + 'TA'+str(i), role=TRUST_ANCHOR, client_connects_to=len(other_nodes))) attributes.append((randomString(6), randomString(10))) addRawAttribute(looper, *trust_anchors[-1], *attributes[-1], diff --git a/sovrin_node/test/conftest.py b/sovrin_node/test/conftest.py index b40d793fc..32e47529c 100644 --- a/sovrin_node/test/conftest.py +++ b/sovrin_node/test/conftest.py @@ -69,7 +69,8 @@ def nodeThetaAdded(looper, nodeSet, tdirWithPoolTxns, tconf, steward, newSteward, newStewardWallet = getClientAddedWithRole(nodeSet, tdir, looper, steward, stewardWallet, - newStewardName, STEWARD) + newStewardName, + role=STEWARD) sigseed = randomString(32).encode() nodeSigner = SimpleSigner(seed=sigseed) diff --git a/sovrin_node/test/did/helper.py b/sovrin_node/test/did/helper.py index 625aecd04..78d1b02d8 100644 --- a/sovrin_node/test/did/helper.py +++ b/sovrin_node/test/did/helper.py @@ -39,8 +39,7 @@ def updateWalletIdrWithFullVerkeySigner(wallet, idr, signer): checkFullVerkeySize(wallet.getVerkey(idr)) -def updateSovrinIdrWithFullKey(looper, senderWallet, senderClient, - ownerWallet, idr, fullKey): +def updateSovrinIdrWithVerkey(looper, senderWallet, senderClient, idr, fullKey): idy = Identity(identifier=idr, verkey=fullKey) senderWallet.updateTrustAnchoredIdentity(idy) # TODO: What if the request fails, there must be some rollback mechanism @@ -53,7 +52,6 @@ def chk(): timeout = plenumWaits.expectedReqAckQuorumTime() looper.run(eventually(chk, retryWait=1, timeout=timeout)) - return ownerWallet def fetchFullVerkeyFromSovrin(looper, senderWallet, senderClient, diff --git a/sovrin_node/test/did/test_did_with_abbreviated_verkey.py b/sovrin_node/test/did/test_did_with_abbreviated_verkey.py index 60773fada..9e5190a61 100644 --- a/sovrin_node/test/did/test_did_with_abbreviated_verkey.py +++ b/sovrin_node/test/did/test_did_with_abbreviated_verkey.py @@ -19,7 +19,7 @@ from sovrin_common.identity import Identity from sovrin_node.test.did.conftest import pf from sovrin_node.test.did.helper import chkVerifyForRetrievedIdentity, \ - updateWalletIdrWithFullKeySigner, updateSovrinIdrWithFullKey, \ + updateWalletIdrWithFullKeySigner, updateSovrinIdrWithVerkey, \ fetchFullVerkeyFromSovrin, checkAbbrVerkeySize, checkDidSize, \ updateWalletIdrWithFullVerkeySigner from sovrin_client.test.helper import createNym @@ -57,8 +57,7 @@ def didUpdatedWithFullVerkey(didAddedWithAbbrvVerkey, looper, trustAnchor, trustAnchorWallet, abbrevIdr, newFullKey, newFullKeySigner, wallet, client): """{ type: NYM, dest: , verkey: }""" - updateSovrinIdrWithFullKey(looper, wallet, client, wallet, - abbrevIdr, newFullKey) + updateSovrinIdrWithVerkey(looper, wallet, client, abbrevIdr, newFullKey) updateWalletIdrWithFullVerkeySigner(wallet, abbrevIdr, newFullKeySigner) diff --git a/sovrin_node/test/did/test_did_with_full_verkey.py b/sovrin_node/test/did/test_did_with_full_verkey.py index 14f174dc6..ddfed4f5a 100644 --- a/sovrin_node/test/did/test_did_with_full_verkey.py +++ b/sovrin_node/test/did/test_did_with_full_verkey.py @@ -18,7 +18,7 @@ from sovrin_common.identity import Identity from sovrin_node.test.did.conftest import pf from sovrin_node.test.did.helper import chkVerifyForRetrievedIdentity, \ - updateSovrinIdrWithFullKey, \ + updateSovrinIdrWithVerkey, \ fetchFullVerkeyFromSovrin, checkFullVerkeySize, \ updateWalletIdrWithFullVerkeySigner from sovrin_client.test.helper import createNym @@ -47,16 +47,13 @@ def didUpdatedWithFullVerkey(didAddedWithFullVerkey, looper, trustAnchor, trustAnchorWallet, fullKeyIdr, newFullKey, newFullKeySigner, wallet, client): """{ type: NYM, dest: , verkey: }""" - # updateSovrinIdrWithFullKey(looper, trustAnchorWallet, trustAnchor, wallet, - # fullKeyIdr, newFullKey) - updateSovrinIdrWithFullKey(looper, wallet, client, wallet, - fullKeyIdr, newFullKey) + updateSovrinIdrWithVerkey(looper, wallet, client, fullKeyIdr, newFullKey) updateWalletIdrWithFullVerkeySigner(wallet, fullKeyIdr, newFullKeySigner) @pf -def newVerkeyFetched(didAddedWithFullVerkey, looper, trustAnchor, trustAnchorWallet, - fullKeyIdr, wallet): +def newVerkeyFetched(didAddedWithFullVerkey, looper, trustAnchor, + trustAnchorWallet, fullKeyIdr, wallet): """{ type: GET_NYM, dest: }""" fetchFullVerkeyFromSovrin(looper, trustAnchorWallet, trustAnchor, wallet, fullKeyIdr) @@ -71,7 +68,7 @@ def testRetrieveFullVerkey(didAddedWithFullVerkey, looper, trustAnchor, """{ type: GET_NYM, dest: }""" identity = Identity(identifier=fullKeyIdr) req = trustAnchorWallet.requestIdentity(identity, - sender=trustAnchorWallet.defaultId) + sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): diff --git a/sovrin_node/test/did/test_did_with_no_verkey.py b/sovrin_node/test/did/test_did_with_no_verkey.py index 0819968c6..e128aac2b 100644 --- a/sovrin_node/test/did/test_did_with_no_verkey.py +++ b/sovrin_node/test/did/test_did_with_no_verkey.py @@ -32,7 +32,7 @@ from sovrin_common.identity import Identity from sovrin_node.test.did.conftest import pf from sovrin_node.test.did.helper import chkVerifyForRetrievedIdentity, \ - updateSovrinIdrWithFullKey + updateSovrinIdrWithVerkey from sovrin_client.test.helper import createNym from plenum.test import waits as plenumWaits @@ -48,8 +48,8 @@ def didAddedWithoutVerkey(addedTrustAnchor, looper, trustAnchor, trustAnchorWall def didUpdatedWithVerkey(didAddedWithoutVerkey, looper, trustAnchor, trustAnchorWallet, noKeyIdr, wallet): """{ type: NYM, dest: , verkey: }""" - updateSovrinIdrWithFullKey(looper, trustAnchorWallet, trustAnchor, wallet, - noKeyIdr, wallet.getVerkey(noKeyIdr)) + updateSovrinIdrWithVerkey(looper, trustAnchorWallet, trustAnchor, + noKeyIdr, wallet.getVerkey(noKeyIdr)) @pf @@ -82,7 +82,8 @@ def testRetrieveEmptyVerkey(didAddedWithoutVerkey, looper, trustAnchor, trustAnchorWallet, noKeyIdr): """{ type: GET_NYM, dest: }""" identity = Identity(identifier=noKeyIdr) - req = trustAnchorWallet.requestIdentity(identity, sender=trustAnchorWallet.defaultId) + req = trustAnchorWallet.requestIdentity(identity, + sender=trustAnchorWallet.defaultId) trustAnchor.submitReqs(req) def chk(): diff --git a/sovrin_node/test/helper.py b/sovrin_node/test/helper.py index 3970e915d..00fbcb304 100644 --- a/sovrin_node/test/helper.py +++ b/sovrin_node/test/helper.py @@ -233,7 +233,7 @@ class TestUpgrader(Upgrader): Node.processRequest, Node.processPropagate, Node.propagate, Node.forward, Node.send, Node.processInstanceChange, Node.checkPerformance, Node.getReplyFromLedger]) -class TestNode(TempStorage, Node, TestNodeCore): +class TestNode(TempStorage, TestNodeCore, Node): def __init__(self, *args, **kwargs): Node.__init__(self, *args, **kwargs) TestNodeCore.__init__(self, *args, **kwargs) @@ -243,6 +243,9 @@ def getUpgrader(self): return TestUpgrader(self.id, self.name, self.dataLocation, self.config, self.configLedger) + def getDomainReqHandler(self): + return Node.getDomainReqHandler(self) + def onStopping(self, *args, **kwargs): if self.cleanupOnStopping: self.cleanupDataLocation() diff --git a/sovrin_node/test/replay/test_successive_batch_no_state_change.py b/sovrin_node/test/replay/test_successive_batch_no_state_change.py new file mode 100644 index 000000000..ecabac206 --- /dev/null +++ b/sovrin_node/test/replay/test_successive_batch_no_state_change.py @@ -0,0 +1,216 @@ +import types +from collections import OrderedDict + +import pytest + +from plenum.common.constants import VERKEY +from plenum.common.signer_simple import SimpleSigner +from plenum.common.types import PrePrepare, Commit +from plenum.common.signer_did import DidSigner +from plenum.common.util import randomString +from plenum.test.delayers import cDelay +from plenum.test.helper import waitForSufficientRepliesForRequests +from plenum.test.test_node import getPrimaryReplica +from sovrin_client.client.wallet.wallet import Wallet +from sovrin_common.identity import Identity +from stp_core.loop.eventually import eventually + + +@pytest.fixture(scope="module") +def tconf(tconf, request): + # Delaying performance checks since delaying 3PC messages in the test + old_freq = tconf.PerfCheckFreq + tconf.PerfCheckFreq = 40 + + def reset(): + tconf.PerfCheckFreq = old_freq + + request.addfinalizer(reset) + return tconf + + +def test_successive_batch_do_no_change_state(looper, tdirWithPoolTxns, + tdirWithDomainTxnsUpdated, + tconf, nodeSet, + trustee, trusteeWallet): + """ + Send 2 NYM txns in different batches such that the second batch does not + change state so that state root remains same, but keep the identifier + and reqId different. Make sure the first request is not ordered by the + primary before PRE-PREPARE for the second is sent. + Also check reject and commit + :return: + """ + prim_node = getPrimaryReplica(nodeSet, 0).node + other_nodes = [n for n in nodeSet if n != prim_node] + # Delay only first PRE-PREPARE + pp_seq_no_to_delay = 1 + + def specific_pre_prepare(wrappedMsg): + nonlocal pp_seq_no_to_delay + msg, sender = wrappedMsg + if isinstance(msg, PrePrepare) and \ + msg.instId == 0 and \ + msg.ppSeqNo == pp_seq_no_to_delay: + return 5 + + def delay_commits(wrappedMsg): + msg, sender = wrappedMsg + if isinstance(msg, Commit) and msg.instId == 0: + return 10 + + def new_identity(): + wallet = Wallet(randomString(5)) + signer = DidSigner() + new_idr, _ = wallet.addIdentifier(signer=signer) + verkey = wallet.getVerkey(new_idr) + idy = Identity(identifier=new_idr, + verkey=verkey, + role=None) + return idy + + def submit_id_req(idy): + nonlocal all_reqs + trusteeWallet.updateTrustAnchoredIdentity(idy) + reqs = trusteeWallet.preparePending() + all_reqs.extend(reqs) + trustee.submitReqs(*reqs) + + def check_verkey(i, vk): + for node in nodeSet: + data = node.reqHandler.idrCache.getNym(i, isCommitted=True) + assert data[VERKEY] == vk + + def check_uncommitted(count): + for node in nodeSet: + assert len(node.reqHandler.idrCache.unCommitted) == count + + for node in other_nodes: + node.nodeIbStasher.delay(specific_pre_prepare) + + idy = new_identity() + new_idr = idy.identifier + verkey = idy.verkey + + all_reqs = [] + + # Setting the same verkey twice but in different batches with different + # request ids + for _ in range(3): + submit_id_req(idy) + looper.runFor(.2) + + waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs, + add_delay_to_timeout=5) + + # Number of uncommitted entries is 0 + looper.run(eventually(check_uncommitted, 0)) + + check_verkey(new_idr, verkey) + + pp_seq_no_to_delay = 4 + for node in other_nodes: + node.nodeIbStasher.delay(specific_pre_prepare) + + # Setting the verkey to `x`, then `y` and then back to `x` but in different + # batches with different with different request ids. The idea is to change + # state root to `t` then `t'` and then back to `t` and observe that no + # errors are encountered + + idy = new_identity() + new_idr = idy.identifier + verkey = idy.verkey + submit_id_req(idy) + looper.runFor(.2) + + new_verkey = SimpleSigner().verkey + idy.verkey = new_verkey + submit_id_req(idy) + looper.runFor(.2) + + idy.verkey = verkey + submit_id_req(idy) + looper.runFor(.2) + + waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs, + add_delay_to_timeout=5) + + # Number of uncommitted entries is 0 + looper.run(eventually(check_uncommitted, 0)) + + check_verkey(new_idr, verkey) + + # Dleay COMMITs so that IdrCache can be checked for correct + # number of entries + + uncommitteds = {} + methods = {} + for node in nodeSet: + node.nodeIbStasher.delay(delay_commits) + + cache = node.reqHandler.idrCache + uncommitteds[cache._name] = [] + + cre = cache.currentBatchCreated + com = cache.onBatchCommitted + methods[cache._name] = (cre, com) + + # Patch methods to record and check roots after commit + + def patched_cre(self, stateRoot): + uncommitteds[self._name].append(stateRoot) + return methods[self._name][0](stateRoot) + + def patched_com(self, stateRoot): + assert uncommitteds[self._name][0] == stateRoot + rv = methods[self._name][1](stateRoot) + uncommitteds[self._name] = uncommitteds[self._name][1:] + return rv + + cache.currentBatchCreated = types.MethodType(patched_cre, cache) + cache.onBatchCommitted = types.MethodType(patched_com, cache) + + # Set verkey of multiple identities + more = 5 + keys = {} + for _ in range(more): + idy = new_identity() + keys[idy.identifier] = idy.verkey + submit_id_req(idy) + looper.runFor(.01) + + # Correct number of uncommitted entries + looper.run(eventually(check_uncommitted, more, retryWait=1)) + + waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs, + add_delay_to_timeout=10) + + # Number of uncommitted entries is 0 + looper.run(eventually(check_uncommitted, 0)) + + # The verkeys are correct + for i, v in keys.items(): + check_verkey(i, v) + + keys = {} + for _ in range(3): + idy = new_identity() + keys[idy.identifier] = idy.verkey + submit_id_req(idy) + looper.runFor(.01) + + # Correct number of uncommitted entries + looper.run(eventually(check_uncommitted, 3, retryWait=1)) + + # Check batch reject + for node in nodeSet: + cache = node.reqHandler.idrCache + initial = cache.unCommitted + cache.batchRejected() + # After reject, last entry is removed + assert cache.unCommitted == initial[:-1] + root = cache.unCommitted[0][0] + cache.onBatchCommitted(root) + # Calling commit with same root results in Assertion error + with pytest.raises(AssertionError): + cache.onBatchCommitted(root) diff --git a/sovrin_node/test/suspension/test_suspension.py b/sovrin_node/test/suspension/test_suspension.py index a8f5504aa..ed4989bcb 100644 --- a/sovrin_node/test/suspension/test_suspension.py +++ b/sovrin_node/test/suspension/test_suspension.py @@ -1,5 +1,6 @@ import pytest +from plenum.common.signer_did import DidSigner from sovrin_node.test.suspension.helper import sendChangeVerkey, checkIdentityRequestFailed, \ checkIdentityRequestSucceed, sendSuspendRole, changeVerkey, suspendRole from stp_core.loop.eventually import eventually @@ -22,32 +23,35 @@ @pytest.fixture(scope="module") def anotherTrustee(nodeSet, tdir, looper, trustee, trusteeWallet): return getClientAddedWithRole(nodeSet, tdir, looper, - trustee, trusteeWallet, 'newTrustee', TRUSTEE) + trustee, trusteeWallet, 'newTrustee', + role=TRUSTEE) @pytest.fixture(scope="module") def anotherTGB(nodeSet, tdir, looper, trustee, trusteeWallet): return getClientAddedWithRole(nodeSet, tdir, looper, - trustee, trusteeWallet, 'newTGB', TGB) + trustee, trusteeWallet, 'newTGB', role=TGB) @pytest.fixture(scope="module") def anotherSteward(nodeSet, tdir, looper, trustee, trusteeWallet): return getClientAddedWithRole(nodeSet, tdir, looper, - trustee, trusteeWallet, 'newSteward', STEWARD) + trustee, trusteeWallet, 'newSteward', + role=STEWARD) @pytest.fixture(scope="module") def anotherSteward1(nodeSet, tdir, looper, trustee, trusteeWallet): return getClientAddedWithRole(nodeSet, tdir, looper, - trustee, trusteeWallet, 'newSteward1', STEWARD) + trustee, trusteeWallet, 'newSteward1', + role=STEWARD) @pytest.fixture(scope="module") def anotherTrustAnchor(nodeSet, tdir, looper, trustee, trusteeWallet): return getClientAddedWithRole(nodeSet, tdir, looper, trustee, trusteeWallet, 'newTrustAnchor', - TRUST_ANCHOR) + role=TRUST_ANCHOR) def testTrusteeAddingAnotherTrustee(anotherTrustee): @@ -111,10 +115,12 @@ def testTrusteeCannotChangeVerkey(trustee, trusteeWallet, looper, nodeSet, for identity in (anotherTrustee, anotherTGB, anotherSteward, anotherTrustAnchor): # Trustee cannot change verkey _, wallet = identity - changeVerkey(looper, trustee, trusteeWallet, wallet.defaultId, '', + signer = DidSigner() + changeVerkey(looper, trustee, trusteeWallet, wallet.defaultId, + signer.verkey, nAckReasonContains='TRUSTEE cannot update verkey') # Identity owner can change verkey - changeVerkey(looper, *identity, wallet.defaultId, '') + changeVerkey(looper, *identity, wallet.defaultId, signer.verkey) # Keep the test below at the end of the suite since it will make one of the diff --git a/sovrin_node/test/upgrade/conftest.py b/sovrin_node/test/upgrade/conftest.py index b88bfc1e2..7eb812c27 100644 --- a/sovrin_node/test/upgrade/conftest.py +++ b/sovrin_node/test/upgrade/conftest.py @@ -25,7 +25,6 @@ def validUpgrade(nodeIds, tconf): for i in nodeIds: schedule[i] = datetime.isoformat(startAt) startAt = startAt + timedelta(seconds=acceptableDiff + 3) - # TODO select or create a timeout from 'waits' if it is needed return dict(name='upgrade-13', version=bumpedVersion(), action=START, schedule=schedule, sha256='aad1242', timeout=1) @@ -57,4 +56,4 @@ def invalidUpgrade(nodeIds, tconf): @pytest.fixture(scope="module") def steward(nodeSet, tdir, looper, trustee, trusteeWallet): return getClientAddedWithRole(nodeSet, tdir, looper, - trustee, trusteeWallet, 'newSteward', STEWARD) + trustee, trusteeWallet, 'newSteward', STEWARD) \ No newline at end of file diff --git a/sovrin_node/test/upgrade/test_migration_tool.py b/sovrin_node/test/upgrade/test_migration_tool.py index 0ce008357..5bfae84ae 100644 --- a/sovrin_node/test/upgrade/test_migration_tool.py +++ b/sovrin_node/test/upgrade/test_migration_tool.py @@ -1,4 +1,4 @@ -import migration_tool +import scripts.migration_tool as migration_tool TEST_MIGRATION_SCRIPTS = ['0_3_100_0_test', '10_2_1_test1', '0_9_2_test3', '0_3_100_1_test4', '0_1_10_2_test5']