From ddee8ba988461f4e845226a710c78ae5f7331534 Mon Sep 17 00:00:00 2001 From: Lovesh Harchandani Date: Mon, 29 Jan 2018 03:08:11 -0700 Subject: [PATCH] changes as per new plenum (#535) * changes as per new plenum Signed-off-by: Lovesh * bump plenum version Signed-off-by: ashcherbakov --- indy_client/test/cli/conftest.py | 2 +- .../test/cli/test_send_get_nym_validation.py | 9 +- indy_client/test/conftest.py | 6 +- indy_client/test/test_nym_attrib.py | 3 +- indy_common/types.py | 24 ++-- indy_common/util.py | 3 +- indy_node/server/client_authn.py | 28 +--- indy_node/server/domain_req_handler.py | 10 +- indy_node/server/node.py | 121 ++---------------- indy_node/test/conftest.py | 2 +- .../test_successive_batch_no_state_change.py | 3 - setup.py | 2 +- 12 files changed, 51 insertions(+), 162 deletions(-) diff --git a/indy_client/test/cli/conftest.py b/indy_client/test/cli/conftest.py index ad46547c7..26a6fe41b 100644 --- a/indy_client/test/cli/conftest.py +++ b/indy_client/test/cli/conftest.py @@ -125,7 +125,7 @@ def susanMap(): return getDefaultUserMap("Susan") -@pytest.fixture(scope="module") +@pytest.fixture(scope="module") # noqa def faberMap(agentIpAddress, faberAgentPort): ha = "{}:{}".format(agentIpAddress, faberAgentPort) return {'inviter': 'Faber College', diff --git a/indy_client/test/cli/test_send_get_nym_validation.py b/indy_client/test/cli/test_send_get_nym_validation.py index 225a2187b..2a691860a 100644 --- a/indy_client/test/cli/test_send_get_nym_validation.py +++ b/indy_client/test/cli/test_send_get_nym_validation.py @@ -10,7 +10,8 @@ from indy_node.test.helper import check_str_is_base58_compatible CURRENT_VERKEY_FOR_NYM = 'Current verkey for NYM {dest} is {verkey}' -CURRENT_VERKEY_FOR_NYM_WITH_ROLE = 'Current verkey for NYM {dest} is {verkey} with role {role}' +CURRENT_VERKEY_FOR_NYM_WITH_ROLE = 'Current verkey for NYM {dest} is ' \ + '{verkey} with role {role}' CURRENT_VERKEY_IS_SAME_AS_IDENTIFIER = \ 'Current verkey is same as identifier {dest}' NYM_NOT_FOUND = 'NYM {dest} not found' @@ -48,7 +49,8 @@ def test_get_nym_returns_role( be, do, poolNodesStarted, trusteeCli): current_role = Roles.TRUST_ANCHOR uuidIdentifier, abbrevVerkey = createHalfKeyIdentifierAndAbbrevVerkey() - addNym(be, do, trusteeCli, idr=uuidIdentifier, verkey=abbrevVerkey, role=current_role) + addNym(be, do, trusteeCli, idr=uuidIdentifier, verkey=abbrevVerkey, + role=current_role) parameters = { 'dest': uuidIdentifier, @@ -59,7 +61,8 @@ def test_get_nym_returns_role( do('send GET_NYM dest={dest}', mapper=parameters, expect=CURRENT_VERKEY_FOR_NYM_WITH_ROLE, within=2) new_role = '' - addNym(be, do, trusteeCli, idr=uuidIdentifier, verkey=abbrevVerkey, role=new_role) + addNym(be, do, trusteeCli, idr=uuidIdentifier, verkey=abbrevVerkey, + role=new_role) do('send GET_NYM dest={dest}', mapper=parameters, expect=CURRENT_VERKEY_FOR_NYM, within=2) diff --git a/indy_client/test/conftest.py b/indy_client/test/conftest.py index ab80f2242..96bed99f8 100644 --- a/indy_client/test/conftest.py +++ b/indy_client/test/conftest.py @@ -24,7 +24,6 @@ import pytest -from plenum.common.signer_simple import SimpleSigner from plenum.common.constants import VERKEY, ALIAS, STEWARD, TXN_ID, TRUSTEE, TYPE from indy_client.client.wallet.wallet import Wallet @@ -33,8 +32,7 @@ from indy_client.test.cli.helper import newCLI, addTrusteeTxnsToGenesis, addTxnToGenesisFile from indy_node.test.helper import makePendingTxnsRequest, buildStewardClient, \ TestNode -from indy_client.test.helper import addRole, \ - genTestClient, TestClient, createNym +from indy_client.test.helper import addRole, genTestClient, TestClient, createNym # noinspection PyUnresolvedReferences from plenum.test.conftest import tdir, client_tdir, nodeReg, up, ready, \ @@ -43,7 +41,7 @@ poolTxnNodeNames, allPluginsPath, tdirWithNodeKeepInited, tdirWithPoolTxns, \ poolTxnStewardData, poolTxnStewardNames, getValueFromModule, \ txnPoolNodesLooper, patchPluginManager, tdirWithClientPoolTxns, \ - warncheck, warnfilters as plenum_warnfilters, setResourceLimits + warncheck, warnfilters as plenum_warnfilters, setResourceLimits, do_post_node_creation # noinspection PyUnresolvedReferences from indy_common.test.conftest import tconf, general_conf_tdir, poolTxnTrusteeNames, \ diff --git a/indy_client/test/test_nym_attrib.py b/indy_client/test/test_nym_attrib.py index ac0f99b4b..179c503ba 100644 --- a/indy_client/test/test_nym_attrib.py +++ b/indy_client/test/test_nym_attrib.py @@ -424,7 +424,8 @@ def testStewardCannotAddUsersAttribute(nodeSet, looper, steward, checkRejects, steward, reqs[0].reqId, - "UnauthorizedClientRequest('Only identity owner/guardian can add attribute for that identity'", + "UnauthorizedClientRequest('Only identity owner/guardian can " + "add attribute for that identity'", retryWait=1, timeout=timeout)) diff --git a/indy_common/types.py b/indy_common/types.py index 8981690e3..cbe52e0ed 100644 --- a/indy_common/types.py +++ b/indy_common/types.py @@ -4,8 +4,7 @@ from plenum.common.constants import TARGET_NYM, NONCE, RAW, ENC, HASH, NAME, VERSION, ORIGIN, FORCE from plenum.common.messages.fields import IterableField, AnyMapField, \ - NonEmptyStringField, LedgerIdField as PLedgerIdField, \ - LedgerInfoField as PLedgerInfoField + NonEmptyStringField from plenum.common.messages.node_message_factory import node_message_factory from plenum.common.messages.message_base import MessageValidator @@ -239,21 +238,22 @@ class ClientMessageValidator(PClientMessageValidator): ) -class LedgerIdField(PLedgerIdField): - ledger_ids = PLedgerIdField.ledger_ids + (CONFIG_LEDGER_ID,) - - -class LedgerInfoField(PLedgerInfoField): - _ledger_id_class = LedgerIdField +# THE CODE BELOW MIGHT BE NEEDED IN THE FUTURE, THEREFORE KEEPING IT +# class LedgerIdField(PLedgerIdField): +# ledger_ids = PLedgerIdField.ledger_ids + (CONFIG_LEDGER_ID,) +# +# +# class LedgerInfoField(PLedgerInfoField): +# _ledger_id_class = LedgerIdField # TODO: it is a workaround which helps extend some fields from # downstream projects, should be removed after we find a better way # to do this -node_message_factory.update_schemas_by_field_type( - PLedgerIdField, LedgerIdField) -node_message_factory.update_schemas_by_field_type( - PLedgerInfoField, LedgerInfoField) +# node_message_factory.update_schemas_by_field_type( +# PLedgerIdField, LedgerIdField) +# node_message_factory.update_schemas_by_field_type( +# PLedgerInfoField, LedgerInfoField) class SafeRequest(Request, ClientMessageValidator): diff --git a/indy_common/util.py b/indy_common/util.py index 1e2d73e7f..d46a93824 100644 --- a/indy_common/util.py +++ b/indy_common/util.py @@ -7,7 +7,8 @@ from base58 import b58decode from common.serializers.serialization import serialize_msg_for_signing from plenum.common.types import f -from plenum.common.util import isHex, error, cryptonymToHex +from plenum.common.util import isHex, cryptonymToHex +from common.error import error from stp_core.crypto.nacl_wrappers import Verifier diff --git a/indy_node/server/client_authn.py b/indy_node/server/client_authn.py index b174d708c..fffe4d919 100644 --- a/indy_node/server/client_authn.py +++ b/indy_node/server/client_authn.py @@ -1,15 +1,12 @@ from copy import deepcopy from hashlib import sha256 -from indy_node.server.config_req_handler import ConfigReqHandler from plenum.common.types import OPERATION from plenum.common.constants import TXN_TYPE, RAW, ENC, HASH from plenum.server.client_authn import NaclAuthNr, CoreAuthNr, CoreAuthMixin -from indy_common.constants import ATTRIB, GET_TXNS -from indy_node.server.pool_req_handler import PoolRequestHandler -from indy_node.server.domain_req_handler import DomainReqHandler - +from indy_common.constants import ATTRIB, POOL_UPGRADE, SCHEMA, CLAIM_DEF, \ + GET_NYM, GET_ATTR, GET_SCHEMA, GET_CLAIM_DEF, POOL_CONFIG from indy_node.persistence.idr_cache import IdrCache @@ -18,23 +15,10 @@ class LedgerBasedAuthNr(CoreAuthMixin, NaclAuthNr): Transaction-based client authenticator. """ - write_types = CoreAuthMixin.write_types.union( - PoolRequestHandler.write_types - ).union( - DomainReqHandler.write_types - ).union( - ConfigReqHandler.write_types - ) - - query_types = CoreAuthMixin.query_types.union( - {GET_TXNS, } - ).union( - PoolRequestHandler.query_types - ).union( - DomainReqHandler.query_types - ).union( - ConfigReqHandler.query_types - ) + write_types = CoreAuthMixin.write_types.union({ATTRIB, SCHEMA, CLAIM_DEF, + POOL_CONFIG, POOL_UPGRADE}) + query_types = CoreAuthMixin.query_types.union({GET_NYM, GET_ATTR, GET_SCHEMA, + GET_CLAIM_DEF}) def __init__(self, cache: IdrCache): NaclAuthNr.__init__(self) diff --git a/indy_node/server/domain_req_handler.py b/indy_node/server/domain_req_handler.py index 7a2be3aa3..67d4841aa 100644 --- a/indy_node/server/domain_req_handler.py +++ b/indy_node/server/domain_req_handler.py @@ -93,11 +93,6 @@ def _doStaticValidationNym(self, identifier, reqId, operation): "{} not a valid role". format(role)) - @staticmethod - def _validate_attrib_keys(operation): - dataKeys = {RAW, ENC, HASH}.intersection(set(operation.keys())) - return len(dataKeys) == 1 - def _doStaticValidationAttrib(self, identifier, reqId, operation): if not self._validate_attrib_keys(operation): raise InvalidClientRequest(identifier, reqId, @@ -125,6 +120,11 @@ def validate(self, req: Request, config=None): elif typ == CLAIM_DEF: self._validate_claim_def(req) + @staticmethod + def _validate_attrib_keys(operation): + dataKeys = {RAW, ENC, HASH}.intersection(set(operation.keys())) + return len(dataKeys) == 1 + def _validateNym(self, req: Request): origin = req.identifier op = req.operation diff --git a/indy_node/server/node.py b/indy_node/server/node.py index dcadbff41..27803df40 100644 --- a/indy_node/server/node.py +++ b/indy_node/server/node.py @@ -2,12 +2,10 @@ from ledger.compact_merkle_tree import CompactMerkleTree from ledger.genesis_txn.genesis_txn_initiator_from_file import GenesisTxnInitiatorFromFile -from plenum.persistence.leveldb_hash_store import LevelDbHashStore from indy_node.server.validator_info_tool import ValidatorNodeInfoTool -from state.pruning_state import PruningState from plenum.common.constants import VERSION, NODE_PRIMARY_STORAGE_SUFFIX, \ - ENC, RAW, DOMAIN_LEDGER_ID, LedgerState + ENC, RAW, DOMAIN_LEDGER_ID from plenum.common.exceptions import InvalidClientRequest from plenum.common.ledger import Ledger from plenum.common.types import f, \ @@ -15,13 +13,9 @@ from plenum.persistence.storage import initStorage, initKeyValueStorage from plenum.server.node import Node as PlenumNode from indy_common.config_util import getConfig -from indy_common.constants import TXN_TYPE, allOpKeys, ATTRIB, GET_ATTR, \ - DATA, GET_NYM, reqOpKeys, GET_TXNS, GET_SCHEMA, GET_CLAIM_DEF, ACTION, \ +from indy_common.constants import TXN_TYPE, ATTRIB, DATA, ACTION, \ NODE_UPGRADE, COMPLETE, FAIL, CONFIG_LEDGER_ID, POOL_UPGRADE, POOL_CONFIG,\ IN_PROGRESS -from indy_common.constants import validTxnTypes, IDENTITY_TXN_TYPES, \ - CONFIG_TXN_TYPES -from indy_common.txn_util import getTxnOrderedFields from indy_common.types import Request, SafeRequest from indy_common.config_helper import NodeConfigHelper from indy_node.persistence.attribute_store import AttributeStore @@ -43,7 +37,6 @@ class Node(PlenumNode, HasPoolManager): keygenScript = "init_indy_keys" _client_request_class = SafeRequest _info_tool_class = ValidatorNodeInfoTool - ledger_ids = PlenumNode.ledger_ids + [CONFIG_LEDGER_ID] def __init__(self, name, @@ -72,9 +65,11 @@ def __init__(self, plugins_dir = plugins_dir or config_helper.plugins_dir node_info_dir = node_info_dir or config_helper.node_info_dir - # TODO: 2 ugly lines ahead, don't know how to avoid + # TODO: 4 ugly lines ahead, don't know how to avoid self.idrCache = None self.attributeStore = None + self.upgrader = None + self.poolCfg = None super().__init__(name=name, nodeRegistry=nodeRegistry, @@ -96,21 +91,6 @@ def __init__(self, # TODO: ugly line ahead, don't know how to avoid self.clientAuthNr = clientAuthNr or self.defaultAuthNr() - self.configLedger = self.getConfigLedger() - self.ledgerManager.addLedger( - CONFIG_LEDGER_ID, - self.configLedger, - postCatchupCompleteClbk=self.postConfigLedgerCaughtUp, - postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) - self.on_new_ledger_added(CONFIG_LEDGER_ID) - self.states[CONFIG_LEDGER_ID] = self.loadConfigState() - self.upgrader = self.getUpgrader() - self.poolCfg = self.getPoolConfig() - self.configReqHandler = self.getConfigReqHandler() - self.initConfigState() - self.register_req_handler(CONFIG_LEDGER_ID, self.configReqHandler) - self.requestExecuter[CONFIG_LEDGER_ID] = self.executeConfigTxns - self.nodeMsgRouter.routes[Request] = self.processNodeRequest self.nodeAuthNr = self.defaultNodeAuthNr() @@ -140,14 +120,6 @@ def getPrimaryStorage(self): dataDir=self.dataLocation, config=self.config) - def send_ledger_status_to_newly_connected_node(self, node_name): - super().send_ledger_status_to_newly_connected_node(node_name) - # If the domain ledger is already synced send config ledger status - # else after the domain ledger is caught up, config ledger status - # will be sent - if self.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID].state == LedgerState.synced: - self.sendConfigLedgerStatus(node_name) - def getUpgrader(self): return Upgrader(self.id, self.name, @@ -177,22 +149,6 @@ def getIdrCache(self): ) return self.idrCache - def getConfigLedger(self): - hashStore = LevelDbHashStore( - dataDir=self.dataLocation, fileNamePrefix='config') - return Ledger(CompactMerkleTree(hashStore=hashStore), - dataDir=self.dataLocation, - fileName=self.config.configTransactionsFile, - ensureDurability=self.config.EnsureLedgerDurability) - - def loadConfigState(self): - return PruningState( - initKeyValueStorage( - self.config.configStateStorage, - self.dataLocation, - self.config.configStateDbName) - ) - def loadAttributeStore(self): return AttributeStore( initKeyValueStorage( @@ -201,6 +157,11 @@ def loadAttributeStore(self): self.config.attrDbName) ) + def setup_config_req_handler(self): + self.upgrader = self.getUpgrader() + self.poolCfg = self.getPoolConfig() + super().setup_config_req_handler() + def getConfigReqHandler(self): return ConfigReqHandler(self.configLedger, self.states[CONFIG_LEDGER_ID], @@ -209,36 +170,9 @@ def getConfigReqHandler(self): self.poolManager, self.poolCfg) - def initConfigState(self): - self.initStateFromLedger(self.states[CONFIG_LEDGER_ID], - self.configLedger, self.configReqHandler) - - @property - def ledgers(self): - ledgers = super().ledgers - ledgers.append(self.configLedger) - return ledgers - - def start_config_ledger_sync(self): - self._sync_ledger(CONFIG_LEDGER_ID) - self.ledgerManager.processStashedLedgerStatuses(CONFIG_LEDGER_ID) - def post_txn_from_catchup_added_to_domain_ledger(self, txn): pass - def sendConfigLedgerStatus(self, nodeName): - self.sendLedgerStatus(nodeName, CONFIG_LEDGER_ID) - - @property - def configLedgerStatus(self): - return self.build_ledger_status(CONFIG_LEDGER_ID) - - def getLedgerStatus(self, ledgerId: int): - if ledgerId == CONFIG_LEDGER_ID: - return self.configLedgerStatus - else: - return super().getLedgerStatus(ledgerId) - def postPoolLedgerCaughtUp(self, **kwargs): # The only reason to override this is to set the correct node id in # the upgrader since when the upgrader is initialized, node might not @@ -246,13 +180,10 @@ def postPoolLedgerCaughtUp(self, **kwargs): self.upgrader.nodeId = self.id super().postPoolLedgerCaughtUp(**kwargs) - def catchup_next_ledger_after_pool(self): - self.start_config_ledger_sync() - def postConfigLedgerCaughtUp(self, **kwargs): self.poolCfg.processLedger() self.upgrader.processLedger() - self.start_domain_ledger_sync() + super().postConfigLedgerCaughtUp(**kwargs) self.acknowledge_upgrade() def acknowledge_upgrade(self): @@ -316,13 +247,6 @@ def processNodeRequest(self, request: Request, frm: str): # corresponding client request(REQUEST) self.recordAndPropagate(request, frm) - def postRecvTxnFromCatchup(self, ledgerId: int, txn: Any): - if ledgerId == CONFIG_LEDGER_ID: - # Since no config ledger transactions are applied to the state - return None - else: - return super().postRecvTxnFromCatchup(ledgerId, txn) - def validateNodeMsg(self, wrappedMsg): msg, frm = wrappedMsg if all(attr in msg.keys() @@ -381,15 +305,8 @@ def executeDomainTxns(self, ppTime, reqs: List[Request], stateRoot, :param ppTime: the time at which PRE-PREPARE was sent :param req: the client REQUEST """ - req_handler = self.get_req_handler(DOMAIN_LEDGER_ID) - return self.commitAndSendReplies(req_handler, ppTime, reqs, - stateRoot, txnRoot) - - def executeConfigTxns(self, ppTime, reqs: List[Request], stateRoot, - txnRoot) -> List: - req_handler = self.get_req_handler(CONFIG_LEDGER_ID) - return self.commitAndSendReplies(req_handler, ppTime, reqs, - stateRoot, txnRoot) + return self.default_executer(DOMAIN_LEDGER_ID, ppTime, reqs, + stateRoot, txnRoot) def update_txn_with_extra_data(self, txn): """ @@ -415,15 +332,3 @@ def closeAllKVStores(self): self.idrCache.close() if self.attributeStore: self.attributeStore.close() - - def onBatchCreated(self, ledgerId, stateRoot): - if ledgerId == CONFIG_LEDGER_ID: - self.configReqHandler.onBatchCreated(stateRoot) - else: - super().onBatchCreated(ledgerId, stateRoot) - - def onBatchRejected(self, ledgerId): - if ledgerId == CONFIG_LEDGER_ID: - self.configReqHandler.onBatchRejected() - else: - super().onBatchRejected(ledgerId) diff --git a/indy_node/test/conftest.py b/indy_node/test/conftest.py index 37e80678e..e3e95a63b 100644 --- a/indy_node/test/conftest.py +++ b/indy_node/test/conftest.py @@ -41,7 +41,7 @@ poolTxnData, dirName, poolTxnNodeNames, allPluginsPath, tdirWithNodeKeepInited, \ poolTxnStewardData, poolTxnStewardNames, getValueFromModule, \ patchPluginManager, txnPoolNodesLooper, warncheck, \ - warnfilters as plenum_warnfilters + warnfilters as plenum_warnfilters, do_post_node_creation # noinspection PyUnresolvedReferences from indy_common.test.conftest import general_conf_tdir, tconf, poolTxnTrusteeNames, \ diff --git a/indy_node/test/replay/test_successive_batch_no_state_change.py b/indy_node/test/replay/test_successive_batch_no_state_change.py index 27d4df039..73f32de8a 100644 --- a/indy_node/test/replay/test_successive_batch_no_state_change.py +++ b/indy_node/test/replay/test_successive_batch_no_state_change.py @@ -46,7 +46,6 @@ def test_successive_batch_do_no_change_state(looper, Also check reject and commit :return: """ - prim_node = getPrimaryReplica(nodeSet, 0).node all_reqs = [] # Delay only first PRE-PREPARE @@ -132,8 +131,6 @@ def check_uncommitted(count): check_verkey(new_idr, verkey) - pp_seq_no_to_delay = 4 - new_client.deregisterObserver(name='temp') # Setting the verkey to `x`, then `y` and then back to `x` but in different diff --git a/setup.py b/setup.py index 3c7c6f898..b963598cd 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ data_files=[( (BASE_DIR, ['data/nssm_original.exe']) )], - install_requires=['indy-plenum-dev==1.2.226', + install_requires=['indy-plenum-dev==1.2.228', 'indy-anoncreds-dev==1.0.32', 'python-dateutil', 'timeout-decorator'],