diff --git a/setup.py b/setup.py index 21a16f804..186447043 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ def run(self): data_files=[( (BASE_DIR, ['data/nssm_original.exe']) )], - install_requires=['indy-plenum-dev==1.1.136', + install_requires=['indy-plenum-dev==1.1.137', 'indy-anoncreds-dev==1.0.25', 'python-dateutil', 'timeout-decorator'], diff --git a/sovrin_client/client/client.py b/sovrin_client/client/client.py index b213d59e7..b9cba6fe0 100644 --- a/sovrin_client/client/client.py +++ b/sovrin_client/client/client.py @@ -12,7 +12,7 @@ from plenum.common.startable import Status from plenum.common.constants import REPLY, NAME, VERSION, REQACK, REQNACK, \ - TXN_ID, TARGET_NYM, NONCE, STEWARD, OP_FIELD_NAME, REJECT + TXN_ID, TARGET_NYM, NONCE, STEWARD, OP_FIELD_NAME, REJECT, TYPE from plenum.common.types import f from plenum.common.util import libnacl from plenum.server.router import Router @@ -21,13 +21,14 @@ from stp_zmq.simple_zstack import SimpleZStack from sovrin_common.constants import TXN_TYPE, ATTRIB, DATA, GET_NYM, ROLE, \ - NYM, GET_TXNS, LAST_TXN, TXNS, SCHEMA, CLAIM_DEF, SKEY, DISCLO,\ - GET_ATTR, TRUST_ANCHOR + NYM, GET_TXNS, LAST_TXN, TXNS, SCHEMA, CLAIM_DEF, SKEY, DISCLO, \ + GET_ATTR, TRUST_ANCHOR, GET_CLAIM_DEF, GET_SCHEMA, SIGNATURE_TYPE, REF from sovrin_client.persistence.client_req_rep_store_file import ClientReqRepStoreFile from sovrin_client.persistence.client_txn_log import ClientTxnLog from sovrin_common.config_util import getConfig from stp_core.types import HA +from sovrin_common.state import domain logger = getlogger() @@ -124,8 +125,21 @@ def requestConfirmed(self, identifier: str, reqId: int) -> bool: def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]: return super().hasConsensus(identifier, reqId) - def getTxnsByNym(self, nym: str): - raise NotImplementedError + def prepare_for_state(self, result): + request_type = result[TYPE] + if request_type == GET_NYM: + return domain.prepare_nym_for_state(result) + if request_type == GET_ATTR: + path, value, hashed_value, value_bytes = \ + domain.prepare_get_attr_for_state(result) + return path, value_bytes + if request_type == GET_CLAIM_DEF: + return domain.prepare_claim_def_for_state(result) + if request_type == GET_SCHEMA: + return domain.prepare_schema_for_state(result) + raise ValueError("Cannot make state key for " + "request of type {}" + .format(request_type)) def getTxnsByType(self, txnType): return self.txnLog.getTxnsByType(txnType) diff --git a/sovrin_common/state/__init__.py b/sovrin_common/state/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sovrin_common/state/domain.py b/sovrin_common/state/domain.py new file mode 100644 index 000000000..bbdafe444 --- /dev/null +++ b/sovrin_common/state/domain.py @@ -0,0 +1,156 @@ +import json +from hashlib import sha256 +from common.serializers.serialization import domain_state_serializer +from plenum.common.constants import RAW, ENC, HASH, TXN_TIME, TXN_TYPE, TARGET_NYM, DATA, NAME, VERSION +from plenum.common.types import f +from sovrin_common.serialization import attrib_raw_data_serializer +from sovrin_common.constants import ATTRIB, GET_ATTR, REF, SIGNATURE_TYPE + +MARKER_ATTR = "\01" +MARKER_SCHEMA = "\02" +MARKER_CLAIM_DEF = "\03" +LAST_SEQ_NO = "lsn" +VALUE = "val" +LAST_UPDATE_TIME = "lut" + + +def make_state_path_for_nym(did) -> bytes: + # TODO: This is duplicated in plenum.DimainRequestHandler + return sha256(did.encode()).digest() + + +def make_state_path_for_attr(did, attr_name) -> bytes: + nameHash = sha256(attr_name.encode()).hexdigest() + return "{DID}:{MARKER}:{ATTR_NAME}"\ + .format(DID=did, + MARKER=MARKER_ATTR, + ATTR_NAME=nameHash).encode() + + +def make_state_path_for_schema(authors_did, schema_name, schema_version) -> bytes: + return "{DID}:{MARKER}:{SCHEMA_NAME}:{SCHEMA_VERSION}" \ + .format(DID=authors_did, + MARKER=MARKER_SCHEMA, + SCHEMA_NAME=schema_name, + SCHEMA_VERSION=schema_version).encode() + + +def make_state_path_for_claim_def(authors_did, schema_seq_no, signature_type) -> bytes: + return "{DID}:{MARKER}:{SIGNATURE_TYPE}:{SCHEMA_SEQ_NO}" \ + .format(DID=authors_did, + MARKER=MARKER_CLAIM_DEF, + SIGNATURE_TYPE=signature_type, + SCHEMA_SEQ_NO=schema_seq_no).encode() + + +def prepare_nym_for_state(txn): + # TODO: this is semi-duplicated in plenum.DomainRequestHandler + data = txn.get(DATA) + parsed = domain_state_serializer.deserialize(data) + parsed.pop(TARGET_NYM, None) + value = domain_state_serializer.serialize(parsed) + nym = txn[TARGET_NYM] + key = make_state_path_for_nym(nym) + return key, value + + +def prepare_attr_for_state(txn): + """ + Make key(path)-value pair for state from ATTRIB or GET_ATTR + :return: state path, state value, value for attribute store + """ + assert txn[TXN_TYPE] in {ATTRIB, GET_ATTR} + nym = txn.get(TARGET_NYM) + attr_key, value = parse_attr_txn(txn) + hashed_value = hash_of(value) if value else '' + seq_no = txn[f.SEQ_NO.nm] + txn_time = txn[TXN_TIME] + value_bytes = encode_state_value(hashed_value, seq_no, txn_time) + path = make_state_path_for_attr(nym, attr_key) + return path, value, hashed_value, value_bytes + + +def prepare_claim_def_for_state(txn): + origin = txn.get(f.IDENTIFIER.nm) + schema_seq_no = txn.get(REF) + if schema_seq_no is None: + raise ValueError("'{}' field is absent, " + "but it must contain schema seq no".format(REF)) + data = txn.get(DATA) + if data is None: + raise ValueError("'{}' field is absent, " + "but it must contain components of keys" + .format(DATA)) + signature_type = txn.get(SIGNATURE_TYPE, 'CL') + path = make_state_path_for_claim_def(origin, schema_seq_no, signature_type) + seq_no = txn[f.SEQ_NO.nm] + txn_time = txn[TXN_TIME] + value_bytes = encode_state_value(data, seq_no, txn_time) + return path, value_bytes + + +def prepare_schema_for_state(txn): + origin = txn.get(f.IDENTIFIER.nm) + data = txn.get(DATA) + schema_name = data[NAME] + schema_version = data[VERSION] + path = make_state_path_for_schema(origin, schema_name, schema_version) + seq_no = txn[f.SEQ_NO.nm] + txn_time = txn[TXN_TIME] + value_bytes = encode_state_value(data, seq_no, txn_time) + return path, value_bytes + + +def encode_state_value(value, seqNo, txnTime): + return domain_state_serializer.serialize({ + LAST_SEQ_NO: seqNo, + LAST_UPDATE_TIME: txnTime, + VALUE: value + }) + + +def decode_state_value(ecnoded_value): + decoded = domain_state_serializer.deserialize(ecnoded_value) + value = decoded.get(VALUE) + last_seq_no = decoded.get(LAST_SEQ_NO) + last_update_time = decoded.get(LAST_UPDATE_TIME) + return value, last_seq_no, last_update_time + + +def hash_of(text) -> str: + if not isinstance(text, (str, bytes)): + text = domain_state_serializer.serialize(text) + if not isinstance(text, bytes): + text = text.encode() + return sha256(text).hexdigest() + + +def parse_attr_txn(txn): + raw = txn.get(RAW) + if raw: + data = attrib_raw_data_serializer.deserialize(raw) + # To exclude user-side formatting issues + re_raw = attrib_raw_data_serializer.serialize(data, + toBytes=False) + key, _ = data.popitem() + return key, re_raw + enc = txn.get(ENC) + if enc: + return hash_of(enc), enc + hsh = txn.get(HASH) + if hsh: + return hsh, None + raise ValueError("One of 'raw', 'enc', 'hash' " + "fields of ATTR must present") + + +def prepare_get_attr_for_state(txn): + keys = [RAW, ENC, HASH] + for key in keys: + if txn[key]: + txn = txn.copy() + data = txn.pop(DATA) + txn[key] = data + return prepare_attr_for_state(txn) + raise ValueError("There is no any of {} in txn {}" + .format(keys, txn)) diff --git a/sovrin_node/server/domain_req_handler.py b/sovrin_node/server/domain_req_handler.py index 3dd2a34de..6a6fad2b1 100644 --- a/sovrin_node/server/domain_req_handler.py +++ b/sovrin_node/server/domain_req_handler.py @@ -23,17 +23,13 @@ from sovrin_common.types import Request from stp_core.common.log import getlogger from sovrin_node.persistence.idr_cache import IdrCache +from sovrin_common.state import domain + logger = getlogger() class DomainReqHandler(PHandler): - MARKER_ATTR = "\01" - MARKER_SCHEMA = "\02" - MARKER_CLAIM_DEF = "\03" - LAST_SEQ_NO = "lsn" - VALUE = "val" - LAST_UPDATE_TIME = "lut" def __init__(self, ledger, state, requestProcessor, idrCache, attributeStore, bls_store): @@ -113,10 +109,7 @@ def _doStaticValidationNym(self, identifier, reqId, operation): @staticmethod def _validate_attrib_keys(operation): dataKeys = {RAW, ENC, HASH}.intersection(set(operation.keys())) - if len(dataKeys) != 1: - return False - else: - return True + return len(dataKeys) == 1 def _doStaticValidationAttrib(self, identifier, reqId, operation): if not self._validate_attrib_keys(operation): @@ -226,7 +219,8 @@ def handleGetNymReq(self, request: Request, frm: str): data = self.stateSerializer.serialize(nymData) seq_no = nymData[f.SEQ_NO.nm] update_time = nymData[TXN_TIME] - proof = self.make_proof(self.nym_to_state_key(nym)) + path = domain.make_state_path_for_nym(nym) + proof = self.make_proof(path) else: data = None seq_no = None @@ -299,19 +293,6 @@ def handleGetAttrsReq(self, request: Request, frm: str): update_time=lastUpdateTime, proof=proof) - def make_proof(self, path): - proof = self.state.generate_state_proof(path, serialize=True) - root_hash = self.state.committedHeadHash - # TODO: move to serialization.py - encoded_proof = base64.b64encode(proof) - encoded_root_hash = state_roots_serializer.serialize(bytes(root_hash)) - multi_sig = self.bls_store.get(encoded_root_hash) - return { - ROOT_HASH: encoded_root_hash, - MULTI_SIGNATURE: multi_sig, # [["participants"], "signature", "encoded_pool_root_hash" ] - PROOF_NODES: encoded_proof - } - def lookup(self, path, isCommitted=True) -> (str, int): """ Queries state for data on specified path @@ -323,12 +304,9 @@ def lookup(self, path, isCommitted=True) -> (str, int): encoded = self.state.get(path, isCommitted) if encoded is None: raise KeyError - decoded = self.stateSerializer.deserialize(encoded) - value = decoded.get(self.VALUE) - lastSeqNo = decoded.get(self.LAST_SEQ_NO) - lastUpdateTime = decoded.get(self.LAST_UPDATE_TIME) + value, last_seq_no, last_update_time = domain.decode_state_value(encoded) proof = self.make_proof(path) - return value, lastSeqNo, lastUpdateTime, proof + return value, last_seq_no, last_update_time, proof def _addAttr(self, txn) -> None: """ @@ -339,50 +317,19 @@ def _addAttr(self, txn) -> None: the trie stores a blank value for the key did+hash """ assert txn[TXN_TYPE] == ATTRIB - nym = txn.get(TARGET_NYM) - attr_key, value = self._parse_attr(txn) - hashedVal = self._hashOf(value) if value else '' - seqNo = txn[f.SEQ_NO.nm] - txnTime = txn[TXN_TIME] - valueBytes = self._encodeValue(hashedVal, seqNo, txnTime) - path = self._makeAttrPath(nym, attr_key) - self.state.set(path, valueBytes) - self.attributeStore.set(hashedVal, value) + path, value, hashed_value, value_bytes = domain.prepare_attr_for_state(txn) + self.state.set(path, value_bytes) + self.attributeStore.set(hashed_value, value) def _addSchema(self, txn) -> None: assert txn[TXN_TYPE] == SCHEMA - origin = txn.get(f.IDENTIFIER.nm) - - data = txn.get(DATA) - schemaName = data[NAME] - schemaVersion = data[VERSION] - path = self._makeSchemaPath(origin, schemaName, schemaVersion) - - seqNo = txn[f.SEQ_NO.nm] - txnTime = txn[TXN_TIME] - valueBytes = self._encodeValue(data, seqNo, txnTime) - self.state.set(path, valueBytes) + path, value_bytes = domain.prepare_schema_for_state(txn) + self.state.set(path, value_bytes) def _addClaimDef(self, txn) -> None: assert txn[TXN_TYPE] == CLAIM_DEF - origin = txn.get(f.IDENTIFIER.nm) - - schemaSeqNo = txn.get(REF) - if schemaSeqNo is None: - raise ValueError("'{}' field is absent, " - "but it must contain schema seq no".format(REF)) - data = txn.get(DATA) - if data is None: - raise ValueError("'{}' field is absent, " - "but it must contain components of keys" - .format(DATA)) - - signatureType = txn.get(SIGNATURE_TYPE, 'CL') - path = self._makeClaimDefPath(origin, schemaSeqNo, signatureType) - seqNo = txn[f.SEQ_NO.nm] - txnTime = txn[TXN_TIME] - valueBytes = self._encodeValue(data, seqNo, txnTime) - self.state.set(path, valueBytes) + path, value_bytes = domain.prepare_claim_def_for_state(txn) + self.state.set(path, value_bytes) def getAttr(self, did: str, @@ -390,7 +337,7 @@ def getAttr(self, isCommitted=True) -> (str, int, int, list): assert did is not None assert key is not None - path = self._makeAttrPath(did, key) + path = domain.make_state_path_for_attr(did, key) try: hashed_val, lastSeqNo, lastUpdateTime, proof = \ self.lookup(path, isCommitted) @@ -416,7 +363,7 @@ def getSchema(self, assert author is not None assert schemaName is not None assert schemaVersion is not None - path = self._makeSchemaPath(author, schemaName, schemaVersion) + path = domain.make_state_path_for_schema(author, schemaName, schemaVersion) try: keys, seqno, lastUpdateTime, proof = self.lookup(path, isCommitted) return keys, seqno, lastUpdateTime, proof @@ -430,54 +377,13 @@ def getClaimDef(self, isCommitted=True) -> (str, int, int, list): assert author is not None assert schemaSeqNo is not None - path = self._makeClaimDefPath(author, schemaSeqNo, signatureType) + path = domain.make_state_path_for_claim_def(author, schemaSeqNo, signatureType) try: keys, seqno, lastUpdateTime, proof = self.lookup(path, isCommitted) return keys, seqno, lastUpdateTime, proof except KeyError: return None, None, None, None - @staticmethod - def _hashOf(text) -> str: - if not isinstance(text, (str, bytes)): - text = DomainReqHandler.stateSerializer.serialize(text) - if not isinstance(text, bytes): - text = text.encode() - return sha256(text).hexdigest() - - @staticmethod - def _makeAttrPath(did, attrName) -> bytes: - nameHash = DomainReqHandler._hashOf(attrName) - return "{DID}:{MARKER}:{ATTR_NAME}" \ - .format(DID=did, - MARKER=DomainReqHandler.MARKER_ATTR, - ATTR_NAME=nameHash).encode() - - @staticmethod - def _makeSchemaPath(did, schemaName, schemaVersion) -> bytes: - return "{DID}:{MARKER}:{SCHEMA_NAME}:{SCHEMA_VERSION}" \ - .format(DID=did, - MARKER=DomainReqHandler.MARKER_SCHEMA, - SCHEMA_NAME=schemaName, - SCHEMA_VERSION=schemaVersion) \ - .encode() - - @staticmethod - def _makeClaimDefPath(did, schemaSeqNo, signatureType) -> bytes: - return "{DID}:{MARKER}:{SIGNATURE_TYPE}:{SCHEMA_SEQ_NO}" \ - .format(DID=did, - MARKER=DomainReqHandler.MARKER_CLAIM_DEF, - SIGNATURE_TYPE=signatureType, - SCHEMA_SEQ_NO=schemaSeqNo)\ - .encode() - - def _encodeValue(self, value, seqNo, txnTime): - return self.stateSerializer.serialize({ - DomainReqHandler.LAST_SEQ_NO: seqNo, - DomainReqHandler.LAST_UPDATE_TIME: txnTime, - DomainReqHandler.VALUE: value - }) - @staticmethod def transform_txn_for_ledger(txn): """ @@ -486,23 +392,24 @@ def transform_txn_for_ledger(txn): hash in the ledger """ if txn[TXN_TYPE] == ATTRIB: - txn = DomainReqHandler.hash_attrib_txn(txn) + txn = DomainReqHandler.transform_attrib_for_ledger(txn) return txn @staticmethod - def hash_attrib_txn(txn): - # Creating copy of result so that `RAW`, `ENC` or `HASH` can be - # replaced by their hashes. We do not insert actual attribute data - # in the ledger but only the hash of it. + def transform_attrib_for_ledger(txn): + """ + Creating copy of result so that `RAW`, `ENC` or `HASH` can be + replaced by their hashes. We do not insert actual attribute data + in the ledger but only the hash of it. + """ txn = deepcopy(txn) - - attr_key, value = DomainReqHandler._parse_attr(txn) - hashedVal = DomainReqHandler._hashOf(value) if value else '' + attr_key, value = domain.parse_attr_txn(txn) + hashed_val = domain.hash_of(value) if value else '' if RAW in txn: - txn[RAW] = hashedVal + txn[RAW] = hashed_val elif ENC in txn: - txn[ENC] = hashedVal + txn[ENC] = hashed_val elif HASH in txn: txn[HASH] = txn[HASH] return txn @@ -519,22 +426,3 @@ def make_result(request, data, last_seq_no, update_time, proof): }} # Do not inline please, it makes debugging easier return result - - @staticmethod - def _parse_attr(txn): - raw = txn.get(RAW) - if raw: - data = attrib_raw_data_serializer.deserialize(raw) - # To exclude user-side formatting issues - re_raw = attrib_raw_data_serializer.serialize(data, - toBytes=False) - key, _ = data.popitem() - return key, re_raw - enc = txn.get(ENC) - if enc: - return DomainReqHandler._hashOf(enc), enc - hsh = txn.get(HASH) - if hsh: - return hsh, None - raise ValueError("One of 'raw', 'enc', 'hash' " - "fields of ATTR must present") diff --git a/sovrin_node/test/state_proof/test_state_proofs_for_get_requests.py b/sovrin_node/test/state_proof/test_state_proofs_for_get_requests.py index 6338b03e5..ed07792cf 100644 --- a/sovrin_node/test/state_proof/test_state_proofs_for_get_requests.py +++ b/sovrin_node/test/state_proof/test_state_proofs_for_get_requests.py @@ -19,6 +19,7 @@ from sovrin_node.server.domain_req_handler import DomainReqHandler from state.pruning_state import PruningState from storage.kv_in_memory import KeyValueStorageInMemory +from sovrin_common.state import domain @pytest.fixture() @@ -65,10 +66,7 @@ def save_multi_sig(request_handler): def is_proof_verified(request_handler, proof, path, value, seq_no, txn_time, ): - encoded_value = request_handler._encodeValue(value, - seq_no, - txn_time) - + encoded_value = domain.encode_state_value(value, seq_no, txn_time) proof_nodes = base64.b64decode(proof[PROOF_NODES]) root_hash = base58.b58decode(proof[ROOT_HASH]) verified = request_handler.state.verify_state_proof( @@ -113,10 +111,10 @@ def test_state_proofs_for_get_attr(request_handler): assert attr_value == raw_attribute # Verifying signed state proof - path = request_handler._makeAttrPath(nym, attr_key) + path = domain.make_state_path_for_attr(nym, attr_key) assert is_proof_verified(request_handler, proof, path, - request_handler._hashOf(attr_value), seq_no, txn_time) + domain.hash_of(attr_value), seq_no, txn_time) def test_state_proofs_for_get_claim_def(request_handler): @@ -159,7 +157,7 @@ def test_state_proofs_for_get_claim_def(request_handler): assert result[DATA] == key_components # Verifying signed state proof - path = request_handler._makeClaimDefPath(nym, schema_seqno, signature_type) + path = domain.make_state_path_for_claim_def(nym, schema_seqno, signature_type) assert is_proof_verified(request_handler, proof, path, key_components, seq_no, txn_time) @@ -202,7 +200,7 @@ def test_state_proofs_for_get_schema(request_handler): assert result[DATA] == data # Verifying signed state proof - path = request_handler._makeSchemaPath(nym, schema_name, schema_version) + path = domain.make_state_path_for_schema(nym, schema_name, schema_version) assert is_proof_verified(request_handler, proof, path, data, seq_no, txn_time)