Support of state proofs in client [INDY-790] (hyperledger#367)

* add test for state proof in get_attr request

* add constants for state proofs

* add test for state proof in get_claim_def

* add tet for state proofs in get_schema

* add state proof to replies for get_attr, get_schema and get_claim_def transactions

* make get_nym return state proof

* base58 proof nodes and root_hash in state proof

* conert root_hash to bytes before encoding to base58

* do not add ORIGIN to data for get_schema reply

* pass bls hash store to DomainReqHandler in state proof test

* add multisignature for root for get requests

* add test that checks that state proof is returned for get_attr request

* return multisignature in state_proof

* use state proof constants from plenum

* add seqNo to the data passed to NYM request handling function

* Add seqNo to reply for get_nym request

On the same level as identifier, data and state proof to make it look like replies to other get_* requests

* use nymData instead of data to get seqno in get_nym

* do not take issuer from schemas data

* add generation of bls keys to init_sovrin_keys

* make IdrCache return None instead of empty string

* fix formatting

* add test for state proof in get_nym

* add divider to schema path to avoid collisions

* make IdrCache store seqno of nym

* remove setVerkey and setRole methods

it is better to always use set instead

* domain_req_handler should work with seqno in nym

* update tests of state proof

* use base64 for proof nodes instead of base58

because it works much quicker

* fix tests

* fix tests

* fix tests

* blskey support, fix tests

* fix tests

* fix getting seq_no from None

* fixed tests

* add txn_time to state for all txn except nym

* send nym request in test_state_proof_returned_for_get_nym

* switch off checking of TXN_TIME

* save txn_time in idr cache

* fix state proof test

* add test for blskey in send NODE CLI command

* use initNodeKeysForBothStacks in init_sovrin_keys

* fix API

* simplify return statement in _validate_attrib_keys

* implement make_state_key

* remove unused method

* move creation of state pathes to commons

* move creation of state value to commons

* add decode_state_value to commons

* move parsing of attrib to commons

* move parsing of claim def to commons

* move parsing of schema to commons

* move make_proof to plenum

* add parsing of nym to commons

* post merge cleaning

* up plenum to 1.1.137

* add special method for parsing get_attr txns

* remove duplicated method and fix formatting

* fix tests in test_state_proofs_for_get_requests

* fix domain.prepare_nym_for_state

* exclude target nym from get nym data

* return target nym, but remove it on proof check

setup.py

@@ -87,7 +87,7 @@ def run(self):
     data_files=[(
         (BASE_DIR, ['data/nssm_original.exe'])
     )],
-    install_requires=['indy-plenum-dev==1.1.136',
+    install_requires=['indy-plenum-dev==1.1.137',
                       'indy-anoncreds-dev==1.0.25',
                       'python-dateutil',
                       'timeout-decorator'],

sovrin_client/client/client.py

@@ -12,7 +12,7 @@
 from plenum.common.startable import Status
 
 from plenum.common.constants import REPLY, NAME, VERSION, REQACK, REQNACK, \
-    TXN_ID, TARGET_NYM, NONCE, STEWARD, OP_FIELD_NAME, REJECT
+    TXN_ID, TARGET_NYM, NONCE, STEWARD, OP_FIELD_NAME, REJECT, TYPE
 from plenum.common.types import f
 from plenum.common.util import libnacl
 from plenum.server.router import Router
@@ -21,13 +21,14 @@
 from stp_zmq.simple_zstack import SimpleZStack
 
 from sovrin_common.constants import TXN_TYPE, ATTRIB, DATA, GET_NYM, ROLE, \
-    NYM, GET_TXNS, LAST_TXN, TXNS, SCHEMA, CLAIM_DEF, SKEY, DISCLO,\
-    GET_ATTR, TRUST_ANCHOR
+    NYM, GET_TXNS, LAST_TXN, TXNS, SCHEMA, CLAIM_DEF, SKEY, DISCLO, \
+    GET_ATTR, TRUST_ANCHOR, GET_CLAIM_DEF, GET_SCHEMA, SIGNATURE_TYPE, REF
 
 from sovrin_client.persistence.client_req_rep_store_file import ClientReqRepStoreFile
 from sovrin_client.persistence.client_txn_log import ClientTxnLog
 from sovrin_common.config_util import getConfig
 from stp_core.types import HA
+from sovrin_common.state import domain
 
 logger = getlogger()
 
@@ -124,8 +125,21 @@ def requestConfirmed(self, identifier: str, reqId: int) -> bool:
     def hasConsensus(self, identifier: str, reqId: int) -> Optional[str]:
         return super().hasConsensus(identifier, reqId)
 
-    def getTxnsByNym(self, nym: str):
-        raise NotImplementedError
+    def prepare_for_state(self, result):
+        request_type = result[TYPE]
+        if request_type == GET_NYM:
+            return domain.prepare_nym_for_state(result)
+        if request_type == GET_ATTR:
+            path, value, hashed_value, value_bytes = \
+                domain.prepare_get_attr_for_state(result)
+            return path, value_bytes
+        if request_type == GET_CLAIM_DEF:
+            return domain.prepare_claim_def_for_state(result)
+        if request_type == GET_SCHEMA:
+            return domain.prepare_schema_for_state(result)
+        raise ValueError("Cannot make state key for "
+                         "request of type {}"
+                         .format(request_type))
 
     def getTxnsByType(self, txnType):
         return self.txnLog.getTxnsByType(txnType)

sovrin_common/state/domain.py

@@ -0,0 +1,156 @@
+import json
+from hashlib import sha256
+from common.serializers.serialization import domain_state_serializer
+from plenum.common.constants import RAW, ENC, HASH, TXN_TIME, TXN_TYPE, TARGET_NYM, DATA, NAME, VERSION
+from plenum.common.types import f
+from sovrin_common.serialization import attrib_raw_data_serializer
+from sovrin_common.constants import ATTRIB, GET_ATTR, REF, SIGNATURE_TYPE
+
+MARKER_ATTR = "\01"
+MARKER_SCHEMA = "\02"
+MARKER_CLAIM_DEF = "\03"
+LAST_SEQ_NO = "lsn"
+VALUE = "val"
+LAST_UPDATE_TIME = "lut"
+
+
+def make_state_path_for_nym(did) -> bytes:
+    # TODO: This is duplicated in plenum.DimainRequestHandler
+    return sha256(did.encode()).digest()
+
+
+def make_state_path_for_attr(did, attr_name) -> bytes:
+    nameHash = sha256(attr_name.encode()).hexdigest()
+    return "{DID}:{MARKER}:{ATTR_NAME}"\
+        .format(DID=did,
+                MARKER=MARKER_ATTR,
+                ATTR_NAME=nameHash).encode()
+
+
+def make_state_path_for_schema(authors_did, schema_name, schema_version) -> bytes:
+    return "{DID}:{MARKER}:{SCHEMA_NAME}:{SCHEMA_VERSION}" \
+        .format(DID=authors_did,
+                MARKER=MARKER_SCHEMA,
+                SCHEMA_NAME=schema_name,
+                SCHEMA_VERSION=schema_version).encode()
+
+
+def make_state_path_for_claim_def(authors_did, schema_seq_no, signature_type) -> bytes:
+    return "{DID}:{MARKER}:{SIGNATURE_TYPE}:{SCHEMA_SEQ_NO}" \
+        .format(DID=authors_did,
+                MARKER=MARKER_CLAIM_DEF,
+                SIGNATURE_TYPE=signature_type,
+                SCHEMA_SEQ_NO=schema_seq_no).encode()
+
+
+def prepare_nym_for_state(txn):
+    # TODO: this is semi-duplicated in plenum.DomainRequestHandler
+    data = txn.get(DATA)
+    parsed = domain_state_serializer.deserialize(data)
+    parsed.pop(TARGET_NYM, None)
+    value = domain_state_serializer.serialize(parsed)
+    nym = txn[TARGET_NYM]
+    key = make_state_path_for_nym(nym)
+    return key, value
+
+
+def prepare_attr_for_state(txn):
+    """
+    Make key(path)-value pair for state from ATTRIB or GET_ATTR
+    :return: state path, state value, value for attribute store
+    """
+    assert txn[TXN_TYPE] in {ATTRIB, GET_ATTR}
+    nym = txn.get(TARGET_NYM)
+    attr_key, value = parse_attr_txn(txn)
+    hashed_value = hash_of(value) if value else ''
+    seq_no = txn[f.SEQ_NO.nm]
+    txn_time = txn[TXN_TIME]
+    value_bytes = encode_state_value(hashed_value, seq_no, txn_time)
+    path = make_state_path_for_attr(nym, attr_key)
+    return path, value, hashed_value, value_bytes
+
+
+def prepare_claim_def_for_state(txn):
+    origin = txn.get(f.IDENTIFIER.nm)
+    schema_seq_no = txn.get(REF)
+    if schema_seq_no is None:
+        raise ValueError("'{}' field is absent, "
+                         "but it must contain schema seq no".format(REF))
+    data = txn.get(DATA)
+    if data is None:
+        raise ValueError("'{}' field is absent, "
+                         "but it must contain components of keys"
+                         .format(DATA))
+    signature_type = txn.get(SIGNATURE_TYPE, 'CL')
+    path = make_state_path_for_claim_def(origin, schema_seq_no, signature_type)
+    seq_no = txn[f.SEQ_NO.nm]
+    txn_time = txn[TXN_TIME]
+    value_bytes = encode_state_value(data, seq_no, txn_time)
+    return path, value_bytes
+
+
+def prepare_schema_for_state(txn):
+    origin = txn.get(f.IDENTIFIER.nm)
+    data = txn.get(DATA)
+    schema_name = data[NAME]
+    schema_version = data[VERSION]
+    path = make_state_path_for_schema(origin, schema_name, schema_version)
+    seq_no = txn[f.SEQ_NO.nm]
+    txn_time = txn[TXN_TIME]
+    value_bytes = encode_state_value(data, seq_no, txn_time)
+    return path, value_bytes
+
+
+def encode_state_value(value, seqNo, txnTime):
+    return domain_state_serializer.serialize({
+        LAST_SEQ_NO: seqNo,
+        LAST_UPDATE_TIME: txnTime,
+        VALUE: value
+    })
+
+
+def decode_state_value(ecnoded_value):
+    decoded = domain_state_serializer.deserialize(ecnoded_value)
+    value = decoded.get(VALUE)
+    last_seq_no = decoded.get(LAST_SEQ_NO)
+    last_update_time = decoded.get(LAST_UPDATE_TIME)
+    return value, last_seq_no, last_update_time
+
+
+def hash_of(text) -> str:
+    if not isinstance(text, (str, bytes)):
+        text = domain_state_serializer.serialize(text)
+    if not isinstance(text, bytes):
+        text = text.encode()
+    return sha256(text).hexdigest()
+
+
+def parse_attr_txn(txn):
+    raw = txn.get(RAW)
+    if raw:
+        data = attrib_raw_data_serializer.deserialize(raw)
+        # To exclude user-side formatting issues
+        re_raw = attrib_raw_data_serializer.serialize(data,
+                                                      toBytes=False)
+        key, _ = data.popitem()
+        return key, re_raw
+    enc = txn.get(ENC)
+    if enc:
+        return hash_of(enc), enc
+    hsh = txn.get(HASH)
+    if hsh:
+        return hsh, None
+    raise ValueError("One of 'raw', 'enc', 'hash' "
+                     "fields of ATTR must present")
+
+
+def prepare_get_attr_for_state(txn):
+    keys = [RAW, ENC, HASH]
+    for key in keys:
+        if txn[key]:
+            txn = txn.copy()
+            data = txn.pop(DATA)
+            txn[key] = data
+            return prepare_attr_for_state(txn)
+    raise ValueError("There is no any of {} in txn {}"
+                     .format(keys, txn))