diff --git a/src/Nethermind/Chains/beverly-hills.json b/src/Nethermind/Chains/beverly-hills.json new file mode 100644 index 00000000000..f5e45ac53b2 --- /dev/null +++ b/src/Nethermind/Chains/beverly-hills.json @@ -0,0 +1,886 @@ +{ + "name": "Beverly Hills", + "dataDir": "bevHills", + "engine": { + "Ethash": { + } + }, + "params": { + "gasLimitBoundDivisor": "0x0400", + "accountStartNonce": "0x0", + "networkID" : "0x16062", + "eip140Transition": "0x0", + "eip145Transition": "0x0", + "eip150Transition": "0x0", + "eip155Transition": "0x0", + "eip160Transition": "0x0", + "eip161abcTransition": "0x0", + "eip161dTransition": "0x0", + "eip211Transition": "0x0", + "eip214Transition": "0x0", + "eip658Transition": "0x0", + "eip1014Transition": "0x0", + "eip1052Transition": "0x0", + "eip1283Transition": "0x0", + "eip1283DisableTransition": "0x0", + "eip152Transition": "0x0", + "eip1108Transition": "0x0", + "eip1344Transition": "0x0", + "eip1884Transition": "0x0", + "eip2028Transition": "0x0", + "eip2200Transition": "0x0", + "eip2315Transition": "0x0", + "eip2537Transition": "0x0", + "eip2565Transition": "0x0", + "eip2929Transition": "0x0", + "eip2930Transition": "0x0", + "verkleTransition": "0x0", + "eip1559Transition": "0x0", + "eip3198Transition": "0x0", + "eip3529Transition": "0x0", + "eip3541Transition": "0x0", + "verkleTreeTransitionTimestamp": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "nonce": "0x1234", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty": "0x01", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "1676038202", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x17D7840" + }, + "nodes": [ + "enode://906aaa403764b718fb6d23a80ad890eddfe6d33922eba7b9bc45e160e1079becd8f5f478f36607c7082d0e1015d43aec73ca35423e97ec69e5f55f0cfcd4f29f@188.166.72.218:3030", + "enode://99561977db2c6c24d6af793fb7c81ed16eccf37548c34590f4f973bbebe6b7ad297c8fbb1f7c7e12224a7e3f29ed5bd4c5a0d0a0d937a54eabee65173127ab50@167.71.70.196:30303" + ], + "accounts": { + "0x0000000000000000000000000000000000000000": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000001": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000002": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000003": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000004": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000005": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000006": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000007": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000008": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000009": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000010": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000011": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000012": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000013": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000014": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000015": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000016": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000017": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000018": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000019": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000020": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000021": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000022": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000023": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000024": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000025": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000026": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000027": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000028": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000029": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000030": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000031": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000032": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000033": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000034": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000035": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000036": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000037": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000038": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000039": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000040": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000041": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000042": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000043": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000044": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000045": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000046": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000047": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000048": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000049": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000050": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000051": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000052": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000053": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000054": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000055": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000056": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000057": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000058": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000059": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000060": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000061": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000062": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000063": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000064": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000065": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000066": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000067": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000068": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000069": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000070": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000071": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000072": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000073": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000074": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000075": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000076": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000077": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000078": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000079": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000080": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000081": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000082": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000083": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000084": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000085": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000086": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000087": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000088": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000089": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000090": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000091": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000092": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000093": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000094": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000095": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000096": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000097": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000098": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000099": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009f": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000aa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ab": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ac": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ad": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ae": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000af": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ba": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000be": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ca": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ce": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000da": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000db": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000de": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000df": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ea": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000eb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ec": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ed": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ee": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ef": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fe": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ff": { + "balance": "1" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + }, + "0x6976650c43E63cb1E2f88312a8E57678b323dF36": { + "balance": "1000000000000000000000000000" + }, + "0x02E4650436d2dCa3308eb4d62B0A4a056a01b2Ec": { + "balance": "1000000000000000000000000000" + }, + "0xDCdaB38e9106D9Fd56d468D1E5044F0a59F231df": { + "balance": "1000000000000000000000000000" + }, + "0x950e6058D70C778b1F28B91559B55feB5B9be819": { + "balance": "1000000000000000000000000000" + }, + "0x24C32bd9715979A114bb5842126d3BEA2F1347B3": { + "balance": "1000000000000000000000000000" + }, + "0xA9500ed21AA63Af1aF82433C8ffa7bE23D6F3029": { + "balance": "1000000000000000000000000000" + } + } +} diff --git a/src/Nethermind/Chains/kaustinen.json b/src/Nethermind/Chains/kaustinen.json new file mode 100644 index 00000000000..7bfae6fd459 --- /dev/null +++ b/src/Nethermind/Chains/kaustinen.json @@ -0,0 +1,103 @@ +{ + "name": "Kaustinen", + "dataDir": "kaustinen", + "engine": { + "Ethash": { + } + }, + "params": { + "gasLimitBoundDivisor": "0x0400", + "accountStartNonce": "0x0", + "networkID" : "0x10F2C", + "eip140Transition": "0x0", + "eip145Transition": "0x0", + "eip150Transition": "0x0", + "eip150Hash": "0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0", + "eip155Transition": "0x0", + "eip160Transition": "0x0", + "eip161abcTransition": "0x0", + "eip161dTransition": "0x0", + "eip211Transition": "0x0", + "eip214Transition": "0x0", + "eip658Transition": "0x0", + "eip1014Transition": "0x0", + "eip1052Transition": "0x0", + "eip1283Transition": "0x0", + "eip1283DisableTransition": "0x0", + "eip152Transition": "0x0", + "eip1108Transition": "0x0", + "eip1344Transition": "0x0", + "eip1884Transition": "0x0", + "eip2028Transition": "0x0", + "eip2200Transition": "0x0", + "eip2315Transition": "0x0", + "eip2537Transition": "0x0", + "eip2565Transition": "0x0", + "eip2929Transition": "0x0", + "eip2930Transition": "0x0", + "verkleTransition": "0x0", + "eip1559Transition": "0x0", + "eip3198Transition": "0x0", + "eip3529Transition": "0x0", + "eip3541Transition": "0x0", + "verkleTreeTransitionTimestamp": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "nonce": "0x56", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty": "0x01", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "1679652600", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x2fefd8" + }, + "nodes": [ + "enode://4b669a76a6bb82d6074bc0296a792375fe257091885911051fb3b3f5fcaec051892d9c4baf937f6e28f0cf975d47cf2303f96e43b46e4963361fe6baecc86e6a@178.62.227.218:30303" + ], + "accounts": { + "0xf97e180c050e5Ab072211Ad2C213Eb5AEE4DF134": { + "balance": "10000000000000000000000000" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + } + } +} diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index 74d9c5dc1a6..54eec518024 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -47,6 +47,7 @@ + @@ -80,4 +81,4 @@ - \ No newline at end of file + diff --git a/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationTracer.cs b/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationTracer.cs index 927ac980204..33fa9b64bff 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationTracer.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationTracer.cs @@ -64,6 +64,7 @@ public UserOperationTxTracer( public byte[] Output { get; private set; } public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => true; public bool IsTracingOpLevelStorage => true; public bool IsTracingMemory => false; @@ -185,6 +186,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool } } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotImplementedException(); + } + public void ReportOperationError(EvmExceptionType error) { } diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 10612330c41..45f16df8bde 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -22,6 +22,7 @@ using Nethermind.State; using Nethermind.Trie.Pruning; using Nethermind.TxPool; +using Nethermind.Verkle.Tree; namespace Nethermind.Api { @@ -43,6 +44,8 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory IUnclesValidator? UnclesValidator { get; set; } IHeaderValidator? HeaderValidator { get; set; } IManualBlockProductionTrigger ManualBlockProductionTrigger { get; } + VerkleStateStore? VerkleTrieStore { get; set; } + ReadOnlyVerkleStateStore? ReadOnlyVerkleTrieStore { get; set; } IReadOnlyTrieStore? ReadOnlyTrieStore { get; set; } IRewardCalculatorSource? RewardCalculatorSource { get; set; } /// diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 9854452bfac..49ee5de548c 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -18,6 +18,7 @@ using Nethermind.Sockets; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Api { @@ -52,5 +53,6 @@ public interface IApiWithNetwork : IApiWithBlockchain IWebSocketsManager WebSocketsManager { get; set; } ISubscriptionFactory? SubscriptionFactory { get; set; } ISnapProvider? SnapProvider { get; set; } + IVerkleSyncProvider? VerkleProvider { get; set; } } } diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index fe323844e39..5a10aff2ca6 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -55,6 +55,9 @@ using Nethermind.Sockets; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.Blocks; +using Nethermind.Synchronization.VerkleSync; +using Nethermind.Verkle; +using Nethermind.Verkle.Tree; namespace Nethermind.Api { @@ -77,13 +80,15 @@ public IBlockchainBridge CreateBlockchainBridge() ReadOnlyBlockTree readOnlyTree = BlockTree!.AsReadOnly(); LazyInitializer.EnsureInitialized(ref _readOnlyDbProvider, () => new ReadOnlyDbProvider(DbProvider, false)); - // TODO: reuse the same trie cache here - ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv = new( - _readOnlyDbProvider, - ReadOnlyTrieStore, - readOnlyTree, - SpecProvider, - LogManager); + ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv = SpecProvider!.GenesisSpec.IsVerkleTreeEipEnabled switch + { + true => + // TODO: reuse the same trie cache here + new ReadOnlyTxProcessingEnv(_readOnlyDbProvider, ReadOnlyVerkleTrieStore, readOnlyTree, SpecProvider, LogManager), + false => + // TODO: reuse the same trie cache here + new ReadOnlyTxProcessingEnv(_readOnlyDbProvider, ReadOnlyTrieStore, readOnlyTree, SpecProvider, LogManager) + }; IMiningConfig miningConfig = ConfigProvider.GetConfig(); IBlocksConfig blocksConfig = ConfigProvider.GetConfig(); @@ -134,6 +139,9 @@ public IBlockchainBridge CreateBlockchainBridge() public IManualBlockProductionTrigger ManualBlockProductionTrigger { get; set; } = new BuildBlocksWhenRequested(); + public VerkleStateStore? VerkleTrieStore { get; set; } + public ReadOnlyVerkleStateStore? ReadOnlyVerkleTrieStore { get; set; } + public IIPResolver? IpResolver { get; set; } public IJsonSerializer EthereumJsonSerializer { get; set; } public IKeyStore? KeyStore { get; set; } @@ -231,6 +239,7 @@ public ISealEngine SealEngine public IList Publishers { get; } = new List(); // this should be called publishers public CompositePruningTrigger PruningTrigger { get; } = new(); public ISnapProvider? SnapProvider { get; set; } + public IVerkleSyncProvider? VerkleProvider { get; set; } public IProcessExitSource? ProcessExit { get; set; } } } diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs index c8de8362917..d768668623b 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs @@ -12,17 +12,22 @@ using Nethermind.Consensus.Validators; using Nethermind.Consensus.Withdrawals; using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Db; using Nethermind.Db.Blooms; using Nethermind.Evm; using Nethermind.Evm.TransactionProcessing; +using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Specs; +using Nethermind.Specs.Forks; using Nethermind.State; using Nethermind.State.Repositories; using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; using NUnit.Framework; namespace Nethermind.Blockchain.Test.Producers @@ -112,5 +117,261 @@ public void Test() autoResetEvent.WaitOne(1000).Should().BeTrue("1"); blockTree.Head.Number.Should().Be(1); } + + [Test, Timeout(Timeout.MaxTestTime)] + public void TestVerkle() + { + ISpecProvider specProvider = MainnetSpecProvider.Instance; + DbProvider dbProvider = new(DbModeHint.Mem); + dbProvider.RegisterDb(DbNames.BlockInfos, new MemDb()); + dbProvider.RegisterDb(DbNames.Blocks, new MemDb()); + dbProvider.RegisterDb(DbNames.Headers, new MemDb()); + dbProvider.RegisterDb(DbNames.State, new MemDb()); + dbProvider.RegisterDb(DbNames.Code, new MemDb()); + dbProvider.RegisterDb(DbNames.Metadata, new MemDb()); + dbProvider.RegisterDb(DbNames.Leaf, new MemDb()); + dbProvider.RegisterDb(DbNames.InternalNodes, new MemDb()); + dbProvider.RegisterDb(DbNames.ForwardDiff, new MemDb()); + dbProvider.RegisterDb(DbNames.ReverseDiff, new MemDb()); + dbProvider.RegisterDb(DbNames.StateRootToBlock, new MemDb()); + + BlockTree blockTree = new( + dbProvider, + new ChainLevelInfoRepository(dbProvider), + specProvider, + NullBloomStorage.Instance, + SimpleConsoleLogManager.Instance); + VerkleStateTree stateTree = new (dbProvider, SimpleConsoleLogManager.Instance); + VerkleStateReader stateReader = new(stateTree, dbProvider.GetDb(DbNames.Code), SimpleConsoleLogManager.Instance); + VerkleWorldState worldState = new (stateTree, dbProvider.RegisteredDbs[DbNames.Code], SimpleConsoleLogManager.Instance); + BlockhashProvider blockHashProvider = new(blockTree, SimpleConsoleLogManager.Instance); + VirtualMachine virtualMachine = new( + blockHashProvider, + specProvider, + SimpleConsoleLogManager.Instance); + TransactionProcessor txProcessor = new( + specProvider, + worldState, + virtualMachine, + SimpleConsoleLogManager.Instance); + BlockProcessor blockProcessor = new( + specProvider, + Always.Valid, + NoBlockRewards.Instance, + new BlockProcessor.BlockValidationTransactionsExecutor(txProcessor, worldState), + worldState, + NullReceiptStorage.Instance, + NullWitnessCollector.Instance, + SimpleConsoleLogManager.Instance); + BlockchainProcessor blockchainProcessor = new( + blockTree, + blockProcessor, + NullRecoveryStep.Instance, + stateReader, + SimpleConsoleLogManager.Instance, + BlockchainProcessor.Options.Default); + BuildBlocksWhenRequested trigger = new(); + ManualTimestamper timestamper = new ManualTimestamper(); + DevBlockProducer devBlockProducer = new( + EmptyTxSource.Instance, + blockchainProcessor, + worldState, + blockTree, + trigger, + timestamper, + specProvider, + new BlocksConfig(), + SimpleConsoleLogManager.Instance); + + blockchainProcessor.Start(); + devBlockProducer.Start(); + ProducedBlockSuggester suggester = new ProducedBlockSuggester(blockTree, devBlockProducer); + + AutoResetEvent autoResetEvent = new(false); + + blockTree.NewHeadBlock += (s, e) => autoResetEvent.Set(); + blockTree.SuggestBlock(Build.A.Block.Genesis.WithStateRoot(Keccak.Zero).TestObject); + + autoResetEvent.WaitOne(1000).Should().BeTrue("genesis"); + + trigger.BuildBlock(); + autoResetEvent.WaitOne(1000).Should().BeTrue("1"); + blockTree.Head.Number.Should().Be(1); + } + + [Test, Timeout(Timeout.MaxTestTime)] + public void TestVerkleBlocksWithExecutionWitness() + { + ISpecProvider specProvider = new TestSpecProvider(Prague.Instance); + DbProvider dbProvider = new(DbModeHint.Mem); + dbProvider.RegisterDb(DbNames.BlockInfos, new MemDb()); + dbProvider.RegisterDb(DbNames.Blocks, new MemDb()); + dbProvider.RegisterDb(DbNames.Headers, new MemDb()); + dbProvider.RegisterDb(DbNames.State, new MemDb()); + dbProvider.RegisterDb(DbNames.Code, new MemDb()); + dbProvider.RegisterDb(DbNames.Metadata, new MemDb()); + dbProvider.RegisterDb(DbNames.Leaf, new MemDb()); + dbProvider.RegisterDb(DbNames.InternalNodes, new MemDb()); + dbProvider.RegisterDb(DbNames.ForwardDiff, new MemDb()); + dbProvider.RegisterDb(DbNames.ReverseDiff, new MemDb()); + dbProvider.RegisterDb(DbNames.StateRootToBlock, new MemDb()); + + BlockTree blockTree = new( + dbProvider, + new ChainLevelInfoRepository(dbProvider), + specProvider, + NullBloomStorage.Instance, + SimpleConsoleLogManager.Instance); + VerkleStateTree stateTree = new (dbProvider, SimpleConsoleLogManager.Instance); + VerkleStateReader stateReader = new(stateTree, dbProvider.GetDb(DbNames.Code), SimpleConsoleLogManager.Instance); + VerkleWorldState worldState = new (stateTree, dbProvider.RegisteredDbs[DbNames.Code], SimpleConsoleLogManager.Instance); + + worldState.CreateAccount(TestItem.AddressA, 1000.Ether()); + worldState.CreateAccount(TestItem.AddressB, 1000.Ether()); + worldState.CreateAccount(TestItem.AddressC, 1000.Ether()); + + byte[] code = Bytes.FromHexString("0xabcd"); + worldState.InsertCode(TestItem.AddressA, code, specProvider.GenesisSpec); + + worldState.Set(new StorageCell(TestItem.AddressA, UInt256.One), Bytes.FromHexString("0xabcdef")); + + worldState.Commit(specProvider.GenesisSpec); + worldState.CommitTree(0); + + BlockhashProvider blockHashProvider = new(blockTree, SimpleConsoleLogManager.Instance); + VirtualMachine virtualMachine = new( + blockHashProvider, + specProvider, + SimpleConsoleLogManager.Instance); + TransactionProcessor txProcessor = new( + specProvider, + worldState, + virtualMachine, + SimpleConsoleLogManager.Instance); + BlockProcessor blockProcessor = new( + specProvider, + Always.Valid, + NoBlockRewards.Instance, + new BlockProcessor.BlockValidationTransactionsExecutor(txProcessor, worldState), + worldState, + NullReceiptStorage.Instance, + NullWitnessCollector.Instance, + SimpleConsoleLogManager.Instance); + BlockchainProcessor blockchainProcessor = new( + blockTree, + blockProcessor, + NullRecoveryStep.Instance, + stateReader, + SimpleConsoleLogManager.Instance, + BlockchainProcessor.Options.Default); + BuildBlocksWhenRequested trigger = new(); + ManualTimestamper timestamper = new ManualTimestamper(); + DevBlockProducer devBlockProducer = new( + EmptyTxSource.Instance, + blockchainProcessor, + worldState, + blockTree, + trigger, + timestamper, + specProvider, + new BlocksConfig(), + SimpleConsoleLogManager.Instance); + + blockchainProcessor.Start(); + devBlockProducer.Start(); + ProducedBlockSuggester suggester = new ProducedBlockSuggester(blockTree, devBlockProducer); + + AutoResetEvent autoResetEvent = new(false); + + blockTree.NewHeadBlock += (s, e) => autoResetEvent.Set(); + blockTree.SuggestBlock(Build.A.Block.Genesis.WithStateRoot(worldState.StateRoot).TestObject); + + autoResetEvent.WaitOne(1000).Should().BeTrue("genesis"); + + trigger.BuildBlock(); + autoResetEvent.WaitOne(1000).Should().BeTrue("1"); + blockTree.Head.Number.Should().Be(1); + } + + [Test, Timeout(Timeout.MaxTestTime)] + public void TestVerkleBlocksWithExecutionWitnessAndStatelessValidation() + { + ISpecProvider specProvider = new TestSpecProvider(Prague.Instance); + DbProvider dbProvider = new(DbModeHint.Mem); + dbProvider.RegisterDb(DbNames.BlockInfos, new MemDb()); + dbProvider.RegisterDb(DbNames.Blocks, new MemDb()); + dbProvider.RegisterDb(DbNames.Headers, new MemDb()); + dbProvider.RegisterDb(DbNames.State, new MemDb()); + dbProvider.RegisterDb(DbNames.Code, new MemDb()); + dbProvider.RegisterDb(DbNames.Metadata, new MemDb()); + dbProvider.RegisterDb(DbNames.Leaf, new MemDb()); + dbProvider.RegisterDb(DbNames.InternalNodes, new MemDb()); + dbProvider.RegisterDb(DbNames.ForwardDiff, new MemDb()); + dbProvider.RegisterDb(DbNames.ReverseDiff, new MemDb()); + dbProvider.RegisterDb(DbNames.StateRootToBlock, new MemDb()); + + BlockTree blockTree = new( + dbProvider, + new ChainLevelInfoRepository(dbProvider), + specProvider, + NullBloomStorage.Instance, + SimpleConsoleLogManager.Instance); + VerkleStateTree stateTree = new (dbProvider, SimpleConsoleLogManager.Instance); + VerkleStateReader stateReader = new(stateTree, dbProvider.GetDb(DbNames.Code), SimpleConsoleLogManager.Instance); + VerkleWorldState worldState = new (stateTree, dbProvider.RegisteredDbs[DbNames.Code], SimpleConsoleLogManager.Instance); + BlockhashProvider blockHashProvider = new(blockTree, SimpleConsoleLogManager.Instance); + VirtualMachine virtualMachine = new( + blockHashProvider, + specProvider, + SimpleConsoleLogManager.Instance); + TransactionProcessor txProcessor = new( + specProvider, + worldState, + virtualMachine, + SimpleConsoleLogManager.Instance); + BlockProcessor blockProcessor = new( + specProvider, + Always.Valid, + NoBlockRewards.Instance, + new BlockProcessor.BlockValidationTransactionsExecutor(txProcessor, worldState), + worldState, + NullReceiptStorage.Instance, + NullWitnessCollector.Instance, + SimpleConsoleLogManager.Instance); + BlockchainProcessor blockchainProcessor = new( + blockTree, + blockProcessor, + NullRecoveryStep.Instance, + stateReader, + SimpleConsoleLogManager.Instance, + BlockchainProcessor.Options.Default); + BuildBlocksWhenRequested trigger = new(); + ManualTimestamper timestamper = new ManualTimestamper(); + DevBlockProducer devBlockProducer = new( + EmptyTxSource.Instance, + blockchainProcessor, + worldState, + blockTree, + trigger, + timestamper, + specProvider, + new BlocksConfig(), + SimpleConsoleLogManager.Instance); + + blockchainProcessor.Start(); + devBlockProducer.Start(); + ProducedBlockSuggester suggester = new ProducedBlockSuggester(blockTree, devBlockProducer); + + AutoResetEvent autoResetEvent = new(false); + + blockTree.NewHeadBlock += (s, e) => autoResetEvent.Set(); + blockTree.SuggestBlock(Build.A.Block.Genesis.WithStateRoot(Keccak.Zero).TestObject); + + autoResetEvent.WaitOne(1000).Should().BeTrue("genesis"); + + trigger.BuildBlock(); + autoResetEvent.WaitOne(1000).Should().BeTrue("1"); + blockTree.Head.Number.Should().Be(1); + } } } diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/VerkleBlockProducerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/VerkleBlockProducerTests.cs new file mode 100644 index 00000000000..a5958f47842 --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/VerkleBlockProducerTests.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading.Tasks; +using Nethermind.Core.Test.Blockchain; +using NUnit.Framework; + +namespace Nethermind.Blockchain.Test.Producers; + +public class VerkleBlockProducerTests +{ + [Test] + public async Task TestBlockProductionWithWitness() + { + TestVerkleBlockchain chain = await TestVerkleBlockchain.Create(); + await chain.BuildSomeBlocksWithDeployment(5); + await chain.BuildSomeBlocks(100); + } +} diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs index 65c6d055cca..a9ef7a85c7c 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs @@ -326,7 +326,7 @@ void SetAccountStates(IEnumerable
missingAddresses) SetAccountStates(testCase.MissingAddresses); - BlockReceiptsTracer receiptsTracer = new(); + BlockReceiptsTracer receiptsTracer = new(true, false); receiptsTracer.StartNewBlockTrace(blockToProduce); txExecutor.ProcessTransactions(blockToProduce, ProcessingOptions.ProducingBlock, receiptsTracer, spec); diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTreeMethodOptions.cs b/src/Nethermind/Nethermind.Blockchain/BlockTreeMethodOptions.cs index 2eaab2ba9be..7e1b288a20f 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTreeMethodOptions.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTreeMethodOptions.cs @@ -69,6 +69,11 @@ public enum BlockTreeSuggestOptions /// Force not to set as main block ///
ForceDontSetAsMain = 8, + + /// + /// If block should be processed in a stateless manner + /// + ShouldProcessStateless = 16, } public static class BlockTreeSuggestOptionsExtensions diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index 0a763dd2f73..042235c462f 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -103,5 +103,11 @@ public interface ISyncConfig : IConfig [ConfigItem(Description = "[EXPERIMENTAL] Optimize db for write during sync. Significantly reduce total writes written and some sync time if you are not network limited.", DefaultValue = "Default")] public ITunableDb.TuneType TuneDbMode { get; set; } + + [ConfigItem(Description = "[EXPERIMENTAL] Enables Verkle Sync protocol.", DefaultValue = "false")] + public bool VerkleSync { get; set; } + + [ConfigItem(Description = "Number of account range partition to create. Increase verkle sync request concurrency. Value must be between 1 to 256 (inclusive).", DefaultValue = "8")] + int VerkleSyncAccountRangePartitionCount { get; set; } } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/IVerkleSyncPeer.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/IVerkleSyncPeer.cs new file mode 100644 index 00000000000..58d6f3781bd --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/IVerkleSyncPeer.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Blockchain.Synchronization; + +public interface IVerkleSyncPeer +{ + Task GetSubTreeRange(SubTreeRange range, CancellationToken token); + Task GetLeafNodes(GetLeafNodesRequest request, CancellationToken token); + Task GetLeafNodes(LeafToRefreshRequest request, CancellationToken token); +} diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index d99576e4861..62dcbe1b4dc 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -61,6 +61,9 @@ public string? PivotHash public bool NonValidatorNode { get; set; } = false; public ITunableDb.TuneType TuneDbMode { get; set; } = ITunableDb.TuneType.Default; + public bool VerkleSync { get; set; } = false; + public int VerkleSyncAccountRangePartitionCount { get; set; } = 8; + public override string ToString() { return diff --git a/src/Nethermind/Nethermind.Blockchain/TrieStoreBoundaryWatcher.cs b/src/Nethermind/Nethermind.Blockchain/TrieStoreBoundaryWatcher.cs index 8aebd45a381..1a687e5de8a 100644 --- a/src/Nethermind/Nethermind.Blockchain/TrieStoreBoundaryWatcher.cs +++ b/src/Nethermind/Nethermind.Blockchain/TrieStoreBoundaryWatcher.cs @@ -13,11 +13,11 @@ namespace Nethermind.Blockchain /// public class TrieStoreBoundaryWatcher : IDisposable { - private readonly ITrieStore _trieStore; + private readonly IStoreWithReorgBoundary _trieStore; private readonly IBlockTree _blockTree; private readonly ILogger _logger; - public TrieStoreBoundaryWatcher(ITrieStore trieStore, IBlockTree blockTree, ILogManager logManager) + public TrieStoreBoundaryWatcher(IStoreWithReorgBoundary trieStore, IBlockTree blockTree, ILogManager logManager) { _trieStore = trieStore; _blockTree = blockTree; @@ -27,7 +27,7 @@ public TrieStoreBoundaryWatcher(ITrieStore trieStore, IBlockTree blockTree, ILog private void OnReorgBoundaryReached(object? sender, ReorgBoundaryReached e) { - if (_logger.IsDebug) _logger.Debug($"Saving reorg boundary {e.BlockNumber}"); + _logger.Info($"Saving reorg boundary {e.BlockNumber}"); _blockTree.BestPersistedState = e.BlockNumber; } diff --git a/src/Nethermind/Nethermind.Config/BlocksConfig.cs b/src/Nethermind/Nethermind.Config/BlocksConfig.cs index 1462905a388..fa6fa8aa490 100644 --- a/src/Nethermind/Nethermind.Config/BlocksConfig.cs +++ b/src/Nethermind/Nethermind.Config/BlocksConfig.cs @@ -21,6 +21,7 @@ public class BlocksConfig : IBlocksConfig public bool RandomizedBlocks { get; set; } public ulong SecondsPerSlot { get; set; } = 12; + public bool StatelessProcessing { get; set; } = false; public string ExtraData diff --git a/src/Nethermind/Nethermind.Config/IBlocksConfig.cs b/src/Nethermind/Nethermind.Config/IBlocksConfig.cs index 554c0286121..c7acd5c4267 100644 --- a/src/Nethermind/Nethermind.Config/IBlocksConfig.cs +++ b/src/Nethermind/Nethermind.Config/IBlocksConfig.cs @@ -35,6 +35,9 @@ public interface IBlocksConfig : IConfig [ConfigItem(Description = "Seconds per slot.", DefaultValue = "12")] ulong SecondsPerSlot { get; set; } + [ConfigItem(Description = "Should process block stateless if possible.", DefaultValue = "false")] + bool StatelessProcessing { get; set; } + byte[] GetExtraDataBytes(); } } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs index 52dc575fab0..52c1d90705f 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs @@ -25,7 +25,6 @@ namespace Nethermind.Consensus.AuRa { public class AuRaBlockProcessor : BlockProcessor { - private readonly ISpecProvider _specProvider; private readonly IBlockTree _blockTree; private readonly AuRaContractGasLimitOverride? _gasLimitOverride; private readonly ContractRewriter? _contractRewriter; @@ -57,7 +56,6 @@ public AuRaBlockProcessor( logManager, withdrawalProcessor) { - _specProvider = specProvider; _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _txFilter = txFilter ?? NullTxFilter.Instance; diff --git a/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs b/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs index 4db8a7d0aa6..8c44e529e3f 100644 --- a/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs @@ -66,12 +66,9 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd if (logger.IsInfo) logger.Info("Starting Neth Dev block producer & sealer"); - ReadOnlyTxProcessingEnv producerEnv = new( - readOnlyDbProvider, - getFromApi.ReadOnlyTrieStore, - readOnlyBlockTree, - getFromApi.SpecProvider, - getFromApi.LogManager); + ReadOnlyTxProcessingEnv producerEnv = _nethermindApi.SpecProvider!.GenesisSpec.IsVerkleTreeEipEnabled + ? new ReadOnlyTxProcessingEnv(readOnlyDbProvider, getFromApi.ReadOnlyVerkleTrieStore, readOnlyBlockTree, getFromApi.SpecProvider, getFromApi.LogManager) + : new ReadOnlyTxProcessingEnv(readOnlyDbProvider, getFromApi.ReadOnlyTrieStore, readOnlyBlockTree, getFromApi.SpecProvider, getFromApi.LogManager); BlockProcessor producerProcessor = new( getFromApi!.SpecProvider, diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockExtensions.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockExtensions.cs index e9d9643a978..d5f295ba49c 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockExtensions.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockExtensions.cs @@ -12,8 +12,8 @@ internal static class BlockExtensions { public static Block CreateCopy(this Block block, BlockHeader header) => block is BlockToProduce blockToProduce - ? new BlockToProduce(header, blockToProduce.Transactions, blockToProduce.Uncles, blockToProduce.Withdrawals) - : new Block(header, block.Transactions, block.Uncles, block.Withdrawals); + ? new BlockToProduce(header, blockToProduce.Transactions, blockToProduce.Uncles, blockToProduce.Withdrawals, blockToProduce.ExecutionWitness) + : new Block(header, block.Transactions, block.Uncles, block.Withdrawals, block.ExecutionWitness); public static IEnumerable GetTransactions(this Block block) => block is BlockToProduce blockToProduce diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs index 05b11f9d7b7..abca0c79bc8 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs @@ -23,6 +23,7 @@ public class BlockProductionTransactionsExecutor : IBlockProductionTransactionsE private readonly IWorldState _stateProvider; private readonly BlockProductionTransactionPicker _blockProductionTransactionPicker; private readonly ILogger _logger; + private readonly ILogManager _logManager; public BlockProductionTransactionsExecutor( ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, @@ -46,6 +47,20 @@ public BlockProductionTransactionsExecutor( _stateProvider = stateProvider; _blockProductionTransactionPicker = new BlockProductionTransactionPicker(specProvider); _logger = logManager.GetClassLogger(); + _logManager = logManager; + } + + private BlockProductionTransactionsExecutor( + ITransactionProcessorAdapter transactionProcessor, + IWorldState stateProvider, + BlockProductionTransactionPicker blockProductionTransactionPicker, + ILogManager logManager) + { + _transactionProcessor = transactionProcessor; + _stateProvider = stateProvider; + _blockProductionTransactionPicker = blockProductionTransactionPicker; + _logger = logManager.GetClassLogger(); + _logManager = logManager; } protected EventHandler? _transactionProcessed; @@ -55,6 +70,12 @@ event EventHandler? IBlockProcessor.IBlockTransactionsExec remove => _transactionProcessed -= value; } + public IBlockProcessor.IBlockTransactionsExecutor WithNewStateProvider(IWorldState worldState) + { + return new BlockProductionTransactionsExecutor(_transactionProcessor, worldState, + _blockProductionTransactionPicker, _logManager); + } + event EventHandler? IBlockProductionTransactionsExecutor.AddingTransaction { add => _blockProductionTransactionPicker.AddingTransaction += value; @@ -64,7 +85,8 @@ event EventHandler? IBlockProductionTransactionsExecutor.Addi public virtual TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec) { IEnumerable transactions = GetTransactions(block); - + var logg = SimpleConsoleLogger.Instance; + logg.Info($"---------------------full this is old-----------------{_stateProvider.StateRoot}"); int i = 0; LinkedHashSet transactionsInBlock = new(ByHashTxComparer.Instance); foreach (Transaction currentTx in transactions) @@ -74,6 +96,7 @@ public virtual TxReceipt[] ProcessTransactions(Block block, ProcessingOptions pr } _stateProvider.Commit(spec, receiptsTracer); + logg.Info($"---------------------full this is new-----------------{_stateProvider.StateRoot}"); SetTransactions(block, transactionsInBlock); return receiptsTracer.TxReceipts.ToArray(); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockStatelessValidationTransactionsExecutor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockStatelessValidationTransactionsExecutor.cs new file mode 100644 index 00000000000..678f9097ae8 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockStatelessValidationTransactionsExecutor.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Crypto; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Consensus.Processing; + +public partial class BlockProcessor +{ + public class BlockStatelessValidationTransactionsExecutor : IBlockProcessor.IBlockTransactionsExecutor + { + private ITransactionProcessorAdapter _transactionProcessor; + private IWorldState? _stateProvider; + + public BlockStatelessValidationTransactionsExecutor(ITransactionProcessor transactionProcessor) + : this(new ExecuteTransactionProcessorAdapter(transactionProcessor)) + { + } + + public BlockStatelessValidationTransactionsExecutor(ITransactionProcessorAdapter transactionProcessor) + { + _transactionProcessor = transactionProcessor; + _stateProvider = null; + } + + public BlockStatelessValidationTransactionsExecutor(ITransactionProcessor transactionProcessor, IWorldState worldState) + : this(new ExecuteTransactionProcessorAdapter(transactionProcessor), worldState) + { + } + + public BlockStatelessValidationTransactionsExecutor(ITransactionProcessorAdapter transactionProcessor, IWorldState worldState) + { + _transactionProcessor = transactionProcessor; + _stateProvider = worldState; + } + + public event EventHandler? TransactionProcessed; + + public IBlockProcessor.IBlockTransactionsExecutor WithNewStateProvider(IWorldState worldState) + { + _transactionProcessor = _transactionProcessor.WithNewStateProvider(worldState); + return new BlockStatelessValidationTransactionsExecutor(_transactionProcessor, worldState); + } + + public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec) + { + // var ecdsa = new EthereumEcdsa(69420, LimboLogs.Instance); + if (!block.IsGenesis) + { + var logg = SimpleConsoleLogger.Instance; + logg.Info($"---------------------this is old-----------------{_stateProvider.StateRoot}"); + for (int i = 0; i < block.Transactions.Length; i++) + { + Transaction currentTx = block.Transactions[i]; + // currentTx.SenderAddress = ecdsa.RecoverAddress(currentTx); + ProcessTransaction(block, currentTx, i, receiptsTracer, _stateProvider, processingOptions); + } + _stateProvider.Commit(spec); + _stateProvider.RecalculateStateRoot(); + logg.Info($"---------------------this is new-----------------{_stateProvider.StateRoot}"); + } + return receiptsTracer.TxReceipts.ToArray(); + } + + private void ProcessTransaction(Block block, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, IWorldState worldState, ProcessingOptions processingOptions) + { + _transactionProcessor.ProcessTransaction(block, currentTx, receiptsTracer, processingOptions, worldState); + TransactionProcessed?.Invoke(this, new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index])); + } + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs index 4489c88109d..631187a755f 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs @@ -30,6 +30,10 @@ public BlockValidationTransactionsExecutor(ITransactionProcessorAdapter transact } public event EventHandler? TransactionProcessed; + public IBlockProcessor.IBlockTransactionsExecutor WithNewStateProvider(IWorldState worldState) + { + return new BlockValidationTransactionsExecutor(_transactionProcessor, worldState); + } public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec) { diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs index dfdecd8f4d0..160dceecb80 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Numerics; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; @@ -11,27 +12,37 @@ using Nethermind.Consensus.Withdrawals; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; +using Nethermind.Core.Verkle; using Nethermind.Crypto; using Nethermind.Evm.Tracing; using Nethermind.Int256; using Nethermind.Logging; +using Nethermind.Serialization.Json; using Nethermind.Specs.Forks; using Nethermind.State; +using Nethermind.Verkle.Curve; namespace Nethermind.Consensus.Processing; public partial class BlockProcessor : IBlockProcessor { private readonly ILogger _logger; - private readonly ISpecProvider _specProvider; + protected readonly ISpecProvider _specProvider; protected readonly IWorldState _stateProvider; private readonly IReceiptStorage _receiptStorage; private readonly IWitnessCollector _witnessCollector; - private readonly IWithdrawalProcessor _withdrawalProcessor; + protected readonly IWithdrawalProcessor _withdrawalProcessor; private readonly IBlockValidator _blockValidator; private readonly IRewardCalculator _rewardCalculator; - private readonly IBlockProcessor.IBlockTransactionsExecutor _blockTransactionsExecutor; + protected readonly IBlockProcessor.IBlockTransactionsExecutor _blockTransactionsExecutor; + + // TODO: will be removed in future + public IBlockProcessor.IBlockTransactionsExecutor StatelessBlockTransactionsExecutor; + public bool ShouldVerifyIncomingWitness { get; set; } = false; + public bool ShouldDoStatelessStuff { get; set; } = false; + public bool ShouldGenerateWitness { get; set; } = false; private const int MaxUncommittedBlocks = 64; @@ -39,7 +50,7 @@ public partial class BlockProcessor : IBlockProcessor /// We use a single receipt tracer for all blocks. Internally receipt tracer forwards most of the calls /// to any block-specific tracers. /// - private readonly BlockReceiptsTracer _receiptsTracer; + protected readonly BlockReceiptsTracer _receiptsTracer; public BlockProcessor( ISpecProvider? specProvider, @@ -63,7 +74,7 @@ public BlockProcessor( _blockTransactionsExecutor = blockTransactionsExecutor ?? throw new ArgumentNullException(nameof(blockTransactionsExecutor)); - _receiptsTracer = new BlockReceiptsTracer(); + _receiptsTracer = new BlockReceiptsTracer(true, true); } public event EventHandler BlockProcessed; @@ -144,7 +155,7 @@ the previous head state.*/ public event EventHandler? BlocksProcessing; // TODO: move to branch processor - private void InitBranch(Keccak branchStateRoot, bool incrementReorgMetric = true) + protected virtual void InitBranch(Keccak branchStateRoot, bool incrementReorgMetric = true) { /* Please note that we do not reset the state if branch state root is null. That said, I do not remember in what cases we receive null here.*/ @@ -188,6 +199,7 @@ private void RestoreBranch(Keccak branchingPointStateRoot) { if (_logger.IsTrace) _logger.Trace($"Processing block {suggestedBlock.ToString(Block.Format.Short)} ({options})"); + // here we assume (and its a good assumption) that DAO is before stateless execution starts ApplyDaoTransition(suggestedBlock); Block block = PrepareBlockForProcessing(suggestedBlock); TxReceipt[] receipts = ProcessBlock(block, blockTracer, options); @@ -211,6 +223,29 @@ private void ValidateProcessedBlock(Block suggestedBlock, ProcessingOptions opti } } + // TODO: remove + private (IBlockProcessor.IBlockTransactionsExecutor, IWorldState) GetOrCreateExecutorAndState(Block block, ExecutionWitness witness) + { + IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor; + IWorldState worldState; + if (block.IsGenesis) + { + blockTransactionsExecutor = StatelessBlockTransactionsExecutor; + worldState = _stateProvider; + + } + else + { + block.Header.MaybeParent!.TryGetTarget(out BlockHeader maybeParent); + Banderwagon stateRoot = Banderwagon.FromBytes(maybeParent!.StateRoot!.Bytes)!.Value; + worldState = new VerkleWorldState(witness, stateRoot, LimboLogs.Instance); + blockTransactionsExecutor = StatelessBlockTransactionsExecutor.WithNewStateProvider(worldState); + } + + return (blockTransactionsExecutor, worldState); + } + // TODO: remove + // TODO: block processor pipeline protected virtual TxReceipt[] ProcessBlock( Block block, @@ -219,6 +254,24 @@ protected virtual TxReceipt[] ProcessBlock( { IReleaseSpec spec = _specProvider.GetSpec(block.Header); + if (ShouldVerifyIncomingWitness) + { + if (!block.IsGenesis) + { + block.Header.MaybeParent!.TryGetTarget(out BlockHeader maybeParent); + Banderwagon stateRoot = Banderwagon.FromBytes(maybeParent!.StateRoot!.Bytes)!.Value; + try + { + VerkleWorldState? incomingWorldState = new (block.ExecutionWitness, stateRoot, LimboLogs.Instance); + _logger.Info($"Incoming Witness - VerkleWorldState StateRoot:{incomingWorldState.StateRoot}"); + } + catch (Exception e) + { + _logger.Error("Verkle proof verification failed for incoming witness.", e); + } + } + } + _receiptsTracer.SetOtherTracer(blockTracer); _receiptsTracer.StartNewBlockTrace(block); TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, options, _receiptsTracer, spec); @@ -228,12 +281,48 @@ protected virtual TxReceipt[] ProcessBlock( _withdrawalProcessor.ProcessWithdrawals(block, spec); _receiptsTracer.EndBlockTrace(); + ExecutionWitness? witness = null; + if (ShouldGenerateWitness) + { + byte[][] witnessKeys = _receiptsTracer.WitnessKeys.ToArray(); + VerkleWorldState? verkleWorldState = _stateProvider as VerkleWorldState; + witness = witnessKeys.Length == 0 ? null : verkleWorldState?.GenerateExecutionWitness(witnessKeys, out _); + // IJsonSerializer ser = new EthereumJsonSerializer(); + // _logger.Info($"BLOCK PROCESSOR WITNESS: {spec.IsVerkleTreeEipEnabled} {!block.IsGenesis} {options} {ser.Serialize(witness)}"); + if (options.ContainsFlag(ProcessingOptions.ProducingBlock) && spec.IsVerkleTreeEipEnabled && + !block.IsGenesis) block.Body.ExecutionWitness = witness; + } + _stateProvider.Commit(spec); _stateProvider.RecalculateStateRoot(); block.Header.StateRoot = _stateProvider.StateRoot; block.Header.Hash = block.Header.CalculateHash(); + if (ShouldGenerateWitness && ShouldDoStatelessStuff) + { + try + { + (IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor, IWorldState worldState) = + GetOrCreateExecutorAndState(block, witness!); + _receiptsTracer.StartNewBlockTrace(block); + TxReceipt[] receiptsSl = blockTransactionsExecutor.ProcessTransactions(block, options, _receiptsTracer, spec); + + block.Header.ReceiptsRoot = receipts.GetReceiptsRoot(spec, block.ReceiptsRoot); + ApplyMinerRewards(block, blockTracer, spec); + _withdrawalProcessor.ProcessWithdrawals(block, spec); + _receiptsTracer.EndBlockTrace(); + + worldState.Commit(spec); + worldState.RecalculateStateRoot(); + } + catch (Exception e) + { + _logger.Error($"Failed while doing stateless stuff", e); + return receipts; + } + } + return receipts; } @@ -274,12 +363,17 @@ private Block PrepareBlockForProcessing(Block suggestedBlock) WithdrawalsRoot = bh.WithdrawalsRoot, IsPostMerge = bh.IsPostMerge, }; + if (bh.MaybeParent is not null) + { + bh.MaybeParent.TryGetTarget(out BlockHeader maybeParent); + headerForProcessing.MaybeParent = new WeakReference(maybeParent); + } return suggestedBlock.CreateCopy(headerForProcessing); } // TODO: block processor pipeline - private void ApplyMinerRewards(Block block, IBlockTracer tracer, IReleaseSpec spec) + protected void ApplyMinerRewards(Block block, IBlockTracer tracer, IReleaseSpec spec) { if (_logger.IsTrace) _logger.Trace("Applying miner rewards:"); BlockReward[] rewards = _rewardCalculator.CalculateRewards(block); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs index 0e4ac8037d9..78cb729d4cb 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs @@ -31,6 +31,8 @@ public class BlockchainProcessor : IBlockchainProcessor, IBlockProcessingQueue public ITracerBag Tracers => _compositeBlockTracer; private readonly IBlockProcessor _blockProcessor; + private readonly IBlockProcessor? _statelessBlockProcessor; + private bool _canProcessStatelessBlocks = false; private readonly IBlockPreprocessorStep _recoveryStep; private readonly IStateReader _stateReader; private readonly Options _options; @@ -88,6 +90,40 @@ public BlockchainProcessor( _stats = new ProcessingStats(_logger); } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + public BlockchainProcessor( + IBlockTree? blockTree, + IBlockProcessor? blockProcessor, + IBlockProcessor? statelessBlockProcessor, + IBlockPreprocessorStep? recoveryStep, + IStateReader stateReader, + ILogManager? logManager, + Options options) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); + _blockProcessor = blockProcessor ?? throw new ArgumentNullException(nameof(blockProcessor)); + _statelessBlockProcessor = statelessBlockProcessor ?? throw new ArgumentNullException(nameof(statelessBlockProcessor)); + _canProcessStatelessBlocks = true; + _recoveryStep = recoveryStep ?? throw new ArgumentNullException(nameof(recoveryStep)); + _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader)); + _options = options; + + _blockTree.NewBestSuggestedBlock += OnNewBestBlock; + _blockTree.NewHeadBlock += OnNewHeadBlock; + + _stats = new ProcessingStats(_logger); + } + private void OnNewHeadBlock(object? sender, BlockEventArgs e) { _lastProcessedBlock = DateTime.UtcNow; @@ -109,7 +145,7 @@ private void OnNewBestBlock(object sender, BlockEventArgs blockEventArgs) public void Enqueue(Block block, ProcessingOptions processingOptions) { - if (_logger.IsTrace) _logger.Trace($"Enqueuing a new block {block.ToString(Block.Format.Short)} for processing."); + if (_logger.IsTrace) _logger.Trace($"Enqueuing a new block {block.ToString(Block.Format.Full)} for processing."); int currentRecoveryQueueSize = Interlocked.Add(ref _currentRecoveryQueueSize, block.Transactions.Length); Keccak? blockHash = block.Hash!; @@ -270,7 +306,7 @@ private void RunProcessingLoop() Block block = blockRef.Block; - if (_logger.IsTrace) _logger.Trace($"Processing block {block.ToString(Block.Format.Short)})."); + if (_logger.IsTrace) _logger.Trace($"Processing block in processor {block.ToString(Block.Format.Full)})."); _stats.Start(); Block processedBlock = Process(block, blockRef.ProcessingOptions, _compositeBlockTracer.GetTracer()); @@ -329,7 +365,8 @@ private void FireProcessingQueueEmpty() bool shouldProcess = suggestedBlock.IsGenesis || _blockTree.IsBetterThanHead(suggestedBlock.Header) - || options.ContainsFlag(ProcessingOptions.ForceProcessing); + || options.ContainsFlag(ProcessingOptions.ForceProcessing) + || options.ContainsFlag(ProcessingOptions.StatelessProcessing); if (!shouldProcess) { @@ -458,7 +495,7 @@ void DeleteInvalidBlocks(Keccak invalidBlockHash) TraceFailingBranch( processingBranch, options, - new BlockReceiptsTracer(), + new BlockReceiptsTracer(true, false), DumpOptions.Receipts); TraceFailingBranch( @@ -522,7 +559,14 @@ private void PrepareBlocksToProcess(Block suggestedBlock, ProcessingOptions opti if (!_stateReader.HasStateForBlock(parentOfFirstBlock)) { - throw new InvalidOperationException("Attempted to process a blockchain without having starting state"); + bool canThisBlockBeProcessedStateless = _canProcessStatelessBlocks && + (blocksToProcess[0].ExecutionWitness is not null); + // here we assume that if a block has execution witness - then all the following block will + // also have execution witness + if (!canThisBlockBeProcessedStateless) + { + throw new InvalidOperationException("Attempted to process a blockchain without having starting state"); + } } } } @@ -564,6 +608,7 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin branchingPoint = _blockTree.FindParentHeader(toBeProcessed.Header, BlockTreeLookupOptions.TotalDifficultyNotNeeded); + toBeProcessed.Header.MaybeParent = new WeakReference(branchingPoint); if (branchingPoint is null) { // genesis block @@ -595,6 +640,7 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin break; } + // TODO: check if we have a separate condition that we need to account for in Stateless Processing if (isFastSyncTransition) { // If we hit this condition, it means that something is wrong in MultiSyncModeSelector. @@ -657,6 +703,10 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin private bool RunSimpleChecksAheadOfProcessing(Block suggestedBlock, ProcessingOptions options) { /* a bit hacky way to get the invalid branch out of the processing loop */ + // if (suggestedBlock.ExecutionWitness is not null) + // { + // _logger.Info($"RunSimpleChecksAheadOfProcessing: ExecutionWitness Present {suggestedBlock.ToString(Block.Format.Short)}"); + // } if (suggestedBlock.Number != 0 && !_blockTree.IsKnownBlock(suggestedBlock.Number - 1, suggestedBlock.ParentHash)) { @@ -671,6 +721,7 @@ private bool RunSimpleChecksAheadOfProcessing(Block suggestedBlock, ProcessingOp if (_logger.IsDebug) _logger.Debug( $"Skipping processing block {suggestedBlock.ToString(Block.Format.FullHashAndNumber)} without total difficulty"); + // suggestedBlock.Header.TotalDifficulty = 1; throw new InvalidOperationException( "Block without total difficulty calculated was suggested for processing"); } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs index 277fc7c295b..68bf1496b2f 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs @@ -7,6 +7,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Consensus.Processing { @@ -47,6 +48,7 @@ public interface IBlockTransactionsExecutor { TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec); event EventHandler TransactionProcessed; + IBlockTransactionsExecutor WithNewStateProvider(IWorldState worldState); } } } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs index 555e1c9e1dd..3e7939580b8 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingOptions.cs @@ -51,7 +51,10 @@ public enum ProcessingOptions /// MarkAsProcessed = 128, - All = 255, + /// + /// Process block using execution witnesses + /// + StatelessProcessing = 256, /// /// Combination of switches for block producers when they preprocess block for state root calculation. diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs index f99c5f01b12..eae77fabaf9 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs @@ -12,6 +12,7 @@ using Nethermind.Logging; using Nethermind.State; using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; // ReSharper disable MemberCanBePrivate.Global // ReSharper disable UnusedAutoPropertyAccessor.Global @@ -37,6 +38,16 @@ public ReadOnlyTxProcessingEnv( { } + public ReadOnlyTxProcessingEnv( + IDbProvider? dbProvider, + ReadOnlyVerkleStateStore? trieStore, + IBlockTree? blockTree, + ISpecProvider? specProvider, + ILogManager? logManager) + : this(dbProvider?.AsReadOnly(false), trieStore, blockTree?.AsReadOnly(), specProvider, logManager) + { + } + public ReadOnlyTxProcessingEnv( IReadOnlyDbProvider? readOnlyDbProvider, IReadOnlyTrieStore? readOnlyTrieStore, @@ -59,6 +70,28 @@ public ReadOnlyTxProcessingEnv( TransactionProcessor = new TransactionProcessor(specProvider, StateProvider, Machine, logManager); } + public ReadOnlyTxProcessingEnv( + IReadOnlyDbProvider? readOnlyDbProvider, + ReadOnlyVerkleStateStore? readOnlyTrieStore, + IReadOnlyBlockTree? readOnlyBlockTree, + ISpecProvider? specProvider, + ILogManager? logManager) + { + if (specProvider is null) throw new ArgumentNullException(nameof(specProvider)); + + DbProvider = readOnlyDbProvider ?? throw new ArgumentNullException(nameof(readOnlyDbProvider)); + ReadOnlyDb codeDb = readOnlyDbProvider.CodeDb.AsReadOnly(true); + + StateReader = new VerkleStateReader(new VerkleStateTree(readOnlyTrieStore, logManager), codeDb, logManager); + StateProvider = new VerkleWorldState(new VerkleStateTree(readOnlyTrieStore, logManager), codeDb, logManager); + + BlockTree = readOnlyBlockTree ?? throw new ArgumentNullException(nameof(readOnlyBlockTree)); + BlockhashProvider = new BlockhashProvider(BlockTree, logManager); + + Machine = new VirtualMachine(BlockhashProvider, specProvider, logManager); + TransactionProcessor = new TransactionProcessor(specProvider, StateProvider, Machine, logManager); + } + public IReadOnlyTransactionProcessor Build(Keccak stateRoot) => new ReadOnlyTransactionProcessor(TransactionProcessor, StateProvider, stateRoot); } } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/StatelessBlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/StatelessBlockProcessor.cs new file mode 100644 index 00000000000..3ec78a2eedb --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/StatelessBlockProcessor.cs @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Linq; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Crypto; +using Nethermind.Evm.Tracing; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Verkle.Curve; + +namespace Nethermind.Consensus.Processing; + +public class StatelessBlockProcessor: BlockProcessor +{ + private readonly ILogger _logger; + private readonly ILogManager _logManager; + + public StatelessBlockProcessor( + ISpecProvider? specProvider, + IBlockValidator? blockValidator, + IRewardCalculator? rewardCalculator, + IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor, + IWorldState? stateProvider, + IReceiptStorage? receiptStorage, + IWitnessCollector? witnessCollector, + ILogManager? logManager, + IWithdrawalProcessor? withdrawalProcessor = null) + : base( + specProvider, + blockValidator, + rewardCalculator, + blockTransactionsExecutor, + stateProvider, + receiptStorage, + witnessCollector, + logManager, + withdrawalProcessor) + { + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager));; + _logger = _logManager.GetClassLogger(); + } + + protected override void InitBranch(Keccak branchStateRoot, bool incrementReorgMetric = true) + { + + } + + private (IBlockProcessor.IBlockTransactionsExecutor, IWorldState) GetOrCreateExecutorAndState(Block block) + { + IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor; + IWorldState worldState; + if (!block.IsGenesis) + { + block.Header.MaybeParent!.TryGetTarget(out BlockHeader maybeParent); + Banderwagon stateRoot = Banderwagon.FromBytes(maybeParent!.StateRoot!.Bytes)!.Value; + worldState = new VerkleWorldState(block.ExecutionWitness!, stateRoot, _logManager); + blockTransactionsExecutor = _blockTransactionsExecutor.WithNewStateProvider(worldState); + } + else + { + blockTransactionsExecutor = _blockTransactionsExecutor; + worldState = _stateProvider; + } + + return (blockTransactionsExecutor, worldState); + } + + protected override TxReceipt[] ProcessBlock( + Block block, + IBlockTracer blockTracer, + ProcessingOptions options) + { + (IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor, IWorldState worldState) = + GetOrCreateExecutorAndState(block); + + IReleaseSpec spec = _specProvider.GetSpec(block.Header); + + _receiptsTracer.SetOtherTracer(blockTracer); + _receiptsTracer.StartNewBlockTrace(block); + TxReceipt[] receipts = blockTransactionsExecutor.ProcessTransactions(block, options, _receiptsTracer, spec); + + block.Header.ReceiptsRoot = receipts.GetReceiptsRoot(spec, block.ReceiptsRoot); + ApplyMinerRewards(block, blockTracer, spec); + _withdrawalProcessor.ProcessWithdrawals(block, spec); + _receiptsTracer.EndBlockTrace(); + + // if we are producing blocks - then calculate and add the witness to the block + // but we need to remember that the witness needs to be generate for the parent block state (for pre state) + // byte[][]? usedKeysCac = _receiptsTracer.WitnessKeys.ToArray(); + // _logger.Info($"UsedKeys {usedKeysCac.Length}"); + // foreach (var key in usedKeysCac) + // { + // _logger.Info($"{key.ToHexString()}"); + // } + if (options.ContainsFlag(ProcessingOptions.ProducingBlock) && spec.IsVerkleTreeEipEnabled && !block.IsGenesis) + { + throw new InvalidOperationException($"{nameof(StatelessBlockProcessor)} cannot produce new block."); + } + + worldState.Commit(spec); + worldState.RecalculateStateRoot(); + + block.Header.StateRoot = worldState.StateRoot; + block.Header.Hash = block.Header.CalculateHash(); + + return receipts; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs index f3f54fb7849..d87736f2d5f 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs @@ -178,9 +178,11 @@ public bool IsProducingBlocks(ulong? maxProducingInterval) { if (TrySetState(parent.StateRoot)) { + // prepare the block - set deterministic fields and get the transactions we want to include in the block Block block = PrepareBlock(parent, payloadAttributes); if (PreparedBlockCanBeMined(block)) { + // execute the transactions and then get the stateRoot, txRoot, receipts, witnesses, etc. Block? processedBlock = ProcessPreparedBlock(block, blockTracer); if (processedBlock is null) { diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs index 3d88b53ba6a..a71ab59e5d0 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Config; @@ -12,10 +13,13 @@ using Nethermind.Consensus.Withdrawals; using Nethermind.Core.Specs; using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; using Nethermind.Trie.Pruning; using Nethermind.TxPool; +using Nethermind.Verkle; +using Nethermind.Verkle.Tree; namespace Nethermind.Consensus.Producers { @@ -24,6 +28,7 @@ public class BlockProducerEnvFactory : IBlockProducerEnvFactory protected readonly IDbProvider _dbProvider; protected readonly IBlockTree _blockTree; protected readonly IReadOnlyTrieStore _readOnlyTrieStore; + protected readonly ReadOnlyVerkleStateStore _readOnlyVerkleTrieStore; protected readonly ISpecProvider _specProvider; protected readonly IBlockValidator _blockValidator; protected readonly IRewardCalculatorSource _rewardCalculatorSource; @@ -33,6 +38,7 @@ public class BlockProducerEnvFactory : IBlockProducerEnvFactory protected readonly ITransactionComparerProvider _transactionComparerProvider; protected readonly IBlocksConfig _blocksConfig; protected readonly ILogManager _logManager; + protected readonly StateType _stateType; public IBlockTransactionsExecutorFactory TransactionsExecutorFactory { get; set; } @@ -62,6 +68,38 @@ public BlockProducerEnvFactory( _transactionComparerProvider = transactionComparerProvider; _blocksConfig = blocksConfig; _logManager = logManager; + _stateType = StateType.Merkle; + + TransactionsExecutorFactory = new BlockProducerTransactionsExecutorFactory(specProvider, logManager); + } + + public BlockProducerEnvFactory( + IDbProvider dbProvider, + IBlockTree blockTree, + ReadOnlyVerkleStateStore readOnlyTrieStore, + ISpecProvider specProvider, + IBlockValidator blockValidator, + IRewardCalculatorSource rewardCalculatorSource, + IReceiptStorage receiptStorage, + IBlockPreprocessorStep blockPreprocessorStep, + ITxPool txPool, + ITransactionComparerProvider transactionComparerProvider, + IBlocksConfig blocksConfig, + ILogManager logManager) + { + _dbProvider = dbProvider; + _blockTree = blockTree; + _readOnlyVerkleTrieStore = readOnlyTrieStore; + _specProvider = specProvider; + _blockValidator = blockValidator; + _rewardCalculatorSource = rewardCalculatorSource; + _receiptStorage = receiptStorage; + _blockPreprocessorStep = blockPreprocessorStep; + _txPool = txPool; + _transactionComparerProvider = transactionComparerProvider; + _blocksConfig = blocksConfig; + _logManager = logManager; + _stateType = StateType.Verkle; TransactionsExecutorFactory = new BlockProducerTransactionsExecutorFactory(specProvider, logManager); } @@ -106,8 +144,15 @@ public virtual BlockProducerEnv Create(ITxSource? additionalTxSource = null) }; } - protected virtual ReadOnlyTxProcessingEnv CreateReadonlyTxProcessingEnv(ReadOnlyDbProvider readOnlyDbProvider, ReadOnlyBlockTree readOnlyBlockTree) => - new(readOnlyDbProvider, _readOnlyTrieStore, readOnlyBlockTree, _specProvider, _logManager); + protected virtual ReadOnlyTxProcessingEnv CreateReadonlyTxProcessingEnv(ReadOnlyDbProvider readOnlyDbProvider, ReadOnlyBlockTree readOnlyBlockTree) + { + return _stateType switch + { + StateType.Merkle => new ReadOnlyTxProcessingEnv(readOnlyDbProvider, _readOnlyTrieStore, readOnlyBlockTree, _specProvider, _logManager), + StateType.Verkle => new ReadOnlyTxProcessingEnv(readOnlyDbProvider, _readOnlyVerkleTrieStore, readOnlyBlockTree, _specProvider, _logManager), + _ => throw new ArgumentOutOfRangeException() + }; + } protected virtual ITxSource CreateTxSourceForProducer( ITxSource? additionalTxSource, diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockToProduce.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockToProduce.cs index ab7bb2ffa9a..9c2c3252332 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockToProduce.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockToProduce.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.CompilerServices; using Nethermind.Core; +using Nethermind.Core.Verkle; //TODO: Redo clique block producer [assembly: InternalsVisibleTo("Nethermind.Consensus.Clique")] @@ -34,8 +35,9 @@ public BlockToProduce( BlockHeader blockHeader, IEnumerable transactions, IEnumerable uncles, - IEnumerable? withdrawals = null) - : base(blockHeader, Array.Empty(), uncles, withdrawals) + IEnumerable? withdrawals = null, + ExecutionWitness? execWitness = null) + : base(blockHeader, Array.Empty(), uncles, withdrawals, execWitness) { Transactions = transactions; } diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestVerkleBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestVerkleBlockchain.cs new file mode 100644 index 00000000000..f408e7f2573 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestVerkleBlockchain.cs @@ -0,0 +1,553 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Find; +using Nethermind.Blockchain.Receipts; +using Nethermind.Config; +using Nethermind.Consensus; +using Nethermind.Consensus.Comparers; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Transactions; +using Nethermind.Consensus.Validators; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Db; +using Nethermind.Db.Blooms; +using Nethermind.Db.Rocks; +using Nethermind.Evm; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Serialization.Json; +using Nethermind.Specs; +using Nethermind.Specs.Forks; +using Nethermind.Specs.Test; +using Nethermind.State; +using Nethermind.State.Repositories; +using Nethermind.TxPool; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.VerkleDb; +using BlockTree = Nethermind.Blockchain.BlockTree; + +namespace Nethermind.Core.Test.Blockchain; + +public class TestVerkleBlockchain +{ + public const int DefaultTimeout = 400000; + public const int PrivateKeyPoolCount = 10000; + + public PrivateKey[] RandomAddressWithBalance; + + public Random random = new Random(0); + + public VerkleStateReader StateReader { get; private set; } = null!; + public IEthereumEcdsa EthereumEcdsa { get; private set; } = null!; + public INonceManager NonceManager { get; private set; } = null!; + public TransactionProcessor TxProcessor { get; set; } = null!; + public IReceiptStorage ReceiptStorage { get; set; } = null!; + public ITxPool TxPool { get; set; } = null!; + public IDb CodeDb => DbProvider.CodeDb; + public IBlockProcessor BlockProcessor { get; set; } = null!; + public IBlockchainProcessor BlockchainProcessor { get; set; } = null!; + + public IBlockPreprocessorStep BlockPreprocessorStep { get; set; } = null!; + + public IBlockProcessingQueue BlockProcessingQueue { get; set; } = null!; + public IBlockTree BlockTree { get; set; } = null!; + + public IBlockFinder BlockFinder + { + get => _blockFinder ?? BlockTree; + set => _blockFinder = value; + } + + public ILogFinder LogFinder { get; private set; } = null!; + public IJsonSerializer JsonSerializer { get; set; } = null!; + public IWorldState State { get; set; } = null!; + public IReadOnlyStateProvider ReadOnlyState { get; private set; } = null!; + public IDb StateDb => DbProvider.StateDb; + public VerkleStateStore TrieStore { get; set; } = null!; + public IBlockProducer BlockProducer { get; private set; } = null!; + public IDbProvider DbProvider { get; set; } = null!; + public ISpecProvider SpecProvider { get; set; } = null!; + + public ISealEngine SealEngine { get; set; } = null!; + + public ITransactionComparerProvider TransactionComparerProvider { get; set; } = null!; + + public IPoSSwitcher PoSSwitcher { get; set; } = null!; + + public static async Task Create() + { + TestVerkleBlockchain chain = new (); + await chain.Build(); + return chain; + } + + public async Task BuildSomeBlocks(int numOfBlocks) + { + for (int i = 0; i < numOfBlocks; i++) + { + await AddBlock(GenerateRandomTxn(100)); + } + } + + public async Task BuildSomeBlocksWithDeployment(int numOfBlocks) + { + for (int i = 0; i < numOfBlocks; i++) + { + await AddBlock(GenerateRandomTxnWithCode(5)); + } + } + + public string SealEngineType { get; set; } = null!; + + + public SemaphoreSlim _resetEvent = null!; + private ManualResetEvent _suggestedBlockResetEvent = null!; + private AutoResetEvent _oneAtATime = new(true); + private IBlockFinder _blockFinder = null!; + + public static readonly UInt256 InitialValue = 100000.Ether(); + private TrieStoreBoundaryWatcher _trieStoreWatcher = null!; + public IHeaderValidator HeaderValidator { get; set; } = null!; + + public IBlockValidator BlockValidator { get; set; } = null!; + public BuildBlocksWhenRequested BlockProductionTrigger { get; } = new(); + + public ReadOnlyVerkleStateStore ReadOnlyTrieStore { get; private set; } = null!; + + public ManualTimestamper Timestamper { get; protected set; } = null!; + + public ProducedBlockSuggester Suggester { get; protected set; } = null!; + + + public static TransactionBuilder BuildSimpleTransaction => Builders.Build.A.Transaction.SignedAndResolved(TestItem.PrivateKeyA).To(TestItem.AddressB); + + public void GenerateRandomPrivateKeys(int numKeys) + { + RandomAddressWithBalance = new PrivateKey[numKeys]; + for (int i = 0; i < numKeys; i++) RandomAddressWithBalance[i] = new PrivateKey(TestItem.GetRandomKeccak(random).Bytes); + } + + public virtual async Task Build(ISpecProvider? specProvider = null, UInt256? initialValues = null) + { + GenerateRandomPrivateKeys(PrivateKeyPoolCount); + + Timestamper = new ManualTimestamper(new DateTime(2020, 2, 15, 12, 50, 30, DateTimeKind.Utc)); + JsonSerializer = new EthereumJsonSerializer(); + SpecProvider = CreateSpecProvider(specProvider ?? new TestSpecProvider(Prague.Instance)); + EthereumEcdsa = new EthereumEcdsa(SpecProvider.ChainId, LogManager); + + DbProvider = CreateDbProvider(); + TrieStore = new VerkleStateStore(DbProvider, LogManager); + byte[] code = Bytes.FromHexString("0xabcd"); + State = new VerkleWorldState(TrieStore, DbProvider.CodeDb, LogManager); + State.CreateAccount(TestItem.AddressA, (initialValues ?? InitialValue)); + State.CreateAccount(TestItem.AddressB, (initialValues ?? InitialValue)); + State.CreateAccount(TestItem.AddressC, (initialValues ?? InitialValue)); + State.CreateAccount(TestItem.AddressD, (initialValues ?? InitialValue)); + State.CreateAccount(TestItem.AddressE, (initialValues ?? InitialValue)); + State.CreateAccount(TestItem.AddressF, (initialValues ?? InitialValue)); + State.InsertCode(TestItem.AddressA, code, SpecProvider.GenesisSpec); + State.Set(new StorageCell(TestItem.AddressA, UInt256.One), Bytes.FromHexString("0xabcdef")); + State.Commit(SpecProvider.GenesisSpec); + State.CommitTree(0); + + ReadOnlyTrieStore = TrieStore.AsReadOnly(new VerkleMemoryDb()); + StateReader = new VerkleStateReader(ReadOnlyTrieStore, CodeDb, LogManager); + + IDb blockDb = new MemDb(); + IDb headerDb = new MemDb(); + IDb blockInfoDb = new MemDb(); + BlockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), SpecProvider, NullBloomStorage.Instance, SimpleConsoleLogManager.Instance); + ReadOnlyState = new ChainHeadReadOnlyStateProvider(BlockTree, StateReader); + TransactionComparerProvider = new TransactionComparerProvider(SpecProvider, BlockTree); + TxPool = CreateTxPool(); + + IChainHeadInfoProvider chainHeadInfoProvider = + new ChainHeadInfoProvider(SpecProvider, BlockTree, StateReader); + + NonceManager = new NonceManager(chainHeadInfoProvider.AccountStateProvider); + + _trieStoreWatcher = new TrieStoreBoundaryWatcher(TrieStore, BlockTree, SimpleConsoleLogManager.Instance); + + ReceiptStorage = new InMemoryReceiptStorage(); + VirtualMachine virtualMachine = new(new BlockhashProvider(BlockTree, LogManager), SpecProvider, SimpleConsoleLogManager.Instance); + TxProcessor = new TransactionProcessor(SpecProvider, State, virtualMachine, SimpleConsoleLogManager.Instance); + BlockPreprocessorStep = new RecoverSignatures(EthereumEcdsa, TxPool, SpecProvider, LogManager); + HeaderValidator = new HeaderValidator(BlockTree, Always.Valid, SpecProvider, LogManager); + + new ReceiptCanonicalityMonitor(BlockTree, ReceiptStorage, LogManager); + + BlockValidator = new BlockValidator( + new TxValidator(SpecProvider.ChainId), + HeaderValidator, + Always.Valid, + SpecProvider, + LogManager); + + PoSSwitcher = NoPoS.Instance; + ISealer sealer = new NethDevSealEngine(TestItem.AddressD); + SealEngine = new SealEngine(sealer, Always.Valid); + + BloomStorage bloomStorage = new(new BloomConfig(), new MemDb(), new InMemoryDictionaryFileStoreFactory()); + ReceiptsRecovery receiptsRecovery = new(new EthereumEcdsa(SpecProvider.ChainId, SimpleConsoleLogManager.Instance), SpecProvider); + LogFinder = new LogFinder(BlockTree, ReceiptStorage, ReceiptStorage, bloomStorage, SimpleConsoleLogManager.Instance, receiptsRecovery); + BlockProcessor = CreateBlockProcessor(); + + BlockchainProcessor chainProcessor = new(BlockTree, BlockProcessor, BlockPreprocessorStep, StateReader, SimpleConsoleLogManager.Instance, Consensus.Processing.BlockchainProcessor.Options.Default); + BlockchainProcessor = chainProcessor; + BlockProcessingQueue = chainProcessor; + chainProcessor.Start(); + + TxPoolTxSource txPoolTxSource = CreateTxPoolTxSource(); + ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(SpecProvider, BlockFinder); + BlockProducer = CreateTestBlockProducer(txPoolTxSource, sealer, transactionComparerProvider); + await BlockProducer.Start(); + Suggester = new ProducedBlockSuggester(BlockTree, BlockProducer); + + _resetEvent = new SemaphoreSlim(0); + _suggestedBlockResetEvent = new ManualResetEvent(true); + BlockTree.NewHeadBlock += OnNewHeadBlock; + BlockProducer.BlockProduced += (s, e) => + { + _suggestedBlockResetEvent.Set(); + }; + + Block? genesis = GetGenesisBlock(); + BlockTree.SuggestBlock(genesis); + + await WaitAsync(_resetEvent, "Failed to process genesis in time."); + await AirdropToAddresses(); + return this; + } + + + public Transaction[] GenerateRandomTxn(int numTxn) + { + Transaction[] txns = new Transaction[numTxn]; + + for (int i = 0; i < numTxn; i++) + { + int fromTx = random.Next(RandomAddressWithBalance.Length - 1); + int toTx = random.Next(RandomAddressWithBalance.Length - 1); + if (fromTx == toTx) toTx = fromTx + 1; + txns[i] = Builders.Build.A.Transaction.WithTo(RandomAddressWithBalance[fromTx].Address).WithValue(1.GWei()) + .SignedAndResolved(RandomAddressWithBalance[toTx]).TestObject; + } + + return txns; + } + + public Transaction[] GenerateRandomTxnWithCode(int numTxn) + { + List txns = new(); + int jump = RandomAddressWithBalance.Length / numTxn; + int key = 0; + while (key < RandomAddressWithBalance.Length) + { + byte[] code = new byte[TestItem.Random.Next(2000)]; + PrivateKey caller = RandomAddressWithBalance[key]; + UInt256 nonce = StateReader.GetNonce(BlockTree.Head!.StateRoot!, caller.Address); + txns.Add(Builders.Build.A.Transaction.WithTo(null).WithCode(code).WithNonce(nonce).WithGasLimit(200000) + .SignedAndResolved(caller).TestObject); + key += jump; + } + return txns.ToArray(); + } + + protected virtual async Task AirdropToAddresses() + { + int batch = 10; + + List txToAdd = new(); + for (int i = 0; i < batch; i++) + { + txToAdd.Add(Builders.Build.A.Transaction.WithTo(RandomAddressWithBalance[i].Address).WithGasLimit(100000).WithMaxFeePerGas(20.GWei()) + .WithMaxPriorityFeePerGas(5.GWei()) + .WithType(TxType.EIP1559).WithValue(1.Ether()).WithNonce((UInt256)i) + .SignedAndResolved(TestItem.PrivateKeyA).TestObject); + } + await AddBlock(txToAdd.ToArray()); + txToAdd.Clear(); + for (int i = batch; i < 2 * batch; i++) + { + txToAdd.Add(Builders.Build.A.Transaction.WithTo(RandomAddressWithBalance[i].Address).WithGasLimit(100000).WithMaxFeePerGas(30.GWei()) + .WithMaxPriorityFeePerGas(5.GWei()) + .WithType(TxType.EIP1559).WithValue(1.Ether()).WithNonce((UInt256)(i-batch)) + .SignedAndResolved(TestItem.PrivateKeyB).TestObject); + } + await AddBlock(txToAdd.ToArray()); + txToAdd.Clear(); + for (int i = 2 * batch; i < 3 * batch; i++) + { + txToAdd.Add(Builders.Build.A.Transaction.WithTo(RandomAddressWithBalance[i].Address).WithGasLimit(100000).WithMaxFeePerGas(40.GWei()) + .WithMaxPriorityFeePerGas(5.GWei()) + .WithType(TxType.EIP1559).WithValue(1.Ether()).WithNonce((UInt256)(i-2 * batch)) + .SignedAndResolved(TestItem.PrivateKeyC).TestObject); + } + await AddBlock(txToAdd.ToArray()); + txToAdd.Clear(); + for (int i = 3 * batch; i < 4 * batch; i++) + { + txToAdd.Add(Builders.Build.A.Transaction.WithTo(RandomAddressWithBalance[i].Address).WithGasLimit(100000).WithMaxFeePerGas(50.GWei()) + .WithMaxPriorityFeePerGas(5.GWei()) + .WithType(TxType.EIP1559).WithValue(1.Ether()).WithNonce((UInt256)(i-3 * batch)) + .SignedAndResolved(TestItem.PrivateKeyD).TestObject); + } + await AddBlock(txToAdd.ToArray()); + txToAdd.Clear(); + for (int i = 4 * batch; i < RandomAddressWithBalance.Length; i++) + { + txToAdd.Add(Builders.Build.A.Transaction.WithTo(RandomAddressWithBalance[i].Address).WithGasLimit(100000).WithMaxFeePerGas(60.GWei()) + .WithMaxPriorityFeePerGas(5.GWei()) + .WithType(TxType.EIP1559).WithValue(1.Ether()).WithNonce((UInt256)(i-4 * batch)) + .SignedAndResolved(TestItem.PrivateKeyE).TestObject); + } + } + + + + private static ISpecProvider CreateSpecProvider(ISpecProvider specProvider) + { + return specProvider is TestSpecProvider { AllowTestChainOverride: false } + ? specProvider + : new OverridableSpecProvider(specProvider, s => new OverridableReleaseSpec(s) { IsEip3607Enabled = false }); + } + + private void OnNewHeadBlock(object? sender, BlockEventArgs e) + { + _resetEvent.Release(1); + } + + protected virtual IDbProvider CreateDbProvider() => VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + + private async Task WaitAsync(SemaphoreSlim semaphore, string error, int timeout = DefaultTimeout) + { + if (!await semaphore.WaitAsync(timeout)) + { + throw new InvalidOperationException(error); + } + } + + private async Task WaitAsync(EventWaitHandle eventWaitHandle, string error, int timeout = DefaultTimeout) + { + if (!await eventWaitHandle.WaitOneAsync(timeout, CancellationToken.None)) + { + throw new InvalidOperationException(error); + } + } + + protected virtual IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolTxSource, ISealer sealer, ITransactionComparerProvider transactionComparerProvider) + { + BlocksConfig blocksConfig = new(); + + BlockProducerEnvFactory blockProducerEnvFactory = new( + DbProvider, + BlockTree, + ReadOnlyTrieStore, + SpecProvider, + BlockValidator, + NoBlockRewards.Instance, + ReceiptStorage, + BlockPreprocessorStep, + TxPool, + transactionComparerProvider, + blocksConfig, + SimpleConsoleLogManager.Instance); + + BlockProducerEnv env = blockProducerEnvFactory.Create(txPoolTxSource); + return new TestBlockProducer( + env.TxSource, + env.ChainProcessor, + env.ReadOnlyStateProvider, + sealer, + BlockTree, + BlockProductionTrigger, + Timestamper, + SpecProvider, + SimpleConsoleLogManager.Instance, + blocksConfig); + } + + public virtual ILogManager LogManager { get; set; } = SimpleConsoleLogManager.Instance; + + protected virtual TxPool.TxPool CreateTxPool() => + new( + EthereumEcdsa, + new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(SpecProvider), BlockTree, ReadOnlyState), + new TxPoolConfig(), + new TxValidator(SpecProvider.ChainId), + LimboLogs.Instance, + TransactionComparerProvider.GetDefaultComparer()); + + protected virtual TxPoolTxSource CreateTxPoolTxSource() + { + BlocksConfig blocksConfig = new() + { + MinGasPrice = 0 + }; + ITxFilterPipeline txFilterPipeline = TxFilterPipelineBuilder.CreateStandardFilteringPipeline(SimpleConsoleLogManager.Instance, + SpecProvider, blocksConfig); + return new TxPoolTxSource(TxPool, SpecProvider, TransactionComparerProvider, LogManager, txFilterPipeline); + } + + public BlockBuilder GenesisBlockBuilder { get; set; } = null!; + + protected virtual Block GetGenesisBlock() + { + BlockBuilder genesisBlockBuilder = Builders.Build.A.Block.Genesis; + if (GenesisBlockBuilder is not null) + { + genesisBlockBuilder = GenesisBlockBuilder; + } + + genesisBlockBuilder.WithStateRoot(State.StateRoot); + if (SealEngineType == Nethermind.Core.SealEngineType.AuRa) + { + genesisBlockBuilder.WithAura(0, new byte[65]); + } + + return genesisBlockBuilder.TestObject; + } + + protected virtual async Task AddBlocksOnStart() + { + await AddBlock(BuildSimpleTransaction.WithNonce(0).TestObject); + await AddBlock(BuildSimpleTransaction.WithNonce(1).TestObject, BuildSimpleTransaction.WithNonce(2).TestObject); + await AddBlock(BuildSimpleTransaction.WithNonce(3).TestObject, BuildSimpleTransaction.WithNonce(4).TestObject); + } + + protected virtual IBlockProcessor CreateBlockProcessor() => + new StatelessBlockProcessor( + SpecProvider, + BlockValidator, + NoBlockRewards.Instance, + new BlockProcessor.BlockStatelessValidationTransactionsExecutor(TxProcessor), + State, + ReceiptStorage, + NullWitnessCollector.Instance, + SimpleConsoleLogManager.Instance); + + public async Task WaitForNewHead() + { + await WaitAsync(_resetEvent, "Failed to produce new head in time."); + _suggestedBlockResetEvent.Reset(); + } + + public async Task AddBlock(params Transaction[] transactions) + { + await AddBlockInternal(transactions); + + await WaitAsync(_resetEvent, "Failed to produce new head in time."); + _suggestedBlockResetEvent.Reset(); + _oneAtATime.Set(); + } + + public async Task AddBlock(bool shouldWaitForHead = true, params Transaction[] transactions) + { + await AddBlockInternal(transactions); + + if (shouldWaitForHead) + { + await WaitAsync(_resetEvent, "Failed to produce new head in time."); + } + else + { + await WaitAsync(_suggestedBlockResetEvent, "Failed to produce new suggested block in time."); + } + + _oneAtATime.Set(); + } + + private async Task AddBlockInternal(params Transaction[] transactions) + { + // we want it to be last event, so lets re-register + BlockTree.NewHeadBlock -= OnNewHeadBlock; + BlockTree.NewHeadBlock += OnNewHeadBlock; + + await WaitAsync(_oneAtATime, "Multiple block produced at once."); + AcceptTxResult[] txResults = transactions.Select(t => TxPool.SubmitTx(t, TxHandlingOptions.None)).ToArray(); + Timestamper.Add(TimeSpan.FromSeconds(1)); + await BlockProductionTrigger.BuildBlock(); + return txResults; + } + + public void AddTransactions(params Transaction[] txs) + { + for (int i = 0; i < txs.Length; i++) + { + TxPool.SubmitTx(txs[i], TxHandlingOptions.None); + } + } + + public virtual void Dispose() + { + BlockProducer?.StopAsync(); + if (DbProvider != null) + { + CodeDb?.Dispose(); + StateDb?.Dispose(); + } + + _trieStoreWatcher?.Dispose(); + DbProvider?.Dispose(); + } + + /// + /// Creates a simple transfer transaction with value defined by + /// from a rich account to + /// + /// Address to add funds to + /// Value of ether to add to the account + /// + public async Task AddFunds(Address address, UInt256 ether) => + await AddBlock(GetFundsTransaction(address, ether)); + + public async Task AddFunds(params (Address address, UInt256 ether)[] funds) => + await AddBlock(funds.Select((f, i) => GetFundsTransaction(f.address, f.ether, (uint)i)).ToArray()); + + public async Task AddFundsAfterLondon(params (Address address, UInt256 ether)[] funds) => + await AddBlock(funds.Select((f, i) => GetFunds1559Transaction(f.address, f.ether, (uint)i)).ToArray()); + + + + private Transaction GetFundsTransaction(Address address, UInt256 ether, uint index = 0) + { + UInt256 nonce = StateReader.GetNonce(BlockTree.Head!.StateRoot!, TestItem.AddressA); + Transaction tx = Builders.Build.A.Transaction + .SignedAndResolved(TestItem.PrivateKeyA) + .To(address) + .WithNonce(nonce + index) + .WithValue(ether) + .TestObject; + return tx; + } + + private Transaction GetFunds1559Transaction(Address address, UInt256 ether, uint index = 0) + { + UInt256 nonce = StateReader.GetNonce(BlockTree.Head!.StateRoot!, TestItem.AddressA); + Transaction tx = Builders.Build.A.Transaction + .SignedAndResolved(TestItem.PrivateKeyA) + .To(address) + .WithNonce(nonce + index) + .WithMaxFeePerGas(20.GWei()) + .WithMaxPriorityFeePerGas(5.GWei()) + .WithType(TxType.EIP1559) + .WithValue(ether) + .WithChainId(MainnetSpecProvider.Instance.ChainId) + .TestObject; + return tx; + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/AccountBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/AccountBuilder.cs index 1b279572264..5cef48c7a4a 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/AccountBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/AccountBuilder.cs @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Microsoft.AspNetCore.Mvc.Controllers; using Nethermind.Core.Crypto; using Nethermind.Int256; +using Nethermind.Serialization.Rlp; namespace Nethermind.Core.Test.Builders { @@ -27,7 +29,7 @@ public AccountBuilder WithNonce(UInt256 nonce) public AccountBuilder WithCode(byte[] code) { - TestObjectInternal = TestObjectInternal.WithChangedCodeHash(Keccak.Compute(code)); + TestObjectInternal = TestObjectInternal.WithChangedCodeHash(Keccak.Compute(code), code); return this; } diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs index 5792e4a646b..ec4b6b4e24d 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs @@ -5,6 +5,7 @@ using System.Linq; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Core.Verkle; using Nethermind.Crypto; using Nethermind.Int256; using Nethermind.State.Proofs; @@ -260,5 +261,12 @@ public BlockBuilder WithWithdrawals(params Withdrawal[]? withdrawals) return this; } + + public BlockBuilder WithExecutionWitness(ExecutionWitness? witness) + { + TestObjectInternal = TestObjectInternal + .WithReplacedBody(TestObjectInternal.Body.WithChangedExecutionWitness(witness)); + return this; + } } } diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs index ba34d21ceb5..e0ff37c2609 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs @@ -3,17 +3,23 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Net; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Crypto; using Nethermind.Db; +using Nethermind.Db.Rocks; using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Serialization.Rlp; using Nethermind.State; using Nethermind.State.Snap; using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Sync; namespace Nethermind.Core.Test.Builders { @@ -23,12 +29,12 @@ public static class Tree { public static Keccak AccountAddress0 = new Keccak("0000000000000000000000000000000000000000000000000000000001101234"); - private static readonly Account _account0 = Build.An.Account.WithBalance(0).TestObject; - private static readonly Account _account1 = Build.An.Account.WithBalance(1).TestObject; - private static readonly Account _account2 = Build.An.Account.WithBalance(2).TestObject; - private static readonly Account _account3 = Build.An.Account.WithBalance(3).TestObject; - private static readonly Account _account4 = Build.An.Account.WithBalance(4).TestObject; - private static readonly Account _account5 = Build.An.Account.WithBalance(5).TestObject; + public static readonly Account _account0 = Build.An.Account.WithBalance(0).TestObject; + public static readonly Account _account1 = Build.An.Account.WithBalance(1).TestObject; + public static readonly Account _account2 = Build.An.Account.WithBalance(2).TestObject; + public static readonly Account _account3 = Build.An.Account.WithBalance(3).TestObject; + public static readonly Account _account4 = Build.An.Account.WithBalance(4).TestObject; + public static readonly Account _account5 = Build.An.Account.WithBalance(5).TestObject; public static PathWithAccount[] AccountsWithPaths = new PathWithAccount[] { @@ -61,6 +67,17 @@ public static StateTree GetStateTree(ITrieStore? store) return stateTree; } + public static VerkleStateTree GetVerkleStateTree(IVerkleTrieStore? store) + { + store ??= new VerkleStateStore(VerkleDbFactory.InitDatabase(DbMode.MemDb, null), LimboLogs.Instance); + + var stateTree = new VerkleStateTree(store, LimboLogs.Instance); + + // FillStateTreeWithTestAccounts(stateTree); + + return stateTree; + } + public static void FillStateTreeWithTestAccounts(StateTree stateTree) { stateTree.Set(AccountsWithPaths[0].Path, AccountsWithPaths[0].Account); @@ -72,6 +89,35 @@ public static void FillStateTreeWithTestAccounts(StateTree stateTree) stateTree.Commit(0); } + public static byte[] stem0 = new Keccak("0000000000000000000000000000000000000000000000000000000001101234").Bytes[1..]; + public static byte[] stem1 = new Keccak("0000000000000000000000000000000000000000000000000000000001112345").Bytes[1..]; + public static byte[] stem2 = new Keccak("0000000000000000000000000000000000000000000000000000000001113456").Bytes[1..]; + public static byte[] stem3 = new Keccak("0000000000000000000000000000000000000000000000000000000001114567").Bytes[1..]; + public static byte[] stem4 = new Keccak("0000000000000000000000000000000000000000000000000000000001123456").Bytes[1..]; + public static byte[] stem5 = new Keccak("0000000000000000000000000000000000000000000000000000000001123457").Bytes[1..]; + + public static PathWithSubTree[] SubTreesWithPaths = new PathWithSubTree[] + { + new PathWithSubTree(stem0, _account0.ToVerkleDict()), + new PathWithSubTree(stem1, _account1.ToVerkleDict()), + new PathWithSubTree(stem2, _account2.ToVerkleDict()), + new PathWithSubTree(stem3, _account3.ToVerkleDict()), + new PathWithSubTree(stem4, _account4.ToVerkleDict()), + new PathWithSubTree(stem5, _account5.ToVerkleDict()), + }; + + public static void FillStateTreeWithTestAccounts(VerkleStateTree stateTree) + { + stateTree.InsertStemBatch(stem0, _account0.ToVerkleDict()); + stateTree.InsertStemBatch(stem1, _account1.ToVerkleDict()); + stateTree.InsertStemBatch(stem2, _account2.ToVerkleDict()); + stateTree.InsertStemBatch(stem3, _account3.ToVerkleDict()); + stateTree.InsertStemBatch(stem4, _account4.ToVerkleDict()); + stateTree.InsertStemBatch(stem5, _account5.ToVerkleDict()); + stateTree.Commit(); + stateTree.CommitTree(0); + } + public static (StateTree stateTree, StorageTree storageTree) GetTrees(ITrieStore? store) { store ??= new TrieStore(new MemDb(), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.VerkleTree.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.VerkleTree.cs new file mode 100644 index 00000000000..a762a4bd3aa --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.VerkleTree.cs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.IO; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Builders; + +public static partial class TestItem +{ + public static readonly Stem Stem0 = new Stem("00000000000000000000000000000000000000000000000000000001101234"); + public static readonly Stem Stem1 = new Stem("00000000000000000000000000000000000000000000000000000001112345"); + public static readonly Stem Stem2 = new Stem("00000000000000000000000000000000000000000000000000000001113456"); + public static readonly Stem Stem3 = new Stem("00000000000000000000000000000000000000000000000000000001114567"); + public static readonly Stem Stem4 = new Stem("00000000000000000000000000000000000000000000000000000001123456"); + public static readonly Stem Stem5 = new Stem("00000000000000000000000000000000000000000000000000000001123457"); + + public static readonly Account _account0 = Build.An.Account.WithBalance(0).TestObject; + public static readonly Account _account1 = Build.An.Account.WithBalance(1).TestObject; + public static readonly Account _account2 = Build.An.Account.WithBalance(2).TestObject; + public static readonly Account _account3 = Build.An.Account.WithBalance(3).TestObject; + public static readonly Account _account4 = Build.An.Account.WithBalance(4).TestObject; + public static readonly Account _account5 = Build.An.Account.WithBalance(5).TestObject; + + public static PathWithSubTree[] SubTreesWithPaths = new PathWithSubTree[] + { + new PathWithSubTree(Stem0, _account0.ToVerkleDict()), + new PathWithSubTree(Stem1, _account1.ToVerkleDict()), + new PathWithSubTree(Stem2, _account2.ToVerkleDict()), + new PathWithSubTree(Stem3, _account3.ToVerkleDict()), + new PathWithSubTree(Stem4, _account4.ToVerkleDict()), + new PathWithSubTree(Stem5, _account5.ToVerkleDict()), + }; + + private static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + public static IVerkleTrieStore GetVerkleStore(DbMode dbMode, int history = 128) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + break; + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + break; + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + + return new VerkleStateStore(provider, LimboLogs.Instance,maxNumberOfBlocksInCache: history); + } + + public static VerkleStateTree GetVerkleStateTree(IVerkleTrieStore? store) + { + store ??= GetVerkleStore(DbMode.MemDb); + VerkleStateTree stateTree = new VerkleStateTree(store, LimboLogs.Instance); + FillStateTreeWithTestAccounts(stateTree); + return stateTree; + } + + public static void FillStateTreeWithTestAccounts(VerkleStateTree stateTree) + { + stateTree.InsertStemBatch(Stem0, _account0.ToVerkleDict()); + stateTree.InsertStemBatch(Stem1, _account1.ToVerkleDict()); + stateTree.InsertStemBatch(Stem2, _account2.ToVerkleDict()); + stateTree.InsertStemBatch(Stem3, _account3.ToVerkleDict()); + stateTree.InsertStemBatch(Stem4, _account4.ToVerkleDict()); + stateTree.InsertStemBatch(Stem5, _account5.ToVerkleDict()); + stateTree.Commit(); + stateTree.CommitTree(0); + } + + public static void InsertBigVerkleTree(VerkleTree tree, int numBlocks, long leafPerBlock, long pathPoolCount, out SortedDictionary leafs) + { + Pedersen[] pathPool = new Pedersen[pathPoolCount]; + leafs = new(); + + for (int i = 0; i < pathPoolCount; i++) + { + byte[] key = new byte[32]; + ((UInt256)i).ToBigEndian(key); + Pedersen keccak = new Pedersen(key); + pathPool[i] = keccak; + } + + + for (int leafIndex = 0; leafIndex < 10000; leafIndex++) + { + byte[] value = new byte[32]; + Random.NextBytes(value); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + tree.Insert(path, value); + leafs[path] = value; + } + + tree.Commit(); + tree.CommitTree(0); + + for (int blockNumber = 1; blockNumber <= numBlocks; blockNumber++) + { + for (int accountIndex = 0; accountIndex < leafPerBlock; accountIndex++) + { + byte[] leafValue = new byte[32]; + + + + Random.NextBytes(leafValue); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + + if (leafs.ContainsKey(path)) + { + if (!(Random.NextSingle() > 0.5)) continue; + Console.WriteLine($"blockNumber:{blockNumber} uKey:{path} uValue:{leafValue.ToHexString()}"); + tree.Insert(path, leafValue); + leafs[path] = leafValue; + Console.WriteLine("new values"); + } + else + { + Console.WriteLine($"blockNumber:{blockNumber} nKey:{path} nValue:{leafValue.ToHexString()}"); + tree.Insert(path, leafValue); + leafs[path] = leafValue; + } + } + + tree.Commit(); + tree.CommitTree(blockNumber); + } + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/Encoding/BlockDecoderTests.cs b/src/Nethermind/Nethermind.Core.Test/Encoding/BlockDecoderTests.cs index ed2e3f3fb23..eca3e8eb5dd 100644 --- a/src/Nethermind/Nethermind.Core.Test/Encoding/BlockDecoderTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Encoding/BlockDecoderTests.cs @@ -2,14 +2,22 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Generic; using System.IO; +using FluentAssertions; +using MathNet.Numerics.Random; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; +using Nethermind.Core.Verkle; using Nethermind.Crypto; using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Serialization.Rlp; +using Nethermind.Serialization.Rlp.Verkle; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Proofs; using NUnit.Framework; namespace Nethermind.Core.Test.Encoding; @@ -56,6 +64,14 @@ public BlockDecoderTests() .WithUncles(uncles) .WithWithdrawals(8) .WithMixHash(Keccak.EmptyTreeHash) + .TestObject, + Build.A.Block + .WithNumber(1) + .WithTransactions(transactions) + .WithUncles(uncles) + .WithWithdrawals(8) + .WithMixHash(Keccak.EmptyTreeHash) + .WithExecutionWitness(GenRandomWitness()) .TestObject }; } @@ -111,4 +127,81 @@ public void Get_length_null() BlockDecoder decoder = new(); Assert.That(decoder.GetLength(null, RlpBehaviors.None), Is.EqualTo(1)); } + + [Test] + public void TestWitnessRlp() + { + ExecutionWitnessDecoder decoder = new(); + ExecutionWitness item = GenRandomWitness(); + RlpStream stream = new (decoder.GetLength(item, RlpBehaviors.None)); + decoder.Encode(stream, item); + string dataPrev = stream.Data.ToHexString(); + + ExecutionWitness exec = decoder.Decode(new RlpStream(stream.Data)); + RlpStream stream2 = new (decoder.GetLength(exec, RlpBehaviors.None)); + decoder.Encode(stream2, exec); + string dataNew = stream2.Data.ToHexString(); + + dataPrev.Should().BeEquivalentTo(dataNew); + } + + private static ExecutionWitness GenRandomWitness() + { + var rand = new Random(); + const int suffixDiffLength = 20; + const int stemLength = 20; + List suffixDiffs = new(); + for (int i = 0; i < suffixDiffLength; i++) + { + var d = new SuffixStateDiff() + { + Suffix = (byte)i, CurrentValue = rand.NextBytes(32), NewValue = rand.NextBytes(32) + }; + suffixDiffs.Add(d); + } + + List stateDiff = new List(); + for (int i = 0; i < stemLength; i++) + { + stateDiff.Add(new StemStateDiff() + { + SuffixDiffs = new List(suffixDiffs.ToArray()), + Stem = rand.NextBytes(31) + }); + } + + List otherStems = new(); + for (int i = 0; i < 5; i++) + { + otherStems.Add(new Stem(rand.NextBytes(31))); + } + + byte[] depthExt = rand.NextBytes(5); + + List comm = new(); + for (int i = 0; i < 100; i++) + { + Banderwagon? point; + while (true) + { + point = Banderwagon.FromBytes(rand.NextBytes(32), subgroupCheck: true); + if (point.HasValue) break; + } + comm.Add(point.Value); + } + + Banderwagon? pointd; + while (true) + { + pointd = Banderwagon.FromBytes(rand.NextBytes(32), subgroupCheck: true); + if (pointd.HasValue) break; + } + Banderwagon dd = pointd.Value; + + IpaProofStruct proof = new IpaProofStruct(comm.ToArray()[..8], FrE.One, comm.ToArray()[..8]); + + var outerProof = new WitnessVerkleProof(otherStems.ToArray(), depthExt, comm.ToArray(), dd, proof); + + return new ExecutionWitness(stateDiff, outerProof); + } } diff --git a/src/Nethermind/Nethermind.Core.Test/Nethermind.Core.Test.csproj b/src/Nethermind/Nethermind.Core.Test/Nethermind.Core.Test.csproj index 1dac9a82022..e59a8d12b68 100644 --- a/src/Nethermind/Nethermind.Core.Test/Nethermind.Core.Test.csproj +++ b/src/Nethermind/Nethermind.Core.Test/Nethermind.Core.Test.csproj @@ -24,6 +24,7 @@ + diff --git a/src/Nethermind/Nethermind.Core/Account.cs b/src/Nethermind/Nethermind.Core/Account.cs index 8ed5f59f536..937487ba44a 100644 --- a/src/Nethermind/Nethermind.Core/Account.cs +++ b/src/Nethermind/Nethermind.Core/Account.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Generic; +using System.Linq; using Nethermind.Core.Crypto; using Nethermind.Int256; @@ -35,6 +37,30 @@ public Account(UInt256 balance) CodeHash = Keccak.OfAnEmptyString; StorageRoot = Keccak.EmptyTreeHash; IsTotallyEmpty = Balance.IsZero; + CodeSize = 0; + Version = UInt256.Zero; + } + + public Account(UInt256 balance, UInt256 nonce, Keccak codeHash, UInt256 codeSize, UInt256 version) + { + Balance = balance; + Nonce = nonce; + CodeHash = codeHash; + StorageRoot = Keccak.EmptyTreeHash; + IsTotallyEmpty = Balance.IsZero && Nonce == _accountStartNonce && CodeHash == Keccak.OfAnEmptyString && StorageRoot == Keccak.EmptyTreeHash; + CodeSize = codeSize; + Version = version; + } + + public Account(UInt256 balance, UInt256 nonce, Keccak codeHash) + { + Balance = balance; + Nonce = nonce; + CodeHash = codeHash; + StorageRoot = Keccak.EmptyTreeHash; + IsTotallyEmpty = Balance.IsZero && Nonce == _accountStartNonce && CodeHash == Keccak.OfAnEmptyString && StorageRoot == Keccak.EmptyTreeHash; + CodeSize = 0; + Version = UInt256.Zero; } private Account() @@ -44,6 +70,8 @@ private Account() CodeHash = Keccak.OfAnEmptyString; StorageRoot = Keccak.EmptyTreeHash; IsTotallyEmpty = true; + CodeSize = 0; + Version = UInt256.Zero; } public Account(in UInt256 nonce, in UInt256 balance, Keccak storageRoot, Keccak codeHash) @@ -53,6 +81,8 @@ public Account(in UInt256 nonce, in UInt256 balance, Keccak storageRoot, Keccak StorageRoot = storageRoot; CodeHash = codeHash; IsTotallyEmpty = Balance.IsZero && Nonce == _accountStartNonce && CodeHash == Keccak.OfAnEmptyString && StorageRoot == Keccak.EmptyTreeHash; + CodeSize = 0; + Version = UInt256.Zero; } private Account(in UInt256 nonce, in UInt256 balance, Keccak storageRoot, Keccak codeHash, bool isTotallyEmpty) @@ -62,6 +92,8 @@ private Account(in UInt256 nonce, in UInt256 balance, Keccak storageRoot, Keccak StorageRoot = storageRoot; CodeHash = codeHash; IsTotallyEmpty = isTotallyEmpty; + CodeSize = 0; + Version = UInt256.Zero; } public bool HasCode => !CodeHash.Equals(Keccak.OfAnEmptyString); @@ -70,8 +102,13 @@ private Account(in UInt256 nonce, in UInt256 balance, Keccak storageRoot, Keccak public UInt256 Nonce { get; } public UInt256 Balance { get; } + public UInt256 CodeSize { get; set; } + public UInt256 Version { get; } public Keccak StorageRoot { get; } public Keccak CodeHash { get; } + + // TODO: change codeHash when this set + public byte[]? Code { get; set; } public bool IsTotallyEmpty { get; } public bool IsEmpty => IsTotallyEmpty || (Balance.IsZero && Nonce == _accountStartNonce && CodeHash == Keccak.OfAnEmptyString); public bool IsContract => CodeHash != Keccak.OfAnEmptyString; @@ -91,9 +128,14 @@ public Account WithChangedStorageRoot(Keccak newStorageRoot) return new(Nonce, Balance, newStorageRoot, CodeHash, IsTotallyEmpty && newStorageRoot == Keccak.EmptyTreeHash); } - public Account WithChangedCodeHash(Keccak newCodeHash) + public Account WithChangedCodeHash(Keccak newCodeHash, byte[]? code = null) { - return new(Nonce, Balance, StorageRoot, newCodeHash, IsTotallyEmpty && newCodeHash == Keccak.OfAnEmptyString); + // TODO: does the code and codeHash match? + return new(Nonce, Balance, StorageRoot, newCodeHash, IsTotallyEmpty && newCodeHash == Keccak.OfAnEmptyString) + { + Code = code, + CodeSize = new UInt256((ulong)(code?.Length ?? 0)) + }; } } } diff --git a/src/Nethermind/Nethermind.Core/Block.cs b/src/Nethermind/Nethermind.Core/Block.cs index fe2d08fdb8e..306e20ecf39 100644 --- a/src/Nethermind/Nethermind.Core/Block.cs +++ b/src/Nethermind/Nethermind.Core/Block.cs @@ -7,6 +7,8 @@ using System.Linq; using System.Text; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; using Nethermind.Int256; namespace Nethermind.Core; @@ -24,10 +26,11 @@ public Block( BlockHeader header, IEnumerable transactions, IEnumerable uncles, - IEnumerable? withdrawals = null) + IEnumerable? withdrawals = null, + ExecutionWitness? execWitness = null) { Header = header ?? throw new ArgumentNullException(nameof(header)); - Body = new(transactions.ToArray(), uncles.ToArray(), withdrawals?.ToArray()); + Body = new(transactions.ToArray(), uncles.ToArray(), withdrawals?.ToArray(), execWitness); } public Block(BlockHeader header) : this( @@ -57,6 +60,7 @@ public Transaction[] Transactions public BlockHeader[] Uncles => Body.Uncles; // do not add setter here public Withdrawal[]? Withdrawals => Body.Withdrawals; + public ExecutionWitness? ExecutionWitness => Body.ExecutionWitness; public Keccak? Hash => Header.Hash; // do not add setter here @@ -147,6 +151,25 @@ private string ToFullString() builder.Append(w.ToString(" ")); } + if (ExecutionWitness is not null) + { + builder.AppendLine($" ExecutionWitness"); + builder.AppendLine($" StateDiff: {ExecutionWitness.StateDiff.Count}"); + builder.AppendLine($" WitnessVerkleProof: {ExecutionWitness.VerkleProof!.D}"); + builder.AppendLine($" D: {ExecutionWitness.VerkleProof.D.ToBytes().ToHexString()}"); + builder.AppendLine( + $" IpaProof: {ExecutionWitness.VerkleProof.IpaProof.Encode().ToHexString()}"); + builder.AppendLine( + $" ExtensionPresent: {ExecutionWitness.VerkleProof.DepthExtensionPresent.ToHexString()}"); + + if(ExecutionWitness.VerkleProof.OtherStems is not null) + builder.AppendLine( + $" OtherStems: {string.Join(", ", ExecutionWitness.VerkleProof.OtherStems.Select(x => x.ToString()))}"); + + builder.AppendLine( + $" ExtensionPresent: {string.Join(", ", ExecutionWitness.VerkleProof.CommitmentsByPath.Select(x => x.ToBytes().ToHexString()))}"); + } + return builder.ToString(); } diff --git a/src/Nethermind/Nethermind.Core/BlockBody.cs b/src/Nethermind/Nethermind.Core/BlockBody.cs index 76658c6ab8a..8b30ec5175d 100644 --- a/src/Nethermind/Nethermind.Core/BlockBody.cs +++ b/src/Nethermind/Nethermind.Core/BlockBody.cs @@ -2,16 +2,18 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Nethermind.Core.Verkle; namespace Nethermind.Core { public class BlockBody { - public BlockBody(Transaction[]? transactions, BlockHeader[]? uncles, Withdrawal[]? withdrawals = null) + public BlockBody(Transaction[]? transactions, BlockHeader[]? uncles, Withdrawal[]? withdrawals = null, ExecutionWitness? execWitness = null) { Transactions = transactions ?? Array.Empty(); Uncles = uncles ?? Array.Empty(); Withdrawals = withdrawals; + ExecutionWitness = execWitness; } public BlockBody() : this(null, null, null) { } @@ -21,6 +23,7 @@ public BlockBody() : this(null, null, null) { } public BlockBody WithChangedUncles(BlockHeader[] uncles) => new(Transactions, uncles, Withdrawals); public BlockBody WithChangedWithdrawals(Withdrawal[]? withdrawals) => new(Transactions, Uncles, withdrawals); + public BlockBody WithChangedExecutionWitness(ExecutionWitness? witness) => new(Transactions, Uncles, Withdrawals, witness); public static BlockBody WithOneTransactionOnly(Transaction tx) => new(new[] { tx }, null, null); @@ -30,6 +33,8 @@ public BlockBody() : this(null, null, null) { } public Withdrawal[]? Withdrawals { get; } + public ExecutionWitness? ExecutionWitness { get; set; } + public bool IsEmpty => Transactions.Length == 0 && Uncles.Length == 0 && (Withdrawals?.Length ?? 0) == 0; } } diff --git a/src/Nethermind/Nethermind.Core/BlockHeader.cs b/src/Nethermind/Nethermind.Core/BlockHeader.cs index 5c0bbcfd529..87ccaac14a3 100644 --- a/src/Nethermind/Nethermind.Core/BlockHeader.cs +++ b/src/Nethermind/Nethermind.Core/BlockHeader.cs @@ -3,10 +3,12 @@ using System; using System.Diagnostics; +using System.Linq; using System.Text; using Nethermind.Core.Attributes; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; using Nethermind.Int256; namespace Nethermind.Core; @@ -66,10 +68,9 @@ public BlockHeader( public UInt256 BaseFeePerGas { get; set; } public Keccak? WithdrawalsRoot { get; set; } public UInt256? ExcessDataGas { get; set; } - public bool HasBody => (TxRoot is not null && TxRoot != Keccak.EmptyTreeHash) - || (UnclesHash is not null && UnclesHash != Keccak.OfAnEmptySequenceRlp) - || (WithdrawalsRoot is not null && WithdrawalsRoot != Keccak.EmptyTreeHash); + || (UnclesHash is not null && UnclesHash != Keccak.OfAnEmptySequenceRlp) + || (WithdrawalsRoot is not null && WithdrawalsRoot != Keccak.EmptyTreeHash); public bool HasTransactions => (TxRoot is not null && TxRoot != Keccak.EmptyTreeHash); diff --git a/src/Nethermind/Nethermind.Core/Collections/JournalSet.cs b/src/Nethermind/Nethermind.Core/Collections/JournalSet.cs index b215e408639..22e99efc3a9 100644 --- a/src/Nethermind/Nethermind.Core/Collections/JournalSet.cs +++ b/src/Nethermind/Nethermind.Core/Collections/JournalSet.cs @@ -14,12 +14,24 @@ namespace Nethermind.Core.Collections /// Due to snapshots is not supported. public class JournalSet : IReadOnlySet, ICollection, IJournal { - private readonly List _items = new(); - private readonly HashSet _set = new(); + private readonly List _items; + private readonly HashSet _set; public int TakeSnapshot() => Position; private int Position => Count - 1; + public JournalSet(IEqualityComparer comparer) + { + _items = new List(); + _set = new HashSet(comparer); + } + + public JournalSet() + { + _items = new List(); + _set = new HashSet(); + } + public void Restore(int snapshot) { if (snapshot >= Count) diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 718024991ec..a16109dd7eb 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -114,6 +114,67 @@ public int Compare(Span x, Span y) return y.Length > x.Length ? 1 : 0; } + + public static int CompareDiffLength(Span x, Span y) + { + if (Unsafe.AreSame(ref MemoryMarshal.GetReference(x), ref MemoryMarshal.GetReference(y)) && + x.Length == y.Length) + { + return 0; + } + + if (x.Length == 0) + { + return y.Length == 0 ? 0 : -1; + } + + for (int i = 0; i < x.Length; i++) + { + if (y.Length <= i) + { + return 1; + } + + int result = x[i].CompareTo(y[i]); + if (result != 0) + { + return result; + } + } + + return y.Length > x.Length ? -1 : 0; + } + + public int CompareGreaterThan(Span x, Span y) + { + if (x.Length == 0) + { + return y.Length == 0 ? 0 : 1; + } + + Span ulongX = MemoryMarshal.Cast(x); + Span ulongY = MemoryMarshal.Cast(y); + + for (int i = 0; i < ulongX.Length; i++) + { + if (ulongX[i] > ulongY[i]) + return 1; + + if (ulongX[i] < ulongY[i]) + return -1; + } + + for (int i = ulongX.Length * Unsafe.SizeOf(); i < x.Length; i++) + { + if (x[i] > y[i]) + return 1; + + if (x[i] < y[i]) + return -1; + } + + return y.Length > x.Length ? 1 : 0; + } } public static readonly byte[] Zero32 = new byte[32]; diff --git a/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs index 4230c89b29e..58750bd892c 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; namespace Nethermind.Core.Extensions { @@ -11,5 +12,14 @@ public static class EnumerableExtensions { public static ISet AsSet(this IEnumerable enumerable) => enumerable is ISet set ? set : enumerable.ToHashSet(); + + public static string ToString(this IEnumerable enumerable) + { + StringBuilder builder = new StringBuilder(); + builder.Append('['); + builder.AppendJoin(", ", enumerable); + builder.Append(']'); + return builder.ToString(); + } } } diff --git a/src/Nethermind/Nethermind.Core/Extensions/IntExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/IntExtensions.cs index a1867c8bf92..16db18af475 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/IntExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/IntExtensions.cs @@ -46,5 +46,12 @@ public static byte[] ToBigEndianByteArray(this int value) return bytes; } + + public static byte[] ToByteArrayLittleEndian(this int value) + { + byte[] bytes = new byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return bytes; + } } } diff --git a/src/Nethermind/Nethermind.Core/IVerkleWitness.cs b/src/Nethermind/Nethermind.Core/IVerkleWitness.cs new file mode 100644 index 00000000000..bfdcdd0c74c --- /dev/null +++ b/src/Nethermind/Nethermind.Core/IVerkleWitness.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + + +using Nethermind.Int256; + +namespace Nethermind.Core; + +public interface IVerkleWitness : IJournal +{ + public byte[][] GetAccessedKeys(); + public long AccessForCodeOpCodes(Address caller); + public long AccessValueTransfer(Address caller, Address callee); + + public long AccessForContractCreationInit(Address contractAddress, bool isValueTransfer); + + public long AccessContractCreated(Address contractAddress); + + public long AccessBalance(Address address); + + public long AccessCodeHash(Address address); + + public long AccessStorage(Address address, UInt256 key, bool isWrite); + + public long AccessCodeChunk(Address address, byte chunkId, bool isWrite); + + public long AccessCompleteAccount(Address address, bool isWrite = false); + + public long AccessForTransaction(Address originAddress, Address destinationAddress, bool isValueTransfer); + public long AccessForProofOfAbsence(Address address); +} diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index 5a7b9a74981..5dae6aef221 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -8,6 +8,10 @@ true + + + + diff --git a/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs b/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs index 3f1ad4dd96c..f9c6ee6d655 100644 --- a/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs @@ -262,6 +262,11 @@ public interface IReleaseSpec : IEip1559Spec, IReceiptSpec /// bool IsEip4844Enabled { get; } + /// + /// State - Verkle Trees + /// + bool IsVerkleTreeEipEnabled { get; } + /// /// Should transactions be validated against chainId. /// @@ -271,8 +276,9 @@ public interface IReleaseSpec : IEip1559Spec, IReceiptSpec public ulong WithdrawalTimestamp { get; } public ulong Eip4844TransitionTimestamp { get; } + public ulong VerkleTreeTransitionTimeStamp { get; } - // STATE related + // STATE related public bool ClearEmptyAccountWhenTouched => IsEip158Enabled; // VM diff --git a/src/Nethermind/Nethermind.Core/Verkle/AccountHeader.cs b/src/Nethermind/Nethermind.Core/Verkle/AccountHeader.cs new file mode 100644 index 00000000000..73339edada0 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/AccountHeader.cs @@ -0,0 +1,71 @@ +using System; +using Nethermind.Core.Caching; +using Nethermind.Int256; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Core.Verkle; + +public readonly struct AccountHeader +{ + public const int Version = 0; + public const int Balance = 1; + public const int Nonce = 2; + public const int CodeHash = 3; + public const int CodeSize = 4; + + private const int MainStorageOffsetExponent = 31; + private const int MainStorageOffsetBase = 256; + private const int HeaderStorageOffset = 64; + private const int CodeOffset = 128; + private const int VerkleNodeWidth = 256; + + private static readonly UInt256 MainStorageOffset = (UInt256)MainStorageOffsetBase << MainStorageOffsetExponent; + + private static readonly LruCache<(byte[], UInt256), Pedersen> _keyCache = new(1000000, 10000, "Verkle Key Cache"); + + private static Pedersen GetTreeKeyPrefix(ReadOnlySpan address20, UInt256 treeIndex) + { + if (_keyCache.TryGet((address20.ToArray(), treeIndex), out Pedersen value)) return value; + value = Pedersen.Compute(address20, treeIndex); + _keyCache.Set((address20.ToArray(), treeIndex), value); + return value; + } + + public static Pedersen GetTreeKeyPrefixAccount(byte[] address) => GetTreeKeyPrefix(address, 0); + + public static Pedersen GetTreeKey(byte[] address, UInt256 treeIndex, byte subIndexBytes) + { + Pedersen treeKeyPrefix = GetTreeKeyPrefix(address, treeIndex); + treeKeyPrefix.SuffixByte = subIndexBytes; + return treeKeyPrefix; + } + + public static Pedersen GetTreeKeyForCodeChunk(byte[] address, UInt256 chunk) + { + UInt256 chunkOffset = CodeOffset + chunk; + UInt256 treeIndex = chunkOffset / VerkleNodeWidth; + UInt256.Mod(chunkOffset, VerkleNodeWidth, out UInt256 subIndex); + return GetTreeKey(address, treeIndex, subIndex.ToBigEndian()[31]); + } + + public static Pedersen GetTreeKeyForStorageSlot(byte[] address, UInt256 storageKey) + { + UInt256 pos; + + if (storageKey < CodeOffset - HeaderStorageOffset) pos = HeaderStorageOffset + storageKey; + else pos = MainStorageOffset + storageKey; + + UInt256 treeIndex = pos / VerkleNodeWidth; + + UInt256.Mod(pos, VerkleNodeWidth, out UInt256 subIndex); + return GetTreeKey(address, treeIndex, subIndex.ToBigEndian()[31]); + } + + public static void FillTreeAndSubIndexForChunk(UInt256 chunkId, ref Span subIndexBytes, out UInt256 treeIndex) + { + UInt256 chunkOffset = CodeOffset + chunkId; + treeIndex = chunkOffset / VerkleNodeWidth; + UInt256.Mod(chunkOffset, VerkleNodeWidth, out UInt256 subIndex); + subIndex.ToBigEndian(subIndexBytes); + } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/CodeChunkEnumerator.cs b/src/Nethermind/Nethermind.Core/Verkle/CodeChunkEnumerator.cs new file mode 100644 index 00000000000..7f9d2af887a --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/CodeChunkEnumerator.cs @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Core.Verkle; + +public ref struct CodeChunkEnumerator +{ + const byte PushOffset = 95; + const byte Push1 = PushOffset + 1; + const byte Push32 = PushOffset + 32; + + private Span _code; + private byte _rollingOverPushLength = 0; + private readonly byte[] _bufferChunk = new byte[32]; + private readonly Span _bufferChunkCodePart; + + public CodeChunkEnumerator(Span code) + { + _code = code; + _bufferChunkCodePart = _bufferChunk.AsSpan().Slice(1); + } + + // Try get next chunk + public bool TryGetNextChunk(out byte[] chunk) + { + chunk = _bufferChunk; + + // we don't have chunks left + if (_code.IsEmpty) + { + return false; + } + + // we don't have full chunk + if (_code.Length < 31) + { + // need to have trailing zeroes + _bufferChunkCodePart.Fill(0); + + // set number of push bytes + _bufferChunk[0] = _rollingOverPushLength; + + // copy main bytes + _code.CopyTo(_bufferChunkCodePart); + + // we are done + _code = Span.Empty; + } + else + { + // fill up chunk to store + + // get current chunk of code + Span currentChunk = _code.Slice(0, 31); + + // copy main bytes + currentChunk.CopyTo(_bufferChunkCodePart); + + switch (_rollingOverPushLength) + { + case 32 or 31: // all bytes are roll over + + // set number of push bytes + _bufferChunk[0] = 31; + + // if 32, then we will roll over with 1 to even next chunk + _rollingOverPushLength -= 31; + break; + default: + // set number of push bytes + _bufferChunk[0] = _rollingOverPushLength; + _rollingOverPushLength = 0; + + // check if we have a push instruction in remaining code + // ignore the bytes we rolled over, they are not instructions + for (int i = _bufferChunk[0]; i < 31;) + { + byte instruction = currentChunk[i]; + i++; + if (instruction is >= Push1 and <= Push32) + { + // we calculate data to ignore in code + i += instruction - PushOffset; + + // check if we rolled over the chunk + _rollingOverPushLength = (byte)Math.Max(i - 31, 0); + } + } + break; + } + + // move to next chunk + _code = _code.Slice(31); + } + return true; + } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/Committer.cs b/src/Nethermind/Nethermind.Core/Verkle/Committer.cs new file mode 100644 index 00000000000..92041b32712 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/Committer.cs @@ -0,0 +1,75 @@ +using System.Diagnostics; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; + +namespace Nethermind.Core.Verkle +{ + public struct Committer + { + private static readonly CRS Constants = CRS.Instance; + + public static Banderwagon Commit(FrE[] value) + { + return Banderwagon.MultiScalarMul(Constants.BasisG, value); + } + + public static Banderwagon ScalarMul(FrE value, int index) + { + return Constants.BasisG[index] * value; + } + } + + public class Commitment + { + private FrE? _pointAsField; + + public byte[] ToBytes() => Point.ToBytes(); + + public Commitment(Banderwagon point) + { + Point = point; + } + + public Commitment() + { + Point = Banderwagon.Identity; + } + public Banderwagon Point { get; private set; } + public FrE PointAsField + { + get + { + if (_pointAsField is null) SetCommitmentToField(); + Debug.Assert(_pointAsField is not null, nameof(_pointAsField) + " != null"); + return _pointAsField.Value; + } + private set => _pointAsField = value; + } + + public Commitment Dup() + { + return new Commitment(Point); + } + + private void SetCommitmentToField() + { + PointAsField = Point.MapToScalarField(); + } + + public void AddPoint(Banderwagon point) + { + Point += point; + _pointAsField = null; + SetCommitmentToField(); + } + + public FrE UpdateCommitmentGetDelta(Banderwagon point) + { + FrE prevPointAsField = PointAsField; + Point += point; + _pointAsField = null; + SetCommitmentToField(); + return PointAsField - prevPointAsField; + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/ExecutionWitness.cs b/src/Nethermind/Nethermind.Core/Verkle/ExecutionWitness.cs new file mode 100644 index 00000000000..9b3f68acb87 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/ExecutionWitness.cs @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using FastEnumUtility; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Proofs; + +namespace Nethermind.Core.Verkle; + + +public class ExecutionWitness +{ + public List StateDiff { get; set; } = new(0); + public WitnessVerkleProof? VerkleProof { get; set; } = null; + + public ExecutionWitness() { } + + public ExecutionWitness(List stateDiff, WitnessVerkleProof proof) + { + StateDiff = stateDiff; + VerkleProof = proof; + } +} + +public class WitnessVerkleProof +{ + public Stem[]? OtherStems { get; set; } + public byte[] DepthExtensionPresent { get; set; } + public Banderwagon[] CommitmentsByPath { get; set; } + public Banderwagon D { get; set; } + public IpaProofStruct IpaProof { get; set; } + + public WitnessVerkleProof( + Stem[] otherStems, + byte[] depthExtensionPresent, + Banderwagon[] commitmentsByPath, + Banderwagon d, + IpaProofStruct ipaProof) + { + OtherStems = otherStems; + DepthExtensionPresent = depthExtensionPresent; + CommitmentsByPath = commitmentsByPath; + D = d; + IpaProof = ipaProof; + } + + public static implicit operator WitnessVerkleProof(VerkleProof proof) + { + Stem[] otherStems = proof.VerifyHint.DifferentStemNoProof.Select(x => new Stem(x)).ToArray(); + + byte[] depthExtensionPresent = new byte[proof.VerifyHint.ExtensionPresent.Length]; + for (int i = 0; i < depthExtensionPresent.Length; i++) + { + depthExtensionPresent[i] = (byte)(proof.VerifyHint.Depths[i] << 3); + depthExtensionPresent[i] = + (byte)(depthExtensionPresent[i] | (proof.VerifyHint.ExtensionPresent[i].ToByte())); + } + + return new WitnessVerkleProof(otherStems, + depthExtensionPresent, + proof.CommsSorted, + proof.Proof.D, + proof.Proof.IpaProof + ); + } +} + + +public struct StateDiff +{ + public List SuffixDiffs { get; set; } +} + +public struct StemStateDiff +{ + public Stem Stem { get; set; } + public List SuffixDiffs { get; set; } +} + +public struct SuffixStateDiff +{ + public byte Suffix { get; set; } + public byte[]? CurrentValue { get; set; } + public byte[]? NewValue { get; set; } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/Leaf.cs b/src/Nethermind/Nethermind.Core/Verkle/Leaf.cs new file mode 100644 index 00000000000..3e45f9abf2d --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/Leaf.cs @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Verkle; + +[DebuggerStepThrough] +[DebuggerDisplay("{ToString()}")] +public readonly struct Leaf : IEquatable, IComparable +{ + private readonly Vector256 Bytes; + + public const int MemorySize = 32; + + public Span BytesAsSpan => MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref Unsafe.AsRef(in Bytes), 1)); + + public ReadOnlySpan Span => MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in Bytes), 1)); + + /// + /// 0x0000000000000000000000000000000000000000000000000000000000000000 + /// + public static Leaf Zero { get; } = default; + + /// + /// 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + /// + public static Leaf MaxValue { get; } = new("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + public Leaf(byte[]? bytes) + { + if (bytes is null || bytes.Length == 0) + { + Bytes = default; + return; + } + + Debug.Assert(bytes.Length == MemorySize); + Bytes = Unsafe.As>(ref MemoryMarshal.GetArrayDataReference(bytes)); + } + + public Leaf(string? hex) + { + if (hex is null || hex.Length == 0) + { + Bytes = default; + return; + } + + byte[] bytes = Nethermind.Core.Extensions.Bytes.FromHexString(hex); + Debug.Assert(bytes.Length == MemorySize); + Bytes = Unsafe.As>(ref MemoryMarshal.GetArrayDataReference(bytes)); + } + + public Leaf(Span bytes) + : this((ReadOnlySpan)bytes) { } + + public Leaf(ReadOnlySpan bytes) + { + if (bytes.Length == 0) + { + Bytes = default; + return; + } + + Debug.Assert(bytes.Length == Leaf.MemorySize); + Bytes = Unsafe.As>(ref MemoryMarshal.GetReference(bytes)); + } + + public override bool Equals(object? obj) => obj is Leaf leaf && Equals(leaf); + + public bool Equals(Leaf other) => Bytes.Equals(other.Bytes); + + public bool Equals(Pedersen? other) => BytesAsSpan.SequenceEqual(other?.Bytes); + + public override int GetHashCode() + { + long v0 = Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)); + long v1 = Unsafe.Add(ref Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)), 1); + long v2 = Unsafe.Add(ref Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)), 2); + long v3 = Unsafe.Add(ref Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)), 3); + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } + + public int CompareTo(Leaf other) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(BytesAsSpan, other.BytesAsSpan); + } + + public override string ToString() + { + return ToString(true); + } + + public string ToShortString(bool withZeroX = true) + { + string hash = BytesAsSpan.ToHexString(withZeroX); + return $"{hash[..(withZeroX ? 8 : 6)]}...{hash[^6..]}"; + } + + public string ToString(bool withZeroX) + { + return BytesAsSpan.ToHexString(withZeroX); + } + + public static bool operator ==(Leaf left, Leaf right) => left.Equals(right); + + public static bool operator !=(Leaf left, Leaf right) => !(left == right); + public static bool operator >(Leaf left, Leaf right) => left.CompareTo(right) > 0; + public static bool operator <(Leaf left, Leaf right) => left.CompareTo(right) < 0; + public static bool operator >=(Leaf left, Leaf right) => left.CompareTo(right) >= 0; + public static bool operator <=(Leaf left, Leaf right) => left.CompareTo(right) <= 0; + + public Pedersen ToPedersen() + { + return new Pedersen(BytesAsSpan.ToArray()); + } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/LeafIterator.cs b/src/Nethermind/Nethermind.Core/Verkle/LeafIterator.cs new file mode 100644 index 00000000000..a673a260da7 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/LeafIterator.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using LeafEnumerator = System.Collections.Generic.IEnumerator>; + +namespace Nethermind.Verkle.Tree.Utils; + +public class LeafIterator +{ + public LeafIterator(LeafEnumerator enumerator, int priority) + { + Enumerator = enumerator; + Priority = priority; + } + public readonly LeafEnumerator Enumerator; + public readonly int Priority; +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/LeafKey.cs b/src/Nethermind/Nethermind.Core/Verkle/LeafKey.cs new file mode 100644 index 00000000000..81b201c80fb --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/LeafKey.cs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Nethermind.Core.Verkle; + +public readonly struct LeafKey: IEquatable, IComparable +{ + public byte[] Bytes { get; } + + private LeafKey(byte[] bytes) + { + Bytes = bytes; + } + + public int CompareTo(LeafKey other) + { + return Core.Extensions.Bytes.Comparer.Compare(Bytes, other.Bytes); + } + + public bool Equals(LeafKey other) + { + if (ReferenceEquals(Bytes, other.Bytes)) + { + return true; + } + + if (Bytes is null) return other.Bytes is null; + + return other.Bytes is not null && Core.Extensions.Bytes.AreEqual(Bytes, other.Bytes); + } + + public override bool Equals(object? obj) + { + return obj is LeafKey key && Equals(key); + } + + public override int GetHashCode() + { + if (Bytes is null) return 0; + + long v0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes)); + long v1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long))); + long v2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2)); + long v3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 3)); + + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/Pedersen.cs b/src/Nethermind/Nethermind.Core/Verkle/Pedersen.cs new file mode 100644 index 00000000000..83cdc19a674 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/Pedersen.cs @@ -0,0 +1,474 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Int256; + +namespace Nethermind.Core.Verkle; + +[DebuggerStepThrough] +[DebuggerDisplay("{ToString()}")] +public readonly struct ValuePedersen : IEquatable, IComparable, IEquatable +{ + private readonly Vector256 Bytes; + + public const int MemorySize = 32; + + public Span BytesAsSpan => MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref Unsafe.AsRef(in Bytes), 1)); + + public ReadOnlySpan Span => MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in Bytes), 1)); + + /// + /// 0x0000000000000000000000000000000000000000000000000000000000000000 + /// + public static ValuePedersen Zero { get; } = default; + + /// + /// 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + /// + public static ValuePedersen MaxValue { get; } = new("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + public static implicit operator ValuePedersen(Pedersen? keccak) + { + return new ValuePedersen(keccak?.Bytes); + } + + public ValuePedersen(byte[]? bytes) + { + if (bytes is null || bytes.Length == 0) + { + Bytes = default; + return; + } + + Debug.Assert(bytes.Length == MemorySize); + Bytes = Unsafe.As>(ref MemoryMarshal.GetArrayDataReference(bytes)); + } + + public ValuePedersen(string? hex) + { + if (hex is null || hex.Length == 0) + { + Bytes = default; + return; + } + + byte[] bytes = Nethermind.Core.Extensions.Bytes.FromHexString(hex); + Debug.Assert(bytes.Length == MemorySize); + Bytes = Unsafe.As>(ref MemoryMarshal.GetArrayDataReference(bytes)); + } + + public ValuePedersen(Span bytes) + : this((ReadOnlySpan)bytes) { } + + public ValuePedersen(ReadOnlySpan bytes) + { + if (bytes.Length == 0) + { + Bytes = default; + return; + } + + Debug.Assert(bytes.Length == ValuePedersen.MemorySize); + Bytes = Unsafe.As>(ref MemoryMarshal.GetReference(bytes)); + } + + public override bool Equals(object? obj) => obj is ValuePedersen keccak && Equals(keccak); + + public bool Equals(ValuePedersen other) => Bytes.Equals(other.Bytes); + + public bool Equals(Pedersen? other) => BytesAsSpan.SequenceEqual(other?.Bytes); + + public override int GetHashCode() + { + long v0 = Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)); + long v1 = Unsafe.Add(ref Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)), 1); + long v2 = Unsafe.Add(ref Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)), 2); + long v3 = Unsafe.Add(ref Unsafe.As, long>(ref Unsafe.AsRef(in Bytes)), 3); + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } + + public int CompareTo(ValuePedersen other) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(BytesAsSpan, other.BytesAsSpan); + } + + public override string ToString() + { + return ToString(true); + } + + public string ToShortString(bool withZeroX = true) + { + string hash = BytesAsSpan.ToHexString(withZeroX); + return $"{hash[..(withZeroX ? 8 : 6)]}...{hash[^6..]}"; + } + + public string ToString(bool withZeroX) + { + return BytesAsSpan.ToHexString(withZeroX); + } + + public static bool operator ==(ValuePedersen left, ValuePedersen right) => left.Equals(right); + + public static bool operator !=(ValuePedersen left, ValuePedersen right) => !(left == right); + public static bool operator >(ValuePedersen left, ValuePedersen right) => left.CompareTo(right) > 0; + public static bool operator <(ValuePedersen left, ValuePedersen right) => left.CompareTo(right) < 0; + public static bool operator >=(ValuePedersen left, ValuePedersen right) => left.CompareTo(right) >= 0; + public static bool operator <=(ValuePedersen left, ValuePedersen right) => left.CompareTo(right) <= 0; + + public Pedersen ToPedersen() + { + return new Pedersen(BytesAsSpan.ToArray()); + } +} + +/// +/// Used as dictionary key with implicit conversion to de-virtualize comparisons +/// +[DebuggerStepThrough] +public readonly struct PedersenKey : IEquatable, IComparable +{ + public byte[] Bytes { get; } + + private PedersenKey(byte[] bytes) + { + Bytes = bytes; + } + + public static implicit operator PedersenKey(Pedersen k) => new(k.Bytes); + + public int CompareTo(PedersenKey other) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(Bytes, other.Bytes); + } + + public bool Equals(PedersenKey other) + { + if (ReferenceEquals(Bytes, other.Bytes)) + { + return true; + } + + if (Bytes is null) + { + return other.Bytes is null; + } + + if (other.Bytes is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(Bytes, other.Bytes); + } + + public override bool Equals(object? obj) + { + return obj is PedersenKey key && Equals(key); + } + + public override int GetHashCode() + { + if (Bytes is null) return 0; + + long v0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes)); + long v1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long))); + long v2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2)); + long v3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 3)); + + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } +} + +[DebuggerStepThrough] +public class Pedersen : IEquatable, IComparable +{ + public const int Size = 32; + + public const int MemorySize = + MemorySizes.SmallObjectOverhead + + MemorySizes.RefSize + + MemorySizes.ArrayOverhead + + Size - + MemorySizes.SmallObjectFreeDataSize; + + /// + /// 0x0000000000000000000000000000000000000000000000000000000000000000 + /// + public static Pedersen Zero { get; } = new(new byte[Size]); + + /// + /// 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + /// + public static Pedersen MaxValue { get; } = new("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + public byte[] Bytes { get; } + public ReadOnlySpan BytesAsSpan => Bytes; + public ReadOnlySpan StemAsSpan => new(Bytes, 0, 31); + + public ReadOnlySpan NodeKeyAsSpan(int i) => new(Bytes, 0, i); + + public byte SuffixByte + { + get => Bytes[31]; + set => Bytes[31] = value; + } + + public Pedersen(string hexString) + : this(Nethermind.Core.Extensions.Bytes.FromHexString(hexString)) { } + + public Pedersen(byte[] bytes) + { + if (bytes.Length != Size) + { + throw new ArgumentException($"{nameof(Pedersen)} must be {Size} bytes and was {bytes.Length} bytes", nameof(bytes)); + } + + Bytes = bytes.AsSpan().ToArray(); + } + + public override string ToString() + { + return ToString(true); + } + + public string ToShortString(bool withZeroX = true) + { + string hash = Bytes.ToHexString(withZeroX); + return $"{hash[..(withZeroX ? 8 : 6)]}...{hash[^6..]}"; + } + + public string ToString(bool withZeroX) + { + return Bytes.ToHexString(withZeroX); + } + + public static implicit operator Pedersen(byte[] bytes) + { + return new Pedersen(bytes); + } + + [DebuggerStepThrough] + public static Pedersen Compute(ReadOnlySpan address20, UInt256 treeIndex) + { + return new Pedersen(PedersenHash.ComputeHashBytes(address20, treeIndex)); + } + + [DebuggerStepThrough] + public static Pedersen Compute(byte[] address20, UInt256 treeIndex) + { + return new Pedersen(PedersenHash.ComputeHashBytes(address20, treeIndex)); + } + + public bool Equals(Pedersen? other) + { + if (other is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(other.Bytes, Bytes); + } + + public int CompareTo(Pedersen? other) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(Bytes, other?.Bytes); + } + + public override bool Equals(object? obj) + { + return obj?.GetType() == typeof(Pedersen) && Equals((Pedersen)obj); + } + + public override int GetHashCode() + { + long v0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes)); + long v1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long))); + long v2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2)); + long v3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 3)); + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } + + public static bool operator ==(Pedersen? a, Pedersen? b) + { + if (a is null) + { + return b is null; + } + + if (b is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(a.Bytes, b.Bytes); + } + + public static bool operator !=(Pedersen? a, Pedersen? b) + { + return !(a == b); + } + + public static bool operator >(Pedersen? k1, Pedersen? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) > 0; + } + + public static bool operator <(Pedersen? k1, Pedersen? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) < 0; + } + + public static bool operator >=(Pedersen? k1, Pedersen? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) >= 0; + } + + public static bool operator <=(Pedersen? k1, Pedersen? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) <= 0; + } + + public PedersenStructRef ToStructRef() => new(Bytes); +} + +public ref struct PedersenStructRef +{ + public const int Size = 32; + + public int MemorySize => MemorySizes.ArrayOverhead + Size; + + public Span Bytes { get; } + + public PedersenStructRef(Span bytes) + { + if (bytes.Length != Size) + { + throw new ArgumentException($"{nameof(Pedersen)} must be {Size} bytes and was {bytes.Length} bytes", nameof(bytes)); + } + + Bytes = bytes; + } + + public override string ToString() + { + return ToString(true); + } + + public string ToShortString(bool withZeroX = true) + { + string hash = Bytes.ToHexString(withZeroX); + return $"{hash[..(withZeroX ? 8 : 6)]}...{hash[^6..]}"; + } + + public string ToString(bool withZeroX) + { + return Bytes.ToHexString(withZeroX); + } + + [DebuggerStepThrough] + public static PedersenStructRef Compute(byte[] address20, UInt256 treeIndex) + { + var result = new PedersenStructRef(); + PedersenHash.ComputeHashBytesToSpan(address20, treeIndex, result.Bytes); + return result; + } + + [DebuggerStepThrough] + public static PedersenStructRef Compute(ReadOnlySpan address20, UInt256 treeIndex) + { + var result = new PedersenStructRef(); + PedersenHash.ComputeHashBytesToSpan(address20, treeIndex, result.Bytes); + return result; + } + + private static PedersenStructRef InternalCompute(ReadOnlySpan address20, UInt256 treeIndex) + { + var result = new PedersenStructRef(); + PedersenHash.ComputeHashBytesToSpan(address20, treeIndex, result.Bytes); + return result; + } + + public bool Equals(Pedersen? other) + { + if (other is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(other.Bytes, Bytes); + } + + public bool Equals(PedersenStructRef other) => Nethermind.Core.Extensions.Bytes.AreEqual(other.Bytes, Bytes); + + public override bool Equals(object? obj) + { + return obj?.GetType() == typeof(Pedersen) && Equals((Pedersen)obj); + } + + public override int GetHashCode() + { + return MemoryMarshal.Read(Bytes); + } + + public static bool operator ==(PedersenStructRef a, Pedersen? b) + { + if (b is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(a.Bytes, b.Bytes); + } + + public static bool operator ==(Pedersen? a, PedersenStructRef b) + { + if (a is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(a.Bytes, b.Bytes); + } + + public static bool operator ==(PedersenStructRef a, PedersenStructRef b) + { + return Nethermind.Core.Extensions.Bytes.AreEqual(a.Bytes, b.Bytes); + } + + public static bool operator !=(PedersenStructRef a, Pedersen b) + { + return !(a == b); + } + + public static bool operator !=(Pedersen a, PedersenStructRef b) + { + return !(a == b); + } + + public static bool operator !=(PedersenStructRef a, PedersenStructRef b) + { + return !(a == b); + } + + public Pedersen ToKeccak() => new(Bytes.ToArray()); +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/PedersenHash.cs b/src/Nethermind/Nethermind.Core/Verkle/PedersenHash.cs new file mode 100644 index 00000000000..22284459901 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/PedersenHash.cs @@ -0,0 +1,76 @@ +using System; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using Nethermind.Int256; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; + +namespace Nethermind.Core.Verkle; + +public static class PedersenHash +{ + public static byte[] Hash(UInt256[] inputElements) + { + int inputLength = inputElements.Length; + FrE[] pedersenVec = new FrE[1 + 2 * inputLength]; + pedersenVec[0] = FrE.SetElement((ulong)(2 + 256 * inputLength * 32)); + + for (int i = 0; i < inputElements.Length; i++) + { + pedersenVec[2 * i + 1] = FrE.SetElement(inputElements[i].u0, inputElements[i].u1); + pedersenVec[2 * i + 2] = FrE.SetElement(inputElements[i].u2, inputElements[i].u3); + } + CRS crs = CRS.Instance; + + Banderwagon res = Banderwagon.Identity; + for (int i = 0; i < pedersenVec.Length; i++) + { + res += crs.BasisG[i] * pedersenVec[i]; + } + + return res.ToBytesLittleEndian(); + } + + public static void ComputeHashBytesToSpan(ReadOnlySpan address20, UInt256 treeIndex, Span output) + { + Hash(address20, treeIndex).CopyTo(output); + } + + public static byte[] ComputeHashBytes(ReadOnlySpan address20, UInt256 treeIndex) => + Hash(address20, treeIndex); + + public static byte[] Hash(ReadOnlySpan address20, UInt256 treeIndex) + { + ulong u0, u1, u2, u3; + if (address20.Length == 32) + { + UInt256 temp = new UInt256(address20); + u0 = temp.u0; + u1 = temp.u1; + u2 = temp.u2; + u3 = temp.u3; + } + else + { + u0 = 0; + Span u1Bytes = new byte[8]; + address20[..4].CopyTo(u1Bytes[4..]); + u1 = BinaryPrimitives.ReadUInt64LittleEndian(u1Bytes); + u2 = BinaryPrimitives.ReadUInt64LittleEndian(address20.Slice(4, 8)); + u3 = BinaryPrimitives.ReadUInt64LittleEndian(address20.Slice(12, 8)); + } + + + CRS crs = CRS.Instance; + + Banderwagon res = crs.BasisG[0] * FrE.SetElement(2 + 256 * 64) + + crs.BasisG[1] * FrE.SetElement(u0, u1) + + crs.BasisG[2] * FrE.SetElement(u2, u3) + + crs.BasisG[3] * FrE.SetElement(treeIndex.u0, treeIndex.u1) + + crs.BasisG[4] * FrE.SetElement(treeIndex.u2, treeIndex.u3); + + return res.ToBytesLittleEndian(); + } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/Stem.cs b/src/Nethermind/Nethermind.Core/Verkle/Stem.cs new file mode 100644 index 00000000000..d8f0395c7fb --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/Stem.cs @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Verkle; + +public unsafe struct ValueStem +{ + internal const int Size = 31; + public fixed byte Bytes[Size]; + + public Span BytesAsSpan => MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref this, 1)); +} + +/// +/// Used as dictionary key with implicit conversion to de-virtualize comparisons +/// +[DebuggerStepThrough] +public readonly struct StemKey : IEquatable, IComparable +{ + public const int Size = 31; + public byte[] Bytes { get; } + + private StemKey(byte[] bytes) + { + if (bytes.Length != Size) + { + throw new ArgumentException($"{nameof(Stem)} must be {Size} bytes and was {bytes.Length} bytes", nameof(bytes)); + } + Bytes = bytes; + } + + public static implicit operator StemKey(Stem k) => new(k.Bytes); + + public int CompareTo(StemKey other) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(Bytes, other.Bytes); + } + + public bool Equals(StemKey other) + { + if (ReferenceEquals(Bytes, other.Bytes)) + { + return true; + } + + if (Bytes is null) + { + return other.Bytes is null; + } + + if (other.Bytes is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(Bytes, other.Bytes); + } + + public override bool Equals(object? obj) + { + return obj is StemKey key && Equals(key); + } + + public override int GetHashCode() + { + if (Bytes is null) return 0; + + long v0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes)); + long v1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long))); + long v2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2)); + long v3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 3 - 1)); + v3 <<= 1; // because one bit extra from previous long + + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } +} + +[DebuggerStepThrough] +public class Stem : IEquatable, IComparable +{ + public const int Size = 31; + + /// + /// 0x0000000000000000000000000000000000000000000000000000000000000000 + /// + public static Stem Zero { get; } = new(new byte[Size]); + + /// + /// 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + /// + public static Stem MaxValue { get; } = new("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + public byte[] Bytes { get; } + public ReadOnlySpan BytesAsSpan => Bytes; + + public Stem(string hexString) + : this(Core.Extensions.Bytes.FromHexString(hexString)) { } + + public Stem(byte[] bytes) + { + if (bytes.Length != Size) + { + throw new ArgumentException($"{nameof(Stem)} must be {Size} bytes and was {bytes.Length} bytes", nameof(bytes)); + } + + Bytes = bytes; + } + + public override string ToString() + { + return ToString(true); + } + + public string ToShortString(bool withZeroX = true) + { + string hash = Bytes.ToHexString(withZeroX); + return $"{hash[..(withZeroX ? 8 : 6)]}...{hash[^6..]}"; + } + + public string ToString(bool withZeroX) + { + return Bytes.ToHexString(withZeroX); + } + + public static implicit operator Stem(byte[] bytes) + { + return new Stem(bytes); + } + + public bool Equals(Stem? other) + { + if (other is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(other.Bytes, Bytes); + } + + public int CompareTo(Stem? other) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(Bytes, other?.Bytes); + } + + public override bool Equals(object? obj) + { + return obj?.GetType() == typeof(Stem) && Equals((Stem)obj); + } + + public override int GetHashCode() + { + long v0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes)); + long v1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long))); + long v2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2)); + long v3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 3 - 1)); + v3 <<= 1; // because one bit extra from previous long + + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } + + public static bool operator ==(Stem? a, Stem? b) + { + if (a is null) + { + return b is null; + } + + if (b is null) + { + return false; + } + + return Nethermind.Core.Extensions.Bytes.AreEqual(a.Bytes, b.Bytes); + } + + public static bool operator !=(Stem? a, Stem? b) + { + return !(a == b); + } + + public static bool operator >(Stem? k1, Stem? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) > 0; + } + + public static bool operator <(Stem? k1, Stem? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) < 0; + } + + public static bool operator >=(Stem? k1, Stem? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) >= 0; + } + + public static bool operator <=(Stem? k1, Stem? k2) + { + return Nethermind.Core.Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) <= 0; + } + +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/Structures.cs b/src/Nethermind/Nethermind.Core/Verkle/Structures.cs new file mode 100644 index 00000000000..9c986dc4885 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/Structures.cs @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using System.Text; +using FastEnumUtility; +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Proofs; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Core.Verkle; + +public struct VerkleProof +{ + public VerificationHint VerifyHint; + public Banderwagon[] CommsSorted; + public VerkleProofStruct Proof; + + public override string ToString() + { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.Append("\n####[Verkle Proof]####\n"); + stringBuilder.Append("\n###[Verify Hint]###\n"); + stringBuilder.Append(VerifyHint.ToString()); + stringBuilder.Append("\n###[Comms Sorted]###\n"); + foreach (Banderwagon comm in CommsSorted) + { + stringBuilder.AppendJoin(", ", comm.ToBytesLittleEndian().Reverse().ToArray()); + stringBuilder.Append('\n'); + } + stringBuilder.Append("\n###[Inner Proof]###\n"); + stringBuilder.Append(Proof.ToString()); + return stringBuilder.ToString(); + } + + public byte[] Encode() + { + List encoded = new List(); + encoded.AddRange(VerifyHint.Encode()); + + encoded.AddRange(CommsSorted.Length.ToByteArrayLittleEndian()); + foreach (Banderwagon comm in CommsSorted) + { + encoded.AddRange(comm.ToBytesLittleEndian().Reverse()); + } + + encoded.AddRange(Proof.Encode()); + + return encoded.ToArray(); + } + + public static VerkleProof Decode(byte[] proof) + { + return new VerkleProof(); + } +} + +public struct VerificationHint +{ + public byte[] Depths; + public ExtPresent[] ExtensionPresent; + public byte[][] DifferentStemNoProof; + + public override string ToString() + { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.Append("\n##[Depths]##\n"); + stringBuilder.AppendJoin(", ", Depths); + stringBuilder.Append("\n##[ExtensionPresent]##\n"); + stringBuilder.AppendJoin(", ", ExtensionPresent.Select(x => x.ToString())); + stringBuilder.Append("\n##[DifferentStemNoProof]##\n"); + foreach (byte[] stem in DifferentStemNoProof) + { + stringBuilder.AppendJoin(", ", stem); + } + return stringBuilder.ToString(); + } + + public byte[] Encode() + { + List encoded = new List(); + + encoded.AddRange(DifferentStemNoProof.Length.ToByteArrayLittleEndian()); + foreach (byte[] stem in DifferentStemNoProof) + { + encoded.AddRange(stem); + } + + encoded.AddRange(Depths.Length.ToByteArrayLittleEndian()); + + foreach ((byte depth, ExtPresent extPresent) in Depths.Zip(ExtensionPresent)) + { + byte extPresentByte = (byte)(extPresent.ToByte() | (depth << 3)); + encoded.Add(extPresentByte); + } + + return encoded.ToArray(); + } +} + +public struct UpdateHint +{ + public Dictionary DepthAndExtByStem { get; set; } + public Dictionary, Banderwagon> CommByPath { get; set; } + public SortedDictionary, Stem> DifferentStemNoProof { get; set; } +} + +public enum ExtPresent : byte +{ + None = 0, + DifferentStem = 1, + Present = 2 +} + +public struct SuffixPoly +{ + public FrE[] C1 { get; set; } + public FrE[] C2 { get; set; } +} diff --git a/src/Nethermind/Nethermind.Core/Verkle/VerkleCommitment.cs b/src/Nethermind/Nethermind.Core/Verkle/VerkleCommitment.cs new file mode 100644 index 00000000000..b215d395786 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Verkle/VerkleCommitment.cs @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Verkle; + +public class VerkleCommitment: IEquatable, IComparable +{ + public const int Size = 32; + + public const int MemorySize = + MemorySizes.SmallObjectOverhead + + MemorySizes.RefSize + + MemorySizes.ArrayOverhead + + Size - + MemorySizes.SmallObjectFreeDataSize; + + /// + /// 0x0000000000000000000000000000000000000000000000000000000000000000 + /// + public static VerkleCommitment Zero { get; } = new(new byte[Size]); + public static VerkleCommitment EmptyTreeHash = Zero; + + /// + /// 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + /// + public static VerkleCommitment MaxValue { get; } = new("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + public byte[] Bytes { get; } + + public VerkleCommitment(string hexString) + : this(Extensions.Bytes.FromHexString(hexString)) { } + + public VerkleCommitment(byte[] bytes) + { + if (bytes.Length != Size) + { + throw new ArgumentException($"{nameof(VerkleCommitment)} must be {Size} bytes and was {bytes.Length} bytes", nameof(bytes)); + } + + Bytes = bytes; + } + + public override string ToString() + { + return ToString(true); + } + + public string ToShortString(bool withZeroX = true) + { + string hash = Bytes.ToHexString(withZeroX); + return $"{hash[..(withZeroX ? 8 : 6)]}...{hash[^6..]}"; + } + + public string ToString(bool withZeroX) + { + return Bytes.ToHexString(withZeroX); + } + + public bool Equals(VerkleCommitment? other) + { + if (other is null) + { + return false; + } + + return Extensions.Bytes.AreEqual(other.Bytes, Bytes); + } + + public int CompareTo(VerkleCommitment? other) + { + return Extensions.Bytes.Comparer.Compare(Bytes, other?.Bytes); + } + + public override bool Equals(object? obj) + { + return obj?.GetType() == typeof(VerkleCommitment) && Equals((VerkleCommitment)obj); + } + + public override int GetHashCode() + { + long v0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes)); + long v1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long))); + long v2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2)); + long v3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 3)); + v0 ^= v1; + v2 ^= v3; + v0 ^= v2; + + return (int)v0 ^ (int)(v0 >> 32); + } + + public static bool operator ==(VerkleCommitment? a, VerkleCommitment? b) + { + if (a is null) + { + return b is null; + } + + if (b is null) + { + return false; + } + + return Extensions.Bytes.AreEqual(a.Bytes, b.Bytes); + } + + public static bool operator !=(VerkleCommitment? a, VerkleCommitment? b) + { + return !(a == b); + } + + public static bool operator >(VerkleCommitment? k1, VerkleCommitment? k2) + { + return Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) > 0; + } + + public static bool operator <(VerkleCommitment? k1, VerkleCommitment? k2) + { + return Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) < 0; + } + + public static bool operator >=(VerkleCommitment? k1, VerkleCommitment? k2) + { + return Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) >= 0; + } + + public static bool operator <=(VerkleCommitment? k1, VerkleCommitment? k2) + { + return Extensions.Bytes.Comparer.Compare(k1?.Bytes, k2?.Bytes) <= 0; + } +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index 3bebbc3c727..4a26a7900c7 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -119,6 +119,26 @@ public void Flush() /// public void Clear() { throw new NotSupportedException(); } + public IEnumerable> GetIterator() + { + using Iterator iterator = _mainDb.CreateIterator(true, _columnFamily); + return _mainDb.GetAllCore(iterator); + } + + public IEnumerable> GetIterator(byte[] start) + { + using Iterator iterator = _mainDb.CreateIterator(true, _columnFamily); + iterator.Seek(start); + return _mainDb.GetAllCore(iterator); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + using Iterator iterator = _mainDb.CreateIterator(true, _columnFamily); + iterator.Seek(start); + return _mainDb.GetAllCore(iterator); + } + private void UpdateWriteMetrics() => _mainDb.UpdateWriteMetrics(); private void UpdateReadMetrics() => _mainDb.UpdateReadMetrics(); diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 1d8dbd20691..913729a4c03 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -578,6 +578,23 @@ protected internal Iterator CreateIterator(bool ordered = false, ColumnFamilyHan } } + private Iterator CreateIterator(byte[] start, byte[] end, bool ordered = false, ColumnFamilyHandle? ch = null) + { + ReadOptions readOptions = new(); + readOptions.SetTailing(!ordered); + readOptions.SetIterateLowerBound(start); + readOptions.SetIterateUpperBound(end); + try + { + return _db.NewIterator(ch, readOptions); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + public IEnumerable GetAllValues(bool ordered = false) { if (_isDisposing) @@ -626,6 +643,39 @@ internal IEnumerable GetAllValuesCore(Iterator iterator) } } + private IEnumerable> GetAllCoreBounded(Iterator iterator) + { + if (_isDisposing) + { + throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}"); + } + + while (iterator.Valid()) + { + yield return new KeyValuePair(iterator.Key(), iterator.Value()); + + try + { + iterator.Next(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + + try + { + iterator.Dispose(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + public IEnumerable> GetAllCore(Iterator iterator) { if (_isDisposing) @@ -796,6 +846,25 @@ public void Clear() Delete(); } + public IEnumerable> GetIterator() + { + Iterator iterator = CreateIterator(true); + return GetAllCore(iterator); + } + + public IEnumerable> GetIterator(byte[] start) + { + Iterator iterator = CreateIterator(true); + iterator.Seek(start); + return GetAllCoreBounded(iterator); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + Iterator iterator = CreateIterator(start, end, true); + return GetAllCoreBounded(iterator); + } + private void Delete() { try diff --git a/src/Nethermind/Nethermind.Db.Rocks/VerkleDbFactory.cs b/src/Nethermind/Nethermind.Db.Rocks/VerkleDbFactory.cs new file mode 100644 index 00000000000..4a1ec6401b6 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/VerkleDbFactory.cs @@ -0,0 +1,65 @@ +using System; +using System.IO; +using System.IO.Abstractions; +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.Db.Rocks.Config; +using Nethermind.Logging; + +namespace Nethermind.Db.Rocks +{ + public enum DbMode + { + [ConfigItem(Description = "Diagnostics mode which uses an in-memory DB")] + MemDb, + [ConfigItem(Description = "Diagnostics mode which uses an Persistant DB")] + PersistantDb, + [ConfigItem(Description = "Diagnostics mode which uses a read-only DB")] + ReadOnlyDb + } + + public class VerkleDbFactory + { + private static (IDbProvider DbProvider, RocksDbFactory RocksDbFactory, MemDbFactory MemDbFactory) InitDbApi(DbMode diagnosticMode, string baseDbPath, bool storeReceipts) + { + DbConfig dbConfig = new DbConfig(); + DisposableStack disposeStack = new DisposableStack(); + IDbProvider dbProvider; + RocksDbFactory rocksDbFactory; + MemDbFactory memDbFactory; + switch (diagnosticMode) + { + case DbMode.ReadOnlyDb: + DbProvider rocksDbProvider = new DbProvider(DbModeHint.Persisted); + dbProvider = new ReadOnlyDbProvider(rocksDbProvider, storeReceipts); // ToDo storeReceipts as createInMemoryWriteStore - bug? + disposeStack.Push(rocksDbProvider); + rocksDbFactory = new RocksDbFactory(dbConfig, NullLogManager.Instance, Path.Combine(baseDbPath, "debug")); + memDbFactory = new MemDbFactory(); + break; + case DbMode.MemDb: + dbProvider = new DbProvider(DbModeHint.Mem); + rocksDbFactory = new RocksDbFactory(dbConfig, NullLogManager.Instance, Path.Combine(baseDbPath, "debug")); + memDbFactory = new MemDbFactory(); + break; + case DbMode.PersistantDb: + dbProvider = new DbProvider(DbModeHint.Persisted); + rocksDbFactory = new RocksDbFactory(dbConfig, NullLogManager.Instance, baseDbPath); + memDbFactory = new MemDbFactory(); + break; + default: + throw new ArgumentException(); + + } + + return (dbProvider, rocksDbFactory, memDbFactory); + } + + public static IDbProvider InitDatabase(DbMode dbMode, string? dbPath) + { + (IDbProvider dbProvider, RocksDbFactory rocksDbFactory, MemDbFactory memDbFactory) = InitDbApi(dbMode, dbPath ?? "testDb", true); + StandardDbInitializer dbInitializer = new StandardDbInitializer(dbProvider, rocksDbFactory, memDbFactory, new FileSystem()); + dbInitializer.InitStandardDbs(true); + return dbProvider; + } + } +} diff --git a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs index eb7e7cf258e..5a3107c449c 100644 --- a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs +++ b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs @@ -74,6 +74,20 @@ public bool KeyExists(ReadOnlySpan key) public IDb Innermost => this; // record db is just a helper DB here public void Flush() { } public void Clear() { } + public IEnumerable> GetIterator() + { + throw new NotImplementedException(); + } + + public IEnumerable> GetIterator(byte[] start) + { + throw new NotImplementedException(); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + throw new NotImplementedException(); + } public IEnumerable> GetAll(bool ordered = false) => _recordDb.GetAll(); diff --git a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs index 96879227c90..5c5f3ef14d5 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs @@ -25,6 +25,8 @@ namespace Nethermind.Db.Test [Parallelizable(ParallelScope.None)] public class DbOnTheRocksTests { + public static Random Random { get; } = new(); + [Test] public void Smoke_test() { @@ -70,6 +72,34 @@ public void Can_get_all_on_empty() } } + [Test] + public void RangeIteratorTest() + { + IDbConfig config = new DbConfig(); + DbOnTheRocks db = new("testRangeIterator1", GetRocksDbSettings("testRangeIterator1", "TestRangeIterator1"), + config, LimboLogs.Instance); + + byte[] key = new byte[32]; + byte[] value = new byte[32]; + + Random.NextBytes(key); + Random.NextBytes(value); + + for (int i = 0; i < 10000; i++) + { + Random.NextBytes(key); + Random.NextBytes(value); + db.Set(key, value.AsSpan().ToArray()); + } + + var xx = db.GetIterator(); + + foreach (KeyValuePair x in xx) + { + Console.WriteLine($"Key:{x.Key.ToHexString()} Value:{x.Value.ToHexString()}"); + } + } + [Test] public async Task Dispose_while_writing_does_not_cause_access_violation_exception() { diff --git a/src/Nethermind/Nethermind.Db/CompressingDb.cs b/src/Nethermind/Nethermind.Db/CompressingDb.cs index 5ca39059933..fe13fa2d787 100644 --- a/src/Nethermind/Nethermind.Db/CompressingDb.cs +++ b/src/Nethermind/Nethermind.Db/CompressingDb.cs @@ -130,6 +130,20 @@ public IEnumerable GetAllValues(bool ordered = false) => public void Flush() => _wrapped.Flush(); public void Clear() => _wrapped.Clear(); + public IEnumerable> GetIterator() + { + return _wrapped.GetIterator(); + } + + public IEnumerable> GetIterator(byte[] start) + { + return _wrapped.GetIterator(start); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + return _wrapped.GetIterator(start, end); + } public long GetSize() => _wrapped.GetSize(); diff --git a/src/Nethermind/Nethermind.Db/DbNames.cs b/src/Nethermind/Nethermind.Db/DbNames.cs index a0f4652b6eb..e374436abef 100644 --- a/src/Nethermind/Nethermind.Db/DbNames.cs +++ b/src/Nethermind/Nethermind.Db/DbNames.cs @@ -16,5 +16,11 @@ public static class DbNames public const string Witness = "witness"; public const string CHT = "canonicalHashTrie"; public const string Metadata = "metadata"; + public const string Leaf = "leaf"; + public const string InternalNodes = "internalNodes"; + public const string ForwardDiff = "forwardDiff"; + public const string ReverseDiff = "reverseDiff"; + public const string Preimages = "preimages"; + public const string StateRootToBlock = "stateRoots"; } } diff --git a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs index f94f7b27dd4..475306b0497 100755 --- a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs +++ b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs @@ -130,6 +130,21 @@ public void Clear() cloningDb?.Clear(); } + public IEnumerable> GetIterator() + { + return _currentDb.GetIterator(); + } + + public IEnumerable> GetIterator(byte[] start) + { + return _currentDb.GetIterator(start); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + return _currentDb.GetIterator(start, end); + } + /// public bool CanStartPruning => _pruningContext is null; // we can start pruning only if no pruning is in progress diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index 9cff4079e90..34bd1b86fb3 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -25,5 +25,9 @@ public interface IDb : IKeyValueStoreWithBatching, IDisposable void Clear(); public IReadOnlyDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyDb(this, createInMemWriteStore); + + public IEnumerable> GetIterator(); + public IEnumerable> GetIterator(byte[] start); + public IEnumerable> GetIterator(byte[] start, byte[] end); } } diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 5625941f1f5..e2d91387818 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -32,6 +32,13 @@ public interface IDbProvider : IDisposable public IDb MetadataDb => GetDb(DbNames.Metadata); + public IDb LeafDb => GetDb(DbNames.Leaf); + public IDb InternalNodesDb => GetDb(DbNames.InternalNodes); + + public IDb ForwardDiff => GetDb(DbNames.ForwardDiff); + public IDb ReverseDiff => GetDb(DbNames.ReverseDiff); + public IDb StateRootToBlocks => GetDb(DbNames.StateRootToBlock); + T GetDb(string dbName) where T : class, IDb; void RegisterDb(string dbName, T db) where T : class, IDb; diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index 79c56952cd7..43665b6e009 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -19,23 +19,27 @@ public class MemDb : IFullDb, IDbWithSpan public long ReadsCount { get; private set; } public long WritesCount { get; private set; } + private readonly SortedSet? _sortedKeys; private readonly SpanConcurrentDictionary _db; - public MemDb(string name) - : this(0, 0) + public MemDb(string name, bool sorted = false) + : this(0, 0, sorted) { Name = name; } - public MemDb() : this(0, 0) + public MemDb(bool sorted = false) : this(0, 0, sorted) { + Name = ""; } - public MemDb(int writeDelay, int readDelay) + public MemDb(int writeDelay, int readDelay, bool sorted = false) { + Name = ""; _writeDelay = writeDelay; _readDelay = readDelay; _db = new SpanConcurrentDictionary(Bytes.SpanEqualityComparer); + if (sorted) _sortedKeys = new SortedSet(Bytes.Comparer); } public string Name { get; } @@ -69,6 +73,7 @@ public KeyValuePair[] this[byte[][] keys] public virtual void Remove(ReadOnlySpan key) { _db.TryRemove(key, out _); + _sortedKeys?.Remove(key.ToArray()); } public bool KeyExists(ReadOnlySpan key) @@ -85,11 +90,63 @@ public void Flush() public void Clear() { _db.Clear(); + _sortedKeys?.Clear(); } - public IEnumerable> GetAll(bool ordered = false) => _db; + public virtual IEnumerable> GetIterator() + { + if (_sortedKeys is null) throw new ArgumentException($"cannot get ordered data"); + using SortedSet.Enumerator keyEnumerator = _sortedKeys.GetEnumerator(); + while (keyEnumerator.MoveNext()) + yield return new KeyValuePair(keyEnumerator.Current, Get(keyEnumerator.Current)); + } + + public virtual IEnumerable> GetIterator(byte[] start) + { + if (_sortedKeys is null) throw new ArgumentException($"cannot get ordered data"); + return GetIterator(start, _sortedKeys.Max); + } + + public virtual IEnumerable> GetIterator(byte[] start, byte[] end) + { + if (_sortedKeys is null) throw new ArgumentException($"cannot get ordered data"); + using SortedSet.Enumerator keyEnumerator = _sortedKeys + .GetViewBetween(start, end) + .GetEnumerator(); + + while (keyEnumerator.MoveNext()) + yield return new KeyValuePair(keyEnumerator.Current, Get(keyEnumerator.Current)); + } + + public IEnumerable> GetAll(bool ordered = false) + { + return ordered ? GetAllSortedIterator() : _db; + } - public IEnumerable GetAllValues(bool ordered = false) => Values; + private IEnumerable> GetAllSortedIterator() + { + if (_sortedKeys is null) throw new ArgumentException($"cannot get ordered data"); + using SortedSet.Enumerator iterator = _sortedKeys.GetEnumerator(); + while (iterator.MoveNext()) + { + yield return new KeyValuePair(iterator.Current, _db[iterator.Current!]); + } + } + + private IEnumerable GetAllValuesSortedIterator() + { + if (_sortedKeys is null) throw new ArgumentException($"cannot get ordered data"); + using SortedSet.Enumerator iterator = _sortedKeys.GetEnumerator(); + while (iterator.MoveNext()) + { + yield return _db[iterator.Current!]; + } + } + + public IEnumerable GetAllValues(bool ordered = false) + { + return ordered ? GetAllValuesSortedIterator() : Values; + } public virtual IBatch StartBatch() { @@ -144,6 +201,7 @@ public virtual void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags WritesCount++; _db[key] = value; + _sortedKeys?.Add(key.ToArray()); } } } diff --git a/src/Nethermind/Nethermind.Db/MemDbFactory.cs b/src/Nethermind/Nethermind.Db/MemDbFactory.cs index fa7510ef645..de017419a72 100644 --- a/src/Nethermind/Nethermind.Db/MemDbFactory.cs +++ b/src/Nethermind/Nethermind.Db/MemDbFactory.cs @@ -7,6 +7,6 @@ public class MemDbFactory : IMemDbFactory { public IColumnsDb CreateColumnsDb(string dbName) => new MemColumnsDb(dbName); - public IDb CreateDb(string dbName) => new MemDb(dbName); + public IDb CreateDb(string dbName) => new MemDb(dbName, true); } } diff --git a/src/Nethermind/Nethermind.Db/Metrics.cs b/src/Nethermind/Nethermind.Db/Metrics.cs index 9952dd6f773..487c889e69a 100644 --- a/src/Nethermind/Nethermind.Db/Metrics.cs +++ b/src/Nethermind/Nethermind.Db/Metrics.cs @@ -118,6 +118,62 @@ public static class Metrics [Description("Number of Metadata DB writes.")] public static long MetadataDbWrites { get; set; } + [CounterMetric] + [Description("Number of Leaf DB reads.")] + public static long LeafDbReads { get; set; } + + [CounterMetric] + [Description("Number of Leaf DB writes.")] + public static long LeafDbWrites { get; set; } + + [CounterMetric] + [Description("Number of Stem DB reads.")] + public static long StemDbReads { get; set; } + + [CounterMetric] + [Description("Number of Stem DB writes.")] + public static long StemDbWrites { get; set; } + + [CounterMetric] + [Description("Number of Branch DB reads.")] + public static long BranchDbReads { get; set; } + + [CounterMetric] + [Description("Number of Branch DB writes.")] + public static long BranchDbWrites { get; set; } + + [CounterMetric] + [Description("Number of ForwardDiff DB reads.")] + public static long ForwardDiffDbReads { get; set; } + + [CounterMetric] + [Description("Number of ForwardDiff DB writes.")] + public static long ForwardDiffDbWrites { get; set; } + + [CounterMetric] + [Description("Number of ReverseDiff DB reads.")] + public static long ReverseDiffDbReads { get; set; } + + [CounterMetric] + [Description("Number of ReverseDiff DB writes.")] + public static long ReverseDiffDbWrites { get; set; } + + [CounterMetric] + [Description("Number of StateRootToBlock DB reads.")] + public static long StateRootToBlockDbReads { get; set; } + + [CounterMetric] + [Description("Number of StateRootToBlock DB writes.")] + public static long StateRootToBlockDbWrites { get; set; } + + [CounterMetric] + [Description("Number of Preimages DB reads.")] + public static long PreimagesDbReads { get; set; } + + [CounterMetric] + [Description("Number of Preimages DB writes.")] + public static long PreimagesDbWrites { get; set; } + [GaugeMetric] [Description("Indicator if StadeDb is being pruned.")] public static int StateDbPruning { get; set; } diff --git a/src/Nethermind/Nethermind.Db/NullDb.cs b/src/Nethermind/Nethermind.Db/NullDb.cs index 4c39c9da25c..7caf75a290a 100644 --- a/src/Nethermind/Nethermind.Db/NullDb.cs +++ b/src/Nethermind/Nethermind.Db/NullDb.cs @@ -51,6 +51,20 @@ public bool KeyExists(ReadOnlySpan key) public IDb Innermost => this; public void Flush() { } public void Clear() { } + public IEnumerable> GetIterator() + { + yield break; + } + + public IEnumerable> GetIterator(byte[] start) + { + yield break; + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + yield break; + } public IEnumerable> GetAll(bool ordered = false) => Enumerable.Empty>(); diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index fe54b92e6fa..73a0caec256 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -90,6 +90,21 @@ public void Flush() public void Clear() { throw new InvalidOperationException(); } + public IEnumerable> GetIterator() + { + return _wrappedDb.GetIterator(); + } + + public IEnumerable> GetIterator(byte[] start) + { + return _wrappedDb.GetIterator(start); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + return _wrappedDb.GetIterator(start, end); + } + public virtual void ClearTempChanges() { _memDb.Clear(); diff --git a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs index 4e607a4a745..4cc0841b3e9 100644 --- a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs +++ b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs @@ -98,6 +98,21 @@ public void Clear() File.Delete(DbPath); } + public IEnumerable> GetIterator() + { + throw new NotImplementedException(); + } + + public IEnumerable> GetIterator(byte[] start) + { + throw new NotImplementedException(); + } + + public IEnumerable> GetIterator(byte[] start, byte[] end) + { + throw new NotImplementedException(); + } + public IEnumerable> GetAll(bool ordered = false) => _cache; public IEnumerable GetAllValues(bool ordered = false) => _cache.Values; diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs index 8f5f0d7f199..e32c6a7c585 100644 --- a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs +++ b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs @@ -65,6 +65,13 @@ private void RegisterAll(bool useReceiptsDb) RegisterCustomDb(DbNames.Receipts, () => new ReadOnlyColumnsDb(new MemColumnsDb(), false)); } RegisterDb(BuildRocksDbSettings(DbNames.Metadata, () => Metrics.MetadataDbReads++, () => Metrics.MetadataDbWrites++)); + // TODO: convert this to two column families + RegisterDb(BuildRocksDbSettings(DbNames.Leaf, () => Metrics.LeafDbReads++, () => Metrics.LeafDbWrites++)); + RegisterDb(BuildRocksDbSettings(DbNames.InternalNodes, () => Metrics.BranchDbReads++, () => Metrics.BranchDbWrites++)); + RegisterDb(BuildRocksDbSettings(DbNames.ForwardDiff, () => Metrics.ForwardDiffDbReads++, () => Metrics.ForwardDiffDbWrites++)); + RegisterDb(BuildRocksDbSettings(DbNames.ReverseDiff, () => Metrics.ReverseDiffDbReads++, () => Metrics.ReverseDiffDbWrites++)); + RegisterDb(BuildRocksDbSettings(DbNames.StateRootToBlock, () => Metrics.StateRootToBlockDbReads++, () => Metrics.StateRootToBlockDbWrites++)); + RegisterDb(BuildRocksDbSettings(DbNames.Preimages, () => Metrics.PreimagesDbReads++, () => Metrics.PreimagesDbWrites++)); } private RocksDbSettings BuildRocksDbSettings(string dbName, Action updateReadsMetrics, Action updateWriteMetrics, bool deleteOnStart = false) diff --git a/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeDataAnalyzerHelperTests.cs b/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeDataAnalyzerHelperTests.cs index 729e432b1cf..adec2f7d36f 100644 --- a/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeDataAnalyzerHelperTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeDataAnalyzerHelperTests.cs @@ -20,7 +20,7 @@ public void Validate_CodeBitmap_With_Push10() (byte)Instruction.JUMPDEST }; - var bitmap = CodeDataAnalyzerHelper.CreateCodeBitmap(code); + var bitmap = CodeDataAnalyzerHelper.CreateCodeBitmap(new ByteCode(code)); bitmap[0].Should().Be(127); bitmap[1].Should().Be(224); } @@ -35,7 +35,7 @@ public void Validate_CodeBitmap_With_Push30() (byte)Instruction.JUMPDEST }; - var bitmap = CodeDataAnalyzerHelper.CreateCodeBitmap(code); + var bitmap = CodeDataAnalyzerHelper.CreateCodeBitmap(new ByteCode(code)); bitmap[0].Should().Be(127); bitmap[1].Should().Be(255); bitmap[2].Should().Be(255); diff --git a/src/Nethermind/Nethermind.Evm.Test/CodeTests.cs b/src/Nethermind/Nethermind.Evm.Test/CodeTests.cs new file mode 100644 index 00000000000..6a5f97cb73d --- /dev/null +++ b/src/Nethermind/Nethermind.Evm.Test/CodeTests.cs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Specs; +using Nethermind.Specs.Forks; +using Nethermind.State; +using NUnit.Framework; + +namespace Nethermind.Evm.Test; + +[TestFixture] +public class CodeTests +{ + [Test] + public void ICodeTests() + { + TestSpecProvider specProvider = new (Prague.Instance); + VerkleStateTree tree = TestItem.Tree.GetVerkleStateTree(null); + MemDb codeDb = new (); + VerkleWorldState worldState = new (tree, codeDb, LimboLogs.Instance); + + byte[] code = new byte[2000]; + TestItem.Random.NextBytes(code); + + worldState.CreateAccount(TestItem.AddressA, 200000); + worldState.CreateAccount(TestItem.AddressB, 200000); + worldState.Commit(specProvider.SpecToReturn); + worldState.InsertCode(TestItem.AddressA, code, specProvider.SpecToReturn, false); + worldState.Commit(specProvider.SpecToReturn); + worldState.CommitTree(0); + + ByteCode byteCode = new (code); + VerkleCode verkleCode = new (worldState, TestItem.AddressA); + + Assert.IsTrue(byteCode.Length == verkleCode.Length); + + byte[] byteCodeBytes = byteCode.ToBytes(); + byte[] verkleCodeBytes = verkleCode.ToBytes(); + byteCodeBytes.Should().BeEquivalentTo(verkleCodeBytes); + + for (int i = 0; i < code.Length; i++) AssetEqualSlice(byteCode, verkleCode, i, code.Length); + for (int i = 0; i < code.Length; i++) AssetEqualZeroPaddedSpan(byteCode, verkleCode, i, code.Length); + } + + private void AssetEqualSlice(ByteCode byteCode, VerkleCode verkleCode, int start, int length) + { + Span byteCodeSlice = byteCode.Slice(start, Math.Min(length - start, length)); + Span verkleCodeSlice = verkleCode.Slice(start, Math.Min(length - start, length)); + byteCodeSlice.ToArray().Should().BeEquivalentTo(verkleCodeSlice.ToArray()); + } + + private void AssetEqualZeroPaddedSpan(ByteCode byteCode, VerkleCode verkleCode, int start, int length) + { + ZeroPaddedSpan byteCodeSlice = byteCode.SliceWithZeroPadding((UInt256)start, length); + ZeroPaddedSpan verkleCodeSlice = verkleCode.SliceWithZeroPadding((UInt256)start, length); + Assert.IsTrue(byteCodeSlice.Length == verkleCodeSlice.Length); + Assert.IsTrue(byteCodeSlice.PaddingLength == verkleCodeSlice.PaddingLength); + Assert.IsTrue(byteCodeSlice.PadDirection == verkleCodeSlice.PadDirection); + Assert.IsTrue(byteCodeSlice.Span.SequenceEqual(verkleCodeSlice.Span)); + } +} diff --git a/src/Nethermind/Nethermind.Evm.Test/IntrinsicGasCalculatorTests.cs b/src/Nethermind/Nethermind.Evm.Test/IntrinsicGasCalculatorTests.cs index 64bfc890bba..e5992dee8ba 100644 --- a/src/Nethermind/Nethermind.Evm.Test/IntrinsicGasCalculatorTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/IntrinsicGasCalculatorTests.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; -using System.Collections.Immutable; using System.IO; using FluentAssertions; using Nethermind.Core; @@ -12,6 +11,7 @@ using Nethermind.Core.Test.Builders; using Nethermind.Int256; using Nethermind.Specs.Forks; +using Nethermind.Verkle.Tree; using NUnit.Framework; namespace Nethermind.Evm.Test @@ -19,6 +19,7 @@ namespace Nethermind.Evm.Test [TestFixture] public class IntrinsicGasCalculatorTests { + private const long IntrinsicWitnessGasCode = 13300; public static IEnumerable<(Transaction Tx, long cost, string Description)> TestCaseSource() { yield return (Build.A.Transaction.SignedAndResolved().TestObject, 21000, "empty"); @@ -46,6 +47,8 @@ public class IntrinsicGasCalculatorTests public void Intrinsic_cost_is_calculated_properly((Transaction Tx, long Cost, string Description) testCase) { IntrinsicGasCalculator.Calculate(testCase.Tx, Berlin.Instance).Should().Be(testCase.Cost); + VerkleWitness witness = new VerkleWitness(); + IntrinsicGasCalculator.Calculate(testCase.Tx, Prague.Instance, ref witness).Should().Be(testCase.Cost + IntrinsicWitnessGasCode); } [TestCaseSource(nameof(AccessTestCaseSource))] @@ -75,7 +78,15 @@ void Test(IReleaseSpec spec, bool supportsAccessLists) } else { - IntrinsicGasCalculator.Calculate(tx, spec).Should().Be(21000 + testCase.Cost, spec.Name); + if (spec.IsVerkleTreeEipEnabled) + { + VerkleWitness witness = new VerkleWitness(); + IntrinsicGasCalculator.Calculate(tx, spec, ref witness).Should().Be(21000 + IntrinsicWitnessGasCode + testCase.Cost, spec.Name); + } + else + { + IntrinsicGasCalculator.Calculate(tx, spec).Should().Be(21000 + testCase.Cost, spec.Name); + } } } @@ -89,6 +100,7 @@ void Test(IReleaseSpec spec, bool supportsAccessLists) Test(Istanbul.Instance, false); Test(MuirGlacier.Instance, false); Test(Berlin.Instance, true); + Test(Prague.Instance, true); } [TestCaseSource(nameof(DataTestCaseSource))] @@ -98,9 +110,23 @@ public void Intrinsic_cost_of_data_is_calculated_properly((byte[] Data, int OldC void Test(IReleaseSpec spec, bool isAfterRepricing) { - IntrinsicGasCalculator.Calculate(tx, spec).Should() - .Be(21000 + (isAfterRepricing ? testCase.NewCost : testCase.OldCost), spec.Name, - testCase.Data.ToHexString()); + long expectedGas = 21000 + (isAfterRepricing ? testCase.NewCost : testCase.OldCost); + long actualGas; + switch (spec.IsVerkleTreeEipEnabled) + { + case true: + expectedGas += IntrinsicWitnessGasCode; + VerkleWitness witness = new VerkleWitness(); + actualGas = IntrinsicGasCalculator.Calculate(tx, spec, ref witness); + break; + case false: + actualGas = IntrinsicGasCalculator.Calculate(tx, spec); + break; + } + + actualGas.Should() + .Be(expectedGas, spec.Name, testCase.Data.ToHexString()); + } Test(Homestead.Instance, false); @@ -113,6 +139,7 @@ void Test(IReleaseSpec spec, bool isAfterRepricing) Test(Istanbul.Instance, true); Test(MuirGlacier.Instance, true); Test(Berlin.Instance, true); + Test(Prague.Instance, true); } } } diff --git a/src/Nethermind/Nethermind.Evm.Test/Nethermind.Evm.Test.csproj b/src/Nethermind/Nethermind.Evm.Test/Nethermind.Evm.Test.csproj index ad6a02a3d0c..a848891235f 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Nethermind.Evm.Test.csproj +++ b/src/Nethermind/Nethermind.Evm.Test/Nethermind.Evm.Test.csproj @@ -25,6 +25,7 @@ + diff --git a/src/Nethermind/Nethermind.Evm.Test/SModTests.cs b/src/Nethermind/Nethermind.Evm.Test/SModTests.cs index 17b2f524b65..9a5d4887485 100644 --- a/src/Nethermind/Nethermind.Evm.Test/SModTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/SModTests.cs @@ -2,12 +2,20 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Int256; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test { + [TestFixture(StateType.Merkle)] + [TestFixture(StateType.Verkle)] public class SModTests : VirtualMachineTestsBase { + public SModTests(StateType stateType): base(stateType) + { + + } + [TestCase(1, 1, 0)] [TestCase(1, 0, 0)] [TestCase(1, -1, 0)] diff --git a/src/Nethermind/Nethermind.Evm.Test/SimdTests.cs b/src/Nethermind/Nethermind.Evm.Test/SimdTests.cs index f648b10f041..c59d51ca16a 100644 --- a/src/Nethermind/Nethermind.Evm.Test/SimdTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/SimdTests.cs @@ -6,19 +6,22 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Specs; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test { - [TestFixture(true)] - [TestFixture(false)] + [TestFixture(true, StateType.Merkle)] + [TestFixture(false, StateType.Merkle)] + [TestFixture(true, StateType.Verkle)] + [TestFixture(false, StateType.Verkle)] [Parallelizable(ParallelScope.Self)] public class SimdTests : VirtualMachineTestsBase { private readonly bool _simdDisabled; protected override long BlockNumber => MainnetSpecProvider.ConstantinopleFixBlockNumber; - public SimdTests(bool simdDisabled) + public SimdTests(bool simdDisabled, StateType stateType): base(stateType) { _simdDisabled = simdDisabled; } diff --git a/src/Nethermind/Nethermind.Evm.Test/SltTests.cs b/src/Nethermind/Nethermind.Evm.Test/SltTests.cs index 3e62d281eda..e67c5d7225a 100644 --- a/src/Nethermind/Nethermind.Evm.Test/SltTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/SltTests.cs @@ -2,12 +2,20 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Int256; +using Nethermind.State; using NUnit.Framework; namespace Nethermind.Evm.Test { + + [TestFixture(StateType.Merkle)] + [TestFixture(StateType.Verkle)] public class SltTests : VirtualMachineTestsBase { + public SltTests(StateType stateType): base(stateType) + { + + } [TestCase(1, 1, 0)] [TestCase(1, 0, 0)] [TestCase(1, -1, 0)] diff --git a/src/Nethermind/Nethermind.Evm.Test/TestAllTracerWithOutput.cs b/src/Nethermind/Nethermind.Evm.Test/TestAllTracerWithOutput.cs index d05e3ea7f0b..202a1e32ab5 100644 --- a/src/Nethermind/Nethermind.Evm.Test/TestAllTracerWithOutput.cs +++ b/src/Nethermind/Nethermind.Evm.Test/TestAllTracerWithOutput.cs @@ -13,6 +13,7 @@ namespace Nethermind.Evm.Test public class TestAllTracerWithOutput : ITxTracer { public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => true; public bool IsTracingActions => true; public bool IsTracingOpLevelStorage => true; public bool IsTracingMemory => true; @@ -57,6 +58,10 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool { } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + } + public void ReportOperationError(EvmExceptionType error) { } diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/BlockReceiptsTracerTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/BlockReceiptsTracerTests.cs index 8cbada7d500..d96bbf170a6 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/BlockReceiptsTracerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/BlockReceiptsTracerTests.cs @@ -19,7 +19,7 @@ public void Sets_state_root_if_provided_on_success() { Block block = Build.A.Block.WithTransactions(Build.A.Transaction.TestObject).TestObject; - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(NullBlockTracer.Instance); tracer.StartNewBlockTrace(block); tracer.StartNewTxTrace(block.Transactions[0]); @@ -33,7 +33,7 @@ public void Sets_tx_type() { Block block = Build.A.Block.WithTransactions(Build.A.Transaction.WithChainId(TestBlockchainIds.ChainId).WithType(TxType.AccessList).TestObject).TestObject; - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(NullBlockTracer.Instance); tracer.StartNewBlockTrace(block); tracer.StartNewTxTrace(block.Transactions[0]); @@ -47,7 +47,7 @@ public void Sets_state_root_if_provided_on_failure() { Block block = Build.A.Block.WithTransactions(Build.A.Transaction.TestObject).TestObject; - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(NullBlockTracer.Instance); tracer.StartNewBlockTrace(block); tracer.StartNewTxTrace(block.Transactions[0]); @@ -62,7 +62,7 @@ public void Invokes_other_tracer_mark_as_failed_if_other_block_tracer_is_tx_trac Block block = Build.A.Block.WithTransactions(Build.A.Transaction.TestObject).TestObject; IBlockTracer otherTracer = Substitute.For(); - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(otherTracer); tracer.StartNewBlockTrace(block); tracer.StartNewTxTrace(block.Transactions[0]); @@ -77,7 +77,7 @@ public void Invokes_other_tracer_mark_as_success_if_other_block_tracer_is_tx_tra Block block = Build.A.Block.WithTransactions(Build.A.Transaction.TestObject).TestObject; IBlockTracer otherTracer = Substitute.For(); - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(otherTracer); tracer.StartNewBlockTrace(block); tracer.StartNewTxTrace(block.Transactions[0]); diff --git a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs index 0ac0d44fa5d..db772fcb9e8 100644 --- a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs @@ -167,7 +167,7 @@ public void Should_stop_when_cancellation(bool withCancellation) CancellationBlockTracer cancellationBlockTracer = new(feesTracer, token); - BlockReceiptsTracer blockTracer = new(); + BlockReceiptsTracer blockTracer = new(true, false); blockTracer.SetOtherTracer(cancellationBlockTracer); blockTracer.StartNewBlockTrace(block); @@ -230,7 +230,7 @@ public void Check_fees_with_free_transaction() private void ExecuteAndTrace(Block block, IBlockTracer otherTracer) { - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(otherTracer); tracer.StartNewBlockTrace(block); diff --git a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs index bfb4470881d..255b14a43fb 100644 --- a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs @@ -668,7 +668,7 @@ private BlockReceiptsTracer BuildTracer(Block block, Transaction tx, bool stateD } IBlockTracer otherTracer = types != ParityTraceTypes.None ? new ParityLikeBlockTracer(tx.Hash, ParityTraceTypes.Trace | ParityTraceTypes.StateDiff) : (IBlockTracer)NullBlockTracer.Instance; - BlockReceiptsTracer tracer = new(); + BlockReceiptsTracer tracer = new(true, false); tracer.SetOtherTracer(otherTracer); return tracer; } diff --git a/src/Nethermind/Nethermind.Evm.Test/VerkleTreeTests.cs b/src/Nethermind/Nethermind.Evm.Test/VerkleTreeTests.cs new file mode 100644 index 00000000000..5e518eea61c --- /dev/null +++ b/src/Nethermind/Nethermind.Evm.Test/VerkleTreeTests.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Specs; +using Nethermind.State; +using NUnit.Framework; + +namespace Nethermind.Evm.Test; + +public class VerkleTreeTests: VirtualMachineTestsBase +{ + public VerkleTreeTests() : base(StateType.Verkle) { } + + protected override long BlockNumber => MainnetSpecProvider.GrayGlacierBlockNumber; + protected override ulong Timestamp => MainnetSpecProvider.PragueBlockTimestamp; + + [Test] + public void TestGasCostUpdateForPushOpCodes() + { + + } + [Test] + public void TestGasCostUpdateForContractCreationComplete() + { + + } + + [Test] + public void TestGasCostUpdateForContractCreationInit() + { + + } + [Test] + public void TestGasCostUpdateForBalanceOpCode() + { + + } + [Test] + public void TestGasCostUpdateForCodeOpCodes() + { + + } + [Test] + public void TestGasCostUpdateForStorageAccessAndUpdate() + { + + } + [Test] + public void TestGasCostForProofOfAbsence() + { + + } + [Test] + public void TestGasCostUpdateForCodeChunksAccess() + { + + } + [Test] + public void TestGasCostUpdateForCodeCopy() + { + + } + [Test] + public void TestGasCostUpdateForPush1() + { + + } + [Test] + public void TestGasCostUpdateForPushX() + { + + } + [Test] + public void TestGasCostUpdateForCreateOpCodes() + { + + } + +} diff --git a/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs b/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs index d9b4ce524a2..2ed23c7cf8a 100644 --- a/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs +++ b/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs @@ -11,6 +11,7 @@ using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Db; +using Nethermind.Db.Rocks; using Nethermind.Int256; using Nethermind.Evm.Tracing; using Nethermind.Evm.Tracing.GethStyle; @@ -24,6 +25,9 @@ namespace Nethermind.Evm.Test { public class VirtualMachineTestsBase { + public VirtualMachineTestsBase(StateType stateType = StateType.Merkle) { _stateType = stateType; } + public VirtualMachineTestsBase() { _stateType = StateType.Merkle; } + protected const string SampleHexData1 = "a01234"; protected const string SampleHexData2 = "b15678"; protected const string HexZero = "00"; @@ -32,6 +36,7 @@ public class VirtualMachineTestsBase private IEthereumEcdsa _ethereumEcdsa; protected ITransactionProcessor _processor; private IDb _stateDb; + private readonly StateType _stateType; protected VirtualMachine Machine { get; private set; } protected IWorldState TestState { get; private set; } @@ -59,10 +64,24 @@ public virtual void Setup() { ILogManager logManager = GetLogManager(); - IDb codeDb = new MemDb(); - _stateDb = new MemDb(); - ITrieStore trieStore = new TrieStore(_stateDb, logManager); - TestState = new WorldState(trieStore, codeDb, logManager); + switch (_stateType) + { + case StateType.Merkle: + IDb codeDb = new MemDb(); + _stateDb = new MemDb(); + ITrieStore trieStore = new TrieStore(_stateDb, logManager); + TestState = new WorldState(trieStore, codeDb, logManager); + break; + case StateType.Verkle: + IDbProvider provider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + _stateDb = provider.StateDb; + VerkleStateTree vTree = new VerkleStateTree(provider, LimboLogs.Instance); + TestState = new VerkleWorldState(vTree, new MemDb(), logManager); + break; + default: + throw new ArgumentOutOfRangeException(); + } + _ethereumEcdsa = new EthereumEcdsa(SpecProvider.ChainId, logManager); IBlockhashProvider blockhashProvider = TestBlockhashProvider.Instance; Machine = new VirtualMachine(blockhashProvider, SpecProvider, logManager); diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeDataAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeDataAnalyzer.cs index 129c04faec5..05e17ce3e31 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeDataAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeDataAnalyzer.cs @@ -9,9 +9,9 @@ namespace Nethermind.Evm.CodeAnalysis public class CodeDataAnalyzer : ICodeInfoAnalyzer { private byte[]? _codeBitmap; - public byte[] MachineCode { get; set; } + public ICode MachineCode { get; set; } - public CodeDataAnalyzer(byte[] code) + public CodeDataAnalyzer(ICode code) { MachineCode = code; } @@ -54,7 +54,7 @@ public static class CodeDataAnalyzerHelper /// Collects data locations in code. /// An unset bit means the byte is an opcode, a set bit means it's data. /// - public static byte[] CreateCodeBitmap(byte[] code) + public static byte[] CreateCodeBitmap(ICode code) { // The bitmap is 4 bytes longer than necessary, in case the code // ends with a PUSH32, the algorithm will push zeroes onto the diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs index 3be8b18785b..708e93b1562 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs @@ -3,12 +3,132 @@ using System; using System.Collections; +using System.Linq; using System.Reflection.PortableExecutable; using System.Threading; +using DotNetty.Common.Utilities; +using Nethermind.Core; using Nethermind.Evm.Precompiles; +using Nethermind.Int256; +using Nethermind.Serialization.Rlp; +using Nethermind.State; namespace Nethermind.Evm.CodeAnalysis { + + public interface ICode + { + byte this[int index] { get; } + int Length { get; } + + ZeroPaddedSpan SliceWithZeroPadding(in UInt256 startIndex, int length, + PadDirection padDirection = PadDirection.Right); + + byte[] ToBytes(); + Span Slice(int index, int length); + } + public readonly struct ByteCode: ICode + { + + public ByteCode(byte[] code) + { + MachineCode = code; + } + + public byte[] MachineCode { get; } + + public byte this[int index] + { + get => MachineCode[index]; + } + + public int Length => MachineCode.Length; + + public ZeroPaddedSpan SliceWithZeroPadding(in UInt256 startIndex, int length, PadDirection padDirection = PadDirection.Right) => MachineCode.SliceWithZeroPadding(startIndex, length, padDirection); + + public byte[] ToBytes() => MachineCode.ToArray(); + + public Span Slice(int index, int length) => MachineCode.Slice(index, length); + } + + public readonly struct VerkleCode: ICode + { + private IWorldState WorldState { get; } + private Address Owner { get; } + public VerkleCode(IWorldState worldState, Address codeOwner) + { + if (worldState.StateType != StateType.Verkle) throw new NotSupportedException("verkle state needed"); + Owner = codeOwner; + WorldState = worldState; + Length = (int)worldState.GetAccount(codeOwner).CodeSize; + } + + public byte this[int index] + { + get + { + int chunkId = index / 31; + int chunkLoc = index % 31; + return WorldState.GetCodeChunk(Owner, (UInt256)chunkId)[chunkLoc]; + } + } + + public int Length { get; init; } + + public ZeroPaddedSpan SliceWithZeroPadding(in UInt256 startIndex, int length, + PadDirection padDirection = PadDirection.Right) + { + if (startIndex >= Length || startIndex > int.MaxValue) + { + return new ZeroPaddedSpan(default, length, PadDirection.Right); + } + + Span toReturn = Slice((int)startIndex, length); + return new ZeroPaddedSpan(toReturn, length - toReturn.Length, padDirection); + + } + + public byte[] ToBytes() => Slice(0, Length).ToArray(); + + public Span Slice(int index, int length) + { + if (index >= Length) + return Array.Empty(); + + int endIndex = index + length - 1; + if (endIndex >= length) + { + endIndex = Length - 1; + } + + int startChunkId = index / 31; + int startChunkLoc = (index % 31) + 1; + + int endChunkId = endIndex / 31; + int endChunkLoc = (endIndex % 31) + 1; + + byte[] codeSlice = new byte[(endIndex - index) + 1]; + Span codeSliceSpan = codeSlice; + if (startChunkId == endChunkId) + { + WorldState.GetCodeChunk(Owner, (UInt256)startChunkId)[startChunkLoc..(endChunkLoc + 1)].CopyTo(codeSliceSpan); + } + else + { + WorldState.GetCodeChunk(Owner, (UInt256)startChunkId)[startChunkLoc..].CopyTo(codeSliceSpan); + codeSliceSpan = codeSliceSpan.Slice(32 - startChunkLoc); + for (int i = (startChunkId+1); i < endChunkId; i++) + { + WorldState.GetCodeChunk(Owner, (UInt256)i)[1..].CopyTo(codeSliceSpan); + codeSliceSpan = codeSliceSpan.Slice(31); + } + WorldState.GetCodeChunk(Owner, (UInt256)endChunkId)[1..(endChunkLoc + 1)].CopyTo(codeSliceSpan); + } + return codeSlice; + } + } + + public class CodeInfo { private const int SampledCodeLength = 10_001; @@ -16,13 +136,18 @@ public class CodeInfo private const int NumberOfSamples = 100; private static Random _rand = new(); - public byte[] MachineCode { get; set; } + public ICode MachineCode { get; set; } public IPrecompile? Precompile { get; set; } private ICodeInfoAnalyzer? _analyzer; + public CodeInfo(IWorldState worldState, Address codeOwner) + { + MachineCode = new VerkleCode(worldState, codeOwner); + } + public CodeInfo(byte[] code) { - MachineCode = code; + MachineCode = new ByteCode(code); } public bool IsPrecompile => Precompile is not null; @@ -30,7 +155,7 @@ public CodeInfo(byte[] code) public CodeInfo(IPrecompile precompile) { Precompile = precompile; - MachineCode = Array.Empty(); + MachineCode = new ByteCode(Array.Empty()); } public bool ValidateJump(int destination, bool isSubroutine) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpdestAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpdestAnalyzer.cs index d3757b8f7b9..f29437fe6f8 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpdestAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpdestAnalyzer.cs @@ -9,12 +9,12 @@ namespace Nethermind.Evm.CodeAnalysis { public class JumpdestAnalyzer : ICodeInfoAnalyzer { - private byte[] MachineCode { get; set; } + private ICode MachineCode { get; set; } private BitArray? _validJumpDestinations; private BitArray? _validJumpSubDestinations; - public JumpdestAnalyzer(byte[] code) + public JumpdestAnalyzer(ICode code) { MachineCode = code; } diff --git a/src/Nethermind/Nethermind.Evm/EvmState.cs b/src/Nethermind/Nethermind.Evm/EvmState.cs index fd00178f4e8..d6ebde60a16 100644 --- a/src/Nethermind/Nethermind.Evm/EvmState.cs +++ b/src/Nethermind/Nethermind.Evm/EvmState.cs @@ -38,7 +38,7 @@ public StackPool(int maxCallStackDepth = VirtualMachine.MaxCallDepth * 2) /// /// The word 'return' acts here once as a verb 'to return stack to the pool' and once as a part of the - /// compound noun 'return stack' which is a stack of subroutine return values. + /// compound noun 'return stack' which is a stack of subroutine return values. /// /// /// @@ -106,6 +106,11 @@ private int[] RentReturnStack() // As we can add here from VM, we need it as ICollection public ICollection Logs => _logs; + /// + /// Verkle Tree: to maintain a list of all accessed subtrees and leaves + /// + public IVerkleWitness? VerkleTreeWitness { get; } + private readonly JournalSet
_accessedAddresses; private readonly JournalSet _accessedStorageCells; private readonly JournalCollection _logs; @@ -114,6 +119,7 @@ private int[] RentReturnStack() private readonly int _accessedStorageKeysSnapshot; private readonly int _destroyListSnapshot; private readonly int _logsSnapshot; + private readonly int _verkleWitnessSnapshot; public int DataStackHead = 0; @@ -179,6 +185,7 @@ internal EvmState( _accessedStorageCells = stateForAccessLists._accessedStorageCells; _destroyList = stateForAccessLists._destroyList; _logs = stateForAccessLists._logs; + VerkleTreeWitness = stateForAccessLists.VerkleTreeWitness; } else { @@ -187,13 +194,14 @@ internal EvmState( _accessedStorageCells = new JournalSet(); _destroyList = new JournalSet
(); _logs = new JournalCollection(); + VerkleTreeWitness = env.VerkleWitness; } _accessedAddressesSnapshot = _accessedAddresses.TakeSnapshot(); _accessedStorageKeysSnapshot = _accessedStorageCells.TakeSnapshot(); _destroyListSnapshot = _destroyList.TakeSnapshot(); _logsSnapshot = _logs.TakeSnapshot(); - + _verkleWitnessSnapshot = VerkleTreeWitness?.TakeSnapshot() ?? 0; } public Address From @@ -295,6 +303,7 @@ private void Restore() _destroyList.Restore(_destroyListSnapshot); _accessedAddresses.Restore(_accessedAddressesSnapshot); _accessedStorageCells.Restore(_accessedStorageKeysSnapshot); + VerkleTreeWitness?.Restore(_verkleWitnessSnapshot); } } } diff --git a/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs b/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs index afd332d5a39..3fe35afbb46 100644 --- a/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs +++ b/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs @@ -20,7 +20,8 @@ public ExecutionEnvironment TxExecutionContext txExecutionContext, UInt256 transferValue, UInt256 value, - int callDepth = 0) + int callDepth = 0, + IVerkleWitness? verkleWitness = null) { CodeInfo = codeInfo; ExecutingAccount = executingAccount; @@ -31,6 +32,7 @@ public ExecutionEnvironment TransferValue = transferValue; Value = value; CallDepth = callDepth; + VerkleWitness = verkleWitness; } /// @@ -38,6 +40,8 @@ public ExecutionEnvironment /// public readonly CodeInfo CodeInfo; + public readonly IVerkleWitness? VerkleWitness; + /// /// Currently executing account (in DELEGATECALL this will be equal to caller). /// diff --git a/src/Nethermind/Nethermind.Evm/GasCostOf.cs b/src/Nethermind/Nethermind.Evm/GasCostOf.cs index 6875a781100..fe91b2f94a6 100644 --- a/src/Nethermind/Nethermind.Evm/GasCostOf.cs +++ b/src/Nethermind/Nethermind.Evm/GasCostOf.cs @@ -61,5 +61,11 @@ public static class GasCostOf public const long AccessStorageListEntry = 1900; // eip-2930 public const long TLoad = WarmStateRead; // eip-1153 public const long TStore = WarmStateRead; // eip-1153 + + public const long WitnessChunkRead = 200; // verkle-trees + public const long WitnessChunkWrite = 500; // verkle-trees + public const long WitnessChunkFill = 6200; // verkle-trees + public const long WitnessBranchRead = 1900; // verkle-trees + public const long WitnessBranchWrite = 3000; // verkle-trees } } diff --git a/src/Nethermind/Nethermind.Evm/GasUtils.cs b/src/Nethermind/Nethermind.Evm/GasUtils.cs new file mode 100644 index 00000000000..350aba0ded6 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/GasUtils.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Runtime.CompilerServices; + +namespace Nethermind.Evm; + +public static class GasUtils +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool UpdateGas(long gasCost, ref long gasAvailable) + { + if (gasAvailable < gasCost) return false; + gasAvailable -= gasCost; + return true; + } +} diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs index 248f0855796..94400f63078 100644 --- a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs +++ b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs @@ -7,6 +7,7 @@ using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; using Nethermind.Int256; +using Nethermind.Verkle.Tree; namespace Nethermind.Evm { @@ -21,6 +22,14 @@ public static long Calculate(Transaction transaction, IReleaseSpec releaseSpec) return result; } + public static long Calculate(Transaction transaction, IReleaseSpec releaseSpec, ref VerkleWitness witness) + { + long result = Calculate(transaction, releaseSpec); + if (releaseSpec.IsVerkleTreeEipEnabled) + result += witness.AccessForTransaction(transaction.SenderAddress!, transaction.To!, !transaction.Value.IsZero); + return result; + } + private static long CreateCost(Transaction transaction, IReleaseSpec releaseSpec) { long createCost = 0; diff --git a/src/Nethermind/Nethermind.Evm/Nethermind.Evm.csproj b/src/Nethermind/Nethermind.Evm/Nethermind.Evm.csproj index bf4f06ca167..6f60182aac1 100644 --- a/src/Nethermind/Nethermind.Evm/Nethermind.Evm.csproj +++ b/src/Nethermind/Nethermind.Evm/Nethermind.Evm.csproj @@ -11,6 +11,7 @@ + diff --git a/src/Nethermind/Nethermind.Evm/Tracing/AccessTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/AccessTxTracer.cs index 9cdc92be7f6..991368a61cb 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/AccessTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/AccessTxTracer.cs @@ -21,6 +21,7 @@ public class AccessTxTracer : ITxTracer public bool IsTracingState => false; public bool IsTracingStorage => false; public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -82,6 +83,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool throw new NotImplementedException(); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotImplementedException(); + } + public void ReportOperationError(EvmExceptionType error) { throw new NotImplementedException(); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs index 5d86ede3a7a..7193818625f 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/AlwaysCancelTxTracer.cs @@ -29,6 +29,7 @@ public static AlwaysCancelTxTracer Instance } public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => true; public bool IsTracingActions => true; public bool IsTracingOpLevelStorage => true; public bool IsTracingMemory => true; @@ -48,6 +49,9 @@ public static AlwaysCancelTxTracer Instance public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool isPostMerge = false) => throw new OperationCanceledException(ErrorMessage); + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) => + throw new OperationCanceledException(ErrorMessage); + public void ReportOperationError(EvmExceptionType error) => throw new OperationCanceledException(ErrorMessage); public void ReportOperationRemainingGas(long gas) => throw new OperationCanceledException(ErrorMessage); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs index 225e84ea564..05245909c13 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs @@ -5,16 +5,20 @@ using System.Collections.Generic; using System.Linq; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Int256; -using Nethermind.State; namespace Nethermind.Evm.Tracing { public class BlockReceiptsTracer : IBlockTracer, ITxTracer, IJournal { private Block _block = null!; - public bool IsTracingReceipt => true; + private readonly bool _isTracingReceipt; + private readonly bool _isTracingVerkleWitness; + public bool IsTracingReceipt => _isTracingReceipt | _currentTxTracer.IsTracingReceipt; + public bool IsTracingVerkleWitness => _isTracingVerkleWitness | _currentTxTracer.IsTracingVerkleWitness; public bool IsTracingActions => _currentTxTracer.IsTracingActions; public bool IsTracingOpLevelStorage => _currentTxTracer.IsTracingOpLevelStorage; public bool IsTracingMemory => _currentTxTracer.IsTracingMemory; @@ -31,9 +35,16 @@ public class BlockReceiptsTracer : IBlockTracer, ITxTracer, IJournal private IBlockTracer _otherTracer = NullBlockTracer.Instance; + public BlockReceiptsTracer(bool traceReceipts, bool traceWitness) + { + _isTracingReceipt = traceReceipts; + _isTracingVerkleWitness = traceWitness; + } + public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEntry[] logs, Keccak stateRoot = null) { - _txReceipts.Add(BuildReceipt(recipient, gasSpent, StatusCode.Success, logs, stateRoot)); + if(_isTracingReceipt) + _txReceipts.Add(BuildReceipt(recipient, gasSpent, StatusCode.Success, logs, stateRoot)); // hacky way to support nested receipt tracers if (_otherTracer is ITxTracer otherTxTracer) @@ -49,7 +60,8 @@ public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEn public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string error, Keccak stateRoot = null) { - _txReceipts.Add(BuildFailedReceipt(recipient, gasSpent, error, stateRoot)); + if(_isTracingReceipt) + _txReceipts.Add(BuildFailedReceipt(recipient, gasSpent, error, stateRoot)); // hacky way to support nested receipt tracers if (_otherTracer is ITxTracer otherTxTracer) @@ -97,6 +109,23 @@ private TxReceipt BuildReceipt(Address recipient, long spentGas, byte statusCode public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool isPostMerge = false) => _currentTxTracer.StartOperation(depth, gas, opcode, pc, isPostMerge); + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + if(_isTracingVerkleWitness) + _witnessKeys.Add(verkleWitnessKeys); + + // hacky way to support nested witness tracers + if (_otherTracer is ITxTracer otherTxTracer) + { + otherTxTracer.SetVerkleWitnessKeys(verkleWitnessKeys); + } + + if (_currentTxTracer.IsTracingReceipt) + { + _currentTxTracer.SetVerkleWitnessKeys(verkleWitnessKeys); + } + } + public void ReportOperationError(EvmExceptionType error) => _currentTxTracer.ReportOperationError(error); @@ -194,15 +223,24 @@ public void ReportFees(UInt256 fees, UInt256 burntFees) private ITxTracer _currentTxTracer = NullTxTracer.Instance; private int _currentIndex; private readonly List _txReceipts = new(); + private readonly List> _witnessKeys = new(); + private SortedSet _aggregatedWitnessKeys = new(Bytes.Comparer); private Transaction? _currentTx; public IReadOnlyList TxReceipts => _txReceipts; public TxReceipt LastReceipt => _txReceipts[^1]; + public SortedSet WitnessKeys => _aggregatedWitnessKeys; + public IReadOnlyList LastWitness => _witnessKeys[^1]; public bool IsTracingRewards => _otherTracer.IsTracingRewards; public int TakeSnapshot() => _txReceipts.Count; public void Restore(int snapshot) { int numToRemove = _txReceipts.Count - snapshot; + for (int i = 0; i < numToRemove; i++) + { + _witnessKeys.RemoveAt(_witnessKeys.Count - 1); + } + for (int i = 0; i < numToRemove; i++) { _txReceipts.RemoveAt(_txReceipts.Count - 1); @@ -224,6 +262,7 @@ public void StartNewBlockTrace(Block block) _block = block; _currentIndex = 0; _txReceipts.Clear(); + _witnessKeys.Clear(); _otherTracer.StartNewBlockTrace(block); } @@ -244,13 +283,22 @@ public void EndTxTrace() public void EndBlockTrace() { _otherTracer.EndBlockTrace(); + if (_witnessKeys.Count > 0) + { + _aggregatedWitnessKeys = new SortedSet(Bytes.Comparer); + foreach (IReadOnlyList? keys in _witnessKeys) + { + _aggregatedWitnessKeys.AddRange(keys); + } + } + if (_txReceipts.Count > 0) { Bloom blockBloom = new(); _block.Header.Bloom = blockBloom; - for (var index = 0; index < _txReceipts.Count; index++) + for (int index = 0; index < _txReceipts.Count; index++) { - var receipt = _txReceipts[index]; + TxReceipt? receipt = _txReceipts[index]; blockBloom.Accumulate(receipt.Bloom!); } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CallOutputTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CallOutputTracer.cs index 52d5627529c..dbbf125f135 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CallOutputTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CallOutputTracer.cs @@ -13,6 +13,7 @@ namespace Nethermind.Evm.Tracing public class CallOutputTracer : ITxTracer { public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -54,6 +55,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool throw new NotSupportedException(); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotSupportedException(); + } + public void ReportOperationError(EvmExceptionType error) { throw new NotSupportedException(); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs index c77dee5bfb3..e86b8f08ec9 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs @@ -17,6 +17,7 @@ public class CancellationTxTracer : ITxTracer private readonly CancellationToken _token; private readonly bool _isTracingReceipt; private readonly bool _isTracingActions; + private readonly bool _isTracingVerkleWitness; private readonly bool _isTracingOpLevelStorage; private readonly bool _isTracingMemory; private readonly bool _isTracingInstructions; @@ -43,6 +44,12 @@ public bool IsTracingReceipt init => _isTracingReceipt = value; } + public bool IsTracingVerkleWitness + { + get => _isTracingVerkleWitness || _innerTracer.IsTracingVerkleWitness; + init => _isTracingVerkleWitness = value; + } + public bool IsTracingActions { get => _isTracingActions || _innerTracer.IsTracingActions; @@ -196,6 +203,15 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool } } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + _token.ThrowIfCancellationRequested(); + if (_innerTracer.IsTracingInstructions) + { + _innerTracer.SetVerkleWitnessKeys(verkleWitnessKeys); + } + } + public void ReportOperationError(EvmExceptionType error) { _token.ThrowIfCancellationRequested(); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs index 9d208fdc914..74978649d39 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CompositeTxTracer.cs @@ -20,11 +20,11 @@ public CompositeTxTracer(params ITxTracer[] txTracers) : this((IList) public CompositeTxTracer(IList txTracers) { _txTracers = txTracers; - for (int index = 0; index < txTracers.Count; index++) + foreach (ITxTracer? t in txTracers) { - ITxTracer t = txTracers[index]; IsTracingState |= t.IsTracingState; IsTracingReceipt |= t.IsTracingReceipt; + IsTracingVerkleWitness |= t.IsTracingVerkleWitness; IsTracingActions |= t.IsTracingActions; IsTracingOpLevelStorage |= t.IsTracingOpLevelStorage; IsTracingMemory |= t.IsTracingMemory; @@ -42,6 +42,7 @@ public CompositeTxTracer(IList txTracers) public bool IsTracingState { get; } public bool IsTracingStorage { get; } public bool IsTracingReceipt { get; } + public bool IsTracingVerkleWitness { get; } public bool IsTracingActions { get; } public bool IsTracingOpLevelStorage { get; } public bool IsTracingMemory { get; } @@ -161,6 +162,18 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool } } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + for (int index = 0; index < _txTracers.Count; index++) + { + ITxTracer innerTracer = _txTracers[index]; + if (innerTracer.IsTracingInstructions) + { + innerTracer.SetVerkleWitnessKeys(verkleWitnessKeys); + } + } + } + public void ReportOperationError(EvmExceptionType error) { for (int index = 0; index < _txTracers.Count; index++) diff --git a/src/Nethermind/Nethermind.Evm/Tracing/EstimateGasTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/EstimateGasTracer.cs index fd94fc3dc7f..026cddde772 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/EstimateGasTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/EstimateGasTracer.cs @@ -20,6 +20,7 @@ public EstimateGasTracer() } public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => true; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -67,6 +68,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool throw new NotSupportedException(); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotSupportedException(); + } + public void ReportOperationError(EvmExceptionType error) { throw new NotSupportedException(); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/FeesTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/FeesTracer.cs index b261e5b5b90..4cc7b968075 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/FeesTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/FeesTracer.cs @@ -13,6 +13,7 @@ public class FeesTracer : IBlockTracer, ITxTracer { public bool IsTracingRewards => false; public bool IsTracingState => false; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -90,6 +91,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool throw new NotImplementedException(); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotImplementedException(); + } + public void ReportOperationError(EvmExceptionType error) { throw new NotImplementedException(); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/GasEstimator.cs b/src/Nethermind/Nethermind.Evm/Tracing/GasEstimator.cs index 32e6aa347cc..72bfb6db3b2 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/GasEstimator.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/GasEstimator.cs @@ -102,6 +102,7 @@ public OutOfGasTracer() OutOfGas = false; } public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -137,6 +138,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool { } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + + } + public void ReportOperationError(EvmExceptionType error) { OutOfGas |= error == EvmExceptionType.OutOfGas; diff --git a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethLikeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethLikeTxTracer.cs index 6d9b2454e01..f5c4f67ae6e 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethLikeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethLikeTxTracer.cs @@ -27,6 +27,7 @@ public GethLikeTxTracer(GethTraceOptions options) bool IStateTracer.IsTracingState => false; bool IStorageTracer.IsTracingStorage => false; public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; bool ITxTracer.IsTracingActions => false; public bool IsTracingOpLevelStorage { get; } public bool IsTracingMemory { get; } @@ -85,6 +86,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool } } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotSupportedException(); + } + public void ReportOperationError(EvmExceptionType error) { _traceEntry.Error = GetErrorDescription(error); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs index 5d5b3ac06fb..1d025220b1a 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/ITxTracer.cs @@ -24,6 +24,15 @@ public interface ITxTracer : IWorldStateTracer /// bool IsTracingReceipt { get; } + /// + /// Defines whether SetVerkleWitnessKeys will be called + /// + /// + /// Controls + /// - + /// + bool IsTracingVerkleWitness { get; } + /// /// High level calls with information on the target account /// @@ -160,6 +169,13 @@ public interface ITxTracer : IWorldStateTracer /// Depends on void StartOperation(int depth, long gas, Instruction opcode, int pc, bool isPostMerge = false); + /// + /// + /// + /// + /// Depends on + void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys); + /// /// /// diff --git a/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs index 242d8dbe714..97d9acd4da5 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/NullTxTracer.cs @@ -17,6 +17,7 @@ public class NullTxTracer : ITxTracer private NullTxTracer() { } public bool IsTracingReceipt => false; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -39,6 +40,9 @@ public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool isPostMerge = false) => throw new InvalidOperationException(ErrorMessage); + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) => + throw new InvalidOperationException(ErrorMessage); + public void ReportOperationError(EvmExceptionType error) => throw new InvalidOperationException(ErrorMessage); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs index b5300031b50..7b3a5558e88 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/ParityStyle/ParityLikeTxTracer.cs @@ -65,6 +65,7 @@ public ParityLikeTxTracer(Block block, Transaction? tx, ParityTraceTypes parityT } public bool IsTracingReceipt { get; } + public bool IsTracingVerkleWitness => false; public bool IsTracingActions { get; } public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -273,6 +274,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool _currentVmTrace.Ops.Add(operationTrace); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotSupportedException(); + } + public void ReportOperationError(EvmExceptionType error) { if (error != EvmExceptionType.InvalidJumpDestination && diff --git a/src/Nethermind/Nethermind.Evm/Tracing/Proofs/ProofTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/Proofs/ProofTxTracer.cs index 9d967083e87..e157b8e59fb 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/Proofs/ProofTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/Proofs/ProofTxTracer.cs @@ -29,6 +29,7 @@ public ProofTxTracer(bool treatSystemAccountDifferently) public bool IsTracingBlockHash => true; public bool IsTracingAccess => false; public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -149,6 +150,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool throw new NotSupportedException(); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotImplementedException(); + } + public void ReportOperationError(EvmExceptionType error) { throw new NotSupportedException(); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/VerkleWitnessTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/VerkleWitnessTracer.cs new file mode 100644 index 00000000000..5deeacea2b3 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/Tracing/VerkleWitnessTracer.cs @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Int256; + +namespace Nethermind.Evm.Tracing; + +public class VerkleWitnessTracer: ITxTracer +{ + public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; + public bool IsTracingActions => false; + public bool IsTracingOpLevelStorage => false; + public bool IsTracingMemory => false; + public bool IsTracingInstructions => false; + public bool IsTracingRefunds => false; + public bool IsTracingCode => false; + public bool IsTracingStack => false; + public bool IsTracingState => false; + public bool IsTracingStorage => false; + public bool IsTracingBlockHash => false; + public bool IsTracingAccess => false; + public bool IsTracingFees => false; + public IReadOnlyList VerkleWitnessKeys { get; set; } + + public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEntry[] logs, Keccak stateRoot = null) + { + throw new NotSupportedException(); + } + + public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string error, Keccak stateRoot = null) + { + throw new NotSupportedException(); + } + + public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool isPostMerge = false) + { + throw new NotSupportedException(); + } + + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + VerkleWitnessKeys = verkleWitnessKeys; + } + + public void ReportOperationError(EvmExceptionType error) + { + throw new NotSupportedException(); + } + + public void ReportOperationRemainingGas(long gas) + { + throw new NotSupportedException(); + } + + public void SetOperationStack(List stackTrace) + { + throw new NotSupportedException(); + } + + public void ReportStackPush(in ReadOnlySpan stackItem) + { + throw new NotSupportedException(); + } + + public void SetOperationMemory(List memoryTrace) + { + throw new NotSupportedException(); + } + + public void SetOperationMemorySize(ulong newSize) + { + throw new NotSupportedException(); + } + + public void ReportMemoryChange(long offset, in ReadOnlySpan data) + { + throw new NotSupportedException(); + } + + public void ReportStorageChange(in ReadOnlySpan key, in ReadOnlySpan value) + { + throw new NotSupportedException(); + } + + public void SetOperationStorage(Address address, UInt256 storageIndex, ReadOnlySpan newValue, ReadOnlySpan currentValue) + { + throw new NotSupportedException(); + } + + public void LoadOperationStorage(Address address, UInt256 storageIndex, ReadOnlySpan value) + { + throw new NotSupportedException(); + } + + public void ReportSelfDestruct(Address address, UInt256 balance, Address refundAddress) + { + throw new NotSupportedException(); + } + + public void ReportBalanceChange(Address address, UInt256? before, UInt256? after) + { + throw new NotSupportedException(); + } + + public void ReportCodeChange(Address address, byte[] before, byte[] after) + { + throw new NotSupportedException(); + } + + public void ReportNonceChange(Address address, UInt256? before, UInt256? after) + { + throw new NotSupportedException(); + } + + public void ReportAccountRead(Address address) + { + throw new NotSupportedException(); + } + + public void ReportStorageChange(in StorageCell storageCell, byte[] before, byte[] after) + { + throw new NotSupportedException(); + } + + public void ReportStorageRead(in StorageCell storageCell) + { + throw new NotImplementedException(); + } + + public void ReportAction(long gas, UInt256 value, Address from, Address to, ReadOnlyMemory input, ExecutionType callType, bool isPrecompileCall = false) + { + throw new NotSupportedException(); + } + + public void ReportActionEnd(long gas, ReadOnlyMemory output) + { + throw new NotSupportedException(); + } + + public void ReportActionError(EvmExceptionType exceptionType) + { + throw new NotSupportedException(); + } + + public void ReportActionEnd(long gas, Address deploymentAddress, ReadOnlyMemory deployedCode) + { + throw new NotSupportedException(); + } + + public void ReportBlockHash(Keccak blockHash) + { + throw new NotSupportedException(); + } + + public void ReportByteCode(byte[] byteCode) + { + throw new NotSupportedException(); + } + + public void ReportGasUpdateForVmTrace(long refund, long gasAvailable) + { + throw new NotSupportedException(); + } + + public void ReportRefund(long refund) + { + throw new NotSupportedException(); + } + + public void ReportExtraGasPressure(long extraGasPressure) + { + throw new NotImplementedException(); + } + + public void ReportAccess(IReadOnlySet
accessedAddresses, IReadOnlySet accessedStorageCells) + { + throw new NotImplementedException(); + } + + public void ReportFees(UInt256 fees, UInt256 burntFees) + { + throw new NotImplementedException(); + } +} diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs index 6c66de2e7ba..373c27698c0 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { @@ -17,5 +18,10 @@ public BuildUpTransactionProcessorAdapter(ITransactionProcessor transactionProce public void Execute(Transaction transaction, BlockHeader block, ITxTracer txTracer) => _transactionProcessor.BuildUp(transaction, block, txTracer); + + public ITransactionProcessorAdapter WithNewStateProvider(IWorldState worldState) + { + return new BuildUpTransactionProcessorAdapter(_transactionProcessor.WithNewStateProvider(worldState)); + } } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/CallAndRestoreTransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/CallAndRestoreTransactionProcessorAdapter.cs index 8ede3451467..fe8ee53c191 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/CallAndRestoreTransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/CallAndRestoreTransactionProcessorAdapter.cs @@ -3,6 +3,7 @@ // using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { @@ -17,5 +18,10 @@ public CallAndRestoreTransactionProcessorAdapter(ITransactionProcessor transacti public void Execute(Transaction transaction, BlockHeader block, ITxTracer txTracer) => _transactionProcessor.CallAndRestore(transaction, block, txTracer); + + public ITransactionProcessorAdapter WithNewStateProvider(IWorldState worldState) + { + return new CallAndRestoreTransactionProcessorAdapter(_transactionProcessor.WithNewStateProvider(worldState)); + } } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ChangeableTransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ChangeableTransactionProcessorAdapter.cs index bc8948b8bba..94b23f56c71 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ChangeableTransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ChangeableTransactionProcessorAdapter.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { @@ -26,5 +27,10 @@ public void Execute(Transaction transaction, BlockHeader block, ITxTracer txTrac { CurrentAdapter.Execute(transaction, block, txTracer); } + + public ITransactionProcessorAdapter WithNewStateProvider(IWorldState worldState) + { + return new BuildUpTransactionProcessorAdapter(TransactionProcessor.WithNewStateProvider(worldState)); + } } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ExecuteTransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ExecuteTransactionProcessorAdapter.cs index c9f144a7b3f..5ecaef831b0 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ExecuteTransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ExecuteTransactionProcessorAdapter.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { @@ -17,5 +18,10 @@ public ExecuteTransactionProcessorAdapter(ITransactionProcessor transactionProce public void Execute(Transaction transaction, BlockHeader block, ITxTracer txTracer) => _transactionProcessor.Execute(transaction, block, txTracer); + + public ITransactionProcessorAdapter WithNewStateProvider(IWorldState worldState) + { + return new ExecuteTransactionProcessorAdapter(_transactionProcessor.WithNewStateProvider(worldState)); + } } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessor.cs index 715f1d7e765..d7db8ff62dd 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessor.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { @@ -27,5 +28,7 @@ public interface ITransactionProcessor /// Call transaction, no validations, commit state /// void Trace(Transaction transaction, BlockHeader block, ITxTracer txTracer); + + ITransactionProcessor WithNewStateProvider(IWorldState worldState); } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessorAdapter.cs index a502c273267..48a3a0aef84 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ITransactionProcessorAdapter.cs @@ -3,11 +3,14 @@ using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { public interface ITransactionProcessorAdapter { void Execute(Transaction transaction, BlockHeader block, ITxTracer txTracer); + + public ITransactionProcessorAdapter WithNewStateProvider(IWorldState worldState); } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs index 09cda3ea1ca..33a600d9d85 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs @@ -36,6 +36,11 @@ public void BuildUp(Transaction transaction, BlockHeader block, ITxTracer txTrac public void Trace(Transaction transaction, BlockHeader block, ITxTracer txTracer) => _transactionProcessor.Trace(transaction, block, txTracer); + public ITransactionProcessor WithNewStateProvider(IWorldState worldState) + { + return _transactionProcessor.WithNewStateProvider(worldState); + } + public bool IsContractDeployed(Address address) => _stateProvider.IsContract(address); diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TraceTransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TraceTransactionProcessorAdapter.cs index 7ae895330fa..ddc22f2fe3e 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TraceTransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TraceTransactionProcessorAdapter.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Evm.Tracing; +using Nethermind.State; namespace Nethermind.Evm.TransactionProcessing { @@ -17,5 +18,10 @@ public TraceTransactionProcessorAdapter(ITransactionProcessor transactionProcess public void Execute(Transaction transaction, BlockHeader block, ITxTracer txTracer) => _transactionProcessor.Trace(transaction, block, txTracer); + + public ITransactionProcessorAdapter WithNewStateProvider(IWorldState worldState) + { + return new TraceTransactionProcessorAdapter(_transactionProcessor.WithNewStateProvider(worldState)); + } } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 4610042986e..cbe30d07fb0 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -4,6 +4,7 @@ using System; using System.IO; using System.Linq; +using Microsoft.IdentityModel.Tokens; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; @@ -15,6 +16,7 @@ using Nethermind.Specs; using Nethermind.State; using Nethermind.State.Tracing; +using Nethermind.Verkle.Tree; using Transaction = Nethermind.Core.Transaction; namespace Nethermind.Evm.TransactionProcessing @@ -23,6 +25,7 @@ public class TransactionProcessor : ITransactionProcessor { private readonly EthereumEcdsa _ecdsa; private readonly ILogger _logger; + private readonly ILogManager? _logManager; private readonly ISpecProvider _specProvider; private readonly IWorldState _worldState; private readonly IVirtualMachine _virtualMachine; @@ -62,6 +65,7 @@ public TransactionProcessor( IVirtualMachine? virtualMachine, ILogManager? logManager) { + _logManager = logManager; _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _worldState = worldState ?? throw new ArgumentNullException(nameof(worldState)); @@ -92,6 +96,11 @@ public void Trace(Transaction transaction, BlockHeader block, ITxTracer txTracer Execute(transaction, block, txTracer, ExecutionOptions.NoValidation); } + public ITransactionProcessor WithNewStateProvider(IWorldState worldState) + { + return new TransactionProcessor(_specProvider, worldState, _virtualMachine, _logManager); + } + private void QuickFail(Transaction tx, BlockHeader block, ITxTracer txTracer, bool eip658NotEnabled, string? reason) { @@ -130,6 +139,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra bool notSystemTransaction = !transaction.IsSystem(); bool deleteCallerAccount = false; + VerkleWitness witness = new(); + if (!notSystemTransaction) { spec = new SystemTransactionReleaseSpec(spec); @@ -161,6 +172,7 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra return; } + // TODO: do we need to check Intrinsic Gas before this? if (!noValidation && _worldState.IsInvalidContractSender(spec, caller)) { TraceLogInvalidTx(transaction, "SENDER_IS_CONTRACT"); @@ -187,7 +199,7 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra return; } - long intrinsicGas = IntrinsicGasCalculator.Calculate(transaction, spec); + long intrinsicGas = IntrinsicGasCalculator.Calculate(transaction, spec, ref witness); if (_logger.IsTrace) _logger.Trace($"Intrinsic gas calculated for {transaction.Hash}: " + intrinsicGas); if (notSystemTransaction) @@ -228,7 +240,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra TraceLogInvalidTx(transaction, $"SENDER_ACCOUNT_DOES_NOT_EXIST {caller}"); if (!commit || noValidation || effectiveGasPrice == UInt256.Zero) { - deleteCallerAccount = !commit || restore; + // TODO: move it to checking EIP-4758 + deleteCallerAccount = (!commit || restore) && !spec.IsVerkleTreeEipEnabled; _worldState.CreateAccount(caller, UInt256.Zero); } } @@ -296,18 +309,21 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra transaction.GetRecipient(transaction.IsContractCreation ? _worldState.GetNonce(caller) : 0); if (transaction.IsContractCreation) { + if (spec.IsVerkleTreeEipEnabled) + { + long gasContractCreationInit = witness.AccessForContractCreationInit(recipient, !transaction.Value.IsZero); + if (!GasUtils.UpdateGas(gasContractCreationInit, ref unspentGas)) + { + TraceLogInvalidTx(transaction, $"INSUFFICIENT_GAS: UNSPENT_GAS = {unspentGas}"); + QuickFail(transaction, block, txTracer, eip658NotEnabled, "insufficient unspent gas"); + return; + } + } // if transaction is a contract creation then recipient address is the contract deployment address - Address contractAddress = recipient; - PrepareAccountForContractDeployment(contractAddress!, spec); - } - - if (recipient is null) - { - // this transaction is not a contract creation so it should have the recipient known and not null - throw new InvalidDataException("Recipient has not been resolved properly before tx execution"); + PrepareAccountForContractDeployment(recipient!, spec); } - recipientOrNull = recipient; + recipientOrNull = recipient ?? throw new InvalidDataException("Recipient has not been resolved properly before tx execution"); ExecutionEnvironment env = new ( @@ -320,7 +336,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra inputData: data ?? Array.Empty(), codeInfo: machineCode is null ? _virtualMachine.GetCachedCodeInfo(_worldState, recipient, spec) - : new CodeInfo(machineCode) + : new CodeInfo(machineCode), + verkleWitness: witness ); ExecutionType executionType = transaction.IsContractCreation ? ExecutionType.Create : ExecutionType.Transaction; @@ -379,8 +396,22 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra _worldState.InsertCode(recipient, substate.Output, spec); unspentGas -= codeDepositGasCost; } + + if (spec.IsVerkleTreeEipEnabled) + { + long gasContractCreationCompleted = witness.AccessContractCreated(recipient); + if (!GasUtils.UpdateGas(gasContractCreationCompleted, ref unspentGas)) + { + TraceLogInvalidTx(transaction, $"INSUFFICIENT_GAS: UNSPENT_GAS = {unspentGas}"); + QuickFail(transaction, block, txTracer, eip658NotEnabled, "insufficient unspent gas"); + return; + } + } } + if (spec.IsVerkleTreeEipEnabled && !substate.DestroyList.IsNullOrEmpty()) + throw new InvalidOperationException("DestroyList should be empty for Verkle Trees"); + foreach (Address toBeDestroyed in substate.DestroyList) { if (_logger.IsTrace) _logger.Trace($"Destroying account {toBeDestroyed}"); @@ -405,6 +436,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra Address gasBeneficiary = block.GasBeneficiary; bool gasBeneficiaryNotDestroyed = substate?.DestroyList.Contains(gasBeneficiary) != true; + // TODO: check and modify + witness.AccessForGasBeneficiary(gasBeneficiary); if (statusCode == StatusCode.Failure || gasBeneficiaryNotDestroyed) { if (notSystemTransaction) @@ -492,6 +525,8 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra substate.Logs.Any() ? substate.Logs.ToArray() : Array.Empty(), stateRoot); } } + + if (txTracer.IsTracingVerkleWitness) txTracer.SetVerkleWitnessKeys(witness.GetAccessedKeys()); } private void PrepareAccountForContractDeployment(Address contractAddress, IReleaseSpec spec) @@ -515,7 +550,7 @@ private void PrepareAccountForContractDeployment(Address contractAddress, IRelea } // we clean any existing storage (in case of a previously called self destruct) - _worldState.UpdateStorageRoot(contractAddress, Keccak.EmptyTreeHash); + if (!spec.IsVerkleTreeEipEnabled) _worldState.UpdateStorageRoot(contractAddress, Keccak.EmptyTreeHash); } } diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index e084ad78aa7..cecba1e89f1 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -128,8 +128,13 @@ public TransactionSubstate Run(EvmState state, IWorldState worldState, ITxTracer { if (_txTracer.IsTracingActions && !currentState.IsContinuation) { - _txTracer.ReportAction(currentState.GasAvailable, currentState.Env.Value, currentState.From, currentState.To, currentState.ExecutionType.IsAnyCreate() ? currentState.Env.CodeInfo.MachineCode : currentState.Env.InputData, currentState.ExecutionType); - if (_txTracer.IsTracingCode) _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode); + _txTracer.ReportAction(currentState.GasAvailable, currentState.Env.Value, currentState.From, + currentState.To, + currentState.ExecutionType.IsAnyCreate() + ? currentState.Env.CodeInfo.MachineCode.ToBytes() + : currentState.Env.InputData, currentState.ExecutionType); + if (_txTracer.IsTracingCode) + _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode.ToBytes()); } callResult = ExecuteCall(currentState, previousCallResult, previousCallOutput, previousCallOutputDestination, spec); @@ -387,15 +392,24 @@ public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IR CodeInfo cachedCodeInfo = _codeCache.Get(codeHash); if (cachedCodeInfo is null) { - byte[] code = worldState.GetCode(codeHash); - - if (code is null) + try { - throw new NullReferenceException($"Code {codeHash} missing in the state for address {codeSource}"); + byte[] code = worldState.GetCode(codeHash); + cachedCodeInfo = new CodeInfo(code); + _codeCache.Set(codeHash, cachedCodeInfo); + } + catch (InvalidOperationException) + { + // this means that code is not there + if (worldState.StateType == StateType.Verkle) + { + cachedCodeInfo = new CodeInfo(worldState, codeSource); + } + else + { + throw new NullReferenceException($"Code {codeHash} missing in the state for address {codeSource}"); + } } - - cachedCodeInfo = new CodeInfo(code); - _codeCache.Set(codeHash, cachedCodeInfo); } else { @@ -458,27 +472,72 @@ private static void UpdateGasUp(long refund, ref long gasAvailable) gasAvailable += refund; } - private bool ChargeAccountAccessGas(ref long gasAvailable, EvmState vmState, Address address, IReleaseSpec spec, bool chargeForWarm = true) + private bool ChargeAccountAccessGas(ref long gasAvailable, EvmState vmState, Address address, IReleaseSpec spec, bool chargeForWarm = true, bool valueTransfer = false, Instruction opCode = Instruction.STOP) { // Console.WriteLine($"Accessing {address}"); bool result = true; - if (spec.UseHotAndColdStorage) + if (spec.IsVerkleTreeEipEnabled) { - if (_txTracer.IsTracingAccess) // when tracing access we want cost as if it was warmed up from access list + bool isAddressPreCompile = address.IsPrecompile(spec); + switch (opCode) { - vmState.WarmUp(address); - } + case Instruction.BALANCE: + { + result = UpdateGas(vmState.VerkleTreeWitness.AccessBalance(address), ref gasAvailable); + break; + } + case Instruction.EXTCODESIZE: + case Instruction.EXTCODECOPY: + case Instruction.SELFDESTRUCT: + case Instruction.CALL: + case Instruction.CALLCODE: + case Instruction.DELEGATECALL: + case Instruction.STATICCALL: + { + if (!isAddressPreCompile) + { + result = UpdateGas(vmState.VerkleTreeWitness.AccessForCodeOpCodes(address), ref gasAvailable); + if (!result) break; + } - if (vmState.IsCold(address) && !address.IsPrecompile(spec)) - { - result = UpdateGas(GasCostOf.ColdAccountAccess, ref gasAvailable); - vmState.WarmUp(address); - } - else if (chargeForWarm) - { - result = UpdateGas(GasCostOf.WarmStateRead, ref gasAvailable); + if (valueTransfer) + { + result = UpdateGas( + vmState.VerkleTreeWitness.AccessBalance(address), + ref gasAvailable); + } + break; + } + case Instruction.EXTCODEHASH: + { + result = UpdateGas( + vmState.VerkleTreeWitness.AccessCodeHash(address), + ref gasAvailable); + break; + } + default: + { + throw new ArgumentOutOfRangeException(nameof(opCode), opCode, null); + } } + return result; + } + + if (!spec.UseHotAndColdStorage) return true; + if (_txTracer.IsTracingAccess) // when tracing access we want cost as if it was warmed up from access list + { + vmState.WarmUp(address); + } + + if (vmState.IsCold(address) && !address.IsPrecompile(spec)) + { + result = UpdateGas(GasCostOf.ColdAccountAccess, ref gasAvailable); + vmState.WarmUp(address); + } + else if (chargeForWarm) + { + result = UpdateGas(GasCostOf.WarmStateRead, ref gasAvailable); } return result; @@ -500,6 +559,12 @@ private bool ChargeStorageAccessGas( // Console.WriteLine($"Accessing {storageCell} {storageAccessType}"); bool result = true; + if (spec.IsVerkleTreeEipEnabled) + { + long storageGas = vmState.VerkleTreeWitness.AccessStorage(storageCell.Address, storageCell.Index, + storageAccessType == StorageAccessType.SSTORE); + if (!UpdateGas(storageGas, ref gasAvailable)) return false; + } if (spec.UseHotAndColdStorage) { if (_txTracer.IsTracingAccess) // when tracing access we want cost as if it was warmed up from access list @@ -596,10 +661,25 @@ private CallResult ExecuteCall(EvmState vmState, byte[]? previousCallResult, Zer ref readonly ExecutionEnvironment env = ref vmState.Env; ref readonly TxExecutionContext txCtx = ref env.TxExecutionContext; + vmState.InitStacks(); + EvmStack stack = new(vmState.DataStack.AsSpan(), vmState.DataStackHead, _txTracer); + long gasAvailable = vmState.GasAvailable; + int programCounter = vmState.ProgramCounter; + ICode code = env.CodeInfo.MachineCode; + if (!vmState.IsContinuation) { if (!_state.AccountExists(env.ExecutingAccount)) { + // TODO: this part is completely wrong - just for consensus with geth + if (spec.IsVerkleTreeEipEnabled && vmState.ExecutionType == ExecutionType.Transaction) + { + if (env.TransferValue.IsZero) + { + long gasProofOfAbsence = vmState.VerkleTreeWitness.AccessForProofOfAbsence(env.Caller); + if (!UpdateGas(gasProofOfAbsence, ref gasAvailable)) goto OutOfGas; + } + } _state.CreateAccount(env.ExecutingAccount, env.TransferValue); } else @@ -615,14 +695,15 @@ private CallResult ExecuteCall(EvmState vmState, byte[]? previousCallResult, Zer if (vmState.Env.CodeInfo.MachineCode.Length == 0) { + UpdateCurrentState(vmState, programCounter, gasAvailable, stack.Head); goto Empty; } - vmState.InitStacks(); - EvmStack stack = new(vmState.DataStack.AsSpan(), vmState.DataStackHead, _txTracer); - long gasAvailable = vmState.GasAvailable; - int programCounter = vmState.ProgramCounter; - Span code = env.CodeInfo.MachineCode.AsSpan(); + byte CalculateCodeChunkIdFromPc(int pc) + { + int chunkId = pc / 31; + return (byte)chunkId; + } static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) { @@ -651,6 +732,14 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) while (programCounter < code.Length) { + if (spec.IsVerkleTreeEipEnabled) + { + if (vmState.ExecutionType is not (ExecutionType.Create or ExecutionType.Create2)) + { + long gas = vmState.VerkleTreeWitness.AccessCodeChunk(vmState.To, CalculateCodeChunkIdFromPc(programCounter), false); + if (!UpdateGas(gas, ref gasAvailable)) goto OutOfGas; + } + } Instruction instruction = (Instruction)code[programCounter]; // Console.WriteLine(instruction); if (traceOpcodes) @@ -1140,7 +1229,7 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) if (gasCost != 0 && !UpdateGas(gasCost, ref gasAvailable)) goto OutOfGas; Address address = stack.PopAddress(); - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec)) goto OutOfGas; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec, opCode: instruction)) goto OutOfGas; UInt256 balance = _state.GetBalance(address); stack.PushUInt256(in balance); @@ -1228,6 +1317,23 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) ZeroPaddedSpan codeSlice = code.SliceWithZeroPadding(src, (int)length); vmState.Memory.Save(in dest, codeSlice); if (_txTracer.IsTracingInstructions) _txTracer.ReportMemoryChange((long)dest, codeSlice); + + if (spec.IsVerkleTreeEipEnabled) + { + // TODO: modify - add the chunk that gets jumped when PUSH32 is called. + if (src > length) + { + src = length; + } + byte startChunkId = CalculateCodeChunkIdFromPc((int)src); + byte endChunkId = CalculateCodeChunkIdFromPc((int)src + codeSlice.Length); + + for (byte ch = startChunkId; ch <= endChunkId; ch++) + { + long gas = vmState.VerkleTreeWitness.AccessCodeChunk(vmState.To, ch, false); + if (!UpdateGas(gas, ref gasAvailable)) goto OutOfGas; + } + } } break; @@ -1246,9 +1352,9 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) if (!UpdateGas(gasCost, ref gasAvailable)) goto OutOfGas; Address address = stack.PopAddress(); - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec)) goto OutOfGas; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec, opCode: instruction)) goto OutOfGas; - byte[] accountCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; + ICode accountCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; UInt256 codeSize = (UInt256)accountCode.Length; stack.PushUInt256(in codeSize); break; @@ -1264,19 +1370,34 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) if (!UpdateGas(gasCost + GasCostOf.Memory * EvmPooledMemory.Div32Ceiling(length), ref gasAvailable)) goto OutOfGas; - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec)) goto OutOfGas; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec, opCode: instruction)) goto OutOfGas; if (length > UInt256.Zero) { if (!UpdateMemoryCost(vmState, ref gasAvailable, in dest, length)) goto OutOfGas; - byte[] externalCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; + ICode externalCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; ZeroPaddedSpan callDataSlice = externalCode.SliceWithZeroPadding(src, (int)length); vmState.Memory.Save(in dest, callDataSlice); if (_txTracer.IsTracingInstructions) { _txTracer.ReportMemoryChange((long)dest, callDataSlice); } + + if (spec.IsVerkleTreeEipEnabled) + { + // TODO: modify - add the chunk that gets jumped when PUSH32 is called. + if (src > length) src = length; + + byte startChunkId = CalculateCodeChunkIdFromPc((int)src); + byte endChunkId = CalculateCodeChunkIdFromPc((int)src + callDataSlice.Length); + + for (byte ch = startChunkId; ch <= endChunkId; ch++) + { + long gas = vmState.VerkleTreeWitness.AccessCodeChunk(address, ch, false); + if (!UpdateGas(gas, ref gasAvailable)) goto OutOfGas; + } + } } break; @@ -1573,7 +1694,11 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) Span originalValue = _state.GetOriginal(storageCell); bool originalIsZero = originalValue.IsZero(); - bool currentSameAsOriginal = Bytes.AreEqual(originalValue, currentValue); + bool currentSameAsOriginal = originalValue.WithoutLeadingZeros().SequenceEqual(currentValue.WithoutLeadingZeros()); + if (originalValue.Length == 0 && currentIsZero) + { + currentSameAsOriginal = true; + } if (currentSameAsOriginal) { if (currentIsZero) @@ -1782,6 +1907,13 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) else { stack.PushByte(code[programCounterInt]); + + if (spec.IsVerkleTreeEipEnabled && programCounterInt % 31 == 0) + { + // TODO: modify - add the chunk that gets jumped when PUSH32 is called. + long gas = vmState.VerkleTreeWitness.AccessCodeChunk(vmState.To, CalculateCodeChunkIdFromPc(programCounterInt + 1), false); + if (!UpdateGas(gas, ref gasAvailable)) goto OutOfGas; + } } programCounter++; @@ -1828,6 +1960,21 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) stack.PushLeftPaddedBytes(code.Slice(programCounterInt, usedFromCode), length); programCounter += length; + + if (spec.IsVerkleTreeEipEnabled) + { + // TODO: modify - add the chunk that gets jumped when PUSH32 is called. + byte startChunkId = CalculateCodeChunkIdFromPc(programCounterInt); + int endOffset = programCounterInt + length > code.Length ? code.Length : programCounterInt + length; + // var endChunkId = CalculateChunkIdFromPc(programCounterInt + usedFromCode); + byte endChunkId = CalculateCodeChunkIdFromPc(endOffset - 1); + + for (byte ch = startChunkId; ch <= endChunkId; ch++) + { + long gas = vmState.VerkleTreeWitness.AccessCodeChunk(vmState.To, ch, false); + if (!UpdateGas(gas, ref gasAvailable)) goto OutOfGas; + } + } break; } case Instruction.DUP1: @@ -1978,6 +2125,12 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) ? ContractAddress.From(env.ExecutingAccount, _state.GetNonce(env.ExecutingAccount)) : ContractAddress.From(env.ExecutingAccount, salt, initCode); + if (spec.IsVerkleTreeEipEnabled) + { + long gasWitness = vmState.VerkleTreeWitness.AccessForContractCreationInit(contractAddress, !vmState.Env.Value.IsZero); + if (!UpdateGas(gasWitness, ref gasAvailable)) goto OutOfGas; + } + if (spec.UseHotAndColdStorage) { // EIP-2929 assumes that warm-up cost is included in the costs of CREATE and CREATE2 @@ -1998,13 +2151,10 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) break; } - if (accountExists) - { - _state.UpdateStorageRoot(contractAddress, Keccak.EmptyTreeHash); - } - else if (_state.IsDeadAccount(contractAddress)) + if (!spec.IsVerkleTreeEipEnabled) { - _state.ClearStorage(contractAddress); + if (accountExists) _state.UpdateStorageRoot(contractAddress, Keccak.EmptyTreeHash); + else if (_state.IsDeadAccount(contractAddress)) _state.ClearStorage(contractAddress); } _state.SubtractFromBalance(env.ExecutingAccount, value, spec); @@ -2033,6 +2183,12 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) false, accountExists); + if (spec.IsVerkleTreeEipEnabled) + { + long gasWitness = vmState.VerkleTreeWitness.AccessContractCreated(contractAddress); + if (!UpdateGas(gasWitness, ref gasAvailable)) goto OutOfGas; + } + UpdateCurrentState(vmState, programCounter, gasAvailable, stack.Head); return new CallResult(callState); } @@ -2062,7 +2218,7 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) Address codeSource = stack.PopAddress(); // Console.WriteLine($"CALLIN {codeSource}"); - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, codeSource, spec)) goto OutOfGas; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, codeSource, spec, opCode: instruction)) goto OutOfGas; UInt256 callValue; switch (instruction) @@ -2228,9 +2384,10 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) Metrics.SelfDestructs++; Address inheritor = stack.PopAddress(); - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, inheritor, spec, false)) goto OutOfGas; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, inheritor, spec, false, opCode: instruction)) goto OutOfGas; - vmState.DestroyList.Add(env.ExecutingAccount); + // TODO: replace this by EIP-4758 + if (!spec.IsVerkleTreeEipEnabled) vmState.DestroyList.Add(env.ExecutingAccount); UInt256 ownerBalance = _state.GetBalance(env.ExecutingAccount); if (_txTracer.IsTracingActions) _txTracer.ReportSelfDestruct(env.ExecutingAccount, ownerBalance, inheritor); @@ -2333,11 +2490,11 @@ static void UpdateCurrentState(EvmState state, int pc, long gas, int stackHead) { if (!spec.ExtCodeHashOpcodeEnabled) goto InvalidInstruction; - var gasCost = spec.GetExtCodeHashCost(); + long gasCost = spec.GetExtCodeHashCost(); if (!UpdateGas(gasCost, ref gasAvailable)) goto OutOfGas; Address address = stack.PopAddress(); - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec)) goto OutOfGas; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, address, spec, opCode: instruction)) goto OutOfGas; if (!_state.AccountExists(address) || _state.IsDeadAccount(address)) { @@ -2523,27 +2680,14 @@ private void EndInstructionTraceError(long gasAvailable, EvmExceptionType evmExc private static ExecutionType GetCallExecutionType(Instruction instruction, bool isPostMerge = false) { - ExecutionType executionType; - if (instruction == Instruction.CALL) - { - executionType = ExecutionType.Call; - } - else if (instruction == Instruction.DELEGATECALL) - { - executionType = ExecutionType.DelegateCall; - } - else if (instruction == Instruction.STATICCALL) + ExecutionType executionType = instruction switch { - executionType = ExecutionType.StaticCall; - } - else if (instruction == Instruction.CALLCODE) - { - executionType = ExecutionType.CallCode; - } - else - { - throw new NotSupportedException($"Execution type is undefined for {instruction.GetName(isPostMerge)}"); - } + Instruction.CALL => ExecutionType.Call, + Instruction.DELEGATECALL => ExecutionType.DelegateCall, + Instruction.STATICCALL => ExecutionType.StaticCall, + Instruction.CALLCODE => ExecutionType.CallCode, + _ => throw new NotSupportedException($"Execution type is undefined for {instruction.GetName(isPostMerge)}") + }; return executionType; } diff --git a/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs b/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs index a92123951f4..7e763e7da45 100644 --- a/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs +++ b/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs @@ -23,6 +23,7 @@ using NSubstitute; using NUnit.Framework; using Nethermind.Core.Extensions; +using Nethermind.Merge.Plugin.Handlers; namespace Nethermind.HealthChecks.Test { diff --git a/src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs b/src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs index 9e5ea1a3dfa..8b317263f9f 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs +++ b/src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs @@ -77,7 +77,7 @@ public bool Equals(TimeUnit? other) return false; } - if ((object)this == other) + if (ReferenceEquals(this, other)) { return true; } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs index b5b41c08b59..d72484ed998 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs @@ -33,18 +33,36 @@ public async Task Execute(CancellationToken _) protected virtual async Task BuildProducer() { - _api.BlockProducerEnvFactory = new BlockProducerEnvFactory(_api.DbProvider!, - _api.BlockTree!, - _api.ReadOnlyTrieStore!, - _api.SpecProvider!, - _api.BlockValidator!, - _api.RewardCalculatorSource!, - _api.ReceiptStorage!, - _api.BlockPreprocessor, - _api.TxPool!, - _api.TransactionComparerProvider!, - _api.Config(), - _api.LogManager); + if (_api.SpecProvider != null && _api.SpecProvider.GenesisSpec.IsVerkleTreeEipEnabled) + { + _api.BlockProducerEnvFactory = new BlockProducerEnvFactory(_api.DbProvider!, + _api.BlockTree!, + _api.ReadOnlyVerkleTrieStore!, + _api.SpecProvider!, + _api.BlockValidator!, + _api.RewardCalculatorSource!, + _api.ReceiptStorage!, + _api.BlockPreprocessor, + _api.TxPool!, + _api.TransactionComparerProvider!, + _api.Config(), + _api.LogManager); + } + else + { + _api.BlockProducerEnvFactory = new BlockProducerEnvFactory(_api.DbProvider!, + _api.BlockTree!, + _api.ReadOnlyTrieStore!, + _api.SpecProvider!, + _api.BlockValidator!, + _api.RewardCalculatorSource!, + _api.ReceiptStorage!, + _api.BlockPreprocessor, + _api.TxPool!, + _api.TransactionComparerProvider!, + _api.Config(), + _api.LogManager); + } if (_api.ChainSpec is null) throw new StepDependencyException(nameof(_api.ChainSpec)); IConsensusPlugin? consensusPlugin = _api.GetConsensusPlugin(); diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index 23a17d39cca..566ce6ea068 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -38,6 +38,8 @@ using Nethermind.Trie; using Nethermind.Trie.Pruning; using Nethermind.TxPool; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.VerkleDb; using Nethermind.Wallet; namespace Nethermind.Init.Steps @@ -106,56 +108,71 @@ private Task InitBlockchain() .WitnessedBy(witnessCollector); TrieStore trieStore; + IWorldState worldState; + IStateReader stateReader; IKeyValueStoreWithBatching stateWitnessedBy = setApi.MainStateDbWithCache.WitnessedBy(witnessCollector); - if (pruningConfig.Mode.IsMemory()) + if (!getApi.SpecProvider.GenesisSpec.IsVerkleTreeEipEnabled) { - IPersistenceStrategy persistenceStrategy = Persist.IfBlockOlderThan(pruningConfig.PersistenceInterval); // TODO: this should be based on time - if (pruningConfig.Mode.IsFull()) + if (pruningConfig.Mode.IsMemory()) { - PruningTriggerPersistenceStrategy triggerPersistenceStrategy = new((IFullPruningDb)getApi.DbProvider!.StateDb, getApi.BlockTree!, getApi.LogManager); - getApi.DisposeStack.Push(triggerPersistenceStrategy); - persistenceStrategy = persistenceStrategy.Or(triggerPersistenceStrategy); - } + IPersistenceStrategy persistenceStrategy = Persist.IfBlockOlderThan(pruningConfig.PersistenceInterval); // TODO: this should be based on time + if (pruningConfig.Mode.IsFull()) + { + PruningTriggerPersistenceStrategy triggerPersistenceStrategy = new((IFullPruningDb)getApi.DbProvider!.StateDb, getApi.BlockTree!, getApi.LogManager); + getApi.DisposeStack.Push(triggerPersistenceStrategy); + persistenceStrategy = persistenceStrategy.Or(triggerPersistenceStrategy); + } - setApi.TrieStore = trieStore = new TrieStore( - stateWitnessedBy, - Prune.WhenCacheReaches(pruningConfig.CacheMb.MB()), // TODO: memory hint should define this - persistenceStrategy, - getApi.LogManager); + setApi.TrieStore = trieStore = new TrieStore( + stateWitnessedBy, + Prune.WhenCacheReaches(pruningConfig.CacheMb.MB()), // TODO: memory hint should define this + persistenceStrategy, + getApi.LogManager); - if (pruningConfig.Mode.IsFull()) - { - IFullPruningDb fullPruningDb = (IFullPruningDb)getApi.DbProvider!.StateDb; - fullPruningDb.PruningStarted += (_, args) => + if (pruningConfig.Mode.IsFull()) { - cachedStateDb.PersistCache(args.Context); - trieStore.PersistCache(args.Context, args.Context.CancellationTokenSource.Token); - }; + IFullPruningDb fullPruningDb = (IFullPruningDb)getApi.DbProvider!.StateDb; + fullPruningDb.PruningStarted += (_, args) => + { + cachedStateDb.PersistCache(args.Context); + trieStore.PersistCache(args.Context, args.Context.CancellationTokenSource.Token); + }; + } + } + else + { + setApi.TrieStore = trieStore = new TrieStore( + stateWitnessedBy, + No.Pruning, + Persist.EveryBlock, + getApi.LogManager); } - } - else - { - setApi.TrieStore = trieStore = new TrieStore( - stateWitnessedBy, - No.Pruning, - Persist.EveryBlock, - getApi.LogManager); - } - TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(trieStore, _api.BlockTree!, _api.LogManager); - getApi.DisposeStack.Push(trieStoreBoundaryWatcher); - getApi.DisposeStack.Push(trieStore); + TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(trieStore, _api.BlockTree!, _api.LogManager); + getApi.DisposeStack.Push(trieStoreBoundaryWatcher); + getApi.DisposeStack.Push(trieStore); - ITrieStore readOnlyTrieStore = setApi.ReadOnlyTrieStore = trieStore.AsReadOnly(cachedStateDb); + ITrieStore readOnlyTrieStore = setApi.ReadOnlyTrieStore = trieStore.AsReadOnly(cachedStateDb); - IWorldState worldState = setApi.WorldState = new WorldState( - trieStore, - codeDb, - getApi.LogManager); + worldState = setApi.WorldState = new WorldState( + trieStore, + codeDb, + getApi.LogManager); - ReadOnlyDbProvider readOnly = new(getApi.DbProvider, false); + ReadOnlyDbProvider readOnly = new(getApi.DbProvider, false); - IStateReader stateReader = setApi.StateReader = new StateReader(readOnlyTrieStore, readOnly.GetDb(DbNames.Code), getApi.LogManager); + stateReader = setApi.StateReader = new StateReader(readOnlyTrieStore, readOnly.GetDb(DbNames.Code), getApi.LogManager); + + } + else + { + VerkleStateStore verkleStateStore = setApi.VerkleTrieStore = new VerkleStateStore(getApi.DbProvider, getApi.LogManager); + TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(verkleStateStore, _api.BlockTree!, _api.LogManager); + getApi.DisposeStack.Push(trieStoreBoundaryWatcher); + ReadOnlyVerkleStateStore readOnlyVerkleStateStore = setApi.ReadOnlyVerkleTrieStore = verkleStateStore.AsReadOnly(new VerkleMemoryDb()); + worldState = setApi.WorldState = new VerkleWorldState(new VerkleStateTree(verkleStateStore, getApi.LogManager), codeDb, getApi.LogManager); + stateReader = setApi.StateReader = new VerkleStateReader(new VerkleStateTree(readOnlyVerkleStateStore, getApi.LogManager), codeDb, getApi.LogManager); + } setApi.TransactionComparerProvider = new TransactionComparerProvider(getApi.SpecProvider!, getApi.BlockTree.AsReadOnly()); setApi.ChainHeadStateProvider = new ChainHeadReadOnlyStateProvider(getApi.BlockTree, stateReader); @@ -170,11 +187,21 @@ private Task InitBlockchain() try { _logger!.Info("Collecting trie stats and verifying that no nodes are missing..."); - TrieStore noPruningStore = new(stateWitnessedBy, No.Pruning, Persist.EveryBlock, getApi.LogManager); - IWorldState diagStateProvider = new WorldState(noPruningStore, codeDb, getApi.LogManager) + IWorldState diagStateProvider; + if (getApi.SpecProvider.GenesisSpec.IsVerkleTreeEipEnabled) { - StateRoot = getApi.BlockTree!.Head?.StateRoot ?? Keccak.EmptyTreeHash - }; + VerkleStateStore verkleStateStore = setApi.VerkleTrieStore = new VerkleStateStore(getApi.DbProvider, getApi.LogManager); + diagStateProvider = new VerkleWorldState(new VerkleStateTree(verkleStateStore, getApi.LogManager), codeDb, getApi.LogManager); + } + else + { + TrieStore noPruningStore = new TrieStore(stateWitnessedBy, No.Pruning, Persist.EveryBlock, getApi.LogManager); + diagStateProvider = new WorldState(noPruningStore, codeDb, getApi.LogManager) + { + StateRoot = getApi.BlockTree!.Head?.StateRoot ?? Keccak.EmptyTreeHash + }; + } + TrieStats stats = diagStateProvider.CollectStats(getApi.DbProvider.CodeDb, _api.LogManager); _logger.Info($"Starting from {getApi.BlockTree.Head?.Number} {getApi.BlockTree.Head?.StateRoot}{Environment.NewLine}" + stats); } @@ -349,7 +376,7 @@ protected virtual BlockProcessor CreateBlockProcessor() if (_api.RewardCalculatorSource is null) throw new StepDependencyException(nameof(_api.RewardCalculatorSource)); if (_api.TransactionProcessor is null) throw new StepDependencyException(nameof(_api.TransactionProcessor)); - return new BlockProcessor( + BlockProcessor processor = new BlockProcessor( _api.SpecProvider, _api.BlockValidator, _api.RewardCalculatorSource.Get(_api.TransactionProcessor!), @@ -357,7 +384,13 @@ protected virtual BlockProcessor CreateBlockProcessor() _api.WorldState, _api.ReceiptStorage, _api.WitnessCollector, - _api.LogManager); + _api.LogManager) + { + StatelessBlockTransactionsExecutor = new BlockProcessor.BlockStatelessValidationTransactionsExecutor(_api.TransactionProcessor, + _api.WorldState!) + }; + + return processor; } // TODO: remove from here - move to consensus? diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 56d41cdf365..2194527d5af 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -39,6 +39,7 @@ using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Init.Steps; @@ -100,16 +101,17 @@ private async Task Initialize(CancellationToken cancellationToken) CanonicalHashTrie cht = new CanonicalHashTrie(_api.DbProvider!.ChtDb); - ProgressTracker progressTracker = new(_api.BlockTree!, _api.DbProvider.StateDb, _api.LogManager, _syncConfig.SnapSyncAccountRangePartitionCount); - _api.SnapProvider = new SnapProvider(progressTracker, _api.DbProvider, _api.LogManager); + SnapProgressTracker snapProgressTracker = new(_api.BlockTree!, _api.DbProvider.StateDb, _api.LogManager, _syncConfig.SnapSyncAccountRangePartitionCount); + _api.SnapProvider = new SnapProvider(snapProgressTracker, _api.DbProvider, _api.LogManager); - SyncProgressResolver syncProgressResolver = new( - _api.BlockTree!, - _api.ReceiptStorage!, - _api.ReadOnlyTrieStore!, - progressTracker, - _syncConfig, - _api.LogManager); + VerkleProgressTracker verkleProgressTracker = new(_api.BlockTree!, _api.DbProvider.StateDb, _api.LogManager, _syncConfig.VerkleSyncAccountRangePartitionCount); + _api.VerkleProvider = new VerkleSyncProvider(verkleProgressTracker, _api.DbProvider, _api.LogManager); + + ISyncProgressResolver syncProgressResolver = _api.SpecProvider!.GenesisSpec.IsVerkleTreeEipEnabled switch + { + true => new SyncProgressResolver(_api.BlockTree!, _api.ReceiptStorage!, _api.ReadOnlyVerkleTrieStore!, verkleProgressTracker, _syncConfig, _api.LogManager), + false => new SyncProgressResolver(_api.BlockTree!, _api.ReceiptStorage!, _api.ReadOnlyTrieStore!, snapProgressTracker, _syncConfig, _api.LogManager) + }; _api.SyncProgressResolver = syncProgressResolver; _api.BetterPeerStrategy = new TotalDifficultyBetterPeerStrategy(_api.LogManager); @@ -157,6 +159,7 @@ private async Task Initialize(CancellationToken cancellationToken) _api.SyncModeSelector, _syncConfig, _api.SnapProvider, + _api.VerkleProvider, _api.BlockDownloaderFactory, _api.Pivot, syncReport, @@ -267,7 +270,7 @@ await StartDiscovery().ContinueWith(initDiscoveryTask => ThisNodeInfo.AddInfo("Node address :", $"{_api.Enode.Address} (do not use as an account)"); } - protected virtual MultiSyncModeSelector CreateMultiSyncModeSelector(SyncProgressResolver syncProgressResolver) + protected virtual MultiSyncModeSelector CreateMultiSyncModeSelector(ISyncProgressResolver syncProgressResolver) => new(syncProgressResolver, _api.SyncPeerPool!, _syncConfig, No.BeaconSync, _api.BetterPeerStrategy!, _api.LogManager, _api.ChainSpec?.SealEngineType == SealEngineType.Clique); private Task StartDiscovery() diff --git a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs index 9ee8920ae69..967f47f25a5 100644 --- a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs +++ b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs @@ -92,7 +92,7 @@ public virtual async Task Execute(CancellationToken cancellationToken) if (_api.ReceiptStorage is null) throw new StepDependencyException(nameof(_api.ReceiptStorage)); if (_api.GasPriceOracle is null) throw new StepDependencyException(nameof(_api.GasPriceOracle)); if (_api.EthSyncingInfo is null) throw new StepDependencyException(nameof(_api.EthSyncingInfo)); - if (_api.ReadOnlyTrieStore is null) throw new StepDependencyException(nameof(_api.ReadOnlyTrieStore)); + if (_api.ReadOnlyTrieStore is null && _api.ReadOnlyVerkleTrieStore is null) throw new StepDependencyException(nameof(_api.ReadOnlyTrieStore)); EthModuleFactory ethModuleFactory = new( _api.TxPool, @@ -118,36 +118,27 @@ public virtual async Task Execute(CancellationToken cancellationToken) if (_api.PeerPool is null) throw new StepDependencyException(nameof(_api.PeerPool)); if (_api.WitnessRepository is null) throw new StepDependencyException(nameof(_api.WitnessRepository)); - ProofModuleFactory proofModuleFactory = new(_api.DbProvider, _api.BlockTree, _api.ReadOnlyTrieStore, _api.BlockPreprocessor, _api.ReceiptFinder, _api.SpecProvider, _api.LogManager); + ProofModuleFactory proofModuleFactory = _api.SpecProvider.GenesisSpec.IsVerkleTreeEipEnabled switch + { + true => new ProofModuleFactory(_api.DbProvider, _api.BlockTree, _api.ReadOnlyVerkleTrieStore!, _api.BlockPreprocessor, _api.ReceiptFinder, _api.SpecProvider, _api.LogManager), + false => new ProofModuleFactory(_api.DbProvider, _api.BlockTree, _api.ReadOnlyTrieStore!, _api.BlockPreprocessor, _api.ReceiptFinder, _api.SpecProvider, _api.LogManager), + }; rpcModuleProvider.RegisterBounded(proofModuleFactory, 2, rpcConfig.Timeout); - DebugModuleFactory debugModuleFactory = new( - _api.DbProvider, - _api.BlockTree, - rpcConfig, - _api.BlockValidator, - _api.BlockPreprocessor, - _api.RewardCalculatorSource, - _api.ReceiptStorage, - new ReceiptMigration(_api), - _api.ReadOnlyTrieStore, - _api.ConfigProvider, - _api.SpecProvider, - _api.SyncModeSelector, - _api.LogManager); + DebugModuleFactory debugModuleFactory = _api.SpecProvider.GenesisSpec.IsVerkleTreeEipEnabled switch + { + true => new DebugModuleFactory(_api.DbProvider, _api.BlockTree, rpcConfig, _api.BlockValidator, _api.BlockPreprocessor, _api.RewardCalculatorSource, _api.ReceiptStorage, new ReceiptMigration(_api), _api.ReadOnlyVerkleTrieStore!, + _api.ConfigProvider, _api.SpecProvider, _api.SyncModeSelector, _api.LogManager), + false => new DebugModuleFactory(_api.DbProvider, _api.BlockTree, rpcConfig, _api.BlockValidator, _api.BlockPreprocessor, _api.RewardCalculatorSource, _api.ReceiptStorage, new ReceiptMigration(_api), _api.ReadOnlyTrieStore!, + _api.ConfigProvider, _api.SpecProvider, _api.SyncModeSelector, _api.LogManager), + }; rpcModuleProvider.RegisterBoundedByCpuCount(debugModuleFactory, rpcConfig.Timeout); - TraceModuleFactory traceModuleFactory = new( - _api.DbProvider, - _api.BlockTree, - _api.ReadOnlyTrieStore, - rpcConfig, - _api.BlockPreprocessor, - _api.RewardCalculatorSource, - _api.ReceiptStorage, - _api.SpecProvider, - _api.PoSSwitcher, - _api.LogManager); + TraceModuleFactory traceModuleFactory = _api.SpecProvider.GenesisSpec.IsVerkleTreeEipEnabled switch + { + true => new TraceModuleFactory(_api.DbProvider, _api.BlockTree, _api.ReadOnlyVerkleTrieStore!, rpcConfig, _api.BlockPreprocessor, _api.RewardCalculatorSource, _api.ReceiptStorage, _api.SpecProvider, _api.PoSSwitcher, _api.LogManager), + false => new TraceModuleFactory(_api.DbProvider, _api.BlockTree, _api.ReadOnlyTrieStore!, rpcConfig, _api.BlockPreprocessor, _api.RewardCalculatorSource, _api.ReceiptStorage, _api.SpecProvider, _api.PoSSwitcher, _api.LogManager), + }; rpcModuleProvider.RegisterBoundedByCpuCount(traceModuleFactory, rpcConfig.Timeout); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs index bab7705398e..868d4eb1c31 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs @@ -28,14 +28,19 @@ using System.Threading.Tasks; using Nethermind.Consensus.Processing; using Nethermind.State.Tracing; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.VerkleDb; using NSubstitute; namespace Nethermind.JsonRpc.Test.Modules.Proof { [Parallelizable(ParallelScope.None)] - [TestFixture(true, true)] - [TestFixture(true, false)] - [TestFixture(false, false)] + // [TestFixture(true, true, TreeType.VerkleTree)] + // [TestFixture(true, false, TreeType.VerkleTree)] + // [TestFixture(false, false, TreeType.VerkleTree)] + [TestFixture(true, true, StateType.Merkle)] + [TestFixture(true, false, StateType.Merkle)] + [TestFixture(false, false, StateType.Merkle)] public class ProofRpcModuleTests { private readonly bool _createSystemAccount; @@ -44,11 +49,13 @@ public class ProofRpcModuleTests private IBlockTree _blockTree; private IDbProvider _dbProvider; private TestSpecProvider _specProvider; + private StateType _stateType; - public ProofRpcModuleTests(bool createSystemAccount, bool useNonZeroGasPrice) + public ProofRpcModuleTests(bool createSystemAccount, bool useNonZeroGasPrice, StateType stateType) { _createSystemAccount = createSystemAccount; _useNonZeroGasPrice = useNonZeroGasPrice; + _stateType = stateType; } [SetUp] @@ -58,15 +65,14 @@ public async Task Setup() _specProvider = new TestSpecProvider(London.Instance); _blockTree = Build.A.BlockTree(_specProvider).WithTransactions(receiptStorage).OfChainLength(10).TestObject; _dbProvider = await TestMemDbProvider.InitAsync(); - - ProofModuleFactory moduleFactory = new( - _dbProvider, - _blockTree, - new TrieStore(_dbProvider.StateDb, LimboLogs.Instance).AsReadOnly(), - new CompositeBlockPreprocessorStep(new RecoverSignatures(new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance), NullTxPool.Instance, _specProvider, LimboLogs.Instance)), - receiptStorage, - _specProvider, - LimboLogs.Instance); + ProofModuleFactory moduleFactory = _stateType switch + { + StateType.Merkle => new(_dbProvider, _blockTree, new TrieStore(_dbProvider.StateDb, LimboLogs.Instance).AsReadOnly(), + new CompositeBlockPreprocessorStep(new RecoverSignatures(new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance), NullTxPool.Instance, _specProvider, LimboLogs.Instance)), receiptStorage, _specProvider, LimboLogs.Instance), + StateType.Verkle => new(_dbProvider, _blockTree, new VerkleStateStore(_dbProvider, LimboLogs.Instance).AsReadOnly(new VerkleMemoryDb()), + new CompositeBlockPreprocessorStep(new RecoverSignatures(new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance), NullTxPool.Instance, _specProvider, LimboLogs.Instance)), receiptStorage, _specProvider, LimboLogs.Instance), + _ => throw new ArgumentOutOfRangeException() + }; _proofRpcModule = moduleFactory.Create(); } @@ -77,7 +83,7 @@ public void Can_get_transaction(bool withHeader) { Keccak txHash = _blockTree.FindBlock(1).Transactions[0].Hash; TransactionWithProof txWithProof = _proofRpcModule.proof_getTransactionByHash(txHash, withHeader).Data; - Assert.NotNull(txWithProof.Transaction); + Assert.That(txWithProof.Transaction, Is.Not.Null); Assert.That(txWithProof.TxProof.Length, Is.EqualTo(2)); if (withHeader) { @@ -204,14 +210,14 @@ public void Get_receipt_when_block_has_few_receipts(bool withHeader, string expe _receiptFinder.Get(Arg.Any()).Returns(receipts); _receiptFinder.FindBlockHash(Arg.Any()).Returns(_blockTree.FindBlock(1).Hash); - ProofModuleFactory moduleFactory = new ProofModuleFactory( - _dbProvider, - _blockTree, - new TrieStore(_dbProvider.StateDb, LimboLogs.Instance).AsReadOnly(), - new CompositeBlockPreprocessorStep(new RecoverSignatures(new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance), NullTxPool.Instance, _specProvider, LimboLogs.Instance)), - _receiptFinder, - _specProvider, - LimboLogs.Instance); + ProofModuleFactory moduleFactory = _stateType switch + { + StateType.Merkle => new(_dbProvider, _blockTree, new TrieStore(_dbProvider.StateDb, LimboLogs.Instance).AsReadOnly(), + new CompositeBlockPreprocessorStep(new RecoverSignatures(new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance), NullTxPool.Instance, _specProvider, LimboLogs.Instance)), _receiptFinder, _specProvider, LimboLogs.Instance), + StateType.Verkle => new(_dbProvider, _blockTree, new VerkleStateStore(_dbProvider, LimboLogs.Instance).AsReadOnly(new VerkleMemoryDb()), + new CompositeBlockPreprocessorStep(new RecoverSignatures(new EthereumEcdsa(TestBlockchainIds.ChainId, LimboLogs.Instance), NullTxPool.Instance, _specProvider, LimboLogs.Instance)), _receiptFinder, _specProvider, LimboLogs.Instance), + _ => throw new ArgumentOutOfRangeException() + }; _proofRpcModule = moduleFactory.Create(); ReceiptWithProof receiptWithProof = _proofRpcModule.proof_getTransactionReceipt(txHash, withHeader).Data; @@ -232,7 +238,7 @@ public void Get_receipt_when_block_has_few_receipts(bool withHeader, string expe [Test] public void Can_call() { - WorldState stateProvider = CreateInitialState(null); + IWorldState stateProvider = CreateInitialState(null); Keccak root = stateProvider.StateRoot; Block block = Build.A.Block.WithParent(_blockTree.Head).WithStateRoot(root).TestObject; @@ -257,7 +263,7 @@ public void Can_call() [Test] public void Can_call_by_hash() { - WorldState stateProvider = CreateInitialState(null); + IWorldState stateProvider = CreateInitialState(null); Keccak root = stateProvider.StateRoot; Block block = Build.A.Block.WithParent(_blockTree.Head).WithStateRoot(root).TestObject; @@ -759,7 +765,7 @@ public void Can_call_with_mix_of_everything_and_storage_from_another_account_wro private CallResultWithProof TestCallWithCode(byte[] code, Address? from = null) { - WorldState stateProvider = CreateInitialState(code); + IWorldState stateProvider = CreateInitialState(code); Keccak root = stateProvider.StateRoot; Block block = Build.A.Block.WithParent(_blockTree.Head!).WithStateRoot(root).WithBeneficiary(TestItem.AddressD).TestObject; @@ -797,7 +803,7 @@ private CallResultWithProof TestCallWithCode(byte[] code, Address? from = null) private void TestCallWithStorageAndCode(byte[] code, UInt256 gasPrice, Address? from = null) { - WorldState stateProvider = CreateInitialState(code); + IWorldState stateProvider = CreateInitialState(code); for (int i = 0; i < 10000; i++) { @@ -868,9 +874,9 @@ private void TestCallWithStorageAndCode(byte[] code, UInt256 gasPrice, Address? Assert.True(response.Contains("\"result\"")); } - private WorldState CreateInitialState(byte[] code) + private IWorldState CreateInitialState(byte[] code) { - WorldState stateProvider = new(new TrieStore(_dbProvider.StateDb, LimboLogs.Instance), _dbProvider.CodeDb, LimboLogs.Instance); + IWorldState stateProvider = new WorldState(new TrieStore(_dbProvider.StateDb, LimboLogs.Instance), _dbProvider.CodeDb, LimboLogs.Instance); AddAccount(stateProvider, TestItem.AddressA, 1.Ether()); AddAccount(stateProvider, TestItem.AddressB, 1.Ether()); @@ -889,13 +895,13 @@ private WorldState CreateInitialState(byte[] code) return stateProvider; } - private void AddAccount(WorldState stateProvider, Address account, UInt256 initialBalance) + private static void AddAccount(IWorldState stateProvider, Address account, UInt256 initialBalance) { stateProvider.CreateAccount(account, initialBalance); stateProvider.Commit(MuirGlacier.Instance, NullStateTracer.Instance); } - private void AddCode(WorldState stateProvider, Address account, byte[] code) + private static void AddCode(IWorldState stateProvider, Address account, byte[] code) { stateProvider.InsertCode(account, code, MuirGlacier.Instance); stateProvider.Commit(MainnetSpecProvider.Instance.GenesisSpec, NullStateTracer.Instance); diff --git a/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs b/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs index de57492245c..eb692693fe2 100644 --- a/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs +++ b/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs @@ -48,7 +48,7 @@ public interface IJsonRpcConfig : IConfig [ConfigItem( Description = "Defines which RPC modules should be enabled. Built in modules are: Admin, Clique, Consensus, Db, Debug, Deposit, Erc20, Eth, Evm, Health Mev, NdmConsumer, NdmProvider, Net, Nft, Parity, Personal, Proof, Subscribe, Trace, TxPool, Vault, Web3.", - DefaultValue = "[Eth, Subscribe, Trace, TxPool, Web3, Personal, Proof, Net, Parity, Health, Rpc]")] + DefaultValue = "[Eth, Subscribe, Trace, TxPool, Web3, Personal, Proof, Net, Parity, Health, Rpc, Evm]")] string[] EnabledModules { get; set; } [ConfigItem( diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs index 552fd207632..5c8930f3d77 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs @@ -14,8 +14,10 @@ using Nethermind.Db; using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; +using Nethermind.State; using Nethermind.Synchronization.ParallelSync; using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; using Newtonsoft.Json; namespace Nethermind.JsonRpc.Modules.DebugModule @@ -28,6 +30,7 @@ public class DebugModuleFactory : ModuleFactoryBase private readonly IReceiptStorage _receiptStorage; private readonly IReceiptsMigration _receiptsMigration; private readonly IReadOnlyTrieStore _trieStore; + private readonly ReadOnlyVerkleStateStore _verkleTrieStore; private readonly IConfigProvider _configProvider; private readonly ISpecProvider _specProvider; private readonly ILogManager _logManager; @@ -36,6 +39,7 @@ public class DebugModuleFactory : ModuleFactoryBase private readonly IReadOnlyBlockTree _blockTree; private readonly ISyncModeSelector _syncModeSelector; private ILogger _logger; + protected readonly StateType _stateType; public DebugModuleFactory( IDbProvider dbProvider, @@ -66,16 +70,50 @@ public DebugModuleFactory( _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); _syncModeSelector = syncModeSelector ?? throw new ArgumentNullException(nameof(syncModeSelector)); _logger = logManager.GetClassLogger(); + _stateType = StateType.Merkle; + } + + public DebugModuleFactory( + IDbProvider dbProvider, + IBlockTree blockTree, + IJsonRpcConfig jsonRpcConfig, + IBlockValidator blockValidator, + IBlockPreprocessorStep recoveryStep, + IRewardCalculatorSource rewardCalculator, + IReceiptStorage receiptStorage, + IReceiptsMigration receiptsMigration, + ReadOnlyVerkleStateStore trieStore, + IConfigProvider configProvider, + ISpecProvider specProvider, + ISyncModeSelector syncModeSelector, + ILogManager logManager) + { + _dbProvider = dbProvider.AsReadOnly(false); + _blockTree = blockTree.AsReadOnly(); + _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); + _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); + _recoveryStep = recoveryStep ?? throw new ArgumentNullException(nameof(recoveryStep)); + _rewardCalculatorSource = rewardCalculator ?? throw new ArgumentNullException(nameof(rewardCalculator)); + _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); + _receiptsMigration = receiptsMigration ?? throw new ArgumentNullException(nameof(receiptsMigration)); + _verkleTrieStore = (trieStore ?? throw new ArgumentNullException(nameof(trieStore))); + _configProvider = configProvider ?? throw new ArgumentNullException(nameof(configProvider)); + _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + _syncModeSelector = syncModeSelector ?? throw new ArgumentNullException(nameof(syncModeSelector)); + _logger = logManager.GetClassLogger(); + _stateType = StateType.Verkle; } public override IDebugRpcModule Create() { - ReadOnlyTxProcessingEnv txEnv = new( - _dbProvider, - _trieStore, - _blockTree, - _specProvider, - _logManager); + + ReadOnlyTxProcessingEnv txEnv = _stateType switch + { + StateType.Merkle => new ReadOnlyTxProcessingEnv(_dbProvider, _trieStore, _blockTree, _specProvider, _logManager), + StateType.Verkle => new ReadOnlyTxProcessingEnv(_dbProvider, _verkleTrieStore, _blockTree, _specProvider, _logManager), + _ => throw new ArgumentOutOfRangeException() + }; ChangeableTransactionProcessorAdapter transactionProcessorAdapter = new(txEnv.TransactionProcessor); BlockProcessor.BlockValidationTransactionsExecutor transactionsExecutor = new(transactionProcessorAdapter, txEnv.StateProvider); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs index 23f704e44dc..6c79cac5f8d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using Nethermind.Blockchain; -using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Rewards; @@ -14,7 +13,9 @@ using Nethermind.Db; using Nethermind.JsonRpc.Data; using Nethermind.Logging; +using Nethermind.State; using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; using Newtonsoft.Json; namespace Nethermind.JsonRpc.Modules.Proof @@ -28,6 +29,8 @@ public class ProofModuleFactory : ModuleFactoryBase private readonly IReadOnlyBlockTree _blockTree; private readonly ReadOnlyDbProvider _dbProvider; private readonly IReadOnlyTrieStore _trieStore; + private readonly ReadOnlyVerkleStateStore _verkleTrieStore; + protected readonly StateType _stateType; public ProofModuleFactory( IDbProvider dbProvider, @@ -45,19 +48,41 @@ public ProofModuleFactory( _dbProvider = dbProvider.AsReadOnly(false); _blockTree = blockTree.AsReadOnly(); _trieStore = trieStore; + _stateType = StateType.Merkle; + } + + public ProofModuleFactory( + IDbProvider dbProvider, + IBlockTree blockTree, + ReadOnlyVerkleStateStore trieStore, + IBlockPreprocessorStep recoveryStep, + IReceiptFinder receiptFinder, + ISpecProvider specProvider, + ILogManager logManager) + { + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + _recoveryStep = recoveryStep ?? throw new ArgumentNullException(nameof(recoveryStep)); + _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); + _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _dbProvider = dbProvider.AsReadOnly(false); + _blockTree = blockTree.AsReadOnly(); + _verkleTrieStore = trieStore; + _stateType = StateType.Verkle; } public override IProofRpcModule Create() { - ReadOnlyTxProcessingEnv txProcessingEnv = new( - _dbProvider, _trieStore, _blockTree, _specProvider, _logManager); + ReadOnlyTxProcessingEnv txProcessingEnv = _stateType switch + { + StateType.Merkle => new ReadOnlyTxProcessingEnv(_dbProvider, _trieStore, _blockTree, _specProvider, _logManager), + StateType.Verkle => new ReadOnlyTxProcessingEnv(_dbProvider, _verkleTrieStore, _blockTree, _specProvider, _logManager), + _ => throw new ArgumentOutOfRangeException() + }; ReadOnlyChainProcessingEnv chainProcessingEnv = new( txProcessingEnv, Always.Valid, _recoveryStep, NoBlockRewards.Instance, new InMemoryReceiptStorage(), _dbProvider, _specProvider, _logManager); - Tracer tracer = new( - txProcessingEnv.StateProvider, - chainProcessingEnv.ChainProcessor); + Tracer tracer = new Tracer(txProcessingEnv.StateProvider, chainProcessingEnv.ChainProcessor); return new ProofRpcModule(tracer, _blockTree, _receiptFinder, _specProvider, _logManager); } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs index 05427d8a01b..41e7d2daa80 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs @@ -148,7 +148,7 @@ public ResultWrapper proof_getTransactionReceipt(Keccak txHash Block block = searchResult.Object; TxReceipt receipt = _receiptFinder.Get(block).ForTransaction(txHash); - BlockReceiptsTracer receiptsTracer = new(); + BlockReceiptsTracer receiptsTracer = new(true, false); receiptsTracer.SetOtherTracer(NullBlockTracer.Instance); _tracer.Trace(block, receiptsTracer); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs index ab664333549..6be31e35820 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs @@ -16,7 +16,9 @@ using Nethermind.Evm.TransactionProcessing; using Nethermind.JsonRpc.Data; using Nethermind.Logging; +using Nethermind.State; using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; using Newtonsoft.Json; namespace Nethermind.JsonRpc.Modules.Trace @@ -26,6 +28,7 @@ public class TraceModuleFactory : ModuleFactoryBase private readonly ReadOnlyDbProvider _dbProvider; private readonly IReadOnlyBlockTree _blockTree; private readonly IReadOnlyTrieStore _trieNodeResolver; + private readonly ReadOnlyVerkleStateStore _verkleTrieStore; private readonly IJsonRpcConfig _jsonRpcConfig; private readonly IReceiptStorage _receiptStorage; private readonly ISpecProvider _specProvider; @@ -33,6 +36,7 @@ public class TraceModuleFactory : ModuleFactoryBase private readonly IBlockPreprocessorStep _recoveryStep; private readonly IRewardCalculatorSource _rewardCalculatorSource; private readonly IPoSSwitcher _poSSwitcher; + protected readonly StateType _stateType; public TraceModuleFactory( IDbProvider dbProvider, @@ -57,12 +61,43 @@ public TraceModuleFactory( _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); logManager.GetClassLogger(); + _stateType = StateType.Merkle; + } + + public TraceModuleFactory( + IDbProvider dbProvider, + IBlockTree blockTree, + ReadOnlyVerkleStateStore trieNodeResolver, + IJsonRpcConfig jsonRpcConfig, + IBlockPreprocessorStep recoveryStep, + IRewardCalculatorSource rewardCalculatorSource, + IReceiptStorage receiptFinder, + ISpecProvider specProvider, + IPoSSwitcher poSSwitcher, + ILogManager logManager) + { + _dbProvider = dbProvider.AsReadOnly(false); + _blockTree = blockTree.AsReadOnly(); + _verkleTrieStore = trieNodeResolver; + _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); + _recoveryStep = recoveryStep ?? throw new ArgumentNullException(nameof(recoveryStep)); + _rewardCalculatorSource = rewardCalculatorSource ?? throw new ArgumentNullException(nameof(rewardCalculatorSource)); + _receiptStorage = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); + _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + logManager.GetClassLogger(); + _stateType = StateType.Verkle; } public override ITraceRpcModule Create() { - ReadOnlyTxProcessingEnv txProcessingEnv = - new(_dbProvider, _trieNodeResolver, _blockTree, _specProvider, _logManager); + ReadOnlyTxProcessingEnv txProcessingEnv = _stateType switch + { + StateType.Merkle => new ReadOnlyTxProcessingEnv(_dbProvider, _trieNodeResolver, _blockTree, _specProvider, _logManager), + StateType.Verkle => new ReadOnlyTxProcessingEnv(_dbProvider, _verkleTrieStore, _blockTree, _specProvider, _logManager), + _ => throw new ArgumentOutOfRangeException() + }; IRewardCalculator rewardCalculator = new MergeRpcRewardCalculator(_rewardCalculatorSource.Get(txProcessingEnv.TransactionProcessor), diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs index 83f1c4ed436..89f4d22c30a 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs @@ -65,7 +65,7 @@ protected async Task CreateBlockChain(ISpecProvider specPro ILogManager? logManager = null) => await CreateBaseBlockChain(null, null, logManager).Build(specProvider); - private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConfig? syncConfig = null, TimeSpan? newPayloadTimeout = null, int newPayloadCacheSize = 50) + private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConfig? syncConfig = null, TimeSpan? newPayloadTimeout = null, int newPayloadCacheSize = 50, bool processStateless=false) { IPeerRefresher peerRefresher = Substitute.For(); @@ -102,6 +102,7 @@ private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConf invalidChainTracker, chain.BeaconSync, chain.LogManager, + processStateless, newPayloadTimeout, newPayloadCacheSize), new ForkchoiceUpdatedHandler( diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs index 10feed5a360..ad19bc91bec 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs @@ -875,7 +875,9 @@ public async Task MultiSyncModeSelector_should_fix_block_tree_levels_if_needed() private MultiSyncModeSelector CreateMultiSyncModeSelector(MergeTestBlockchain chain) { - SyncProgressResolver syncProgressResolver = new(chain.BlockTree, chain.ReceiptStorage, chain.TrieStore, new ProgressTracker(chain.BlockTree, chain.StateDb, LimboLogs.Instance), new SyncConfig(), LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(chain.BlockTree, chain.ReceiptStorage, chain.TrieStore, + new SnapProgressTracker(chain.BlockTree, chain.StateDb, LimboLogs.Instance), new SyncConfig(), + LimboLogs.Instance); BlockHeader peerHeader = chain.BlockTree.Head!.Header; ISyncPeer syncPeer = Substitute.For(); syncPeer.HeadHash.Returns(peerHeader.Hash); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs index ae0a6c32812..4953e435561 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs @@ -4,13 +4,16 @@ using System; using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Runtime.InteropServices.ComTypes; using System.Text; using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Blockchain.Find; +using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -18,6 +21,8 @@ using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Int256; using Nethermind.JsonRpc; using Nethermind.JsonRpc.Modules; @@ -26,10 +31,12 @@ using Nethermind.JsonRpc.Test.Modules; using Nethermind.Logging; using Nethermind.Merge.Plugin.Data; +using Nethermind.Serialization.Json; using Nethermind.Specs; using Nethermind.Specs.Forks; using Nethermind.State; using Nethermind.Trie; +using Nethermind.Verkle.Curve; using Newtonsoft.Json; using NSubstitute; using NUnit.Framework; @@ -241,6 +248,83 @@ public async Task executePayloadV1_accepts_previously_prepared_block_multiple_ti bestSuggestedHeaderHash.Should().NotBe(startingBestSuggestedHeader!.Hash!); } + [Test] + public void TestExecutePayloadSerializationFile() + { + Banderwagon root = Banderwagon + .FromBytes(Bytes.FromHexString("0x53053c27d4c8bb2f9a2bf0d74995a752a180f2c47a4a44cfdadf53f761eb043b")).Value; + IJsonSerializer serializer = new EthereumJsonSerializer(); + using StreamReader encodedPayload = new ("/home/eurus/verkle-testnet-neth/kaustinen/nethermind/src/Nethermind/Nethermind.Merge.Plugin.Test/payload.json"); + ExecutionPayload? payload = serializer.Deserialize(encodedPayload.BaseStream); + payload.ParentHash.ToString().Should().BeEquivalentTo("0xb6dd1f34a549560d7272a5c870c9f5c9d0d49837ffe3d9666689bfab9b64f916"); + payload.PrevRandao.ToString().Should().BeEquivalentTo("0x2ecbb0b8141068e728ff32c2da01934e3dc60befbfa3f5ff6274b0e0e6ebb205"); + payload.ReceiptsRoot.ToString().Should().BeEquivalentTo("0x570c8a20b5e928939ef9adf0a34b20e4b6b7783c33db3c86cfe4d8cbe039ce55"); + payload.StateRoot.ToString().Should().BeEquivalentTo("0x10db87ce544358c098b9635d0c9bf27a22b449c55cde30f49fe6b01ead621c60"); + payload.BlockHash.ToString().Should().BeEquivalentTo("0x899ab8d14548e82b4b98237f036d9080b19b768553e81af16689810c4ec18774"); + payload.ExtraData.ToHexString(true).Should().BeEquivalentTo("0x"); + payload.FeeRecipient.ToString().Should().BeEquivalentTo("0xf97e180c050e5ab072211ad2c213eb5aee4df134"); + + payload.BaseFeePerGas.ToBigEndian().WithoutLeadingZeros().ToHexString(true, true, false).Should().BeEquivalentTo("0x8"); + + payload.BlockNumber.ToHexString(true).Should().BeEquivalentTo("0xaa3b3"); + payload.GasLimit.ToHexString(true).Should().BeEquivalentTo("0x1c9c380"); + payload.GasUsed.ToHexString(true).Should().BeEquivalentTo("0x9a1dc"); + payload.Timestamp.ToHexString(true).Should().BeEquivalentTo("0x64a2e768"); + + VerkleWorldState worldState = new (payload.ExecutionWitness, root, LimboLogs.Instance); + Console.WriteLine($"StateRoot: {worldState.StateRoot}"); + } + + [Test] + public void TestAndExecutePayload() + { + IJsonSerializer serializer = new EthereumJsonSerializer(); + + using StreamReader encodedBlockPayload = new ("/home/eurus/verkle-testnet-neth/kaustinen/nethermind/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadBlock.json"); + ExecutionPayload? blockPayload = serializer.Deserialize(encodedBlockPayload.BaseStream); + blockPayload.TryGetBlock(out Block block, 1); + + using StreamReader encodedParentPayload = new ("/home/eurus/verkle-testnet-neth/kaustinen/nethermind/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadParent.json"); + ExecutionPayload? parentPayload = serializer.Deserialize(encodedParentPayload.BaseStream); + parentPayload.TryGetBlock(out Block parentBlock, 1); + + block.Header.MaybeParent = new WeakReference(parentBlock.Header); + + Banderwagon blockStateRoot = Banderwagon.FromBytes(blockPayload.StateRoot.Bytes)!.Value; + Banderwagon parentStateRoot = Banderwagon.FromBytes(parentPayload.StateRoot.Bytes)!.Value; + + VerkleWorldState worldState = new (blockPayload.ExecutionWitness, parentStateRoot, LimboLogs.Instance); + worldState.StateRoot.Bytes.Should().BeEquivalentTo(parentStateRoot.ToBytes()); + + var txn = new ExecuteTransactionProcessorAdapter(new TransactionProcessor(new TestSpecProvider(Prague.Instance), + worldState, + new VirtualMachine(new TestBlockhashProvider(), new TestSpecProvider(Prague.Instance), LimboLogs.Instance), + LimboLogs.Instance)); + IBlockProcessor.IBlockTransactionsExecutor _blockTransactionsExecutor = + new BlockProcessor.BlockStatelessValidationTransactionsExecutor(txn, worldState); + + var _receiptsTracer = new BlockReceiptsTracer(true, true); + + _receiptsTracer.StartNewBlockTrace(block); + TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, ProcessingOptions.ForceProcessing, _receiptsTracer, Prague.Instance); + _receiptsTracer.EndBlockTrace(); + Console.WriteLine($"NEW STATE ROOT: {worldState.StateRoot}"); + } + + public class TestBlockhashProvider : IBlockhashProvider + { + public static TestBlockhashProvider Instance = new(); + + public TestBlockhashProvider() + { + } + + public Keccak GetBlockhash(BlockHeader currentBlock, in long number) + { + return Keccak.Compute(number.ToString()); + } + } + [Test] public async Task block_should_not_be_canonical_before_forkchoiceUpdatedV1() { diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.VerkleSetup.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.VerkleSetup.cs new file mode 100644 index 00000000000..2bdd3d039e4 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.VerkleSetup.cs @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Config; +using Nethermind.Consensus; +using Nethermind.Consensus.Comparers; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Blockchain; +using Nethermind.Core.Timers; +using Nethermind.Db; +using Nethermind.Facade.Eth; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Merge.Plugin.BlockProduction; +using Nethermind.Merge.Plugin.Handlers; +using Nethermind.Merge.Plugin.Synchronization; +using Nethermind.Specs; +using Nethermind.Specs.Forks; +using Nethermind.State; + +namespace Nethermind.Merge.Plugin.Test; + +public partial class EngineModuleTests +{ + protected virtual MergeTestVerkleBlockchain CreateBaseVerkleBlockChain(IMergeConfig? mergeConfig = null, + IPayloadPreparationService? mockedPayloadService = null, ILogManager? logManager = null) => + new(mergeConfig, mockedPayloadService, logManager); + + protected async Task CreateShanghaiVerkleBlockChain(IMergeConfig? mergeConfig = null, + IPayloadPreparationService? mockedPayloadService = null) + => await CreateVerkleBlockChain(mergeConfig, mockedPayloadService, Shanghai.Instance); + + + protected async Task CreateVerkleBlockChain(IMergeConfig? mergeConfig = null, + IPayloadPreparationService? mockedPayloadService = null, IReleaseSpec? releaseSpec = null) + => await CreateBaseVerkleBlockChain(mergeConfig, mockedPayloadService) + .Build(new TestSingleReleaseSpecProvider(releaseSpec ?? London.Instance)); + + protected async Task CreateVerkleBlockChain(ISpecProvider specProvider, + ILogManager? logManager = null) + => await CreateBaseVerkleBlockChain(null, null, logManager).Build(specProvider); + + public class MergeTestVerkleBlockchain : TestVerkleBlockchain + { + public IMergeConfig MergeConfig { get; set; } + + public PostMergeBlockProducer? PostMergeBlockProducer { get; set; } + + public IPayloadPreparationService? PayloadPreparationService { get; set; } + + public ISealValidator? SealValidator { get; set; } + + public IBeaconPivot? BeaconPivot { get; set; } + + public BeaconSync? BeaconSync { get; set; } + + private int _blockProcessingThrottle = 0; + + public MergeTestVerkleBlockchain ThrottleBlockProcessor(int delayMs) + { + _blockProcessingThrottle = delayMs; + if (BlockProcessor is TestBlockProcessorInterceptor testBlockProcessor) + { + testBlockProcessor.DelayMs = delayMs; + } + return this; + } + + public MergeTestVerkleBlockchain(IMergeConfig? mergeConfig = null, IPayloadPreparationService? mockedPayloadPreparationService = null, ILogManager? logManager = null) + { + GenesisBlockBuilder = Core.Test.Builders.Build.A.Block.Genesis.Genesis.WithTimestamp(1UL); + MergeConfig = mergeConfig ?? new MergeConfig() { TerminalTotalDifficulty = "0" }; + PayloadPreparationService = mockedPayloadPreparationService; + LogManager = logManager ?? LogManager; + } + + protected override Task AddBlocksOnStart() => Task.CompletedTask; + + public sealed override ILogManager LogManager { get; set; } = LimboLogs.Instance; + + public IEthSyncingInfo? EthSyncingInfo { get; protected set; } + + protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolTxSource, ISealer sealer, ITransactionComparerProvider transactionComparerProvider) + { + SealEngine = new MergeSealEngine(SealEngine, PoSSwitcher, SealValidator!, LogManager); + IBlockProducer preMergeBlockProducer = + base.CreateTestBlockProducer(txPoolTxSource, sealer, transactionComparerProvider); + BlocksConfig blocksConfig = new() { MinGasPrice = 0 }; + TargetAdjustedGasLimitCalculator targetAdjustedGasLimitCalculator = new(SpecProvider, blocksConfig); + ISyncConfig syncConfig = new SyncConfig(); + EthSyncingInfo = new EthSyncingInfo(BlockTree, ReceiptStorage, syncConfig, LogManager); + PostMergeBlockProducerFactory? blockProducerFactory = new( + SpecProvider, + SealEngine, + Timestamper, + blocksConfig, + LogManager, + targetAdjustedGasLimitCalculator); + + BlockProducerEnvFactory blockProducerEnvFactory = new( + DbProvider, + BlockTree, + ReadOnlyTrieStore, + SpecProvider, + BlockValidator, + NoBlockRewards.Instance, + ReceiptStorage, + BlockPreprocessorStep, + TxPool, + transactionComparerProvider, + blocksConfig, + LogManager); + + + BlockProducerEnv blockProducerEnv = blockProducerEnvFactory.Create(); + PostMergeBlockProducer? postMergeBlockProducer = blockProducerFactory.Create( + blockProducerEnv, BlockProductionTrigger); + PostMergeBlockProducer = postMergeBlockProducer; + PayloadPreparationService ??= new PayloadPreparationService( + postMergeBlockProducer, + new BlockImprovementContextFactory(BlockProductionTrigger, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot)), + TimerFactory.Default, + LogManager, + TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot), + 50000); // by default we want to avoid cleanup payload effects in testing + return new MergeBlockProducer(preMergeBlockProducer, postMergeBlockProducer, PoSSwitcher); + } + + protected override IBlockProcessor CreateBlockProcessor() + { + BlockValidator = CreateBlockValidator(); + IBlockProcessor processor = new BlockProcessor( + SpecProvider, + BlockValidator, + NoBlockRewards.Instance, + new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), + State, + ReceiptStorage, + NullWitnessCollector.Instance, + LogManager); + + return new TestBlockProcessorInterceptor(processor, _blockProcessingThrottle); + } + + private IBlockValidator CreateBlockValidator() + { + IBlockCacheService blockCacheService = new BlockCacheService(); + PoSSwitcher = new PoSSwitcher(MergeConfig, SyncConfig.Default, new MemDb(), BlockTree, SpecProvider, LogManager); + SealValidator = new MergeSealValidator(PoSSwitcher, Always.Valid); + HeaderValidator preMergeHeaderValidator = new HeaderValidator(BlockTree, SealValidator, SpecProvider, LogManager); + HeaderValidator = new MergeHeaderValidator(PoSSwitcher, preMergeHeaderValidator, BlockTree, SpecProvider, SealValidator, LogManager); + + return new BlockValidator( + new TxValidator(SpecProvider.ChainId), + HeaderValidator, + Always.Valid, + SpecProvider, + LogManager); + } + + public IManualBlockFinalizationManager BlockFinalizationManager { get; } = new ManualBlockFinalizationManager(); + + public override async Task Build(ISpecProvider? specProvider = null, UInt256? initialValues = null) + { + TestVerkleBlockchain chain = await base.Build(specProvider, initialValues); + return chain; + } + + public async Task Build(ISpecProvider? specProvider = null) => + (MergeTestVerkleBlockchain)await Build(specProvider, null); + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs index 3b684d5e40a..f0a63d7c19b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs @@ -24,6 +24,7 @@ using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; using Nethermind.Trie.Pruning; using NSubstitute; using NUnit.Framework; @@ -101,12 +102,12 @@ public MultiSyncModeSelector Selector if (_selector is null) { MemDb stateDb = new(); - ProgressTracker progressTracker = new(BlockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(BlockTree, stateDb, LimboLogs.Instance); SyncProgressResolver syncProgressResolver = new( BlockTree, NullReceiptStorage.Instance, new TrieStore(stateDb, LimboLogs.Instance), - progressTracker, + snapProgressTracker, SyncConfig, LimboLogs.Instance); TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/payload.json b/src/Nethermind/Nethermind.Merge.Plugin.Test/payload.json new file mode 100644 index 00000000000..02af974e82d --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/payload.json @@ -0,0 +1,324 @@ +{ + "baseFeePerGas": "0x8", + "blockHash": "0x899ab8d14548e82b4b98237f036d9080b19b768553e81af16689810c4ec18774", + "blockNumber": "0xaa3b3", + "executionWitness": { + "stateDiff": [ + { + "stem": "0x0124c6de98c61db111fe7b869fbc743f8a8b096fbebff5dd53c6b0f57604fd", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + }, + { + "currentValue": null, + "suffix": 128 + }, + { + "currentValue": null, + "suffix": 129 + } + ] + }, + { + "stem": "0x350989b82e026fe741d383873c5497e21fa909cc03c74e9d6259d16868f61f", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + } + ] + }, + { + "stem": "0x56da9b49ce556e49e7abc395ec410b65f1aa84204a1a610aa56a72f4dcaee7", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + }, + { + "currentValue": null, + "suffix": 128 + }, + { + "currentValue": null, + "suffix": 129 + } + ] + }, + { + "stem": "0x754db1b8f1e7982ce989a892da791bd2e45ff12f68fa71b83111bee7f87ea6", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 89 + }, + { + "currentValue": null, + "suffix": 204 + } + ] + }, + { + "stem": "0x7f1821a5b70c22154f2904c11d02d32124905d41a39c5623512df8b6a979fd", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 103 + } + ] + }, + { + "stem": "0x8fbcbc0bd2e718154829cc39f913f9d25bd4528c1c49b035462a87964a799c", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 100 + }, + { + "currentValue": null, + "suffix": 108 + } + ] + }, + { + "stem": "0xa5a8d873e5dbe853393dd66f89a69f78f8176fdabd8b9c742dbeacb2b37223", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + } + ] + }, + { + "stem": "0xae1b106992d792cbeb5d96559894c2a8dda72c9414cfb3532325f4633047ae", + "suffixDiffs": [ + { + "currentValue": "0x0000000000000000000000000000000000000000000000000000000000000000", + "suffix": 0 + }, + { + "currentValue": "0xd235cd28da96ceab171500000000000000000000000000000000000000000000", + "suffix": 1 + }, + { + "currentValue": "0xcb77630000000000000000000000000000000000000000000000000000000000", + "suffix": 2 + }, + { + "currentValue": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "suffix": 3 + }, + { + "currentValue": null, + "suffix": 4 + } + ] + }, + { + "stem": "0xbf101a6e1c8e83c11bd203a582c7981b91097ec55cbd344ce09005c1f26d19", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 1 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + }, + { + "currentValue": null, + "suffix": 4 + } + ] + }, + { + "stem": "0xee29a4d97600bfe797521fe5210fee34b3cf8ca8aaf8714d4180ab04aac69e", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 1 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + }, + { + "currentValue": null, + "suffix": 4 + } + ] + }, + { + "stem": "0xee9707ef2846473a6fe4e10781a3b8b225ea950a7c89fb88f01c3544a41dfb", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 1 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + }, + { + "currentValue": null, + "suffix": 4 + } + ] + }, + { + "stem": "0xfc57e03b7448ccefe96c60879b2a4a140a94f8c37bdd9a43092cfd5dbf753c", + "suffixDiffs": [ + { + "currentValue": null, + "suffix": 0 + }, + { + "currentValue": null, + "suffix": 2 + }, + { + "currentValue": null, + "suffix": 3 + } + ] + } + ], + "verkleProof": { + "commitmentsByPath": [ + "0x4215606d4d8862f6471a87e74ee3607c78ebb2faed060c63f7c531b74d4f3d49", + "0x1a650407d16467c98e2a8053feb761f4ab9571ab19263616a30e9058614e0350", + "0x61e498acd6dc6b1582067656d4da546b36c7e53bfa8085366992cb64463590de", + "0x31c4c2ad78f22661bb6a9fa721cc09ff32a6ac189d7968dcbfc2ec54bb0dacc9", + "0x34f2efbe4b3588de95bf5fd921c9175fae4c4f59f12dba66083f8795a95515b0", + "0x5e41c1946b11c4f4a334101b9b473d311b727699f58e979a6c1a15e69159a8cf", + "0x373cc1a64e24142442bd8226c9fa0bd154773cced889ae137fe078d16bc95ebb", + "0x3bd5b6c5bd73f2440d86c44215c4708c55a3e8c9d24e2fc40af93e8ff66201b2", + "0x721838639a5ede258bd53be557b57e10ce3840604eb2477ec8b2d69de553dd04", + "0x4e58a2a03cf9d112ea3d14899b4ef61de07fce858dff5d1d8c209a95d1de79b2", + "0x024b2325723f4f961be477bf25aaca2bff35fc6cc5add4ebdcecbfc10000131b", + "0x5a7d61e68c3dc7e15c99985c8e4e12789e0991582ab00775ad64243d8a88fd79", + "0x20d4581449039bcfa973f8f391fe899fe848cabbbbd21a299b22e92c652587db", + "0x5546d41c594f811094005963a9e5603ac705328f53792664a962a5dd4d6db125", + "0x4ea3fc5fa3c85b5bf953bb57fd561d5aa69d6411397c8b96f700c86791fa717b", + "0x3a963ee86f9918682bf45539df4bd984ca7dfce77dacfdddba4cf4fdc3d72ae2", + "0x551242fd9715dd994f823a18bc3ae9bd923b05765549373465940f0e1c7d6f89", + "0x5ce730e2c3000b777f62b84da98b60f9f0a4e8761cf2d5cb4517bbfa4f9e3608", + "0x61f041909463d666f2ecbfd00454ab025a45c8e82b296c5222954e322bf4c822", + "0x1bacd7e58989c5f25547a731fbf1c50d3b29bf506baaa28a2ef183a705800fb2", + "0x1a596bd126c23305c77fc4a0cde4ae89f0985aaebe4f9e5561785243453ddb59", + "0x5cf220f97f87c6cf55fcc67ea6d07608da0654b1f4ea012d8747335b9cf7080f", + "0x409f56b3b62066e1f65ef8331c4854bdedd1095fdd46357d21c2c6fa999024bc", + "0x56d4a6c96b764d8cc6cc9de40c6f0c2ee6d488c1a315c1275cb00222136016c6", + "0x28cd81de95b0d982737569d9eae5644facc4a6e6090c6b9d6ff1e132b5d5ab09", + "0x604f355a5876e84afa7de62f0e609bcdb75b683285cdf4032809d26c564f479b" + ], + "d": "0x1df465ecd01c4b30cd1603a6150f144aa62aa0dbaac960c73144accd25a126b3", + "depthExtensionPresent": "0x181918181818181a18181818", + "ipaProof": { + "cl": [ + "0x14dfe0c14d09476bbf4c1c3749ed4e26a380b8d22ca1a52179b5998326241250", + "0x1b6c79febb8279661859c284473aeb0789509e6886e65b2566bd5630abe1fab2", + "0x55a0959b620fe6ec7b591805f63671ceeabc6cdad87fcadfd1c9e46f3e0dd34e", + "0x58b7450308264181eb222cfa5f372ad22f741680e1607604af63159484351547", + "0x34219806fa5b9a96aad5e64da699ead51c5469c7f19fa99c0d47c80367afc06b", + "0x2841686240ed01c15c64b38a7423d70cf07f45ca558771e70dd40d5d532407ab", + "0x395957ace8da3ceccecf82c79e5e367846e3a171dac4fb04860ad4a16d962502", + "0x59218d85c8a7eb7e5026d50d822e9aae230f1be0a470a4392dec9a15da42100f" + ], + "cr": [ + "0x097f09638dc268c24dcb10d25e172936a444f995af81ff6c19342ff09428e459", + "0x1354b79a1320b7b9163bf8c815911baab501f64555b7c6b181a9014a7dcc3d94", + "0x175c6f9a3d371c4d6aa67b5ce1a589c5ec192bf28f08ad2bafd0f146f9c8cc26", + "0x5e685a15f2b6edcf4b05270127d27202450a869968bd052bd4c85cb49cd13e53", + "0x08000f34f41efcc5795264a0dd87e0ea7753ece9370d08b4a08dc5f31543e16f", + "0x2e66b06fadd61a70eb883aa55a6527ec5c1c753657176264b191db07c4a30203", + "0x1c683afd90442aa52e9805197bfcea3ba8a9df8204a057e17af42aabee707fef", + "0x25cbefc88ed8d3c1f1a9b2dff8220256bc5c4f2ba79934a4f53b4c664ce6aeab" + ], + "finalEvaluation": "0x076e23992cf87011ddbcf6210d4165d7db91dada52c22c98222f71c718da054b" + }, + "otherStems": [ + "0x3509896f30cee15c99bed51bf9873f49e03989af64daa8928aeb2c66d3c494" + ] + } + }, + "extraData": "0x", + "feeRecipient": "0xf97e180c050e5ab072211ad2c213eb5aee4df134", + "gasLimit": "0x1c9c380", + "gasUsed": "0x9a1dc", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0xb6dd1f34a549560d7272a5c870c9f5c9d0d49837ffe3d9666689bfab9b64f916", + "prevRandao": "0x2ecbb0b8141068e728ff32c2da01934e3dc60befbfa3f5ff6274b0e0e6ebb205", + "receiptsRoot": "0x570c8a20b5e928939ef9adf0a34b20e4b6b7783c33db3c86cfe4d8cbe039ce55", + "stateRoot": "0x10db87ce544358c098b9635d0c9bf27a22b449c55cde30f49fe6b01ead621c60", + "timestamp": "0x64a2e768", + "transactions": [ + "0x02f8f283010f2c836377cb843b9b1360843b9b1368830186a094b02a2eda1b317fbd16760128836b0ac59b560e9d80b8803d6000593e5960002080556000606e55612264e36000607955600060ef55600060f65586600060f65538600060f7557f6e9c241a2e01bdcaadb0cfa60554e169e86fad15e84a85fdafb28b08cd81798260ed527f701bef124ed1997b7931753ef29b9cbbcace80bd8b0d17a261aecc72690e7ec461010d527fb69bc9047e599fc080a02eb3b7ad3473f5b1816640c621c373d75f2d0be5a13a4e8b5356525d4943e4fca01b98c120f3ae57fa7d323b0f1b29f8c70bee0306e350c08c1cb172d67675dad9", + "0x01f8d983010f2c836377cc843b9b1368830186a08080b8807f20bb94da595c48633d724d44041fb73b29b72b3bcb87cc3b19f8de6f03b2964f6374547a1f527ffcd2ff630c58876481b9b44bc78722a52b7875345070041d56d85c126a6c91396374547a3f527f2de419bdc23342b9928776d2cbaed6554ac21173f6c002fa22ad5ec888dac8a96374547a5f527ffadf169d545f8cddc859c080a0f0301f01dc2dec1a4ad7e9bd14700149eb5262cf0f5e96bdafd7f5740ec7809aa00dda0a4611c6d823fca9779c74c97b95bef95e4c7fdaab229df3f3865e770a96", + "0x01f8ed83010f2c836377cd843b9b1368830186a094000000000000000000000000000000000000000080b880b3667fa88d3430aaa13edb5edc39e7c47652fa6d336f8175e4b3c0e3ce084257ecdf636035527f298164230e75a65e557449a0ccb528b1a2b7dd1c8294b54eeb0d17b60144c0d16055527fab62115da215c0a2526d0fb51a62ce89efeba0b49df025ca1ba9964815089619607552604260955360ef60965360f660975360ea60c080a0d6fffa21eda056e9199f48b614108a386dbff6de26d95dbb27148005d9f2c6fba050bc3e4384b00834b8a3eb7a9977e8e54e6333ce08be98d717b986d65775b496", + "0x02f8de83010f2c836377ce843b9b1360843b9b1368830186a08080b8806000606c556000606455fd32603860205360d86021536000602253608560235360ad602453604e60255360a3602653606960275360586028536056602953607c602a53603a602b53601b602c5360a7602d5360a7602e53600c602f53607f60305360c160315360ee60325360ea6033537f05e62eb3c89fcde08c9f2eec36ab16c001a08eca5d7b9983b1c39b568e6d75e1d3ccf85bc3643ae143a722092faa8f810501a045ccbe3dae3437179479425a91f1dd99e1dcd158e8a43f12625a3e1d1bc69444", + "0x01f8d983010f2c836377cf843b9b1368830186a08080b8807c7f0b25e2382d2d3431002dcb3409ec207dd1c5b6ddfcab393d36af0a4279eb3361605a527f73d87511c62568ca2efc697b8d7bebcfcf705f81228fb1e96527d2bea3051d5f607a527f118e8ada3d7225ab1941d78f555bc9570f7240d20d12dd2fe13f65648271498e609a527fc85db05e3d00ed64305ba3a36c2d0c736df9c080a05a8059cc5d24646c26447ecfc371bfeed4685127bc7ca4a804c525d8ee82699aa031de254e4387b020730fa1ea265b04601d0915e8a82bb5842283813492121680", + "0x02f8de83010f2c836377d0843b9b1360843b9b1368830186a08080b8806000605955600060cc55da600060b755de8f4124600060565560006018552b988e6000603d55607b60d953600560da53604d60db53601f60dc53609a60dd53606d60de5360ae60df5360ae60e05360cb60e153605660e253601960e353602060e453603060e55360ba60e653607960e753608960e85360d460e95360a460ea53c080a08af055939c62f8ff98ec40c9bc0126a618f1c1c12aaeeffedd2d1bb5d21a411ca01595bfcc2626718a8c11e6947d6d73c912669f1e5798b389dd378ee0c74846ba", + "0x01f8ed83010f2c836377d1843b9b1368830186a094b02a2eda1b317fbd16760128836b0ac59b560e9d80b880ea817f6c1b0d0f6505b6cd9d2163515a7fd61a3c7ad906ab0cc8cd87dea234b8073b9d602a5260b9604a53602c604b5360b5604c536031604d5313600060ec55600060d255a27fc982926b6692aab859a93079f3cbfdff37f3367dd0500d861ab4424b7f7ff2316041527fb7b46b0c2774d5cba242f98b321d5c04eff045e5b4c001a0f2d38978b3bcbeda530b5a6321b902454e9604d987482ff193febcacece67e8fa07b0522f7dc667719d84be3df39acab5ad92bea4b4b5a2071e2fb81f74f9c3d8b", + "0x02f8f283010f2c836377d2843b9b1360843b9b1368830186a09487463bc5da4e24d37392d82f05ee5eda89c39c3780b88060006038557f26bea0fc21026c74bfae7292ab938fe9948f20f1229795d370fd19b8c6a526ce601d527fd42d20365b71180dc59070eb8647970c9d856ef9a8c067843293b1b73a6bd192603d526003605d536009605e536020605f53600160605360d060615360a860625360716063536099606453603a606553607c60665360c080a0edd7e52d489cb67f0d3ae4646461df6da12587d24c74fd01c936b237abb067a3a02a2dbd90002d2b9bf66cf9a203b658d089308f82b447dcdf8dfd11e91f15d09e", + "0x02f8de83010f2c836377d3843b9b1360843b9b1368830186a08080b8806000606755d67fe54fc4abfd26331b6d1bf4b7711c32a8d3f5ce4f53fb20571b4d5a431742b76460bb527fa51642115e3b9357c6ffa1bafbaf6fa76b378bf7ae7dfe429446e6c5d553196260db52604b60fb53604c60fc53604260fd5360d860fe5360a260ff53603761010053602d6101015360066101025360ac6101035360c001a04a25b25e7d110673d5fa53b8dce877e47e3f1a3fd45d25105f26f67e0810325fa04689e0f8beb485abe079080fa21f378bbcc0b3065607802534f92e777f3f12f5" + ] +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadBlock.json b/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadBlock.json new file mode 100644 index 00000000000..dccd5cb3842 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadBlock.json @@ -0,0 +1,328 @@ +{ + "executionWitness": { + "stateDiff": [ + { + "stem": "0x08bfb04d8c1f5d29740d4f1ced7bc0fc2cc32be1828ef0f79054a25160f6dd", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0x23d5c1eecd42bdf9e69d4650a0307a866dd3e0fc3219be8ec4d0c156db6c5d", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 84 + } + ] + }, + { + "stem": "0x2d57fda1128d037216ca3d8ea04bb726e8f3042d75a6c35ea986180b6f4e2e", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0x304468977deed2f4621bdd4cd8504521317728a58e3bf5ccf58f40a49243bc", + "suffixDiffs": [ + { + "suffix": 140 + }, + { + "suffix": 224 + } + ] + }, + { + "stem": "0x356bbcf89efe77c00f03a154882905ef40fe0b709a31e9606f16b9edb9061b", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0x37a1a4d881e6e104ae7a6233379db6165f013eb59aebb79cfd6011ed19b2c0", + "suffixDiffs": [ + { + "suffix": 89 + }, + { + "suffix": 204 + } + ] + }, + { + "stem": "0x432ca7f685afcf9ef5c74ed12058f15113a2af93cc2047d0c3307a499b581f", + "suffixDiffs": [ + { + "suffix": 100 + }, + { + "suffix": 108 + } + ] + }, + { + "stem": "0x803530d457314476d4f2ef5d2fc5b8367f139f1427fa02506ddd5de41b5053", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0x9e2c3ed0375dde737196a05ca46ed32d59624212ff061cf130d854531458e0", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0xab1b35958fd283a110b678e45eabdad5f8de1ee2176c67756e3bacb5ae0841", + "suffixDiffs": [ + { + "suffix": 91 + } + ] + }, + { + "stem": "0xae1b106992d792cbeb5d96559894c2a8dda72c9414cfb3532325f4633047ae", + "suffixDiffs": [ + { + "suffix": 0, + "currentValue": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + { + "suffix": 1, + "currentValue": "0xaabb5990fd2efd0f171500000000000000000000000000000000000000000000" + }, + { + "suffix": 2, + "currentValue": "0xfe19660000000000000000000000000000000000000000000000000000000000" + }, + { + "suffix": 3, + "currentValue": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0xaea3df91f870e7cb2a599d0bc40b09b12e2bc9199ae2bd81a16bbe1f7ed866", + "suffixDiffs": [ + { + "suffix": 112 + } + ] + }, + { + "stem": "0xd6673436c9b986d093e6de0e47bcdcb6ebd22083302324302ce739de98d430", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0xd9a67d12d7e68796c9cd72dcf733b728e74a3926fe8044b741b0d577ede26b", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0xe3e8773e67e7af3e4e1259207e724bc9dff9ec6794644f3da1c380495ee81d", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0xee9707ef2846473a6fe4e10781a3b8b225ea950a7c89fb88f01c3544a41dfb", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + } + ], + "verkleProof": { + "otherStems": [], + "depthExtensionPresent": "0x181818181818181818181a1818181818", + "commitmentsByPath": [ + "0x675758cb74e79e7c05b3fcfa2018916dc18e51a99818f8c0cefb26987a86c953", + "0x6838f3626476d019bad0019e025dbe076db66734dabeaef70e359e05f63ce7be", + "0x0be338a9d82c4bae5308bc99dc5d66942a19c91b214b1b9a0fa66427bf21f6c1", + "0x5f626d03392e0cc823e3180ed3909a39255599bba2471f1747154183542669b1", + "0x0a30d4e6057c84d040635b664d0e647bd94c3981eaf187f0fc5c2cd29b33dae5", + "0x4928e6bc1549cd4437246f71467c7c2da953d9112d61359cd611e509160bf090", + "0x10fa3122af7ad7ed0b49dda46b408886e78b7b45215c3bee9c885c8e3ebb23b4", + "0x33e8d41ca0c872299a0e31fd86c44ff40d267ac68caed728c69c444acf417fee", + "0x3ee51ae771eb08f3b479a05cb31a9df4488b49efc3a3d2190b5cb220c15a0b46", + "0x39be46860628f141a4313af741417b0f1ec1112c4fcd528b03a57429daf380bd", + "0x16a0ed4623c39442b466b0cb7a4ca4a1a82596bb4f06dad011b9e44473fb5e8d", + "0x4760aca815af3f0293ee95588056dcaf10a32c01b3e7738d7d0d0481c81eec73", + "0x536511a411dcf8a044eea72188246a833d5448ba22e4b8ebed7d94a0f3bef483", + "0x271f8186ff219f3c05e9ecaaa7a65e75da77f76e18f9349e80e5b5b365926b20", + "0x6487c5eef2ebf7c4fdef754426fcbf26cbcbf859abf01f220f67f4ef94a9c048", + "0x42d654724a04e289067323f7a9562e62d915b0394fec2a1842ef22b509a5539f", + "0x4456f20777cb8be0ed690ba07955c80e300f189422c7de5b1455974731b56660", + "0x053f5a1125505f46f47e1cb40f103fb5a0ed5c46d08fa70e2f9356b65e559a5a", + "0x22c99109c25b08261b4187b59dd79e450c1bc556aa4a6428f0e09fe1c90c6048", + "0x5f6ef69706ddea6212f8be8c9f359a1caaf15e8a45dc3ba2f56ffdf8034ad231", + "0x195b8aedc7c439481841e7ce3847e40ccbfb77a59903c9097454ac7dd8e41a62", + "0x55679ab78cc7042cba849ad10bf7c788b835520b09d2abf4077cebe5c1e5a9dd", + "0x66e8e08729d51870728b98ad013e63e151ebe63f4caab33032407b94239793db", + "0x44da54c431b7b074f8c204274baaa4946069fe82ef22345b8a63d745a1a8ca52", + "0x0fa476826980e9ad15ddedf19759b0a3e8c5c78f5217a16ca8bedd2816d777ab", + "0x6068f1d569479ef1772aad11d8c6709717791ef38236a1f039b40f2fb94aa84e", + "0x52c3cd35a6d860fbf7c51c1baab7bf1fff6d364756144d810979bb60f4bf0cc9", + "0x28d953cebd909099683a09ab3b765c41d78802072d570478577e3c4686e12c64", + "0x58e981af8b641b15d5570a8b2289e570ae9cb71feb969163f023f18f73e907d2", + "0x250ba68bd526cf6a8dcdc41773589bb4e86ed56d20cb37ef4338b7dd3c01e268", + "0x08eda5be6df962f90e39589a52a0af5edd98b9e96b6edbe27e43de613309f4cd", + "0x22d31ea9c4e889467e64141fd5cfbc76eb47d4fc4f626ea25cf02e5938786cc9", + "0x2e199b034fe133445dbca3ced7e379cdc1e037106e961f7c0fb22a75836c5800" + ], + "d": "0x418ad1c9a0463fb607aa9af3ac1549af664e521778f67de8d5b47e5455345d0a", + "ipaProof": { + "cl": [ + "0x6f43c0a59d8c383d3758011bac5d3da3201fc719625ba68d79343cca97295be6", + "0x3f9e5eda2d42103d4493bc211ac74067285d347077e33ff0c1466f0f20627533", + "0x00be0f56a1f582c63dbf393a7f54f5ebd894d3a3566400c8c1c0f498e46dedaa", + "0x08de323081323495675edc913fbf07c9da1746dd0ca67cf4c6b91f23e9037897", + "0x172fa608d539d19e0118ff69c91458552abeb3f2819956f5abbd6f1688f57a0c", + "0x46b471cf7dadcfa6ac831ec0aac38691d1c05e5b46304f6312e9960a0d424612", + "0x2ab31a005d97d9528958774e1e8fc12d544339f73a1ce66d4a8ac7f5db0e46ac", + "0x72b88e2b31c4a0ef4dab9770d77d25f84cb64ec89687f4eba6c4aef62cbccbb6" + ], + "cr": [ + "0x2082aaad27f4fb67c6a229913f15aad5159e0c10cd04ce0cc77daf76832c4c2d", + "0x5c1bee0b59c98c81862b6bf7583aba9f21ac8ea8ceff55afea4ec43816eb705c", + "0x14c9043bf69c2eb75b401d3dff9ebb8d37b046ef57c7df68b9e468be9eb0770d", + "0x352f1a98da9b6843b4bd384cba4d6300275634cf67aa7f2c25d7f45a55edfc47", + "0x20f6875bae263b7966f1122265398112e337c9ecc498ff43888c01f346528f85", + "0x0be64eba0819d05907665e301a432a9d23d1f4961e80920635ba5bbd094cbfdf", + "0x6336ac960e025f7ad4b39a54466b5bbba05354ae0b9475b441282160f6bf8ba8", + "0x0725fac4ca14511fd1950280ec1c849464b08b6c0472b47cfc9ee40441da8dbb" + ], + "finalEvaluation": "0x01a47a72f34a92bee9f7f7be408bf302c13cff254633b4c9e6d28d69a7c8c768" + } + } + }, + "baseFeePerGas": "0x8", + "blockHash": "0xd9355393d47ba3604e077530b7b798bb82c7be2c6b328768e1980b95bbd59532", + "blockNumber": "0xaea3f", + "extraData": "0x", + "feeRecipient": "0xf97e180c050e5ab072211ad2c213eb5aee4df134", + "gasLimit": "0x1c9c380", + "gasUsed": "0xc284c", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0xba1695b85858cbaccc61dcd2914229361a145c5dbc3eb818b111b8aec474745b", + "prevRandao": "0x9b057f348fb3f798a6a10fe4f72c4b69bc3c43871e6a7ef393cb4e9cb5a85b91", + "receiptsRoot": "0xd02dd60adc8c185f1b2f69814227882da699df65d794b78b0f089b7cf7848cf2", + "stateRoot": "0x3e5e2be7a5040a2e9eaa0b06c2dcebfd3b2a235e32c9513dd376e360c89130bd", + "timestamp": "0x64a653bc", + "transactions": [ + "0x02f8f283010f2c836619fe843b9b1360843b9b1368830186a09469908232acef702f580c0bb2e4bab6e1034288a980b8808ca1986000601b557ffc71a3525a3cd19397d529766d359875a48d2b8eec6f02e5234649ec0eea841f604d527f5e51b7df5b386c6b95496ae5bbe9e5c61feba8f4d74e6a8fee5a59053d334bf7606d527f596f16f11968166e940e5161f0accd9564ade899c375f83b9827fc546ab19ad0608d527f95c648427b4029096ba8f7c001a0305572e4cce750ac3dbe156f9c34afbe4e6b9510598e41aae5ffbc2bb8082d1ca07806f7fc4687487cdbc892ce5c754b5052c57a46539e6942aa7c9bb8389f374c", + "0xf8d7836619ff843b9b1368830186a08080b8803d6000593e5960002080556086604d536049604e53607c604f5360946050536065605153fd6000603155ff600060565585245b600060dd556000604a55d687a660006097557f287174210997d79218d80d014647ac6d39f6bd23b6930f920f3e0b36628103546020527f4e134bb64663a3bc200fba06ab0a2b7ad282b61279d083021e7ba02458d3b2d7d871545e14df79c93ea9e5089df9cef176bf39a31f7dea2636299ca00d48b7c0ef459cdb2b6e751eff5164215abe5ed73a94710ff689ca03348fbe6f", + "0x02f8de83010f2c83661a00843b9b1360843b9b1368830186a08080b8806000606c556000606455fd32603860205360d86021536000602253608560235360ad602453604e60255360a3602653606960275360586028536056602953607c602a53603a602b53601b602c5360a7602d5360a7602e53600c602f53607f60305360c160315360ee60325360ea6033537f05e62eb3c89fcde08c9f2eec36ab16c080a0bb2b771f46bc6fde4b6b576c8a6589edc059e4b2b1baa71db1f4685bddb06d20a06e1e9ad9b472d8554c0481cd44b007d6ee399d94a0c30d71728e022726f62a69", + "0x02f8de83010f2c83661a01843b9b1360843b9b1368830186a08080b8806000605955600060cc55da600060b755de8f4124600060565560006018552b988e6000603d55607b60d953600560da53604d60db53601f60dc53609a60dd53606d60de5360ae60df5360ae60e05360cb60e153605660e253601960e353602060e453603060e55360ba60e653607960e753608960e85360d460e95360a460ea53c001a0f4093c897752f007045e9b72daff26ad0035dd14f434d99b466157ae81c266d7a063a1dca63c050652d6b4848b319e2329da019ad107b7a3dc8e4df4fc7c2810ec", + "0x02f8f283010f2c83661a02843b9b1360843b9b1368830186a094b02a2eda1b317fbd16760128836b0ac59b560e9d80b8807fc03ca4ffe71dfbca55e7b05eabd9870b5ab1566568f3552d4aecdff4448fe27460e15260e26101015360056101025360fe61010353603a6101045360b061010553607e6101065360de6101075360df6101085360cd6101095360e861010a5360c861010b5360bd61010c53602461010d5360bb61010e53607360ea5360cf60c080a01437b1e61daffdb5fc021ebfa2124149893c8530f465fde3da4a980beb51b77fa008b0aba78422e42429d5eb499ac5208e916d94a290046d0c0b42eb4769e45ac3", + "0x02f8ca83010f2c83661a03843b9b1360843b9b1368830186a08080b86c60006014551e7f9b9881691f540c5e8a089b9bcf16c5069959065f836de454b33018c6ffe2a6c0606a526091608a53601b608b536008608c53606f608d5360ea608e5360b6608f5360b4609053496000602d55fa146000604e556000606055600063754026cf55600060d855c001a0dce74865b956e03fdd5ceb1908887839a0d30ede8af8c825c6f5154f69399e98a07f221d144f74940239d5149ec2302bb27c6b1ddd9ca8d856785559cc863a44a2", + "0x01f8d983010f2c83661a04843b9b1368830186a08080b880600060e0556000608c554f600060bb557f06e2c82b5f8c0996ec13be81c9833bc13c254877c132bef0e975ff5508cba2d5608a5260cd60aa5360bf60ab53609e60ac5360dc60ad53607660ae53608560af5360e460b053609360b153603460b2537f6f6b064ebd6243ce2deda33a93b308096cbf22af9a32656c787ea81657dac001a0bae739288547808b07ea3ed0351a6081a604e4f1fcee25ed299ab6ef37f428ffa01df4a08544fe9c63c6113aefd84799e5e3a60648ec22e2fff5fd6d9bac4415c5", + "0x01f8d983010f2c83661a05843b9b1368830186a08080b880b4667fe339b5a82151c11fe7afa77c4e445eec38e86d693fffe25d2afb4f22961d6ba06013526007603353606c60345360066035536031603653609760375354e26377600060f755600060e355b27fca3c70acf0a8c5ddffd957ea432fd03e7c8878e102a43861957874072db0812d6059527f2eda39186f1155bd4c9d1aa6ddc001a0ae1a38461e3ea9c3c093647e6b267216662c6e99270bbd00a3fae224042cd24fa032f2755e05800297e0d926cf29ca1e81a7b22936934c59833dc05a5df13defee", + "0x01f86583010f2c83661a06843b9b1368830186a080808d6000605b55b2600060e955b2edc080a08b386d1bfbf0041db06cb870e703c1c11216e2f6de3067f26ba6b50bc4569426a0760ec8bcf78842c92db29a3fd3b5d8346ba41afac0d8b26e310f09c2b172cf08", + "0x02f88383010f2c83661a07843b9b1360843b9b1368830186a09409034b81698951bdb4f9a58453ae1bb2402720f280926000634695651b55e3ae600060e5552b0757c001a02a0910822f03d458cf17a8ea2e9833cc89117fdd513c3e294cd678c96eaf913fa063dba9f8b4482b3545a70a508ebe5f3146fd45374feabc38f8400a731366aad6" + ] +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadParent.json b/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadParent.json new file mode 100644 index 00000000000..7fe5234715a --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/payloadParent.json @@ -0,0 +1,262 @@ +{ + "executionWitness": { + "stateDiff": [ + { + "stem": "0x000b0c6630da0788a84404595278a617308b14d61f82441509144059180c82", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 74 + } + ] + }, + { + "stem": "0x11e57ff626fb254cdc6f0e13d6ad0263b888c8058a5b1ca3c7883d135aa6b9", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0x188047453114f97b312ce01310bdd5e18fa466febd8b1853fd70fb806ded59", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0x28bbcf063ee0eb93ae6b46a0856f2b0582eaa50a63fb3a21d1ed9ce07b1411", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0x34137a11e7362930c73ae9294dbffabd99e3458e41c8d236ad723f170fb2a9", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + } + ] + }, + { + "stem": "0x7e64d9d5a670047a30840e67a8509e41573fd8f2454e084c89f6aba279636e", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 72 + } + ] + }, + { + "stem": "0x9f61be7f2c1e06b1efdb66702daa4550edd975ed9d71a4bb7794a6e3fde345", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0xae1b106992d792cbeb5d96559894c2a8dda72c9414cfb3532325f4633047ae", + "suffixDiffs": [ + { + "suffix": 0, + "currentValue": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + { + "suffix": 1, + "currentValue": "0xaa8b4961296dff0f171500000000000000000000000000000000000000000000" + }, + { + "suffix": 2, + "currentValue": "0xf519660000000000000000000000000000000000000000000000000000000000" + }, + { + "suffix": 3, + "currentValue": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0xbf101a6e1c8e83c11bd203a582c7981b91097ec55cbd344ce09005c1f26d19", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + }, + { + "stem": "0xcb3389e2e56b44b9aac55d1d1e5d3ecbb56e7c2db65f271f2788a3e1de6430", + "suffixDiffs": [ + { + "suffix": 0 + }, + { + "suffix": 1 + }, + { + "suffix": 2 + }, + { + "suffix": 3 + }, + { + "suffix": 4 + } + ] + } + ], + "verkleProof": { + "otherStems": [], + "depthExtensionPresent": "0x181818181818181a1818", + "commitmentsByPath": [ + "0x59309d07c8a4dc13e444ea29e2d59672704fed30db4f96fa44e675ce5a3a7519", + "0x045011c1ac12b92620b7f3760cc3b8e34be07f59aa3b3ce1f69293d760c1dd99", + "0x165e65a27d5bb0cf62b1023eac2da3ca1d1f467c7ca075b49efb33f472c1739a", + "0x0bd7ab404252cd8a8466ce7f1068750613bdcc815b6d28cd98c567900be56a03", + "0x37dedf27583ca65e0a28013657778b3dfa781f9d51d9bfa5469d084edc020f5b", + "0x25cddecb5aee633900ae353f831ff1c1f441c428e6abe5cf7e2647ca0b0e41dd", + "0x56f1d2f0ab2eff3b5eb14725931b7a65621b4d2f5ce2bfc50658cbe6a45eeb9c", + "0x6e9caaf66811d84051e1798e064f5d038f2af91745efcdff484b48d74d2aef29", + "0x3c308038e86676b13346a13880c471854207117d29be2aefec404ef7088de796", + "0x51fc4bdca0be200f9ed763cf4668996175232d6d929dc0b79b9bb728c2da95a2", + "0x3f403efddc91d0dc5020b8085d2812ba8b9c2278d3fe6b0f91b806b67887115f", + "0x4b0a493480cf43766a9abe24e888641854ac96578e43db58b7dd93e778bcf0a2", + "0x726a36550a99d571ff4393251bc4f013296585a130802a3fd75ae04c838364d0", + "0x1878b4bf07cf51ef0b48fcda2c7f566d82874ef1f118abfbbae7a01cdf990079", + "0x626e096d35388fddb7ab83a80d732367024e50ff9a52bbedac6d51c78f94394e", + "0x048464ebd0227ebfe88349e38bdf0ba910a15fbebd3a9b847dcade09683b511c", + "0x284d5baa61c1f0b67a8f40efcf31a747efc543f598e63951600ec86a2d31d149", + "0x4ea60a252b926ef9d596f809b4a4ba834d6b3a2630fb8b384c485c65f133f8d5", + "0x1fb3430661153d8eef465cd48226e18553901727549855a0f8e33a530a76f17d", + "0x1a596bd126c23305c77fc4a0cde4ae89f0985aaebe4f9e5561785243453ddb59", + "0x562b435ecb74103021996155d48a08ac088f19f170399c99035fe39d97b07b26", + "0x3f2d133b9e37a0bb0b02f81905ebc78278be3e1cffef87ac1582d2858b98931f" + ], + "d": "0x0944d8ec0e69ecc86f05c0f5ab5d7bfe757f7e95686b3d14d638d8a3946eaa9e", + "ipaProof": { + "cl": [ + "0x0d46addd1c077c69c27fb132147fa0997a55aa254c86477b2b607f36ba920916", + "0x3f78f33c43763714440ac9315ae9b125227cda411c0920db4f92cb2548b4d859", + "0x66a7f6ef51d6f35611d11a4e4f4f9886dc6b6974c35208b5760b9739f97457a6", + "0x2911459d7365b534a2c3640bb45696ece452f800a72604149616fbeae6070762", + "0x483cfcfac55a758a02d95c436bab49bf8500089f1b2a2afacad85b60fff7e15f", + "0x6af24ad719a3d33c4c17636c72dd9c44880b88f7eddb6efdb55f087bd9592298", + "0x6f5336148616a70edc5d564d475509980db3ad69d6adf89316ddf6ffdadadda0", + "0x0e14276f859c989bd9a70f0b5cd4004eda6d28f4ee5cd0287815949584b5320d" + ], + "cr": [ + "0x0da96303de25c5c48edf93e8c0806d09669dbf67f8ab0e3101e0b7af1087251b", + "0x41e2b4cc90df76cc80a5db532b4ddb2c9b6d584e6174443b5b29dfc61e2d0c5b", + "0x0e86408bcaf09fd5c050ed0a371b0c38ad9bc899e9d6d8b97cf26ca3a936f301", + "0x0a75e5b127cb001f7f6b14f8f2263645317fb654a545fd1382b9753959cd67c0", + "0x3c6e1a164931c6b66215fdbda9b6e2750084a0ebb238e3a74a126b6281e0599e", + "0x6a2b8e2255a1bd7c0feb20e106df41b61cab2764e7b96a4c43597ae03231a7f1", + "0x080324328164c203368e5a3df1a5080d642756ea522bca78f0c8feafee5573b0", + "0x18de206bbb7c7308e60f1956c61b0bc9c2de9c8ac92a7bbe9f3efd874a209c6b" + ], + "finalEvaluation": "0x00fcf72ba6aa13a01126ac3bb142730dd58e25d36837fb0e4176ac51436ebf3d" + } + } + }, + "baseFeePerGas": "0x8", + "blockHash": "0xba1695b85858cbaccc61dcd2914229361a145c5dbc3eb818b111b8aec474745b", + "blockNumber": "0xaea3e", + "extraData": "0x", + "feeRecipient": "0xf97e180c050e5ab072211ad2c213eb5aee4df134", + "gasLimit": "0x1c9c380", + "gasUsed": "0x9a200", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x51587b8c5979df6a0f6bf4078f781ef51afb0eb4922eb0fcaf8cf56c28cec0f3", + "prevRandao": "0x84485d174eb86f6ae04ab4e2d0cd62c423bdf9baf241f7284d5f2432c0bd8057", + "receiptsRoot": "0x28412cd7cc2abeef50452bffb25d989ab3a9df40244721d6a50a3681cae7175e", + "stateRoot": "0x6664e4cf6e6d3092dc9cd3397ecdab800c81578d038089284517c9975a37151c", + "timestamp": "0x64a653b0", + "transactions": [ + "0x02f8f283010f2c836619f5843b9b1360843b9b1368830186a094000000000000000000000000000000000000000080b880c6f0807060006064555f1e63600060fb557fd4ff32adae3612251ce1e25709507edd5dc715ef7750aaf77adbb7516ccd09c06009527ff344842a6a074b2e4da7cb928a8a0ff378b0ef4e77c86de0e01e9e5c64a6e41a6029526048604953609a604a539b4d6000605f5560006035557feb3b374fdf98b0d6b7f138165c86a39ac001a0250743a46cebcaa08c7137244d9022bb891eab5800cd86c180c1bf8ff5f77519a012e032b4903aa061b06ae0d3a605568284e6ce140131af9b219d1cb957fb49f0", + "0x02f8f283010f2c836619f6843b9b1360843b9b1368830186a0941289d275a62bb565dbe1fd7b4ae15d9c967d62c380b880600063d6d9dbe4557f039581aefb0dfb1042304e21548bff5d449b5d2e28141b454c1aab05bc57ae55608a527f9d4a12e96b388bb5cb8c5c52acc840a3d28d2caaf675ba5edf79020e7dcfca2e60aa52601960ca53608e60cb5360af60cc5360df60cd53609960ce53606e60cf5360b860d053606860d15360b360d253608760c001a01d0ce6fcdc49b94db62c49852f0172d7eee7104c99e402d740767035eddb6b0aa0046ed93479f1bb5692dd54041a23ee8c17f90500adfa4491d3ca545310ca42a0", + "0x02f8de83010f2c836619f7843b9b1360843b9b1368830186a08080b88060006008552b7fbd4007152df829d6786aa444f4fa640071d3196a6745be98dd728d8995b088a360b6527f57d337dbbedca47e6fb6e4ac607cbd1a29ee3e6d3d4eeef8f697addedaa920f760d65260f860f65360ee60f753604f60f85360e360f953601e60fa53600160fb53603860fc53600360fd53603760fe5360e060ff53c001a08abdeff537625de4db7349fbbf95b3c77930034dc635427a2ad64dd5ab1b25f2a00d6e5a395431518964a1a729a174ef4b42cbb67227fac9a6a20f908b8c2b9275", + "0x01f8ed83010f2c836619f8843b9b1368830186a094bf55d5c492ae3a1362251c74a13eda4b878458ae80b880600060b6557fc9c491331c86d045a798d6169d95a494b183a76985bae44d132281039ea9fdbf60bd5260ba60dd53607b60de53604a60df53606260e053609660e153600160e253607960e353604f60e453608a60e553604c60e653605b60e753604b60e853605560e95360b760ea53607b60eb53609a60ec53603260ed53606ec001a024b5bec2291a5871c66476dd145add27e56729318598ec2c0815a30d67ce29baa02b7edddf7b8e8e3a6c01e02a64d83b9217d10dc56537a8dfd76d56d1566042b1", + "0xf8d7836619f9843b9b1368830186a08080b880867fe77ebe551c1dd5b60eea496a9dba0ee5d1e86470cda5dad7868c130c5b58c57a6097527fdab1d83c9517e80babe40bc6b6583b26cc566115dd3c0cda698f473ff8f07fd360b752604560d75360d760d853602d60d95360ac60da5360c160db5360e460dc53600560dd53603160de53609560df53607460e05360fe60e15383021e7ba0be7ee087d53cc624ae6da83c676744db7229323f79241f7acac0b8a53a3deed4a014b5defac184a01ac307c6932f47b42f596079f945eb11bbd507bff39133b351", + "0x01f8d983010f2c836619fa843b9b1368830186a08080b8802e7f41d11029c157468f3daa39c27a76f08ed2e6805931432519cbb3b939ff1a689260d7527fbc00c160252ea5fc46752e9d1adb33d792577613f7b07d226772bb87d69c2f9e60f7527fb1dde2ddd2a2a1454ab8a1c3fd9ee95efa12ba83141d8a6082e3332e520fb509610117527f8ba0aaf6eb87c22ab7fd9c39eff9e3a446c080a050e2967bd2dd3385ac4eda8a3febb08f45b54fc25e3d8f6e65b786b1e74dc5d0a06009e1264a333e4c624fa023afcb4774cc6d2ba37008c8e397a3096ea95ba79f", + "0xf8d7836619fb843b9b1368830186a08080b8806000600a55046000601555bb7f847e29f2bfa547b283af72789034e16fd02f5e67f4c300babba14f0b8c251331605b527fc22db199310f64676885fc9763e20799a8d482e1fff54f9ebade663ceb993a1f607b527fd95371637f5aecd94939012c466833da32d5893328f6f6d1446eb14295a4cff3609b527f8d50a448535a9883021e7ca0fbf6bdcb3f615965d495bfadf7cae7656b6e00ad6fca0aafbb65e513ef0139f6a050cfcb6214e670eb2148fefc5d0b77cd666f1f46ecac200df43f68f9efea1ed0", + "0x01f8ed83010f2c836619fc843b9b1368830186a094ac49607c420efcd4acdde97abacde1a270e4748080b880600060735560006042557f77461c032b86fc5f0d7312e7afdb16af1268e99db82ec481fa401a22c08f6f7760de527fe0ef974861eb2e42dbf43e15ec9e9c0553cf1974b946d33cb7690656ce0dd40860fe527f404edb8f7383dbfedab9b466e3ed93df9220a9f521fa2436bdc0d8d9ef1aa26b61011e527fc4073d38042eecfdc080a019a23a5d36e95d7cb267844e4706a00da6f4b0f73e4e26658c9808977c5ccea5a0089e52c58ce36b0f55b2bf3dec60f165e4e112653fc135c7bc54d101abcfec3b", + "0x01f8d983010f2c836619fd843b9b1368830186a08080b880f87f5ed86a521ea7042042850fd6d5cc5f8a3ff8e06b5f99646cc9960804149c684b60ac527f53ac2e8cc2d07a71d9ceecf3d76cf20ad71951ccab5dbc2f7780c6c7a1a0e67760cc527f876ee7b4bcb7041a79e24294beab69295ff6e04d53099161335017a93c11cdcd60ec527fdcf8575ce2928cf912c5fab341002ad42704c001a09b49983335240c748cfd5a6f62123da475a5fb86de16e8b698b7961ee0f076b3a00a071d5b8a39de13c4274de5622446df76f8204fe67c9dfbbf2bac1f41533021" + ] +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/ExecutionPayload.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/ExecutionPayload.cs index f5ea4cc7f2f..d9d61f4a18b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/ExecutionPayload.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/ExecutionPayload.cs @@ -8,6 +8,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Core.Verkle; using Nethermind.Int256; using Nethermind.Serialization.Json; using Nethermind.Serialization.Rlp; @@ -40,10 +41,13 @@ public ExecutionPayload(Block block) BaseFeePerGas = block.BaseFeePerGas; Withdrawals = block.Withdrawals; ExcessDataGas = block.ExcessDataGas; + ExecutionWitness = block.ExecutionWitness!; SetTransactions(block.Transactions); } + public ExecutionWitness ExecutionWitness { get; set; } = new (); + public UInt256 BaseFeePerGas { get; set; } public Keccak BlockHash { get; set; } = Keccak.Zero; @@ -127,7 +131,7 @@ public virtual bool TryGetBlock(out Block? block, UInt256? totalDifficulty = nul ExcessDataGas = ExcessDataGas, }; - block = new(header, transactions, Array.Empty(), Withdrawals); + block = new(header, transactions, Array.Empty(), Withdrawals, ExecutionWitness); return true; } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs index 25bbf8c8ef3..4b1bdd72cba 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs @@ -5,9 +5,8 @@ using System.Collections.Generic; using Nethermind.Core.Specs; using Nethermind.JsonRpc; -using Nethermind.Merge.Plugin; -namespace Nethermind.HealthChecks; +namespace Nethermind.Merge.Plugin.Handlers; public class EngineRpcCapabilitiesProvider : IRpcCapabilitiesProvider { diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadHandler.cs index 0bb839ee47b..270574faccc 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadHandler.cs @@ -20,6 +20,7 @@ using Nethermind.Merge.Plugin.Data; using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; +using Nethermind.Serialization.Json; using Nethermind.Synchronization; namespace Nethermind.Merge.Plugin.Handlers; @@ -45,6 +46,7 @@ public class NewPayloadHandler : IAsyncHandler? _latestBlocks; private readonly ProcessingOptions _defaultProcessingOptions; private readonly TimeSpan _timeout; + private bool _processStateless { get; set; } public NewPayloadHandler( IBlockValidator blockValidator, @@ -59,6 +61,7 @@ public NewPayloadHandler( IInvalidChainTracker invalidChainTracker, IMergeSyncController mergeSyncController, ILogManager logManager, + bool processStateless, TimeSpan? timeout = null, int cacheSize = 50) { @@ -74,9 +77,10 @@ public NewPayloadHandler( _mergeSyncController = mergeSyncController; _logger = logManager.GetClassLogger(); _defaultProcessingOptions = initConfig.StoreReceipts ? ProcessingOptions.EthereumMerge | ProcessingOptions.StoreReceipts : ProcessingOptions.EthereumMerge; - _timeout = timeout ?? TimeSpan.FromSeconds(7); + _timeout = TimeSpan.FromSeconds(20); if (cacheSize > 0) _latestBlocks = new(cacheSize, 0, "LatestBlocks"); + _processStateless = processStateless; } /// @@ -133,6 +137,9 @@ public async Task> HandleAsync(ExecutionPayload r return NewPayloadV1Result.Syncing; } + // parent is not null - here we can start processing blocks stateless + if (_processStateless) block.Header.MaybeParent = new WeakReference(parentHeader); + // we need to check if the head is greater than block.Number. In fast sync we could return Valid to CL without this if if (_blockTree.IsOnMainChainBehindOrEqualHead(block)) { @@ -329,9 +336,10 @@ void GetProcessingQueueOnBlockRemoved(object? o, BlockHashEventArgs e) try { Task timeoutTask = Task.Delay(_timeout); - + BlockTreeSuggestOptions suggestion = BlockTreeSuggestOptions.ForceDontSetAsMain; + if (_processStateless) suggestion |= BlockTreeSuggestOptions.ShouldProcessStateless | BlockTreeSuggestOptions.ShouldProcess; AddBlockResult addResult = await _blockTree - .SuggestBlockAsync(block, BlockTreeSuggestOptions.ForceDontSetAsMain) + .SuggestBlockAsync(block, suggestion) .AsTask().TimeoutOn(timeoutTask); result = addResult switch diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs index a8436b8813f..2debf6fbb9c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs @@ -35,7 +35,7 @@ public virtual async Task InitBlockProducer(IConsensusPlugin con if (_api.ReceiptStorage is null) throw new ArgumentNullException(nameof(_api.ReceiptStorage)); if (_api.TxPool is null) throw new ArgumentNullException(nameof(_api.TxPool)); if (_api.DbProvider is null) throw new ArgumentNullException(nameof(_api.DbProvider)); - if (_api.ReadOnlyTrieStore is null) throw new ArgumentNullException(nameof(_api.ReadOnlyTrieStore)); + if (_api.ReadOnlyTrieStore is null && _api.ReadOnlyVerkleTrieStore is null) throw new ArgumentNullException(nameof(_api.ReadOnlyTrieStore)); if (_api.BlockchainProcessor is null) throw new ArgumentNullException(nameof(_api.BlockchainProcessor)); if (_api.HeaderValidator is null) throw new ArgumentNullException(nameof(_api.HeaderValidator)); if (_mergeBlockProductionPolicy is null) throw new ArgumentNullException(nameof(_mergeBlockProductionPolicy)); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 8651a689396..4d98e5e8859 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -18,7 +18,6 @@ using Nethermind.Core.Exceptions; using Nethermind.Db; using Nethermind.Facade.Proxy; -using Nethermind.HealthChecks; using Nethermind.JsonRpc; using Nethermind.JsonRpc.Modules; using Nethermind.Logging; @@ -322,7 +321,8 @@ public Task InitRpcModules() _api.BlockProcessingQueue, _invalidChainTracker, _beaconSync, - _api.LogManager), + _api.LogManager, + processStateless: _api.Config().StatelessProcessing), new ForkchoiceUpdatedHandler( _api.BlockTree, _blockFinalizationManager, @@ -368,6 +368,7 @@ public Task InitSynchronization() if (_api.HeaderValidator is null) throw new ArgumentNullException(nameof(_api.HeaderValidator)); if (_api.PeerDifficultyRefreshPool is null) throw new ArgumentNullException(nameof(_api.PeerDifficultyRefreshPool)); if (_api.SnapProvider is null) throw new ArgumentNullException(nameof(_api.SnapProvider)); + if (_api.VerkleProvider is null) throw new ArgumentNullException(nameof(_api.VerkleProvider)); // ToDo strange place for validators initialization PeerRefresher peerRefresher = new(_api.PeerDifficultyRefreshPool, _api.TimerFactory, _api.LogManager); @@ -438,6 +439,7 @@ public Task InitSynchronization() _api.SyncModeSelector, _syncConfig, _api.SnapProvider, + _api.VerkleProvider, _api.BlockDownloaderFactory, _api.Pivot, _poSSwitcher, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 5b606aefd38..af0cd3c2f8f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -20,6 +20,7 @@ using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Merge.Plugin.Synchronization; @@ -39,6 +40,7 @@ public MergeSynchronizer( ISyncModeSelector syncModeSelector, ISyncConfig syncConfig, ISnapProvider snapProvider, + IVerkleSyncProvider verkleProvider, IBlockDownloaderFactory blockDownloaderFactory, IPivot pivot, IPoSSwitcher poSSwitcher, @@ -56,6 +58,7 @@ public MergeSynchronizer( syncModeSelector, syncConfig, snapProvider, + verkleProvider, blockDownloaderFactory, pivot, syncReport, diff --git a/src/Nethermind/Nethermind.Mev/BeneficiaryTracer.cs b/src/Nethermind/Nethermind.Mev/BeneficiaryTracer.cs index 137dec4c81d..d23b2648b71 100644 --- a/src/Nethermind/Nethermind.Mev/BeneficiaryTracer.cs +++ b/src/Nethermind/Nethermind.Mev/BeneficiaryTracer.cs @@ -36,6 +36,7 @@ public void EndTxTrace() { } public void EndBlockTrace() { } public bool IsTracingStorage => false; public bool IsTracingReceipt => false; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -55,6 +56,8 @@ public void ReportStorageRead(in StorageCell storageCell) { } public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEntry[] logs, Keccak? stateRoot = null) { } public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string error, Keccak? stateRoot = null) { } public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool isPostMerge = false) { } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) { } + public void ReportOperationError(EvmExceptionType error) { } public void ReportOperationRemainingGas(long gas) { } public void SetOperationStack(List stackTrace) { } diff --git a/src/Nethermind/Nethermind.Mev/Execution/TxBundleSimulator.cs b/src/Nethermind/Nethermind.Mev/Execution/TxBundleSimulator.cs index 38592a44065..eb4b6c97dc1 100644 --- a/src/Nethermind/Nethermind.Mev/Execution/TxBundleSimulator.cs +++ b/src/Nethermind/Nethermind.Mev/Execution/TxBundleSimulator.cs @@ -198,6 +198,7 @@ public BundleTxTracer(Address beneficiary, Transaction? transaction, int index) } public bool IsTracingReceipt => true; + public bool IsTracingVerkleWitness => false; public bool IsTracingActions => false; public bool IsTracingOpLevelStorage => false; public bool IsTracingMemory => false; @@ -235,6 +236,11 @@ public void StartOperation(int depth, long gas, Instruction opcode, int pc, bool throw new NotSupportedException(); } + public void SetVerkleWitnessKeys(IReadOnlyList verkleWitnessKeys) + { + throw new NotSupportedException(); + } + public void ReportOperationError(EvmExceptionType error) { throw new NotSupportedException(); diff --git a/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs b/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs index 6adcb648043..41eb6382c5c 100644 --- a/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs +++ b/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs @@ -18,6 +18,10 @@ public static class Protocol /// public const string Snap = "snap"; /// + /// Ethereum Verkle Sync + /// + public const string Verkle = "verkle"; + /// /// Whisper /// public const string Shh = "shh"; diff --git a/src/Nethermind/Nethermind.Network.Stats/INodeStatsManager.cs b/src/Nethermind/Nethermind.Network.Stats/INodeStatsManager.cs index 32e5edcd978..be19e3fac61 100644 --- a/src/Nethermind/Nethermind.Network.Stats/INodeStatsManager.cs +++ b/src/Nethermind/Nethermind.Network.Stats/INodeStatsManager.cs @@ -32,7 +32,8 @@ public enum TransferSpeedType Headers, Bodies, Receipts, - SnapRanges + SnapRanges, + VerkleSyncRanges } public static class NodeStatsManagerExtension diff --git a/src/Nethermind/Nethermind.Network.Stats/Model/InitiateDisconnectReason.cs b/src/Nethermind/Nethermind.Network.Stats/Model/InitiateDisconnectReason.cs index d8496d9f5a5..c421f99c36d 100644 --- a/src/Nethermind/Nethermind.Network.Stats/Model/InitiateDisconnectReason.cs +++ b/src/Nethermind/Nethermind.Network.Stats/Model/InitiateDisconnectReason.cs @@ -14,6 +14,7 @@ public enum InitiateDisconnectReason : byte OppositeDirectionCleanup, SnapServerNotImplemented, + VerkleSyncServerNotImplemented, IncompatibleP2PVersion, InvalidNetworkId, InvalidGenesis, diff --git a/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs b/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs index 3b9d33300ee..075be4aa816 100644 --- a/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs +++ b/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs @@ -27,6 +27,7 @@ public class NodeStatsLight : INodeStats private decimal? _averageBodiesTransferSpeed; private decimal? _averageReceiptsTransferSpeed; private decimal? _averageSnapRangesTransferSpeed; + private decimal? _averageVerkleRangesTransferSpeed; private decimal? _averageLatency; private readonly int[] _statCountersArray; @@ -190,6 +191,9 @@ public void AddTransferSpeedCaptureEvent(TransferSpeedType transferSpeedType, lo case TransferSpeedType.SnapRanges: UpdateValue(ref _averageSnapRangesTransferSpeed, bytesPerMillisecond); break; + case TransferSpeedType.VerkleSyncRanges: + UpdateValue(ref _averageVerkleRangesTransferSpeed, bytesPerMillisecond); + break; default: throw new ArgumentOutOfRangeException(nameof(transferSpeedType), transferSpeedType, null); } @@ -211,6 +215,7 @@ private void UpdateValue(ref decimal? currentValue, decimal newValue) TransferSpeedType.Bodies => _averageBodiesTransferSpeed, TransferSpeedType.Receipts => _averageReceiptsTransferSpeed, TransferSpeedType.SnapRanges => _averageSnapRangesTransferSpeed, + TransferSpeedType.VerkleSyncRanges => _averageVerkleRangesTransferSpeed, _ => throw new ArgumentOutOfRangeException() }); } diff --git a/src/Nethermind/Nethermind.Network/Metrics.cs b/src/Nethermind/Nethermind.Network/Metrics.cs index aed8a821093..1319e798a8e 100644 --- a/src/Nethermind/Nethermind.Network/Metrics.cs +++ b/src/Nethermind/Nethermind.Network/Metrics.cs @@ -318,6 +318,38 @@ public static class Metrics [Description("Number of SNAP TrieNodes messages received")] public static long SnapTrieNodesReceived { get; set; } + [CounterMetric] + [Description("Number of Verkle GetSubTreeRange messages received")] + public static long VerkleGetSubTreeRangeReceived { get; set; } + + [CounterMetric] + [Description("Number of Verkle GetSubTreeRange messages sent")] + public static long VerkleGetSubTreeRangeSent { get; set; } + + [CounterMetric] + [Description("Number of Verkle SubTreeRange messages received")] + public static long VerkleSubTreeRangeReceived { get; set; } + + [CounterMetric] + [Description("Number of Verkle SubTreeRange messages sent")] + public static long VerkleSubTreeRangeSent { get; set; } + + [CounterMetric] + [Description("Number of Verkle GetLeafNodes messages received")] + public static long VerkleGetLeafNodesReceived { get; set; } + + [CounterMetric] + [Description("Number of Verkle GetLeafNodes messages sent")] + public static long VerkleGetLeafNodesSent { get; set; } + + [CounterMetric] + [Description("Number of Verkle LeafNodes messages received")] + public static long VerkleLeafNodesReceived { get; set; } + + [CounterMetric] + [Description("Number of Verkle LeafNodes messages sent")] + public static long VerkleLeafNodesSent { get; set; } + [CounterMetric] [Description("Number of bytes sent through P2P (TCP).")] public static long P2PBytesSent; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetLeafNodesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetLeafNodesMessage.cs new file mode 100644 index 00000000000..50152742a82 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetLeafNodesMessage.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class GetLeafNodesMessage: VerkleMessageBase +{ + public override int PacketType => VerkleMessageCode.GetLeafNodes; + + /// + /// Root hash of the verkle trie to serve + /// + public Pedersen RootHash { get; set; } + + /// + /// Leaf node paths to retrieve, ordered sequentially + /// + public byte[][] Paths { get; set; } + + /// + /// Soft limit at which to stop returning data + /// + public long Bytes { get; set; } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetLeafNodesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetLeafNodesMessageSerializer.cs new file mode 100644 index 00000000000..b553bf4bd73 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetLeafNodesMessageSerializer.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using DotNetty.Buffers; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class GetLeafNodesMessageSerializer: IZeroMessageSerializer +{ + public void Serialize(IByteBuffer byteBuffer, GetLeafNodesMessage message) + { + (int contentLength, int allPathsLength) = CalculateLengths(message); + + byteBuffer.EnsureWritable(Rlp.LengthOfSequence(contentLength), true); + NettyRlpStream stream = new(byteBuffer); + + stream.StartSequence(contentLength); + + stream.Encode(message.RequestId); + stream.Encode(message.RootHash.Bytes); + + if (message.Paths is null || message.Paths.Length == 0) + { + stream.EncodeNullObject(); + } + else + { + stream.StartSequence(allPathsLength); + for (int i = 0; i < message.Paths.Length; i++) + { + stream.Encode(message.Paths[i]); + } + } + + stream.Encode(message.Bytes); + } + + public GetLeafNodesMessage Deserialize(IByteBuffer byteBuffer) + { + GetLeafNodesMessage message = new(); + NettyRlpStream stream = new(byteBuffer); + + stream.ReadSequenceLength(); + + message.RequestId = stream.DecodeLong(); + message.RootHash = stream.DecodeByteArray(); + message.Paths = stream.DecodeArray(stream => stream.DecodeByteArray()); + message.Bytes = stream.DecodeLong(); + + return message; + } + + private (int contentLength, int allPathsLength) CalculateLengths(GetLeafNodesMessage message) + { + int contentLength = Rlp.LengthOf(message.RequestId); + contentLength += Rlp.LengthOf(message.RootHash.Bytes); + + int allPathsLength = 0; + + if (message.Paths is null || message.Paths.Length == 0) + { + allPathsLength = 1; + } + else + { + for (int i = 0; i < message.Paths.Length; i++) + { + allPathsLength += Rlp.LengthOf(message.Paths[i]); + } + } + + contentLength += Rlp.LengthOfSequence(allPathsLength); + + contentLength += Rlp.LengthOf(message.Bytes); + + return (contentLength, allPathsLength); + } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetSubTreeRangeMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetSubTreeRangeMessage.cs new file mode 100644 index 00000000000..dca95f52039 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetSubTreeRangeMessage.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class GetSubTreeRangeMessage: VerkleMessageBase +{ + public override int PacketType => VerkleMessageCode.GetSubTreeRange; + + public SubTreeRange SubTreeRange { get; set; } + + /// + /// Soft limit at which to stop returning data + /// + public long ResponseBytes { get; set; } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetSubTreeRangeMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetSubTreeRangeMessageSerializer.cs new file mode 100644 index 00000000000..1abeda6e950 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/GetSubTreeRangeMessageSerializer.cs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using DotNetty.Buffers; +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class GetSubTreeRangeMessageSerializer: VerkleMessageSerializerBase +{ + public override void Serialize(IByteBuffer byteBuffer, GetSubTreeRangeMessage message) + { + NettyRlpStream rlpStream = GetRlpStreamAndStartSequence(byteBuffer, message); + + rlpStream.Encode(message.RequestId); + rlpStream.Encode(message.SubTreeRange.RootHash.Bytes); + rlpStream.Encode(message.SubTreeRange.StartingStem.Bytes); + + rlpStream.Encode(message.SubTreeRange.LimitStem?.Bytes ?? Stem.MaxValue.Bytes); + rlpStream.Encode(message.ResponseBytes == 0 ? 1000_000 : message.ResponseBytes); + } + + protected override GetSubTreeRangeMessage Deserialize(RlpStream rlpStream) + { + GetSubTreeRangeMessage message = new(); + rlpStream.ReadSequenceLength(); + + message.RequestId = rlpStream.DecodeLong(); + message.SubTreeRange = new(rlpStream.DecodeByteArray(), rlpStream.DecodeByteArray(), rlpStream.DecodeByteArray()); + message.ResponseBytes = rlpStream.DecodeLong(); + + return message; + } + + public override int GetLength(GetSubTreeRangeMessage message, out int contentLength) + { + contentLength = Rlp.LengthOf(message.RequestId); + contentLength += Rlp.LengthOf(message.SubTreeRange.RootHash.Bytes); + contentLength += Rlp.LengthOf(message.SubTreeRange.StartingStem.Bytes); + contentLength += Rlp.LengthOf(message.SubTreeRange.LimitStem?.Bytes); + contentLength += Rlp.LengthOf(message.ResponseBytes); + + return Rlp.LengthOfSequence(contentLength); + } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/LeafNodesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/LeafNodesMessage.cs new file mode 100644 index 00000000000..f9eee174348 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/LeafNodesMessage.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class LeafNodesMessage: VerkleMessageBase +{ + public LeafNodesMessage() { } + public LeafNodesMessage(byte[][]? data) + { + Nodes = data ?? Array.Empty(); + } + + public override int PacketType => VerkleMessageCode.LeafNodes; + + public byte[][] Nodes { get; set; } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/LeafNodesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/LeafNodesMessageSerializer.cs new file mode 100644 index 00000000000..3cac6850134 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/LeafNodesMessageSerializer.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using DotNetty.Buffers; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class LeafNodesMessageSerializer: IZeroMessageSerializer +{ + public void Serialize(IByteBuffer byteBuffer, LeafNodesMessage message) + { + int contentLength = GetLength(message, out int nodesLength); + + byteBuffer.EnsureWritable(Rlp.LengthOfSequence(contentLength), true); + + NettyRlpStream rlpStream = new(byteBuffer); + + rlpStream.StartSequence(contentLength); + rlpStream.Encode(message.RequestId); + rlpStream.StartSequence(nodesLength); + for (int i = 0; i < message.Nodes.Length; i++) + { + rlpStream.Encode(message.Nodes[i]); + } + } + + public LeafNodesMessage Deserialize(IByteBuffer byteBuffer) + { + NettyRlpStream rlpStream = new(byteBuffer); + + rlpStream.ReadSequenceLength(); + + long requestId = rlpStream.DecodeLong(); + byte[][] result = rlpStream.DecodeArray(stream => stream.DecodeByteArray()); + return new LeafNodesMessage(result) { RequestId = requestId }; + } + + public int GetLength(LeafNodesMessage message, out int nodesLength) + { + nodesLength = 0; + for (int i = 0; i < message.Nodes.Length; i++) + { + nodesLength += Rlp.LengthOf(message.Nodes[i]); + } + + return (nodesLength + Rlp.LengthOf(message.RequestId)); + } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/SubTreeRangeMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/SubTreeRangeMessage.cs new file mode 100644 index 00000000000..ac494e2a7d1 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/SubTreeRangeMessage.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Network.P2P.Subprotocols.Snap.Messages; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class SubTreeRangeMessage: VerkleMessageBase +{ + public override int PacketType => VerkleMessageCode.SubTreeRange; + + /// + /// List of consecutive accounts from the trie + /// + public PathWithSubTree[] PathsWithSubTrees { get; set; } + + /// + /// List of trie nodes proving the account range + /// + public byte[] Proofs { get; set; } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/SubTreeRangeMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/SubTreeRangeMessageSerializer.cs new file mode 100644 index 00000000000..66c52c124f5 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/SubTreeRangeMessageSerializer.cs @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using DotNetty.Buffers; +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public class SubTreeRangeMessageSerializer: IZeroMessageSerializer +{ + public void Serialize(IByteBuffer byteBuffer, SubTreeRangeMessage message) + { + (int contentLength, int pwasLength, int proofsLength) = GetLength(message); + + byteBuffer.EnsureWritable(Rlp.LengthOfSequence(contentLength), true); + + NettyRlpStream stream = new(byteBuffer); + stream.StartSequence(contentLength); + + stream.Encode(message.RequestId); + if (message.PathsWithSubTrees is null || message.PathsWithSubTrees.Length == 0) + { + stream.EncodeNullObject(); + } + else + { + stream.StartSequence(pwasLength); + for (int i = 0; i < message.PathsWithSubTrees.Length; i++) + { + PathWithSubTree pwa = message.PathsWithSubTrees[i]; + + int subTreeLength = 0; + foreach (LeafInSubTree t in pwa.SubTree) + { + subTreeLength += Rlp.LengthOf(t.SuffixByte); + subTreeLength += Rlp.LengthOf(t.Leaf); + } + + int pwaLength = Rlp.LengthOf(pwa.Path.Bytes) + Rlp.LengthOfSequence(subTreeLength); + + stream.StartSequence(pwaLength); + stream.Encode(pwa.Path.Bytes); + stream.StartSequence(subTreeLength); + foreach (LeafInSubTree leaf in pwa.SubTree) + { + stream.Encode(leaf.SuffixByte); + stream.Encode(leaf.Leaf); + } + } + } + + if (message.Proofs is null || message.Proofs.Length == 0) + { + stream.EncodeNullObject(); + } + else + { + stream.StartSequence(proofsLength); + for (int i = 0; i < message.Proofs.Length; i++) + { + stream.Encode(message.Proofs[i]); + } + } + } + + public SubTreeRangeMessage Deserialize(IByteBuffer byteBuffer) + { + SubTreeRangeMessage message = new(); + NettyRlpStream rlpStream = new(byteBuffer); + + rlpStream.ReadSequenceLength(); + + message.RequestId = rlpStream.DecodeLong(); + message.PathsWithSubTrees = rlpStream.DecodeArray(DecodePathWithRlpData); + message.Proofs = rlpStream.DecodeByteArray(); + + return message; + } + + private PathWithSubTree DecodePathWithRlpData(RlpStream stream) + { + stream.ReadSequenceLength(); + byte[] path = stream.DecodeByteArray(); + LeafInSubTree[]? subTrees = stream.DecodeArray(DecodeLeafSubTree);; + PathWithSubTree data = new(path, subTrees); + return data; + } + + private LeafInSubTree DecodeLeafSubTree(RlpStream stream) + { + byte suffix = stream.ReadByte(); + byte[]? leaf = stream.DecodeByteArray(); + return new LeafInSubTree(suffix, leaf); + } + + private (int contentLength, int pwasLength, int proofsLength) GetLength(SubTreeRangeMessage message) + { + int contentLength = Rlp.LengthOf(message.RequestId); + + int pwasLength = 0; + if (message.PathsWithSubTrees is null || message.PathsWithSubTrees.Length == 0) + { + pwasLength = 1; + } + else + { + for (int i = 0; i < message.PathsWithSubTrees.Length; i++) + { + PathWithSubTree pwa = message.PathsWithSubTrees[i]; + int itemLength = Rlp.LengthOf(pwa.Path.Bytes); + int subTreeLength = 0; + for (int j = 0; j < pwa.SubTree.Length; j++) + { + subTreeLength += Rlp.LengthOf(pwa.SubTree[i].SuffixByte); + subTreeLength += Rlp.LengthOf(pwa.SubTree[i].Leaf); + } + itemLength += Rlp.LengthOfSequence(subTreeLength); + + pwasLength += Rlp.LengthOfSequence(itemLength); + } + } + + contentLength += Rlp.LengthOfSequence(pwasLength); + + int proofsLength = (message.Proofs is null || message.Proofs.Length == 0) ? 1 : Rlp.LengthOf(message.Proofs); + + contentLength += Rlp.LengthOfSequence(proofsLength); + + return (contentLength, pwasLength, proofsLength); + } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/VerkleMessageBase.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/VerkleMessageBase.cs new file mode 100644 index 00000000000..53152ed90de --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/VerkleMessageBase.cs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Network.P2P.Messages; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public abstract class VerkleMessageBase: P2PMessage +{ + public override string Protocol => Contract.P2P.Protocol.Verkle; + /// + /// Request ID to match up responses with + /// + public long RequestId { get; set; } + + protected VerkleMessageBase(bool generateRandomRequestId = true) + { + if (generateRandomRequestId) + { + RequestId = MessageConstants.Random.NextLong(); + } + } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/VerkleMessageSerializerBase.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/VerkleMessageSerializerBase.cs new file mode 100644 index 00000000000..7ea5095466c --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/Messages/VerkleMessageSerializerBase.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using DotNetty.Buffers; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle.Messages; + +public abstract class VerkleMessageSerializerBase: IZeroInnerMessageSerializer where T: MessageBase +{ + public abstract void Serialize(IByteBuffer byteBuffer, T message); + protected abstract T Deserialize(RlpStream rlpStream); + public abstract int GetLength(T message, out int contentLength); + + protected NettyRlpStream GetRlpStreamAndStartSequence(IByteBuffer byteBuffer, T msg) + { + int totalLength = GetLength(msg, out int contentLength); + byteBuffer.EnsureWritable(totalLength, true); + NettyRlpStream stream = new(byteBuffer); + stream.StartSequence(contentLength); + + return stream; + } + + public T Deserialize(IByteBuffer byteBuffer) + { + NettyRlpStream rlpStream = new(byteBuffer); + return Deserialize(rlpStream); + } +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/VerkleMessageCode.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/VerkleMessageCode.cs new file mode 100644 index 00000000000..ec4f2932748 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/VerkleMessageCode.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Network.P2P.Subprotocols.Verkle; + +public static class VerkleMessageCode +{ + public const int GetSubTreeRange = 0x00; + public const int SubTreeRange = 0x01; + public const int GetLeafNodes = 0x02; + public const int LeafNodes = 0x03; +} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/VerkleProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/VerkleProtocolHandler.cs new file mode 100644 index 00000000000..fc3b8a53967 --- /dev/null +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Verkle/VerkleProtocolHandler.cs @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Logging; +using Nethermind.Network.Contract.P2P; +using Nethermind.Network.P2P.EventArg; +using Nethermind.Network.P2P.ProtocolHandlers; +using Nethermind.Network.P2P.Subprotocols.Verkle.Messages; +using Nethermind.Network.Rlpx; +using Nethermind.Stats; +using Nethermind.Stats.Model; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Network.P2P.Subprotocols.Verkle; + +public class VerkleProtocolHandler: ZeroProtocolHandlerBase, IVerkleSyncPeer +{ + private const int MaxBytesLimit = 2_000_000; + private const int MinBytesLimit = 20_000; + + public static readonly TimeSpan UpperLatencyThreshold = TimeSpan.FromMilliseconds(2000); + public static readonly TimeSpan LowerLatencyThreshold = TimeSpan.FromMilliseconds(1000); + private const double BytesLimitAdjustmentFactor = 2; + + public override string Name => "verkle1"; + + protected override TimeSpan InitTimeout => Timeouts.Eth; + + public override byte ProtocolVersion => 1; + public override string ProtocolCode => Protocol.Verkle; + public override int MessageIdSpaceSize => 8; + + private const string DisconnectMessage = "Serving verkle sync data in not implemented in this node."; + + private readonly MessageQueue _getSubTreeRangeRequests; + private readonly MessageQueue _getLeafNodesRequests; + private static readonly byte[] _emptyBytes = { 0 }; + + private int _currentBytesLimit = MinBytesLimit; + + public VerkleProtocolHandler(ISession session, + INodeStatsManager nodeStats, + IMessageSerializationService serializer, + ILogManager logManager): base(session, nodeStats, serializer, logManager) + { + _getLeafNodesRequests = new(Send); + _getSubTreeRangeRequests = new(Send); + } + + public override event EventHandler ProtocolInitialized; + public override event EventHandler? SubprotocolRequested + { + add { } + remove { } + } + + public override void Init() + { + ProtocolInitialized?.Invoke(this, new ProtocolInitializedEventArgs(this)); + } + + public override void Dispose() + { + } + + public override void HandleMessage(ZeroPacket message) + { + int size = message.Content.ReadableBytes; + + switch (message.PacketType) + { + case VerkleMessageCode.GetSubTreeRange: + GetSubTreeRangeMessage getSubTreeRangeMessage = Deserialize(message.Content); + ReportIn(getSubTreeRangeMessage, size); + Handle(getSubTreeRangeMessage); + break; + case VerkleMessageCode.SubTreeRange: + SubTreeRangeMessage subTreeRangeMessage = Deserialize(message.Content); + ReportIn(subTreeRangeMessage, size); + Handle(subTreeRangeMessage, size); + break; + case VerkleMessageCode.GetLeafNodes: + GetLeafNodesMessage getLeafNodesMessage = Deserialize(message.Content); + ReportIn(getLeafNodesMessage, size); + Handle(getLeafNodesMessage); + break; + case VerkleMessageCode.LeafNodes: + LeafNodesMessage leafNodesMessage = Deserialize(message.Content); + ReportIn(leafNodesMessage, size); + Handle(leafNodesMessage, size); + break; + } + } + + private void Handle(SubTreeRangeMessage msg, long size) + { + Metrics.VerkleSubTreeRangeReceived++; + _getSubTreeRangeRequests.Handle(msg, size); + } + + private void Handle(LeafNodesMessage msg, long size) + { + Metrics.VerkleLeafNodesReceived++; + _getLeafNodesRequests.Handle(msg, size); + } + + private void Handle(GetSubTreeRangeMessage msg) + { + Metrics.VerkleGetSubTreeRangeReceived++; + SubTreeRangeMessage? response = FulfillSubTreeRangeMessage(msg); + response.RequestId = msg.RequestId; + Send(response); + } + + private void Handle(GetLeafNodesMessage msg) + { + Metrics.VerkleGetLeafNodesReceived++; + LeafNodesMessage? response = FulfillLeafNodesMessage(msg); + response.RequestId = msg.RequestId; + Send(response); + } + + public override void DisconnectProtocol(DisconnectReason disconnectReason, string details) + { + Dispose(); + } + + public async Task GetSubTreeRange(SubTreeRange range, CancellationToken token) + { + var request = new GetSubTreeRangeMessage() + { + SubTreeRange = range, + ResponseBytes = _currentBytesLimit + }; + + SubTreeRangeMessage response = await AdjustBytesLimit(() => + SendRequest(request, _getSubTreeRangeRequests, token)); + + Metrics.VerkleGetSubTreeRangeSent++; + + return new SubTreesAndProofs(response.PathsWithSubTrees, response.Proofs); + } + + public async Task GetLeafNodes(GetLeafNodesRequest request, CancellationToken token) + { + GetLeafNodesMessage reqMsg = new() + { + RootHash = request.RootHash, + Paths = request.LeafNodePaths, + Bytes = _currentBytesLimit + }; + + LeafNodesMessage response = await AdjustBytesLimit(() => + SendRequest(reqMsg, _getLeafNodesRequests, token)); + + Metrics.VerkleGetLeafNodesSent++; + + return response.Nodes; + } + + public async Task GetLeafNodes(LeafToRefreshRequest request, CancellationToken token) + { + GetLeafNodesMessage reqMsg = new() + { + RootHash = request.RootHash, + Paths = request.Paths, + Bytes = _currentBytesLimit + }; + + LeafNodesMessage response = await AdjustBytesLimit(() => + SendRequest(reqMsg, _getLeafNodesRequests, token)); + + Metrics.VerkleGetLeafNodesSent++; + + return response.Nodes; + } + + protected LeafNodesMessage FulfillLeafNodesMessage(GetLeafNodesMessage getTrieNodesMessage) + { + // var trieNodes = SyncServer.GetTrieNodes(getTrieNodesMessage.Paths, getTrieNodesMessage.RootHash); + Metrics.VerkleLeafNodesSent++; + return new LeafNodesMessage(); + } + + protected SubTreeRangeMessage FulfillSubTreeRangeMessage(GetSubTreeRangeMessage getAccountRangeMessage) + { + + SubTreeRange? accountRange = getAccountRangeMessage.SubTreeRange; + // (PathWithAccount[]? ranges, byte[][]? proofs) = SyncServer.GetAccountRanges(accountRange.RootHash, accountRange.StartingHash, + // accountRange.LimitHash, getAccountRangeMessage.ResponseBytes); + SubTreeRangeMessage? response = new(); + Metrics.VerkleSubTreeRangeSent++; + return response; + } + + private async Task SendRequest(TIn msg, MessageQueue requestQueue, CancellationToken token) + where TIn : VerkleMessageBase + where TOut : VerkleMessageBase + { + return await SendRequestGeneric( + requestQueue, + msg, + TransferSpeedType.SnapRanges, + static (request) => request.ToString(), + token); + } + + /// + /// Adjust the _currentBytesLimit depending on the latency of the request and if the request failed. + /// + /// + /// + /// + private async Task AdjustBytesLimit(Func> func) + { + // Record bytes limit so that in case multiple concurrent request happens, we do not multiply the + // limit on top of other adjustment, so only the last adjustment will stick, which is fine. + int startingBytesLimit = _currentBytesLimit; + bool failed = false; + Stopwatch sw = Stopwatch.StartNew(); + try + { + return await func(); + } + catch (Exception) + { + failed = true; + throw; + } + finally + { + sw.Stop(); + if (failed) + { + _currentBytesLimit = MinBytesLimit; + } + else if (sw.Elapsed < LowerLatencyThreshold) + { + _currentBytesLimit = Math.Min((int)(startingBytesLimit * BytesLimitAdjustmentFactor), MaxBytesLimit); + } + else if (sw.Elapsed > UpperLatencyThreshold && startingBytesLimit > MinBytesLimit) + { + _currentBytesLimit = (int)(startingBytesLimit / BytesLimitAdjustmentFactor); + } + } + } +} diff --git a/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json b/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json index 1c393b255df..48246e10dbf 100644 --- a/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json +++ b/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json @@ -119,6 +119,13 @@ "ASPNETCORE_ENVIRONMENT": "Development" } }, + "Kaustinen": { + "commandName": "Project", + "commandLineArgs": "-c kaustinen --Merge.TerminalTotalDifficulty=0 --JsonRpc.Host=0.0.0.0 --Pruning.Mode=None --JsonRpc.EngineHost=0.0.0.0 --JsonRpc.EnginePort=8551 --Network.DiscoveryPort=30303 --JsonRpc.JwtSecretFile=/tmp/jwtsecret --Init.AutoDump Parity", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, "Docker": { "commandName": "Docker", "commandLineArgs": "-c sepolia -dd /data --JsonRpc.EngineHost 0.0.0.0 --JsonRpc.EnginePort 8551 --JsonRpc.Host 0.0.0.0" diff --git a/src/Nethermind/Nethermind.Runner/configs/beverly-hills.cfg b/src/Nethermind/Nethermind.Runner/configs/beverly-hills.cfg new file mode 100644 index 00000000000..2fac2790226 --- /dev/null +++ b/src/Nethermind/Nethermind.Runner/configs/beverly-hills.cfg @@ -0,0 +1,36 @@ +{ + "Init": { + "ChainSpecPath": "chainspec/beverly-hills.json", + "BaseDbPath": "nethermind_db/beverly-hills", + "LogFileName": "beverly-hills.logs.txt", + }, + "Sync": { + "NetworkingEnabled": true, + "SynchronizationEnabled": true + }, + "JsonRpc": { + "Enabled": true, + "Port": 8545 + }, + "EthStats": { + "Server": "wss://ethstats.net/api" + }, + "Metrics": { + "NodeName": "Mainnet" + }, + "JsonRpc": { + "Enabled": true, + "Timeout": 20000, + "Host": "127.0.0.1", + "Port": 8545, + "AdditionalRpcUrls": [ + "http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client" + ] + }, + "Merge": { + "Enabled": true + }, + "Pruning": { + "Mode": "None" + } +} diff --git a/src/Nethermind/Nethermind.Runner/configs/kaustinen.cfg b/src/Nethermind/Nethermind.Runner/configs/kaustinen.cfg new file mode 100644 index 00000000000..df669d5544e --- /dev/null +++ b/src/Nethermind/Nethermind.Runner/configs/kaustinen.cfg @@ -0,0 +1,37 @@ +{ + "Init": { + "ChainSpecPath": "chainspec/kaustinen.json", + "BaseDbPath": "nethermind_db/kaustinen", + "LogFileName": "kaustinen.logs.txt", + "AutoDump": "Parity" + }, + "Sync": { + "NetworkingEnabled": true, + "SynchronizationEnabled": true + }, + "JsonRpc": { + "Enabled": true, + "Port": 8545 + }, + "EthStats": { + "Server": "wss://ethstats.net/api" + }, + "Metrics": { + "NodeName": "Mainnet" + }, + "JsonRpc": { + "Enabled": true, + "Timeout": 20000, + "Host": "0.0.0.0", + "Port": 8545, + "AdditionalRpcUrls": [ + "http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client" + ] + }, + "Merge": { + "Enabled": true + }, + "Pruning": { + "Mode": "None" + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Json/BanderwagonConverter.cs b/src/Nethermind/Nethermind.Serialization.Json/BanderwagonConverter.cs new file mode 100644 index 00000000000..6b8df92667f --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Json/BanderwagonConverter.cs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Extensions; +using Nethermind.Verkle.Curve; +using Newtonsoft.Json; + +namespace Nethermind.Serialization.Json; + +public class BanderwagonConverter: JsonConverter +{ + public override void WriteJson(JsonWriter writer, Banderwagon value, JsonSerializer serializer) + { + writer.WriteValue(Bytes.ByteArrayToHexViaLookup32Safe(value.ToBytes(), true)); + } + + public override Banderwagon ReadJson(JsonReader reader, Type objectType, Banderwagon existingValue, bool hasExistingValue, JsonSerializer serializer) + { + string s = (string)reader.Value; + return string.IsNullOrWhiteSpace(s) ? new Banderwagon() : Banderwagon.FromBytes(Bytes.FromHexString(s), subgroupCheck: false).Value; + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs b/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs index 2eaba4cb364..b8b2bdc0f7c 100644 --- a/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs +++ b/src/Nethermind/Nethermind.Serialization.Json/EthereumJsonSerializer.cs @@ -48,6 +48,9 @@ public EthereumJsonSerializer(int? maxDepth = null, params JsonConverter[] conve new PublicKeyConverter(), new TxTypeConverter(), new MemoryByteConverter(), + new StemConverter(), + new IpaProofConverter(), + new BanderwagonConverter(), }); public IList BasicConverters { get; } = CommonConverters.ToList(); @@ -69,6 +72,9 @@ public EthereumJsonSerializer(int? maxDepth = null, params JsonConverter[] conve new PublicKeyConverter(), new TxTypeConverter(), new MemoryByteConverter(), + new StemConverter(), + new IpaProofConverter(), + new BanderwagonConverter(), }; public T Deserialize(Stream stream) diff --git a/src/Nethermind/Nethermind.Serialization.Json/IpaProofConverter.cs b/src/Nethermind/Nethermind.Serialization.Json/IpaProofConverter.cs new file mode 100644 index 00000000000..26c6c495ecf --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Json/IpaProofConverter.cs @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Linq; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Proofs; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Nethermind.Serialization.Json; + + +public class IpaProofConverter: JsonConverter +{ + public override void WriteJson(JsonWriter writer, IpaProofStruct value, JsonSerializer serializer) + { + if (value is null) + { + value = new IpaProofStruct(Array.Empty(), FrE.Zero, Array.Empty()); + } + + writer.WriteStartObject(); + + writer.WritePropertyName("cl"); + writer.WriteStartArray(); + for (int i = 0; i < value.L.Length; i++) + { + writer.WriteValue(value.L[i].ToBytes().ToHexString(true)); + } + writer.WriteEnd(); + + writer.WritePropertyName("cr"); + writer.WriteStartArray(); + for (int i = 0; i < value.R.Length; i++) + { + writer.WriteValue(value.R[i].ToBytes().ToHexString(true)); + } + writer.WriteEnd(); + + writer.WritePropertyName("finalEvaluation"); + serializer.Serialize(writer, value.A.ToBytesBigEndian()); + + writer.WriteEndObject(); + } + + public override IpaProofStruct ReadJson(JsonReader reader, Type objectType, IpaProofStruct existingValue, bool hasExistingValue, JsonSerializer serializer) + { + Banderwagon[] cl = Array.Empty(); + Banderwagon[] cr = Array.Empty(); + if (reader.TokenType == JsonToken.StartObject) + { + for (int i = 0; i < 2; i++) + { + reader.Read(); + if (string.Equals((string)reader.Value, "cl", StringComparison.InvariantCultureIgnoreCase)) + { + reader.Read(); + cl = serializer.Deserialize(reader) ?? + throw new InvalidOperationException(); + } + else if (string.Equals((string)reader.Value, "cr", StringComparison.InvariantCultureIgnoreCase)) + { + reader.Read(); + cr = serializer.Deserialize(reader) ?? + throw new InvalidOperationException(); + } + } + } + reader.Read(); + FrE finalEvaluation = FrE.FromBytes(Bytes.FromHexString(reader.ReadAsString()), true); + reader.Read(); + return new IpaProofStruct(cl, finalEvaluation, cr); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Json/StemConverter.cs b/src/Nethermind/Nethermind.Serialization.Json/StemConverter.cs new file mode 100644 index 00000000000..ff35f142f15 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Json/StemConverter.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Newtonsoft.Json; + +namespace Nethermind.Serialization.Json; + +public class StemConverter: JsonConverter +{ + public override void WriteJson(JsonWriter writer, Stem value, JsonSerializer serializer) + { + if (value is null) + { + writer.WriteNull(); + } + else + { + writer.WriteValue(Bytes.ByteArrayToHexViaLookup32Safe(value.Bytes, true)); + } + } + + public override Stem ReadJson(JsonReader reader, Type objectType, Stem existingValue, bool hasExistingValue, JsonSerializer serializer) + { + string s = (string)reader.Value; + return string.IsNullOrWhiteSpace(s) ? null : new Stem(Bytes.FromHexString(s)); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index 425f84c664b..d9b30157b82 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -4,6 +4,8 @@ using System.Collections.Generic; using System.Linq; using Nethermind.Core; +using Nethermind.Core.Verkle; +using Nethermind.Serialization.Rlp.Verkle; namespace Nethermind.Serialization.Rlp { @@ -12,6 +14,7 @@ public class BlockDecoder : IRlpValueDecoder, IRlpStreamDecoder private readonly HeaderDecoder _headerDecoder = new(); private readonly TxDecoder _txDecoder = new(); private readonly WithdrawalDecoder _withdrawalDecoder = new(); + // private readonly ExecutionWitnessDecoder _witnessDecoder = new(); public Block? Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { @@ -67,6 +70,12 @@ public class BlockDecoder : IRlpValueDecoder, IRlpStreamDecoder rlpStream.Check(withdrawalsCheck); } + // ExecutionWitness? witness = null; + // if (rlpStream.Position != blockCheck) + // { + // witness = _witnessDecoder.Decode(rlpStream); + // } + if ((rlpBehaviors & RlpBehaviors.AllowExtraBytes) != RlpBehaviors.AllowExtraBytes) { rlpStream.Check(blockCheck); @@ -75,6 +84,7 @@ public class BlockDecoder : IRlpValueDecoder, IRlpStreamDecoder return new(header, transactions, uncleHeaders, withdrawals); } + // private (int Total, int Txs, int Uncles, int? Withdrawals, int? ExecutionWitness) GetContentLength(Block item, RlpBehaviors rlpBehaviors) private (int Total, int Txs, int Uncles, int? Withdrawals) GetContentLength(Block item, RlpBehaviors rlpBehaviors) { int contentLength = _headerDecoder.GetLength(item.Header, rlpBehaviors); @@ -94,6 +104,13 @@ public class BlockDecoder : IRlpValueDecoder, IRlpStreamDecoder contentLength += Rlp.LengthOfSequence(withdrawalsLength.Value); } + // int? wintessLength = null; + // if (item.ExecutionWitness is not null) + // { + // wintessLength = ExecutionWitnessDecoder.GetContentLength(item.ExecutionWitness).contentLength; + // contentLength += Rlp.LengthOfSequence(wintessLength.Value); + // } + return (contentLength, txLength, unclesLength, withdrawalsLength); } @@ -193,11 +210,18 @@ public int GetLength(Block? item, RlpBehaviors rlpBehaviors) decoderContext.Check(withdrawalsCheck); } + // ExecutionWitness? witness = null; + // if (decoderContext.Position != blockCheck) + // { + // witness = _witnessDecoder.Decode(ref decoderContext); + // } + if ((rlpBehaviors & RlpBehaviors.AllowExtraBytes) != RlpBehaviors.AllowExtraBytes) { decoderContext.Check(blockCheck); } + // return new(header, transactions, uncleHeaders, withdrawals, witness); return new(header, transactions, uncleHeaders, withdrawals); } @@ -245,6 +269,11 @@ public void Encode(RlpStream stream, Block? item, RlpBehaviors rlpBehaviors = Rl stream.Encode(item.Withdrawals[i]); } } + + // if (witnessLength.HasValue) + // { + // _witnessDecoder.Encode(stream, item.ExecutionWitness); + // } } } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 2f4fec645e0..e11621e07f4 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -1039,6 +1039,23 @@ public byte[] DecodeByteArray() return DecodeByteArraySpan().ToArray(); } + public byte DecodeByte() + { + byte byteValue = PeekByte(); + if (byteValue < 128) + { + SkipBytes(1); + return byteValue; + } + + ReadOnlySpan bytes = DecodeByteArraySpan(); + return bytes.Length == 0 ? (byte)0 + : bytes.Length == 1 ? bytes[0] == (byte)128 + ? (byte)0 + : bytes[0] + : bytes[1]; + } + public Span DecodeByteArraySpan() { int prefix = ReadByte(); @@ -1467,7 +1484,7 @@ public static int LengthOf(string value) public static int LengthOf(byte value) { - return 1; + return (value >= 128) ? 2 : 1; } public static int LengthOf(bool value) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/ExecutionWitnessDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/ExecutionWitnessDecoder.cs new file mode 100644 index 00000000000..556132cc770 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/ExecutionWitnessDecoder.cs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core.Verkle; + +namespace Nethermind.Serialization.Rlp.Verkle; + +public class ExecutionWitnessDecoder: IRlpStreamDecoder, IRlpValueDecoder +{ + private static readonly WitnessVerkleProofDecoder _proofDecoder = new WitnessVerkleProofDecoder(); + private static readonly StemStateDiffDecoder _diffDecoder = new StemStateDiffDecoder(); + public int GetLength(ExecutionWitness item, RlpBehaviors rlpBehaviors)=> + Rlp.LengthOfSequence(GetContentLength(item).contentLength); + + public static (int contentLength, int diffLength) GetContentLength(ExecutionWitness item) + { + int contentLength = 0; + + int diffLength = item.StateDiff.Sum(diff => _diffDecoder.GetLength(diff, RlpBehaviors.None)); + contentLength += Rlp.LengthOfSequence(diffLength); + + contentLength += _proofDecoder.GetLength(item.VerkleProof, RlpBehaviors.None); + + return (contentLength, diffLength); + } + + public ExecutionWitness Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + var resultLength = rlpStream.PeekNumberOfItemsRemaining(checkPosition); + List result = new (resultLength); + for (int i = 0; i < resultLength; i++) + { + result.Add(_diffDecoder.Decode(rlpStream, rlpBehaviors)); + } + + var proof = _proofDecoder.Decode(rlpStream); + + return new ExecutionWitness(result, proof); + } + + public void Encode(RlpStream stream, ExecutionWitness item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + (int contentLength, int diffLength) = GetContentLength(item); + stream.StartSequence(contentLength); + stream.StartSequence(diffLength); + foreach (var diff in item.StateDiff) + { + _diffDecoder.Encode(stream, diff); + } + + _proofDecoder.Encode(stream, item.VerkleProof); + } + + public ExecutionWitness Decode(ref Rlp.ValueDecoderContext rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + var resultLength = rlpStream.PeekNumberOfItemsRemaining(checkPosition); + List result = new (resultLength); + for (int i = 0; i < resultLength; i++) + { + result.Add(_diffDecoder.Decode(ref rlpStream, rlpBehaviors)); + } + + var proof = _proofDecoder.Decode(ref rlpStream); + + return new ExecutionWitness(result, proof); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/IpaProofStructDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/IpaProofStructDecoder.cs new file mode 100644 index 00000000000..14846431bb7 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/IpaProofStructDecoder.cs @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Proofs; + +namespace Nethermind.Serialization.Rlp.Verkle; + +public class IpaProofStructDecoder: IRlpStreamDecoder, IRlpValueDecoder +{ + public int GetLength(IpaProofStruct item, RlpBehaviors rlpBehaviors)=> + Rlp.LengthOfSequence(GetContentLength(item)); + + public static int GetContentLength(IpaProofStruct item) + { + int contentLength = 0; + + int cl = item.L.Length * 33; + contentLength += Rlp.LengthOfSequence(cl); + + contentLength += 33; + + int cr = item.R.Length * 33; + contentLength += Rlp.LengthOfSequence(cr); + + return contentLength; + } + + public IpaProofStruct Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + Banderwagon[] resultL = new Banderwagon[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < resultL.Length; i++) + { + resultL[i] = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + } + FrE fre = FrE.FromBytes(rlpStream.DecodeByteArray()); + + checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + Banderwagon[] resultR = new Banderwagon[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < resultR.Length; i++) + { + resultR[i] = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + } + + return new IpaProofStruct(resultL, fre, resultR); + } + + public void Encode(RlpStream stream, IpaProofStruct item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + stream.StartSequence(GetContentLength(item)); + stream.StartSequence(item.L.Length * 33); + foreach (Banderwagon point in item.L) stream.Encode(point.ToBytes()); + stream.Encode(item.A.ToBytes()); + stream.StartSequence(item.R.Length * 33); + foreach (Banderwagon point in item.R) stream.Encode(point.ToBytes()); + } + + public IpaProofStruct Decode(ref Rlp.ValueDecoderContext rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + Banderwagon[] resultL = new Banderwagon[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < resultL.Length; i++) + { + resultL[i] = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + } + FrE fre = FrE.FromBytes(rlpStream.DecodeByteArray()); + + checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + Banderwagon[] resultR = new Banderwagon[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < resultR.Length; i++) + { + resultR[i] = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + } + + return new IpaProofStruct(resultL, fre, resultR); + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/StemStateDiffDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/StemStateDiffDecoder.cs new file mode 100644 index 00000000000..3e607b1c669 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/StemStateDiffDecoder.cs @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core.Verkle; + +namespace Nethermind.Serialization.Rlp.Verkle; + +public class StemStateDiffDecoder: IRlpStreamDecoder, IRlpValueDecoder +{ + private static readonly SuffixStateDiffDecoder _suffixStateDiffDecoder = SuffixStateDiffDecoder.Instance; + public int GetLength(StemStateDiff item, RlpBehaviors rlpBehaviors)=> + Rlp.LengthOfSequence(GetContentLength(item).contentLength); + + public static (int contentLength, int suffixDiffContentLength) GetContentLength(StemStateDiff item) + { + int contentLength = 0; + contentLength += Rlp.LengthOf(item.Stem.Bytes); + + int suffixDiffContentLength = item.SuffixDiffs.Sum(suffixDiff => + _suffixStateDiffDecoder.GetLength(suffixDiff, RlpBehaviors.None)); + + contentLength += Rlp.LengthOfSequence(suffixDiffContentLength); + return (contentLength, suffixDiffContentLength); + } + + public StemStateDiff Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + Stem stem = new Stem(rlpStream.DecodeByteArray()); + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + SuffixStateDiff[] result = new SuffixStateDiff[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < result.Length; i++) + { + result[i] = _suffixStateDiffDecoder.Decode(rlpStream, rlpBehaviors); + } + + return new StemStateDiff { Stem = stem, SuffixDiffs = new List(result) }; + } + + public void Encode(RlpStream stream, StemStateDiff item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + (int contentLength, int suffixDiffContentLength) = GetContentLength(item); + stream.StartSequence(contentLength); + stream.Encode(item.Stem.Bytes); + + stream.StartSequence(suffixDiffContentLength); + foreach (SuffixStateDiff suffixDiff in item.SuffixDiffs) + { + _suffixStateDiffDecoder.Encode(stream, suffixDiff); + } + } + + public StemStateDiff Decode(ref Rlp.ValueDecoderContext rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + Stem stem = new Stem(rlpStream.DecodeByteArray()); + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + SuffixStateDiff[] result = new SuffixStateDiff[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < result.Length; i++) + { + result[i] = _suffixStateDiffDecoder.Decode(ref rlpStream, rlpBehaviors); + } + + return new StemStateDiff { Stem = stem, SuffixDiffs = new List(result) }; + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/SuffixStateDiffDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/SuffixStateDiffDecoder.cs new file mode 100644 index 00000000000..fd5d943bd54 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/SuffixStateDiffDecoder.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; + +namespace Nethermind.Serialization.Rlp.Verkle; + +public class SuffixStateDiffDecoder: IRlpStreamDecoder, IRlpValueDecoder +{ + public static SuffixStateDiffDecoder Instance => new SuffixStateDiffDecoder(); + + public int GetLength(SuffixStateDiff item, RlpBehaviors rlpBehaviors) => + Rlp.LengthOfSequence(GetContentLength(item)); + + public static int GetContentLength(SuffixStateDiff item) + { + return Rlp.LengthOf(item.Suffix) + Rlp.LengthOf(item.CurrentValue) + Rlp.LengthOf(item.NewValue); + } + + public SuffixStateDiff Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + return new SuffixStateDiff() + { + Suffix = rlpStream.DecodeByte(), + CurrentValue = rlpStream.DecodeByteArray(), + NewValue = rlpStream.DecodeByteArray() + }; + } + + public void Encode(RlpStream stream, SuffixStateDiff item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + stream.StartSequence(GetContentLength(item)); + stream.Encode(item.Suffix); + stream.Encode(item.CurrentValue); + stream.Encode(item.NewValue); + } + + public SuffixStateDiff Decode(ref Rlp.ValueDecoderContext rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + return new SuffixStateDiff() + { + Suffix = rlpStream.DecodeByte(), + CurrentValue = rlpStream.DecodeByteArray(), + NewValue = rlpStream.DecodeByteArray() + }; + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/WitnessVerkleProofDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/WitnessVerkleProofDecoder.cs new file mode 100644 index 00000000000..8b205eb9fcf --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Verkle/WitnessVerkleProofDecoder.cs @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Curve; + +namespace Nethermind.Serialization.Rlp.Verkle; + +public class WitnessVerkleProofDecoder: IRlpStreamDecoder, IRlpValueDecoder +{ + private static readonly IpaProofStructDecoder _ipaDecoder = new IpaProofStructDecoder(); + public int GetLength(WitnessVerkleProof item, RlpBehaviors rlpBehaviors) => + Rlp.LengthOfSequence(GetContentLength(item)); + + public static int GetContentLength(WitnessVerkleProof item) + { + int contentLength = 0; + + int stemContentLength = item.OtherStems?.Length * 32 ?? 0; + contentLength += Rlp.LengthOfSequence(stemContentLength); + + contentLength += Rlp.LengthOf(item.DepthExtensionPresent); + + int commitmentsLength = item.CommitmentsByPath.Length * 33; + contentLength += Rlp.LengthOfSequence(commitmentsLength); + + contentLength += 33; + + contentLength += _ipaDecoder.GetLength(item.IpaProof, RlpBehaviors.None); + + return contentLength; + } + + public WitnessVerkleProof Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + int otherStem = rlpStream.PeekNumberOfItemsRemaining(checkPosition); + Stem[]? stems; + if (otherStem > 0) + { + stems = new Stem[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < stems.Length; i++) + { + stems[i] = new Stem(rlpStream.DecodeByteArray()); + } + } + else + { + stems = null; + } + + var depth = rlpStream.DecodeByteArray(); + + checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + Banderwagon[] commitments = new Banderwagon[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < commitments.Length; i++) + { + commitments[i] = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + } + + Banderwagon d = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + + var proof = _ipaDecoder.Decode(rlpStream); + + return new WitnessVerkleProof(stems, depth, commitments, d, proof); + } + + public void Encode(RlpStream stream, WitnessVerkleProof item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + stream.StartSequence(GetContentLength(item)); + int stemContentLength = item.OtherStems?.Length * 32 ?? 0; + stream.StartSequence(stemContentLength); + if (item.OtherStems != null) + { + foreach (Stem? stem in item.OtherStems) + { + stream.Encode(stem.Bytes); + } + } + + stream.Encode(item.DepthExtensionPresent); + + int commitmentsLength = item.CommitmentsByPath.Length * 33; + stream.StartSequence(commitmentsLength); + foreach (Banderwagon commit in item.CommitmentsByPath) + { + stream.Encode(commit.ToBytes()); + } + + stream.Encode(item.D.ToBytes()); + + _ipaDecoder.Encode(stream, item.IpaProof); + } + + public WitnessVerkleProof Decode(ref Rlp.ValueDecoderContext rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + rlpStream.ReadSequenceLength(); + int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + int otherStem = rlpStream.PeekNumberOfItemsRemaining(checkPosition); + Stem[]? stems; + if (otherStem > 0) + { + stems = new Stem[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < stems.Length; i++) + { + stems[i] = new Stem(rlpStream.DecodeByteArray()); + } + } + else + { + stems = null; + } + + var depth = rlpStream.DecodeByteArray(); + + checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; + Banderwagon[] commitments = new Banderwagon[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + for (int i = 0; i < commitments.Length; i++) + { + commitments[i] = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + } + + Banderwagon d = Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false).Value; + + var proof = _ipaDecoder.Decode(ref rlpStream); + + return new WitnessVerkleProof(stems, depth, commitments, d, proof); + } +} diff --git a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs index bfea158d537..9e2dbe58692 100644 --- a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs @@ -651,6 +651,7 @@ public void Eip_transitions_loaded_correctly() Eip3855TransitionTimestamp = 1000000012, Eip3860TransitionTimestamp = 1000000012, Eip1153TransitionTimestamp = 1000000024, + VerkleTreeTransitionTimestamp = 1000000030, } }; @@ -727,5 +728,6 @@ void TestTransitions(ForkActivation activation, Action changes) r.IsEip3860Enabled = true; }); TestTransitions((40001L, 1000000024), r => { r.IsEip1153Enabled = true; }); + TestTransitions((40001L, 1000000030), r => { r.IsVerkleTreeEipEnabled = true; }); } } diff --git a/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs b/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs index fde167d5f5e..d902daa9368 100644 --- a/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs @@ -147,11 +147,20 @@ public ulong Eip4844TransitionTimestamp } } + private ulong? _overridenVerkleTreeTransitionTimeStamp; + public ulong VerkleTreeTransitionTimeStamp + { + get => _overridenVerkleTreeTransitionTimeStamp ?? _spec.VerkleTreeTransitionTimeStamp; + set => _overridenVerkleTreeTransitionTimeStamp = value; + } + public bool IsEip1153Enabled => _spec.IsEip1153Enabled; public bool IsEip3651Enabled => _spec.IsEip3651Enabled; public bool IsEip3855Enabled => _spec.IsEip3855Enabled; public bool IsEip3860Enabled => _spec.IsEip3860Enabled; public bool IsEip4895Enabled => _spec.IsEip4895Enabled; public ulong WithdrawalTimestamp => _spec.WithdrawalTimestamp; + public bool IsVerkleTreeEipEnabled => _spec.IsVerkleTreeEipEnabled; + } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs index 76fbd2f8398..cc5992743b2 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs @@ -124,5 +124,7 @@ public class ChainParameters public ulong? Eip4895TransitionTimestamp { get; set; } public ulong? Eip4844TransitionTimestamp { get; set; } + + public ulong? VerkleTreeTransitionTimestamp { get; set; } } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpec.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpec.cs index bd2074c021c..9e9d2fb3f7a 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpec.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpec.cs @@ -11,7 +11,7 @@ namespace Nethermind.Specs.ChainSpecStyle { /// /// https://github.com/ethereum/wiki/wiki/Ethereum-Chain-Spec-Format - /// https://wiki.parity.io/Chain-specification + /// https://wiki.parity.io/Chain-specification /// [DebuggerDisplay("{Name}, ChainId = {ChainId}")] public class ChainSpec @@ -78,5 +78,7 @@ public class ChainSpec public UInt256? TerminalTotalDifficulty { get; set; } public ulong? ShanghaiTimestamp { get; set; } + public ulong? CancunTimestamp { get; set; } + public ulong? PragueTimestamp { get; set; } } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs index bacd27adf5c..802678f4b05 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs @@ -237,6 +237,9 @@ private static ReleaseSpec CreateReleaseSpec(ChainSpec chainSpec, long releaseSt releaseSpec.IsEip4844Enabled = (chainSpec.Parameters.Eip4844TransitionTimestamp ?? ulong.MaxValue) <= releaseStartTimestamp; releaseSpec.Eip4844TransitionTimestamp = chainSpec.Parameters.Eip4844TransitionTimestamp ?? ulong.MaxValue; + releaseSpec.IsVerkleTreeEipEnabled = (chainSpec.Parameters.VerkleTreeTransitionTimestamp ?? ulong.MaxValue) <= releaseStartTimestamp; + releaseSpec.VerkleTreeTransitionTimeStamp = chainSpec.Parameters.VerkleTreeTransitionTimestamp ?? ulong.MaxValue; + return releaseSpec; } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs index 607d324dc64..d8d244b63c8 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs @@ -139,6 +139,7 @@ private void LoadParameters(ChainSpecJson chainSpecJson, ChainSpec chainSpec) Eip3860TransitionTimestamp = chainSpecJson.Params.Eip3860TransitionTimestamp, Eip4895TransitionTimestamp = chainSpecJson.Params.Eip4895TransitionTimestamp, Eip4844TransitionTimestamp = chainSpecJson.Params.Eip4844TransitionTimestamp, + VerkleTreeTransitionTimestamp = chainSpecJson.Params.VerkleTreeTransitionTimestamp, TransactionPermissionContract = chainSpecJson.Params.TransactionPermissionContract, TransactionPermissionContractTransition = chainSpecJson.Params.TransactionPermissionContractTransition, ValidateChainIdTransition = chainSpecJson.Params.ValidateChainIdTransition, @@ -215,6 +216,8 @@ chainSpec.Parameters.Eip1283DisableTransition is null chainSpec.ArrowGlacierBlockNumber = difficultyBombDelaysBlockNumbers?.Skip(4).FirstOrDefault(); chainSpec.GrayGlacierBlockNumber = difficultyBombDelaysBlockNumbers?.Skip(5).FirstOrDefault(); chainSpec.ShanghaiTimestamp = chainSpec.Parameters.Eip3651TransitionTimestamp; + chainSpec.CancunTimestamp = chainSpec.Parameters.Eip4844TransitionTimestamp; + chainSpec.PragueTimestamp = chainSpec.Parameters.VerkleTreeTransitionTimestamp; // TheMerge parameters chainSpec.MergeForkIdBlockNumber = chainSpec.Parameters.MergeForkIdTransition; diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecParamsJson.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecParamsJson.cs index 2f42d69e61a..b8c541d790c 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecParamsJson.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecParamsJson.cs @@ -140,5 +140,6 @@ internal class ChainSpecParamsJson public ulong? Eip4895TransitionTimestamp { get; set; } public ulong? Eip4844TransitionTimestamp { get; set; } + public ulong? VerkleTreeTransitionTimestamp { get; set; } } } diff --git a/src/Nethermind/Nethermind.Specs/Forks/00_Olympic.cs b/src/Nethermind/Nethermind.Specs/Forks/00_Olympic.cs index 4180bb4cd52..ed13b3c7329 100644 --- a/src/Nethermind/Nethermind.Specs/Forks/00_Olympic.cs +++ b/src/Nethermind/Nethermind.Specs/Forks/00_Olympic.cs @@ -24,6 +24,7 @@ protected Olympic() IsEip3607Enabled = true; MaximumUncleCount = 2; Eip1559TransitionBlock = long.MaxValue; + VerkleTreeTransitionTimeStamp = ulong.MaxValue; ValidateChainId = true; ValidateReceipts = true; } diff --git a/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs b/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs index a110de27f0d..1f2142edaed 100644 --- a/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs +++ b/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs @@ -17,6 +17,6 @@ protected Cancun() IsEip4844Enabled = true; } - public new static IReleaseSpec Instance => LazyInitializer.EnsureInitialized(ref _instance, () => new Cancun()); + public static new IReleaseSpec Instance => LazyInitializer.EnsureInitialized(ref _instance, () => new Cancun()); } } diff --git a/src/Nethermind/Nethermind.Specs/Forks/17_Prague.cs b/src/Nethermind/Nethermind.Specs/Forks/17_Prague.cs new file mode 100644 index 00000000000..cec82e40b71 --- /dev/null +++ b/src/Nethermind/Nethermind.Specs/Forks/17_Prague.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; +using Nethermind.Core.Specs; + +namespace Nethermind.Specs.Forks; + +public class Prague : Cancun +{ + private static IReleaseSpec _instance; + + protected Prague() + { + Name = "Prague"; + IsVerkleTreeEipEnabled = true; + } + + public static new IReleaseSpec Instance => LazyInitializer.EnsureInitialized(ref _instance, () => new Prague()); +} diff --git a/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs b/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs index 9cb46d7439c..fbf44293edc 100644 --- a/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs @@ -79,5 +79,8 @@ public ReleaseSpec Clone() public bool IsEip3860Enabled { get; set; } public bool IsEip4895Enabled { get; set; } public bool IsEip4844Enabled { get; set; } + public bool IsVerkleTreeEipEnabled { get; set; } + public ulong VerkleTreeTransitionTimeStamp { get; set; } + } } diff --git a/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs b/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs index 1dd493212d3..0132a6200c8 100644 --- a/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs @@ -119,6 +119,7 @@ public bool IsEip158IgnoredAccount(Address address) public ulong WithdrawalTimestamp => _spec.WithdrawalTimestamp; public ulong Eip4844TransitionTimestamp => _spec.Eip4844TransitionTimestamp; + public ulong VerkleTreeTransitionTimeStamp => _spec.VerkleTreeTransitionTimeStamp; public Address Eip1559FeeCollector => _spec.Eip1559FeeCollector; public bool IsEip1153Enabled => _spec.IsEip1153Enabled; @@ -126,5 +127,7 @@ public bool IsEip158IgnoredAccount(Address address) public bool IsEip3855Enabled => _spec.IsEip3855Enabled; public bool IsEip3860Enabled => _spec.IsEip3860Enabled; public bool IsEip4895Enabled => _spec.IsEip4895Enabled; + public bool IsVerkleTreeEipEnabled => _spec.IsVerkleTreeEipEnabled; + } } diff --git a/src/Nethermind/Nethermind.State.Test/Nethermind.State.Test.csproj b/src/Nethermind/Nethermind.State.Test/Nethermind.State.Test.csproj index e46f2d637e4..f38cea07e01 100644 --- a/src/Nethermind/Nethermind.State.Test/Nethermind.State.Test.csproj +++ b/src/Nethermind/Nethermind.State.Test/Nethermind.State.Test.csproj @@ -24,6 +24,7 @@ + diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index 60cfcd8bafa..49ed996854e 100644 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -115,8 +115,8 @@ public void RecreateAccountStateFromOneRangeWithNonExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AddRangeResult result = snapProvider.AddAccountRange(1, rootHash, Keccak.Zero, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray()); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -139,8 +139,8 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray()); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -156,8 +156,8 @@ public void RecreateAccountStateFromOneRangeWithoutProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -174,8 +174,8 @@ public void RecreateAccountStateFromMultipleRange() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes); _inputTree.Accept(accountProofCollector, _inputTree.RootHash); @@ -224,8 +224,8 @@ public void MissingAccountFromRange() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes); _inputTree.Accept(accountProofCollector, _inputTree.RootHash); diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs index 3ad5f6ac2a1..32f0e0a2a48 100644 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs +++ b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs @@ -52,8 +52,8 @@ public void RecreateStorageStateFromOneRangeWithNonExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddStorageRange(1, null, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -71,8 +71,8 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddStorageRange(1, null, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -86,8 +86,8 @@ public void RecreateStorageStateFromOneRangeWithoutProof() MemDb db = new MemDb(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddStorageRange(1, null, rootHash, TestItem.Tree.SlotsWithPaths[0].Path, TestItem.Tree.SlotsWithPaths); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -102,8 +102,8 @@ public void RecreateAccountStateFromMultipleRange() MemDb db = new MemDb(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new Keccak[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); @@ -137,8 +137,8 @@ public void MissingAccountFromRange() MemDb db = new MemDb(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new Keccak[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); diff --git a/src/Nethermind/Nethermind.State.Test/VerkleTreeTests.cs b/src/Nethermind/Nethermind.State.Test/VerkleTreeTests.cs new file mode 100644 index 00000000000..4f87c23477e --- /dev/null +++ b/src/Nethermind/Nethermind.State.Test/VerkleTreeTests.cs @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle; +using NUnit.Framework; + +namespace Nethermind.Store.Test; + +[TestFixture, Parallelizable(ParallelScope.All)] +public class VerkleTreeTests +{ + [Test] + public void Create_commit_change_balance_get() + { + Account account = new(1); + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleStateTree stateTree = new VerkleStateTree(dbProvider, LimboLogs.Instance); + stateTree.Set(TestItem.AddressA, account); + stateTree.Commit(); + stateTree.CommitTree(0); + + account = account.WithChangedBalance(2); + stateTree.Set(TestItem.AddressA, account); + stateTree.Commit(); + stateTree.CommitTree(0); + + Account accountRestored = stateTree.Get(TestItem.AddressA); + Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2)); + } + + [Test] + public void Create_create_commit_change_balance_get() + { + Account account = new(1); + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleStateTree stateTree = new VerkleStateTree(dbProvider, LimboLogs.Instance); + stateTree.Set(TestItem.AddressA, account); + stateTree.Set(TestItem.AddressB, account); + stateTree.Commit(); + stateTree.CommitTree(0); + + account = account.WithChangedBalance(2); + stateTree.Set(TestItem.AddressA, account); + stateTree.Commit(); + stateTree.CommitTree(0); + + Account accountRestored = stateTree.Get(TestItem.AddressA); + Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2)); + } + + [Test] + public void TestGenesis() + { + Account account = new(1); + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleStateTree stateTree = new VerkleStateTree(dbProvider, LimboLogs.Instance); + stateTree.Set(new Address("0x0000000000000000000000000000000000000000"), account); + stateTree.Commit(); + stateTree.CommitTree(0); + Console.WriteLine(EnumerableExtensions.ToString(stateTree.StateRoot.Bytes)); + } + + // [Test] + // public void Create_commit_reset_change_balance_get() + // { + // Account account = new(1); + // IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + // VerkleStateTree stateTree = new VerkleStateTree(dbProvider); + // stateTree.Set(TestItem.AddressA, account); + // stateTree.Flush(0); + // + // Keccak rootHash = new Keccak(stateTree.RootHash); + // + // stateTree.Get(TestItem.AddressA); + // account = account.WithChangedBalance(2); + // stateTree.Set(TestItem.AddressA, account); + // stateTree.Flush(0); + // } + // + // [TestCase(true, false)] + // [TestCase(false, true)] + // public void Commit_with_skip_root_should_skip_root(bool skipRoot, bool hasRoot) + // { + // MemDb db = new(); + // TrieStore trieStore = new TrieStore(db, LimboLogs.Instance); + // Account account = new(1); + // + // IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + // VerkleStateTree stateTree = new VerkleStateTree(dbProvider); + // stateTree.Set(TestItem.AddressA, account); + // stateTree.UpdateRootHash(); + // Keccak stateRoot = stateTree.RootHash; + // stateTree.Flush(0, skipRoot); + // + // if (hasRoot) + // { + // trieStore.LoadRlp(stateRoot).Length.Should().BeGreaterThan(0); + // } + // else + // { + // trieStore.Invoking(ts => ts.LoadRlp(stateRoot)).Should().Throw(); + // } + // } +} diff --git a/src/Nethermind/Nethermind.State.Test/VerkleWitnessAccessCostTest.cs b/src/Nethermind/Nethermind.State.Test/VerkleWitnessAccessCostTest.cs new file mode 100644 index 00000000000..e99ad2abd94 --- /dev/null +++ b/src/Nethermind/Nethermind.State.Test/VerkleWitnessAccessCostTest.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Test.Builders; +using Nethermind.State; +using Nethermind.Verkle.Tree; +using NUnit.Framework; + +namespace Nethermind.Store.Test; + +public class VerkleWitnessAccessCostTest +{ + + [Test] + public void TestAccessForTransaction() + { + VerkleWitness calculator = new VerkleWitness(); + long gas = calculator.AccessForTransaction(TestItem.AddressA, TestItem.AddressB, false); + Console.WriteLine(gas); + } + + [Test] + public void TestAccessForTransactionWithValue() + { + VerkleWitness calculator = new VerkleWitness(); + long gas = calculator.AccessForTransaction(TestItem.AddressA, TestItem.AddressB, true); + Console.WriteLine(gas); + } +} diff --git a/src/Nethermind/Nethermind.State/IWorldState.cs b/src/Nethermind/Nethermind.State/IWorldState.cs index cea4e411e11..f46dbf0664d 100644 --- a/src/Nethermind/Nethermind.State/IWorldState.cs +++ b/src/Nethermind/Nethermind.State/IWorldState.cs @@ -15,6 +15,7 @@ namespace Nethermind.State; /// public interface IWorldState : IJournal, IReadOnlyStateProvider { + StateType StateType { get; } /// /// Return the original persistent storage value from the storage cell /// @@ -109,4 +110,5 @@ public interface IWorldState : IJournal, IReadOnlyStateProvider /// /// void TouchCode(Keccak codeHash); + public byte[] GetCodeChunk(Address codeOwner, UInt256 chunkId); } diff --git a/src/Nethermind/Nethermind.State/Nethermind.State.csproj b/src/Nethermind/Nethermind.State/Nethermind.State.csproj index e49821c1d47..ab832f98588 100644 --- a/src/Nethermind/Nethermind.State/Nethermind.State.csproj +++ b/src/Nethermind/Nethermind.State/Nethermind.State.csproj @@ -14,5 +14,6 @@ + diff --git a/src/Nethermind/Nethermind.State/StateType.cs b/src/Nethermind/Nethermind.State/StateType.cs new file mode 100644 index 00000000000..a5c8fdfb885 --- /dev/null +++ b/src/Nethermind/Nethermind.State/StateType.cs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.State; + +public enum StateType +{ + Merkle = 0, + Verkle = 1 +} diff --git a/src/Nethermind/Nethermind.State/VerklePersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/VerklePersistentStorageProvider.cs new file mode 100644 index 00000000000..9d0e0b0d318 --- /dev/null +++ b/src/Nethermind/Nethermind.State/VerklePersistentStorageProvider.cs @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Core.Resettables; +using Nethermind.Core.Verkle; +using Nethermind.Logging; +using Nethermind.State.Tracing; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.State; + +internal class VerklePersistentStorageProvider : PartialStorageProviderBase +{ + private readonly VerkleStateTree _verkleTree; + private readonly ILogManager? _logManager; + /// + /// EIP-1283 + /// + private readonly ResettableDictionary _originalValues = new ResettableDictionary(); + private readonly ResettableHashSet _committedThisRound = new ResettableHashSet(); + + public VerklePersistentStorageProvider(VerkleStateTree tree, ILogManager? logManager) + : base(logManager) + { + _verkleTree = tree; + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + } + + /// + /// Reset the storage state + /// + public override void Reset() + { + base.Reset(); + _originalValues.Clear(); + _committedThisRound.Clear(); + } + + /// + /// Get the current value at the specified location + /// + /// Storage location + /// Value at location + protected override byte[] GetCurrentValue(in StorageCell storageCell) => + TryGetCachedValue(storageCell, out byte[]? bytes) ? bytes! : LoadFromTree(storageCell); + + /// + /// Return the original persistent storage value from the storage cell + /// + /// + /// + public byte[] GetOriginal(StorageCell storageCell) + { + if (!_originalValues.ContainsKey(storageCell)) + { + throw new InvalidOperationException("Get original should only be called after get within the same caching round"); + } + + if (_transactionChangesSnapshots.TryPeek(out int snapshot)) + { + if (_intraBlockCache.TryGetValue(storageCell, out StackList stack)) + { + if (stack.TryGetSearchedItem(snapshot, out int lastChangeIndexBeforeOriginalSnapshot)) + { + return _changes[lastChangeIndexBeforeOriginalSnapshot]!.Value; + } + } + } + + return _originalValues[storageCell]; + } + + + /// + /// Called by Commit + /// Used for persistent storage specific logic + /// + /// Storage tracer + protected override void CommitCore(IStorageTracer tracer) + { + if (_logger.IsTrace) _logger.Trace("Committing storage changes"); + + if (_changes[_currentPosition] is null) + { + throw new InvalidOperationException($"Change at current position {_currentPosition} was null when commiting {nameof(PartialStorageProviderBase)}"); + } + + if (_changes[_currentPosition + 1] is not null) + { + throw new InvalidOperationException($"Change after current position ({_currentPosition} + 1) was not null when commiting {nameof(PartialStorageProviderBase)}"); + } + + bool isTracing = tracer.IsTracingStorage; + Dictionary? trace = null; + if (isTracing) + { + trace = new Dictionary(); + } + + for (int i = 0; i <= _currentPosition; i++) + { + Change change = _changes[_currentPosition - i]; + if (!isTracing && change!.ChangeType == ChangeType.JustCache) + { + continue; + } + + if (_committedThisRound.Contains(change!.StorageCell)) + { + if (isTracing && change.ChangeType == ChangeType.JustCache) + { + trace![change.StorageCell] = new ChangeTrace(change.Value, trace[change.StorageCell].After); + } + + continue; + } + + if (isTracing && change.ChangeType == ChangeType.JustCache) + { + tracer!.ReportStorageRead(change.StorageCell); + } + + _committedThisRound.Add(change.StorageCell); + + if (change.ChangeType == ChangeType.Destroy) + { + continue; + } + + int forAssertion = _intraBlockCache[change.StorageCell].Pop(); + if (forAssertion != _currentPosition - i) + { + throw new InvalidOperationException($"Expected checked value {forAssertion} to be equal to {_currentPosition} - {i}"); + } + + switch (change.ChangeType) + { + case ChangeType.Destroy: + break; + case ChangeType.JustCache: + break; + case ChangeType.Update: + if (_logger.IsTrace) + { + _logger.Trace($" Update {change.StorageCell.Address}_{change.StorageCell.Index} V = {change.Value.ToHexString(true)}"); + } + + Db.Metrics.StorageTreeWrites++; + _verkleTree.SetStorage(change.StorageCell, change.Value); + if (isTracing) + { + trace![change.StorageCell] = new ChangeTrace(change.Value); + } + + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + + base.CommitCore(tracer); + _originalValues.Reset(); + _committedThisRound.Reset(); + + if (isTracing) + { + ReportChanges(tracer!, trace!); + } + } + + /// + /// Commit persistent storage trees + /// + /// Current block number + public void CommitTrees(long blockNumber) { } + + private byte[] LoadFromTree(StorageCell storageCell) + { + Db.Metrics.StorageTreeReads++; + Pedersen key = AccountHeader.GetTreeKeyForStorageSlot(storageCell.Address.Bytes, storageCell.Index); + byte[] value = (_verkleTree.Get(key) ?? Array.Empty()).ToArray(); + PushToRegistryOnly(storageCell, value); + return value; + } + + private void PushToRegistryOnly(StorageCell cell, byte[] value) + { + SetupRegistry(cell); + IncrementChangePosition(); + _intraBlockCache[cell].Push(_currentPosition); + _originalValues[cell] = value; + _changes[_currentPosition] = new Change(ChangeType.JustCache, cell, value); + } + + private static void ReportChanges(IStorageTracer tracer, Dictionary trace) + { + foreach ((StorageCell address, ChangeTrace change) in trace) + { + byte[] before = change.Before; + byte[] after = change.After; + + if (!Bytes.AreEqual(before, after)) + { + tracer.ReportStorageChange(address, before, after); + } + } + } + + public override void ClearStorage(Address address) + { + throw new NotSupportedException("Verkle Trees does not support deletion of data from the tree"); + } +} diff --git a/src/Nethermind/Nethermind.State/VerkleStateReader.cs b/src/Nethermind/Nethermind.State/VerkleStateReader.cs new file mode 100644 index 00000000000..350b2d60a47 --- /dev/null +++ b/src/Nethermind/Nethermind.State/VerkleStateReader.cs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Db; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Trie; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using Metrics = Nethermind.Db.Metrics; + +namespace Nethermind.State; + +public class VerkleStateReader : IStateReader +{ + private readonly IKeyValueStore _codeDb; + private readonly ILogger _logger; + private readonly VerkleStateTree _state; + + public VerkleStateReader(VerkleStateTree verkleTree, IKeyValueStore? codeDb, ILogManager? logManager) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); + _state = verkleTree; + } + + public VerkleStateReader(IVerkleTrieStore verkleTree, IKeyValueStore? codeDb, ILogManager? logManager) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); + _state = new VerkleStateTree(verkleTree, logManager);; + } + + public Account? GetAccount(Keccak stateRoot, Address address) + { + return GetState(stateRoot, address); + } + + public byte[] GetStorage(Address address, in UInt256 index) + { + return _state.Get(address, index); + } + + public byte[] GetStorage(Keccak storageRoot, in UInt256 index) + { + throw new NotImplementedException(); + } + + public UInt256 GetBalance(Keccak stateRoot, Address address) + { + return GetState(stateRoot, address)?.Balance ?? UInt256.Zero; + } + + public byte[]? GetCode(Keccak codeHash) + { + if (codeHash == Keccak.OfAnEmptyString) + { + return Array.Empty(); + } + + return _codeDb[codeHash.Bytes]; + } + + public void RunTreeVisitor(ITreeVisitor treeVisitor, Keccak rootHash, VisitingOptions? visitingOptions = null) + { + _state.Accept(treeVisitor, rootHash, visitingOptions); + } + + public byte[] GetCode(Keccak stateRoot, Address address) + { + Account? account = GetState(stateRoot, address); + return account is null ? Array.Empty() : GetCode(account.CodeHash); + } + + private Account? GetState(Keccak stateRoot, Address address) + { + if (stateRoot == Keccak.EmptyTreeHash) + { + return null; + } + + Metrics.StateTreeReads++; + Account? account = _state.Get(address, stateRoot); + return account; + } +} diff --git a/src/Nethermind/Nethermind.State/VerkleStateTree.cs b/src/Nethermind/Nethermind.State/VerkleStateTree.cs new file mode 100644 index 00000000000..867627e8025 --- /dev/null +++ b/src/Nethermind/Nethermind.State/VerkleStateTree.cs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.State; + +public class VerkleStateTree : VerkleTree +{ + + public VerkleStateTree(IDbProvider dbProvider, ILogManager logManager) : base(dbProvider, logManager) + { + } + + public VerkleStateTree(IVerkleTrieStore stateStore, ILogManager logManager) : base(stateStore, logManager) + { + } + + [DebuggerStepThrough] + public Account? Get(Address address, Keccak? rootHash = null) + { + Span key = new byte[32]; + Pedersen headerTreeKey = AccountHeader.GetTreeKeyPrefixAccount(address.Bytes); + headerTreeKey.StemAsSpan.CopyTo(key); + key[31] = AccountHeader.Version; + UInt256 version = new((Get(new Pedersen(key.ToArray())) ?? Array.Empty()).ToArray()); + key[31] = AccountHeader.Balance; + UInt256 balance = new((Get(new Pedersen(key.ToArray())) ?? Array.Empty()).ToArray()); + key[31] = AccountHeader.Nonce; + UInt256 nonce = new((Get(new Pedersen(key.ToArray())) ?? Array.Empty()).ToArray()); + key[31] = AccountHeader.CodeHash; + byte[]? codeHash = (Get(new Pedersen(key.ToArray())) ?? Keccak.OfAnEmptyString.Bytes).ToArray(); + key[31] = AccountHeader.CodeSize; + UInt256 codeSize = new((Get(new Pedersen(key.ToArray())) ?? Array.Empty()).ToArray()); + + return new Account(balance, nonce, new Keccak(codeHash), codeSize, version); + } + + public void Set(Address address, Account? account) + { + Pedersen headerTreeKey = AccountHeader.GetTreeKeyPrefixAccount(address.Bytes); + if (account != null) InsertStemBatch(headerTreeKey.StemAsSpan, account.ToVerkleDict()); + } + + public byte[] Get(Address address, in UInt256 index, Keccak? storageRoot = null) + { + Pedersen key = AccountHeader.GetTreeKeyForStorageSlot(address.Bytes, index); + return (Get(key) ?? Array.Empty()).ToArray(); + } + + public void Set(Address address, in UInt256 index, byte[] value) + { + Pedersen key = AccountHeader.GetTreeKeyForStorageSlot(address.Bytes, index); + Insert(key, value); + } + + public void SetCode(Address address, byte[] code) + { + UInt256 chunkId = 0; + CodeChunkEnumerator codeEnumerator = new CodeChunkEnumerator(code); + while (codeEnumerator.TryGetNextChunk(out byte[] chunk)) + { + Pedersen key = AccountHeader.GetTreeKeyForCodeChunk(address.Bytes, chunkId); +#if DEBUG + Console.WriteLine("K: " + key.ToString()); + Console.WriteLine("V: " + chunk.ToHexString()); +#endif + Insert(key, chunk); + chunkId += 1; + } + } + + public void SetStorage(StorageCell cell, byte[] value) + { + Pedersen key = AccountHeader.GetTreeKeyForStorageSlot(cell.Address.Bytes, cell.Index); +#if DEBUG + Console.WriteLine("K: " + key.ToString()); + Console.WriteLine("V: " + value.ToHexString()); +#endif + Insert(key, value); + } + + public static VerkleStateTree CreateStatelessTreeFromExecutionWitness(ExecutionWitness? execWitness, Banderwagon root, ILogManager logManager) + { + VerkleStateStore? stateStore = new (new MemDb(), new MemDb(), new MemDb(), logManager); + VerkleStateTree? tree = new (stateStore, logManager); + if (!tree.InsertIntoStatelessTree(execWitness, root)) + { + throw new InvalidDataException("stateless tree cannot be created: invalid proof"); + } + + return tree; + } + +} diff --git a/src/Nethermind/Nethermind.State/VerkleStorageProvider.cs b/src/Nethermind/Nethermind.State/VerkleStorageProvider.cs new file mode 100644 index 00000000000..e089813c204 --- /dev/null +++ b/src/Nethermind/Nethermind.State/VerkleStorageProvider.cs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using Nethermind.Core; +using Nethermind.Logging; +using Nethermind.State.Tracing; +using Nethermind.Verkle.Tree; + +namespace Nethermind.State; + +public class VerkleStorageProvider +{ + private readonly VerklePersistentStorageProvider _persistentStorageProvider; + private readonly VerkleTransientStorageProvider _transientStorageProvider; + + public VerkleStorageProvider(VerkleStateTree tree, ILogManager? logManager) + { + _persistentStorageProvider = new VerklePersistentStorageProvider(tree, logManager); + _transientStorageProvider = new VerkleTransientStorageProvider(logManager); + } + + public void ClearStorage(Address address) + { + _persistentStorageProvider.ClearStorage(address); + _transientStorageProvider.ClearStorage(address); + } + + public void Commit() + { + _persistentStorageProvider.Commit(); + _transientStorageProvider.Commit(); + } + + public void Commit(IStorageTracer stateTracer) + { + _persistentStorageProvider.Commit(stateTracer); + _transientStorageProvider.Commit(stateTracer); + } + + public void CommitTrees(long blockNumber) + { + _persistentStorageProvider.CommitTrees(blockNumber); + } + + public byte[] Get(in StorageCell storageCell) + { + return _persistentStorageProvider.Get(storageCell); + } + + public byte[] GetOriginal(in StorageCell storageCell) + { + return _persistentStorageProvider.GetOriginal(storageCell); + } + + public byte[] GetTransientState(in StorageCell storageCell) + { + return _transientStorageProvider.Get(storageCell); + } + + public void Reset() + { + _persistentStorageProvider.Reset(); + _transientStorageProvider.Reset(); + } + + internal void Restore(int snapshot) + { + Restore(new Snapshot.Storage(snapshot, Snapshot.EmptyPosition)); + } + + public void Restore(Snapshot.Storage snapshot) + { + _persistentStorageProvider.Restore(snapshot.PersistentStorageSnapshot); + _transientStorageProvider.Restore(snapshot.TransientStorageSnapshot); + } + + public void Set(in StorageCell storageCell, byte[] newValue) + { + Debug.Assert(newValue.Length == 32); + _persistentStorageProvider.Set(storageCell, newValue); + } + + public void SetTransientState(in StorageCell storageCell, byte[] newValue) + { + Debug.Assert(newValue.Length == 32); + _transientStorageProvider.Set(storageCell, newValue); + } + + public Snapshot.Storage TakeSnapshot(bool newTransactionStart) + { + int persistentSnapshot = _persistentStorageProvider.TakeSnapshot(newTransactionStart); + int transientSnapshot = _transientStorageProvider.TakeSnapshot(newTransactionStart); + + return new Snapshot.Storage(persistentSnapshot, transientSnapshot); + } +} diff --git a/src/Nethermind/Nethermind.State/VerkleTransientStorageProvider.cs b/src/Nethermind/Nethermind.State/VerkleTransientStorageProvider.cs new file mode 100644 index 00000000000..0aaba416c1c --- /dev/null +++ b/src/Nethermind/Nethermind.State/VerkleTransientStorageProvider.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Logging; + +namespace Nethermind.State; + +internal class VerkleTransientStorageProvider : PartialStorageProviderBase +{ + public VerkleTransientStorageProvider(ILogManager? logManager) + : base(logManager) { } + + protected override byte[] GetCurrentValue(in StorageCell storageCell) => + TryGetCachedValue(storageCell, out byte[]? bytes) ? bytes! : _zeroValue; + + public override void ClearStorage(Address address) + { + throw new NotSupportedException("Verkle Trees does not support deletion of data from the tree"); + } +} diff --git a/src/Nethermind/Nethermind.State/VerkleWorldState.cs b/src/Nethermind/Nethermind.State/VerkleWorldState.cs new file mode 100644 index 00000000000..184d752b418 --- /dev/null +++ b/src/Nethermind/Nethermind.State/VerkleWorldState.cs @@ -0,0 +1,840 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Resettables; +using Nethermind.Core.Specs; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State.Tracing; +using Nethermind.State.Witnesses; +using Nethermind.Trie; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; + +namespace Nethermind.State; + +public class VerkleWorldState : IWorldState +{ + public StateType StateType => StateType.Verkle; + + private const int StartCapacity = Resettable.StartCapacity; + private readonly ResettableDictionary> _intraBlockCache = new ResettableDictionary>(); + private readonly ResettableHashSet
_committedThisRound = new ResettableHashSet
(); + + private readonly List _keptInCache = new List(); + private readonly ILogger _logger; + private readonly IKeyValueStore _codeDb; + + private int _capacity = StartCapacity; + private Change?[] _changes = new Change?[StartCapacity]; + private int _currentPosition = Resettable.EmptyPosition; + + private readonly VerkleStateTree _tree; + private readonly VerkleStorageProvider _storageProvider; + + // private readonly LruCache<> + + public VerkleWorldState(VerkleStateTree verkleTree, IKeyValueStore? codeDb, ILogManager? logManager) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); + _tree = verkleTree; + _storageProvider = new VerkleStorageProvider(verkleTree, logManager); + } + + public VerkleWorldState(IVerkleTrieStore verkleStateStore, IKeyValueStore? codeDb, ILogManager? logManager) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); + _tree = new VerkleStateTree(verkleStateStore, logManager); + _storageProvider = new VerkleStorageProvider(_tree, logManager); + } + + // create a state provider using execution witness + public VerkleWorldState(ExecutionWitness? executionWitness, Banderwagon root, ILogManager? logManager) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _tree = VerkleStateTree.CreateStatelessTreeFromExecutionWitness(executionWitness, root, logManager); + _codeDb = new MemDb(); + _storageProvider = new VerkleStorageProvider(_tree, logManager); + } + + public void Accept(ITreeVisitor? visitor, Keccak? stateRoot, VisitingOptions? visitingOptions = null) + { + _tree.Accept(visitor, stateRoot, visitingOptions); + } + + public void RecalculateStateRoot() + { + // probably cache and store state root in the tree, and use this to fetch and update rootHash from db + // no good reason to fetch root everytime from db + } + + public Keccak StateRoot + { + get => new Keccak(_tree.StateRoot.Bytes); + set => _tree.StateRoot = new VerkleCommitment(value.Bytes); + } + + public ExecutionWitness GenerateExecutionWitness(byte[][] keys, out Banderwagon rootPoint) + { + _logger.Info($"GenerateExecutionWitness: {keys.Length}"); + return _tree.GenerateExecutionWitnessFromStore(keys, out rootPoint); + } + + public bool AccountExists(Address address) + { + if (_intraBlockCache.ContainsKey(address)) + { + return _changes[_intraBlockCache[address].Peek()]!.ChangeType != ChangeType.Delete; + } + + return GetAndAddToCache(address) is not null; + } + + public bool IsEmptyAccount(Address address) + { + Account account = GetThroughCache(address); + if (account is null) + { + throw new InvalidOperationException($"Account {address} is null when checking if empty"); + } + + return account.IsEmpty; + } + + public Account GetAccount(Address address) + { + return GetThroughCache(address) ?? Account.TotallyEmpty; + } + + public bool IsDeadAccount(Address address) + { + Account? account = GetThroughCache(address); + return account?.IsEmpty ?? true; + } + + public UInt256 GetNonce(Address address) + { + Account? account = GetThroughCache(address); + return account?.Nonce ?? UInt256.Zero; + } + + public Keccak GetStorageRoot(Address address) + { + throw new InvalidOperationException($"no storage root in verkle trees"); + } + + public UInt256 GetBalance(Address address) + { + Account? account = GetThroughCache(address); + return account?.Balance ?? UInt256.Zero; + } + + public void InsertCode(Address address, ReadOnlyMemory code, IReleaseSpec releaseSpec, bool isGenesis = false) + { + Keccak codeHash; + if (code.Length == 0) + { + codeHash = Keccak.OfAnEmptyString; + } + else + { + codeHash = Keccak.Compute(code.Span); + _codeDb[codeHash.Bytes] = code.ToArray(); + } + + Account? account = GetThroughCache(address); + if (account is null) + { + throw new InvalidOperationException($"Account {address} is null when updating code hash"); + } + + if (account.CodeHash != codeHash) + { + if (_logger.IsTrace) _logger.Trace($" Update {address} C {account.CodeHash} -> {codeHash}"); + Account changedAccount = account.WithChangedCodeHash(codeHash, _codeDb[codeHash.Bytes]); + PushUpdate(address, changedAccount); + } + else if (releaseSpec.IsEip158Enabled && !isGenesis) + { + if (_logger.IsTrace) _logger.Trace($" Touch {address} (code hash)"); + if (account.IsEmpty) + { + PushTouch(address, account, releaseSpec, account.Balance.IsZero); + } + } + } + + private void SetNewBalance(Address address, in UInt256 balanceChange, IReleaseSpec releaseSpec, bool isSubtracting) + { + + Account GetThroughCacheCheckExists() + { + Account result = GetThroughCache(address); + if (result is null) + { + if (_logger.IsError) _logger.Error("Updating balance of a non-existing account"); + throw new InvalidOperationException("Updating balance of a non-existing account"); + } + + return result; + } + + bool isZero = balanceChange.IsZero; + if (isZero) + { + if (releaseSpec.IsEip158Enabled) + { + Account touched = GetThroughCacheCheckExists(); + if (_logger.IsTrace) _logger.Trace($" Touch {address} (balance)"); + if (touched.IsEmpty) + { + PushTouch(address, touched, releaseSpec, true); + } + } + + return; + } + + Account account = GetThroughCacheCheckExists(); + + if (isSubtracting && account.Balance < balanceChange) + { + throw new InsufficientBalanceException(address); + } + + UInt256 newBalance = isSubtracting ? account.Balance - balanceChange : account.Balance + balanceChange; + + Account changedAccount = account.WithChangedBalance(newBalance); + if (_logger.IsTrace) _logger.Trace($" Update {address} B {account.Balance} -> {newBalance} ({(isSubtracting ? "-" : "+")}{balanceChange})"); + PushUpdate(address, changedAccount); + } + + public void SubtractFromBalance(Address address, in UInt256 balanceChange, IReleaseSpec releaseSpec) + { + SetNewBalance(address, balanceChange, releaseSpec, true); + } + + public void AddToBalance(Address address, in UInt256 balanceChange, IReleaseSpec releaseSpec) + { + SetNewBalance(address, balanceChange, releaseSpec, false); + } + + /// + /// This is a coupling point between storage provider and state provider. + /// This is pointing at the architectural change likely required where Storage and State Provider are represented by a single world state class. + /// + /// + /// + public void UpdateStorageRoot(Address address, Keccak storageRoot) + { + throw new InvalidOperationException($"no storage root in verkle trees"); + } + + public void IncrementNonce(Address address) + { + Account? account = GetThroughCache(address); + if (account is null) + { + throw new InvalidOperationException($"Account {address} is null when incrementing nonce"); + } + + Account changedAccount = account.WithChangedNonce(account.Nonce + 1); + if (_logger.IsTrace) _logger.Trace($" Update {address} N {account.Nonce} -> {changedAccount.Nonce}"); + PushUpdate(address, changedAccount); + } + + public void DecrementNonce(Address address) + { + Account account = GetThroughCache(address); + if (account is null) + { + throw new InvalidOperationException($"Account {address} is null when decrementing nonce."); + } + + Account changedAccount = account.WithChangedNonce(account.Nonce - 1); + if (_logger.IsTrace) _logger.Trace($" Update {address} N {account.Nonce} -> {changedAccount.Nonce}"); + PushUpdate(address, changedAccount); + } + + public void TouchCode(Keccak codeHash) + { + if (_codeDb is WitnessingStore witnessingStore) + { + witnessingStore.Touch(codeHash.Bytes); + } + } + + public Keccak GetCodeHash(Address address) + { + Account account = GetThroughCache(address); + return account?.CodeHash ?? Keccak.OfAnEmptyString; + } + + public byte[] GetCode(Keccak codeHash) + { + byte[]? code = codeHash == Keccak.OfAnEmptyString ? Array.Empty() : _codeDb[codeHash.Bytes]; + if (code is null) + { + throw new InvalidOperationException($"Code {codeHash} is missing from the database."); + } + + return code; + } + + public byte[] GetCodeChunk(Address codeOwner, UInt256 codeChunk) + { + Pedersen? treeKey = AccountHeader.GetTreeKeyForCodeChunk(codeOwner.Bytes, codeChunk); + byte[]? chunk = _tree.Get(treeKey); + if (chunk is null) + { + throw new InvalidOperationException($"Code Chunk {chunk} is missing from the database."); + } + return chunk; + } + + public byte[] GetCode(Address address) + { + Account? account = GetThroughCache(address); + if (account is null) + { + return Array.Empty(); + } + + return GetCode(account.CodeHash); + } + + public void DeleteAccount(Address address) + { + throw new NotSupportedException("Verkle Trees does not support deletion of data from the tree"); + } + + public void ClearStorage(Address address) + { + throw new NotSupportedException("Verkle Trees does not support deletion of data from the tree"); + } + + public void CreateAccount(Address address, in UInt256 balance) + { + if (_logger.IsTrace) _logger.Trace($"Creating account: {address} with balance {balance}"); + Account account = balance.IsZero ? Account.TotallyEmpty : new Account(balance); + PushNew(address, account); + } + + + public void CreateAccount(Address address, in UInt256 balance, in UInt256 nonce) + { + if (_logger.IsTrace) _logger.Trace($"Creating account: {address} with balance {balance} and nonce {nonce}"); + Account account = (balance.IsZero && nonce.IsZero) ? Account.TotallyEmpty : new Account(nonce, balance, Keccak.EmptyTreeHash, Keccak.OfAnEmptyString); + PushNew(address, account); + } + + + + private readonly struct ChangeTrace + { + public ChangeTrace(Account? before, Account? after) + { + After = after; + Before = before; + } + + public ChangeTrace(Account? after) + { + After = after; + Before = null; + } + + public Account? Before { get; } + public Account? After { get; } + } + + + + private void ReportChanges(IStateTracer stateTracer, Dictionary trace) + { + foreach ((Address address, ChangeTrace change) in trace) + { + bool someChangeReported = false; + + Account? before = change.Before; + Account? after = change.After; + + UInt256? beforeBalance = before?.Balance; + UInt256? afterBalance = after?.Balance; + + UInt256? beforeNonce = before?.Nonce; + UInt256? afterNonce = after?.Nonce; + + Keccak? beforeCodeHash = before?.CodeHash; + Keccak? afterCodeHash = after?.CodeHash; + + if (beforeCodeHash != afterCodeHash) + { + byte[]? beforeCode = beforeCodeHash is null + ? null + : beforeCodeHash == Keccak.OfAnEmptyString + ? Array.Empty() + : _codeDb[beforeCodeHash.Bytes]; + byte[]? afterCode = afterCodeHash is null + ? null + : afterCodeHash == Keccak.OfAnEmptyString + ? Array.Empty() + : _codeDb[afterCodeHash.Bytes]; + + if (!((beforeCode?.Length ?? 0) == 0 && (afterCode?.Length ?? 0) == 0)) + { + stateTracer.ReportCodeChange(address, beforeCode, afterCode); + } + + someChangeReported = true; + } + + if (afterBalance != beforeBalance) + { + stateTracer.ReportBalanceChange(address, beforeBalance, afterBalance); + someChangeReported = true; + } + + if (afterNonce != beforeNonce) + { + stateTracer.ReportNonceChange(address, beforeNonce, afterNonce); + someChangeReported = true; + } + + if (!someChangeReported) + { + stateTracer.ReportAccountRead(address); + } + } + } + + private Account? GetState(Address address) + { + Db.Metrics.StateTreeReads++; + Pedersen headerTreeKey = AccountHeader.GetTreeKeyPrefixAccount(address.Bytes); + headerTreeKey.SuffixByte = AccountHeader.Version; + IEnumerable? versionVal = _tree.Get(headerTreeKey); + if (versionVal is null) return null; + UInt256 version = new((versionVal ?? Array.Empty()).ToArray()); + headerTreeKey.SuffixByte = AccountHeader.Balance; + UInt256 balance = new((_tree.Get(headerTreeKey) ?? Array.Empty()).ToArray()); + headerTreeKey.SuffixByte = AccountHeader.Nonce; + UInt256 nonce = new ((_tree.Get(headerTreeKey) ?? Array.Empty()).ToArray()); + headerTreeKey.SuffixByte = AccountHeader.CodeHash; + byte[]? codeHash = (_tree.Get(headerTreeKey) ?? Keccak.OfAnEmptyString.Bytes).ToArray(); + headerTreeKey.SuffixByte = AccountHeader.CodeSize; + UInt256 codeSize = new ((_tree.Get(headerTreeKey) ?? Array.Empty()).ToArray()); + + return new Account(balance, nonce, new Keccak(codeHash), codeSize, version); + } + + private void SetState(Address address, Account? account) + { + Db.Metrics.StateTreeWrites++; + + Pedersen headerTreeKey = AccountHeader.GetTreeKeyPrefixAccount(address.Bytes); + if (account != null) _tree.InsertStemBatch(headerTreeKey.StemAsSpan, account.ToVerkleDict()); + if (account!.Code is null) return; + _tree.SetCode(address, account.Code); + } + + private readonly HashSet
_readsForTracing = new HashSet
(); + + private Account? GetAndAddToCache(Address address) + { + Account? account = GetState(address); + if (account is not null) + { + PushJustCache(address, account); + } + else + { + // just for tracing - potential perf hit, maybe a better solution? + _readsForTracing.Add(address); + } + + return account; + } + + private Account? GetThroughCache(Address address) + { + if (_intraBlockCache.ContainsKey(address)) + { + return _changes[_intraBlockCache[address].Peek()]!.Account; + } + + Account account = GetAndAddToCache(address); + return account; + } + + private void PushJustCache(Address address, Account account) + { + Push(ChangeType.JustCache, address, account); + } + + private void PushUpdate(Address address, Account account) + { + Push(ChangeType.Update, address, account); + } + + private void PushTouch(Address address, Account account, IReleaseSpec releaseSpec, bool isZero) + { + if (isZero && releaseSpec.IsEip158IgnoredAccount(address)) return; + Push(ChangeType.Touch, address, account); + } + + private void Push(ChangeType changeType, Address address, Account? touchedAccount) + { + SetupCache(address); + if (changeType == ChangeType.Touch + && _changes[_intraBlockCache[address].Peek()]!.ChangeType == ChangeType.Touch) + { + return; + } + + IncrementChangePosition(); + _intraBlockCache[address].Push(_currentPosition); + _changes[_currentPosition] = new Change(changeType, address, touchedAccount); + } + + private void PushNew(Address address, Account account) + { + SetupCache(address); + IncrementChangePosition(); + _intraBlockCache[address].Push(_currentPosition); + _changes[_currentPosition] = new Change(ChangeType.New, address, account); + } + + private void IncrementChangePosition() + { + Resettable.IncrementPosition(ref _changes, ref _capacity, ref _currentPosition); + } + + private void SetupCache(Address address) + { + if (!_intraBlockCache.ContainsKey(address)) + { + _intraBlockCache[address] = new Stack(); + } + } + + public byte[] GetOriginal(in StorageCell storageCell) + { + return _storageProvider.GetOriginal(storageCell); + } + public byte[] Get(in StorageCell storageCell) + { + return _storageProvider.Get(storageCell).WithoutLeadingZeros().ToArray(); + } + public void Set(in StorageCell storageCell, byte[] newValue) + { + _storageProvider.Set(storageCell, newValue.PadLeft(32)); + } + public byte[] GetTransientState(in StorageCell storageCell) + { + return _storageProvider.GetTransientState(storageCell).WithoutLeadingZeros().ToArray(); + } + public void SetTransientState(in StorageCell storageCell, byte[] newValue) + { + _storageProvider.SetTransientState(storageCell, newValue.PadLeft(32)); + } + public void Reset() + { + if (_logger.IsTrace) _logger.Trace("Clearing state provider caches"); + _intraBlockCache.Reset(); + _committedThisRound.Reset(); + _readsForTracing.Clear(); + _currentPosition = Resettable.EmptyPosition; + Array.Clear(_changes, 0, _changes.Length); + + _storageProvider.Reset(); + } + + public void CommitTree(long blockNumber) + { + _tree.CommitTree(blockNumber); + } + + + // used in EthereumTests + public void SetNonce(Address address, in UInt256 nonce) + { + Account? account = GetThroughCache(address); + if (account is null) + { + throw new InvalidOperationException($"Account {address} is null when incrementing nonce"); + } + + Account changedAccount = account.WithChangedNonce(nonce); + if (_logger.IsTrace) _logger.Trace($" Update {address} N {account.Nonce} -> {changedAccount.Nonce}"); + PushUpdate(address, changedAccount); + } + private enum ChangeType + { + JustCache, + Touch, + Update, + New, + Delete + } + + private class Change + { + public Change(ChangeType type, Address address, Account? account) + { + ChangeType = type; + Address = address; + Account = account; + } + + public ChangeType ChangeType { get; } + public Address Address { get; } + public Account? Account { get; } + } + + public Snapshot TakeSnapshot(bool newTransactionStart = false) + { + return new Snapshot(_currentPosition, _storageProvider.TakeSnapshot(newTransactionStart)); + } + + public void Commit(IReleaseSpec releaseSpec, bool isGenesis = false) + { + _storageProvider.Commit(); + Commit(releaseSpec, (IStateTracer)NullStateTracer.Instance, isGenesis); + } + + public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool isGenesis = false) + { + _storageProvider.Commit(stateTracer); + Commit(releaseSpec, (IStateTracer)stateTracer, isGenesis); + } + + public void Commit(IReleaseSpec releaseSpec, IStateTracer stateTracer, bool isGenesis = false) + { + if (_currentPosition == -1) + { + if (_logger.IsTrace) _logger.Trace(" no state changes to commit"); + return; + } + + if (_logger.IsTrace) _logger.Trace($"Committing state changes (at {_currentPosition})"); + if (_changes[_currentPosition] is null) + { + throw new InvalidOperationException($"Change at current position {_currentPosition} was null when commiting {nameof(WorldState)}"); + } + + if (_changes[_currentPosition + 1] is not null) + { + throw new InvalidOperationException($"Change after current position ({_currentPosition} + 1) was not null when commiting {nameof(WorldState)}"); + } + + bool isTracing = stateTracer.IsTracingState; + Dictionary trace = null; + if (isTracing) + { + trace = new Dictionary(); + } + + for (int i = 0; i <= _currentPosition; i++) + { + Change change = _changes[_currentPosition - i]; + if (!isTracing && change!.ChangeType == ChangeType.JustCache) + { + continue; + } + + if (_committedThisRound.Contains(change!.Address)) + { + if (isTracing && change.ChangeType == ChangeType.JustCache) + { + trace[change.Address] = new ChangeTrace(change.Account, trace[change.Address].After); + } + + continue; + } + + // because it was not committed yet it means that the just cache is the only state (so it was read only) + if (isTracing && change.ChangeType == ChangeType.JustCache) + { + _readsForTracing.Add(change.Address); + continue; + } + + int forAssertion = _intraBlockCache[change.Address].Pop(); + if (forAssertion != _currentPosition - i) + { + throw new InvalidOperationException($"Expected checked value {forAssertion} to be equal to {_currentPosition} - {i}"); + } + + _committedThisRound.Add(change.Address); + + switch (change.ChangeType) + { + case ChangeType.JustCache: + { + break; + } + case ChangeType.Touch: + case ChangeType.Update: + { + if (change.Account != null && releaseSpec.IsEip158Enabled && change.Account.IsEmpty && !isGenesis) + { + if (_logger.IsTrace) _logger.Trace($" Commit remove empty {change.Address} B = {change.Account.Balance} N = {change.Account.Nonce}"); + SetState(change.Address, null); + if (isTracing) + { + trace[change.Address] = new ChangeTrace(null); + } + } + else + { + if (_logger.IsTrace) + if (change.Account != null) + _logger.Trace($" Commit update {change.Address} B = {change.Account.Balance} N = {change.Account.Nonce} C = {change.Account.CodeHash}"); + SetState(change.Address, change.Account); + if (isTracing) + { + trace[change.Address] = new ChangeTrace(change.Account); + } + } + + break; + } + case ChangeType.New: + { + if (change.Account != null && (!releaseSpec.IsEip158Enabled || !change.Account.IsEmpty || isGenesis)) + { + if (_logger.IsTrace) _logger.Trace($" Commit create {change.Address} B = {change.Account.Balance} N = {change.Account.Nonce}"); + SetState(change.Address, change.Account); + if (isTracing) + { + trace[change.Address] = new ChangeTrace(change.Account); + } + } + + break; + } + case ChangeType.Delete: + { + if (_logger.IsTrace) _logger.Trace($" Commit remove {change.Address}"); + bool wasItCreatedNow = false; + while (_intraBlockCache[change.Address].Count > 0) + { + int previousOne = _intraBlockCache[change.Address].Pop(); + wasItCreatedNow |= _changes[previousOne]!.ChangeType == ChangeType.New; + if (wasItCreatedNow) + { + break; + } + } + + if (!wasItCreatedNow) + { + SetState(change.Address, null); + if (isTracing) + { + trace[change.Address] = new ChangeTrace(null); + } + } + + break; + } + default: + throw new ArgumentOutOfRangeException(); + } + } + + if (isTracing) + { + foreach (Address nullRead in _readsForTracing) + { + // // this may be enough, let us write tests + stateTracer.ReportAccountRead(nullRead); + } + } + + _tree.Commit(); + Resettable.Reset(ref _changes, ref _capacity, ref _currentPosition); + _committedThisRound.Reset(); + _readsForTracing.Clear(); + _intraBlockCache.Reset(); + + if (isTracing) + { + ReportChanges(stateTracer, trace); + } + } + + public void Restore(Snapshot snapshot) + { + Restore(snapshot.StateSnapshot); + _storageProvider.Restore(snapshot.StorageSnapshot); + } + + public void Restore(int snapshot) + { + if (snapshot > _currentPosition) + { + throw new InvalidOperationException($"{nameof(WorldState)} tried to restore snapshot {snapshot} beyond current position {_currentPosition}"); + } + + if (_logger.IsTrace) _logger.Trace($"Restoring state snapshot {snapshot}"); + if (snapshot == _currentPosition) + { + return; + } + + for (int i = 0; i < _currentPosition - snapshot; i++) + { + Change change = _changes[_currentPosition - i]; + if (_intraBlockCache[change!.Address].Count == 1) + { + if (change.ChangeType == ChangeType.JustCache) + { + int actualPosition = _intraBlockCache[change.Address].Pop(); + if (actualPosition != _currentPosition - i) + { + throw new InvalidOperationException($"Expected actual position {actualPosition} to be equal to {_currentPosition} - {i}"); + } + + _keptInCache.Add(change); + _changes[actualPosition] = null; + continue; + } + } + + _changes[_currentPosition - i] = null; // TODO: temp, ??? + int forChecking = _intraBlockCache[change.Address].Pop(); + if (forChecking != _currentPosition - i) + { + throw new InvalidOperationException($"Expected checked value {forChecking} to be equal to {_currentPosition} - {i}"); + } + + if (_intraBlockCache[change.Address].Count == 0) + { + _intraBlockCache.Remove(change.Address); + } + } + + _currentPosition = snapshot; + foreach (Change kept in _keptInCache) + { + _currentPosition++; + _changes[_currentPosition] = kept; + _intraBlockCache[kept.Address].Push(_currentPosition); + } + + _keptInCache.Clear(); + } +} diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index e73f3081a5d..bf16a32e109 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -23,6 +23,8 @@ namespace Nethermind.State { public class WorldState : IWorldState { + public StateType StateType => StateType.Merkle; + private readonly StateProvider _stateProvider; private readonly PersistentStorageProvider _persistentStorageProvider; private readonly TransientStorageProvider _transientStorageProvider; @@ -127,6 +129,9 @@ public void TouchCode(Keccak codeHash) _stateProvider.TouchCode(codeHash); } + public byte[] GetCodeChunk(Address codeOwner, UInt256 chunkId) => + throw new NotSupportedException("Merkle state does not support code chunks"); + public UInt256 GetNonce(Address address) { return _stateProvider.GetNonce(address); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs index 5418a9174d1..cc28320fa05 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs @@ -47,6 +47,7 @@ using NUnit.Framework; using BlockTree = Nethermind.Blockchain.BlockTree; using Nethermind.Core.Specs; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Synchronization.Test { @@ -943,9 +944,9 @@ public virtual IBlockTree BlockTree public ResponseBuilder ResponseBuilder => _responseBuilder ??= new ResponseBuilder(BlockTree, TestHeaderMapping); - private ProgressTracker? _progressTracker; + private SnapProgressTracker? _progressTracker; - private ProgressTracker ProgressTracker => _progressTracker ??= + private SnapProgressTracker SnapProgressTracker => _progressTracker ??= new(BlockTree, _stateDb, LimboLogs.Instance); private ISyncProgressResolver? _syncProgressResolver; @@ -955,7 +956,7 @@ public virtual IBlockTree BlockTree BlockTree, ReceiptStorage, new TrieStore(_stateDb, LimboLogs.Instance), - ProgressTracker, + SnapProgressTracker, syncConfig, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs index 094cf72d964..bdf95e1c44f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs @@ -8,6 +8,7 @@ using Nethermind.Synchronization.FastSync; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; using NSubstitute; using NUnit.Framework; @@ -18,6 +19,7 @@ public class SyncDbTunerTests private ITunableDb.TuneType _tuneType = ITunableDb.TuneType.HeavyWrite; private SyncConfig _syncConfig = null!; private ISyncFeed? _snapSyncFeed; + private ISyncFeed? _verkleSyncFeed; private ISyncFeed? _bodiesSyncFeed; private ISyncFeed? _receiptSyncFeed; private ITunableDb _stateDb = null!; @@ -35,6 +37,7 @@ public void Setup() TuneDbMode = _tuneType }; _snapSyncFeed = Substitute.For?>(); + _verkleSyncFeed = Substitute.For?>(); _bodiesSyncFeed = Substitute.For?>(); _receiptSyncFeed = Substitute.For?>(); _stateDb = Substitute.For(); @@ -45,6 +48,7 @@ public void Setup() _tuner = new SyncDbTuner( _syncConfig, _snapSyncFeed, + _verkleSyncFeed, _bodiesSyncFeed, _receiptSyncFeed, _stateDb, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 042c29b0b81..4b903911a48 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -25,6 +25,7 @@ using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; using Nethermind.Trie.Pruning; using NSubstitute; using NUnit.Framework; @@ -53,15 +54,18 @@ public async Task Setup() NodeStatsManager stats = new(timerFactory, LimboLogs.Instance); _pool = new SyncPeerPool(_blockTree, stats, new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), LimboLogs.Instance, 25); SyncConfig syncConfig = new(); - ProgressTracker progressTracker = new(_blockTree, dbProvider.StateDb, LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(_blockTree, dbProvider.StateDb, LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); + + VerkleProgressTracker verkleProgressTracker = new(_blockTree, dbProvider.StateDb, LimboLogs.Instance); + VerkleSyncProvider verkleProvider = new(verkleProgressTracker, dbProvider, LimboLogs.Instance); TrieStore trieStore = new(_stateDb, LimboLogs.Instance); SyncProgressResolver resolver = new( _blockTree, _receiptStorage, trieStore, - progressTracker, + snapProgressTracker, syncConfig, LimboLogs.Instance); TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); @@ -88,6 +92,7 @@ public async Task Setup() syncModeSelector, syncConfig, snapProvider, + verkleProvider, blockDownloaderFactory, pivot, syncReport, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs index 4f2ee0f0b5e..7ee7b0f610d 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs @@ -400,7 +400,7 @@ public ScenarioBuilder IfThisNodeJustFinishedFastBlocksAndFastSync(FastBlocksSta SyncProgressResolver.FindBestProcessedBlock().Returns(0); SyncProgressResolver.IsFastBlocksFinished().Returns(fastBlocksState); SyncProgressResolver.ChainDifficulty.Returns(UInt256.Zero); - SyncProgressResolver.IsSnapGetRangesFinished().Returns(snapRangesFinished); + SyncProgressResolver.IsGetRangesFinished().Returns(snapRangesFinished); return "just after fast blocks and fast sync"; } ); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs index fd034f3aad7..78610b6b5ef 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs @@ -21,8 +21,8 @@ public class ProgressTrackerTests public async Task Did_not_have_race_issue() { BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block.TestObject).TestObject; - ProgressTracker progressTracker = new ProgressTracker(blockTree, new MemDb(), LimboLogs.Instance); - progressTracker.EnqueueStorageRange(new StorageRange() + SnapProgressTracker snapProgressTracker = new SnapProgressTracker(blockTree, new MemDb(), LimboLogs.Instance); + snapProgressTracker.EnqueueStorageRange(new StorageRange() { Accounts = Array.Empty(), }); @@ -32,9 +32,9 @@ public async Task Did_not_have_race_issue() { for (int i = 0; i < loopIteration; i++) { - (SnapSyncBatch snapSyncBatch, bool ok) = progressTracker.GetNextRequest(); + (SnapSyncBatch snapSyncBatch, bool ok) = snapProgressTracker.GetNextRequest(); ok.Should().BeFalse(); - progressTracker.EnqueueStorageRange(snapSyncBatch.StorageRangeRequest!); + snapProgressTracker.EnqueueStorageRange(snapSyncBatch.StorageRangeRequest!); } }); @@ -42,7 +42,7 @@ public async Task Did_not_have_race_issue() { for (int i = 0; i < loopIteration; i++) { - progressTracker.IsSnapGetRangesFinished().Should().BeFalse(); + snapProgressTracker.IsGetRangesFinished().Should().BeFalse(); } }); @@ -54,33 +54,33 @@ public async Task Did_not_have_race_issue() public void Will_create_multiple_get_address_range_request() { BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block.TestObject).TestObject; - ProgressTracker progressTracker = new ProgressTracker(blockTree, new MemDb(), LimboLogs.Instance, 4); + SnapProgressTracker snapProgressTracker = new SnapProgressTracker(blockTree, new MemDb(), LimboLogs.Instance, 4); - (SnapSyncBatch request, bool finished) = progressTracker.GetNextRequest(); + (SnapSyncBatch request, bool finished) = snapProgressTracker.GetNextRequest(); request.AccountRangeRequest.Should().NotBeNull(); request.AccountRangeRequest!.StartingHash.Bytes[0].Should().Be(0); request.AccountRangeRequest.LimitHash!.Bytes[0].Should().Be(64); finished.Should().BeFalse(); - (request, finished) = progressTracker.GetNextRequest(); + (request, finished) = snapProgressTracker.GetNextRequest(); request.AccountRangeRequest.Should().NotBeNull(); request.AccountRangeRequest!.StartingHash.Bytes[0].Should().Be(64); request.AccountRangeRequest.LimitHash!.Bytes[0].Should().Be(128); finished.Should().BeFalse(); - (request, finished) = progressTracker.GetNextRequest(); + (request, finished) = snapProgressTracker.GetNextRequest(); request.AccountRangeRequest.Should().NotBeNull(); request.AccountRangeRequest!.StartingHash.Bytes[0].Should().Be(128); request.AccountRangeRequest.LimitHash!.Bytes[0].Should().Be(192); finished.Should().BeFalse(); - (request, finished) = progressTracker.GetNextRequest(); + (request, finished) = snapProgressTracker.GetNextRequest(); request.AccountRangeRequest.Should().NotBeNull(); request.AccountRangeRequest!.StartingHash.Bytes[0].Should().Be(192); request.AccountRangeRequest.LimitHash!.Bytes[0].Should().Be(255); finished.Should().BeFalse(); - (request, finished) = progressTracker.GetNextRequest(); + (request, finished) = snapProgressTracker.GetNextRequest(); request.Should().BeNull(); finished.Should().BeFalse(); } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index 50291fceec5..e382b9122ce 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -114,8 +114,8 @@ public void RecreateAccountStateFromOneRangeWithNonExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AddRangeResult result = snapProvider.AddAccountRange(1, rootHash, Keccak.Zero, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray()); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -134,8 +134,8 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray()); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -151,8 +151,8 @@ public void RecreateAccountStateFromOneRangeWithoutProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths); Assert.That(result, Is.EqualTo(AddRangeResult.OK)); @@ -169,8 +169,8 @@ public void RecreateAccountStateFromMultipleRange() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(Keccak.Zero.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[1].Path.Bytes); @@ -207,8 +207,8 @@ public void RecreateAccountStateFromMultipleRange_InReverseOrder() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[4].Path.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); @@ -242,8 +242,8 @@ public void RecreateAccountStateFromMultipleRange_OutOfOrder() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[4].Path.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); @@ -277,8 +277,8 @@ public void RecreateAccountStateFromMultipleOverlappingRange() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(Keccak.Zero.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[2].Path.Bytes); @@ -321,8 +321,8 @@ public void CorrectlyDetermineHasMoreChildren() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(Keccak.Zero.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[1].Path.Bytes); @@ -375,8 +375,8 @@ public void CorrectlyDetermineMaxKeccakExist() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(ac1.Path.Bytes, tree); byte[][] lastProof = CreateProofForPath(ac2.Path.Bytes, tree); @@ -414,8 +414,8 @@ public void MissingAccountFromRange() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); byte[][] firstProof = CreateProofForPath(Keccak.Zero.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[1].Path.Bytes); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs index de6b1882a05..3e99d3587a2 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs @@ -52,8 +52,8 @@ public void RecreateStorageStateFromOneRangeWithNonExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddStorageRange(1, null, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); @@ -72,8 +72,8 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof() MemDb db = new(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddStorageRange(1, null, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); @@ -88,8 +88,8 @@ public void RecreateStorageStateFromOneRangeWithoutProof() MemDb db = new MemDb(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); var result = snapProvider.AddStorageRange(1, null, rootHash, TestItem.Tree.SlotsWithPaths[0].Path, TestItem.Tree.SlotsWithPaths); @@ -105,8 +105,8 @@ public void RecreateAccountStateFromMultipleRange() MemDb db = new MemDb(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new Keccak[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); @@ -140,8 +140,8 @@ public void MissingAccountFromRange() MemDb db = new MemDb(); DbProvider dbProvider = new(DbModeHint.Mem); dbProvider.RegisterDb(DbNames.State, db); - ProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new Keccak[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs index 8f357d3c055..f05c894c2a6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs @@ -11,6 +11,7 @@ using Nethermind.Logging; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; using Nethermind.Trie.Pruning; using NSubstitute; using NUnit.Framework; @@ -29,10 +30,11 @@ public void Header_block_is_0_when_no_header_was_suggested() IDb stateDb = new MemDb(); SyncConfig syncConfig = new(); syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, + syncConfig, LimboLogs.Instance); blockTree.BestSuggestedHeader.Returns((BlockHeader)null); Assert.That(syncProgressResolver.FindBestHeader(), Is.EqualTo(0)); } @@ -45,10 +47,10 @@ public void Best_block_is_0_when_no_block_was_suggested() IDb stateDb = new MemDb(); SyncConfig syncConfig = new(); syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); blockTree.BestSuggestedBody.Returns((Block)null); Assert.That(syncProgressResolver.FindBestFullBlock(), Is.EqualTo(0)); } @@ -61,10 +63,10 @@ public void Best_state_is_head_when_there_are_no_suggested_blocks() IDb stateDb = new MemDb(); SyncConfig syncConfig = new(); syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); var head = Build.A.Block.WithHeader(Build.A.BlockHeader.WithNumber(5).WithStateRoot(TestItem.KeccakA).TestObject).TestObject; blockTree.Head.Returns(head); blockTree.BestSuggestedHeader.Returns(head.Header); @@ -80,10 +82,10 @@ public void Best_state_is_suggested_if_there_is_suggested_block_with_state() IDb stateDb = new MemDb(); SyncConfig syncConfig = new(); syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); var head = Build.A.Block.WithHeader(Build.A.BlockHeader.WithNumber(5).WithStateRoot(TestItem.KeccakA).TestObject).TestObject; var suggested = Build.A.BlockHeader.WithNumber(6).WithStateRoot(TestItem.KeccakB).TestObject; blockTree.Head.Returns(head); @@ -103,10 +105,10 @@ public void Best_state_is_head_if_there_is_suggested_block_without_state() IDb stateDb = new MemDb(); SyncConfig syncConfig = new(); syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); var head = Build.A.Block.WithHeader(Build.A.BlockHeader.WithNumber(5).WithStateRoot(TestItem.KeccakA).TestObject).TestObject; var suggested = Build.A.BlockHeader.WithNumber(6).WithStateRoot(TestItem.KeccakB).TestObject; blockTree.Head.Returns(head); @@ -126,10 +128,10 @@ public void Is_fast_block_finished_returns_true_when_no_fast_block_sync_is_used( SyncConfig syncConfig = new(); syncConfig.FastBlocks = false; syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); Assert.True(syncProgressResolver.IsFastBlocksHeadersFinished()); Assert.True(syncProgressResolver.IsFastBlocksBodiesFinished()); Assert.True(syncProgressResolver.IsFastBlocksReceiptsFinished()); @@ -147,12 +149,12 @@ public void Is_fast_block_headers_finished_returns_false_when_headers_not_downlo syncConfig.DownloadBodiesInFastSync = true; syncConfig.DownloadReceiptsInFastSync = true; syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(2).WithStateRoot(TestItem.KeccakA).TestObject); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); Assert.False(syncProgressResolver.IsFastBlocksHeadersFinished()); } @@ -168,13 +170,13 @@ public void Is_fast_block_bodies_finished_returns_false_when_blocks_not_download syncConfig.DownloadBodiesInFastSync = true; syncConfig.DownloadReceiptsInFastSync = true; syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); blockTree.LowestInsertedBodyNumber.Returns(2); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); Assert.False(syncProgressResolver.IsFastBlocksBodiesFinished()); } @@ -190,7 +192,7 @@ public void Is_fast_block_receipts_finished_returns_false_when_receipts_not_down syncConfig.DownloadBodiesInFastSync = true; syncConfig.DownloadReceiptsInFastSync = true; syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); blockTree.LowestInsertedBodyNumber.Returns(1); @@ -198,7 +200,7 @@ public void Is_fast_block_receipts_finished_returns_false_when_receipts_not_down TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); SyncProgressResolver syncProgressResolver = new( - blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); Assert.False(syncProgressResolver.IsFastBlocksReceiptsFinished()); } @@ -213,14 +215,14 @@ public void Is_fast_block_bodies_finished_returns_true_when_bodies_not_downloade syncConfig.DownloadBodiesInFastSync = false; syncConfig.DownloadReceiptsInFastSync = true; syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); blockTree.LowestInsertedBodyNumber.Returns(2); receiptStorage.LowestInsertedReceiptBlockNumber.Returns(1); TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); Assert.True(syncProgressResolver.IsFastBlocksBodiesFinished()); } @@ -235,7 +237,7 @@ public void Is_fast_block_receipts_finished_returns_true_when_receipts_not_downl syncConfig.DownloadBodiesInFastSync = true; syncConfig.DownloadReceiptsInFastSync = false; syncConfig.PivotNumber = "1"; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(blockTree, stateDb, LimboLogs.Instance); blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); blockTree.LowestInsertedBodyNumber.Returns(1); @@ -243,7 +245,7 @@ public void Is_fast_block_receipts_finished_returns_true_when_receipts_not_downl TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); SyncProgressResolver syncProgressResolver = new( - blockTree, receiptStorage, trieStore, progressTracker, syncConfig, LimboLogs.Instance); + blockTree, receiptStorage, trieStore, snapProgressTracker, syncConfig, LimboLogs.Instance); Assert.True(syncProgressResolver.IsFastBlocksReceiptsFinished()); } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 7a1330b2f21..55866d07670 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -43,6 +43,7 @@ using BlockTree = Nethermind.Blockchain.BlockTree; using Nethermind.Synchronization.SnapSync; using Nethermind.Config; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Synchronization.Test { @@ -342,11 +343,13 @@ private SyncTestContext CreateSyncManager(int index) new BlocksConfig(), logManager); - ProgressTracker progressTracker = new(tree, dbProvider.StateDb, LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(tree, dbProvider.StateDb, LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); + VerkleProgressTracker verkleProgressTracker = new VerkleProgressTracker(tree, dbProvider.StateDb, LimboLogs.Instance); + VerkleSyncProvider verkleProvider = new(verkleProgressTracker, dbProvider, LimboLogs.Instance); SyncProgressResolver resolver = new( - tree, receiptStorage, trieStore, progressTracker, syncConfig, logManager); + tree, receiptStorage, trieStore, snapProgressTracker, syncConfig, logManager); TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); MultiSyncModeSelector selector = new(resolver, syncPeerPool, syncConfig, No.BeaconSync, bestPeerStrategy, logManager); Pivot pivot = new(syncConfig); @@ -370,6 +373,7 @@ private SyncTestContext CreateSyncManager(int index) StaticSelector.Full, syncConfig, snapProvider, + verkleProvider, blockDownloaderFactory, pivot, syncReport, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index f2314f57578..455ae8e41cd 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -41,6 +41,7 @@ using NSubstitute; using NUnit.Framework; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Synchronization.Test { @@ -320,15 +321,18 @@ ISyncConfig GetSyncConfig() => PoSSwitcher poSSwitcher = new(mergeConfig, syncConfig, dbProvider.MetadataDb, BlockTree, new TestSingleReleaseSpecProvider(Constantinople.Instance), _logManager); IBeaconPivot beaconPivot = new BeaconPivot(syncConfig, dbProvider.MetadataDb, BlockTree, _logManager); - ProgressTracker progressTracker = new(BlockTree, dbProvider.StateDb, LimboLogs.Instance); - SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + SnapProgressTracker snapProgressTracker = new(BlockTree, dbProvider.StateDb, LimboLogs.Instance); + SnapProvider snapProvider = new(snapProgressTracker, dbProvider, LimboLogs.Instance); + + VerkleProgressTracker verkleProgressTracker = new(BlockTree, dbProvider.StateDb, LimboLogs.Instance); + VerkleSyncProvider verkleProvider = new(verkleProgressTracker, dbProvider, LimboLogs.Instance); TrieStore trieStore = new(stateDb, LimboLogs.Instance); SyncProgressResolver syncProgressResolver = new( BlockTree, NullReceiptStorage.Instance, trieStore, - progressTracker, + snapProgressTracker, syncConfig, _logManager); @@ -372,6 +376,7 @@ ISyncConfig GetSyncConfig() => syncModeSelector, syncConfig, snapProvider, + verkleProvider, blockDownloaderFactory, pivot, poSSwitcher, @@ -404,6 +409,7 @@ ISyncConfig GetSyncConfig() => syncModeSelector, syncConfig, snapProvider, + verkleProvider, blockDownloaderFactory, pivot, syncReport, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/RecreateStateFromAccountRangesTests.cs new file mode 100644 index 00000000000..7a8832baac4 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/RecreateStateFromAccountRangesTests.cs @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +#nullable disable + +using System; +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Synchronization.VerkleSync; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree.Proofs; +using Nethermind.Verkle.Tree.Sync; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.VerkleSync +{ + [TestFixture] + public class RecreateVerkleStateFromAccountRangesTests + { + private VerkleStateTree _inputTree; + + [OneTimeSetUp] + public void Setup() + { + _inputTree = TestItem.Tree.GetVerkleStateTree(null); + } + + [Test] + public void RecreateAccountStateFromOneRangeWithNonExistenceProof() + { + VerkleProof proof = + _inputTree.CreateVerkleRangeProof(Keccak.Zero.Bytes[..31], TestItem.Tree.stem5, out Banderwagon rootPoint); + + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + VerkleSyncProvider verkleSyncProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + AddRangeResult result = verkleSyncProvider.AddSubTreeRange(1, rootPoint, Keccak.Zero.Bytes[..31], TestItem.Tree.SubTreesWithPaths, proof, TestItem.Tree.stem5); + Assert.That(result, Is.EqualTo(AddRangeResult.OK)); + + Console.WriteLine(dbProvider.InternalNodesDb.GetSize()); + } + + [Test] + public void RecreateAccountStateFromOneRangeWithExistenceProof() + { + VerkleProof proof = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem0, TestItem.Tree.stem5, out Banderwagon rootPoint); + + + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + VerkleSyncProvider verkleSyncProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + AddRangeResult result = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem0, TestItem.Tree.SubTreesWithPaths, proof, TestItem.Tree.stem5); + Assert.That(result, Is.EqualTo(AddRangeResult.OK)); + } + + [Test] + public void RecreateAccountStateFromMultipleRange() + { + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + VerkleSyncProvider verkleSyncProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + + PathWithSubTree[] pathWithSubTrees = TestItem.Tree.SubTreesWithPaths; + + VerkleProof proof1 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem0, TestItem.Tree.stem1, out Banderwagon rootPoint); + VerkleProof proof2 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem2, TestItem.Tree.stem3, out rootPoint); + VerkleProof proof3 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem4, TestItem.Tree.stem5, out rootPoint); + + AddRangeResult result1 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem0, pathWithSubTrees[..2], proof1, TestItem.Tree.stem1); + Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result2 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem2, pathWithSubTrees[2..4], proof2, TestItem.Tree.stem3); + Assert.That(result2, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result3 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem4, pathWithSubTrees[4..], proof3, TestItem.Tree.stem5); + Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); + } + + [Test] + public void RecreateAccountStateFromMultipleRange_InReverseOrder() + { + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + VerkleSyncProvider verkleSyncProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + PathWithSubTree[] pathWithSubTrees = TestItem.Tree.SubTreesWithPaths; + + VerkleProof proof1 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem0, TestItem.Tree.stem1, out Banderwagon rootPoint); + VerkleProof proof2 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem2, TestItem.Tree.stem3, out rootPoint); + VerkleProof proof3 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem4, TestItem.Tree.stem5, out rootPoint); + + AddRangeResult result1 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem4, pathWithSubTrees[4..], proof3, TestItem.Tree.stem5); + Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result2 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem2, pathWithSubTrees[2..4], proof2, TestItem.Tree.stem3); + Assert.That(result2, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result3 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem0, pathWithSubTrees[..2], proof1, TestItem.Tree.stem1); + Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); + } + + [Test] + public void RecreateAccountStateFromMultipleRange_OutOfOrder() + { + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + VerkleSyncProvider verkleSyncProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + PathWithSubTree[] pathWithSubTrees = TestItem.Tree.SubTreesWithPaths; + + VerkleProof proof1 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem0, TestItem.Tree.stem1, out Banderwagon rootPoint); + VerkleProof proof2 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem2, TestItem.Tree.stem3, out rootPoint); + VerkleProof proof3 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem4, TestItem.Tree.stem5, out rootPoint); + + AddRangeResult result2 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem2, pathWithSubTrees[2..4], proof2, TestItem.Tree.stem3); + Assert.That(result2, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result1 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem0, pathWithSubTrees[..2], proof1, TestItem.Tree.stem1); + Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result3 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem4, pathWithSubTrees[4..], proof3, TestItem.Tree.stem5); + Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); + } + + [Test] + public void RecreateAccountStateFromMultipleOverlappingRange() + { + IDbProvider dbProvider = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleProgressTracker progressTracker = new(null, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + VerkleSyncProvider verkleSyncProvider = new(progressTracker, dbProvider, LimboLogs.Instance); + + PathWithSubTree[] pathWithSubTrees = TestItem.Tree.SubTreesWithPaths; + + VerkleProof proof1 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem0, TestItem.Tree.stem2, out Banderwagon rootPoint); + VerkleProof proof2 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem2, TestItem.Tree.stem3, out rootPoint); + VerkleProof proof3 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem3, TestItem.Tree.stem4, out rootPoint); + VerkleProof proof4 = + _inputTree.CreateVerkleRangeProof(TestItem.Tree.stem4, TestItem.Tree.stem5, out rootPoint); + + AddRangeResult result2 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem0, pathWithSubTrees[..3], proof1, TestItem.Tree.stem2); + Assert.That(result2, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result1 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem2, pathWithSubTrees[2..4], proof2, TestItem.Tree.stem3); + Assert.That(result1, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result3 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem3, pathWithSubTrees[3..5], proof3, TestItem.Tree.stem4); + Assert.That(result3, Is.EqualTo(AddRangeResult.OK)); + + AddRangeResult result4 = verkleSyncProvider.AddSubTreeRange(1, rootPoint, TestItem.Tree.stem4, pathWithSubTrees[4..6], proof4, TestItem.Tree.stem5); + Assert.That(result4, Is.EqualTo(AddRangeResult.OK)); + } + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleProgressTrackerTest.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleProgressTrackerTest.cs new file mode 100644 index 00000000000..4030d97536c --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleProgressTrackerTest.cs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Synchronization.VerkleSync; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.VerkleSync; + +public class VerkleProgressTrackerTest +{ + [Test] + public void Will_create_multiple_get_address_range_request() + { + BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block.TestObject).TestObject; + VerkleProgressTracker progressTracker = new(blockTree, new MemDb(), LimboLogs.Instance, 4); + + (VerkleSyncBatch request, bool finished) = progressTracker.GetNextRequest(); + request.SubTreeRangeRequest.Should().NotBeNull(); + request.SubTreeRangeRequest!.StartingStem.Bytes[0].Should().Be(0); + request.SubTreeRangeRequest.LimitStem!.Bytes[0].Should().Be(64); + finished.Should().BeFalse(); + + (request, finished) = progressTracker.GetNextRequest(); + request.SubTreeRangeRequest.Should().NotBeNull(); + request.SubTreeRangeRequest!.StartingStem.Bytes[0].Should().Be(64); + request.SubTreeRangeRequest.LimitStem!.Bytes[0].Should().Be(128); + finished.Should().BeFalse(); + + (request, finished) = progressTracker.GetNextRequest(); + request.SubTreeRangeRequest.Should().NotBeNull(); + request.SubTreeRangeRequest!.StartingStem.Bytes[0].Should().Be(128); + request.SubTreeRangeRequest.LimitStem!.Bytes[0].Should().Be(192); + finished.Should().BeFalse(); + + (request, finished) = progressTracker.GetNextRequest(); + request.SubTreeRangeRequest.Should().NotBeNull(); + request.SubTreeRangeRequest!.StartingStem.Bytes[0].Should().Be(192); + request.SubTreeRangeRequest.LimitStem!.Bytes[0].Should().Be(255); + finished.Should().BeFalse(); + + (request, finished) = progressTracker.GetNextRequest(); + request.Should().BeNull(); + finished.Should().BeFalse(); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncBatchTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncBatchTests.cs new file mode 100644 index 00000000000..bb72d8ab044 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncBatchTests.cs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Synchronization.VerkleSync; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.VerkleSync; + +public class VerkleSyncBatchTests +{ + [Test] + public void TestAccountRangeToString() + { + VerkleSyncBatch batch = new() + { + SubTreeRangeRequest = new SubTreeRange(Pedersen.Zero, Stem.MaxValue.Bytes, Keccak.Compute("abc").Bytes[..31], 999) + }; + + Console.WriteLine(batch.ToString()); + batch.ToString().Should().Be("SubTreeRange: (999, 0x0000000000000000000000000000000000000000000000000000000000000000, 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff, 0x4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c)"); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/AnalyzeResponsePerPeerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/AnalyzeResponsePerPeerTests.cs new file mode 100644 index 00000000000..25621578ba4 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/AnalyzeResponsePerPeerTests.cs @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Logging; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.VerkleSync; +using NSubstitute; +using NUnit.Framework; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.Test.VerkleSync.VerkleSyncFeed; + +[TestFixture] +internal class AnalyzeResponsePerPeerTests +{ + [Test] + public void Test01() + { + PeerInfo peer1 = new(null!); + PeerInfo peer2 = new(null!); + + ISyncModeSelector selector = Substitute.For(); + IVerkleSyncProvider verkleProvider = Substitute.For(); + + Synchronization.VerkleSync.VerkleSyncFeed feed = new(selector, verkleProvider, LimboLogs.Instance); + + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + + var result = feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + + Assert.That(result, Is.EqualTo(SyncResponseHandlingResult.LesserQuality)); + + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + result = feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + Assert.That(result, Is.EqualTo(SyncResponseHandlingResult.LesserQuality)); + } + + [Test] + public void Test02() + { + PeerInfo peer1 = new(null!); + PeerInfo peer2 = new(null!); + + ISyncModeSelector selector = Substitute.For(); + IVerkleSyncProvider verkleProvider = Substitute.For(); + + Synchronization.VerkleSync.VerkleSyncFeed feed = new(selector, verkleProvider, LimboLogs.Instance); + + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + + var result = feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + + Assert.That(result, Is.EqualTo(SyncResponseHandlingResult.LesserQuality)); + + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + result = feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + Assert.That(result, Is.EqualTo(SyncResponseHandlingResult.OK)); + } + + [Test] + public void Test03() + { + PeerInfo peer1 = new(null!); + PeerInfo peer2 = new(null!); + + ISyncModeSelector selector = Substitute.For(); + IVerkleSyncProvider verkleProvider = Substitute.For(); + + Synchronization.VerkleSync.VerkleSyncFeed feed = new(selector, verkleProvider, LimboLogs.Instance); + + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer2); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + feed.AnalyzeResponsePerPeer(AddRangeResult.ExpiredRootHash, peer1); + var result = feed.AnalyzeResponsePerPeer(AddRangeResult.DifferentRootHash, peer1); + Assert.That(result, Is.EqualTo(SyncResponseHandlingResult.OK)); + + verkleProvider.Received(1).UpdatePivot(); + } + + [Test] + public void Test04() + { + PeerInfo peer1 = new(null!); + + ISyncModeSelector selector = Substitute.For(); + IVerkleSyncProvider verkleProvider = Substitute.For(); + + Synchronization.VerkleSync.VerkleSyncFeed feed = new(selector, verkleProvider, LimboLogs.Instance); + + for (int i = 0; i < 200; i++) + { + feed.AnalyzeResponsePerPeer(AddRangeResult.OK, peer1); + } + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/VerkleSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/VerkleSyncFeedTests.cs new file mode 100644 index 00000000000..8c6320c31d0 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/VerkleSyncFeedTests.cs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Core.Verkle; +using Nethermind.Logging; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.Peers; +using Nethermind.Synchronization.VerkleSync; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.VerkleSync.VerkleSyncFeed; + +public class VerkleSyncFeedTests +{ + [Test] + public void WhenAccountRequestEmpty_ReturnNoProgress() + { + IVerkleSyncProvider verkleProvider = Substitute.For(); + Synchronization.VerkleSync.VerkleSyncFeed feed = new( + Substitute.For(), verkleProvider, LimboLogs.Instance); + + verkleProvider.AddSubTreeRange(Arg.Any(), Arg.Any()) + .Returns(AddRangeResult.ExpiredRootHash); + + VerkleSyncBatch response = new VerkleSyncBatch(); + response.SubTreeRangeRequest = new SubTreeRange(Pedersen.Zero, Stem.Zero.Bytes); + response.SubTreeRangeResponse = new SubTreesAndProofs(Array.Empty(), Array.Empty()); + PeerInfo peer = new PeerInfo(Substitute.For()); + + feed.HandleResponse(response, peer).Should().Be(SyncResponseHandlingResult.NoProgress); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/VerkleSyncFeedTestsBase.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/VerkleSyncFeedTestsBase.cs new file mode 100644 index 00000000000..845df7b216f --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleSync/VerkleSyncFeed/VerkleSyncFeedTestsBase.cs @@ -0,0 +1,317 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Stats.Model; +using Nethermind.Synchronization.FastSync; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.Peers; +using Nethermind.Synchronization.VerkleSync; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.VerkleSync.VerkleSyncFeed; + +public class VerkleSyncFeedTestsBase +{ + private const int TimeoutLength = 2000; + + protected static IBlockTree _blockTree; + protected static IBlockTree BlockTree => LazyInitializer.EnsureInitialized(ref _blockTree, () => Build.A.BlockTree().OfChainLength(100).TestObject); + + protected ILogger _logger; + protected ILogManager _logManager; + + private readonly int _defaultPeerCount; + private readonly int _defaultPeerMaxRandomLatency; + + public VerkleSyncFeedTestsBase(int defaultPeerCount = 1, int defaultPeerMaxRandomLatency = 0) + { + _defaultPeerCount = defaultPeerCount; + _defaultPeerMaxRandomLatency = defaultPeerMaxRandomLatency; + } + + public static (string Name, Action Action)[] Scenarios => VerkleTrieScenarios.Scenarios; + + [SetUp] + public void Setup() + { + _logManager = LimboLogs.Instance; + _logger = LimboTraceLogger.Instance; + TrieScenarios.InitOnce(); + } + + [TearDown] + public void TearDown() + { + (_logger as ConsoleAsyncLogger)?.Flush(); + } + + protected static void SetStorage(IVerkleTrieStore trieStore, Address address, byte i) + { + VerkleStateTree tree = new VerkleStateTree(trieStore, LimboLogs.Instance); + for (int j = 0; j < i; j++) + { + StorageCell cell = new(address, i); + tree.SetStorage(cell, ((UInt256)j).ToLittleEndian()); + } + } + + protected class SafeContext + { + public ISyncModeSelector SyncModeSelector; + public SyncPeerMock[] SyncPeerMocks; + public ISyncPeerPool Pool; + public Synchronization.VerkleSync.VerkleSyncFeed Feed; + public VerkleSyncDispatcher StateSyncDispatcher; + } + + public static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + public static IDbProvider GetDbProviderForVerkleTrees(DbMode dbMode) + { + switch (dbMode) + { + case DbMode.MemDb: + return VerkleDbFactory.InitDatabase(dbMode, null); + case DbMode.PersistantDb: + return VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + protected class DbContext + { + private readonly ILogger _logger; + + public DbContext(ILogger logger, ILogManager logManager, DbMode dbMode) + { + RemoteDbProvider = GetDbProviderForVerkleTrees(DbMode.MemDb); + LocalDbProvider = GetDbProviderForVerkleTrees(DbMode.MemDb); + + _logger = logger; + + RemoteTrieStore = new VerkleStateStore(RemoteDbProvider, LimboLogs.Instance); + LocalTrieStore = new VerkleStateStore(LocalDbProvider, LimboLogs.Instance); + + RemoteStateTree = new VerkleStateTree(RemoteTrieStore, LimboLogs.Instance); + LocalStateTree = new VerkleStateTree(LocalTrieStore, LimboLogs.Instance); + } + + public IDbProvider RemoteDbProvider; + public IDbProvider LocalDbProvider; + + public IDb RemoteCodeDb => RemoteDbProvider.CodeDb; + public IDb LocalCodeDb => LocalDbProvider.CodeDb; + public IVerkleTrieStore RemoteTrieStore { get; } + public IVerkleTrieStore LocalTrieStore { get; } + + public IDb RemoteLeafDb => RemoteDbProvider.LeafDb; + public IDb LocalLeafDb => LocalDbProvider.LeafDb; + + public IDb RemoteInternalDb => RemoteDbProvider.InternalNodesDb; + public IDb LocalInternalDb => LocalDbProvider.InternalNodesDb; + public VerkleStateTree RemoteStateTree { get; } + public VerkleStateTree LocalStateTree { get; } + + public void CompareTrees(string stage, bool skipLogs = false) + { + if (!skipLogs) _logger.Info($"==================== {stage} ===================="); + LocalStateTree.StateRoot = RemoteStateTree.StateRoot; + + if (!skipLogs) _logger.Info("-------------------- REMOTE --------------------"); + VerkleTreeDumper dumper = new VerkleTreeDumper(); + RemoteStateTree.Accept(dumper, RemoteStateTree.StateRoot); + string remote = dumper.ToString(); + if (!skipLogs) _logger.Info(remote); + if (!skipLogs) _logger.Info("-------------------- LOCAL --------------------"); + dumper.Reset(); + LocalStateTree.Accept(dumper, LocalStateTree.StateRoot); + string local = dumper.ToString(); + if (!skipLogs) _logger.Info(local); + + if (stage == "END") + { + Assert.That(local, Is.EqualTo(remote), $"{remote}{Environment.NewLine}{local}"); + VerkleTrieStatsCollector collector = new(LocalCodeDb, LimboLogs.Instance); + LocalStateTree.Accept(collector, LocalStateTree.StateRoot); + Assert.That(collector.Stats.MissingLeaf, Is.EqualTo(0)); + } + + // Assert.AreEqual(dbContext._remoteCodeDb.Keys.OrderBy(k => k, Bytes.Comparer).ToArray(), dbContext._localCodeDb.Keys.OrderBy(k => k, Bytes.Comparer).ToArray(), "keys"); + // Assert.AreEqual(dbContext._remoteCodeDb.Values.OrderBy(k => k, Bytes.Comparer).ToArray(), dbContext._localCodeDb.Values.OrderBy(k => k, Bytes.Comparer).ToArray(), "values"); + // + // Assert.AreEqual(dbContext._remoteDb.Keys.OrderBy(k => k, Bytes.Comparer).ToArray(), _localDb.Keys.OrderBy(k => k, Bytes.Comparer).ToArray(), "keys"); + // Assert.AreEqual(dbContext._remoteDb.Values.OrderBy(k => k, Bytes.Comparer).ToArray(), _localDb.Values.OrderBy(k => k, Bytes.Comparer).ToArray(), "values"); + } + } + + protected class SyncPeerMock : ISyncPeer + { + public static Func, Task> NotPreimage = request => + { + var result = new byte[request.Count][]; + + int i = 0; + foreach (Keccak _ in request) result[i++] = new byte[] { 1, 2, 3 }; + + return Task.FromResult(result); + }; + + public static Func, Task> EmptyArraysInResponses = request => + { + var result = new byte[request.Count][]; + + int i = 0; + foreach (Keccak _ in request) result[i++] = new byte[0]; + + return Task.FromResult(result); + }; + + private readonly IDb _codeDb; + private readonly IDb _stateDb; + + private Func, Task> _executorResultFunction; + + private Keccak[] _filter; + + private readonly long _maxRandomizedLatencyMs; + + public SyncPeerMock( + IDb stateDb, + IDb codeDb, + Func, Task> executorResultFunction = null, + long? maxRandomizedLatencyMs = null, + Node? node = null + ) + { + _stateDb = stateDb; + _codeDb = codeDb; + + if (executorResultFunction is not null) _executorResultFunction = executorResultFunction; + + Node = node ?? new Node(TestItem.PublicKeyA, "127.0.0.1", 30302, true) { EthDetails = "eth66" }; + _maxRandomizedLatencyMs = maxRandomizedLatencyMs ?? 0; + } + + public int MaxResponseLength { get; set; } = int.MaxValue; + + public Node Node { get; } + public string ClientId => "executorMock"; + public Keccak HeadHash { get; set; } + public long HeadNumber { get; set; } + public UInt256 TotalDifficulty { get; set; } + public bool IsInitialized { get; set; } + public bool IsPriority { get; set; } + public byte ProtocolVersion { get; } + public string ProtocolCode { get; } + + public void Disconnect(InitiateDisconnectReason reason, string details) + { + throw new NotImplementedException(); + } + + public Task GetBlockBodies(IReadOnlyList blockHashes, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task GetBlockHeaders(Keccak blockHash, int maxBlocks, int skip, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task GetBlockHeaders(long number, int maxBlocks, int skip, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task GetHeadBlockHeader(Keccak? hash, CancellationToken token) + { + return Task.FromResult(BlockTree.Head?.Header); + } + + public void NotifyOfNewBlock(Block block, SendBlockMode mode) + { + throw new NotImplementedException(); + } + + public PublicKey Id => Node.Id; + + public void SendNewTransactions(IEnumerable txs, bool sendFullTx) + { + throw new NotImplementedException(); + } + + public Task GetReceipts(IReadOnlyList blockHash, CancellationToken token) + { + throw new NotImplementedException(); + } + + public async Task GetNodeData(IReadOnlyList hashes, CancellationToken token) + { + if (_maxRandomizedLatencyMs != 0) + { + await Task.Delay(TimeSpan.FromMilliseconds(TestContext.CurrentContext.Random.NextLong() % _maxRandomizedLatencyMs)); + } + + if (_executorResultFunction is not null) return await _executorResultFunction(hashes); + + var responses = new byte[hashes.Count][]; + + int i = 0; + foreach (Keccak item in hashes) + { + if (i >= MaxResponseLength) break; + + if (_filter is null || _filter.Contains(item)) responses[i] = _stateDb[item.Bytes] ?? _codeDb[item.Bytes]; + + i++; + } + + return responses; + } + + public void SetFilter(Keccak[] availableHashes) + { + _filter = availableHashes; + } + + public void RegisterSatelliteProtocol(string protocol, T protocolHandler) where T : class + { + throw new NotImplementedException(); + } + + public bool TryGetSatelliteProtocol(string protocol, out T protocolHandler) where T : class + { + protocolHandler = null; + return false; + } + } + +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/VerkleTrieScenarios.cs b/src/Nethermind/Nethermind.Synchronization.Test/VerkleTrieScenarios.cs new file mode 100644 index 00000000000..3c5e47149ac --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/VerkleTrieScenarios.cs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Runtime.CompilerServices; +using System.Threading; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; + +namespace Nethermind.Synchronization.Test; + +public class VerkleTrieScenarios +{ + public static Account Empty; + public static Account AccountJustState0; + public static Account AccountJustState1; + public static Account AccountJustState2; + public static Account Account0; + public static Account Account1; + public static Account Account2; + public static Account Account3; + + public static readonly byte[] Code0 = { 0, 0 }; + public static readonly byte[] Code1 = { 0, 1 }; + public static readonly byte[] Code2 = { 0, 2 }; + public static readonly byte[] Code3 = { 0, 3 }; + + [MethodImpl(MethodImplOptions.Synchronized)] + public static void InitOnce() + { + if (Empty is null) + { + Empty = Build.An.Account.WithBalance(0).TestObject; + // these 4 accounts are considered to be with storage + Account0 = Build.An.Account.WithBalance(1).WithCode(Code0).TestObject; + Account1 = Build.An.Account.WithBalance(2).WithCode(Code1).TestObject; + Account2 = Build.An.Account.WithBalance(3).WithCode(Code2).TestObject; + Account3 = Build.An.Account.WithBalance(4).WithCode(Code3).TestObject; + + AccountJustState0 = Build.An.Account.WithBalance(1).TestObject; + AccountJustState1 = Build.An.Account.WithBalance(2).TestObject; + AccountJustState2 = Build.An.Account.WithBalance(3).TestObject; + } + } + + private static (string Name, Action Action)[] _scenarios; + + public static (string Name, Action Action)[] Scenarios + => LazyInitializer.EnsureInitialized(ref _scenarios, InitScenarios); + + private static (string Name, Action Action)[] InitScenarios() + { + return new (string, Action)[] + { + ("empty", (tree, stateDb, codeDb) => + { + tree.Commit(); + tree.CommitTree(0); + }), + ("set_3_via_address", (tree, stateDb, codeDb) => + { + codeDb[Keccak.Compute(Code0).Bytes] = Code0; + SetAccountWithStorageAndCode(tree, TestItem.AddressA, Account0, true); + SetAccountWithStorageAndCode(tree, TestItem.AddressB, Account0, true); + SetAccountWithStorageAndCode(tree, TestItem.AddressC, Account0, true); + tree.Commit(); + tree.CommitTree(0);; + }), + ("branch_with_same_accounts_at_different_addresses", (tree, stateDb, codeDb) => + { + codeDb[Keccak.Compute(Code0).Bytes] = Code0; + Keccak account1 = new("1baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + Keccak account2 = new("2baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + tree.InsertStemBatch(account1.Bytes.AsSpan()[..31], AccountJustState0.ToVerkleDict()); + tree.InsertStemBatch(account2.Bytes.AsSpan()[..31], AccountJustState0.ToVerkleDict()); + tree.Commit(); + tree.CommitTree(0); + }), + ("just_state", (tree, stateDb, codeDb) => + { + Keccak account1 = new("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"); + Keccak account2 = new("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"); + Keccak account3 = new("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"); + tree.InsertStemBatch(account1.Bytes.AsSpan()[..31], AccountJustState0.ToVerkleDict()); + tree.InsertStemBatch(account2.Bytes.AsSpan()[..31], AccountJustState1.ToVerkleDict()); + tree.InsertStemBatch(account3.Bytes.AsSpan()[..31], AccountJustState2.ToVerkleDict()); + tree.Commit(); + tree.CommitTree(0); + }) + }; + } + + private static void SetAccountWithStorageAndCode(VerkleStateTree tree, Address address, Account account, + bool setStorage = false) + + { + tree.Set(address, account); + if (account.Code is not null) tree.SetCode(address, account.Code); + + if(!setStorage) return; + tree.SetStorage(new StorageCell(address, 1), new byte[] { 1 }); + tree.SetStorage(new StorageCell(address, 2), new byte[] { 2 }); + tree.SetStorage(new StorageCell(address, 3), new byte[] { 3 }); + tree.SetStorage(new StorageCell(address, 4), new byte[] { 4 }); + tree.SetStorage(new StorageCell(address, 1005), new byte[] { 5 }); + tree.SetStorage(new StorageCell(address, 1006), new byte[] { 6 }); + tree.SetStorage(new StorageCell(address, 1007), new byte[] { 7 }); + tree.SetStorage(new StorageCell(address, 1008), new byte[] { 8 }); + } + +} diff --git a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs index 3e67e53effd..767356af385 100644 --- a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs @@ -6,6 +6,7 @@ using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.SnapSync; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Synchronization.DbTuner; @@ -21,6 +22,7 @@ public class SyncDbTuner public SyncDbTuner( ISyncConfig syncConfig, ISyncFeed? snapSyncFeed, + ISyncFeed? verkleSyncFeed, ISyncFeed? bodiesSyncFeed, ISyncFeed? receiptSyncFeed, IDb stateDb, @@ -37,6 +39,11 @@ IDb receiptDb snapSyncFeed.StateChanged += SnapStateChanged; } + if (verkleSyncFeed != null) + { + verkleSyncFeed.StateChanged += VerkleStateChanged; + } + if (bodiesSyncFeed != null) { bodiesSyncFeed.StateChanged += BodiesStateChanged; @@ -81,6 +88,32 @@ private void SnapStateChanged(object? sender, SyncFeedStateEventArgs e) } } + private void VerkleStateChanged(object? sender, SyncFeedStateEventArgs e) + { + if (e.NewState == SyncFeedState.Active) + { + if (_stateDb is ITunableDb stateDb) + { + stateDb.Tune(_tuneType); + } + if (_codeDb is ITunableDb codeDb) + { + codeDb.Tune(_tuneType); + } + } + else if (e.NewState == SyncFeedState.Finished) + { + if (_stateDb is ITunableDb stateDb) + { + stateDb.Tune(ITunableDb.TuneType.Default); + } + if (_codeDb is ITunableDb codeDb) + { + codeDb.Tune(ITunableDb.TuneType.Default); + } + } + } + private void BodiesStateChanged(object? sender, SyncFeedStateEventArgs e) { if (e.NewState == SyncFeedState.Active) diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncProgressResolver.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncProgressResolver.cs index 3608af55b45..0c8f80d0a8d 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncProgressResolver.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncProgressResolver.cs @@ -26,8 +26,7 @@ public interface ISyncProgressResolver long FindBestProcessedBlock(); - bool IsSnapGetRangesFinished(); - + bool IsGetRangesFinished(); UInt256 ChainDifficulty { get; } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index 2038ac4c724..27cd1c7137f 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -57,6 +57,7 @@ public class MultiSyncModeSelector : ISyncModeSelector, IDisposable private readonly long _pivotNumber; private bool FastSyncEnabled => _syncConfig.FastSync; private bool SnapSyncEnabled => _syncConfig.SnapSync && !_isSnapSyncDisabledAfterAnyStateSync; + private bool VerkleSyncEnabled => _syncConfig.VerkleSync; private bool FastBlocksEnabled => _syncConfig.FastSync && _syncConfig.FastBlocks; private bool FastBodiesEnabled => FastBlocksEnabled && _syncConfig.DownloadBodiesInFastSync; private bool FastReceiptsEnabled => FastBlocksEnabled && _syncConfig.DownloadReceiptsInFastSync; @@ -210,6 +211,7 @@ public void Update() CheckAddFlag(best.IsInFullSync, SyncMode.Full, ref newModes); CheckAddFlag(best.IsInStateNodes, SyncMode.StateNodes, ref newModes); CheckAddFlag(best.IsInSnapRanges, SyncMode.SnapSync, ref newModes); + CheckAddFlag(best.IsInVerkleRanges, SyncMode.VerkleSync, ref newModes); CheckAddFlag(best.IsInDisconnected, SyncMode.Disconnected, ref newModes); CheckAddFlag(best.IsInWaitingForBlock, SyncMode.WaitingForBlock, ref newModes); if (IsTheModeSwitchWorthMentioning(newModes)) @@ -564,16 +566,18 @@ private bool ShouldBeInStateNodesMode(Snapshot best) { bool isInStateSync = best.IsInStateSync; bool snapSyncDisabled = !SnapSyncEnabled; - bool snapRangesFinished = _syncProgressResolver.IsSnapGetRangesFinished(); + bool verkleSyncDisabled = !VerkleSyncEnabled; + bool rangesFinished = _syncProgressResolver.IsGetRangesFinished(); - bool result = isInStateSync && (snapSyncDisabled || snapRangesFinished); + bool result = isInStateSync && (snapSyncDisabled || rangesFinished) && (verkleSyncDisabled || rangesFinished); if (_logger.IsTrace) { LogDetailedSyncModeChecks("STATE_NODES", (nameof(isInStateSync), isInStateSync), (nameof(snapSyncDisabled), snapSyncDisabled), - (nameof(snapRangesFinished), snapRangesFinished)); + (nameof(verkleSyncDisabled), verkleSyncDisabled), + (nameof(rangesFinished), rangesFinished)); } return result; @@ -583,7 +587,7 @@ private bool ShouldBeBeInSnapRangesPhase(Snapshot best) { bool isInStateSync = best.IsInStateSync; bool isCloseToHead = best.TargetBlock >= best.Header && (best.TargetBlock - best.Header) < Constants.MaxDistanceFromHead; - bool snapNotFinished = !_syncProgressResolver.IsSnapGetRangesFinished(); + bool snapNotFinished = !_syncProgressResolver.IsGetRangesFinished(); if (_logger.IsTrace) { @@ -600,6 +604,27 @@ private bool ShouldBeBeInSnapRangesPhase(Snapshot best) && snapNotFinished; } + private bool ShouldBeBeInVerkleRangesPhase(Snapshot best) + { + bool isInStateSync = best.IsInStateSync; + bool isCloseToHead = best.TargetBlock >= best.Header && (best.TargetBlock - best.Header) < Constants.MaxDistanceFromHead; + bool verkleRangesNotFinished = !_syncProgressResolver.IsGetRangesFinished(); + + if (_logger.IsTrace) + { + LogDetailedSyncModeChecks("VERKLE_RANGES", + (nameof(VerkleSyncEnabled), VerkleSyncEnabled), + (nameof(isInStateSync), isInStateSync), + (nameof(isCloseToHead), isCloseToHead), + (nameof(verkleRangesNotFinished), verkleRangesNotFinished)); + } + + return VerkleSyncEnabled + && isInStateSync + && isCloseToHead + && verkleRangesNotFinished; + } + private bool HasJustStartedFullSync(Snapshot best) => best.State > _pivotNumber // we have saved some root && (best.State == best.Header || best.Header == best.Block) // and we do not need to catch up to headers anymore @@ -744,7 +769,8 @@ long targetBlock TargetBlock = targetBlock; IsInWaitingForBlock = IsInDisconnected = IsInFastReceipts = IsInFastBodies = IsInFastHeaders - = IsInFastSync = IsInFullSync = IsInStateSync = IsInStateNodes = IsInSnapRanges = IsInBeaconHeaders = false; + = IsInFastSync = IsInFullSync = IsInStateSync = IsInStateNodes = IsInSnapRanges = IsInVerkleRanges + = IsInBeaconHeaders = false; } public bool IsInFastHeaders { get; set; } @@ -754,6 +780,7 @@ long targetBlock public bool IsInStateSync { get; set; } public bool IsInStateNodes { get; set; } public bool IsInSnapRanges { get; set; } + public bool IsInVerkleRanges { get; set; } public bool IsInFullSync { get; set; } public bool IsInDisconnected { get; set; } public bool IsInWaitingForBlock { get; set; } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncMode.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncMode.cs index ca7165555af..e5223bbd544 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncMode.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncMode.cs @@ -23,7 +23,7 @@ public enum SyncMode /// FastBlocks = 4, /// - /// A standard fast sync mode before the peers head - 32 (threshold). It happens after the fast blocks finishes to download from pivot downwards. By default the pivot for fast blocks is 0 so the fast blocks finish immediately. + /// A standard fast sync mode before the peers head - 32 (threshold). It happens after the fast blocks finishes to download from pivot downwards. By default the pivot for fast blocks is 0 so the fast blocks finish immediately. /// FastSync = 8, /// @@ -56,11 +56,15 @@ public enum SyncMode SnapSync = 2048, /// /// Reverse download of headers from beacon pivot to genesis - /// + /// BeaconHeaders = 4096, + /// + /// Reverse download of headers from beacon pivot to genesis + /// + VerkleSync = 8192, All = WaitingForBlock | Disconnected | FastBlocks | FastSync | StateNodes | StateNodes | Full | DbLoad | - FastHeaders | FastBodies | FastReceipts | SnapSync | BeaconHeaders + FastHeaders | FastBodies | FastReceipts | SnapSync | BeaconHeaders | VerkleSync } public static class SyncModeExtensions diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs index bf993353061..96830bebf2e 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs @@ -7,11 +7,9 @@ using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Crypto; -using Nethermind.Db; using Nethermind.Int256; using Nethermind.Logging; -using Nethermind.Synchronization.SnapSync; -using Nethermind.Trie; +using Nethermind.Synchronization.RangeSync; using Nethermind.Trie.Pruning; namespace Nethermind.Synchronization.ParallelSync @@ -25,8 +23,8 @@ public class SyncProgressResolver : ISyncProgressResolver private readonly IBlockTree _blockTree; private readonly IReceiptStorage _receiptStorage; - private readonly ITrieNodeResolver _trieNodeResolver; - private readonly ProgressTracker _progressTracker; + private readonly ISyncTrieStore _trieNodeResolver; + private readonly IRangeFinishTracker _rangeProgressTracker; private readonly ISyncConfig _syncConfig; // ReSharper disable once NotAccessedField.Local @@ -37,8 +35,8 @@ public class SyncProgressResolver : ISyncProgressResolver public SyncProgressResolver(IBlockTree blockTree, IReceiptStorage receiptStorage, - ITrieNodeResolver trieNodeResolver, - ProgressTracker progressTracker, + ISyncTrieStore trieNodeResolver, + IRangeFinishTracker rangeProgressTracker, ISyncConfig syncConfig, ILogManager logManager) { @@ -46,7 +44,7 @@ public SyncProgressResolver(IBlockTree blockTree, _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _trieNodeResolver = trieNodeResolver ?? throw new ArgumentNullException(nameof(trieNodeResolver)); - _progressTracker = progressTracker ?? throw new ArgumentNullException(nameof(progressTracker)); + _rangeProgressTracker = rangeProgressTracker ?? throw new ArgumentNullException(nameof(rangeProgressTracker)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); _bodiesBarrier = _syncConfig.AncientBodiesBarrierCalc; @@ -149,7 +147,7 @@ public bool IsFastBlocksReceiptsFinished() => !IsFastBlocks() || (!_syncConfig.D .LowestInsertedReceiptBlockNumber ?? long.MaxValue) <= _receiptsBarrier); - public bool IsSnapGetRangesFinished() => _progressTracker.IsSnapGetRangesFinished(); + public bool IsGetRangesFinished() => _rangeProgressTracker.IsGetRangesFinished(); public void RecalculateProgressPointers() => _blockTree.RecalculateTreeLevels(); diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs b/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs index 8361c2ffd65..e42916f0314 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs @@ -16,6 +16,7 @@ public enum AllocationContexts State = 8, Witness = 16, Snap = 32, - All = Headers | Bodies | Receipts | Blocks | State | Witness | Snap + Verkle = 64, + All = Headers | Bodies | Receipts | Blocks | State | Witness | Snap | Verkle } } diff --git a/src/Nethermind/Nethermind.Synchronization/RangeSync/AddRangeResult.cs b/src/Nethermind/Nethermind.Synchronization/RangeSync/AddRangeResult.cs new file mode 100644 index 00000000000..05fe67d3a83 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/RangeSync/AddRangeResult.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nethermind.Synchronization.RangeSync +{ + public enum AddRangeResult + { + OK, + MissingRootHashInProofs, + DifferentRootHash, + ExpiredRootHash + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/RangeSync/IRangeProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/RangeSync/IRangeProgressTracker.cs new file mode 100644 index 00000000000..6f770f96464 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/RangeSync/IRangeProgressTracker.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Synchronization.RangeSync; + +public interface IRangeProgressTracker: IRangeFinishTracker +{ + public bool CanSync(); + public void UpdatePivot(); + + public (T request, bool finished) GetNextRequest(); +} + +public interface IRangeFinishTracker +{ + public bool IsGetRangesFinished(); +} diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/Pivot.cs b/src/Nethermind/Nethermind.Synchronization/RangeSync/Pivot.cs similarity index 93% rename from src/Nethermind/Nethermind.Synchronization/SnapSync/Pivot.cs rename to src/Nethermind/Nethermind.Synchronization/RangeSync/Pivot.cs index cbb4cd46fea..8c6d5c0cd26 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/Pivot.cs +++ b/src/Nethermind/Nethermind.Synchronization/RangeSync/Pivot.cs @@ -2,16 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Logging; using Nethermind.State.Snap; -namespace Nethermind.Synchronization.SnapSync +namespace Nethermind.Synchronization.RangeSync { internal class Pivot { diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProgressTracker.cs similarity index 96% rename from src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs rename to src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProgressTracker.cs index 8d25cd66714..064cf219934 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProgressTracker.cs @@ -13,10 +13,11 @@ using Nethermind.Db; using Nethermind.Logging; using Nethermind.State.Snap; +using Nethermind.Synchronization.RangeSync; namespace Nethermind.Synchronization.SnapSync { - public class ProgressTracker + public class SnapProgressTracker: IRangeProgressTracker { private const string NO_REQUEST = "NO REQUEST"; @@ -51,14 +52,14 @@ public class ProgressTracker private ConcurrentQueue AccountsToRefresh { get; set; } = new(); - private readonly Pivot _pivot; + private readonly RangeSync.Pivot _pivot; - public ProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int accountRangePartitionCount = 8) + public SnapProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int accountRangePartitionCount = 8) { _logger = logManager.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _db = db ?? throw new ArgumentNullException(nameof(db)); - _pivot = new Pivot(blockTree, logManager); + _pivot = new RangeSync.Pivot(blockTree, logManager); if (accountRangePartitionCount < 1 || accountRangePartitionCount > 256) throw new ArgumentException("Account range partition must be between 1 to 256."); @@ -230,7 +231,7 @@ public void UpdatePivot() return (request, false); } - bool rangePhaseFinished = IsSnapGetRangesFinished(); + bool rangePhaseFinished = IsGetRangesFinished(); if (rangePhaseFinished) { _logger.Info($"SNAP - State Ranges (Phase 1) finished."); @@ -239,7 +240,7 @@ public void UpdatePivot() LogRequest(NO_REQUEST); - return (null, IsSnapGetRangesFinished()); + return (null, IsGetRangesFinished()); } private bool ShouldRequestAccountRequests() @@ -335,7 +336,7 @@ public void UpdateAccountRangePartitionProgress(Keccak hashLimit, Keccak nextPat partition.MoreAccountsToRight = moreChildrenToRight && nextPath < hashLimit; } - public bool IsSnapGetRangesFinished() + public bool IsGetRangesFinished() { return AccountRangeReadyForRequest.IsEmpty && StoragesToRetrieve.IsEmpty diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 2b13ade9ed6..06790350a45 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -25,21 +25,21 @@ public class SnapProvider : ISnapProvider private readonly ILogManager _logManager; private readonly ILogger _logger; - private readonly ProgressTracker _progressTracker; + private readonly SnapProgressTracker _snapProgressTracker; - public SnapProvider(ProgressTracker progressTracker, IDbProvider dbProvider, ILogManager logManager) + public SnapProvider(SnapProgressTracker snapProgressTracker, IDbProvider dbProvider, ILogManager logManager) { _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); - _progressTracker = progressTracker ?? throw new ArgumentNullException(nameof(progressTracker)); + _snapProgressTracker = snapProgressTracker ?? throw new ArgumentNullException(nameof(snapProgressTracker)); _trieStorePool = new DefaultObjectPool(new TrieStorePoolPolicy(_dbProvider.StateDb, logManager)); _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); _logger = logManager.GetClassLogger(); } - public bool CanSync() => _progressTracker.CanSync(); + public bool CanSync() => _snapProgressTracker.CanSync(); - public (SnapSyncBatch request, bool finished) GetNextRequest() => _progressTracker.GetNextRequest(); + public (SnapSyncBatch request, bool finished) GetNextRequest() => _snapProgressTracker.GetNextRequest(); public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs response) { @@ -61,7 +61,7 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re } } - _progressTracker.ReportAccountRangePartitionFinished(request.LimitHash); + _snapProgressTracker.ReportAccountRangePartitionFinished(request.LimitHash); return result; } @@ -82,11 +82,11 @@ public AddRangeResult AddAccountRange(long blockNumber, Keccak expectedRootHash, { foreach (var item in accountsWithStorage) { - _progressTracker.EnqueueAccountStorage(item); + _snapProgressTracker.EnqueueAccountStorage(item); } - _progressTracker.EnqueueCodeHashes(codeHashes); - _progressTracker.UpdateAccountRangePartitionProgress(hashLimit, accounts[^1].Path, moreChildrenToRight); + _snapProgressTracker.EnqueueCodeHashes(codeHashes); + _snapProgressTracker.UpdateAccountRangePartitionProgress(hashLimit, accounts[^1].Path, moreChildrenToRight); } else if (result == AddRangeResult.MissingRootHashInProofs) { @@ -113,7 +113,7 @@ public AddRangeResult AddStorageRange(StorageRange request, SlotsAndProofs respo { _logger.Trace($"SNAP - GetStorageRange - expired BlockNumber:{request.BlockNumber}, RootHash:{request.RootHash}, (Accounts:{request.Accounts.Length}), {request.StartingHash}"); - _progressTracker.ReportStorageRangeRequestFinished(request); + _snapProgressTracker.ReportStorageRangeRequestFinished(request); return AddRangeResult.ExpiredRootHash; } @@ -140,11 +140,11 @@ public AddRangeResult AddStorageRange(StorageRange request, SlotsAndProofs respo if (requestLength > responseLength) { - _progressTracker.ReportFullStorageRequestFinished(request.Accounts[responseLength..requestLength]); + _snapProgressTracker.ReportFullStorageRequestFinished(request.Accounts[responseLength..requestLength]); } else { - _progressTracker.ReportFullStorageRequestFinished(); + _snapProgressTracker.ReportFullStorageRequestFinished(); } if (result == AddRangeResult.OK && slotCount > 0) @@ -174,20 +174,20 @@ public AddRangeResult AddStorageRange(long blockNumber, PathWithAccount pathWith StartingHash = slots.Last().Path }; - _progressTracker.EnqueueStorageRange(range); + _snapProgressTracker.EnqueueStorageRange(range); } } else if (result == AddRangeResult.MissingRootHashInProofs) { _logger.Trace($"SNAP - AddStorageRange failed, missing root hash {expectedRootHash} in the proofs, startingHash:{startingHash}"); - _progressTracker.EnqueueAccountRefresh(pathWithAccount, startingHash); + _snapProgressTracker.EnqueueAccountRefresh(pathWithAccount, startingHash); } else if (result == AddRangeResult.DifferentRootHash) { _logger.Trace($"SNAP - AddStorageRange failed, expected storage root hash:{expectedRootHash} but was {tree.RootHash}, startingHash:{startingHash}"); - _progressTracker.EnqueueAccountRefresh(pathWithAccount, startingHash); + _snapProgressTracker.EnqueueAccountRefresh(pathWithAccount, startingHash); } return result; @@ -235,11 +235,11 @@ public void RefreshAccounts(AccountsToRefreshRequest request, byte[][] response) StartingHash = requestedPath.StorageStartingHash }; - _progressTracker.EnqueueStorageRange(range); + _snapProgressTracker.EnqueueStorageRange(range); } else { - _progressTracker.EnqueueAccountStorage(requestedPath.PathAndAccount); + _snapProgressTracker.EnqueueAccountStorage(requestedPath.PathAndAccount); } } catch (Exception exc) @@ -254,7 +254,7 @@ public void RefreshAccounts(AccountsToRefreshRequest request, byte[][] response) } } - _progressTracker.ReportAccountRefreshFinished(); + _snapProgressTracker.ReportAccountRefreshFinished(); } finally { @@ -264,7 +264,7 @@ public void RefreshAccounts(AccountsToRefreshRequest request, byte[][] response) private void RetryAccountRefresh(AccountWithStorageStartingHash requestedPath) { - _progressTracker.EnqueueAccountRefresh(requestedPath.PathAndAccount, requestedPath.StorageStartingHash); + _snapProgressTracker.EnqueueAccountRefresh(requestedPath.PathAndAccount, requestedPath.StorageStartingHash); } public void AddCodes(Keccak[] requestedHashes, byte[][] codes) @@ -285,34 +285,34 @@ public void AddCodes(Keccak[] requestedHashes, byte[][] codes) Interlocked.Add(ref Metrics.SnapSyncedCodes, codes.Length); - _progressTracker.ReportCodeRequestFinished(set); + _snapProgressTracker.ReportCodeRequestFinished(set); } public void RetryRequest(SnapSyncBatch batch) { if (batch.AccountRangeRequest is not null) { - _progressTracker.ReportAccountRangePartitionFinished(batch.AccountRangeRequest.LimitHash); + _snapProgressTracker.ReportAccountRangePartitionFinished(batch.AccountRangeRequest.LimitHash); } else if (batch.StorageRangeRequest is not null) { - _progressTracker.ReportStorageRangeRequestFinished(batch.StorageRangeRequest); + _snapProgressTracker.ReportStorageRangeRequestFinished(batch.StorageRangeRequest); } else if (batch.CodesRequest is not null) { - _progressTracker.ReportCodeRequestFinished(batch.CodesRequest); + _snapProgressTracker.ReportCodeRequestFinished(batch.CodesRequest); } else if (batch.AccountsToRefreshRequest is not null) { - _progressTracker.ReportAccountRefreshFinished(batch.AccountsToRefreshRequest); + _snapProgressTracker.ReportAccountRefreshFinished(batch.AccountsToRefreshRequest); } } - public bool IsSnapGetRangesFinished() => _progressTracker.IsSnapGetRangesFinished(); + public bool IsSnapGetRangesFinished() => _snapProgressTracker.IsGetRangesFinished(); public void UpdatePivot() { - _progressTracker.UpdatePivot(); + _snapProgressTracker.UpdatePivot(); } private class TrieStorePoolPolicy : IPooledObjectPolicy diff --git a/src/Nethermind/Nethermind.Synchronization/SyncServer.cs b/src/Nethermind/Nethermind.Synchronization/SyncServer.cs index 0f561331d10..d9777c912e1 100644 --- a/src/Nethermind/Nethermind.Synchronization/SyncServer.cs +++ b/src/Nethermind/Nethermind.Synchronization/SyncServer.cs @@ -40,7 +40,7 @@ public class SyncServer : ISyncServer private readonly IReceiptFinder _receiptFinder; private readonly IBlockValidator _blockValidator; private readonly ISealValidator _sealValidator; - private readonly IReadOnlyKeyValueStore _stateDb; + private readonly IReadOnlyKeyValueStore? _stateDb; private readonly IReadOnlyKeyValueStore _codeDb; private readonly IWitnessRepository _witnessRepository; private readonly IGossipPolicy _gossipPolicy; @@ -56,7 +56,7 @@ public class SyncServer : ISyncServer private BlockHeader? _pivotHeader; public SyncServer( - IReadOnlyKeyValueStore stateDb, + IReadOnlyKeyValueStore? stateDb, IReadOnlyKeyValueStore codeDb, IBlockTree blockTree, IReceiptFinder receiptFinder, @@ -78,7 +78,7 @@ public SyncServer( _pool = pool ?? throw new ArgumentNullException(nameof(pool)); _syncModeSelector = syncModeSelector ?? throw new ArgumentNullException(nameof(syncModeSelector)); _sealValidator = sealValidator ?? throw new ArgumentNullException(nameof(sealValidator)); - _stateDb = stateDb ?? throw new ArgumentNullException(nameof(stateDb)); + _stateDb = stateDb; _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 48cb9e450eb..239968eba95 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -22,6 +22,7 @@ using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.StateSync; +using Nethermind.Synchronization.VerkleSync; namespace Nethermind.Synchronization { @@ -38,6 +39,7 @@ public class Synchronizer : ISynchronizer protected readonly IBlockTree _blockTree; protected readonly ISyncConfig _syncConfig; protected readonly ISnapProvider _snapProvider; + protected readonly IVerkleSyncProvider _verkleSyncProvider; protected readonly ISyncPeerPool _syncPeerPool; protected readonly ILogManager _logManager; protected readonly ISyncReport _syncReport; @@ -53,6 +55,7 @@ public class Synchronizer : ISynchronizer private FastSyncFeed? _fastSyncFeed; private StateSyncFeed? _stateSyncFeed; private SnapSyncFeed? _snapSyncFeed; + private VerkleSyncFeed? _verkleSyncFeed; private FullSyncFeed? _fullSyncFeed; private HeadersSyncFeed? _headersFeed; private BodiesSyncFeed? _bodiesFeed; @@ -68,6 +71,7 @@ public Synchronizer( ISyncModeSelector syncModeSelector, ISyncConfig syncConfig, ISnapProvider snapProvider, + IVerkleSyncProvider verkleSyncProvider, IBlockDownloaderFactory blockDownloaderFactory, IPivot pivot, ISyncReport syncReport, @@ -81,6 +85,7 @@ public Synchronizer( _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); _snapProvider = snapProvider ?? throw new ArgumentNullException(nameof(snapProvider)); + _verkleSyncProvider = verkleSyncProvider ?? throw new ArgumentNullException(nameof(verkleSyncProvider)); _blockDownloaderFactory = blockDownloaderFactory ?? throw new ArgumentNullException(nameof(blockDownloaderFactory)); _pivot = pivot ?? throw new ArgumentNullException(nameof(pivot)); _syncPeerPool = peerPool ?? throw new ArgumentNullException(nameof(peerPool)); @@ -112,6 +117,11 @@ public virtual void Start() StartSnapSyncComponents(); } + if (_syncConfig.VerkleSync) + { + StartVerkleSyncComponents(); + } + StartStateSyncComponents(); } @@ -123,7 +133,7 @@ public virtual void Start() private void SetupDbOptimizer() { - new SyncDbTuner(_syncConfig, _snapSyncFeed, _bodiesFeed, _receiptsFeed, _dbProvider.StateDb, _dbProvider.CodeDb, + new SyncDbTuner(_syncConfig, _snapSyncFeed, _verkleSyncFeed, _bodiesFeed, _receiptsFeed, _dbProvider.StateDb, _dbProvider.CodeDb, _dbProvider.BlocksDb, _dbProvider.ReceiptsDb); } @@ -181,6 +191,24 @@ private void StartSnapSyncComponents() }); } + private void StartVerkleSyncComponents() + { + _verkleSyncFeed = new VerkleSyncFeed(_syncMode, _verkleSyncProvider, _logManager); + VerkleSyncDispatcher dispatcher = new(_verkleSyncFeed!, _syncPeerPool, new VerkleSyncAllocationStrategyFactory(), _logManager); + + Task _ = dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + { + if (t.IsFaulted) + { + if (_logger.IsError) _logger.Error("Verkle sync failed", t.Exception); + } + else + { + if (_logger.IsInfo) _logger.Info("Verkle sync task completed."); + } + }); + } + private void StartFastBlocksComponents() { FastBlocksPeerAllocationStrategyFactory fastFactory = new(); @@ -284,6 +312,7 @@ public Task StopAsync() _fastSyncFeed?.FeedTask ?? Task.CompletedTask, _stateSyncFeed?.FeedTask ?? Task.CompletedTask, _snapSyncFeed?.FeedTask ?? Task.CompletedTask, + _verkleSyncFeed?.FeedTask ?? Task.CompletedTask, _fullSyncFeed?.FeedTask ?? Task.CompletedTask, _headersFeed?.FeedTask ?? Task.CompletedTask, _bodiesFeed?.FeedTask ?? Task.CompletedTask, @@ -297,6 +326,7 @@ public void Dispose() _fastSyncFeed?.Dispose(); _stateSyncFeed?.Dispose(); _snapSyncFeed?.Dispose(); + _verkleSyncFeed?.Dispose(); _fullSyncFeed?.Dispose(); _headersFeed?.Dispose(); _bodiesFeed?.Dispose(); diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/AddRangeResult.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/AddRangeResult.cs new file mode 100644 index 00000000000..76b622b547a --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/AddRangeResult.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Synchronization.VerkleSync +{ + public enum AddRangeResult + { + OK, + MissingRootHashInProofs, + DifferentRootHash, + ExpiredRootHash + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/IVerkleSyncProvider.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/IVerkleSyncProvider.cs new file mode 100644 index 00000000000..e3973337893 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/IVerkleSyncProvider.cs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Synchronization.VerkleSync; + +public interface IVerkleSyncProvider +{ + (VerkleSyncBatch request, bool finished) GetNextRequest(); + bool CanSync(); + + AddRangeResult AddSubTreeRange(SubTreeRange request, SubTreesAndProofs response); + AddRangeResult AddSubTreeRange(long blockNumber, Pedersen expectedRootHash, Stem startingStem, PathWithSubTree[] subTrees, byte[] proofs = null, Stem limitStem = null!); + + void RefreshLeafs(LeafToRefreshRequest request, byte[][] response); + + void RetryRequest(VerkleSyncBatch batch); + + bool IsVerkleGetRangesFinished(); + void UpdatePivot(); +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleProgressTracker.cs new file mode 100644 index 00000000000..2e2855dad96 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleProgressTracker.cs @@ -0,0 +1,295 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Synchronization.RangeSync; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleProgressTracker: IRangeProgressTracker +{ + private const string NO_REQUEST = "NO REQUEST"; + private readonly byte[] SYNC_PROGRESS_KEY = "VerkleSyncProgressKey"u8.ToArray(); + + // This does not need to be a lot as it spawn other requests. In fact 8 is probably too much. It is severely + // bottlenecked by _syncCommit lock in SnapProviderHelper, which in turns is limited by the IO. + // In any case, all partition will be touched when calculating progress, so we can't really put like 1024 for this. + private readonly int _subTreeRangePartitionCount; + + private long _reqCount; + private int _activeSubTreeRequests; + private int _activeLeafRefreshRequests; + + private readonly ILogger _logger; + private readonly IDb _db; + + // Partitions are indexed by its limit keccak/address as they are keep in the request struct and remain the same + // throughout the sync. So its easy. + private Dictionary SubTreeRangePartitions { get; set; } = new(); + + // Using a queue here to evenly distribute request across partitions. Don't want a situation where one really slow + // partition is taking up most of the time at the end of the sync. + private ConcurrentQueue SubTreeRangeReadyForRequest { get; set; } = new(); + private ConcurrentQueue LeafsToRefresh { get; set; } = new(); + + + private readonly RangeSync.Pivot _pivot; + + public VerkleProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int subTreeRangePartitionCount = 8) + { + _logger = logManager.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _db = db ?? throw new ArgumentNullException(nameof(db)); + + _pivot = new RangeSync.Pivot(blockTree, logManager); + + if (subTreeRangePartitionCount < 1 || subTreeRangePartitionCount > 256) + throw new ArgumentException("SubTree range partition must be between 1 to 256."); + + _subTreeRangePartitionCount = subTreeRangePartitionCount; + SetupSubTreeRangePartition(); + + //TODO: maybe better to move to a init method instead of the constructor + GetSyncProgress(); + } + + private void SetupSubTreeRangePartition() + { + // Confusingly dividing the range evenly via UInt256 for example, consistently cause root hash mismatch. + // The mismatch happens on exactly the same partition every time, suggesting tome kind of boundary issues + // either on proof generation or validation. + byte curStartingPath = 0; + int partitionSize = (256 / _subTreeRangePartitionCount); + + for (int i = 0; i < _subTreeRangePartitionCount; i++) + { + SubTreeRangePartition partition = new SubTreeRangePartition(); + + Stem startingPath = new(Stem.Zero.Bytes.ToArray()) { Bytes = { [0] = curStartingPath } }; + + partition.NextSubTreePath = startingPath.Bytes; + partition.SubTreePathStart = startingPath.Bytes; + + curStartingPath += (byte)partitionSize; + + // Special case for the last partition + Stem limitPath = + i == _subTreeRangePartitionCount - 1 + ? Stem.MaxValue + : new Stem(Stem.Zero.Bytes.ToArray()) { Bytes = { [0] = curStartingPath } }; + + partition.SubTreePathLimit = limitPath.Bytes; + + SubTreeRangePartitions[limitPath.Bytes] = partition; + SubTreeRangeReadyForRequest.Enqueue(partition); + } + } + + public bool CanSync() + { + BlockHeader? header = _pivot.GetPivotHeader(); + if (header is null || header.Number == 0) + { + if (_logger.IsInfo) _logger.Info($"No Best Suggested Header available. Verkle Sync not started."); + + return false; + } + + if (_logger.IsInfo) _logger.Info($"Starting the VERKLE_SYNC data sync from the {header.ToString(BlockHeader.Format.Short)} {header.StateRoot} root"); + + return true; + } + + public void UpdatePivot() + { + _pivot.UpdateHeaderForcefully(); + } + + public (VerkleSyncBatch request, bool finished) GetNextRequest() + { + Interlocked.Increment(ref _reqCount); + + var pivotHeader = _pivot.GetPivotHeader(); + var rootHash = pivotHeader.StateRoot; + var blockNumber = pivotHeader.Number; + + VerkleSyncBatch request = new(); + + if (!LeafsToRefresh.IsEmpty) + { + Interlocked.Increment(ref _activeLeafRefreshRequests); + + LogRequest($"LeafsToRefresh:{LeafsToRefresh.Count}"); + + int queueLength = LeafsToRefresh.Count; + byte[][] paths = new byte[queueLength][]; + + for (int i = 0; i < queueLength && LeafsToRefresh.TryDequeue(out var acc); i++) + { + paths[i] = acc; + } + + request.LeafToRefreshRequest = new LeafToRefreshRequest() { RootHash = rootHash.Bytes, Paths = paths }; + + return (request, false); + + } + + if (ShouldRequestSubTreeRequests() && SubTreeRangeReadyForRequest.TryDequeue(out SubTreeRangePartition partition)) + { + Interlocked.Increment(ref _activeSubTreeRequests); + + SubTreeRange range = new( + rootHash.Bytes, + partition.NextSubTreePath.Bytes, + partition.SubTreePathLimit.Bytes, + blockNumber); + + LogRequest("SubTreeRange"); + + request.SubTreeRangeRequest = range; + + return (request, false); + } + + bool rangePhaseFinished = IsGetRangesFinished(); + if (rangePhaseFinished) + { + _logger.Info($"VERKLE_SYNC - State Ranges (Phase 1) finished."); + FinishRangePhase(); + } + + LogRequest(NO_REQUEST); + + return (null, IsGetRangesFinished()); + } + + private bool ShouldRequestSubTreeRequests() + { + return _activeSubTreeRequests < _subTreeRangePartitionCount; + } + + + public void ReportLeafRefreshFinished(LeafToRefreshRequest leafToRefreshRequest = null) + { + if (leafToRefreshRequest is not null) + { + foreach (byte[]? path in leafToRefreshRequest.Paths) + { + LeafsToRefresh.Enqueue(path); + } + } + + Interlocked.Decrement(ref _activeLeafRefreshRequests); + } + + public void EnqueueLeafRefresh(byte[] leaf) + { + LeafsToRefresh.Enqueue(leaf); + } + + public void ReportSubTreeRangePartitionFinished(Stem hashLimit) + { + SubTreeRangePartition partition = SubTreeRangePartitions[hashLimit]; + + if (partition.MoreSubTreesToRight) + { + SubTreeRangeReadyForRequest.Enqueue(partition); + } + Interlocked.Decrement(ref _activeSubTreeRequests); + } + + public void UpdateSubTreePartitionProgress(Stem hashLimit, Stem nextPath, bool moreChildrenToRight) + { + SubTreeRangePartition partition = SubTreeRangePartitions[hashLimit]; + + partition.NextSubTreePath = nextPath; + partition.MoreSubTreesToRight = moreChildrenToRight && nextPath < hashLimit; + } + + public bool IsGetRangesFinished() + { + return SubTreeRangeReadyForRequest.IsEmpty + && LeafsToRefresh.IsEmpty + && _activeSubTreeRequests == 0 + && _activeLeafRefreshRequests == 0; + } + + private void GetSyncProgress() + { + // Note, as before, the progress actually only store MaxValue or 0. So we can't actually resume + // verkle sync on restart. + byte[] progress = _db.Get(SYNC_PROGRESS_KEY); + if (progress is { Length: 31 }) + { + Stem path = new(progress); + + if (path == Stem.MaxValue) + { + _logger.Info($"VERKLE_SYNC - State Ranges (Phase 1) is finished."); + foreach (KeyValuePair partition in SubTreeRangePartitions) + { + partition.Value.MoreSubTreesToRight = false; + } + SubTreeRangeReadyForRequest.Clear(); + } + else + { + _logger.Info($"VERKLE_SYNC - State Ranges (Phase 1) progress loaded from DB:{path}"); + } + } + } + + private void FinishRangePhase() + { + _db.Set(SYNC_PROGRESS_KEY, Stem.MaxValue.Bytes); + } + + private void LogRequest(string reqType) + { + if (_reqCount % 100 == 0) + { + int totalPathProgress = 0; + foreach (KeyValuePair kv in SubTreeRangePartitions) + { + SubTreeRangePartition? partition = kv.Value; + int nextAccount = partition.NextSubTreePath.Bytes[0] * 256 + partition.NextSubTreePath.Bytes[1]; + int startAccount = partition.SubTreePathStart.Bytes[0] * 256 + partition.SubTreePathStart.Bytes[1]; + totalPathProgress += nextAccount - startAccount; + } + + double progress = 100 * totalPathProgress / (double)(256 * 256); + + if (_logger.IsInfo) _logger.Info($"VERKLE_SYNC - progress of State Ranges (Phase 1): {progress:f3}% [{new string('*', (int)progress / 10)}{new string(' ', 10 - (int)progress / 10)}]"); + } + + if (_logger.IsTrace || _reqCount % 1000 == 0) + { + int moreAccountCount = SubTreeRangePartitions.Count(kv => kv.Value.MoreSubTreesToRight); + + _logger.Info( + $"VERKLE_SYNC - ({reqType}, diff:{_pivot.Diff}) {moreAccountCount} - Requests Account:{_activeSubTreeRequests} | Refresh:{_activeLeafRefreshRequests} - Refresh:{LeafsToRefresh.Count}"); + } + } + + + // A partition of the top level account range starting from `SubTreePathStart` to `SubTreePathLimit` (exclusive). + private class SubTreeRangePartition + { + public Stem NextSubTreePath { get; set; } = Stem.Zero; + public Stem SubTreePathStart { get; set; } = Stem.Zero; // Not really needed, but useful + public Stem SubTreePathLimit { get; set; } = Stem.MaxValue; + public bool MoreSubTreesToRight { get; set; } = true; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleProviderHelper.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleProviderHelper.cs new file mode 100644 index 00000000000..692f814c174 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleProviderHelper.cs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core.Crypto; +using Nethermind.State; +using Nethermind.State.Snap; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleProviderHelper +{ + public static (AddRangeResult result, bool moreChildrenToRight) AddSubTreeRange( + VerkleStateTree tree, + long blockNumber, + byte[] expectedRootHash, + byte[] startingHash, + byte[] limitHash, + PathWithSubTree[] subTrees, + byte[][] proofs = null + ) + { + byte[] lastStem = subTrees[^1].Path.Bytes; + + foreach (PathWithSubTree? subTree in subTrees) + { + tree.InsertStemBatch(subTree.Path.Bytes, subTree.SubTree); + tree.Commit(); + } + + if (tree.StateRoot.Bytes != expectedRootHash) + { + return (AddRangeResult.DifferentRootHash, true); + } + + tree.CommitTree(blockNumber); + + return (AddRangeResult.OK, true); + } + + +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncAllocationStrategyFactory.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncAllocationStrategyFactory.cs new file mode 100644 index 00000000000..43942455bb4 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncAllocationStrategyFactory.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain.Synchronization; +using Nethermind.Stats; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.Peers.AllocationStrategies; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleSyncAllocationStrategyFactory: StaticPeerAllocationStrategyFactory +{ + private static readonly IPeerAllocationStrategy DefaultStrategy = + new SatelliteProtocolPeerAllocationStrategy(new TotalDiffStrategy(new BySpeedStrategy(TransferSpeedType.VerkleSyncRanges, true), TotalDiffStrategy.TotalDiffSelectionType.CanBeSlightlyWorse), "verkle"); + + public VerkleSyncAllocationStrategyFactory() : base(DefaultStrategy) + { + } + +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncBatch.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncBatch.cs new file mode 100644 index 00000000000..3857ee7b60a --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncBatch.cs @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleSyncBatch +{ + public SubTreeRange? SubTreeRangeRequest { get; set; } + + public SubTreesAndProofs? SubTreeRangeResponse { get; set; } + + public LeafToRefreshRequest? LeafToRefreshRequest { get; set; } + public byte[][]? LeafToRefreshResponse { get; set; } + + public override string ToString() + { + if (SubTreeRangeRequest is not null) + { + return SubTreeRangeRequest!.ToString(); + } + else if (LeafToRefreshRequest is not null) + { + return LeafToRefreshRequest!.ToString(); + } + else + { + return "Empty snap sync batch"; + } + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncDispatcher.cs new file mode 100644 index 00000000000..353795e5442 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncDispatcher.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Logging; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleSyncDispatcher: SyncDispatcher +{ + public VerkleSyncDispatcher(ISyncFeed? syncFeed, ISyncPeerPool? syncPeerPool, IPeerAllocationStrategyFactory? peerAllocationStrategy, ILogManager? logManager) + : base(syncFeed, syncPeerPool, peerAllocationStrategy, logManager) + { + } + + protected override async Task Dispatch(PeerInfo peerInfo, VerkleSyncBatch batch, + CancellationToken cancellationToken) + { + ISyncPeer peer = peerInfo.SyncPeer; + + //TODO: replace with a constant "snap" + if (peer.TryGetSatelliteProtocol("verkle", out IVerkleSyncPeer? handler)) + { + try + { + if (batch.SubTreeRangeRequest is not null) + { + batch.SubTreeRangeResponse = await handler.GetSubTreeRange(batch.SubTreeRangeRequest, cancellationToken); + } + else if (batch.LeafToRefreshRequest is not null) + { + batch.LeafToRefreshResponse = await handler.GetLeafNodes(batch.LeafToRefreshRequest, cancellationToken); + } + } + catch (Exception e) + { + if (Logger.IsDebug) + Logger.Error($"DEBUG/ERROR Error after dispatching the snap sync request. Request: {batch}", e); + } + } + + await Task.CompletedTask; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncFeed.cs new file mode 100644 index 00000000000..ceca773af13 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncFeed.cs @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Logging; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleSyncFeed: SyncFeed, IDisposable +{ + private readonly object _syncLock = new(); + + private readonly ISyncModeSelector _syncModeSelector; + private readonly IVerkleSyncProvider _syncProvider; + private readonly ILogger _logger; + + private const VerkleSyncBatch EmptyBatch = null; + + public VerkleSyncFeed(ISyncModeSelector syncModeSelector, IVerkleSyncProvider syncProvider, ILogManager logManager) + { + _syncModeSelector = syncModeSelector; + _syncProvider = syncProvider; + + _logger = logManager.GetClassLogger(); + + _syncModeSelector.Changed += SyncModeSelectorOnChanged; + } + + public override Task PrepareRequest(CancellationToken token = default) + { + try + { + (VerkleSyncBatch request, bool finished) = _syncProvider.GetNextRequest(); + if (request is not null) return Task.FromResult(request); + + if (finished) Finish(); + return Task.FromResult(EmptyBatch); + } + catch (Exception e) + { + _logger.Error("Error when preparing a batch", e); + return Task.FromResult(EmptyBatch); + } + } + + public override SyncResponseHandlingResult HandleResponse(VerkleSyncBatch? batch, PeerInfo? peer = null) + { + if (batch is null) + { + if (_logger.IsError) _logger.Error("Received empty batch as a response"); + return SyncResponseHandlingResult.InternalError; + } + + AddRangeResult result = AddRangeResult.OK; + + if (batch.SubTreeRangeResponse is not null) + { + result = _syncProvider.AddSubTreeRange(batch.SubTreeRangeRequest, batch.SubTreeRangeResponse); + } + else if (batch.LeafToRefreshRequest is not null) + { + _syncProvider.RefreshLeafs(batch.LeafToRefreshRequest, batch.LeafToRefreshResponse); + } + else + { + _syncProvider.RetryRequest(batch); + + if (peer is null) return SyncResponseHandlingResult.NotAssigned; + + _logger.Trace($"VERKLE SYNC - timeout {peer}"); + return SyncResponseHandlingResult.LesserQuality; + } + + return AnalyzeResponsePerPeer(result, peer); + } + + + private const int AllowedInvalidResponses = 5; + private readonly LinkedList<(PeerInfo peer, AddRangeResult result)> _resultLog = new(); + public SyncResponseHandlingResult AnalyzeResponsePerPeer(AddRangeResult result, PeerInfo? peer = null) + { + if (peer is null) return SyncResponseHandlingResult.OK; + + const int maxSize = 10 * AllowedInvalidResponses; + + while (_resultLog.Count > maxSize) + { + lock (_syncLock) + { + if (_resultLog.Count > 0) + { + _resultLog.RemoveLast(); + } + } + } + + lock (_syncLock) + { + _resultLog.AddFirst((peer, result)); + } + + if (result == AddRangeResult.OK) return SyncResponseHandlingResult.OK; + + int allLastSuccess = 0; + int allLastFailures = 0; + int peerLastFailures = 0; + + lock (_syncLock) + { + foreach ((PeerInfo peer, AddRangeResult result) item in _resultLog) + { + if (item.result == AddRangeResult.OK) + { + allLastSuccess++; + + if (item.peer == peer) + { + break; + } + } + else + { + allLastFailures++; + + if (item.peer == peer) + { + peerLastFailures++; + + if (peerLastFailures > AllowedInvalidResponses) + { + if (allLastFailures == peerLastFailures) + { + _logger.Trace($"VERKLE SYNC - peer to be punished:{peer}"); + return SyncResponseHandlingResult.LesserQuality; + } + + if (allLastSuccess == 0 && allLastFailures > peerLastFailures) + { + // TODO: here try to get proofs and witnesses from block and update the state when + // pivot changes - this will allow to get rid of healing + _syncProvider.UpdatePivot(); + _resultLog.Clear(); + break; + } + } + } + } + } + } + + return result == AddRangeResult.ExpiredRootHash + ? SyncResponseHandlingResult.NoProgress + : SyncResponseHandlingResult.OK; + } + + public override bool IsMultiFeed => true; + public override AllocationContexts Contexts => AllocationContexts.Verkle; + + private void SyncModeSelectorOnChanged(object? sender, SyncModeChangedEventArgs e) + { + if (CurrentState == SyncFeedState.Dormant) + { + if ((e.Current & SyncMode.VerkleSync) == SyncMode.VerkleSync) + { + if (_syncProvider.CanSync()) + { + Activate(); + } + } + } + } + + public void Dispose() + { + _syncModeSelector.Changed -= SyncModeSelectorOnChanged; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncProvider.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncProvider.cs new file mode 100644 index 00000000000..9eef1d242e7 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncProvider.cs @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.ObjectPool; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Proofs; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; +using ILogger = Nethermind.Logging.ILogger; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleSyncProvider: IVerkleSyncProvider +{ + private readonly ObjectPool _trieStorePool; + private readonly ILogManager _logManager; + private readonly ILogger _logger; + + private readonly VerkleProgressTracker _progressTracker; + + public VerkleSyncProvider(VerkleProgressTracker progressTracker, IDbProvider dbProvider, ILogManager logManager) + { + IDbProvider dbProvider1 = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); + _progressTracker = progressTracker ?? throw new ArgumentNullException(nameof(progressTracker)); + _trieStorePool = new DefaultObjectPool(new TrieStorePoolPolicy(dbProvider1, logManager)); + + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + _logger = logManager.GetClassLogger(); + } + + public bool CanSync() => _progressTracker.CanSync(); + + public AddRangeResult AddSubTreeRange(SubTreeRange request, SubTreesAndProofs response) + { + AddRangeResult result; + + if (response.SubTrees.Length == 0 && response.Proofs.Length == 0) + { + if(_logger.IsTrace) _logger.Trace($"VERKLE_SYNC - GetSubTreeRange - requested expired RootHash:{request.RootHash}"); + + result = AddRangeResult.ExpiredRootHash; + } + else + { + result = AddSubTreeRange(request.BlockNumber.Value, request.RootHash, request.StartingStem, response.SubTrees, response.Proofs, limitStem: request.LimitStem); + + if (result == AddRangeResult.OK) + { + Interlocked.Add(ref Metrics.SnapSyncedAccounts, response.SubTrees.Length); + } + } + + _progressTracker.ReportSubTreeRangePartitionFinished(request.LimitStem); + + return result; + } + + public AddRangeResult AddSubTreeRange(long blockNumber, Pedersen expectedRootHash, Stem startingStem, + PathWithSubTree[] subTrees, byte[]? proofs = null, Stem? limitStem = null) + { + limitStem ??= Keccak.MaxValue.Bytes[..31]; + Banderwagon rootPoint = Banderwagon.FromBytes(expectedRootHash.Bytes) ?? throw new Exception("root point invalid"); + IVerkleTrieStore store = _trieStorePool.Get(); + VerkleTree tree = new VerkleTree(store, LimboLogs.Instance); + try + { + VerkleProof vProof = VerkleProof.Decode(proofs!); + bool correct = + tree.CreateStatelessTreeFromRange(vProof, rootPoint, startingStem, limitStem, + subTrees); + if (!correct) + { + if(_logger.IsTrace) _logger.Trace( + $"VERKLE_SYNC - AddSubTreeRange failed, expected {blockNumber}:{expectedRootHash}, startingHash:{startingStem}"); + return AddRangeResult.DifferentRootHash; + } + + _progressTracker.UpdateSubTreePartitionProgress(limitStem, subTrees[^1].Path, true); + return AddRangeResult.OK; + } + finally + { + _trieStorePool.Return(store); + } + } + + + public AddRangeResult AddSubTreeRange(long blockNumber, Banderwagon rootPoint, byte[] startingStem, + PathWithSubTree[] subTrees, VerkleProof proof, byte[] limitStem) + { + IVerkleTrieStore store = _trieStorePool.Get(); + VerkleTree tree = new VerkleTree(store, LimboLogs.Instance); + bool correct = + tree.CreateStatelessTreeFromRange(proof, rootPoint, startingStem, limitStem, + subTrees); + if (!correct) return AddRangeResult.DifferentRootHash; + return AddRangeResult.OK; + } + + public bool HealTheTreeFromExecutionWitness(ExecutionWitness execWitness, Banderwagon root) + { + IVerkleTrieStore store = _trieStorePool.Get(); + VerkleTree tree = new (store, LimboLogs.Instance); + return tree.InsertIntoStatelessTree(execWitness, root, false); + } + + public void RefreshLeafs(LeafToRefreshRequest request, byte[][] response) + { + throw new NotImplementedException(); + } + + private void RetryLeafRefresh(byte[] leaf) + { + _progressTracker.EnqueueLeafRefresh(leaf); + } + + public void RetryRequest(VerkleSyncBatch batch) + { + if (batch.SubTreeRangeRequest is not null) + { + _progressTracker.ReportSubTreeRangePartitionFinished(batch.SubTreeRangeRequest.LimitStem); + } + else if (batch.LeafToRefreshRequest is not null) + { + _progressTracker.ReportLeafRefreshFinished(batch.LeafToRefreshRequest); + } + } + + public bool IsVerkleGetRangesFinished() => _progressTracker.IsGetRangesFinished(); + + public void UpdatePivot() + { + _progressTracker.UpdatePivot(); + } + + public (VerkleSyncBatch request, bool finished) GetNextRequest() => _progressTracker.GetNextRequest(); + + private class TrieStorePoolPolicy : IPooledObjectPolicy + { + private readonly IDbProvider _dbProvider; + private readonly ILogManager _logManager; + + public TrieStorePoolPolicy(IDbProvider provider, ILogManager logManager) + { + _dbProvider = provider; + _logManager = logManager; + } + + public IVerkleTrieStore Create() + { + return new VerkleStateStore(_dbProvider, _logManager,0); + } + + public bool Return(IVerkleTrieStore obj) + { + return true; + } + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncServer.cs b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncServer.cs new file mode 100644 index 00000000000..4626089e534 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/VerkleSync/VerkleSyncServer.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core.Verkle; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Proofs; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Synchronization.VerkleSync; + +public class VerkleSyncServer +{ + private readonly IVerkleTrieStore _store; + private readonly ILogManager _logManager; + private readonly ILogger _logger; + + private const long HardResponseByteLimit = 2000000; + private const int HardResponseNodeLimit = 10000; + + public VerkleSyncServer(IVerkleTrieStore trieStore, ILogManager logManager) + { + _store = trieStore ?? throw new ArgumentNullException(nameof(trieStore)); + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + _logger = logManager.GetClassLogger(); + } + + public (List, VerkleProof) GetSubTreeRanges(VerkleCommitment rootHash, Stem startingStem, Stem? limitStem, long byteLimit, out Banderwagon rootPoint) + { + rootPoint = default; + if(_logger.IsDebug) _logger.Debug($"Getting SubTreeRanges - RH:{rootHash} S:{startingStem} L:{limitStem} Bytes:{byteLimit}"); + List nodes = _store.GetLeafRangeIterator(startingStem, limitStem?? Stem.MaxValue, rootHash, byteLimit).ToList(); + if(_logger.IsDebug) _logger.Debug($"Nodes Count - {nodes.Count}"); + if (nodes.Count == 0) return (new List(), new VerkleProof()); + + VerkleTree tree = new (_store, _logManager); + VerkleProof vProof = + tree.CreateVerkleRangeProof(startingStem.Bytes, nodes[^1].Path.Bytes, out rootPoint); + return (nodes, vProof); + } +} diff --git a/src/Nethermind/Nethermind.Trie/VisitContext.cs b/src/Nethermind/Nethermind.Trie/VisitContext.cs index 7bbfb14b5c3..cac556ffe52 100644 --- a/src/Nethermind/Nethermind.Trie/VisitContext.cs +++ b/src/Nethermind/Nethermind.Trie/VisitContext.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Generic; using System.Runtime.InteropServices; using System.Threading; @@ -13,16 +14,31 @@ public class TrieVisitContext : IDisposable private readonly int _maxDegreeOfParallelism = 1; private int _visitedNodes; - public int Level { get; internal set; } + public int Level { get; set; } public bool IsStorage { get; internal set; } public int? BranchChildIndex { get; internal set; } public bool ExpectAccounts { get; init; } public int VisitedNodes => _visitedNodes; + public bool KeepTrackOfAbsolutePath { get; init; } + + private List? _absolutePathIndex; + + public List AbsolutePathIndex => _absolutePathIndex ??= new List(); public int MaxDegreeOfParallelism { get => _maxDegreeOfParallelism; - internal init => _maxDegreeOfParallelism = value == 0 ? Environment.ProcessorCount : value; + init => _maxDegreeOfParallelism = value == 0 ? Environment.ProcessorCount : value; + } + + public AbsolutePathStruct AbsolutePathNext(byte[] path) + { + return new AbsolutePathStruct(!KeepTrackOfAbsolutePath ? null : AbsolutePathIndex, path); + } + + public AbsolutePathStruct AbsolutePathNext(byte path) + { + return new AbsolutePathStruct(!KeepTrackOfAbsolutePath ? null : AbsolutePathIndex, path); } public SemaphoreSlim Semaphore @@ -59,6 +75,32 @@ public void AddVisited() } } + public readonly ref struct AbsolutePathStruct + { + public AbsolutePathStruct(List? absolutePath, IReadOnlyCollection? path) + { + _absolutePath = absolutePath; + _pathLength = path!.Count; + _absolutePath?.AddRange(path!); + } + + public AbsolutePathStruct(List? absolutePath, byte path) + { + _absolutePath = absolutePath; + _pathLength = 1; + _absolutePath?.Add(path); + } + + private readonly List? _absolutePath; + private readonly int _pathLength; + + public void Dispose() + { + if (_pathLength > 0) + _absolutePath?.RemoveRange(_absolutePath.Count - _pathLength, _pathLength); + } + } + [StructLayout(LayoutKind.Sequential, Pack = 1)] public struct SmallTrieVisitContext { diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/AccountHeaderTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/AccountHeaderTests.cs new file mode 100644 index 00000000000..d485df42707 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/AccountHeaderTests.cs @@ -0,0 +1,191 @@ +using FluentAssertions; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Int256; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Test; + +[TestFixture] +public class AccountHeaderTests +{ + + [Test] + public void TestGetTreeKey() + { + Span addr = new byte[32]; + for (int i = 0; i < 16; i++) + { + addr[1 + 2 * i] = 255; + } + + UInt256 n = 1; + n = n << 129; + n = n + 3; + byte[] key = PedersenHash.Hash(addr, n); + key[31] = 1; + + key.ToHexString().Should().BeEquivalentTo("f42f932f43faf5d14b292b9009c45c28da61dbf66e20dbedc2e02dfd64ff5a01"); + } + + [Test] + public void SetAccountWithCode() + { + byte[] code = { 1, 2, 3, 4 }; + CodeChunkEnumerator codeEnumerator = new CodeChunkEnumerator(code); + + codeEnumerator.TryGetNextChunk(out byte[] value); + value.Should().NotBeNull(); + value[..5].Should().BeEquivalentTo(new byte[] { 0, 1, 2, 3, 4 }); + value[5..32].Should().BeEquivalentTo(new byte[27]); + + codeEnumerator.TryGetNextChunk(out value).Should().BeFalse(); + } + + [Test] + public void SetAccountWithCodePushOpcodes() + { + byte[] code1 = { 97, 1, 2, 3, 4 }; + CodeChunkEnumerator codeEnumerator = new CodeChunkEnumerator(code1); + + codeEnumerator.TryGetNextChunk(out byte[] value); + value.Should().NotBeNull(); + value[0..6].Should().BeEquivalentTo(new byte[] { 0, 97, 1, 2, 3, 4 }); + value[6..32].Should().BeEquivalentTo(new byte[26]); + + byte[] code2 = + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 100, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45 + }; + byte[] firstCodeChunk = + { + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 100, 30 + }; + byte[] secondCodeChunk = + { + 4, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45 + }; + + codeEnumerator = new CodeChunkEnumerator(code2); + + codeEnumerator.TryGetNextChunk(out value); + value.Should().NotBeNull(); + value.Should().BeEquivalentTo(firstCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value.Should().NotBeNull(); + value[0..16].Should().BeEquivalentTo(secondCodeChunk); + value[16..32].Should().BeEquivalentTo(new byte[16]); + + } + + [Test] + public void SetCodeEdgeCases1() + { + byte[] code = + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 127, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65 + }; + byte[] firstCodeChunk = + { + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 127 + }; + byte[] secondCodeChunk = + { + 31, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61 + }; + byte[] thirdCodeChunk = + { + 1, 62, 63, 64, 65 + }; + CodeChunkEnumerator codeEnumerator = new CodeChunkEnumerator(code); + + codeEnumerator.TryGetNextChunk(out byte[] value); + value.Should().BeEquivalentTo(firstCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value.Should().BeEquivalentTo(secondCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value[0..5].Should().BeEquivalentTo(thirdCodeChunk); + } + + [Test] + public void SetCodeEdgeCases2() + { + byte[] code = + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 126, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65 + }; + byte[] firstCodeChunk = + { + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 126 + }; + byte[] secondCodeChunk = + { + 31, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61 + }; + byte[] thirdCodeChunk = + { + 0, 62, 63, 64, 65 + }; + + CodeChunkEnumerator codeEnumerator = new CodeChunkEnumerator(code); + + codeEnumerator.TryGetNextChunk(out byte[] value); + value.Should().BeEquivalentTo(firstCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value.Should().BeEquivalentTo(secondCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value[0..5].Should().BeEquivalentTo(thirdCodeChunk); + + } + + [Test] + public void SetCodeEdgeCases3() + { + byte[] code = + { + 95, 1, 96, 3, 4, 97, 6, 7, 8, 98, 10, 11, 12, 13, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65 + }; + byte[] firstCodeChunk = + { + 0, 95, 1, 96, 3, 4, 97, 6, 7, 8, 98, 10, 11, 12, 13, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112, 113, + 114, 115, 116 + }; + byte[] secondCodeChunk = + { + 19, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61 + }; + byte[] thirdCodeChunk = + { + 0, 62, 63, 64, 65 + }; + + CodeChunkEnumerator codeEnumerator = new CodeChunkEnumerator(code); + + codeEnumerator.TryGetNextChunk(out byte[] value); + value.Should().BeEquivalentTo(firstCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value.Should().BeEquivalentTo(secondCodeChunk); + + codeEnumerator.TryGetNextChunk(out value); + value[0..5].Should().BeEquivalentTo(thirdCodeChunk); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/HistoryTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/HistoryTests.cs new file mode 100644 index 00000000000..5b65636cdf9 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/HistoryTests.cs @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Utils; +using Nethermind.Verkle.Tree.VerkleDb; +using NUnit.Framework; + +namespace Nethermind.Verkle.Tree.Test; + +public class HistoryTests +{ + + [TearDown] + public void CleanTestData() + { + string dbPath = VerkleTestUtils.GetDbPathForTest(); + if (Directory.Exists(dbPath)) + { + Directory.Delete(dbPath, true); + } + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertGetMultiBlockReverseState(DbMode dbMode) + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(dbMode); + + tree.Insert(VerkleTestUtils._keyVersion, VerkleTestUtils._emptyArray); + tree.Insert(VerkleTestUtils._keyBalance, VerkleTestUtils._emptyArray); + tree.Insert(VerkleTestUtils._keyNonce, VerkleTestUtils._emptyArray); + tree.Insert(VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._valueEmptyCodeHashValue); + tree.Insert(VerkleTestUtils._keyCodeSize, VerkleTestUtils._emptyArray); + tree.Commit(); + tree.CommitTree(0); + VerkleCommitment stateRoot0 = tree.StateRoot; + Console.WriteLine(tree.StateRoot.ToString()); + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + + tree.Insert(VerkleTestUtils._keyVersion, VerkleTestUtils._arrayAll0Last2); + tree.Insert(VerkleTestUtils._keyBalance, VerkleTestUtils._arrayAll0Last2); + tree.Insert(VerkleTestUtils._keyNonce, VerkleTestUtils._arrayAll0Last2); + tree.Insert(VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._valueEmptyCodeHashValue); + tree.Insert(VerkleTestUtils._keyCodeSize, VerkleTestUtils._arrayAll0Last2); + tree.Commit(); + tree.CommitTree(1); + VerkleCommitment stateRoot1 = tree.StateRoot; + Console.WriteLine(tree.StateRoot.ToString()); + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + + tree.Insert(VerkleTestUtils._keyVersion, VerkleTestUtils._arrayAll0Last3); + tree.Insert(VerkleTestUtils._keyBalance, VerkleTestUtils._arrayAll0Last3); + tree.Insert(VerkleTestUtils._keyNonce, VerkleTestUtils._arrayAll0Last3); + tree.Insert(VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._valueEmptyCodeHashValue); + tree.Insert(VerkleTestUtils._keyCodeSize, VerkleTestUtils._arrayAll0Last3); + tree.Commit(); + tree.CommitTree(2); + Console.WriteLine(tree.StateRoot.ToString()); + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + + tree.StateRoot = stateRoot1; + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + + tree.StateRoot = stateRoot0; + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertGetBatchMultiBlockReverseState(DbMode dbMode) + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(dbMode); + + tree.Insert(VerkleTestUtils._keyVersion, VerkleTestUtils._emptyArray); + tree.Insert(VerkleTestUtils._keyBalance, VerkleTestUtils._emptyArray); + tree.Insert(VerkleTestUtils._keyNonce, VerkleTestUtils._emptyArray); + tree.Insert(VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._valueEmptyCodeHashValue); + tree.Insert(VerkleTestUtils._keyCodeSize, VerkleTestUtils._emptyArray); + tree.Commit(); + tree.CommitTree(0); + VerkleCommitment stateRoot0 = tree.StateRoot; + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + + tree.Insert(VerkleTestUtils._keyVersion, VerkleTestUtils._arrayAll0Last2); + tree.Insert(VerkleTestUtils._keyBalance, VerkleTestUtils._arrayAll0Last2); + tree.Insert(VerkleTestUtils._keyNonce, VerkleTestUtils._arrayAll0Last2); + tree.Insert(VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._valueEmptyCodeHashValue); + tree.Insert(VerkleTestUtils._keyCodeSize, VerkleTestUtils._arrayAll0Last2); + tree.Commit(); + tree.CommitTree(1); + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + + tree.Insert(VerkleTestUtils._keyVersion, VerkleTestUtils._arrayAll0Last3); + tree.Insert(VerkleTestUtils._keyBalance, VerkleTestUtils._arrayAll0Last3); + tree.Insert(VerkleTestUtils._keyNonce, VerkleTestUtils._arrayAll0Last3); + tree.Insert(VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._valueEmptyCodeHashValue); + tree.Insert(VerkleTestUtils._keyCodeSize, VerkleTestUtils._arrayAll0Last3); + tree.Commit(); + tree.CommitTree(2); + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last3); + + tree.StateRoot = stateRoot0; + + tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._emptyArray); + } + + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestReverseDiffThenForwardDiff(DbMode dbMode) + { + // VerkleTree tree = VerkleTestUtils.GetFilledVerkleTreeForTest(dbMode); + // + // VerkleMemoryDb memory = tree.GetReverseMergedDiff(3, 1); + // + // tree.ApplyDiffLayer(memory, 3, 1); + // + // tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + // tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + // tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + // tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + // tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last2); + // + // VerkleMemoryDb forwardMemory = tree.GetForwardMergedDiff(1, 3); + // + // tree.ApplyDiffLayer(forwardMemory, 1, 3); + // + // tree.Get(VerkleTestUtils._keyVersion).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last4); + // tree.Get(VerkleTestUtils._keyBalance).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last4); + // tree.Get(VerkleTestUtils._keyNonce).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last4); + // tree.Get(VerkleTestUtils._keyCodeCommitment).Should().BeEquivalentTo(VerkleTestUtils._valueEmptyCodeHashValue); + // tree.Get(VerkleTestUtils._keyCodeSize).Should().BeEquivalentTo(VerkleTestUtils._arrayAll0Last4); + + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestReverseStateOneBlock(DbMode dbMode) + { + // VerkleTree tree = VerkleTestUtils.GetFilledVerkleTreeForTest(dbMode); + // DateTime start = DateTime.Now; + // tree.ReverseState(); + // DateTime end = DateTime.Now; + // Console.WriteLine($"ReverseState() 1 Block: {(end - start).TotalMilliseconds}"); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestForwardStateOneBlock(DbMode dbMode) + { + // VerkleTree tree = VerkleTestUtils.GetFilledVerkleTreeForTest(dbMode); + // tree.ReverseState(); + // VerkleMemoryDb forwardMemory = tree.GetForwardMergedDiff(2, 3); + // DateTime start = DateTime.Now; + // tree.ApplyDiffLayer(forwardMemory, 2, 3); + // DateTime end = DateTime.Now; + // Console.WriteLine($"ForwardState() 1 Block Insert: {(end - start).TotalMilliseconds}"); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestBatchReverseDiffs(DbMode dbMode) + { + // VerkleTree tree = GetHugeVerkleTreeForTest(dbMode); + // for (int i = 2;i <= 1000;i++) { + // DateTime start = DateTime.Now; + // IVerkleDiffDb reverseDiff = tree.GetReverseMergedDiff(1, i); + // DateTime check1 = DateTime.Now; + // tree.ReverseState(reverseDiff, (i -1)); + // DateTime check2 = DateTime.Now; + // Console.WriteLine($"Batch Reverse Diff Fetch(1, {i}): {(check1 - start).TotalMilliseconds}"); + // Console.WriteLine($"Batch Reverse State(2, {i-1}): {(check2 - check1).TotalMilliseconds}"); + //} + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestBatchForwardDiffs(DbMode dbMode) + { + // VerkleTree tree = GetHugeVerkleTreeForTest(dbMode); + // for (int i = 2;i <= 1000;i++) { + // DateTime start = DateTime.Now; + // IVerkleDiffDb forwardDiff = tree.GetForwardMergedDiff(1, i); + // DateTime check1 = DateTime.Now; + // tree.ForwardState(reverseDiff, (i -1)); + // DateTime check2 = DateTime.Now; + // Console.WriteLine($"Batch Forward Diff Fetch(1, {i}): {(check1 - start).TotalMilliseconds}"); + // Console.WriteLine($"Batch Forward State(2, {i-1}): {(check2 - check1).TotalMilliseconds}"); + //} + } + + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/InMemoryDiffTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/InMemoryDiffTests.cs new file mode 100644 index 00000000000..96cbd9f36d5 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/InMemoryDiffTests.cs @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Logging; + +namespace Nethermind.Verkle.Tree.Test; + +[TestFixture] +public class InMemoryDiffTests +{ + public static Random Random { get; } = new(); + public static int numKeys = 2000; + private static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + [TearDown] + public void CleanTestData() + { + string dbPath = GetDbPathForTest(); + if (Directory.Exists(dbPath)) + { + Directory.Delete(dbPath, true); + } + } + + private static VerkleTree GetVerkleTreeForTest(DbMode dbMode, int maxBlockInHistory) + { + IDbProvider provider; + VerkleStateStore store; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + store = new VerkleStateStore(provider, LimboLogs.Instance, maxBlockInHistory); + return new VerkleTree(store, LimboLogs.Instance); + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest() + maxBlockInHistory); + store = new VerkleStateStore(provider, LimboLogs.Instance, maxBlockInHistory); + return new VerkleTree(store, LimboLogs.Instance); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + [TestCase(DbMode.MemDb)] + public void InsertHugeTree(DbMode dbMode) + { + long block = 0; + + VerkleTree treeWithH = GetVerkleTreeForTest(dbMode, 128); + VerkleTree treeWithoutH = GetVerkleTreeForTest(dbMode, 0); + + byte[] key = new byte[32]; + byte[] value = new byte[32]; + for (int i = 0; i < numKeys; i++) + { + Random.NextBytes(key); + Random.NextBytes(value); + treeWithH.Insert(key, value); + treeWithoutH.Insert(key, value); + } + treeWithH.Commit(); + treeWithoutH.Commit(); + treeWithH.CommitTree(block++); + treeWithoutH.CommitTree(block++); + for (int i = 10; i < numKeys; i += 10) + { + Random.NextBytes(key); + Random.NextBytes(value); + for (int j = 0; j < 10; j += 1) + { + Random.NextBytes(key); + Random.NextBytes(value); + treeWithH.Insert(key, value); + treeWithoutH.Insert(key, value); + } + treeWithH.Commit(); + treeWithoutH.Commit(); + treeWithH.CommitTree(block++); + treeWithoutH.CommitTree(block++); + } + + + VerkleTreeDumper dump1 = new VerkleTreeDumper(); + VerkleTreeDumper dump2 = new VerkleTreeDumper(); + + treeWithH.Accept(dump1, treeWithH.StateRoot); + treeWithoutH.Accept(dump2, treeWithoutH.StateRoot); + + string data1 = dump1.ToString(); + string data2 = dump2.ToString(); + + data1.Should().BeEquivalentTo(data2); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/InsertHugeTreeTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/InsertHugeTreeTests.cs new file mode 100644 index 00000000000..2b256dc84c0 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/InsertHugeTreeTests.cs @@ -0,0 +1,134 @@ +// Copyright 2022 Demerzel Solutions Limited +// Licensed under Apache-2.0. For full terms, see LICENSE in the project root. + +using System.Net.Mime; +using System.Reflection; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Logging; +using Nethermind.Verkle.Tree; +using NUnit.Framework; + +namespace Nethermind.Verkle.Tree.Test; + +[TestFixture] +public class InsertHugeTreeTests +{ + public static Random Random { get; } = new(); + public static int numKeys = 1000; + private static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + private static VerkleTree GetVerkleTreeForTest(DbMode dbMode) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + [TearDown] + public void CleanTestData() + { + string dbPath = GetDbPathForTest(); + if (Directory.Exists(dbPath)) + { + Directory.Delete(dbPath, true); + } + } + + // [Test] + // public void CreateDbWith190MillionAccounts() + // { + // const string dbname = "VerkleTrie190ML"; + // string? path = Path.GetDirectoryName(dbname); + // + // byte[] stem = new byte[31]; + // byte[] val = new byte[32]; + // + // IDbProvider provider = VerkleDbFactory.InitDatabase(DbMode.PersistantDb, path); + // VerkleTree verkleTree = new VerkleTree(provider); + // + // for (int i = 0; i < 190000000; i++) + // { + // Console.WriteLine(i); + // Dictionary leafIndexValueMap = new Dictionary(); + // + // Random.NextBytes(val); + // leafIndexValueMap.Add(0, val); + // Random.NextBytes(val); + // leafIndexValueMap.Add(1, val); + // Random.NextBytes(val); + // leafIndexValueMap.Add(2, val); + // Random.NextBytes(val); + // leafIndexValueMap.Add(3, val); + // Random.NextBytes(val); + // leafIndexValueMap.Add(4, val); + // + // Random.NextBytes(stem); + // verkleTree.InsertStemBatch(stem, leafIndexValueMap); + // if (i % 1000 == 0) + // { + // verkleTree.Flush(0); + // } + // } + // + // } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void InsertHugeTree(DbMode dbMode) + { + long block = 0; + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] key = new byte[32]; + byte[] value = new byte[32]; + DateTime start = DateTime.Now; + for (int i = 0; i < numKeys; i++) + { + Random.NextBytes(key); + Random.NextBytes(value); + tree.Insert(key, value); + } + DateTime check1 = DateTime.Now; + tree.Commit(); + tree.CommitTree(block++); + DateTime check2 = DateTime.Now; + Console.WriteLine($"{block} Insert: {(check1 - start).TotalMilliseconds}"); + Console.WriteLine($"{block} Flush: {(check2 - check1).TotalMilliseconds}"); + for (int i = 100; i < numKeys; i += 100) + { + DateTime check5 = DateTime.Now; + Random.NextBytes(key); + Random.NextBytes(value); + for (int j = 0; j < i; j += 1) + { + Random.NextBytes(key); + Random.NextBytes(value); + tree.Insert(key, value); + } + DateTime check3 = DateTime.Now; + tree.Commit(); + tree.CommitTree(block++); + DateTime check4 = DateTime.Now; + Console.WriteLine($"{block} Insert: {(check3 - check5).TotalMilliseconds}"); + Console.WriteLine($"{block} Flush: {(check4 - check3).TotalMilliseconds}"); + } + DateTime check6 = DateTime.Now; + Console.WriteLine($"Loop Time: {(check6 - check2).TotalMilliseconds}"); + Console.WriteLine($"Total Time: {(check6 - start).TotalMilliseconds}"); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/Nethermind.Verkle.Tree.Test.csproj b/src/Nethermind/Nethermind.Verkle.Tree.Test/Nethermind.Verkle.Tree.Test.csproj new file mode 100644 index 00000000000..10025f54a49 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/Nethermind.Verkle.Tree.Test.csproj @@ -0,0 +1,27 @@ + + + + net7.0 + enable + enable + + false + true + + + + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/NodeSerializerTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/NodeSerializerTests.cs new file mode 100644 index 00000000000..cee97fe7bde --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/NodeSerializerTests.cs @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Serializers; + +namespace Nethermind.Verkle.Tree.Test; + +[TestFixture] +public class NodeSerializerTests +{ + + private readonly Random _rand = new(0); + + [Test] + public void TestBranchNode() + { + byte[] commitment = new byte[32]; + _rand.NextBytes(commitment); + _rand.NextBytes(commitment); + InternalNode node = new (VerkleNodeType.BranchNode, new Commitment(Banderwagon.FromBytes(commitment)!.Value)); + AssertEncodeDecodeEqual(node); + } + + [Test] + public void TestFullStemNode() + { + byte[] commitment = new byte[32]; + byte[] c1 = new byte[32]; + byte[] c2 = new byte[32]; + byte[] stem = new byte[31]; + + Banderwagon? c1Point; + Banderwagon? c2Point; + Banderwagon? internalCommitment; + while (true) + { + _rand.NextBytes(commitment); + internalCommitment = Banderwagon.FromBytes(commitment); + if (internalCommitment.HasValue) break; + } + while (true) + { + _rand.NextBytes(c1); + c1Point = Banderwagon.FromBytes(c1); + if (c1Point.HasValue) break; + } + while (true) + { + _rand.NextBytes(c2); + c2Point = Banderwagon.FromBytes(c2); + if (c2Point.HasValue) break; + } + _rand.NextBytes(stem); + + InternalNode node = new(VerkleNodeType.StemNode, stem, new Commitment(c1Point.Value), + new Commitment(c2Point.Value), new Commitment(internalCommitment.Value)); + AssertEncodeDecodeEqual(node); + } + + [Test] + public void TestC1NullStemNode() + { + byte[] commitment = new byte[32]; + byte[] c2 = new byte[32]; + byte[] stem = new byte[31]; + + + Banderwagon? c2Point; + Banderwagon? internalCommitment; + while (true) + { + _rand.NextBytes(commitment); + internalCommitment = Banderwagon.FromBytes(commitment); + if (internalCommitment.HasValue) break; + } + + while (true) + { + _rand.NextBytes(c2); + c2Point = Banderwagon.FromBytes(c2); + if (c2Point.HasValue) break; + } + _rand.NextBytes(stem); + + InternalNode node = new(VerkleNodeType.StemNode, stem, null, + new Commitment(c2Point.Value), new Commitment(internalCommitment.Value)); + AssertEncodeDecodeEqual(node); + } + + [Test] + public void TestC2NullStemNode() + { + byte[] commitment = new byte[32]; + byte[] c1 = new byte[32]; + byte[] stem = new byte[31]; + + Banderwagon? c1Point; + Banderwagon? internalCommitment; + while (true) + { + _rand.NextBytes(commitment); + internalCommitment = Banderwagon.FromBytes(commitment); + if (internalCommitment.HasValue) break; + } + while (true) + { + _rand.NextBytes(c1); + c1Point = Banderwagon.FromBytes(c1); + if (c1Point.HasValue) break; + } + _rand.NextBytes(stem); + + InternalNode node = new(VerkleNodeType.StemNode, stem, new Commitment(c1Point.Value), + null, new Commitment(internalCommitment.Value)); + AssertEncodeDecodeEqual(node); + } + + [Test] + public void TestNullNullStemNode() + { + byte[] commitment = new byte[32]; + byte[] stem = new byte[31]; + + Banderwagon? internalCommitment; + while (true) + { + _rand.NextBytes(commitment); + internalCommitment = Banderwagon.FromBytes(commitment); + if (internalCommitment.HasValue) break; + } + _rand.NextBytes(stem); + + InternalNode node = new(VerkleNodeType.StemNode, stem, null, + null, new Commitment(internalCommitment.Value)); + AssertEncodeDecodeEqual(node); + } + + [Test] + public void TestZeroBanderwagonSerialized() + { + Banderwagon zero = new Banderwagon(); + var ser = zero.ToBytes(); + Banderwagon zero2 = Banderwagon.FromBytes(ser).Value; + Assert.IsTrue(zero == zero2); + } + + private void AssertEncodeDecodeEqual(InternalNode node1) + { + byte[] serNode = InternalNodeSerializer.Instance.Encode(node1).Bytes; + InternalNode node2 = InternalNodeSerializer.Instance.Decode(serNode); + Assert.That(node1.NodeType == node2.NodeType, Is.True); + switch (node2.NodeType) + { + case VerkleNodeType.BranchNode: + Assert.That(node1.InternalCommitment.Point == node2.InternalCommitment.Point, Is.True); + break; + case VerkleNodeType.StemNode: + Assert.That(node1.Stem == node2.Stem, Is.True); + Assert.That(node1.InternalCommitment.Point == node2.InternalCommitment.Point, Is.True); + Assert.That(node1.C1?.Point == node2.C1?.Point, Is.True); + Assert.That(node1.C2?.Point == node2.C2?.Point, Is.True); + break; + default: + throw new ArgumentOutOfRangeException(); + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/PedersenHashTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/PedersenHashTests.cs new file mode 100644 index 00000000000..2254c40b9d2 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/PedersenHashTests.cs @@ -0,0 +1,34 @@ +using FluentAssertions; +using Nethermind.Core.Verkle; +using Nethermind.Int256; + +namespace Nethermind.Verkle.Tree.Test +{ + [TestFixture] + public class PedersenHashTests + { + private readonly byte[] _testAddressBytesZero; + public PedersenHashTests() + { + _testAddressBytesZero = new byte[20]; + } + + [Test] + public void PedersenHashTreeKey0() + { + byte[] hash = PedersenHash.Hash(_testAddressBytesZero, UInt256.Zero); + hash[31] = 0; + Convert.ToHexString(hash).Should().BeEquivalentTo("bf101a6e1c8e83c11bd203a582c7981b91097ec55cbd344ce09005c1f26d1900"); + } + + [Test] + public void PedersenHashTreeKey1() + { + Span address32 = Convert.FromHexString("71562b71999873DB5b286dF957af199Ec94617f7"); + byte[] hash = PedersenHash.Hash(address32, UInt256.Zero); + hash[31] = 0; + Convert.ToHexString(hash).Should().BeEquivalentTo("274cde18dd9dbb04caf16ad5ee969c19fe6ca764d5688b5e1d419f4ac6cd1600"); + } + + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/StatelessTreeTest.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/StatelessTreeTest.cs new file mode 100644 index 00000000000..c20e52ab717 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/StatelessTreeTest.cs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; + +namespace Nethermind.Verkle.Tree.Test; + +public class StatelessTreeTest +{ + public static Random Random { get; } = new(0); + public static int numKeys = 1000; + private static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + private static VerkleTree GetVerkleTreeForTest(DbMode dbMode) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + return new VerkleTree(provider, SimpleConsoleLogManager.Instance); + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + return new VerkleTree(provider, SimpleConsoleLogManager.Instance); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + [TearDown] + public void CleanTestData() + { + string dbPath = GetDbPathForTest(); + if (Directory.Exists(dbPath)) + { + Directory.Delete(dbPath, true); + } + } + + [TestCase(DbMode.MemDb, 10, 20, 30)] + [TestCase(DbMode.MemDb, 100, 200, 300)] + [TestCase(DbMode.MemDb, 1000, 2000, 3000)] + public void InsertAndCreateStatelessTree(DbMode dbMode, int start, int end, int pathCount) + { + VerkleTree initTree = GetVerkleTreeForTest(dbMode); + + Pedersen[] pathPool = new Pedersen[pathCount]; + byte[][] leaf1 = new byte[pathCount][]; + byte[][] leaf2 = new byte[pathCount][]; + + for (int i = 0; i < pathCount; i++) + { + byte[] key = new byte[32]; + ((UInt256)i).ToBigEndian(key); + Pedersen keccak = new (key); + pathPool[i] = keccak; + + byte[] valueOld = new byte[32]; + Random.NextBytes(valueOld); + leaf1[i] = valueOld; + + byte[] valueNew = new byte[32]; + Random.NextBytes(valueNew); + leaf2[i] = valueNew; + } + + + + for (int i = 0; i < pathCount; i++) initTree.Insert(pathPool[i], leaf1[i]); + initTree.Commit(); + initTree.CommitTree(0); + Console.WriteLine($"Commit init tree for block 0: {initTree.StateRoot}"); + + + + ExecutionWitness execWitness = initTree.GenerateExecutionWitness(pathPool[start..end].Select(x => x.Bytes).ToArray(), out Banderwagon rootPoint); + Console.WriteLine($"generated execution witness"); + + + + for (int i = start; i < end; i++) initTree.Insert(pathPool[i], leaf2[i]); + initTree.Commit(); + initTree.CommitTree(1); + VerkleCommitment initTreeStateRootBlock1 = initTree.StateRoot; + Console.WriteLine($"Full Block1 StateRoot: {initTreeStateRootBlock1}"); + + + + VerkleTree statelessTree = GetVerkleTreeForTest(dbMode); + Console.WriteLine($"init stateless tree"); + statelessTree.InsertIntoStatelessTree(execWitness, rootPoint).Should().BeTrue(); + Console.WriteLine($"create stateless tree and now insert block 1"); + + + + for (int i = start; i < end; i++) statelessTree.Insert(pathPool[i], leaf2[i]); + statelessTree.Commit(); + statelessTree.CommitTree(1); + VerkleCommitment statelessTreeStateRootBlock1 = statelessTree.StateRoot; + Console.WriteLine($"Stateless Block1 StateRoot: {statelessTreeStateRootBlock1}"); + + statelessTreeStateRootBlock1.Should().BeEquivalentTo(initTreeStateRootBlock1); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/TestSyncRangesInAHugeVerkleTree.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/TestSyncRangesInAHugeVerkleTree.cs new file mode 100644 index 00000000000..fb532acf2a5 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/TestSyncRangesInAHugeVerkleTree.cs @@ -0,0 +1,516 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using FluentAssertions; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Proofs; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Test; + +public class TestSyncRangesInAHugeVerkleTree +{ + public static Random Random { get; } = new(0); + public static int numKeys = 2000; + private static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + private static IVerkleTrieStore GetVerkleStoreForTest(DbMode dbMode) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + break; + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + break; + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + + return new VerkleStateStore(provider, LimboLogs.Instance); + } + + private static VerkleTree GetVerkleTreeForTest(DbMode dbMode) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + [TearDown] + public void CleanTestData() + { + string dbPath = GetDbPathForTest(); + if (Directory.Exists(dbPath)) + { + Directory.Delete(dbPath, true); + } + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void GetSyncRangeForBigVerkleTree(DbMode dbMode) + { + const int pathPoolCount = 100_000; + const int numBlocks = 200; + const int leafPerBlock = 10; + const int blockToGetIteratorFrom = 180; + + IVerkleTrieStore store = TestItem.GetVerkleStore(dbMode); + VerkleTree tree = new(store, LimboLogs.Instance); + + Pedersen[] pathPool = new Pedersen[pathPoolCount]; + SortedDictionary leafs = new(); + SortedDictionary leafsForSync = new(); + + for (int i = 0; i < pathPoolCount; i++) + { + byte[] key = new byte[32]; + ((UInt256)i).ToBigEndian(key); + Pedersen keccak = new Pedersen(key); + pathPool[i] = keccak; + } + + + for (int leafIndex = 0; leafIndex < 10000; leafIndex++) + { + byte[] value = new byte[32]; + Random.NextBytes(value); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + tree.Insert(path, value); + leafs[path] = value; + leafsForSync[path] = value; + } + + tree.Commit(); + tree.CommitTree(0); + + + VerkleCommitment stateRoot180 = VerkleCommitment.Zero; + for (int blockNumber = 1; blockNumber <= numBlocks; blockNumber++) + { + for (int accountIndex = 0; accountIndex < leafPerBlock; accountIndex++) + { + byte[] leafValue = new byte[32]; + + Random.NextBytes(leafValue); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + + if (leafs.ContainsKey(path)) + { + if (!(Random.NextSingle() > 0.5)) continue; + // Console.WriteLine($"blockNumber:{blockNumber} uKey:{path} uValue:{leafValue.ToHexString()}"); + tree.Insert(path, leafValue); + leafs[path] = leafValue; + if(blockToGetIteratorFrom >= blockNumber) leafsForSync[path] = leafValue; + // Console.WriteLine("new values"); + } + else + { + // Console.WriteLine($"blockNumber:{blockNumber} nKey:{path} nValue:{leafValue.ToHexString()}"); + tree.Insert(path, leafValue); + leafs[path] = leafValue; + if(blockToGetIteratorFrom >= blockNumber) leafsForSync[path] = leafValue; + } + } + + tree.Commit(); + tree.CommitTree(blockNumber); + if (blockNumber == blockToGetIteratorFrom) stateRoot180 = tree.StateRoot; + } + + + Pedersen[] keysArray = leafs.Keys.ToArray(); + int keyLength = keysArray.Length; + using IEnumerator> rangeEnum = + tree._verkleStateStore + .GetLeafRangeIterator( + keysArray[keyLength/4].Bytes, + keysArray[(keyLength*2)/3].Bytes, 180) + .GetEnumerator(); + + + while (rangeEnum.MoveNext()) + { + // Console.WriteLine($"Key:{rangeEnum.Current.Key.ToHexString()} AcValue:{rangeEnum.Current.Value.ToHexString()} ExValue:{leafsForSync[rangeEnum.Current.Key].ToHexString()}"); + Assert.That(rangeEnum.Current.Value.SequenceEqual(leafsForSync[rangeEnum.Current.Key]), Is.True); + } + + using IEnumerator rangeEnumSized = + tree._verkleStateStore + .GetLeafRangeIterator( + keysArray[keyLength/4].StemAsSpan.ToArray(), + keysArray[(keyLength*2)/3].StemAsSpan.ToArray(), stateRoot180, 1000) + .GetEnumerator(); + + + long bytesSent = 0; + while (rangeEnumSized.MoveNext()) + { + Console.WriteLine($"{rangeEnumSized.Current.Path}"); + bytesSent += 31; + bytesSent = rangeEnumSized.Current.SubTree.Aggregate(bytesSent, (current, xx) => current + 33); + // Console.WriteLine($"Key:{rangeEnum.Current.Key.ToHexString()} AcValue:{rangeEnum.Current.Value.ToHexString()} ExValue:{leafsForSync[rangeEnum.Current.Key].ToHexString()}"); + // Assert.That(rangeEnum.Current.Value.SequenceEqual(leafsForSync[rangeEnum.Current.Key]), Is.True); + } + Console.WriteLine($"{bytesSent}"); + } + + [TestCase(DbMode.MemDb)] + public void GetSyncRangeForBigVerkleTreeAndHealTree(DbMode dbMode) + { + const int pathPoolCount = 100_000; + const int numBlocks1 = 200; + const int numBlocks2 = 20; + const int leafPerBlock = 10; + + IVerkleTrieStore remoteStore = TestItem.GetVerkleStore(dbMode); + VerkleTree remoteTree = new(remoteStore, LimboLogs.Instance); + + IVerkleTrieStore localStore = TestItem.GetVerkleStore(dbMode); + VerkleTree localTree = new(localStore, LimboLogs.Instance); + + Pedersen[] pathPool = new Pedersen[pathPoolCount]; + SortedDictionary leafs = new(); + SortedDictionary leafsForSync = new(); + + for (int i = 0; i < pathPoolCount; i++) + { + byte[] key = new byte[32]; + ((UInt256)i).ToBigEndian(key); + Pedersen keccak = new Pedersen(key); + pathPool[i] = keccak; + } + + + for (int leafIndex = 0; leafIndex < 10000; leafIndex++) + { + byte[] value = new byte[32]; + Random.NextBytes(value); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + remoteTree.Insert(path, value); + leafs[path] = value; + leafsForSync[path] = value; + } + + remoteTree.Commit(); + remoteTree.CommitTree(0); + + + for (int blockNumber = 1; blockNumber <= numBlocks1; blockNumber++) + { + for (int accountIndex = 0; accountIndex < leafPerBlock; accountIndex++) + { + byte[] leafValue = new byte[32]; + + Random.NextBytes(leafValue); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + + if (leafs.ContainsKey(path)) + { + if (!(Random.NextSingle() > 0.5)) continue; + // Console.WriteLine($"blockNumber:{blockNumber} uKey:{path} uValue:{leafValue.ToHexString()}"); + remoteTree.Insert(path, leafValue); + leafs[path] = leafValue; + // Console.WriteLine("new values"); + } + else + { + // Console.WriteLine($"blockNumber:{blockNumber} nKey:{path} nValue:{leafValue.ToHexString()}"); + remoteTree.Insert(path, leafValue); + leafs[path] = leafValue; + } + } + + remoteTree.Commit(); + remoteTree.CommitTree(blockNumber); + } + + Banderwagon root = default; + ExecutionWitness executionWitness = default; + SortedSet update = new(Bytes.Comparer); + + int startingHashIndex = 0; + int endHashIndex = 0; + for (int blockNumber = numBlocks1 + 1; blockNumber <= numBlocks1 + 5; blockNumber++) + { + for (int i = 0; i < 19; i++) + { + endHashIndex = startingHashIndex + 1000; + + PathWithSubTree[] range = + remoteTree._verkleStateStore + .GetLeafRangeIterator( + pathPool[startingHashIndex].StemAsSpan.ToArray(), + pathPool[endHashIndex].StemAsSpan.ToArray(), + remoteTree.StateRoot, 10000000) + .ToArray(); + ProcessSubTreeRange(remoteTree, localTree, blockNumber, remoteTree.StateRoot, range); + + startingHashIndex = endHashIndex + 1; + } + + if (update.Count != 0) + { + // use execution witness to heal + bool insertedWitness = localTree.InsertIntoStatelessTree(executionWitness, root, false); + Assert.IsTrue(insertedWitness); + localTree.CommitTree(0); + } + + update = new SortedSet(Bytes.Comparer); + for (int accountIndex = 0; accountIndex < leafPerBlock; accountIndex++) + { + byte[] leafValue = new byte[32]; + Random.NextBytes(leafValue); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + + if (leafs.ContainsKey(path)) + { + if (!(Random.NextSingle() > 0.5)) continue; + // Console.WriteLine($"blockNumber:{blockNumber} uKey:{path} uValue:{leafValue.ToHexString()}"); + remoteTree.Insert(path, leafValue); + leafs[path] = leafValue; + update.Add(path.Bytes); + // Console.WriteLine("new values"); + } + else + { + // Console.WriteLine($"blockNumber:{blockNumber} nKey:{path} nValue:{leafValue.ToHexString()}"); + remoteTree.Insert(path, leafValue); + leafs[path] = leafValue; + update.Add(path.Bytes); + } + } + + remoteTree.Commit(); + remoteTree.CommitTree(blockNumber); + + executionWitness = remoteTree.GenerateExecutionWitness(update.ToArray(), out root); + } + + endHashIndex = startingHashIndex + 1000; + while (endHashIndex < pathPool.Length - 1) + { + endHashIndex = startingHashIndex + 1000; + if (endHashIndex > pathPool.Length - 1) + { + endHashIndex = pathPool.Length - 1; + } + + PathWithSubTree[] range = remoteTree._verkleStateStore.GetLeafRangeIterator( + pathPool[startingHashIndex].StemAsSpan.ToArray(), + pathPool[endHashIndex].StemAsSpan.ToArray(), + remoteTree.StateRoot, 100000000).ToArray(); + ProcessSubTreeRange(remoteTree, localTree, numBlocks1 + numBlocks2, remoteTree.StateRoot, range); + + startingHashIndex += 1000; + } + + + + if (update.Count != 0) + { + // use execution witness to heal + bool insertedWitness = localTree.InsertIntoStatelessTree(executionWitness, root, false); + Assert.IsTrue(insertedWitness); + localTree.CommitTree(0); + } + + VerkleTreeDumper oldTreeDumper = new(); + VerkleTreeDumper newTreeDumper = new(); + + localTree.Accept(oldTreeDumper, localTree.StateRoot); + remoteTree.Accept(newTreeDumper, remoteTree.StateRoot); + + Console.WriteLine("oldTreeDumper"); + Console.WriteLine(oldTreeDumper.ToString()); + Console.WriteLine("newTreeDumper"); + Console.WriteLine(newTreeDumper.ToString()); + + oldTreeDumper.ToString().Should().BeEquivalentTo(newTreeDumper.ToString()); + + Assert.IsTrue(oldTreeDumper.ToString().SequenceEqual(newTreeDumper.ToString())); + } + + private static void ProcessSubTreeRange(VerkleTree remoteTree, VerkleTree localTree, int blockNumber, VerkleCommitment stateRoot, PathWithSubTree[] subTrees) + { + Stem startingStem = subTrees[0].Path; + Stem endStem = subTrees[^1].Path; + // Stem limitHash = Stem.MaxValue; + + VerkleProof proof = remoteTree.CreateVerkleRangeProof(startingStem, endStem, out Banderwagon root); + + bool isTrue = localTree.CreateStatelessTreeFromRange(proof, root, startingStem, endStem, subTrees); + Assert.IsTrue(isTrue); + } + + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void CreateHugeTree(DbMode dbMode) + { + long block = 0; + VerkleTree tree = GetVerkleTreeForTest(dbMode); + Dictionary kvMap = new(Bytes.EqualityComparer); + byte[] key = new byte[32]; + byte[] value = new byte[32]; + DateTime start = DateTime.Now; + for (int i = 0; i < numKeys; i++) + { + Random.NextBytes(key); + Random.NextBytes(value); + kvMap[key.AsSpan().ToArray()] = value.AsSpan().ToArray(); + tree.Insert(key, value); + } + DateTime check1 = DateTime.Now; + tree.Commit(); + tree.CommitTree(block++); + DateTime check2 = DateTime.Now; + Console.WriteLine($"{block} Insert: {(check1 - start).TotalMilliseconds}"); + Console.WriteLine($"{block} Flush: {(check2 - check1).TotalMilliseconds}"); + + SortedSet keys = new(Bytes.Comparer); + for (int i = 10; i < numKeys; i += 10) + { + DateTime check5 = DateTime.Now; + Random.NextBytes(key); + Random.NextBytes(value); + for (int j = (i-10); j < i; j += 1) + { + Random.NextBytes(key); + Random.NextBytes(value); + kvMap[key.AsSpan().ToArray()] = value.AsSpan().ToArray(); + tree.Insert(key, value); + keys.Add(key.AsSpan().ToArray()); + } + DateTime check3 = DateTime.Now; + tree.Commit(); + tree.CommitTree(block++); + DateTime check4 = DateTime.Now; + Console.WriteLine($"{block} Insert: {(check3 - check5).TotalMilliseconds}"); + Console.WriteLine($"{block} Flush: {(check4 - check3).TotalMilliseconds}"); + } + DateTime check6 = DateTime.Now; + Console.WriteLine($"Loop Time: {(check6 - check2).TotalMilliseconds}"); + Console.WriteLine($"Total Time: {(check6 - start).TotalMilliseconds}"); + + + byte[][] keysArray = keys.ToArray(); + using IEnumerator> rangeEnum = + tree._verkleStateStore.GetLeafRangeIterator(keysArray[30], keysArray[90], 180).GetEnumerator(); + + while (rangeEnum.MoveNext()) + { + Console.WriteLine($"Key:{rangeEnum.Current.Key.ToHexString()} Value:{rangeEnum.Current.Value.ToHexString()}"); + Assert.That(rangeEnum.Current.Value.SequenceEqual(kvMap[rangeEnum.Current.Key]), Is.True); + } + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestRangeIterator(DbMode dbMode) + { + const int pathPoolCount = 100_000; + const int leafPerBlock = 10; + + IVerkleTrieStore store = TestItem.GetVerkleStore(dbMode); + VerkleTree tree = new(store, LimboLogs.Instance); + + Pedersen[] pathPool = new Pedersen[pathPoolCount]; + SortedDictionary leafs = new(); + + for (int i = 0; i < pathPoolCount; i++) + { + byte[] key = new byte[32]; + ((UInt256)i).ToBigEndian(key); + Pedersen keccak = new Pedersen(key); + pathPool[i] = keccak; + } + + + for (int leafIndex = 0; leafIndex < 10000; leafIndex++) + { + byte[] value = new byte[32]; + Random.NextBytes(value); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + tree.Insert(path, value); + leafs[path] = value; + Console.WriteLine($"blockNumber:{0} nKey:{path} nValue:{value.ToHexString()}"); + } + + tree.Commit(); + tree.CommitTree(0); + + for (int blockNumber = 1; blockNumber <= 180; blockNumber++) + { + for (int accountIndex = 0; accountIndex < leafPerBlock; accountIndex++) + { + byte[] leafValue = new byte[32]; + + Random.NextBytes(leafValue); + Pedersen path = pathPool[Random.Next(pathPool.Length - 1)]; + + if (leafs.ContainsKey(path)) + { + if (!(Random.NextSingle() > 0.5)) continue; + Console.WriteLine($"blockNumber:{blockNumber} uKey:{path} uValue:{leafValue.ToHexString()}"); + tree.Insert(path, leafValue); + leafs[path] = leafValue; + Console.WriteLine("new values"); + } + else + { + Console.WriteLine($"blockNumber:{blockNumber} nKey:{path} nValue:{leafValue.ToHexString()}"); + tree.Insert(path, leafValue); + leafs[path] = leafValue; + } + } + + tree.Commit(); + tree.CommitTree(blockNumber); + } + + KeyValuePair[] rangeEnum = + tree._verkleStateStore.GetLeafRangeIterator(Pedersen.Zero.Bytes, Pedersen.MaxValue.Bytes, 180) + .ToArray(); + + int index = 0; + foreach (KeyValuePair leaf in leafs) + { + Console.WriteLine($"{leaf.Key} {rangeEnum[index].Key.ToHexString()}"); + Console.WriteLine($"{leaf.Value.ToHexString()} {rangeEnum[index].Value.ToArray()}"); + Assert.IsTrue(leaf.Key.Bytes.SequenceEqual(rangeEnum[index].Key)); + Assert.IsTrue(leaf.Value.SequenceEqual(rangeEnum[index].Value)); + index++; + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/Usings.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/Usings.cs new file mode 100644 index 00000000000..324456763af --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/Usings.cs @@ -0,0 +1 @@ +global using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleDbTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleDbTests.cs new file mode 100644 index 00000000000..4128aca6e66 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleDbTests.cs @@ -0,0 +1,38 @@ +using Nethermind.Core.Extensions; +using NUnit.Framework; + +namespace Nethermind.Verkle.Tree.Test; + +public class VerkleDbTests +{ + [Test] + public void ByteArrayEqualityTestsDictionary() + { + byte[] a = { 1, 2 }; + byte[] b = { 1, 2 }; + + Dictionary table = new Dictionary + { + [a] = b, + }; + Assert.IsFalse(table.TryGetValue(b, out byte[] _)); + + table = new Dictionary(Bytes.EqualityComparer) + { + [a] = b, + }; + Assert.IsTrue(table.TryGetValue(b, out byte[] _)); + } + + // [Test] + // public void TestDiffLayer() + // { + // DiffLayer forwardDiff = new DiffLayer(DiffType.Forward); + // DiffLayer reverseDiff = new DiffLayer(DiffType.Reverse); + // + // MemoryStateDb currentState = new MemoryStateDb(); + // MemoryStateDb changes = new MemoryStateDb(); + // + // + // } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleProofTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleProofTests.cs new file mode 100644 index 00000000000..c0e1a9d0fa6 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleProofTests.cs @@ -0,0 +1,248 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree.Proofs; + +namespace Nethermind.Verkle.Tree.Test; + +public class VerkleProofTest +{ + [Test] + public void TestProofVerifyTwoLeaves() + { + byte[][] keys = + { + VerkleTestUtils._emptyArray, + VerkleTestUtils._arrayAll0Last1, + VerkleTestUtils._maxValue, + }; + VerkleTree tree = VerkleTestUtils.CreateVerkleTreeWithKeysAndValues(keys, keys); + VerkleProof proof = tree.CreateVerkleProof(keys, out Banderwagon root); + + bool verified = VerkleTree.VerifyVerkleProof(proof, new List(keys), new List(keys), root, out _); + Assert.That(verified, Is.True); + } + + [Test] + public void TestVerkleProof() + { + List keys = new() + { + VerkleTestUtils._keyVersion, + VerkleTestUtils._keyBalance, + VerkleTestUtils._keyNonce, + VerkleTestUtils._keyCodeCommitment, + VerkleTestUtils._keyCodeSize + }; + + List values = new() + { + VerkleTestUtils._emptyArray, + VerkleTestUtils._emptyArray, + VerkleTestUtils._emptyArray, + VerkleTestUtils._valueEmptyCodeHashValue, + VerkleTestUtils._emptyArray + }; + VerkleTree tree = VerkleTestUtils.CreateVerkleTreeWithKeysAndValues(keys.ToArray(), values.ToArray()); + + VerkleProof proof = tree.CreateVerkleProof(keys.ToArray(), out Banderwagon root); + bool verified = VerkleTree.VerifyVerkleProof(proof, keys, values, root, out _); + Assert.That(verified, Is.True); + } + + [Test] + public void BasicProofTrue() + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + byte[][] keys = + { + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + new byte[]{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + new byte[]{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + new byte[]{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + }; + + tree.Insert(keys[0], keys[0]); + tree.Insert(keys[1], keys[1]); + tree.Insert(keys[2], keys[2]); + tree.Insert(keys[3], keys[3]); + tree.Commit(); + tree.CommitTree(0); + + VerkleProof proof = tree.CreateVerkleProof(keys, out Banderwagon root); + + const string expectedProof = "00000000040000000a0a0a0a0800000056778fe0bcf12a14820d4c054d85cfcae4bdb7017107b6769cecd42629a3825e38f30e21c" + + "79747190371df99e88b886638be445d44f8f9b56ca7c062ea3299446c650ce85c8b5d3cb5ccef8be82858aa2fa9c2cad512086db5" + + "21bd2823e3fc38107802129c490edadbab32ec891ee6310e4f4f00e0056ce3bb0ffc6840a27577556a6fa8933ab25ddb0fb102fca" + + "932ac58a5f539f83b9bfb6e21ea742aa5ad7403f36f9c0821d7014e7a7b917c1d3b72acf724906a30a8fefb09889c3e4cfbc528a4" + + "0fd3331b653dea7be3fe55bc1a0451e0dbad672ec0236cac46e5b23d13d5562743b585beaf3dc1987c9bdf5701af9c4784a392549" + + "9bd6318b63ccec4b35f1bd427779680f60c2df48a9df458fa989fc0f8a7a38d54363b722138e4e4ac4351c5a0aa5cc5e53b697d8b" + + "57eaa43db3dc3987f9f1e71c31b5098721dad2910465fff50d7fb4d0145c41c53a395f99052a251fcb97ef31da582938a67756697" + + "024068f61bd61a10a2c7d8d2a522fa3834e1516f16bfc4ec7f1808069effeab095a5ff89d9bacad138316aa7c9001ce03830e443d" + + "a2aed1f66b5211ae7912bbe751bb05960d4f6bcdb3d266685d6e1b81c632e66f90df80b76cfe8e619bb29ed3322c2f9743d918f47" + + "062f4d077d5a658ab41c3d9c3add6def200e7f242d5ed840a7389ec6a7ab71f6ce813fb898a530af1a3c800f849bf56aae0c7a12a" + + "f1c0ee210863a29533a0c848de893cd1bc0256d8b3ddd3439ee55bc94eb77f71ac2d994b4fd1f08738f53183ac85b3c6e4ee1f8e9" + + "7e0154df668ec700131d4167b93d6180ed760ded7c1899f6f53116ea6c9b54ab809809ae05e821c2e4b0b3cccbf6d643f5aff2dd6" + + "ea235f2e53efccd6009f560e1c0eb01163e1415b2176a2679f8a3845884f3ffac354449be949b849325ec0d66af841825dbf6bd66" + + "8bb91a49c150be9b911a60e285c2ffa50f0380bcb86ed85bf7114c2c0d0aa8e7e6fb33351464a9de74b4219ebf351933831d1f5b5" + + "3467f856adfa7b478c428027dd408f61ff4eb9d94d0ee8c3e79e0265b0635af17db6aa7ca1b463b70e4c51fffb7f8403c94c9315a" + + "7b48d8a11ffd23510e0936842ae8368dedfb511a01dfc930c96d8ee26235b4acc8ace6a0d8fc3fb9142b69b2b989f97ce36ba4386" + + "8d93add3abe7a012"; + + Assert.That(proof.Encode().ToHexString().SequenceEqual(expectedProof), Is.True); + bool verified = VerkleTree.VerifyVerkleProof(proof, new List(keys), new List(keys), root, out _); + Assert.That(verified, Is.True); + } + + [Test] + public void BasicProofTrueSameStem() + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + byte[][] keys = + { + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3} + }; + + tree.Insert(keys[0], keys[0]); + tree.Insert(keys[1], keys[1]); + tree.Insert(keys[2], keys[2]); + tree.Insert(keys[3], keys[3]); + tree.Commit(); + tree.CommitTree(0); + + VerkleProof proof = tree.CreateVerkleProof(keys, out Banderwagon root); + + const string expectedProof = "00000000010000000a020000000b2cd97f2703f0e0030f8356c66ef9cda8587109aab48ebdf02fd49ceefa716d1731296" + + "d27f24eddf8e4576bcf69395373a282be54e0d16966c5ac77f3423b9f05e4ab9d3080bed0f9d2c6eaedb998ae66bd1cda" + + "c02572f57f0f2f4ea61621e12ef7f19003b84b8e4b35ade794a687661cc6f9f96c8b6915f5b82c47525bc8e327e411d55" + + "3fcecc9a4ff9979a60e66283cc7f5e3677ec7b7c18d5fecfaa11fe2446de5c2a55d5f577bc030ed5e4b70578d87263f2a" + + "05956f70e9e43dd6085a231435b1af4c59950e2dd6cce694dbd44e47c6d0ed560e886d2b137d8962d5fd992ebbbbc4843" + + "aebcbe8f982f8439c91423c41f08e4dba677b3032ccec61bc2fe759df90164cc2b3fa5dd2db8da1baf0991805755a45f1" + + "44b6c163302695e2426739b67afc46e1ccaa67cc903c78009bef1d5983d7e027c443f9d785d63e8e25690b04d0c856699" + + "97e44442bbdeca25f074d79030ddb4b98cefbdfa49e663628a76210c2573ab8b20e77d202b54ac9541d0c2ed7985aca3b" + + "10e5b742edd6b071065cf195545d69cebe14391025b460e79d12d32f2cb61adf99603198931637fffd2d999e19495ea88" + + "95ee3480e2d5b39006b4c19429b100d28c214f016407c599c5dd20ce3f4c5d59ee6817cf8fd65a879519175e5c1d3d931" + + "be44041833bf32fd3d31d28304010e4451565ad226b86a0d24b27b23c3a486cea63d58297a0179dd32134e6e80d85d3d3" + + "022a01ada9aea52c542fd1d20cc47af6a8462c11390b9ff6fae91943fe7c0d86c19bce4ffdcf34bb6a03c94f5de3d5c18" + + "a49048e06d9b8f3d481ada5b82c8b8a9619d538fa039bb99332f6395f387acee1e577bfe092dc02094949c12a8e0d580f" + + "6390d9a41302df30ffaf57e40296c75052bb028fe2b09"; + + Assert.That(proof.Encode().ToHexString().SequenceEqual(expectedProof), Is.True); + bool verified = VerkleTree.VerifyVerkleProof(proof, new List(keys), new List(keys), root, out _); + Assert.That(verified, Is.True); + } + + [Test] + public void ProofOfAbsenceEdgeCase() + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + byte[][] keys = + { + new byte[] + { + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 + }, + }; + + VerkleProof proof = tree.CreateVerkleProof(keys, out Banderwagon root); + + const string expectedProof = "000000000100000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000"; + + proof.Encode().ToHexString().Should().BeEquivalentTo(expectedProof); + List values = new() { null }; + bool verified = VerkleTree.VerifyVerkleProof(proof, new List(keys), values, root, out _); + Assert.That(verified, Is.True); + } + + [TestCase(1, 1)] + public void BenchmarkProofCalculation(int iteration, int warmup) + { + IDbProvider db = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleTree tree = new VerkleTree(db, LimboLogs.Instance); + byte[][] keys = new byte[1000][]; + for (int i = 0; i < 1000; i++) + { + keys[i] = TestItem.GetRandomKeccak().Bytes; + tree.Insert(keys[i], Keccak.Zero.Bytes); + tree.Commit(); + } + tree.CommitTree(0); + + Stopwatch sw = new Stopwatch(); + + for (int i = 0; i < warmup; i++) + { + tree.CreateVerkleProof(keys[..500], out _); + } + + sw.Start(); + + for (int i = 0; i < iteration; i++) + { + tree.CreateVerkleProof(keys[..500], out _); + } + + sw.Stop(); + + Console.WriteLine("Elapsed={0}",sw.Elapsed/iteration); + } + + [Test] + public void RandomProofCalculationAndVerification() + { + Random rand = new(0); + IDbProvider db = VerkleDbFactory.InitDatabase(DbMode.MemDb, null); + VerkleTree tree = new(db, LimboLogs.Instance); + byte[][] values = new byte[1000][]; + for (int i = 0; i < 1000; i++) values[i] = Keccak.EmptyTreeHash.Bytes; + + SortedSet keysSet = new(Bytes.Comparer); + while (keysSet.Count != 1000) + { + keysSet.Add(TestItem.GetRandomKeccak(rand).Bytes); + } + + byte[][] keys = keysSet.ToArray(); + + for (int i = 0; i < 1000; i++) + { + tree.Insert(keys[i], values[i]); + tree.Commit(); + } + tree.CommitTree(0); + + VerkleProof proof = tree.CreateVerkleProof(keys[..500], out Banderwagon root); + bool verified = VerkleTree.VerifyVerkleProof(proof, new(keys[..500]), + new(values[..500]), root, out _); + Assert.That(verified, Is.True); + } + + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleRangeProofTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleRangeProofTests.cs new file mode 100644 index 00000000000..5c2b94528fa --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleRangeProofTests.cs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Db.Rocks; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree.Proofs; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Test; + +public class VerkleRangeProofTests +{ + [Test] + public void TestSyncProofCreationAndVerification() + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + byte[][] stems = + { + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4} + }; + + byte[][] values = + { + VerkleTestUtils._keyVersion, VerkleTestUtils._keyBalance, VerkleTestUtils._keyNonce, + VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._keyCodeSize, + }; + + foreach (byte[] stem in stems) + { + List<(byte, byte[])> batch = new(); + for (byte i = 0; i < 5; i++) batch.Add((i, values[0])); + tree.InsertStemBatch(stem, batch); + } + tree.Commit(); + tree.CommitTree(0); + + VerkleProof proof = tree.CreateVerkleRangeProof(stems[0], stems[^1], out Banderwagon root); + + // bool verified = VerkleTree.VerifyVerkleRangeProof(proof, stems[0], stems[^1], stems, root, out _); + // Assert.That(verified, Is.True); + + VerkleTree newTree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + List subTrees = new List(); + List leafs = new List(); + + for (byte i = 0; i < 5; i++) leafs.Add((i, values[0])); + subTrees.Add(new PathWithSubTree(stems[0], leafs.ToArray())); + subTrees.Add(new PathWithSubTree(stems[1], leafs.ToArray())); + subTrees.Add(new PathWithSubTree(stems[2], leafs.ToArray())); + subTrees.Add(new PathWithSubTree(stems[3], leafs.ToArray())); + + + bool isTrue = + newTree.CreateStatelessTreeFromRange(proof, root, stems[0], stems[^1], subTrees.ToArray()); + Assert.That(isTrue, Is.True); + + VerkleTreeDumper oldTreeDumper = new(); + VerkleTreeDumper newTreeDumper = new(); + + tree.Accept(oldTreeDumper, new VerkleCommitment(root.ToBytes())); + newTree.Accept(newTreeDumper, new VerkleCommitment(root.ToBytes())); + + Console.WriteLine("oldTreeDumper"); + Console.WriteLine(oldTreeDumper.ToString()); + Console.WriteLine("newTreeDumper"); + Console.WriteLine(newTreeDumper.ToString()); + } + + [Test] + public void TestSyncProofCreationAndVerificationLastBytesDiffering() + { + VerkleTree tree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + byte[][] stems = + { + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}, + new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4} + }; + + byte[][] values = + { + VerkleTestUtils._keyVersion, VerkleTestUtils._keyBalance, VerkleTestUtils._keyNonce, + VerkleTestUtils._keyCodeCommitment, VerkleTestUtils._keyCodeSize, + }; + + foreach (byte[] stem in stems) + { + List<(byte, byte[])> batch = new(); + for (byte i = 0; i < 5; i++) batch.Add((i, values[i])); + tree.InsertStemBatch(stem, batch); + } + tree.Commit(); + tree.CommitTree(0); + + VerkleProof proof = tree.CreateVerkleRangeProof(stems[0], stems[^1], out Banderwagon root); + + // bool verified = VerkleTree.VerifyVerkleRangeProof(proof, stems[0], stems[^1], stems, root, out _); + // Assert.That(verified, Is.True); + + VerkleTree newTree = VerkleTestUtils.GetVerkleTreeForTest(DbMode.MemDb); + + List subTrees = new List(); + List leafs = new List(); + + for (byte i = 0; i < 5; i++) leafs.Add((i, values[i])); + subTrees.Add(new PathWithSubTree(stems[0], leafs.ToArray())); + subTrees.Add(new PathWithSubTree(stems[1], leafs.ToArray())); + subTrees.Add(new PathWithSubTree(stems[2], leafs.ToArray())); + subTrees.Add(new PathWithSubTree(stems[3], leafs.ToArray())); + + + bool isTrue = + newTree.CreateStatelessTreeFromRange(proof, root, stems[0], stems[^1], subTrees.ToArray()); + Assert.That(isTrue, Is.True); + + VerkleTreeDumper oldTreeDumper = new(); + VerkleTreeDumper newTreeDumper = new(); + + tree.Accept(oldTreeDumper, new VerkleCommitment(root.ToBytes())); + newTree.Accept(newTreeDumper, new VerkleCommitment(root.ToBytes())); + + Console.WriteLine("oldTreeDumper"); + Console.WriteLine(oldTreeDumper.ToString()); + Console.WriteLine("newTreeDumper"); + Console.WriteLine(newTreeDumper.ToString()); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleTestUtils.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleTestUtils.cs new file mode 100644 index 00000000000..5641753d94a --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleTestUtils.cs @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Logging; +using Nethermind.Verkle.Tree; +using NUnit.Framework; + +namespace Nethermind.Verkle.Tree.Test; + +public static class VerkleTestUtils +{ + public static readonly byte[] _array1To32 = + { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }; + public static readonly byte[] _array1To32Last128 = + { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 128, + }; + public static readonly byte[] _emptyArray = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + public static readonly byte[] _arrayAll1 = + { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + public static readonly byte[] _arrayAll3 = + { + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 + }; + + public static readonly byte[] _splitKeyTest = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + public static readonly byte[] _start40Key = + { + 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + public static readonly byte[] _maxValue = + Convert.FromHexString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + public static readonly byte[] _arrayAll0Last1 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 + }; + + public static readonly byte[] _arrayAll0Last2 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 + }; + + public static readonly byte[] _arrayAll0Last3 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 + }; + + public static readonly byte[] _arrayAll0Last4 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4 + }; + + + public static readonly byte[] _keyVersion = + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, 183, 72, 25, 6, 8, 210, 159, 31, 0, + }; + public static readonly byte[] _keyBalance = + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, 183, 72, 25, 6, 8, 210, 159, 31, 1, + }; + public static readonly byte[] _keyNonce = + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, 183, 72, 25, 6, 8, 210, 159, 31, 2, + }; + public static readonly byte[] _keyCodeCommitment = { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, 183, 72, 25, 6, 8, 210, 159, 31, 3, + }; + public static readonly byte[] _keyCodeSize = + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, 183, 72, 25, 6, 8, 210, 159, 31, 4, + }; + public static readonly byte[] _valueEmptyCodeHashValue = + { + 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112, + }; + + public static readonly byte[] _startWith1 = new byte[] + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + public static readonly byte[] _startWith2 = new byte[] + { + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + public static readonly byte[] _startWith3 = new byte[] + { + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + public static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = "VerkleTrie_TestID_" + TestContext.CurrentContext.Test.ID; + return Path.Combine(tempDir, dbname); + } + + public static VerkleTree GetVerkleTreeForTest(DbMode dbMode) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + public static VerkleTree GetFilledVerkleTreeForTest(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + + tree.Insert(_keyVersion, _emptyArray); + tree.Insert(_keyBalance, _emptyArray); + tree.Insert(_keyNonce, _emptyArray); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _emptyArray); + tree.Commit(); + tree.CommitTree(0); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyBalance).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyNonce).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_emptyArray); + + tree.Insert(_keyVersion, _arrayAll0Last2); + tree.Insert(_keyBalance, _arrayAll0Last2); + tree.Insert(_keyNonce, _arrayAll0Last2); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last2); + tree.Commit(); + tree.CommitTree(1); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last2); + + tree.Insert(_keyVersion, _arrayAll0Last3); + tree.Insert(_keyBalance, _arrayAll0Last3); + tree.Insert(_keyNonce, _arrayAll0Last3); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last3); + tree.Commit(); + tree.CommitTree(2); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last3); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last3); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last3); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last3); + + tree.Insert(_keyVersion, _arrayAll0Last4); + tree.Insert(_keyBalance, _arrayAll0Last4); + tree.Insert(_keyNonce, _arrayAll0Last4); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last4); + tree.Commit(); + tree.CommitTree(3); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last4); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last4); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last4); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last4); + + return tree; + } + + public static VerkleTree CreateVerkleTreeWithKeysAndValues(byte[][] keys, byte[][] values) + { + VerkleTree tree = GetVerkleTreeForTest(DbMode.MemDb); + for (int i = 0; i < keys.Length; i++) + { + tree.Insert(keys[i], values[i]); + } + tree.Commit(); + tree.CommitTree(0); + return tree; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleTreeTests.cs b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleTreeTests.cs new file mode 100644 index 00000000000..e78c83bb45f --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree.Test/VerkleTreeTests.cs @@ -0,0 +1,588 @@ +using System.Diagnostics.CodeAnalysis; +using FluentAssertions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Db.Rocks; +using Nethermind.Logging; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Tree; +using Nethermind.Verkle.Tree.Utils; +using NUnit.Framework; + +namespace Nethermind.Verkle.Tree.Test; + + +[TestFixture, Parallelizable(ParallelScope.All)] +public class VerkleTreeTests +{ + private readonly byte[] _array1To32 = + { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }; + private readonly byte[] _array1To32Last128 = + { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 128, + }; + private readonly byte[] _emptyArray = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + private readonly byte[] _arrayAll1 = + { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + + private readonly byte[] _arrayAll0Last2 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 + }; + + private readonly byte[] _arrayAll0Last3 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 + }; + + private readonly byte[] _arrayAll0Last4 = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4 + }; + + + private readonly Pedersen _keyVersion = new byte[] + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, + 183, 72, 25, 6, 8, 210, 159, 31, 0, + }; + + private readonly Pedersen _keyBalance = new byte[] + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, + 183, 72, 25, 6, 8, 210, 159, 31, 1, + }; + + private readonly Pedersen _keyNonce = new byte[] + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, + 183, 72, 25, 6, 8, 210, 159, 31, 2, + }; + + private readonly Pedersen _keyCodeCommitment = new byte[] + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, + 183, 72, 25, 6, 8, 210, 159, 31, 3, + }; + + private readonly Pedersen _keyCodeSize = new byte[] + { + 121, 85, 7, 198, 131, 230, 143, 90, 165, 129, 173, 81, 186, 89, 19, 191, 13, 107, 197, 120, 243, 229, 224, + 183, 72, 25, 6, 8, 210, 159, 31, 4, + }; + + private readonly byte[] _valueEmptyCodeHashValue = + { + 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, + 123, 250, 216, 4, 93, 133, 164, 112, + }; + + private static string GetDbPathForTest() + { + string tempDir = Path.GetTempPath(); + string dbname = $"VerkleTrie_TestID_{TestContext.CurrentContext.Test.ID}"; + return Path.Combine(tempDir, dbname); + } + + private static VerkleTree GetVerkleTreeForTest(DbMode dbMode) + { + IDbProvider provider; + switch (dbMode) + { + case DbMode.MemDb: + provider = VerkleDbFactory.InitDatabase(dbMode, null); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.PersistantDb: + provider = VerkleDbFactory.InitDatabase(dbMode, GetDbPathForTest()); + return new VerkleTree(provider, LimboLogs.Instance); + case DbMode.ReadOnlyDb: + default: + throw new ArgumentOutOfRangeException(nameof(dbMode), dbMode, null); + } + } + + private VerkleTree GetFilledVerkleTreeForTest(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + + tree.Insert(_keyVersion, _emptyArray); + tree.Insert(_keyBalance, _emptyArray); + tree.Insert(_keyNonce, _emptyArray); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _emptyArray); + tree.Commit(); + tree.CommitTree(0); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyBalance).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyNonce).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_emptyArray); + + tree.Insert(_keyVersion, _arrayAll0Last2); + tree.Insert(_keyBalance, _arrayAll0Last2); + tree.Insert(_keyNonce, _arrayAll0Last2); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last2); + tree.Commit(); + tree.CommitTree(1); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last2); + + tree.Insert(_keyVersion, _arrayAll0Last3); + tree.Insert(_keyBalance, _arrayAll0Last3); + tree.Insert(_keyNonce, _arrayAll0Last3); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last3); + tree.Commit(); + tree.CommitTree(2); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last3); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last3); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last3); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last3); + + tree.Insert(_keyVersion, _arrayAll0Last4); + tree.Insert(_keyBalance, _arrayAll0Last4); + tree.Insert(_keyNonce, _arrayAll0Last4); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last4); + tree.Commit(); + tree.CommitTree(3); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last4); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last4); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last4); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last4); + + return tree; + } + + [TearDown] + public void CleanTestData() + { + string dbPath = GetDbPathForTest(); + if (Directory.Exists(dbPath)) + { + Directory.Delete(dbPath, true); + } + } + + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertKey0Value0(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] key = _emptyArray; + + tree.Insert(key, key); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6B630905CE275E39F223E175242DF2C1E8395E6F46EC71DCE5557012C1334A5C"); + + tree.Get(key).Should().BeEquivalentTo(key); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertKey1Value1(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] key = _array1To32; + + tree.Insert(key, key); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6F5E7CFC3A158A64E5718B0D2F18F564171342380F5808F3D2A82F7E7F3C2778"); + + tree.Get(key).Should().BeEquivalentTo(key); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertSameStemTwoLeaves(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] keyA = _array1To32; + + byte[] keyB = _array1To32Last128; + + tree.Insert(keyA, keyA); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6F5E7CFC3A158A64E5718B0D2F18F564171342380F5808F3D2A82F7E7F3C2778"); + tree.Insert(keyB, keyB); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "14EE5E5C5B698E363055B41DD3334F8168C7FCA4F85C5E30AB39CF9CC2FEEF70"); + + tree.Get(keyA).Should().BeEquivalentTo(keyA); + tree.Get(keyB).Should().BeEquivalentTo(keyB); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertKey1Val1Key2Val2(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] keyA = _emptyArray; + byte[] keyB = _arrayAll1; + + tree.Insert(keyA, keyA); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6B630905CE275E39F223E175242DF2C1E8395E6F46EC71DCE5557012C1334A5C"); + tree.Insert(keyB, keyB); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "5E208CDBA664A7B8FBDC26A1C1185F153A5F721CBA389625C18157CEF7D4931C"); + + tree.Get(keyA).Should().BeEquivalentTo(keyA); + tree.Get(keyB).Should().BeEquivalentTo(keyB); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertLongestPath(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] keyA = _emptyArray; + byte[] keyB = (byte[])_emptyArray.Clone(); + keyB[30] = 1; + + tree.Insert(keyA, keyA); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6B630905CE275E39F223E175242DF2C1E8395E6F46EC71DCE5557012C1334A5C"); + tree.Insert(keyB, keyB); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "3258D722AEA34B5AE7CB24A9B0175EDF0533C651FA09592E823B5969C729FB88"); + + tree.Get(keyA).Should().BeEquivalentTo(keyA); + tree.Get(keyB).Should().BeEquivalentTo(keyB); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertAndTraverseLongestPath(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] keyA = _emptyArray; + tree.Insert(keyA, keyA); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6B630905CE275E39F223E175242DF2C1E8395E6F46EC71DCE5557012C1334A5C"); + + byte[] keyB = (byte[])_emptyArray.Clone(); + keyB[30] = 1; + tree.Insert(keyB, keyB); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "3258D722AEA34B5AE7CB24A9B0175EDF0533C651FA09592E823B5969C729FB88"); + + byte[] keyC = (byte[])_emptyArray.Clone(); + keyC[29] = 1; + tree.Insert(keyC, keyC); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "5B82B26A1A7E00A1E997ABD51FE3075D05F54AA4CB1B3A70607E62064FADAA82"); + + tree.Get(keyA).Should().BeEquivalentTo(keyA); + tree.Get(keyB).Should().BeEquivalentTo(keyB); + tree.Get(keyC).Should().BeEquivalentTo(keyC); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestEmptyTrie(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + tree.Commit(); + tree.StateRoot.Bytes.Should().BeEquivalentTo(FrE.Zero.ToBytes().ToArray()); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestSimpleUpdate(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[] key = _array1To32; + byte[] value = _emptyArray; + tree.Insert(key, value); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "140A25B322EAA1ADACD0EE1BB135ECA7B78FCF02B4B19E4A55B26B7A434F42AC"); + tree.Get(key).Should().BeEquivalentTo(value); + + tree.Insert(key, key); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6F5E7CFC3A158A64E5718B0D2F18F564171342380F5808F3D2A82F7E7F3C2778"); + tree.Get(key).Should().BeEquivalentTo(key); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertGet(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + + tree.Insert(_keyVersion, _emptyArray); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "476C50753A22B270DA9D409C0F9AB655AB2506CE4EF831481DD455F0EA730FEF"); + + tree.Insert(_keyBalance, _arrayAll0Last2); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "6D9E4F2D418DE2822CE9C2F4193C0E155E4CAC6CF4170E44098DC49D4B571B7B"); + + tree.Insert(_keyNonce, _emptyArray); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "5C7AE53FE2AAE9852127140C1E2F5122BB3759A7975C0E7A1AEC7CAF7C711FDE"); + + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "3FD5FA25042DB0304792BFC007514DA5B777516FFBDAA5658AF36D355ABD9BD8"); + + tree.Insert(_keyCodeSize, _emptyArray); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, + "006BD679A8204502DCBF9A002F0B828AECF5A29A3A5CE454E651E3A96CC02FE2"); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyNonce).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_emptyArray); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestValueSameBeforeAndAfterFlush(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + + + tree.Insert(_keyVersion, _emptyArray); + tree.Insert(_keyBalance, _emptyArray); + tree.Insert(_keyNonce, _emptyArray); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _emptyArray); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyBalance).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyNonce).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_emptyArray); + + tree.Commit(); + tree.CommitTree(0); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyBalance).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyNonce).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_emptyArray); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestInsertGetMultiBlock(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + + tree.Insert(_keyVersion, _emptyArray); + tree.Insert(_keyBalance, _emptyArray); + tree.Insert(_keyNonce, _emptyArray); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _emptyArray); + + tree.Commit(); + tree.CommitTree(0); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyBalance).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyNonce).Should().BeEquivalentTo(_emptyArray); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_emptyArray); + + tree.Insert(_keyVersion, _arrayAll0Last2); + tree.Insert(_keyBalance, _arrayAll0Last2); + tree.Insert(_keyNonce, _arrayAll0Last2); + tree.Insert(_keyCodeCommitment, _valueEmptyCodeHashValue); + tree.Insert(_keyCodeSize, _arrayAll0Last2); + + tree.Commit(); + tree.CommitTree(1); + + tree.Get(_keyVersion).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyBalance).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyNonce).Should().BeEquivalentTo(_arrayAll0Last2); + tree.Get(_keyCodeCommitment).Should().BeEquivalentTo(_valueEmptyCodeHashValue); + tree.Get(_keyCodeSize).Should().BeEquivalentTo(_arrayAll0Last2); + } + + [TestCase(DbMode.MemDb)] + [TestCase(DbMode.PersistantDb)] + public void TestBeverlyHillGenesis(DbMode dbMode) + { + VerkleTree tree = GetVerkleTreeForTest(dbMode); + byte[][] keys = + { + new byte[] + { + 80, 91, 197, 250, 186, 158, 22, 244, 39, 111, 133, 220, 198, 184, 196, 37, 196, 170, 83, 13, 248, 137, 214, 145, 207, 141, 22, 250, 127, 178, 242, 98 + }, + new byte[] + { + 80, 91, 197, 250, 186, 158, 22, 244, 39, 111, 133, 220, 198, 184, 196, 37, 196, 170, 83, 13, 248, 137, 214, 145, 207, 141, 22, 250, 127, 178, 242, 0 + }, + new byte[] + { + 80, 91, 197, 250, 186, 158, 22, 244, 39, 111, 133, 220, 198, 184, 196, 37, 196, 170, 83, 13, 248, 137, 214, 145, 207, 141, 22, 250, 127, 178, 242, 1 + }, + new byte[] + { + 80, 91, 197, 250, 186, 158, 22, 244, 39, 111, 133, 220, 198, 184, 196, 37, 196, 170, 83, 13, 248, 137, 214, 145, 207, 141, 22, 250, 127, 178, 242, 2 + }, + new byte[] + { + 80, 91, 197, 250, 186, 158, 22, 244, 39, 111, 133, 220, 198, 184, 196, 37, 196, 170, 83, 13, 248, 137, 214, 145, 207, 141, 22, 250, 127, 178, 242, 3 + }, + new byte[] + { + 80, 28, 126, 51, 3, 54, 20, 30, 142, 44, 127, 93, 139, 146, 112, 200, 182, 35, 165, 99, 140, 74, 215, 203, 100, 29, 142, 136, 89, 75, 19, 0 + }, + new byte[] + { + 80, 28, 126, 51, 3, 54, 20, 30, 142, 44, 127, 93, 139, 146, 112, 200, 182, 35, 165, 99, 140, 74, 215, 203, 100, 29, 142, 136, 89, 75, 19, 1 + }, + new byte[] + { + 80, 28, 126, 51, 3, 54, 20, 30, 142, 44, 127, 93, 139, 146, 112, 200, 182, 35, 165, 99, 140, 74, 215, 203, 100, 29, 142, 136, 89, 75, 19, 2 + }, + new byte[] + { + 80, 28, 126, 51, 3, 54, 20, 30, 142, 44, 127, 93, 139, 146, 112, 200, 182, 35, 165, 99, 140, 74, 215, 203, 100, 29, 142, 136, 89, 75, 19, 3 + } + + }; + byte[][] values = + { + new byte[] + { + 245, 165, 253, 66, 209, 106, 32, 48, 39, 152, 239, 110, 211, 9, 151, 155, 67, 0, 61, 35, 32, 217, 240, 232, 234, 152, 49, 169, 39, 89, 251, 75 + }, + new byte[] + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + new byte[] + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + new byte[] + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + new byte[] + { + 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112 + }, + new byte[] + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + new byte[] + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + new byte[] + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }, + new byte[] + { + 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112 + }, + + }; + byte[][] expectedRootHash = + { + new byte[] + { + 65, 218, 34, 122, 247, 60, 5, 240, 251, 52, 246, 140, 37, 216, 19, 241, 119, 101, 100, 139, 61, 137, 58, 14, 46, 134, 177, 141, 32, 154, 1, 59 + }, + new byte[] + { + 69, 56, 80, 6, 24, 134, 42, 30, 244, 7, 143, 117, 127, 102, 1, 223, 77, 33, 172, 250, 47, 138, 224, 44, 49, 218, 223, 138, 225, 174, 75, 206 + }, + new byte[] + { + 1, 193, 213, 250, 133, 201, 192, 237, 111, 140, 199, 105, 38, 248, 215, 48, 91, 252, 209, 7, 111, 242, 47, 75, 91, 142, 31, 125, 12, 184, 18, 245 + }, + new byte[] + { + 81, 28, 164, 25, 6, 44, 54, 172, 110, 118, 89, 173, 112, 198, 226, 209, 24, 221, 92, 3, 221, 62, 85, 11, 85, 179, 58, 178, 78, 187, 238, 71 + }, + new byte[] + { + 57, 217, 114, 85, 250, 173, 204, 43, 147, 248, 170, 130, 74, 146, 15, 162, 181, 125, 23, 235, 3, 110, 20, 156, 89, 185, 169, 139, 69, 112, 37, 175 + }, + new byte[] + { + 3, 81, 31, 1, 31, 195, 243, 62, 60, 100, 13, 77, 215, 21, 9, 92, 237, 104, 172, 25, 234, 240, 207, 142, 94, 213, 237, 176, 195, 163, 39, 87 + }, + new byte[] + { + 83, 211, 141, 144, 197, 72, 206, 46, 100, 220, 68, 98, 90, 239, 120, 251, 113, 102, 18, 170, 98, 238, 138, 174, 249, 120, 142, 238, 79, 192, 75, 59 + }, + new byte[] + { + 91, 25, 255, 166, 55, 136, 143, 224, 38, 101, 85, 250, 65, 108, 59, 161, 105, 247, 140, 181, 185, 207, 120, 76, 92, 218, 72, 103, 18, 0, 214, 144 + }, + new byte[] + { + 5, 153, 156, 66, 226, 161, 228, 136, 209, 248, 42, 235, 222, 221, 118, 125, 87, 137, 13, 106, 11, 70, 26, 248, 3, 21, 218, 184, 201, 164, 194, 198 + } + + }; + + + for (int i = 0; i < keys.Length; i++) + { + tree.Insert(keys[i], values[i]); + tree.Commit(); + AssertRootHash(tree.StateRoot.Bytes, expectedRootHash[i]); + } + } + + private static void AssertRootHash(byte[] realRootHash, string expectedRootHash) + { + Convert.ToHexString(realRootHash).Should() + .BeEquivalentTo(expectedRootHash); + } + + private static void AssertRootHash(IEnumerable realRootHash, IEnumerable expectedRootHash) + { + realRootHash.Should().BeEquivalentTo(expectedRootHash); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/AccountExtension.cs b/src/Nethermind/Nethermind.Verkle.Tree/AccountExtension.cs new file mode 100644 index 00000000000..c477cc51c14 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/AccountExtension.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Verkle.Tree; + +public static class AccountExtension +{ + public static LeafInSubTree[] ToVerkleDict(this Account account) + { + List subTree = new() + { + new LeafInSubTree(0, account.Version.ToLittleEndian()), + new LeafInSubTree(1, account.Balance.ToLittleEndian()), + new LeafInSubTree(2, account.Nonce.ToLittleEndian()), + new LeafInSubTree(3, account.CodeHash.Bytes) + }; + + if (!account.CodeHash.Bytes.SequenceEqual(Keccak.OfAnEmptyString.Bytes)) + subTree.Add(new LeafInSubTree(4, account.CodeSize.ToLittleEndian())); + return subTree.ToArray(); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleSyncTireStore.cs b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleSyncTireStore.cs new file mode 100644 index 00000000000..f4bda7429d6 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleSyncTireStore.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Verkle.Tree.Interfaces; + +public interface IVerkleSyncTireStore +{ + public IEnumerable> GetLeafRangeIterator(byte[] fromRange, byte[] toRange, long blockNumber); + public IEnumerable GetLeafRangeIterator(Stem fromRange, Stem toRange, VerkleCommitment stateRoot, long bytes); +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTree.cs b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTree.cs new file mode 100644 index 00000000000..a4207665fe8 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTree.cs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Trie; +using Nethermind.Verkle.Tree.Sync; + +namespace Nethermind.Verkle.Tree.Interfaces; + +public interface IVerkleTree +{ + public VerkleCommitment StateRoot { get; set; } + public bool MoveToStateRoot(VerkleCommitment stateRoot); + public byte[]? Get(Pedersen key); + public void Insert(Pedersen key, ReadOnlySpan value); + public void InsertStemBatch(ReadOnlySpan stem, IEnumerable<(byte, byte[])> leafIndexValueMap); + public void InsertStemBatch(ReadOnlySpan stem, IEnumerable leafIndexValueMap); + public void InsertStemBatch(in Stem stem, IEnumerable leafIndexValueMap); + public void Commit(bool forSync = false); + public void CommitTree(long blockNumber); + public void Accept(ITreeVisitor visitor, Keccak rootHash, VisitingOptions? visitingOptions = null); + public void Accept(IVerkleTreeVisitor visitor, VerkleCommitment rootHash, VisitingOptions? visitingOptions = null); +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTreeVisitor.cs b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTreeVisitor.cs new file mode 100644 index 00000000000..4f2e0d948a1 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTreeVisitor.cs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Trie; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree.Interfaces; + +public interface IVerkleTreeVisitor +{ + bool ShouldVisit(byte[] nextNode); + + void VisitTree(VerkleCommitment rootHash, TrieVisitContext trieVisitContext); + + void VisitMissingNode(byte[] nodeKey, TrieVisitContext trieVisitContext); + + void VisitBranchNode(InternalNode node, TrieVisitContext trieVisitContext); + + void VisitStemNode(InternalNode node, TrieVisitContext trieVisitContext); + + void VisitLeafNode(ReadOnlySpan nodeKey, TrieVisitContext trieVisitContext, byte[]? nodeValue); + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTrieStore.cs b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTrieStore.cs new file mode 100644 index 00000000000..c8f1c8708e3 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Interfaces/IVerkleTrieStore.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core.Verkle; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree.Interfaces; + +public interface IVerkleTrieStore: IStoreWithReorgBoundary, IVerkleSyncTireStore +{ + VerkleCommitment StateRoot { get; } + VerkleCommitment GetStateRoot(); + bool MoveToStateRoot(VerkleCommitment stateRoot); + + byte[]? GetLeaf(ReadOnlySpan key); + InternalNode? GetInternalNode(ReadOnlySpan key); + + void Flush(long blockNumber, VerkleMemoryDb memDb); + void Reset(); + + void ReverseState(); + void ApplyDiffLayer(BatchChangeSet changeSet); + bool GetForwardMergedDiff(long fromBlock, long toBlock, [MaybeNullWhen(false)] out VerkleMemoryDb diff); + bool GetReverseMergedDiff(long fromBlock, long toBlock, [MaybeNullWhen(false)] out VerkleMemoryDb diff); + + ReadOnlyVerkleStateStore AsReadOnly(VerkleMemoryDb keyValueStore); +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Metrics.cs b/src/Nethermind/Nethermind.Verkle.Tree/Metrics.cs new file mode 100644 index 00000000000..efb474d1ca7 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Metrics.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.ComponentModel; + +namespace Nethermind.Verkle.Tree; + +public static class Metrics +{ + [Description("Number of trie node hash calculations.")] + public static long TreeNodeHashCalculations { get; set; } + + [Description("Number of trie node RLP encodings.")] + public static long TreeNodeRlpEncodings { get; set; } + + [Description("Number of trie node RLP decodings.")] + public static long TreeNodeRlpDecodings { get; set; } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Nethermind.Verkle.Tree.csproj b/src/Nethermind/Nethermind.Verkle.Tree/Nethermind.Verkle.Tree.csproj new file mode 100644 index 00000000000..3de7b008962 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Nethermind.Verkle.Tree.csproj @@ -0,0 +1,23 @@ + + + + net7.0 + enable + enable + Nethermind.Verkle.Tree + true + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Nodes/InternalNodes.cs b/src/Nethermind/Nethermind.Verkle.Tree/Nodes/InternalNodes.cs new file mode 100644 index 00000000000..e3fef1af2b9 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Nodes/InternalNodes.cs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using System.Text; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Nodes; + +public class InternalNode +{ + public bool IsStem => NodeType == VerkleNodeType.StemNode; + public bool IsBranchNode => NodeType == VerkleNodeType.BranchNode; + public VerkleNodeType NodeType { get; } + public Commitment InternalCommitment { get; } + public byte[] Bytes => InternalCommitment.ToBytes(); + + public bool IsStateless { get; set; } = false; + + public bool ShouldPersist + { + get + { + if (IsBranchNode) return true; + return C1 is not null && C2 is not null && Stem is not null; + } + } + + + /// + /// C1, C2, InitCommitmentHash - only relevant for stem nodes + /// + public Commitment? C1 { get; set; } + public Commitment? C2 { get; set; } + public FrE? InitCommitmentHash { get; } + + private static readonly Banderwagon _initFirstElementCommitment = Committer.ScalarMul(FrE.One, 0); + + public Stem? Stem { get; } + + public InternalNode Clone() + { + return NodeType switch + { + VerkleNodeType.BranchNode => new InternalNode(VerkleNodeType.BranchNode, InternalCommitment.Dup()), + VerkleNodeType.StemNode => new InternalNode(VerkleNodeType.StemNode, Stem!, C1?.Dup(), C2?.Dup(), InternalCommitment.Dup()), + _ => throw new ArgumentOutOfRangeException() + }; + } + + public InternalNode(VerkleNodeType nodeType) + { + NodeType = nodeType; + InternalCommitment = new Commitment(); + } + + public InternalNode(VerkleNodeType nodeType, Commitment commitment) + { + NodeType = nodeType; + InternalCommitment = commitment; + } + + public InternalNode(VerkleNodeType nodeType, Stem stem) + { + NodeType = nodeType; + Stem = stem; + C1 = new Commitment(); + C2 = new Commitment(); + InternalCommitment = new Commitment(); + Banderwagon stemCommitment = GetInitialCommitment(); + InternalCommitment.AddPoint(stemCommitment); + InitCommitmentHash = InternalCommitment.PointAsField; + } + + public InternalNode(VerkleNodeType nodeType, Stem stem, Commitment? c1, Commitment? c2, Commitment internalCommitment) + { + NodeType = nodeType; + Stem = stem; + C1 = c1; + C2 = c2; + InternalCommitment = internalCommitment; + } + + public InternalNode(VerkleNodeType nodeType, Stem stem, byte[] c1, byte[] c2, byte[] extCommit, bool subGroupCheck=false) + { + NodeType = nodeType; + Stem = stem; + C1 = new Commitment(Banderwagon.FromBytes(c1, subGroupCheck)!.Value); + C2 = new Commitment(Banderwagon.FromBytes(c2, subGroupCheck)!.Value); + InternalCommitment = new Commitment(Banderwagon.FromBytes(extCommit, subGroupCheck)!.Value); + } + + private Banderwagon GetInitialCommitment() + { + return _initFirstElementCommitment + + Committer.ScalarMul(FrE.FromBytesReduced(Stem!.Bytes.Reverse().ToArray()), 1); + } + + public FrE UpdateCommitment(Banderwagon point) + { + Debug.Assert(NodeType == VerkleNodeType.BranchNode); + FrE prevCommit = InternalCommitment.PointAsField; + InternalCommitment.AddPoint(point); + return InternalCommitment.PointAsField - prevCommit; + } + + public FrE UpdateCommitment(LeafUpdateDelta deltaLeafCommitment) + { + Debug.Assert(NodeType == VerkleNodeType.StemNode); + FrE deltaC1Commit = FrE.Zero; + FrE deltaC2Commit = FrE.Zero; + + if (deltaLeafCommitment.DeltaC1 is not null) + { + FrE oldC1Value = C1!.PointAsField; + C1.AddPoint(deltaLeafCommitment.DeltaC1.Value); + deltaC1Commit = C1.PointAsField - oldC1Value; + } + if (deltaLeafCommitment.DeltaC2 is not null) + { + FrE oldC2Value = C2!.PointAsField; + C2.AddPoint(deltaLeafCommitment.DeltaC2.Value); + deltaC2Commit = C2.PointAsField - oldC2Value; + } + + Banderwagon deltaCommit = Committer.ScalarMul(deltaC1Commit, 2) + + Committer.ScalarMul(deltaC2Commit, 3); + + return InternalCommitment.UpdateCommitmentGetDelta(deltaCommit); + } + + public override string ToString() + { + StringBuilder builder = new StringBuilder(); + builder.AppendLine($"InternalNode: {InternalCommitment.Point.ToBytes().ToHexString()}"); + builder.AppendLine($"NodeType: {NodeType}"); + if (NodeType == VerkleNodeType.StemNode) + { + builder.AppendLine($"Stem: {Stem.ToString()}"); + builder.AppendLine($"C1: {C1?.Point.ToBytes().ToHexString()}"); + builder.AppendLine($"C2: {C2?.Point.ToBytes().ToHexString()}"); + } + + return builder.ToString(); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Nodes/Nodes.cs b/src/Nethermind/Nethermind.Verkle.Tree/Nodes/Nodes.cs new file mode 100644 index 00000000000..d649a18660c --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Nodes/Nodes.cs @@ -0,0 +1,8 @@ + +namespace Nethermind.Verkle.Tree.Nodes; + +public enum VerkleNodeType : byte +{ + BranchNode = 1, + StemNode = 2 +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Nodes/VerkleNodes.cs b/src/Nethermind/Nethermind.Verkle.Tree/Nodes/VerkleNodes.cs new file mode 100644 index 00000000000..4cfab4432d7 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Nodes/VerkleNodes.cs @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Nodes; + +public static class VerkleNodes +{ + + public static InternalNode CreateStatelessBranchNode(Commitment commitment) + { + return new InternalNode(VerkleNodeType.BranchNode, commitment) { IsStateless = true }; + } + public static InternalNode CreateBranchNode(Commitment commitment) + { + return new InternalNode(VerkleNodeType.BranchNode, commitment); + } + + public static InternalNode CreateStatelessStemNode(Stem stem, Commitment internalCommitment) + { + return new(VerkleNodeType.StemNode, stem, null, null, internalCommitment) { IsStateless = true }; + } + + public static InternalNode CreateStatelessStemNode(Stem stem, Commitment? c1, Commitment? c2, Commitment internalCommitment) + { + return new(VerkleNodeType.StemNode, stem, c1, c2, internalCommitment) { IsStateless = true }; + } + + public static InternalNode CreateStatelessStemNode(Stem stem) + { + return new(VerkleNodeType.StemNode, stem) { IsStateless = true }; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Proofs/Comparer.cs b/src/Nethermind/Nethermind.Verkle.Tree/Proofs/Comparer.cs new file mode 100644 index 00000000000..ef4256f8f50 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Proofs/Comparer.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Runtime.InteropServices; +using Nethermind.Core.Extensions; + +namespace Nethermind.Verkle.Tree.Proofs; + +public class ListComparer : Comparer> +{ + public override int Compare(List? x, List? y) + { + if (x is null) + { + return y is null ? 0 : -1; + } + + return y is null ? 1 : Bytes.BytesComparer.CompareDiffLength(x.ToArray(), y.ToArray()); + } +} + +public class ListEqualityComparer : EqualityComparer> +{ + public override bool Equals(List? x, List? y) + { + return Bytes.SpanEqualityComparer.Equals(CollectionsMarshal.AsSpan(x), CollectionsMarshal.AsSpan(y)); + } + + public override int GetHashCode(List obj) + { + return Bytes.GetSimplifiedHashCode(CollectionsMarshal.AsSpan(obj)); + } +} + +public class ListWithByteComparer : Comparer<(List, byte)> +{ + public override int Compare((List, byte) x, (List, byte) y) + { + ListComparer comp = new ListComparer(); + int val = comp.Compare(x.Item1, y.Item1); + return val == 0 ? x.Item2.CompareTo(y.Item2) : val; + } +} + +public class ListWithByteEqualityComparer : EqualityComparer<(List, byte)> +{ + public override bool Equals((List, byte) x, (List, byte) y) + { + bool isBytesSame = Bytes.SpanEqualityComparer.Equals(CollectionsMarshal.AsSpan(x.Item1), CollectionsMarshal.AsSpan(y.Item1)); + if (isBytesSame) + { + return x.Item2 == y.Item2; + } + return isBytesSame; + } + + public override int GetHashCode((List, byte) obj) + { + return HashCode.Combine(Bytes.GetSimplifiedHashCode(CollectionsMarshal.AsSpan(obj.Item1)), obj.Item2.GetHashCode()); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Proofs/Metrics.cs b/src/Nethermind/Nethermind.Verkle.Tree/Proofs/Metrics.cs new file mode 100644 index 00000000000..445a2dc34a2 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Proofs/Metrics.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Verkle.Tree.Proofs; + +public static class Metrics +{ + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/ReadOnlyVerkleStateStore.cs b/src/Nethermind/Nethermind.Verkle.Tree/ReadOnlyVerkleStateStore.cs new file mode 100644 index 00000000000..99ed81ab739 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/ReadOnlyVerkleStateStore.cs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree; + +public class ReadOnlyVerkleStateStore : IVerkleTrieStore, ISyncTrieStore +{ + private VerkleStateStore _verkleStateStore; + private VerkleMemoryDb _keyValueStore; + + public ReadOnlyVerkleStateStore(VerkleStateStore verkleStateStore, VerkleMemoryDb keyValueStore) + { + _verkleStateStore = verkleStateStore; + _keyValueStore = keyValueStore; + } + + public VerkleCommitment StateRoot => _verkleStateStore.StateRoot; + + public byte[]? GetLeaf(ReadOnlySpan key) + { + if (_keyValueStore.GetLeaf(key, out byte[]? value)) return value; + return _verkleStateStore.GetLeaf(key); + } + public InternalNode? GetInternalNode(ReadOnlySpan key) + { + if (_keyValueStore.GetInternalNode(key, out var value)) return value; + return _verkleStateStore.GetInternalNode(key); + } + public void SetLeaf(ReadOnlySpan leafKey, byte[] leafValue) + { + _keyValueStore.SetLeaf(leafKey, leafValue); + } + public void SetInternalNode(ReadOnlySpan InternalNodeKey, InternalNode internalNodeValue) + { + _keyValueStore.SetInternalNode(InternalNodeKey, internalNodeValue); + } + public void Flush(long blockNumber, VerkleMemoryDb batch) { } + + public void ReverseState() { } + public void ApplyDiffLayer(BatchChangeSet changeSet) { } + + public bool GetForwardMergedDiff(long fromBlock, long toBlock, out VerkleMemoryDb diff) + { + return _verkleStateStore.GetForwardMergedDiff(fromBlock, toBlock, out diff); + } + public bool GetReverseMergedDiff(long fromBlock, long toBlock, out VerkleMemoryDb diff) + { + return _verkleStateStore.GetReverseMergedDiff(fromBlock, toBlock, out diff); + } + + public VerkleCommitment GetStateRoot() + { + return _verkleStateStore.GetStateRoot(); + } + public bool MoveToStateRoot(VerkleCommitment stateRoot) + { + return _verkleStateStore.MoveToStateRoot(stateRoot); + } + + public void Reset() => _verkleStateStore.Reset(); + + public event EventHandler? ReorgBoundaryReached; + + public ReadOnlyVerkleStateStore AsReadOnly(VerkleMemoryDb keyValueStore) + { + return new ReadOnlyVerkleStateStore(_verkleStateStore, keyValueStore); + } + public bool IsFullySynced(Keccak stateRoot) + { + return _verkleStateStore.IsFullySynced(stateRoot); + } + + public IEnumerable> GetLeafRangeIterator(byte[] fromRange, byte[] toRange, long blockNumber) + { + return _verkleStateStore.GetLeafRangeIterator(fromRange, toRange, blockNumber); + } + + public IEnumerable GetLeafRangeIterator(Stem fromRange, Stem toRange, VerkleCommitment stateRoot, long bytes) + { + return _verkleStateStore.GetLeafRangeIterator(fromRange, toRange, stateRoot, bytes); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Serializers/InternalNodeSerializer.cs b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/InternalNodeSerializer.cs new file mode 100644 index 00000000000..f574d1f4dc4 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/InternalNodeSerializer.cs @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree.Serializers; + +public class InternalNodeSerializer : IRlpStreamDecoder, IRlpObjectDecoder +{ + public static InternalNodeSerializer Instance => new InternalNodeSerializer(); + public int GetLength(InternalNode item, RlpBehaviors rlpBehaviors) + { + return item.NodeType switch + { + VerkleNodeType.BranchNode => 1 + 33, // NodeType + InternalCommitment + VerkleNodeType.StemNode => 1 + 32 + 33 + + (item.C1 == null ? 1 : 33) + + (item.C2 == null ? 1 : 33), // NodeType + Stem + InternalCommitment + C1? + C2? + _ => throw new ArgumentOutOfRangeException() + }; + } + + public InternalNode Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + VerkleNodeType nodeType = (VerkleNodeType)rlpStream.ReadByte(); + switch (nodeType) + { + case VerkleNodeType.BranchNode: + InternalNode node = new(VerkleNodeType.BranchNode); + node.UpdateCommitment(Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false)!.Value); + return node; + case VerkleNodeType.StemNode: + byte[] stem = rlpStream.DecodeByteArray(); + + byte[] c1Ser = rlpStream.DecodeByteArray(); + Commitment? c1 = c1Ser.Length == 0 + ? null + : new (Banderwagon.FromBytes(c1Ser, subgroupCheck: false)!.Value); + + byte[] c2Ser = rlpStream.DecodeByteArray(); + Commitment? c2 = c2Ser.Length == 0 + ? null + : new (Banderwagon.FromBytes(c2Ser, subgroupCheck: false)!.Value); + + Commitment extCommit = + new (Banderwagon.FromBytes(rlpStream.DecodeByteArray(), subgroupCheck: false)!.Value); + return new InternalNode(VerkleNodeType.StemNode, stem, c1, c2, extCommit); + default: + throw new ArgumentOutOfRangeException(); + } + } + public void Encode(RlpStream stream, InternalNode item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + switch (item.NodeType) + { + case VerkleNodeType.BranchNode: + stream.WriteByte((byte)VerkleNodeType.BranchNode); + stream.Encode(item.InternalCommitment.Point.ToBytes()); + break; + case VerkleNodeType.StemNode: + stream.WriteByte((byte)VerkleNodeType.StemNode); + stream.Encode(item.Stem!.Bytes); + if (item.C1 is not null) stream.Encode(item.C1.Point.ToBytes()); + else stream.EncodeEmptyByteArray(); + if (item.C2 is not null) stream.Encode(item.C2.Point.ToBytes()); + else stream.EncodeEmptyByteArray(); + stream.Encode(item.InternalCommitment.Point.ToBytes()); + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + public Rlp Encode(InternalNode item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + int length = GetLength(item, rlpBehaviors); + RlpStream stream = new RlpStream(Rlp.LengthOfSequence(length)); + stream.StartSequence(length); + Encode(stream, item, rlpBehaviors); + return new Rlp(stream.Data); + } + public InternalNode Decode(byte[] data, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + RlpStream stream = data.AsRlpStream(); + stream.ReadSequenceLength(); + return Decode(stream, rlpBehaviors); + } + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Serializers/InternalStoreSerializer.cs b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/InternalStoreSerializer.cs new file mode 100644 index 00000000000..e6c9302576a --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/InternalStoreSerializer.cs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree.Serializers; + + +public class InternalStoreSerializer : IRlpStreamDecoder +{ + private static InternalNodeSerializer InternalNodeSerializer => InternalNodeSerializer.Instance; + + public static InternalStoreSerializer Instance => new(); + public int GetLength(InternalStore item, RlpBehaviors rlpBehaviors) + { + int length = Rlp.LengthOf(item.Count); + foreach (KeyValuePair pair in item) + { + length += Rlp.LengthOf(pair.Key); + length += pair.Value == null ? Rlp.EmptyArrayByte : InternalNodeSerializer.GetLength(pair.Value, RlpBehaviors.None); + } + return length; + } + + public InternalStore Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + InternalStore item = new(Bytes.SpanEqualityComparer); + int length = rlpStream.DecodeInt(); + for (int i = 0; i < length; i++) + { + byte[] key = rlpStream.DecodeByteArray(); + if (rlpStream.PeekNextItem().Length == 0) + { + item[key] = null; + rlpStream.SkipItem(); + } + else + { + item[key] = InternalNodeSerializer.Decode(rlpStream); + } + } + return item; + } + public void Encode(RlpStream stream, InternalStore item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + stream.Encode(item.Count); + foreach (KeyValuePair pair in item) + { + stream.Encode(pair.Key); + if (pair.Value is null) stream.EncodeEmptyByteArray(); + else InternalNodeSerializer.Encode(stream, pair.Value); + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Serializers/LeafStoreSerializer.cs b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/LeafStoreSerializer.cs new file mode 100644 index 00000000000..79a5660c45d --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/LeafStoreSerializer.cs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Verkle.Tree.Serializers; + + +public class LeafStoreSerializer +{ + public static LeafStoreSerializer Instance => new(); + public int GetLength(LeafStoreInterface item, RlpBehaviors rlpBehaviors) + { + int length = Rlp.LengthOf(item.Count); + foreach (KeyValuePair pair in item) + { + length += Rlp.LengthOf(pair.Key); + length += Rlp.LengthOf(pair.Value); + } + return length; + } + + public LeafStore Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + LeafStore item = new(Bytes.SpanEqualityComparer); + int length = rlpStream.DecodeInt(); + for (int i = 0; i < length; i++) + { + item[rlpStream.DecodeByteArray()] = rlpStream.DecodeByteArray(); + } + return item; + } + + public void Encode(RlpStream stream, LeafStoreInterface item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + stream.Encode(item.Count); + foreach (KeyValuePair pair in item) + { + stream.Encode(pair.Key.AsSpan()); + stream.Encode(pair.Value.AsSpan()); + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Serializers/VerkleMemoryDbSerializer.cs b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/VerkleMemoryDbSerializer.cs new file mode 100644 index 00000000000..1b9657c8a3f --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Serializers/VerkleMemoryDbSerializer.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree.Serializers; + +public class VerkleMemoryDbSerializer +{ + public static VerkleMemoryDbSerializer Instance => new(); + + public int GetLength(VerkleMemoryDb item, RlpBehaviors rlpBehaviors) + { + int length = 0; + length += LeafStoreSerializer.Instance.GetLength(item.LeafTable, RlpBehaviors.None); + length += InternalStoreSerializer.Instance.GetLength(item.InternalTable, RlpBehaviors.None); + return length; + } + + public int GetLength(ReadOnlyVerkleMemoryDb item, RlpBehaviors rlpBehaviors) + { + int length = 0; + length += LeafStoreSerializer.Instance.GetLength(item.LeafTable, RlpBehaviors.None); + length += InternalStoreSerializer.Instance.GetLength(item.InternalTable, RlpBehaviors.None); + return length; + } + + public VerkleMemoryDb Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + return new VerkleMemoryDb( + LeafStoreSerializer.Instance.Decode(rlpStream), + InternalStoreSerializer.Instance.Decode(rlpStream) + ); + } + public void Encode(RlpStream stream, VerkleMemoryDb item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + LeafStoreSerializer.Instance.Encode(stream, item.LeafTable); + InternalStoreSerializer.Instance.Encode(stream, item.InternalTable); + } + + public void Encode(RlpStream stream, ReadOnlyVerkleMemoryDb item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + LeafStoreSerializer.Instance.Encode(stream, item.LeafTable); + InternalStoreSerializer.Instance.Encode(stream, item.InternalTable); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/StackQueue.cs b/src/Nethermind/Nethermind.Verkle.Tree/StackQueue.cs new file mode 100644 index 00000000000..b88d57841d5 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/StackQueue.cs @@ -0,0 +1,661 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace Nethermind.Verkle.Tree; + +/// +/// A simple of generic objects. +/// This structure implements the functionality of +/// both queue and stack in a single data structure. +/// 1. Queue: Enqueue(), Dequeue(), FIFO Enumerator +/// 2. Stack: Pop(), LIFO Enumerator +/// Internally it is implemented as a circular buffer +/// and the size of buffer is fixed during initialization +/// so Enqueue and Dequeue is O(1) +/// +/// +[DebuggerDisplay("Count = {Count}")] +[Serializable] +public class StackQueue +{ + public T[] _array; + private int _head; // The index from which to dequeue if the queue isn't empty. + private int _tail; // The index at which to enqueue if the queue isn't full. + private int _version; + + // Creates a queue with room for capacity objects. Capacity cannot be changed after initialization + public StackQueue(int capacity) + { + if (capacity < 0) + throw new ArgumentOutOfRangeException(nameof(capacity), capacity, "NeedNonNegNum"); + _array = new T[capacity]; + } + + public int Count { get; private set; } + + public void Clear() + { + if (Count != 0) + { + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + if (_head < _tail) + { + Array.Clear(_array, _head, Count); + } + else + { + Array.Clear(_array, _head, _array.Length - _head); + Array.Clear(_array, 0, _tail); + } + } + + Count = 0; + } + + _head = 0; + _tail = 0; + _version++; + } + + // Adds item to the tail of the queue. Throws if the queue if full + public void Enqueue(T item) + { + if (Count == _array.Length) + { + throw new ArgumentException($"QueueIsFull: {Count}"); + } + + _array[_tail] = item; + MoveNext(ref _tail); + Count++; + _version++; + } + + // Adds item to the tail of the queue. If the queue if full, it will perform a dequeue + // and then insert the new element. It also returns the dequeued element + public bool EnqueueAndReplaceIfFull(T item, [MaybeNullWhen(true)]out T element) + { + if (Count == _array.Length) + { + element = Dequeue(); + Enqueue(item); + return false; + } + + _array[_tail] = item; + MoveNext(ref _tail); + Count++; + element = default; + _version++; + return true; + } + + // GetQueueEnumerator returns an IEnumerator over this StackQueue. This + // enumerator returns elements in FIFO order. + // This Enumerator will support removing. + public QueueEnumerator GetQueueEnumerator() + { + return new QueueEnumerator(this); + } + + // GetQueueEnumerator returns an IEnumerator over this StackQueue. This + // enumerator returns elements int LIFO order. + // This Enumerator will support removing. + public StackEnumerator GetStackEnumerator() + { + return new StackEnumerator(this); + } + + // Removes the object at the head of the queue and returns it. If the queue + // is empty, this method throws an + // InvalidOperationException. + public T Dequeue() + { + int head = _head; + T[] array = _array; + + if (Count == 0) + { + ThrowForEmptyQueue(); + } + + T removed = array[head]; + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + array[head] = default!; + } + MoveNext(ref _head); + Count--; + _version++; + return removed; + } + + public bool TryDequeue([MaybeNullWhen(false)] out T result) + { + int head = _head; + T[] array = _array; + + if (Count == 0) + { + result = default; + return false; + } + + result = array[head]; + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + array[head] = default!; + } + MoveNext(ref _head); + Count--; + _version++; + return true; + } + + // Returns the object at the head of the queue. The object remains in the + // queue. If the queue is empty, this method throws an + // InvalidOperationException. + public T Peek() + { + if (Count == 0) + { + ThrowForEmptyQueue(); + } + + return _array[_head]; + } + + public bool TryPeek([MaybeNullWhen(false)] out T result) + { + if (Count == 0) + { + result = default; + return false; + } + + result = _array[_head]; + return true; + } + + // Remove the last inserted element from the queue + public void Pop(out T element) + { + if (Count == 0) + { + ThrowForEmptyQueue(); + } + + MoveBack(ref _tail); + Count--; + element = _array[_tail]; + } + + public bool TryPop([MaybeNullWhen(false)]out T element) + { + if (Count == 0) + { + element = default; + return false; + } + + MoveBack(ref _tail); + Count--; + element = _array[_tail]; + return true; + } + + // Returns the object at the head of the stack. The object remains in the + // stack. If the stack is empty, this method throws an + // InvalidOperationException. + public T PeekStack() + { + if (Count == 0) + { + ThrowForEmptyQueue(); + } + + int tmp = _tail - 1; + if (tmp == -1) + { + tmp = _array.Length - 1; + } + + return _array[tmp]; + } + + public bool TryPeekStack([MaybeNullWhen(false)] out T result) + { + if (Count == 0) + { + result = default; + return false; + } + + int tmp = _tail - 1; + if (tmp == -1) + { + tmp = _array.Length - 1; + } + result = _array[tmp]; + return true; + } + + // Returns true if the queue contains at least one object equal to item. + // Equality is determined using EqualityComparer.Default.Equals(). + public bool Contains(T item) + { + if (Count == 0) + { + return false; + } + + if (_head < _tail) + { + return Array.IndexOf(_array, item, _head, Count) >= 0; + } + + // We've wrapped around. Check both partitions, the least recently enqueued first. + return + Array.IndexOf(_array, item, _head, _array.Length - _head) >= 0 || + Array.IndexOf(_array, item, 0, _tail) >= 0; + } + + // Iterates over the objects in the queue, returning an array of the + // objects in the StackQueue, or an empty array if the queue is empty. + // The order of elements in the array is first in to last in, the same + // order produced by successive calls to Dequeue. + public T[] ToArray() + { + if (Count == 0) + { + return Array.Empty(); + } + + T[] arr = new T[Count]; + + if (_head < _tail) + { + Array.Copy(_array, _head, arr, 0, Count); + } + else + { + Array.Copy(_array, _head, arr, 0, _array.Length - _head); + Array.Copy(_array, 0, arr, _array.Length - _head, _tail); + } + + return arr; + } + + // Increments the index wrapping it if necessary. + private void MoveNext(ref int index) + { + // It is tempting to use the remainder operator here but it is actually much slower + // than a simple comparison and a rarely taken branch. + // JIT produces better code than with ternary operator ?: + int tmp = index + 1; + if (tmp == _array.Length) + { + tmp = 0; + } + index = tmp; + } + + private void MoveBack(ref int index) + { + // It is tempting to use the remainder operator here but it is actually much slower + // than a simple comparison and a rarely taken branch. + // JIT produces better code than with ternary operator ?: + int tmp = index - 1; + if (tmp == -1) + { + tmp = _array.Length - 1; + } + index = tmp; + } + + public void CopyTo(T[] array, int arrayIndex) + { + ArgumentNullException.ThrowIfNull(array); + + if (arrayIndex < 0 || arrayIndex > array.Length) + { + throw new ArgumentOutOfRangeException(nameof(arrayIndex), arrayIndex, "IndexMustBeLessOrEqual"); + } + + if (array.Length - arrayIndex < Count) + { + throw new ArgumentException("InvalidOffLen"); + } + + int numToCopy = Count; + if (numToCopy == 0) return; + + int firstPart = Math.Min(_array.Length - _head, numToCopy); + Array.Copy(_array, _head, array, arrayIndex, firstPart); + numToCopy -= firstPart; + if (numToCopy > 0) + { + Array.Copy(_array, 0, array, arrayIndex + _array.Length - _head, numToCopy); + } + } + + void CopyTo(Array array, int index) + { + ArgumentNullException.ThrowIfNull(array); + + if (array.Rank != 1) + { + throw new ArgumentException("RankMultiDimNotSupported", nameof(array)); + } + + if (array.GetLowerBound(0) != 0) + { + throw new ArgumentException("NonZeroLowerBound", nameof(array)); + } + + int arrayLen = array.Length; + if (index < 0 || index > arrayLen) + { + throw new ArgumentOutOfRangeException(nameof(index), index, "IndexMustBeLessOrEqual"); + } + + if (arrayLen - index < Count) + { + throw new ArgumentException("InvalidOffLen"); + } + + int numToCopy = Count; + if (numToCopy == 0) return; + + try + { + int firstPart = (_array.Length - _head < numToCopy) ? _array.Length - _head : numToCopy; + Array.Copy(_array, _head, array, index, firstPart); + numToCopy -= firstPart; + + if (numToCopy > 0) + { + Array.Copy(_array, 0, array, index + _array.Length - _head, numToCopy); + } + } + catch (ArrayTypeMismatchException) + { + throw new ArgumentException("InvalidArrayType", nameof(array)); + } + } + + private void ThrowForEmptyQueue() + { + Debug.Assert(Count == 0); + throw new InvalidOperationException("EmptyQueue"); + } + + // Implements an enumerator for a StackQueue. The enumerator uses the + // internal version number of the list to ensure that no modifications are + // made to the list while an enumeration is in progress. + public struct QueueEnumerator : IEnumerator + { + private const int Started = -1; + private const int Ended = -2; + private readonly int Completed { get; init; } + + private readonly StackQueue _q; + private readonly int _version; + private int _index; // -1 = not started, -2 = ended/disposed + private T? _currentElement; + + internal QueueEnumerator(StackQueue q) + { + _q = q; + _version = q._version; + _index = Started; + _currentElement = default; + Completed = _q.Count; + } + + public void Dispose() + { + _index = Ended; + _currentElement = default; + } + + public bool MoveNext() + { + if (_version != _q._version) throw new InvalidOperationException("EnumFailedVersion"); + + if (_index == Ended) + return false; + + _index++; + + if (_index == Completed) + { + // We've run past the last element + _index = Ended; + _currentElement = default; + return false; + } + + // Cache some fields in locals to decrease code size + T[] array = _q._array; + int capacity = array.Length; + + // _index represents the 0-based index into the queue, however the queue + // doesn't have to start from 0 and it may not even be stored contiguously in memory. + + int arrayIndex = _q._head + _index; // this is the actual index into the queue's backing array + if (arrayIndex >= capacity) + { + // NOTE: Originally we were using the modulo operator here, however + // on Intel processors it has a very high instruction latency which + // was slowing down the loop quite a bit. + // Replacing it with simple comparison/subtraction operations sped up + // the average foreach loop by 2x. + + arrayIndex -= capacity; // wrap around if needed + } + + _currentElement = array[arrayIndex]; + return true; + } + + public T Current + { + get + { + if (_index < 0) + ThrowEnumerationNotStartedOrEnded(); + return _currentElement!; + } + } + + private void ThrowEnumerationNotStartedOrEnded() + { + Debug.Assert(_index == Started || _index == Ended); + throw new InvalidOperationException(_index == Started ? "EnumNotStarted" : "EnumEnded"); + } + + object? IEnumerator.Current + { + get { return Current; } + } + + void IEnumerator.Reset() + { + if (_version != _q._version) throw new InvalidOperationException("EnumFailedVersion"); + _index = Started; + _currentElement = default; + } + } + + // Implements an enumerator for a StackQueue. The enumerator uses the + // internal version number of the list to ensure that no modifications are + // made to the list while an enumeration is in progress. + public struct StackEnumerator : IEnumerator + { + private readonly int Started { get; init; } + private readonly int Ended { get; init; } + private const int Completed = -1; + + private readonly StackQueue _q; + private readonly int _version; + private int _index; // -1 = not started, -2 = ended/disposed + private T? _currentElement; + + internal StackEnumerator(StackQueue q) + { + _q = q; + _version = _q._version; + _index = Started = _q.Count; + _currentElement = default; + Ended = _q.Count + 1; + } + + public void Dispose() + { + _index = Ended; + _currentElement = default; + } + + public bool MoveNext() + { + if (_version != _q._version) throw new InvalidOperationException("EnumFailedVersion"); + + if (_index == Ended) + return false; + + _index--; + + if (_index == Completed) + { + // We've run past the last element + _index = Ended; + _currentElement = default; + return false; + } + + // Cache some fields in locals to decrease code size + T[] array = _q._array; + int capacity = array.Length; + + // _index represents the 0-based index into the queue, however the queue + // doesn't have to start from 0 and it may not even be stored contiguously in memory. + + int arrayIndex = _q._head + _index; // this is the actual index into the queue's backing array + if (arrayIndex >= capacity) + { + // NOTE: Originally we were using the modulo operator here, however + // on Intel processors it has a very high instruction latency which + // was slowing down the loop quite a bit. + // Replacing it with simple comparison/subtraction operations sped up + // the average foreach loop by 2x. + + arrayIndex -= capacity; // wrap around if needed + } + + _currentElement = array[arrayIndex]; + return true; + } + + public T Current + { + get + { + if (_index < 0) + ThrowEnumerationNotStartedOrEnded(); + return _currentElement!; + } + } + + private void ThrowEnumerationNotStartedOrEnded() + { + Debug.Assert(_index == Started || _index == Ended); + throw new InvalidOperationException(_index == Started ? "EnumNotStarted" : "EnumEnded"); + } + + object? IEnumerator.Current + { + get { return Current; } + } + + void IEnumerator.Reset() + { + if (_version != _q._version) throw new InvalidOperationException("EnumFailedVersion"); + _index = Started; + _currentElement = default; + } + } + + /// Converts an enumerable to an array using the same logic as List{T}. + /// The enumerable to convert. + /// The number of items stored in the resulting array, 0-indexed. + /// + /// The resulting array. The length of the array may be greater than , + /// which is the actual number of elements in the array. + /// + internal static T[] ToArray(IEnumerable source, out int length) + { + if (source is ICollection ic) + { + int count = ic.Count; + if (count != 0) + { + // Allocate an array of the desired size, then copy the elements into it. Note that this has the same + // issue regarding concurrency as other existing collections like List. If the collection size + // concurrently changes between the array allocation and the CopyTo, we could end up either getting an + // exception from overrunning the array (if the size went up) or we could end up not filling as many + // items as 'count' suggests (if the size went down). This is only an issue for concurrent collections + // that implement ICollection, which as of .NET 4.6 is just ConcurrentDictionary. + T[] arr = new T[count]; + ic.CopyTo(arr, 0); + length = count; + return arr; + } + } + else + { + using IEnumerator en = source.GetEnumerator(); + if (en.MoveNext()) + { + const int DefaultCapacity = 4; + T[] arr = new T[DefaultCapacity]; + arr[0] = en.Current; + int count = 1; + + while (en.MoveNext()) + { + if (count == arr.Length) + { + // This is the same growth logic as in List: + // If the array is currently empty, we make it a default size. Otherwise, we attempt to + // double the size of the array. Doubling will overflow once the size of the array reaches + // 2^30, since doubling to 2^31 is 1 larger than Int32.MaxValue. In that case, we instead + // constrain the length to be Array.MaxLength (this overflow check works because of the + // cast to uint). + int newLength = count << 1; + if ((uint)newLength > Array.MaxLength) + { + newLength = Array.MaxLength <= count ? count + 1 : Array.MaxLength; + } + + Array.Resize(ref arr, newLength); + } + + arr[count++] = en.Current; + } + + length = count; + return arr; + } + } + + length = 0; + return Array.Empty(); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Sync/GetLeafNodesRequest.cs b/src/Nethermind/Nethermind.Verkle.Tree/Sync/GetLeafNodesRequest.cs new file mode 100644 index 00000000000..46096664eaf --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Sync/GetLeafNodesRequest.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Sync; + +public class GetLeafNodesRequest +{ + public Pedersen RootHash { get; set; } + + public byte[][] LeafNodePaths { get; set; } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Sync/LeafToRefreshRequest.cs b/src/Nethermind/Nethermind.Verkle.Tree/Sync/LeafToRefreshRequest.cs new file mode 100644 index 00000000000..e1f463b46bf --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Sync/LeafToRefreshRequest.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Sync; + +public class LeafToRefreshRequest +{ + /// + /// Root hash of the account trie to serve + /// + public Pedersen RootHash { get; set; } + + public byte[][] Paths { get; set; } + + public override string ToString() + { + return $"LeafToRefreshRequest: ({RootHash}, {Paths.Length})"; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Sync/PathWithSubTree.cs b/src/Nethermind/Nethermind.Verkle.Tree/Sync/PathWithSubTree.cs new file mode 100644 index 00000000000..9a714c4baff --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Sync/PathWithSubTree.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Sync; + +public class PathWithSubTree +{ + public PathWithSubTree(Stem stem, LeafInSubTree[] subTree) + { + Path = stem; + SubTree = subTree; + } + + public Stem Path { get; set; } + public LeafInSubTree[] SubTree { get; set; } +} + +public readonly struct LeafInSubTree +{ + public readonly byte SuffixByte; + public readonly byte[]? Leaf; + + public LeafInSubTree(byte suffixByte, byte[]? leaf) + { + SuffixByte = suffixByte; + Leaf = leaf; + } + + public static implicit operator LeafInSubTree((byte, byte[]) leafWithSubIndex) + { + return new LeafInSubTree(leafWithSubIndex.Item1, leafWithSubIndex.Item2); + } + + public static implicit operator LeafInSubTree(KeyValuePair leafWithSubIndex) + { + return new LeafInSubTree(leafWithSubIndex.Key, leafWithSubIndex.Value); + } + + public override string ToString() + { + return $"{SuffixByte}:{Leaf?.ToHexString()}"; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Sync/SubTreeRange.cs b/src/Nethermind/Nethermind.Verkle.Tree/Sync/SubTreeRange.cs new file mode 100644 index 00000000000..512928ba11b --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Sync/SubTreeRange.cs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree.Sync; + +public class SubTreeRange +{ + public SubTreeRange(Pedersen stateRoot, Stem startingStem, Stem? limitStem = null, long? blockNumber = null) + { + RootHash = stateRoot; + StartingStem = startingStem; + BlockNumber = blockNumber; + LimitStem = limitStem; + } + + public long? BlockNumber { get; } + + /// + /// State Root of the verkle trie to serve + /// + public Pedersen RootHash { get; } + + /// + /// Stem of the first sub-tree to retrieve + /// + public Stem StartingStem { get; } + + /// + /// Stem after which to stop serving data + /// + public Stem? LimitStem { get; } + + public override string ToString() + { + return $"SubTreeRange: ({BlockNumber}, {RootHash}, {StartingStem}, {LimitStem})"; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Sync/SubTreesAndProofs.cs b/src/Nethermind/Nethermind.Verkle.Tree/Sync/SubTreesAndProofs.cs new file mode 100644 index 00000000000..18ac7b69fef --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Sync/SubTreesAndProofs.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Verkle.Tree.Sync; + +public class SubTreesAndProofs +{ + public PathWithSubTree[] SubTrees { get; set; } + public byte[] Proofs { get; set; } + + public SubTreesAndProofs(PathWithSubTree[] data, byte[] proofs) + { + SubTrees = data; + Proofs = proofs; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Usings.cs b/src/Nethermind/Nethermind.Verkle.Tree/Usings.cs new file mode 100644 index 00000000000..2367cb8fa17 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Usings.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +global using InternalStore = Nethermind.Core.Collections.SpanConcurrentDictionary; +global using LeafStore = Nethermind.Core.Collections.SpanConcurrentDictionary; +global using LeafStoreSorted = System.Collections.Generic.SortedDictionary; +global using VerkleUtils = Nethermind.Verkle.Tree.Utils.VerkleUtils; +global using VerkleNodeType = Nethermind.Verkle.Tree.Nodes.VerkleNodeType; +global using InternalStoreInterface = System.Collections.Generic.IDictionary; +global using LeafStoreInterface = System.Collections.Generic.IDictionary; +global using LeafEnumerator = System.Collections.Generic.IEnumerator>; +global using Committer = Nethermind.Core.Verkle.Committer; diff --git a/src/Nethermind/Nethermind.Verkle.Tree/Utils/VerkleUtils.cs b/src/Nethermind/Nethermind.Verkle.Tree/Utils/VerkleUtils.cs new file mode 100644 index 00000000000..47d94bcc77c --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/Utils/VerkleUtils.cs @@ -0,0 +1,59 @@ +using System.Numerics; +using Nethermind.Int256; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; + +namespace Nethermind.Verkle.Tree.Utils +{ + public struct LeafUpdateDelta + { + public Banderwagon? DeltaC1 { get; private set; } + public Banderwagon? DeltaC2 { get; private set; } + + public LeafUpdateDelta() + { + DeltaC1 = null; + DeltaC2 = null; + } + + public void UpdateDelta(Banderwagon deltaLeafCommitment, byte index) + { + if (index < 128) + { + if (DeltaC1 is null) DeltaC1 = deltaLeafCommitment; + else DeltaC1 += deltaLeafCommitment; + } + else + { + if (DeltaC2 is null) DeltaC2 = deltaLeafCommitment; + else DeltaC2 += deltaLeafCommitment; + } + } + } + + public static class VerkleUtils + { + private static readonly FrE ValueExistsMarker = FrE.SetElement(BigInteger.Pow(2, 128)); + + public static (FrE, FrE) BreakValueInLowHigh(byte[]? value) + { + if (value is null) return (FrE.Zero, FrE.Zero); + if (value.Length != 32) throw new ArgumentException(); + UInt256 valueFr = new(value); + FrE lowFr = FrE.SetElement(valueFr.u0, valueFr.u1) + ValueExistsMarker; + FrE highFr = FrE.SetElement(valueFr.u2, valueFr.u3); + return (lowFr, highFr); + } + + public static (List, byte?, byte?) GetPathDifference(IEnumerable existingNodeKey, IEnumerable newNodeKey) + { + List samePathIndices = new List(); + foreach ((byte first, byte second) in existingNodeKey.Zip(newNodeKey)) + { + if (first != second) return (samePathIndices, first, second); + samePathIndices.Add(first); + } + return (samePathIndices, null, null); + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/ChangeSet.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/ChangeSet.cs new file mode 100644 index 00000000000..9c413ef451b --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/ChangeSet.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public class ChangeSet +{ + public long BlockNumber; + public VerkleMemoryDb DiffLayer; + + public ChangeSet(long blockNumber, VerkleMemoryDb diffLayer) + { + BlockNumber = blockNumber; + DiffLayer = diffLayer; + } +} + +public class BatchChangeSet +{ + public long FromBlockNumber; + public long ToBlockNumber; + public VerkleMemoryDb DiffLayer; + + public BatchChangeSet(long fromBlockNumber, long toBlockNumber, VerkleMemoryDb diffLayer) + { + FromBlockNumber = fromBlockNumber; + ToBlockNumber = toBlockNumber; + DiffLayer = diffLayer; + } +} + diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/DiffLayer.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/DiffLayer.cs new file mode 100644 index 00000000000..85b7d275dd1 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/DiffLayer.cs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using Nethermind.Db; +using Nethermind.Serialization.Rlp; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Serializers; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public enum DiffType +{ + Forward, + Reverse +} + + +public class DiffLayer +{ + private readonly DiffType _diffType; + private IDb DiffDb { get; } + public DiffLayer(IDb diffDb, DiffType diffType) + { + DiffDb = diffDb; + _diffType = diffType; + } + public void InsertDiff(long blockNumber, VerkleMemoryDb memory) + { + RlpStream stream = new(VerkleMemoryDbSerializer.Instance.GetLength(memory, RlpBehaviors.None)); + VerkleMemoryDbSerializer.Instance.Encode(stream, memory); + if (stream.Data != null) DiffDb.Set(blockNumber, stream.Data); + } + + public void InsertDiff(long blockNumber, ReadOnlyVerkleMemoryDb memory) + { + RlpStream stream = new(VerkleMemoryDbSerializer.Instance.GetLength(memory, RlpBehaviors.None)); + VerkleMemoryDbSerializer.Instance.Encode(stream, memory); + if (stream.Data != null) DiffDb.Set(blockNumber, stream.Data); + } + + public VerkleMemoryDb FetchDiff(long blockNumber) + { + byte[]? diff = DiffDb.Get(blockNumber); + if (diff is null) throw new ArgumentException(null, nameof(blockNumber)); + return VerkleMemoryDbSerializer.Instance.Decode(diff.AsRlpStream()); + } + + public VerkleMemoryDb MergeDiffs(long fromBlock, long toBlock) + { + VerkleMemoryDb mergedDiff = new VerkleMemoryDb(); + switch (_diffType) + { + case DiffType.Reverse: + Debug.Assert(fromBlock > toBlock); + for (long i = toBlock + 1; i <= fromBlock; i++) + { + VerkleMemoryDb reverseDiff = FetchDiff(i); + foreach (KeyValuePair item in reverseDiff.LeafTable) + { + mergedDiff.LeafTable.TryAdd(item.Key, item.Value); + } + foreach (KeyValuePair item in reverseDiff.InternalTable) + { + mergedDiff.InternalTable.TryAdd(item.Key, item.Value); + } + } + break; + case DiffType.Forward: + Debug.Assert(fromBlock < toBlock); + for (long i = toBlock; i >= fromBlock; i--) + { + VerkleMemoryDb forwardDiff = FetchDiff(i); + foreach (KeyValuePair item in forwardDiff.LeafTable) + { + mergedDiff.LeafTable.TryAdd(item.Key, item.Value); + } + foreach (KeyValuePair item in forwardDiff.InternalTable) + { + mergedDiff.InternalTable.TryAdd(item.Key, item.Value); + } + } + break; + default: + throw new NotSupportedException(); + } + return mergedDiff; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleDb.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleDb.cs new file mode 100644 index 00000000000..51536ec0fbb --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleDb.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public interface IVerkleDb +{ + bool GetLeaf(ReadOnlySpan key, out byte[]? value); + bool GetInternalNode(ReadOnlySpan key, out InternalNode? value); + + void SetLeaf(ReadOnlySpan leafKey, byte[] leafValue); + void SetInternalNode(ReadOnlySpan internalNodeKey, InternalNode internalNodeValue); + + void RemoveLeaf(ReadOnlySpan leafKey); + void RemoveInternalNode(ReadOnlySpan internalNodeKey); + + void BatchLeafInsert(IEnumerable> keyLeaf); + void BatchInternalNodeInsert(IEnumerable> internalNodeLeaf); +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleKeyValueDb.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleKeyValueDb.cs new file mode 100644 index 00000000000..f8ef7a49f92 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleKeyValueDb.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Db; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public interface IVerkleKeyValueDb +{ + public IDb LeafDb { get; } + public IDb InternalNodeDb { get; } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleMemDb.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleMemDb.cs new file mode 100644 index 00000000000..dde5a36f703 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/IVerkleMemDb.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using Nethermind.Core.Collections; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public interface IVerkleMemDb +{ + public LeafStore LeafTable { get; } + public InternalStore InternalTable { get; } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleHistoryStore.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleHistoryStore.cs new file mode 100644 index 00000000000..44b261327fc --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleHistoryStore.cs @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Db; +using Nethermind.Logging; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public class VerkleHistoryStore +{ + private readonly ILogger _logger; + private DiffLayer ForwardDiff { get; } + private DiffLayer ReverseDiff { get; } + + public VerkleHistoryStore(IDbProvider dbProvider, ILogManager logManager) + { + ForwardDiff = new DiffLayer(dbProvider.ForwardDiff, DiffType.Forward); + ReverseDiff = new DiffLayer(dbProvider.ReverseDiff, DiffType.Reverse); + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + } + + public VerkleHistoryStore(IDb forwardDiff, IDb reverseDiff, ILogManager logManager) + { + ForwardDiff = new DiffLayer(forwardDiff, DiffType.Forward); + ReverseDiff = new DiffLayer(reverseDiff, DiffType.Reverse); + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + } + + public void InsertDiff(long blockNumber, VerkleMemoryDb postState, VerkleMemoryDb preState) + { + ForwardDiff.InsertDiff(blockNumber, postState); + ReverseDiff.InsertDiff(blockNumber, preState); + } + + public void InsertDiff(long blockNumber, ReadOnlyVerkleMemoryDb postState, VerkleMemoryDb preState) + { + ForwardDiff.InsertDiff(blockNumber, postState); + ReverseDiff.InsertDiff(blockNumber, preState); + } + + public BatchChangeSet GetBatchDiff(long fromBlock, long toBlock) + { + VerkleMemoryDb diff = (fromBlock > toBlock) switch + { + true => ReverseDiff.MergeDiffs(fromBlock, toBlock), + false => ForwardDiff.MergeDiffs(fromBlock, toBlock) + }; + + return new BatchChangeSet(fromBlock, toBlock, diff); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleKeyValueDb.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleKeyValueDb.cs new file mode 100644 index 00000000000..49fcf9bd0ad --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleKeyValueDb.cs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Db; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Serializers; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public class VerkleKeyValueDb : IVerkleDb, IVerkleKeyValueDb +{ + public IDb LeafDb { get; } + public IDb InternalNodeDb { get; } + + public VerkleKeyValueDb(IDbProvider dbProvider) + { + LeafDb = dbProvider.LeafDb; + InternalNodeDb = dbProvider.InternalNodesDb; + } + + public VerkleKeyValueDb(IDb internalNodeDb, IDb leafDb) + { + LeafDb = leafDb; + InternalNodeDb = internalNodeDb; + } + + public byte[]? GetLeaf(ReadOnlySpan key) => LeafDb.Get(key); + + public InternalNode? GetInternalNode(ReadOnlySpan key) + { + byte[]? value = InternalNodeDb[key]; + return value is null ? null : InternalNodeSerializer.Instance.Decode(value); + } + + public bool GetLeaf(ReadOnlySpan key, out byte[]? value) + { + value = GetLeaf(key); + return value is not null; + } + + public bool GetInternalNode(ReadOnlySpan key, out InternalNode? value) + { + value = GetInternalNode(key); + return value is not null; + } + + public void SetLeaf(ReadOnlySpan leafKey, byte[] leafValue) => LeafDb[leafKey] = leafValue; + public void SetInternalNode(ReadOnlySpan internalNodeKey, InternalNode internalNodeValue) => SetInternalNode(internalNodeKey, internalNodeValue, InternalNodeDb); + + public void RemoveLeaf(ReadOnlySpan leafKey) + { + LeafDb.Remove(leafKey); + } + + public void RemoveInternalNode(ReadOnlySpan internalNodeKey) + { + InternalNodeDb.Remove(internalNodeKey); + } + + + public void BatchLeafInsert(IEnumerable> keyLeaf) + { + using IBatch batch = LeafDb.StartBatch(); + foreach ((byte[] key, byte[]? value) in keyLeaf) + { + batch[key] = value; + } + } + + public void BatchInternalNodeInsert(IEnumerable> internalNode) + { + using IBatch batch = InternalNodeDb.StartBatch(); + foreach ((byte[] key, InternalNode? value) in internalNode) + { + SetInternalNode(key, value, batch); + } + } + + private static void SetInternalNode(ReadOnlySpan internalNodeKey, InternalNode? internalNodeValue, IKeyValueStore db) + { + if (internalNodeValue != null) db[internalNodeKey] = InternalNodeSerializer.Instance.Encode(internalNodeValue).Bytes; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleMemoryDb.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleMemoryDb.cs new file mode 100644 index 00000000000..05f1ad1bce9 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleDb/VerkleMemoryDb.cs @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree.VerkleDb; + +public class ReadOnlyVerkleMemoryDb +{ + public LeafStoreSorted LeafTable; + public InternalStore InternalTable; +} + +public class VerkleMemoryDb : IVerkleDb, IVerkleMemDb +{ + public LeafStore LeafTable { get; } + public InternalStore InternalTable { get; } + + public VerkleMemoryDb() + { + LeafTable = new LeafStore(Bytes.SpanEqualityComparer); + InternalTable = new InternalStore(Bytes.SpanEqualityComparer); + } + + public VerkleMemoryDb(LeafStore leafTable, InternalStore internalTable) + { + LeafTable = leafTable; + InternalTable = internalTable; + } + + public bool GetLeaf(ReadOnlySpan key, out byte[]? value) => LeafTable.TryGetValue(key, out value); + public bool GetInternalNode(ReadOnlySpan key, out InternalNode? value) => InternalTable.TryGetValue(key, out value); + + public void SetLeaf(ReadOnlySpan leafKey, byte[] leafValue) => LeafTable[leafKey] = leafValue; + public void SetInternalNode(ReadOnlySpan internalNodeKey, InternalNode internalNodeValue) => InternalTable[internalNodeKey] = internalNodeValue; + + public void RemoveLeaf(ReadOnlySpan leafKey) => LeafTable.Remove(leafKey.ToArray(), out _); + public void RemoveInternalNode(ReadOnlySpan internalNodeKey) => InternalTable.Remove(internalNodeKey.ToArray(), out _); + + public void BatchLeafInsert(IEnumerable> keyLeaf) + { + foreach ((byte[] key, byte[]? value) in keyLeaf) + { + SetLeaf(key, value); + } + } + + public void BatchInternalNodeInsert(IEnumerable> internalNodeKey) + { + foreach ((byte[] key, InternalNode? value) in internalNodeKey) + { + SetInternalNode(key, value); + } + } + + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleRangeProofCollector.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleRangeProofCollector.cs new file mode 100644 index 00000000000..3af284e6dee --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleRangeProofCollector.cs @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers; +using System.Runtime.InteropServices; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Trie; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree; + +public class VerkleRangeProofCollector: IVerkleTreeVisitor +{ + private readonly byte[]? _startStem; + private bool _findFirstNodeInRange = true; + + private readonly byte[]? _limitStem; + private readonly bool _comparePathWithLimitHash = true; + + private bool _shouldContinueTraversing = true; + + private long _currentBytesCount; + private readonly Dictionary _collectedNodes = new(); + + private HashSet? _nodeToVisitFilterInstance; + + private HashSet NodeToVisitFilter => _nodeToVisitFilterInstance ?? + LazyInitializer.EnsureInitialized(ref _nodeToVisitFilterInstance, + () => new HashSet()); + + private readonly int _nodeLimit; + private readonly long _byteLimit; + + private readonly long _hardByteLimit; + public bool _isStoppedDueToHardLimit; + + public VerkleRangeProofCollector(byte[] startStem, byte[] limitStem, bool isAccountVisitor, long byteLimit = -1, long hardByteLimit = 200000, int nodeLimit = 10000) + { + if (Bytes.AreEqual(startStem, Keccak.Zero.Bytes[..31])) + { + _findFirstNodeInRange = false; + } + else + { + _startStem = ArrayPool.Shared.Rent(64); + Nibbles.BytesToNibbleBytes(limitStem, _startStem); + } + + if (Bytes.AreEqual(limitStem, Keccak.MaxValue.Bytes[..31])) + { + _comparePathWithLimitHash = false; + } + else + { + _limitStem = ArrayPool.Shared.Rent(64); + Nibbles.BytesToNibbleBytes(limitStem, _limitStem); + } + + _nodeLimit = nodeLimit; + _byteLimit = byteLimit; + _hardByteLimit = hardByteLimit; + } + + private static int ComparePath(List path, byte[]? hash) + { + Span pathSpan = CollectionsMarshal.AsSpan(path); + // compare the `path` and `hash` to check if a key with prefix `path` would come after the hash or not + return Bytes.Comparer.CompareGreaterThan(hash.AsSpan()[..pathSpan.Length], pathSpan); + } + + public bool ShouldVisit(byte[] nextNode) + { + // if still looking for node just after the startHash, then only visit node that are present in _nodeToVisitFilter + return _findFirstNodeInRange ? NodeToVisitFilter.Contains(nextNode) : _shouldContinueTraversing; + } + + private bool ShouldVisit(List path) + { + if (_collectedNodes.Count >= _nodeLimit) + { + _isStoppedDueToHardLimit = true; + return false; + } + + if (_byteLimit != -1 && _currentBytesCount >= _byteLimit) + { + return false; + } + + if (_currentBytesCount >= _hardByteLimit) + { + _isStoppedDueToHardLimit = true; + return false; + } + + if (!_comparePathWithLimitHash) return true; + int compResult = ComparePath(path, _limitStem); + return compResult != -1; + } + + public void VisitTree(VerkleCommitment rootHash, TrieVisitContext trieVisitContext) { } + + public void VisitMissingNode(byte[] nodeKey, TrieVisitContext trieVisitContext) { } + + public void VisitBranchNode(InternalNode node, TrieVisitContext trieVisitContext) + { + List path = trieVisitContext.AbsolutePathIndex; + + if (_findFirstNodeInRange) + { + NodeToVisitFilter.Remove(path.ToArray()); + + int compRes = ComparePath(path, _startStem); + switch (compRes) + { + case 1: + // if path < _startHash[:path.Count] - return and check for the next node. + return; + case 0: + { + // this is a important case - here the path == _startHash[:path.Count] + // the index of child should be _startHash[path.Count] + byte index = _startStem[path.Count]; + for (int i = index; i < 256; i++) + { + using (trieVisitContext.AbsolutePathNext((byte)i)) + { + NodeToVisitFilter.Add(trieVisitContext.AbsolutePathIndex.ToArray()); + } + } + return; + } + case -1: + // if path > _startHash[:path.Count] -> found the first element after the start range. + // continue visiting and collecting next nodes and set _findFirstNodeInRange = false + _findFirstNodeInRange = false; + break; + } + } + + bool shouldVisitNode = ShouldVisit(path); + if (shouldVisitNode) return; + _shouldContinueTraversing = false; + } + + public void VisitStemNode(InternalNode node, TrieVisitContext trieVisitContext) + { + throw new NotImplementedException(); + } + + public void VisitLeafNode(ReadOnlySpan nodeKey, TrieVisitContext trieVisitContext, byte[]? nodeValue) + { + throw new NotImplementedException(); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.History.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.History.cs new file mode 100644 index 00000000000..b30f00564c3 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.History.cs @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleStateStore +{ + + /// + /// maximum number of blocks that should be stored in cache (not persisted in db) + /// + private int MaxNumberOfBlocksInCache { get; } + private StackQueue<(long, ReadOnlyVerkleMemoryDb)>? BlockCache { get; } + + private VerkleHistoryStore? History { get; } + + + // now the full state back in time by one block. + public void ReverseState() + { + + if (BlockCache is not null && BlockCache.Count != 0) + { + BlockCache.Pop(out _); + return; + } + + VerkleMemoryDb reverseDiff = + History?.GetBatchDiff(LastPersistedBlockNumber, LastPersistedBlockNumber - 1).DiffLayer ?? + throw new ArgumentException("History not Enabled"); + + foreach (KeyValuePair entry in reverseDiff.LeafTable) + { + reverseDiff.GetLeaf(entry.Key, out byte[]? node); + if (node is null) + { + Storage.RemoveLeaf(entry.Key); + } + else + { + Storage.SetLeaf(entry.Key, node); + } + } + + foreach (KeyValuePair entry in reverseDiff.InternalTable) + { + reverseDiff.GetInternalNode(entry.Key, out InternalNode? node); + if (node is null) + { + Storage.RemoveInternalNode(entry.Key); + } + else + { + Storage.SetInternalNode(entry.Key, node); + } + } + LastPersistedBlockNumber -= 1; + } + + // use the batch diff to move the full state back in time to access historical state. + public void ApplyDiffLayer(BatchChangeSet changeSet) + { + if (changeSet.FromBlockNumber != LastPersistedBlockNumber) + { + throw new ArgumentException( + $"This case should not be possible. Diff fromBlock should be equal to persisted block number. FullStateBlock:{LastPersistedBlockNumber}!=fromBlock:{changeSet.FromBlockNumber}", + nameof(changeSet.FromBlockNumber)); + } + + + VerkleMemoryDb reverseDiff = changeSet.DiffLayer; + + foreach (KeyValuePair entry in reverseDiff.LeafTable) + { + reverseDiff.GetLeaf(entry.Key, out byte[]? node); + if (node is null) + { + Storage.RemoveLeaf(entry.Key); + } + else + { + Storage.SetLeaf(entry.Key, node); + } + } + + foreach (KeyValuePair entry in reverseDiff.InternalTable) + { + reverseDiff.GetInternalNode(entry.Key, out InternalNode? node); + if (node is null) + { + Storage.RemoveInternalNode(entry.Key); + } + else + { + Storage.SetInternalNode(entry.Key, node); + } + } + LastPersistedBlockNumber = changeSet.ToBlockNumber; + } + + public bool MoveToStateRoot(VerkleCommitment stateRoot) + { + // TODO: this can be replace with StateRoot - no need to get stateRoot from db + VerkleCommitment currentRoot = GetStateRoot(); + if (currentRoot == stateRoot) return true; + + if (_logger.IsDebug) _logger.Debug($"Trying to move state root from:{currentRoot} to:{stateRoot}"); + + // TODO: this is actually not possible - not sure if return true is correct here + if (stateRoot.Equals(new VerkleCommitment(Keccak.EmptyTreeHash.Bytes))) + { + if (currentRoot.Equals(VerkleCommitment.Zero)) return true; + return false; + } + + // resolve block numbers + long fromBlock = StateRootToBlocks[currentRoot]; + if (fromBlock == -1) + { + if (_logger.IsDebug) _logger.Debug($"Cannot get the block number for currentRoot:{currentRoot}"); + return false; + } + long toBlock = StateRootToBlocks[stateRoot]; + if (toBlock == -1) + { + if (_logger.IsDebug) _logger.Debug($"Cannot get the block number for wantedStateRoot:{stateRoot}"); + return false; + } + + if (_logger.IsDebug) + _logger.Debug($"Block numbers resolved. Trying to move state from:{fromBlock} to:{toBlock}"); + + // TODO: this should be handled when comparing stateRoot before + if (fromBlock == toBlock) return true; + + if (fromBlock > toBlock) + { + long noOfBlockToMove = fromBlock - toBlock; + if (BlockCache is not null && noOfBlockToMove > BlockCache.Count) + { + if (_logger.IsDebug) + _logger.Debug( + $"Number of blocks to move:{noOfBlockToMove}. Removing all the diffs from BlockCache ({noOfBlockToMove} > {BlockCache.Count})"); + + if (History is null) + { + if (_logger.IsDebug) _logger.Debug($"History is null and in this case - state cannot be reverted to wanted state root"); + return false; + } + BlockCache.Clear(); + fromBlock -= BlockCache.Count; + + if (_logger.IsDebug) + _logger.Debug($"now using fromBlock:{fromBlock} toBlock:{toBlock}"); + BatchChangeSet batchDiff = History.GetBatchDiff(fromBlock, toBlock); + ApplyDiffLayer(batchDiff); + } + else + { + if (_logger.IsDebug) + _logger.Debug( + $"Number of blocks to move:{noOfBlockToMove}. Removing all the diffs from BlockCache ({noOfBlockToMove} > {BlockCache?.Count})"); + if (BlockCache is not null) + { + for (int i = 0; i < noOfBlockToMove; i++) + { + BlockCache.Pop(out _); + } + } + else + { + if (_logger.IsDebug) + _logger.Debug( + $"BlockCache is null and in this case - state cannot be reverted to wanted state root"); + return false; + } + } + } + else + { + if (_logger.IsDebug) + _logger.Debug($"Trying to move forward in state - this is not implemented and supported yet"); + return false; + } + + Debug.Assert(GetStateRoot().Equals(stateRoot)); + LatestCommittedBlockNumber = toBlock; + return true; + } + + // This generates and returns a batchForwardDiff, that can be used to move the full state from fromBlock to toBlock. + // for this fromBlock < toBlock - move forward in time + public bool GetForwardMergedDiff(long fromBlock, long toBlock, [MaybeNullWhen(false)]out VerkleMemoryDb diff) + { + if (History is null) + { + diff = default; + return false; + } + diff = History.GetBatchDiff(fromBlock, toBlock).DiffLayer; + return true; + } + + // This generates and returns a batchForwardDiff, that can be used to move the full state from fromBlock to toBlock. + // for this fromBlock > toBlock - move back in time + public bool GetReverseMergedDiff(long fromBlock, long toBlock, [MaybeNullWhen(false)]out VerkleMemoryDb diff) + { + if (History is null) + { + diff = default; + return false; + } + diff = History.GetBatchDiff(fromBlock, toBlock).DiffLayer; + return true; + } + + private readonly StateRootToBlockMap StateRootToBlocks; + + private readonly struct StateRootToBlockMap + { + private readonly IDb _stateRootToBlock; + + public StateRootToBlockMap(IDb stateRootToBlock) + { + _stateRootToBlock = stateRootToBlock; + } + + public long this[VerkleCommitment key] + { + get + { + // if (Pedersen.Zero.Equals(key)) return -1; + byte[]? encodedBlock = _stateRootToBlock[key.Bytes]; + return encodedBlock is null ? -2 : BinaryPrimitives.ReadInt64LittleEndian(encodedBlock); + } + set + { + Span encodedBlock = stackalloc byte[8]; + BinaryPrimitives.WriteInt64LittleEndian(encodedBlock, value); + if(!_stateRootToBlock.KeyExists(key.Bytes)) + _stateRootToBlock.Set(key.Bytes, encodedBlock.ToArray()); + } + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.Persist.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.Persist.cs new file mode 100644 index 00000000000..12642190226 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.Persist.cs @@ -0,0 +1,179 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleStateStore +{ + private bool _lastPersistedReachedReorgBoundary; + private long _latestPersistedBlockNumber; + public long LastPersistedBlockNumber + { + get => _latestPersistedBlockNumber; + private set + { + if (value != _latestPersistedBlockNumber) + { + _latestPersistedBlockNumber = value; + _lastPersistedReachedReorgBoundary = false; + } + } + } + private long LatestCommittedBlockNumber { get; set; } + + private VerkleCommitment? PersistedStateRoot { get; set; } + + // This method is called at the end of each block to flush the batch changes to the storage and generate forward and reverse diffs. + // this should be called only once per block, right now it does not support multiple calls for the same block number. + // if called multiple times, the full state would be fine - but it would corrupt the diffs and historical state will be lost + // TODO: add capability to update the diffs instead of overwriting if Flush(long blockNumber) + // is called multiple times for the same block number, but do we even need this? + public void Flush(long blockNumber, VerkleMemoryDb batch) + { + if (_logger.IsDebug) + _logger.Debug( + $"VSS: Flushing:{blockNumber} InternalDb:{batch.InternalTable.Count} LeafDb:{batch.LeafTable.Count}"); + + if (blockNumber == 0) + { + if (_logger.IsDebug) + _logger.Debug($"VSS: Special case for block 0, Persisting"); + PersistBlockChanges(batch.InternalTable, batch.LeafTable, Storage); + StateRoot = PersistedStateRoot = GetStateRoot(); + LatestCommittedBlockNumber = LastPersistedBlockNumber = 0; + StateRootToBlocks[StateRoot] = blockNumber; + } + else + { + if (blockNumber <= LatestCommittedBlockNumber) + throw new InvalidOperationException("Cannot flush for same block number `multiple times"); + + // create a sorted set for leaves - for snap sync + // TODO: create this sorted set while inserting into the batch - will help reducing allocations + ReadOnlyVerkleMemoryDb cacheBatch = new() + { + InternalTable = batch.InternalTable, + LeafTable = new SortedDictionary(batch.LeafTable, Bytes.Comparer) + }; + + bool shouldPersistBlock; + ReadOnlyVerkleMemoryDb changesToPersist; + long blockNumberPersist; + if (BlockCache is null) + { + shouldPersistBlock = true; + changesToPersist = cacheBatch; + blockNumberPersist = blockNumber; + } + else + { + shouldPersistBlock = !BlockCache.EnqueueAndReplaceIfFull((blockNumber, cacheBatch), + out (long, ReadOnlyVerkleMemoryDb) element); + changesToPersist = element.Item2; + blockNumberPersist = element.Item1; + } + + if (shouldPersistBlock) + { + if (_logger.IsDebug) + _logger.Debug($"VSS: BlockCache is full - got forwardDiff BlockNumber:{blockNumberPersist} IN:{changesToPersist.InternalTable.Count} LN:{changesToPersist.LeafTable.Count}"); + VerkleCommitment root = GetStateRoot(changesToPersist.InternalTable) ?? (new VerkleCommitment(Storage.GetInternalNode(RootNodeKey)?.Bytes ?? throw new ArgumentException())); + if (_logger.IsDebug) _logger.Debug($"VSS: StateRoot after persisting forwardDiff: {root}"); + VerkleMemoryDb reverseDiff = PersistBlockChanges(changesToPersist.InternalTable, changesToPersist.LeafTable, Storage); + if (_logger.IsDebug) _logger.Debug($"VSS: reverseDiff: IN:{reverseDiff.InternalTable.Count} LN:{reverseDiff.LeafTable.Count}"); + History?.InsertDiff(blockNumberPersist, changesToPersist, reverseDiff); + PersistedStateRoot = root; + LastPersistedBlockNumber = blockNumberPersist; + Storage.LeafDb.Flush(); + Storage.InternalNodeDb.Flush(); + } + + LatestCommittedBlockNumber = blockNumber; + StateRoot = GetStateRoot(); + StateRootToBlocks[StateRoot] = blockNumber; + if (_logger.IsDebug) + _logger.Debug( + $"VSS: Completed Flush: PersistedStateRoot:{PersistedStateRoot} LastPersistedBlockNumber:{LastPersistedBlockNumber} LatestCommittedBlockNumber:{LatestCommittedBlockNumber} StateRoot:{StateRoot} blockNumber:{blockNumber}"); + } + AnnounceReorgBoundaries(); + } + + private VerkleMemoryDb PersistBlockChanges(IDictionary internalStore, IDictionary leafStore, VerkleKeyValueDb storage) + { + // we should not have any null values in the Batch db - because deletion of values from verkle tree is not allowed + // nullable values are allowed in MemoryStateDb only for reverse diffs. + VerkleMemoryDb reverseDiff = new(); + + foreach (KeyValuePair entry in leafStore) + { + // in stateless tree - anything can be null + // Debug.Assert(entry.Value is not null, "nullable value only for reverse diff"); + if (storage.GetLeaf(entry.Key, out byte[]? node)) reverseDiff.LeafTable[entry.Key] = node; + else reverseDiff.LeafTable[entry.Key] = null; + + storage.SetLeaf(entry.Key, entry.Value); + } + + foreach (KeyValuePair entry in internalStore) + { + // in stateless tree - anything can be null + // Debug.Assert(entry.Value is not null, "nullable value only for reverse diff"); + if (storage.GetInternalNode(entry.Key, out InternalNode? node)) reverseDiff.InternalTable[entry.Key] = node; + else reverseDiff.InternalTable[entry.Key] = null; + + storage.SetInternalNode(entry.Key, entry.Value); + } + + if (_logger.IsDebug) + _logger.Debug( + $"PersistBlockChanges: ReverseDiff InternalStore:{reverseDiff.InternalTable.Count} LeafStore:{reverseDiff.LeafTable.Count}"); + + return reverseDiff; + } + + private int _isFirst; + private void AnnounceReorgBoundaries() + { + if (LatestCommittedBlockNumber < 1) + { + return; + } + + bool shouldAnnounceReorgBoundary = false; + bool isFirstCommit = Interlocked.Exchange(ref _isFirst, 1) == 0; + if (isFirstCommit) + { + if (_logger.IsDebug) _logger.Debug($"Reached first commit - newest {LatestCommittedBlockNumber}, last persisted {LastPersistedBlockNumber}"); + // this is important when transitioning from fast sync + // imagine that we transition at block 1200000 + // and then we close the app at 1200010 + // in such case we would try to continue at Head - 1200010 + // because head is loaded if there is no persistence checkpoint + // so we need to force the persistence checkpoint + long baseBlock = Math.Max(0, LatestCommittedBlockNumber - 1); + LastPersistedBlockNumber = baseBlock; + shouldAnnounceReorgBoundary = true; + } + else if (!_lastPersistedReachedReorgBoundary) + { + // even after we persist a block we do not really remember it as a safe checkpoint + // until max reorgs blocks after + if (LatestCommittedBlockNumber >= LastPersistedBlockNumber + MaxNumberOfBlocksInCache) + { + shouldAnnounceReorgBoundary = true; + } + } + + if (shouldAnnounceReorgBoundary) + { + ReorgBoundaryReached?.Invoke(this, new ReorgBoundaryReached(LastPersistedBlockNumber)); + _lastPersistedReachedReorgBoundary = true; + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.Sync.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.Sync.cs new file mode 100644 index 00000000000..9f7f4564923 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.Sync.cs @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleStateStore +{ + public bool IsFullySynced(Keccak stateRoot) => StateRootToBlocks[new VerkleCommitment(stateRoot.Bytes)] != -2; + + public IEnumerable GetLeafRangeIterator(Stem fromRange, Stem toRange, VerkleCommitment stateRoot, long bytes) + { + if(bytes == 0) yield break; + + long blockNumber = StateRootToBlocks[stateRoot]; + byte[] fromRangeBytes = new byte[32]; + byte[] toRangeBytes = new byte[32]; + fromRange.BytesAsSpan.CopyTo(fromRangeBytes); + toRange.BytesAsSpan.CopyTo(toRangeBytes); + fromRangeBytes[31] = 0; + toRangeBytes[31] = 255; + + using LeafEnumerator enumerator = GetLeafRangeIterator(fromRangeBytes, toRangeBytes, blockNumber).GetEnumerator(); + + int usedBytes = 0; + + HashSet listOfStem = new(); + Stem currentStem = fromRange; + List subTree = new(256); + + while (enumerator.MoveNext()) + { + KeyValuePair current = enumerator.Current; + if (listOfStem.Contains(current.Key.Slice(0,31))) + { + subTree.Add(new LeafInSubTree(current.Key[31], current.Value)); + usedBytes += 31; + } + else + { + if (subTree.Count != 0) yield return new PathWithSubTree(currentStem, subTree.ToArray()); + if (usedBytes >= bytes) break; + subTree.Clear(); + currentStem = new Stem(current.Key.Slice(0,31).ToArray()); + listOfStem.Add(currentStem); + subTree.Add(new LeafInSubTree(current.Key[31], current.Value)); + usedBytes += 31 + 33; + } + } + if (subTree.Count != 0) yield return new PathWithSubTree(currentStem, subTree.ToArray()); + } + + public IEnumerable> GetLeafRangeIterator(byte[] fromRange, byte[] toRange, long blockNumber) + { + if(BlockCache is null) yield break; + + // this will contain all the iterators that we need to fulfill the GetSubTreeRange request + List iterators = new(); + + // kvMap is used to keep a map of keyValues we encounter - this is for ease of access - but not optimal + // TODO: remove this - merge kvMap and kvEnumMap + Dictionary> kvMap = new(Bytes.EqualityComparer); + // this created a sorted structure for all the keys and the corresponding enumerators. the idea is that get + // the first key (sorted), remove the key, then move the enumerator to next and insert the new key and + // enumerator again + DictionarySortedSet keyEnumMap = new(Bytes.Comparer); + + // TODO: optimize this to start from a specific blockNumber - or better yet get the list of enumerators directly + using StackQueue<(long, ReadOnlyVerkleMemoryDb)>.StackEnumerator blockEnumerator = + BlockCache.GetStackEnumerator(); + try + { + int iteratorPriority = 0; + while (blockEnumerator.MoveNext()) + { + // enumerate till we get to the required block number + if(blockEnumerator.Current.Item1 > blockNumber) continue; + + // TODO: here we construct a set from the LeafTable so that we can do the GetViewBetween + // obviously this is very un-optimal but the idea is to replace the LeafTable with SortedSet in the + // blockCache itself. The reason we want to use GetViewBetween because this is optimal to do seek + DictionarySortedSet currentSet = new (blockEnumerator.Current.Item2.LeafTable, Bytes.Comparer); + + // construct the iterators that starts for the specific range using GetViewBetween + IEnumerator> enumerator = currentSet + .GetViewBetween( + new KeyValuePair(fromRange, Pedersen.Zero.Bytes), + new KeyValuePair(toRange, Pedersen.Zero.Bytes)) + .GetEnumerator(); + + // find the first value in iterator that is not already used + bool isIteratorUsed = false; + while (enumerator.MoveNext()) + { + KeyValuePair current = enumerator.Current; + // add the key and corresponding value + if (kvMap.TryAdd(current.Key, new(iteratorPriority, current.Value))) + { + isIteratorUsed = true; + iterators.Add(enumerator); + // add the new key and the corresponding enumerator + keyEnumMap.Add(current.Key, new(enumerator, iteratorPriority)); + break; + } + } + if (!isIteratorUsed) + { + enumerator.Dispose(); + continue; + } + iteratorPriority++; + } + + LeafEnumerator persistentLeafsIterator = Storage.LeafDb.GetIterator(fromRange, toRange).GetEnumerator(); + bool isPersistentIteratorUsed = false; + while (persistentLeafsIterator.MoveNext()) + { + KeyValuePair current = persistentLeafsIterator.Current; + // add the key and corresponding value + if (kvMap.TryAdd(current.Key, new(iteratorPriority, current.Value))) + { + isPersistentIteratorUsed = true; + iterators.Add(persistentLeafsIterator); + // add the new key and the corresponding enumerator + keyEnumMap.Add(current.Key, new (persistentLeafsIterator, iteratorPriority)); + break; + } + } + if (!isPersistentIteratorUsed) + { + persistentLeafsIterator.Dispose(); + } + + void InsertAndMoveIteratorRecursive(LeafIterator leafIterator) + { + while (leafIterator.Enumerator.MoveNext()) + { + KeyValuePair newKeyValuePair = leafIterator.Enumerator.Current; + byte[] newKeyToInsert = newKeyValuePair.Key; + // now here check if the value already exist and if the priority of value of higher or lower and + // update accordingly + KeyValuePair valueToInsert = new(leafIterator.Priority, newKeyValuePair.Value); + + if (kvMap.TryGetValue(newKeyToInsert, out KeyValuePair valueExisting)) + { + // priority of the new value is smaller (more) than the priority of old value + if (valueToInsert.Key < valueExisting.Key) + { + keyEnumMap.TryGetValue(newKeyValuePair.Key, out LeafIterator? prevIterator); + keyEnumMap.Remove(newKeyValuePair.Key); + + // replace the existing value + keyEnumMap.Add(newKeyValuePair.Key, leafIterator); + kvMap[newKeyValuePair.Key] = valueToInsert; + + // since we replacing the existing value, we need to move the prevIterator iterator to + // next value till we get the new value + InsertAndMoveIteratorRecursive(prevIterator); + break; + } + + // since we were not able to add current value from this iterator, move to next value and try + // to add that + } + else + { + // this is the most simple case + // since there was no existing value - we just insert without modifying other iterators + keyEnumMap.Add(newKeyValuePair.Key, leafIterator); + kvMap.Add(newKeyValuePair.Key, valueToInsert); + break; + } + } + } + + while (keyEnumMap.Count > 0) + { + // get the first value from the sorted set + KeyValuePair value = keyEnumMap.Min; + // remove the corresponding element because it will be used + keyEnumMap.Remove(value.Key); + + // get the enumerator and move it to next and insert the corresponding values recursively + InsertAndMoveIteratorRecursive(value.Value); + + byte[] returnValue = kvMap[value.Key].Value; + kvMap.Remove(value.Key); + + // return the value + yield return new KeyValuePair (value.Key, returnValue); + } + } + finally + { + foreach (LeafEnumerator t in iterators) t.Dispose(); + } + } + + public List? GetLeafRangeIterator(byte[] fromRange, byte[] toRange, VerkleCommitment stateRoot, long bytes) + { + long blockNumber = StateRootToBlocks[stateRoot]; + using IEnumerator> ranges = GetLeafRangeIterator(fromRange, toRange, blockNumber).GetEnumerator(); + + long currentBytes = 0; + + SpanDictionary> rangesToReturn = new(Bytes.SpanEqualityComparer); + + if (!ranges.MoveNext()) return null; + + // handle the first element + Span stem = ranges.Current.Key.AsSpan()[..31]; + rangesToReturn.TryAdd(stem, new List()); + rangesToReturn[stem].Add(new LeafInSubTree(ranges.Current.Key[31], ranges.Current.Value!)); + currentBytes += 64; + + + bool bytesConsumed = false; + while (ranges.MoveNext()) + { + if (currentBytes > bytes) + { + bytesConsumed = true; + break; + } + } + + if (bytesConsumed) + { + // this means the iterator is not empty but the bytes is consumed, now we need to complete the current + // subtree we are processing + while (ranges.MoveNext()) + { + // if stem is present that means we have to complete that subTree + stem = ranges.Current.Key.AsSpan()[..31]; + if (rangesToReturn.TryGetValue(stem, out List? listOfLeafs)) + { + listOfLeafs.Add(new LeafInSubTree(ranges.Current.Key[31], ranges.Current.Value!)); + continue; + } + break; + } + } + + List pathWithSubTrees = new(rangesToReturn.Count); + foreach (KeyValuePair> keyVal in rangesToReturn) + { + pathWithSubTrees.Add(new PathWithSubTree(keyVal.Key, keyVal.Value.ToArray())); + } + + return pathWithSubTrees; + } + + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.cs new file mode 100644 index 00000000000..135b8b9534e --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleStateStore.cs @@ -0,0 +1,149 @@ +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Trie.Pruning; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleStateStore : IVerkleTrieStore, ISyncTrieStore +{ + private static Span RootNodeKey => Array.Empty(); + public VerkleCommitment StateRoot { get; private set; } + + private readonly ILogger _logger; + + public VerkleCommitment GetStateRoot() + { + InternalNode rootNode = RootNode ?? throw new InvalidOperationException("Root node should always be present"); + + byte[] stateRoot = rootNode.Bytes; + return new VerkleCommitment(stateRoot); + } + + private static VerkleCommitment? GetStateRoot(IVerkleDb db) + { + return db.GetInternalNode(RootNodeKey, out InternalNode? node) ? new VerkleCommitment(node!.Bytes) : null; + } + + private static VerkleCommitment? GetStateRoot(InternalStore db) + { + return db.TryGetValue(RootNodeKey, out InternalNode? node) ? new VerkleCommitment(node!.Bytes) : null; + } + + // The underlying key value database + // We try to avoid fetching from this, and we only store at the end of a batch insert + private VerkleKeyValueDb Storage { get; } + + public VerkleStateStore(IDbProvider dbProvider, ILogManager logManager, int maxNumberOfBlocksInCache = 128) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + Storage = new VerkleKeyValueDb(dbProvider); + History = new VerkleHistoryStore(dbProvider, logManager); + StateRootToBlocks = new StateRootToBlockMap(dbProvider.StateRootToBlocks); + BlockCache = maxNumberOfBlocksInCache == 0 + ? null + : new StackQueue<(long, ReadOnlyVerkleMemoryDb)>(maxNumberOfBlocksInCache); + MaxNumberOfBlocksInCache = maxNumberOfBlocksInCache; + InitRootHash(); + } + + public VerkleStateStore( + IDb leafDb, + IDb internalDb, + IDb forwardDiff, + IDb reverseDiff, + IDb stateRootToBlocks, + ILogManager logManager, + int maxNumberOfBlocksInCache = 128) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + Storage = new VerkleKeyValueDb(internalDb, leafDb); + History = new VerkleHistoryStore(forwardDiff, reverseDiff, logManager); + StateRootToBlocks = new StateRootToBlockMap(stateRootToBlocks); + BlockCache = maxNumberOfBlocksInCache == 0 + ? null + : new StackQueue<(long, ReadOnlyVerkleMemoryDb)>(maxNumberOfBlocksInCache); + MaxNumberOfBlocksInCache = maxNumberOfBlocksInCache; + InitRootHash(); + } + + public VerkleStateStore( + IDb leafDb, + IDb internalDb, + IDb stateRootToBlocks, + ILogManager logManager, + int maxNumberOfBlocksInCache = 128) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + Storage = new VerkleKeyValueDb(internalDb, leafDb); + StateRootToBlocks = new StateRootToBlockMap(stateRootToBlocks); + BlockCache = maxNumberOfBlocksInCache == 0 + ? null + : new StackQueue<(long, ReadOnlyVerkleMemoryDb)>(maxNumberOfBlocksInCache); + MaxNumberOfBlocksInCache = maxNumberOfBlocksInCache; + InitRootHash(); + } + public ReadOnlyVerkleStateStore AsReadOnly(VerkleMemoryDb keyValueStore) + { + return new ReadOnlyVerkleStateStore(this, keyValueStore); + } + + public void Reset() => BlockCache?.Clear(); + private InternalNode? RootNode => GetInternalNode(RootNodeKey); + + public event EventHandler? ReorgBoundaryReached; + + private void InitRootHash() + { + InternalNode? node = RootNode; + if (node is not null) + { + StateRoot = new VerkleCommitment(node.InternalCommitment.ToBytes()); + LastPersistedBlockNumber = StateRootToBlocks[StateRoot]; + LatestCommittedBlockNumber = -1; + } + else + { + Storage.SetInternalNode(RootNodeKey, new InternalNode(VerkleNodeType.BranchNode)); + StateRoot = VerkleCommitment.Zero; + LastPersistedBlockNumber = LatestCommittedBlockNumber = -1; + } + + // TODO: why should we store using block number - use stateRoot to index everything + // but i think block number is easy to understand and it maintains a sequence + if (LastPersistedBlockNumber == -2) throw new Exception("StateRoot To BlockNumber Cache Corrupted"); + } + + public byte[]? GetLeaf(ReadOnlySpan key) + { + if (BlockCache is not null) + { + using StackQueue<(long, ReadOnlyVerkleMemoryDb)>.StackEnumerator diffs = BlockCache.GetStackEnumerator(); + while (diffs.MoveNext()) + { + if (diffs.Current.Item2.LeafTable.TryGetValue(key.ToArray(), out byte[]? node)) return node; + } + } + + return Storage.GetLeaf(key, out byte[]? value) ? value : null; + } + + public InternalNode? GetInternalNode(ReadOnlySpan key) + { + if (BlockCache is not null) + { + using StackQueue<(long, ReadOnlyVerkleMemoryDb)>.StackEnumerator diffs = BlockCache.GetStackEnumerator(); + while (diffs.MoveNext()) + { + if (diffs.Current.Item2.InternalTable.TryGetValue(key, out InternalNode? node)) return node.Clone(); + } + } + + return Storage.GetInternalNode(key, out InternalNode? value) ? value : null; + } + + +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Prover.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Prover.cs new file mode 100644 index 00000000000..14c19e435da --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Prover.cs @@ -0,0 +1,419 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Data; +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Polynomial; +using Nethermind.Verkle.Proofs; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleTree +{ + private Dictionary ProofBranchPolynomialCache { get; } = new(Bytes.EqualityComparer); + private Dictionary ProofStemPolynomialCache { get; } = new(); + + public ExecutionWitness GenerateExecutionWitnessFromStore(byte[][] keys, out Banderwagon rootPoint) + { + VerkleTree tree = new(_verkleStateStore, LimboLogs.Instance); + return tree.GenerateExecutionWitness(keys, out rootPoint); + } + + public ExecutionWitness GenerateExecutionWitness(byte[][] keys, out Banderwagon rootPoint) + { + if (keys.Length == 0) + { + rootPoint = default; + return new ExecutionWitness(); + } + VerkleProof proof = CreateVerkleProof(keys, out rootPoint); + + SpanDictionary> stemStateDiff = new(Bytes.SpanEqualityComparer); + foreach (byte[] key in keys) + { + SuffixStateDiff suffixData = new() { Suffix = key[31], CurrentValue = Get(key) }; + if (!stemStateDiff.TryGetValue(key.Slice(0, 31), out List? suffixStateDiffList)) + { + suffixStateDiffList = new(); + stemStateDiff.TryAdd(key.Slice(0, 31), suffixStateDiffList); + } + suffixStateDiffList.Add(suffixData); + } + + List stemStateDiffList = stemStateDiff.Select(stemStateDiffData => + new StemStateDiff { Stem = stemStateDiffData.Key, SuffixDiffs = stemStateDiffData.Value }).ToList(); + + return new ExecutionWitness { VerkleProof = proof, StateDiff = stemStateDiffList }; + } + + public VerkleProof CreateVerkleProof(byte[][] keys, out Banderwagon rootPoint) + { + if (keys.Length == 0) + { + rootPoint = default; + return new VerkleProof(); + } + ProofBranchPolynomialCache.Clear(); + ProofStemPolynomialCache.Clear(); + + Dictionary depthsByStem = new(); + Dictionary extStatus = new(); + + // generate prover path for keys + Dictionary> neededOpenings = new(Bytes.EqualityComparer); + HashSet stemList = new(Bytes.EqualityComparer); + + foreach (byte[] key in keys) + { + for (int i = 0; i < 32; i++) + { + byte[] parentPath = key[..i]; + InternalNode? node = GetInternalNode(parentPath); + if (node != null) + { + switch (node.NodeType) + { + case VerkleNodeType.BranchNode: + CreateBranchProofPolynomialIfNotExist(parentPath); + neededOpenings.TryAdd(parentPath, new HashSet()); + neededOpenings[parentPath].Add(key[i]); + continue; + case VerkleNodeType.StemNode: + Stem keyStem = key[..31]; + depthsByStem.TryAdd(keyStem, (byte)i); + CreateStemProofPolynomialIfNotExist(keyStem); + neededOpenings.TryAdd(parentPath, new HashSet()); + stemList.Add(parentPath); + if (keyStem == node.Stem) + { + neededOpenings[parentPath].Add(key[31]); + extStatus.TryAdd(keyStem, ExtPresent.Present); + } + else + { + extStatus.TryAdd(keyStem, ExtPresent.DifferentStem); + } + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + else + { + byte[] keyStem = key[..31]; + extStatus.TryAdd(keyStem, ExtPresent.None); + depthsByStem.TryAdd(keyStem, (byte)(i)); + } + // reaching here means end of the path for the leaf + break; + } + } + + VerkleProof finalProof = CreateProofStruct(stemList, neededOpenings, addLeafOpenings: true, out rootPoint); + finalProof.VerifyHint.Depths = depthsByStem.Values.ToArray(); + finalProof.VerifyHint.ExtensionPresent = extStatus.Values.ToArray(); + + return finalProof; + } + + public VerkleProof CreateVerkleRangeProof(Stem startStem, Stem endStem, out Banderwagon rootPoint) + { + ProofBranchPolynomialCache.Clear(); + ProofStemPolynomialCache.Clear(); + + Dictionary depthsByStem = new(); + ExtPresent[] extStatus = new ExtPresent[2]; + + // generate prover path for keys + Dictionary> neededOpenings = new(Bytes.EqualityComparer); + HashSet stemList = new(Bytes.EqualityComparer); + + int prefixLength = 0; + while (prefixLength()); + if (i < prefixLength) + { + neededOpenings[parentPath].Add(startStem.Bytes[i]); + continue; + } + + int startIndex = startStem.Bytes[i]; + int endIndex = endStem.Bytes[i]; + if (i > prefixLength) + { + if (keyIndex == 0) endIndex = 255; + else startIndex = 0; + } + + for (int j = startIndex; j <= endIndex; j++) + { + neededOpenings[parentPath].Add((byte)j); + } + continue; + case VerkleNodeType.StemNode: + Stem keyStem = stem; + depthsByStem.TryAdd(keyStem, (byte)i); + CreateStemProofPolynomialIfNotExist(keyStem); + neededOpenings.TryAdd(parentPath, new HashSet()); + stemList.Add(parentPath); + if (keyStem == node.Stem) + { + neededOpenings[parentPath].Add(0); + extStatus[keyIndex++] = ExtPresent.Present; + } + else + { + extStatus[keyIndex++] = ExtPresent.DifferentStem; + } + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + else + { + Stem keyStem = stem; + extStatus[keyIndex++] = ExtPresent.None; + depthsByStem.TryAdd(keyStem, (byte)(i)); + } + // reaching here means end of the path for the leaf + break; + } + } + + VerkleProof finalProof = CreateProofStruct(stemList, neededOpenings, addLeafOpenings: false, out rootPoint); + finalProof.VerifyHint.Depths = depthsByStem.Values.ToArray(); + finalProof.VerifyHint.ExtensionPresent = extStatus; + + return finalProof; + } + + + private VerkleProof CreateProofStruct(IReadOnlySet stemList, Dictionary> neededOpenings, bool addLeafOpenings, out Banderwagon rootPoint) + { + + List queries = new(); + HashSet stemWithNoProofSet = new(Bytes.EqualityComparer); + List sortedCommitments = new(); + + foreach (KeyValuePair> elem in neededOpenings) + { + if (stemList.Contains(elem.Key)) + { + InternalNode? suffix = GetInternalNode(elem.Key); + bool stemWithNoProof = AddStemCommitmentsOpenings(suffix, elem.Value, queries, addLeafOpenings); + if (stemWithNoProof) stemWithNoProofSet.Add(suffix.Stem.Bytes); + continue; + } + + AddBranchCommitmentsOpening(elem.Key, elem.Value, queries); + } + + VerkleProverQuery root = queries.First(); + + rootPoint = root.NodeCommitPoint; + foreach (VerkleProverQuery query in queries.Where(query => root.NodeCommitPoint != query.NodeCommitPoint)) + { + if(sortedCommitments.Count == 0 || sortedCommitments[^1] != query.NodeCommitPoint) + sortedCommitments.Add(query.NodeCommitPoint); + } + + MultiProof proofConstructor = new(CRS.Instance, PreComputedWeights.Instance); + + + Transcript proverTranscript = new("vt"); + VerkleProofStruct proof = proofConstructor.MakeMultiProof(proverTranscript, queries); + + return new VerkleProof + { + CommsSorted = sortedCommitments.ToArray(), + Proof = proof, + VerifyHint = new VerificationHint + { + DifferentStemNoProof = stemWithNoProofSet.ToArray(), + } + }; + } + + private void AddBranchCommitmentsOpening(byte[] branchPath, IEnumerable branchChild, List queries) + { + if (!ProofBranchPolynomialCache.TryGetValue(branchPath, out FrE[] poly)) throw new EvaluateException(); + InternalNode? node = GetInternalNode(branchPath); + queries.AddRange(branchChild.Select(childIndex => new VerkleProverQuery(new LagrangeBasis(poly), node!.InternalCommitment.Point, childIndex, poly[childIndex]))); + } + + private bool AddStemCommitmentsOpenings(InternalNode? suffix, HashSet stemChild, List queries, bool addLeafOpenings) + { + byte[] stemPath = suffix!.Stem!.Bytes; + AddExtensionCommitmentOpenings(stemPath, addLeafOpenings? stemChild: new byte[]{}, suffix, queries); + if (stemChild.Count == 0) return true; + + // this is used for sync proofs - we dont need to include proofs for leaf openings as we send all the leafs + // the client can generate the leaf and verify the commitments + if (!addLeafOpenings) return false; + + ProofStemPolynomialCache.TryGetValue(stemPath, out SuffixPoly hashStruct); + + FrE[] c1Hashes = hashStruct.C1; + FrE[] c2Hashes = hashStruct.C2; + + foreach (byte valueIndex in stemChild) + { + int valueLowerIndex = 2 * (valueIndex % 128); + int valueUpperIndex = valueLowerIndex + 1; + + (FrE valueLow, FrE valueHigh) = VerkleUtils.BreakValueInLowHigh(Get(stemPath.Append(valueIndex).ToArray())); + + int offset = valueIndex < 128 ? 0 : 128; + + Banderwagon commitment; + FrE[] poly; + switch (offset) + { + case 0: + commitment = suffix.C1.Point; + poly = c1Hashes.ToArray(); + break; + case 128: + commitment = suffix.C2.Point; + poly = c2Hashes.ToArray(); + break; + default: + throw new Exception("unreachable"); + } + + VerkleProverQuery openAtValLow = new(new LagrangeBasis(poly), commitment, (byte)valueLowerIndex, valueLow); + VerkleProverQuery openAtValUpper = new(new LagrangeBasis(poly), commitment, (byte)valueUpperIndex, valueHigh); + + queries.Add(openAtValLow); + queries.Add(openAtValUpper); + } + + return false; + } + + private static void AddExtensionCommitmentOpenings(Stem stem, IEnumerable value, InternalNode suffix, List queries) + { + FrE[] extPoly = new FrE[256]; + for (int i = 0; i < 256; i++) + { + extPoly[i] = FrE.Zero; + } + extPoly[0] = FrE.One; + extPoly[1] = FrE.FromBytesReduced(stem.Bytes.Reverse().ToArray()); + extPoly[2] = suffix.C1!.PointAsField; + extPoly[3] = suffix.C2!.PointAsField; + + VerkleProverQuery openAtOne = new(new LagrangeBasis(extPoly), suffix.InternalCommitment.Point, 0, FrE.One); + VerkleProverQuery openAtStem = new(new LagrangeBasis(extPoly), suffix.InternalCommitment.Point, 1, FrE.FromBytesReduced(stem.Bytes.Reverse().ToArray())); + queries.Add(openAtOne); + queries.Add(openAtStem); + + bool openC1 = false; + bool openC2 = false; + foreach (byte valueIndex in value) + { + if (valueIndex < 128) openC1 = true; + else openC2 = true; + } + + if (openC1) + { + VerkleProverQuery openAtC1 = new(new LagrangeBasis(extPoly), suffix.InternalCommitment.Point, 2, suffix.C1.PointAsField); + queries.Add(openAtC1); + } + + if (openC2) + { + VerkleProverQuery openAtC2 = new(new LagrangeBasis(extPoly), suffix.InternalCommitment.Point, 3, suffix.C2.PointAsField); + queries.Add(openAtC2); + } + } + + private void BatchCreateBranchProofPolynomialIfNotExist(HashSet paths) + { + Banderwagon[] commitments = new Banderwagon[256 * paths.Count]; + + int commitmentIndex = 0; + + foreach (byte[] path in paths) + { + for (int i = 0; i < 256; i++) + { + InternalNode? node = GetInternalNode(path.Append((byte)i).ToArray()); + commitments[commitmentIndex++] = node == null ? Banderwagon.Identity : node.InternalCommitment.Point; + } + } + + Span scalars = Banderwagon.BatchMapToScalarField(commitments); + + foreach (byte[] path in paths) + { + ProofBranchPolynomialCache[path] = scalars[..256].ToArray(); + scalars = scalars[256..]; + } + } + + private void CreateBranchProofPolynomialIfNotExist(byte[] path) + { + if (ProofBranchPolynomialCache.ContainsKey(path)) return; + Banderwagon[] commitments = new Banderwagon[256]; + for (int i = 0; i < 256; i++) + { + InternalNode? node = GetInternalNode(path.Append((byte)i).ToArray()); + commitments[i] = node == null ? Banderwagon.Identity : node.InternalCommitment.Point; + } + ProofBranchPolynomialCache[path] = Banderwagon.BatchMapToScalarField(commitments); + } + + private void CreateStemProofPolynomialIfNotExist(Stem stem) + { + if (ProofStemPolynomialCache.ContainsKey(stem)) return; + + List c1Hashes = new(256); + List c2Hashes = new(256); + for (int i = 0; i < 128; i++) + { + (FrE valueLow, FrE valueHigh) = VerkleUtils.BreakValueInLowHigh(Get(stem.Bytes.Append((byte)i).ToArray())); + c1Hashes.Add(valueLow); + c1Hashes.Add(valueHigh); + } + + for (int i = 128; i < 256; i++) + { + (FrE valueLow, FrE valueHigh) = VerkleUtils.BreakValueInLowHigh(Get(stem.Bytes.Append((byte)i).ToArray())); + c2Hashes.Add(valueLow); + c2Hashes.Add(valueHigh); + } + ProofStemPolynomialCache[stem] = new SuffixPoly() + { + C1 = c1Hashes.ToArray(), + C2 = c2Hashes.ToArray() + }; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Stateless.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Stateless.cs new file mode 100644 index 00000000000..41d3dce8f11 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Stateless.cs @@ -0,0 +1,409 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Proofs; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleTree +{ + + private void InsertStemBatchStateless(in Stem stem, IEnumerable leafIndexValueMap) + { + InsertStemBatchStateless(stem.BytesAsSpan, leafIndexValueMap); + } + + private void InsertStemBatchStateless(ReadOnlySpan stem, IEnumerable leafIndexValueMap) + { + Span key = new byte[32]; + stem.CopyTo(key); + foreach (LeafInSubTree leaf in leafIndexValueMap) + { + key[31] = leaf.SuffixByte; + SetLeafCache(key.ToArray(), leaf.Leaf); + } + } + + private void InsertBranchNodeForSync(byte[] path, Commitment commitment) + { + InternalNode node = VerkleNodes.CreateStatelessBranchNode(commitment); + SetInternalNode(path, node); + } + + private void InsertSubTreesForSync(PathWithSubTree[] subTrees) + { + Span key = new byte[32]; + foreach (PathWithSubTree subTree in subTrees) + { + subTree.Path.Bytes.CopyTo(key); + LeafUpdateDelta leafUpdateDelta = new(); + foreach (LeafInSubTree leafs in subTree.SubTree) + { + key[31] = leafs.SuffixByte; + leafUpdateDelta.UpdateDelta(GetLeafDelta(leafs.Leaf, leafs.SuffixByte), leafs.SuffixByte); + SetLeafCache(key.ToArray(), leafs.Leaf); + } + _leafUpdateCache[subTree.Path.Bytes] = leafUpdateDelta; + } + } + + private bool VerifyCommitmentThenInsertStem(byte[] pathOfStem, byte[] stem, Commitment expectedCommitment) + { + InternalNode stemNode = VerkleNodes.CreateStatelessStemNode(stem); + stemNode.UpdateCommitment(_leafUpdateCache[stem]); + if (stemNode.InternalCommitment.Point != expectedCommitment.Point) return false; + SetInternalNode(pathOfStem, stemNode); + return true; + } + + private void InsertPlaceholderForNotPresentStem(Span stem, byte[] pathOfStem, Commitment stemCommitment) + { + InternalNode stemNode = VerkleNodes.CreateStatelessStemNode(stem.ToArray(), stemCommitment); + SetInternalNode(pathOfStem, stemNode); + } + + private void InsertStemBatchForSync(Dictionary> stemBatch, + IDictionary, Banderwagon> commByPath) + { + foreach (KeyValuePair> prefixWithStem in stemBatch) + { + foreach (byte[] stem in prefixWithStem.Value) + { + TraverseContext context = new(stem, _leafUpdateCache[stem]) + { + CurrentIndex = prefixWithStem.Key.Length - 1, + ForSync = true + }; + TraverseBranch(context); + } + + commByPath[new List(prefixWithStem.Key)] = GetInternalNode(prefixWithStem.Key)! + .InternalCommitment.Point; + } + } + public bool InsertIntoStatelessTree(VerkleProof proof, List keys, List values, Banderwagon root) + { + bool verification = VerifyVerkleProof(proof, keys, values, root, out UpdateHint? updateHint); + if (!verification) return false; + InsertAfterVerification(updateHint!.Value, keys, values, root, false); + return true; + } + + public void InsertAfterVerification(UpdateHint hint, List keys, List values, Banderwagon root, bool skipRoot = true) + { + if (!skipRoot) + { + InternalNode rootNode = new(VerkleNodeType.BranchNode, new Commitment(root)); + SetInternalNode(Array.Empty(), rootNode); + } + + AddStatelessInternalNodes(hint); + + for (int i = 0; i < keys.Count; i++) + { + byte[]? value = values[i]; + if(value is null) continue; + SetLeafCache(keys[i], value); + } + } + + public bool InsertIntoStatelessTree(ExecutionWitness? execWitness, Banderwagon root, bool skipRoot = false) + { + if (execWitness is null || execWitness.VerkleProof is null) + { + if (!skipRoot) + { + InternalNode rootNode = new(VerkleNodeType.BranchNode, new Commitment(root)); + SetInternalNode(Array.Empty(), rootNode); + } + + return true; + } + + bool isVerified = VerifyVerkleProof(execWitness, root, out UpdateHint? updateHint); + if (!isVerified) return false; + + if (!skipRoot) + { + InternalNode rootNode = new(VerkleNodeType.BranchNode, new Commitment(root)); + SetInternalNode(Array.Empty(), rootNode); + } + + AddStatelessInternalNodes(updateHint.Value); + + foreach (StemStateDiff stemStateDiff in execWitness.StateDiff) + { + InsertStemBatchStateless(stemStateDiff.Stem, + stemStateDiff.SuffixDiffs.Select(x => new LeafInSubTree(x.Suffix, x.CurrentValue))); + } + + CommitTree(0); + return true; + + } + + private void AddStatelessInternalNodes(UpdateHint hint) + { + List pathList = new(); + foreach ((Stem stem, (ExtPresent extStatus, byte depth)) in hint.DepthAndExtByStem) + { + pathList.Clear(); + for (int i = 0; i < depth - 1; i++) + { + pathList.Add(stem.Bytes[i]); + InternalNode node = VerkleNodes.CreateStatelessBranchNode(new Commitment(hint.CommByPath[pathList])); + SetInternalNode(pathList.ToArray(), node); + } + + pathList.Add(stem.Bytes[depth-1]); + + InternalNode stemNode; + byte[] pathOfStem; + switch (extStatus) + { + case ExtPresent.None: + stemNode = VerkleNodes.CreateStatelessStemNode(stem, new Commitment(), new Commitment(), new Commitment()); + pathOfStem = pathList.ToArray(); + break; + case ExtPresent.DifferentStem: + Stem otherStem = hint.DifferentStemNoProof[pathList]; + Commitment otherInternalCommitment = new(hint.CommByPath[pathList]); + stemNode = VerkleNodes.CreateStatelessStemNode(otherStem, otherInternalCommitment); + pathOfStem = pathList.ToArray(); + break; + case ExtPresent.Present: + Commitment internalCommitment = new(hint.CommByPath[pathList]); + Commitment? c1 = null; + Commitment? c2 = null; + + pathList.Add(2); + if (hint.CommByPath.TryGetValue(pathList, out Banderwagon c1B)) c1 = new Commitment(c1B); + pathList[^1] = 3; + if (hint.CommByPath.TryGetValue(pathList, out Banderwagon c2B)) c2 = new Commitment(c2B); + + stemNode = VerkleNodes.CreateStatelessStemNode(stem, c1, c2, internalCommitment); + pathOfStem = new byte[pathList.Count - 1]; + pathList.CopyTo(0, pathOfStem, 0, pathList.Count - 1); + break; + default: + throw new ArgumentOutOfRangeException(); + } + + SetInternalNode(pathOfStem, stemNode, extStatus != ExtPresent.DifferentStem); + } + } + + public bool CreateStatelessTreeFromRange(VerkleProof proof, Banderwagon rootPoint, Stem startStem, Stem endStem, PathWithSubTree[] subTrees) + { + const int numberOfStems = 2; + List commSortedByPath = new(proof.CommsSorted.Length + 1) { rootPoint }; + commSortedByPath.AddRange(proof.CommsSorted); + + Stem[] stems = { startStem, endStem }; + + // map stems to depth and extension status and create a list of stem with extension present + Dictionary depthsAndExtByStem = new(Bytes.EqualityComparer); + HashSet stemsWithExtension = new(Bytes.EqualityComparer); + for (int i = 0; i < numberOfStems; i++) + { + ExtPresent extPresent = proof.VerifyHint.ExtensionPresent[i]; + depthsAndExtByStem.Add(stems[i].Bytes, (extPresent, proof.VerifyHint.Depths[i])); + if (extPresent == ExtPresent.Present) stemsWithExtension.Add(stems[i].Bytes); + } + + SortedSet> allPaths = new(new ListComparer()); + SortedSet<(List, byte)> allPathsAndZs = new(new ListWithByteComparer()); + Dictionary<(List, byte), FrE> leafValuesByPathAndZ = new(new ListWithByteEqualityComparer()); + SortedDictionary, byte[]> otherStemsByPrefix = new(new ListComparer()); + + int prefixLength = 0; + while (prefixLength < Stem.Size) + { + if (startStem.Bytes[prefixLength] != endStem.Bytes[prefixLength]) break; + prefixLength++; + } + + int keyIndex = 0; + foreach (Stem stem in stems) + { + (ExtPresent extPres, byte depth) = depthsAndExtByStem[stem.Bytes]; + + for (int i = 0; i < depth; i++) + { + allPaths.Add(new List(stem.Bytes[..i])); + if (i < prefixLength) + { + allPathsAndZs.Add((new List(stem.Bytes[..i]), stem.Bytes[i])); + continue; + } + int startIndex = startStem.Bytes[i]; + int endIndex = endStem.Bytes[i]; + if (i > prefixLength) + { + if (keyIndex == 0) endIndex = 255; + else startIndex = 0; + } + + for (int j = startIndex; j <= endIndex; j++) + { + allPathsAndZs.Add((new List(stem.Bytes[..i]), (byte)j)); + } + } + + switch (extPres) + { + case ExtPresent.DifferentStem: + + allPaths.Add(new List(stem.Bytes[..depth])); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 0)); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 1)); + + byte[] otherStem; + + // find the stems that are equal to the stem we are assuming to be without extension + // this happens when we initially added this stem when we were searching for another one + // but then in a future key, we found that we needed this stem too. + byte[][] found = stemsWithExtension.Where(x => x[..depth].SequenceEqual(stem.Bytes[..depth])).ToArray(); + + switch (found.Length) + { + case 0: + found = proof.VerifyHint.DifferentStemNoProof.Where(x => x[..depth].SequenceEqual(stem.Bytes[..depth])).ToArray(); + byte[] encounteredStem = found[^1]; + otherStem = encounteredStem; + + // Add extension node to proof in particular, we only want to open at (1, stem) + leafValuesByPathAndZ[(new List(stem.Bytes[..depth]), 0)] = FrE.One; + leafValuesByPathAndZ.Add((new List(stem.Bytes[..depth]), 1), FrE.FromBytesReduced(encounteredStem.Reverse().ToArray())); + break; + case 1: + otherStem = found[0]; + break; + default: + throw new InvalidDataException($"found more than one instance of stem_with_extension at depth {depth}, see: {string.Join(" | ", found.Select(x => string.Join(", ", x)))}"); + } + + otherStemsByPrefix.Add(stem.Bytes[..depth].ToList(), otherStem); + break; + case ExtPresent.Present: + allPaths.Add(new List(stem.Bytes[..depth])); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 0)); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 1)); + + leafValuesByPathAndZ[(new List(stem.Bytes[..depth]), 0)] = FrE.One; + leafValuesByPathAndZ[(new List(stem.Bytes[..depth]), 1)] = FrE.FromBytesReduced(stem.Bytes.Reverse().ToArray()); + break; + case ExtPresent.None: + leafValuesByPathAndZ[depth == 1 ? (new List(), stem.Bytes[depth - 1]) : (stem.Bytes[..depth].ToList(), stem.Bytes[depth - 1])] = FrE.Zero; + break; + default: + throw new ArgumentOutOfRangeException(); + } + keyIndex++; + } + + Dictionary, Banderwagon> commByPath = new(new ListEqualityComparer()); + foreach ((List path, Banderwagon comm) in allPaths.Zip(commSortedByPath)) + { + commByPath[path] = comm; + } + + HashSet subTreesToCreate = UpdatePathsAndReturnSubTreesToCreate(allPaths, allPathsAndZs, subTrees, startStem.BytesAsSpan, endStem.BytesAsSpan); + InsertSubTreesForSync(subTrees); + + List pathList = new(); + InsertBranchNodeForSync(pathList.ToArray(), new Commitment(commByPath[pathList])); + foreach ((byte[]? stem, (ExtPresent extStatus, byte depth)) in depthsAndExtByStem) + { + pathList.Clear(); + for (int i = 0; i < depth - 1; i++) + { + pathList.Add(stem[i]); + InsertBranchNodeForSync(pathList.ToArray(), new Commitment(commByPath[pathList])); + } + + pathList.Add(stem[depth-1]); + + switch (extStatus) + { + case ExtPresent.None: + InsertPlaceholderForNotPresentStem(stem, pathList.ToArray(), new Commitment()); + break; + case ExtPresent.DifferentStem: + byte[] otherStem = otherStemsByPrefix[pathList]; + InsertPlaceholderForNotPresentStem(otherStem, pathList.ToArray(), new(commByPath[pathList])); + break; + case ExtPresent.Present: + Commitment internalCommitment = new(commByPath[pathList]); + if (!VerifyCommitmentThenInsertStem(pathList.ToArray(), stem, internalCommitment)) + return false; + break; + default: + throw new ArgumentOutOfRangeException(); + } + + } + + byte[][] allStemsWithoutStartAndEndStems = + subTrees.Where(x => !x.Path.Bytes.SequenceEqual(startStem.Bytes) && !x.Path.Bytes.SequenceEqual(endStem.Bytes)) + .Select(x => x.Path.Bytes).ToArray(); + + int stemIndex = 0; + Dictionary> stemBatch = new(Bytes.EqualityComparer); + foreach (byte[] stemPrefix in subTreesToCreate) + { + stemBatch.Add(stemPrefix, new List()); + while (stemIndex < allStemsWithoutStartAndEndStems.Length) + { + if (Bytes.EqualityComparer.Equals(stemPrefix, allStemsWithoutStartAndEndStems[stemIndex][..stemPrefix.Length])) + { + stemBatch[stemPrefix].Add(allStemsWithoutStartAndEndStems[stemIndex]); + stemIndex++; + } + else break; + } + } + + InsertStemBatchForSync(stemBatch, commByPath); + bool verification = VerifyVerkleProofStruct(proof.Proof, allPathsAndZs, leafValuesByPathAndZ, commByPath); + if (!verification) Reset(); + else CommitTree(0); + + return verification; + } + + private static HashSet UpdatePathsAndReturnSubTreesToCreate(IReadOnlySet> allPaths, + ISet<(List, byte)> allPathsAndZs, PathWithSubTree[] stems, ReadOnlySpan startStem, ReadOnlySpan endStem) + { + ISpanEqualityComparer comparer = Bytes.SpanEqualityComparer; + HashSet subTreesToCreate = new(Bytes.EqualityComparer); + foreach (PathWithSubTree subTree in stems) + { + if(comparer.Equals(subTree.Path.Bytes, startStem)) continue; + if(comparer.Equals(subTree.Path.Bytes, endStem)) continue; + for (int i = 0; i < 32; i++) + { + List prefix = new(subTree.Path.Bytes[..i]); + if (allPaths.Contains(prefix)) + { + allPathsAndZs.Add((prefix, subTree.Path.Bytes[i])); + } + else + { + subTreesToCreate.Add(prefix.ToArray()); + break; + } + } + } + + return subTreesToCreate; + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Traverse.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Traverse.cs new file mode 100644 index 00000000000..91464aef7ba --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Traverse.cs @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using Nethermind.Core.Extensions; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Utils; + + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleTree +{ + private void UpdateTreeCommitments(Span stem, LeafUpdateDelta leafUpdateDelta, bool forSync = false) + { + TraverseContext context = new(stem, leafUpdateDelta) { ForSync = forSync }; + Banderwagon rootDelta = TraverseBranch(context); + if(_logger.IsTrace) _logger.Trace($"RootDelta To Apply: {rootDelta.ToBytes().ToHexString()}"); + UpdateRootNode(rootDelta); + } + + private Banderwagon TraverseBranch(TraverseContext traverseContext) + { + byte childIndex = traverseContext.Stem[traverseContext.CurrentIndex]; + byte[] absolutePath = traverseContext.Stem[..(traverseContext.CurrentIndex + 1)].ToArray(); + + InternalNode? child = GetInternalNode(absolutePath); + if (child is null || (child.IsStem && child.InternalCommitment.Point.Equals(new Banderwagon()))) + { + if(_logger.IsTrace) _logger.Trace($"Create New Stem Node"); + // 1. create new suffix node + // 2. update the C1 or C2 - we already know the leafDelta - traverseContext.LeafUpdateDelta + // 3. update ExtensionCommitment + // 4. get the delta for commitment - ExtensionCommitment - 0; + InternalNode stem = new(VerkleNodeType.StemNode, traverseContext.Stem.ToArray()); + FrE deltaFr = stem.UpdateCommitment(traverseContext.LeafUpdateDelta); + FrE deltaHash = deltaFr + stem.InitCommitmentHash!.Value; + + // 1. Add internal.stem node + // 2. return delta from ExtensionCommitment + SetInternalNode(absolutePath, stem); + return Committer.ScalarMul(deltaHash, childIndex); + } + + if (child.IsBranchNode) + { + traverseContext.CurrentIndex += 1; + Banderwagon branchDeltaHash = TraverseBranch(traverseContext); + traverseContext.CurrentIndex -= 1; + if(_logger.IsTrace) _logger.Trace($"TraverseBranch Delta:{branchDeltaHash.ToBytes().ToHexString()}"); + + FrE deltaHash = child.UpdateCommitment(branchDeltaHash); + SetInternalNode(absolutePath, child); + + return Committer.ScalarMul(deltaHash, childIndex); + } + + traverseContext.CurrentIndex += 1; + (Banderwagon stemDeltaHash, bool changeStemToBranch) = TraverseStem(child, traverseContext); + traverseContext.CurrentIndex -= 1; + if(_logger.IsTrace) _logger.Trace($"TraverseStem Delta:{stemDeltaHash.ToBytes().ToHexString()}"); + + if (changeStemToBranch) + { + InternalNode newChild = new(VerkleNodeType.BranchNode); + newChild.InternalCommitment.AddPoint(child.InternalCommitment.Point); + // since this is a new child, this would be just the parentDeltaHash.PointToField + // now since there was a node before and that value is deleted - we need to subtract + // that from the delta as well + FrE deltaHash = newChild.UpdateCommitment(stemDeltaHash); + SetInternalNode(absolutePath, newChild); + return Committer.ScalarMul(deltaHash, childIndex); + } + // in case of stem, no need to update the child commitment - because this commitment is the suffix commitment + // pass on the update to upper level + return stemDeltaHash; + } + + private (Banderwagon, bool) TraverseStem(InternalNode node, TraverseContext traverseContext) + { + Debug.Assert(node.IsStem); + + (List sharedPath, byte? pathDiffIndexOld, byte? pathDiffIndexNew) = + VerkleUtils.GetPathDifference(node.Stem!.Bytes, traverseContext.Stem.ToArray()); + + if (sharedPath.Count != 31) + { + int relativePathLength = sharedPath.Count - traverseContext.CurrentIndex; + // byte[] relativeSharedPath = sharedPath.ToArray()[traverseContext.CurrentIndex..].ToArray(); + byte oldLeafIndex = pathDiffIndexOld ?? throw new ArgumentException(); + byte newLeafIndex = pathDiffIndexNew ?? throw new ArgumentException(); + // node share a path but not the complete stem. + + // the internal node will be denoted by their sharedPath + // 1. create SuffixNode for the traverseContext.Key - get the delta of the commitment + // 2. set this suffix as child node of the BranchNode - get the commitment point + // 3. set the existing suffix as the child - get the commitment point + // 4. update the internal node with the two commitment points + InternalNode newStem = new InternalNode(VerkleNodeType.StemNode, traverseContext.Stem.ToArray()); + FrE deltaFrNewStem = newStem.UpdateCommitment(traverseContext.LeafUpdateDelta); + FrE deltaHashNewStem = deltaFrNewStem + newStem.InitCommitmentHash!.Value; + + // creating the stem node for the new suffix node + byte[] stemKey = new byte[sharedPath.Count + 1]; + sharedPath.CopyTo(stemKey); + stemKey[^1] = newLeafIndex; + SetInternalNode(stemKey, newStem); + Banderwagon newSuffixCommitmentDelta = Committer.ScalarMul(deltaHashNewStem, newLeafIndex); + + stemKey = new byte[sharedPath.Count + 1]; + sharedPath.CopyTo(stemKey); + stemKey[^1] = oldLeafIndex; + SetInternalNode(stemKey, node); + + Banderwagon oldSuffixCommitmentDelta = + Committer.ScalarMul(node.InternalCommitment.PointAsField, oldLeafIndex); + + Banderwagon deltaCommitment = oldSuffixCommitmentDelta + newSuffixCommitmentDelta; + + Banderwagon internalCommitment = FillSpaceWithBranchNodes(sharedPath.ToArray(), relativePathLength, deltaCommitment); + + return (internalCommitment - node.InternalCommitment.Point, true); + } + + byte[] absolutePath = traverseContext.Stem[..traverseContext.CurrentIndex].ToArray(); + byte childIndex = traverseContext.Stem[traverseContext.CurrentIndex - 1]; + if (traverseContext.ForSync) + { + // 1. create new suffix node + // 2. update the C1 or C2 - we already know the leafDelta - traverseContext.LeafUpdateDelta + // 3. update ExtensionCommitment + // 4. get the delta for commitment - ExtensionCommitment - 0; + InternalNode stem = new (VerkleNodeType.StemNode, traverseContext.Stem.ToArray()); + FrE deltaFr = stem.UpdateCommitment(traverseContext.LeafUpdateDelta); + FrE deltaHash = deltaFr + stem.InitCommitmentHash!.Value; + + // 1. Add internal.stem node + // 2. return delta from ExtensionCommitment + SetInternalNode(absolutePath, stem); + return (Committer.ScalarMul(deltaHash, childIndex), false); + } + else + { + InternalNode updatedStemNode = node.Clone(); + FrE deltaFr = updatedStemNode.UpdateCommitment(traverseContext.LeafUpdateDelta); + SetInternalNode(absolutePath, updatedStemNode); + return (Committer.ScalarMul(deltaFr, childIndex), false); + } + } + + private Banderwagon FillSpaceWithBranchNodes(byte[] path, int length, Banderwagon deltaPoint) + { + for (int i = 0; i < length; i++) + { + InternalNode newInternalNode = new(VerkleNodeType.BranchNode); + FrE upwardsDelta = newInternalNode.UpdateCommitment(deltaPoint); + SetInternalNode(path[..^i], newInternalNode); + deltaPoint = Committer.ScalarMul(upwardsDelta, path[path.Length - i - 1]); + } + + return deltaPoint; + } + + public ref struct TraverseContext + { + public LeafUpdateDelta LeafUpdateDelta { get; } + public bool ForSync { get; set; } + public Span Stem { get; } + public int CurrentIndex { get; set; } + + public TraverseContext(Span stem, LeafUpdateDelta delta) + { + Stem = stem; + CurrentIndex = 0; + LeafUpdateDelta = delta; + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Verifier.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Verifier.cs new file mode 100644 index 00000000000..599bf4362a3 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Verifier.cs @@ -0,0 +1,374 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using Nethermind.Core.Verkle; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Polynomial; +using Nethermind.Verkle.Proofs; +using Nethermind.Verkle.Tree.Proofs; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleTree +{ + private static Stem[] GetStemsFromKeys(Span keys, int numberOfStems) + { + Stem[] stems = new Stem[numberOfStems]; + stems[0] = keys[0][..31]; + + int stemIndex = 1; + for (int i = 1; i < keys.Length; i++) + { + byte[] currentStem = keys[i][..31]; + if (stems[stemIndex - 1].Equals(currentStem)) continue; + stems[stemIndex++] = currentStem; + } + return stems; + } + + private static Stem[] GetStemsFromStemStateDiff(List suffixDiffs) + { + Stem[] stems = new Stem[suffixDiffs.Count]; + for (int i = 0; i < suffixDiffs.Count; i++) + { + stems[i] = suffixDiffs[i].Stem; + } + return stems; + } + + public static bool VerifyVerkleProof(ExecutionWitness execWitness, Banderwagon root, [NotNullWhen(true)]out UpdateHint? updateHint) + { + // var logg = SimpleConsoleLogger.Instance; + updateHint = null; + int numberOfStems = execWitness.VerkleProof.DepthExtensionPresent.Length; + + // sorted commitments including root + Banderwagon[] commSortedByPath = new Banderwagon[execWitness.VerkleProof.CommitmentsByPath.Length + 1]; + commSortedByPath[0] = root; + execWitness.VerkleProof.CommitmentsByPath.CopyTo(commSortedByPath, 1); + + Stem[] stems = GetStemsFromStemStateDiff(execWitness.StateDiff); + + Dictionary depthsAndExtByStem = new(); + HashSet stemsWithExtension = new(); + for (int i = 0; i < numberOfStems; i++) + { + byte extAndDepth = execWitness.VerkleProof.DepthExtensionPresent[i]; + byte depth = (byte)(extAndDepth >> 3); + ExtPresent extPresent = (ExtPresent)(extAndDepth & 3); + depthsAndExtByStem.Add(stems[i], (extPresent, depth)); + if (extPresent == ExtPresent.Present) stemsWithExtension.Add(stems[i]); + } + + SortedSet> allPaths = new(new ListComparer()); + SortedSet<(List, byte)> allPathsAndZs = new(new ListWithByteComparer()); + Dictionary<(List, byte), FrE> leafValuesByPathAndZ = new(new ListWithByteEqualityComparer()); + SortedDictionary, Stem> otherStemsByPrefix = new(new ListComparer()); + + foreach (StemStateDiff stemStateDiff in execWitness.StateDiff) + { + Stem stem = stemStateDiff.Stem; + (ExtPresent extPres, byte depth) = depthsAndExtByStem[stem]; + + for (int i = 0; i < depth; i++) + { + allPaths.Add(new List(stem.Bytes[..i])); + allPathsAndZs.Add((new List(stem.Bytes[..i]), stem.Bytes[i])); + } + + switch (extPres) + { + case ExtPresent.DifferentStem: + allPaths.Add(new List(stem.Bytes[..depth])); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 0)); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 1)); + + Stem otherStem; + + // find the stems that are equal to the stem we are assuming to be without extension + // this happens when we initially added this stem when we were searching for another one + // but then in a future key, we found that we needed this stem too. + Stem[] found = stemsWithExtension.Where(x => x.BytesAsSpan[..depth].SequenceEqual(stem.Bytes[..depth])).ToArray(); + + switch (found.Length) + { + case 0: + found = execWitness.VerkleProof.OtherStems.Where(x => x.BytesAsSpan[..depth].SequenceEqual(stem.Bytes[..depth])).ToArray(); + Stem encounteredStem = found[^1]; + otherStem = encounteredStem; + + // Add extension node to proof in particular, we only want to open at (1, stem) + leafValuesByPathAndZ[(new List(stem.Bytes[..depth]), 0)] = FrE.One; + leafValuesByPathAndZ.TryAdd((new List(stem.Bytes[..depth]), 1), FrE.FromBytesReduced(encounteredStem.Bytes.Reverse().ToArray())); + break; + case 1: + otherStem = found[0]; + break; + default: + throw new InvalidDataException($"found more than one instance of stem_with_extension at depth {depth}, see: {string.Join(" | ", found.Select(x => string.Join(", ", x)))}"); + } + + otherStemsByPrefix.TryAdd(stem.Bytes[..depth].ToList(), otherStem); + + break; + case ExtPresent.Present: + allPaths.Add(new List(stem.Bytes[..depth])); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 0)); + allPathsAndZs.Add((new List(stem.Bytes[..depth]), 1)); + + leafValuesByPathAndZ[(new List(stem.Bytes[..depth]), 0)] = FrE.One; + leafValuesByPathAndZ[(new List(stem.Bytes[..depth]), 1)] = FrE.FromBytesReduced(stem.Bytes.Reverse().ToArray()); + + foreach (SuffixStateDiff suffixDiff in stemStateDiff.SuffixDiffs) + { + byte suffix = suffixDiff.Suffix; + byte openingIndex = suffix < 128 ? (byte)2 : (byte)3; + + allPathsAndZs.Add((new List(stem.Bytes[..depth]), openingIndex)); + + + // this should definitely be the stem + openingIndex, but the path is just used for sorting + // and indexing the values - this is directly never used for verification + // so it is a good idea to used values as small as possible without the issues of collision + List suffixTreePath = new(stem.Bytes[..depth]) { openingIndex }; + + allPaths.Add(new List(suffixTreePath.ToArray())); + byte valLowerIndex = (byte)(2 * (suffix % 128)); + byte valUpperIndex = (byte)(valLowerIndex + 1); + + allPathsAndZs.Add((new List(suffixTreePath.ToArray()), valLowerIndex)); + allPathsAndZs.Add((new List(suffixTreePath.ToArray()), valUpperIndex)); + + (FrE valLow, FrE valHigh) = VerkleUtils.BreakValueInLowHigh(suffixDiff.CurrentValue); + + leafValuesByPathAndZ[(new List(suffixTreePath.ToArray()), valLowerIndex)] = valLow; + leafValuesByPathAndZ[(new List(suffixTreePath.ToArray()), valUpperIndex)] = valHigh; + } + break; + case ExtPresent.None: + leafValuesByPathAndZ[ + depth == 1 + ? (new List(), stem.Bytes[depth - 1]) + : (stem.Bytes[..depth].ToList(), stem.Bytes[depth - 1])] = FrE.Zero; + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + + Dictionary, Banderwagon> commByPath = new(new ListEqualityComparer()); + foreach ((List path, Banderwagon comm) in allPaths.Zip(commSortedByPath)) + { + commByPath[path] = comm; + } + + bool isTrue = VerifyVerkleProofStruct(new VerkleProofStruct(execWitness.VerkleProof.IpaProof, execWitness.VerkleProof.D), + allPathsAndZs, leafValuesByPathAndZ, commByPath); + + updateHint = new UpdateHint + { + DepthAndExtByStem = depthsAndExtByStem, + CommByPath = commByPath, + DifferentStemNoProof = otherStemsByPrefix + }; + + return isTrue; + } + + + public static bool VerifyVerkleProof( + IpaProofStruct ipaProof, + Banderwagon d, + Banderwagon[] commsSorted, + byte[] depths, + ExtPresent[] extensionPresent, + Stem[] differentStemNoProof, + List keys, + List values, + Banderwagon root, + [NotNullWhen(true)]out UpdateHint? updateHint) + { + updateHint = null; + + int numberOfStems = depths.Length; + + // sorted commitments including root + List commSortedByPath = new(commsSorted.Length + 1) { root }; + commSortedByPath.AddRange(commsSorted); + + Stem[] stems = GetStemsFromKeys(CollectionsMarshal.AsSpan(keys), numberOfStems); + + // map stems to depth and extension status and create a list of stem with extension present + Dictionary depthsAndExtByStem = new(); + HashSet stemsWithExtension = new(); + for (int i = 0; i < numberOfStems; i++) + { + ExtPresent extPresent = extensionPresent[i]; + depthsAndExtByStem.Add(stems[i], (extPresent, depths[i])); + if (extPresent == ExtPresent.Present) stemsWithExtension.Add(stems[i]); + } + + SortedSet> allPaths = new(new ListComparer()); + SortedSet<(List, byte)> allPathsAndZs = new(new ListWithByteComparer()); + Dictionary<(List, byte), FrE> leafValuesByPathAndZ = new(new ListWithByteEqualityComparer()); + SortedDictionary, Stem> otherStemsByPrefix = new(new ListComparer()); + foreach ((byte[] key, byte[]? value) in keys.Zip(values)) + { + byte[] stem = key[..31]; + (ExtPresent extPres, byte depth) = depthsAndExtByStem[stem]; + + for (int i = 0; i < depth; i++) + { + allPaths.Add(new List(stem[..i])); + allPathsAndZs.Add((new List(stem[..i]), stem[i])); + } + + switch (extPres) + { + case ExtPresent.DifferentStem: + + allPaths.Add(new List(stem[..depth])); + allPathsAndZs.Add((new List(stem[..depth]), 0)); + allPathsAndZs.Add((new List(stem[..depth]), 1)); + + // since the stem was different - value should not have been set + if (value != null) return false; + + Debug.Assert(depth != stem.Length); + + Stem otherStem; + + // find the stems that are equal to the stem we are assuming to be without extension + // this happens when we initially added this stem when we were searching for another one + // but then in a future key, we found that we needed this stem too. + Stem[] found = stemsWithExtension.Where(x => x.BytesAsSpan[..depth].SequenceEqual(stem[..depth])).ToArray(); + + switch (found.Length) + { + case 0: + found = differentStemNoProof.Where(x => x.BytesAsSpan[..depth].SequenceEqual(stem[..depth])).ToArray(); + Stem encounteredStem = found[^1]; + otherStem = encounteredStem; + + // Add extension node to proof in particular, we only want to open at (1, stem) + leafValuesByPathAndZ[(new List(stem[..depth]), 0)] = FrE.One; + leafValuesByPathAndZ.Add((new List(stem[..depth]), 1), FrE.FromBytesReduced(encounteredStem.Bytes.Reverse().ToArray())); + break; + case 1: + otherStem = found[0]; + break; + default: + throw new InvalidDataException($"found more than one instance of stem_with_extension at depth {depth}, see: {string.Join(" | ", found.Select(x => string.Join(", ", x)))}"); + } + + otherStemsByPrefix.Add(stem[..depth].ToList(), otherStem); + break; + case ExtPresent.Present: + allPaths.Add(new List(stem[..depth])); + allPathsAndZs.Add((new List(stem[..depth]), 0)); + allPathsAndZs.Add((new List(stem[..depth]), 1)); + + leafValuesByPathAndZ[(new List(stem[..depth]), 0)] = FrE.One; + leafValuesByPathAndZ[(new List(stem[..depth]), 1)] = FrE.FromBytesReduced(stem.Reverse().ToArray()); + + byte suffix = key[31]; + byte openingIndex = suffix < 128 ? (byte)2 : (byte)3; + + allPathsAndZs.Add((new List(stem[..depth]), openingIndex)); + + + // this should definitely be the stem + openingIndex, but the path is just used for sorting + // and indexing the values - this is directly never used for verification + // so it is a good idea to used values as small as possible without the issues of collision + List suffixTreePath = new(stem[..depth]) { openingIndex }; + + allPaths.Add(new List(suffixTreePath.ToArray())); + byte valLowerIndex = (byte)(2 * (suffix % 128)); + byte valUpperIndex = (byte)(valLowerIndex + 1); + + allPathsAndZs.Add((new List(suffixTreePath.ToArray()), valLowerIndex)); + allPathsAndZs.Add((new List(suffixTreePath.ToArray()), valUpperIndex)); + + (FrE valLow, FrE valHigh) = VerkleUtils.BreakValueInLowHigh(value); + + leafValuesByPathAndZ[(new List(suffixTreePath.ToArray()), valLowerIndex)] = valLow; + leafValuesByPathAndZ[(new List(suffixTreePath.ToArray()), valUpperIndex)] = valHigh; + break; + case ExtPresent.None: + // If the extension was not present, then the value should be None + if (value != null) return false; + + leafValuesByPathAndZ[depth == 1 ? (new List(), stem[depth - 1]) : (stem[..depth].ToList(), stem[depth - 1])] = FrE.Zero; + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + + Dictionary, Banderwagon> commByPath = new(new ListEqualityComparer()); + foreach ((List path, Banderwagon comm) in allPaths.Zip(commSortedByPath)) + { + commByPath[path] = comm; + } + + bool isTrue = VerifyVerkleProofStruct(new VerkleProofStruct(ipaProof, d), allPathsAndZs, leafValuesByPathAndZ, commByPath); + updateHint = new UpdateHint + { + DepthAndExtByStem = depthsAndExtByStem, + CommByPath = commByPath, + DifferentStemNoProof = otherStemsByPrefix + }; + + return isTrue; + } + + public static bool VerifyVerkleProof(VerkleProof proof, List keys, List values, Banderwagon root, [NotNullWhen(true)]out UpdateHint? updateHint) + { + return VerifyVerkleProof(proof.Proof.IpaProof, proof.Proof.D, proof.CommsSorted, proof.VerifyHint.Depths, + proof.VerifyHint.ExtensionPresent, proof.VerifyHint.DifferentStemNoProof.Select(x => new Stem(x)).ToArray(), + keys, values, root, + out updateHint); + } + + private static bool VerifyVerkleProofStruct(VerkleProofStruct proof, SortedSet<(List, byte)> allPathsAndZs, Dictionary<(List, byte), FrE> leafValuesByPathAndZ, Dictionary, Banderwagon> commByPath) + { + Banderwagon[] comms = new Banderwagon[allPathsAndZs.Count]; + int index = 0; + foreach ((List path, byte z) in allPathsAndZs) + { + comms[index++] = commByPath[path]; + } + + SortedDictionary<(List, byte), FrE> ysByPathAndZ = new(new ListWithByteComparer()); + foreach ((List path, byte z) in allPathsAndZs) + { + List childPath = new(path.ToArray()) { z }; + + if (!leafValuesByPathAndZ.TryGetValue((path, z), out FrE y)) + { + y = !commByPath.TryGetValue(childPath, out Banderwagon yPoint) ? FrE.Zero : yPoint.MapToScalarField(); + } + ysByPathAndZ.Add((new List(path.ToArray()), z), y); + } + + IEnumerable zs = allPathsAndZs.Select(elem => elem.Item2); + SortedDictionary<(List, byte), FrE>.ValueCollection ys = ysByPathAndZ.Values; + + List queries = new(comms.Length); + + foreach (((FrE y, byte z), Banderwagon comm) in ys.Zip(zs).Zip(comms)) + { + VerkleVerifierQuery query = new(comm, z, y); + queries.Add(query); + } + + Transcript proverTranscript = new("vt"); + MultiProof proofVerifier = new(CRS.Instance, PreComputedWeights.Instance); + + return proofVerifier.CheckMultiProof(proverTranscript, queries.ToArray(), proof); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Visitor.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Visitor.cs new file mode 100644 index 00000000000..a5890ceefa7 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.Visitor.cs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Core.Verkle; +using Nethermind.Trie; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleTree +{ + public void Accept(ITreeVisitor visitor, Keccak rootHash, VisitingOptions? visitingOptions = null) + { + if (visitor is null) throw new ArgumentNullException(nameof(visitor)); + if (rootHash is null) throw new ArgumentNullException(nameof(rootHash)); + visitingOptions ??= VisitingOptions.Default; + + using TrieVisitContext trieVisitContext = new TrieVisitContext() + { + // hacky but other solutions are not much better, something nicer would require a bit of thinking + // we introduced a notion of an account on the visit context level which should have no knowledge of account really + // but we know that we have multiple optimizations and assumptions on trees + ExpectAccounts = visitingOptions.ExpectAccounts, + MaxDegreeOfParallelism = visitingOptions.MaxDegreeOfParallelism, + KeepTrackOfAbsolutePath = true + }; + + if (!rootHash.Equals(Keccak.EmptyTreeHash)) + { + _verkleStateStore.MoveToStateRoot(new VerkleCommitment(rootHash.Bytes)); + } + else + { + return; + } + + if (visitor is RootCheckVisitor) + { + if (!rootHash.Bytes.SequenceEqual(_verkleStateStore.GetStateRoot().Bytes)) visitor.VisitMissingNode(Keccak.Zero, trieVisitContext); + } + else + { + throw new Exception(); + } + + } + + public void Accept(IVerkleTreeVisitor visitor, VerkleCommitment rootHash, VisitingOptions? visitingOptions = null) + { + if (visitor is null) throw new ArgumentNullException(nameof(visitor)); + if (rootHash is null) throw new ArgumentNullException(nameof(rootHash)); + visitingOptions ??= VisitingOptions.Default; + + using TrieVisitContext trieVisitContext = new TrieVisitContext + { + // hacky but other solutions are not much better, something nicer would require a bit of thinking + // we introduced a notion of an account on the visit context level which should have no knowledge of account really + // but we know that we have multiple optimizations and assumptions on trees + ExpectAccounts = visitingOptions.ExpectAccounts, + MaxDegreeOfParallelism = visitingOptions.MaxDegreeOfParallelism, + KeepTrackOfAbsolutePath = true + }; + + if (!rootHash.Equals(new VerkleCommitment(Keccak.EmptyTreeHash.Bytes))) + { + _verkleStateStore.MoveToStateRoot(rootHash); + } + else + { + return; + } + + visitor.VisitTree(rootHash, trieVisitContext); + + RecurseNodes(visitor, _verkleStateStore.GetInternalNode(Array.Empty()), trieVisitContext); + + } + + private void RecurseNodes(IVerkleTreeVisitor visitor, InternalNode node, TrieVisitContext trieVisitContext) + { + switch (node.NodeType) + { + case VerkleNodeType.BranchNode: + { + visitor.VisitBranchNode(node, trieVisitContext); + trieVisitContext.Level++; + for (int i = 0; i < 256; i++) + { + trieVisitContext.AbsolutePathIndex.Add((byte)i); + InternalNode? childNode = _verkleStateStore.GetInternalNode(trieVisitContext.AbsolutePathIndex.ToArray()); + if (childNode is not null && visitor.ShouldVisit(trieVisitContext.AbsolutePathIndex.ToArray())) + { + RecurseNodes(visitor, childNode!, trieVisitContext); + } + trieVisitContext.AbsolutePathIndex.RemoveAt(trieVisitContext.AbsolutePathIndex.Count - 1); + } + trieVisitContext.Level--; + break; + } + case VerkleNodeType.StemNode: + { + visitor.VisitStemNode(node, trieVisitContext); + Stem stemKey = node.Stem; + Span childKey = stackalloc byte[32]; + stemKey.Bytes.CopyTo(childKey); + trieVisitContext.Level++; + for (int i = 0; i < 256; i++) + { + childKey[31] = (byte)i; + byte[]? childNode = _verkleStateStore.GetLeaf(childKey.ToArray()); + if (childNode is not null && visitor.ShouldVisit(childKey.ToArray())) + { + visitor.VisitLeafNode(childKey.ToArray(), trieVisitContext, childNode); + } + } + trieVisitContext.Level--; + break; + } + default: + throw new ArgumentOutOfRangeException(); + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.cs new file mode 100644 index 00000000000..be6d67d751d --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTree.cs @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Verkle.Curve; +using Nethermind.Verkle.Fields.FrEElement; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Sync; +using Nethermind.Verkle.Tree.Utils; +using Nethermind.Verkle.Tree.VerkleDb; + +namespace Nethermind.Verkle.Tree; + +public partial class VerkleTree: IVerkleTree +{ + private static byte[] RootKey => Array.Empty(); + private readonly ILogger _logger; + + /// + /// _leafUpdateCache, _treeCache, _verkleStateStore - these are use while inserting and commiting data to the tree + /// + /// Insertion - _leafUpdateCache is used to keep track of keys inserted accumulated by the stem + /// Commit - calculate and update the commitment of internal nodes from bottom up and insert into _treeCache + /// CommitTree - flush all the changes stored in _treeCache to _verkleStateStore indexed by the blockNumber + /// + + // aggregate the stem commitments here and then update the entire tree when Commit() is called + private readonly SpanDictionary _leafUpdateCache = new(Bytes.SpanEqualityComparer); + + // cache to maintain recently used or inserted nodes of the tree - should be consistent + private VerkleMemoryDb _treeCache = new(); + + // the store that is responsible to store the tree in a key-value store + public readonly IVerkleTrieStore _verkleStateStore; + + public VerkleTree(IDbProvider dbProvider, ILogManager logManager) + { + _verkleStateStore = new VerkleStateStore(dbProvider, logManager); + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + } + + public VerkleTree(IVerkleTrieStore verkleStateStore, ILogManager logManager) + { + _verkleStateStore = verkleStateStore; + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + } + + public VerkleCommitment StateRoot + { + get => GetStateRoot(); + set => MoveToStateRoot(value); + } + + private VerkleCommitment GetStateRoot() + { + bool inTreeCache = _treeCache.GetInternalNode(Array.Empty(), out InternalNode? value); + return inTreeCache ? new VerkleCommitment(value!.Bytes) : _verkleStateStore.StateRoot; + } + + public bool MoveToStateRoot(VerkleCommitment stateRoot) + { + try + { + if (_logger.IsTrace) _logger.Trace($"MoveToStateRoot: from: {StateRoot} to: {stateRoot}"); + return _verkleStateStore.MoveToStateRoot(stateRoot); + } + catch (Exception e) + { + _logger.Error($"MoveToStateRoot: failed | from: {StateRoot} to: {stateRoot}", e); + return false; + } + } + + public byte[]? Get(Pedersen key) + { + _treeCache.GetLeaf(key.Bytes, out byte[]? value); + value ??= _verkleStateStore.GetLeaf(key.Bytes); + return value; + } + + private void SetLeafCache(Pedersen key, byte[]? value) + { + _treeCache.SetLeaf(key.BytesAsSpan, value); + } + + public void Insert(Pedersen key, ReadOnlySpan value) + { + ReadOnlySpan stem = key.StemAsSpan; + bool present = _leafUpdateCache.TryGetValue(stem, out LeafUpdateDelta leafUpdateDelta); + if (!present) leafUpdateDelta = new LeafUpdateDelta(); + leafUpdateDelta.UpdateDelta(UpdateLeafAndGetDelta(key, value.ToArray()), key.SuffixByte); + _leafUpdateCache[stem] = leafUpdateDelta; + } + + public void InsertStemBatch(ReadOnlySpan stem, IEnumerable<(byte, byte[])> leafIndexValueMap) + { + bool present = _leafUpdateCache.TryGetValue(stem, out LeafUpdateDelta leafUpdateDelta); + if(!present) leafUpdateDelta = new LeafUpdateDelta(); + + Span key = new byte[32]; + stem.CopyTo(key); + foreach ((byte index, byte[] value) in leafIndexValueMap) + { + key[31] = index; + leafUpdateDelta.UpdateDelta(UpdateLeafAndGetDelta(new Pedersen(key.ToArray()), value), key[31]); + } + + _leafUpdateCache[stem.ToArray()] = leafUpdateDelta; + } + + public void InsertStemBatch(ReadOnlySpan stem, IEnumerable leafIndexValueMap) + { + bool present = _leafUpdateCache.TryGetValue(stem, out LeafUpdateDelta leafUpdateDelta); + if(!present) leafUpdateDelta = new LeafUpdateDelta(); + + Span key = new byte[32]; + stem.CopyTo(key); + foreach (LeafInSubTree leaf in leafIndexValueMap) + { + key[31] = leaf.SuffixByte; + leafUpdateDelta.UpdateDelta(UpdateLeafAndGetDelta(new Pedersen(key.ToArray()), leaf.Leaf), key[31]); + } + + _leafUpdateCache[stem.ToArray()] = leafUpdateDelta; + } + + public void InsertStemBatch(in Stem stem, IEnumerable leafIndexValueMap) + { + InsertStemBatch(stem.BytesAsSpan, leafIndexValueMap); + } + + private Banderwagon UpdateLeafAndGetDelta(Pedersen key, byte[] value) + { + byte[]? oldValue = Get(key); + Banderwagon leafDeltaCommitment = GetLeafDelta(oldValue, value, key.SuffixByte); + SetLeafCache(key, value); + return leafDeltaCommitment; + } + + private static Banderwagon GetLeafDelta(byte[]? oldValue, byte[] newValue, byte index) + { + // break the values to calculate the commitments for the leaf + (FrE newValLow, FrE newValHigh) = VerkleUtils.BreakValueInLowHigh(newValue); + (FrE oldValLow, FrE oldValHigh) = VerkleUtils.BreakValueInLowHigh(oldValue); + + int posMod128 = index % 128; + int lowIndex = 2 * posMod128; + int highIndex = lowIndex + 1; + + Banderwagon deltaLow = Committer.ScalarMul(newValLow - oldValLow, lowIndex); + Banderwagon deltaHigh = Committer.ScalarMul(newValHigh - oldValHigh, highIndex); + return deltaLow + deltaHigh; + } + + private static Banderwagon GetLeafDelta(byte[] newValue, byte index) + { + (FrE newValLow, FrE newValHigh) = VerkleUtils.BreakValueInLowHigh(newValue); + + int posMod128 = index % 128; + int lowIndex = 2 * posMod128; + int highIndex = lowIndex + 1; + + Banderwagon deltaLow = Committer.ScalarMul(newValLow, lowIndex); + Banderwagon deltaHigh = Committer.ScalarMul(newValHigh, highIndex); + return deltaLow + deltaHigh; + } + + public void Commit(bool forSync = false) + { + if (_logger.IsDebug) _logger.Debug($"VT Commit: SubTree Count:{_leafUpdateCache.Count}"); + foreach (KeyValuePair leafDelta in _leafUpdateCache) + { + if (_logger.IsTrace) + _logger.Trace( + $"VT Commit: Stem:{leafDelta.Key.ToHexString()} DeltaCommitment C1:{leafDelta.Value.DeltaC1?.ToBytes().ToHexString()} C2{leafDelta.Value.DeltaC2?.ToBytes().ToHexString()}"); + UpdateTreeCommitments(leafDelta.Key, leafDelta.Value, forSync); + } + _leafUpdateCache.Clear(); + } + + public void CommitTree(long blockNumber) + { + _verkleStateStore.Flush(blockNumber, _treeCache); + _treeCache = new VerkleMemoryDb(); + } + + private void UpdateRootNode(Banderwagon rootDelta) + { + InternalNode root = GetInternalNode(RootKey) ?? throw new InvalidOperationException("root should be present"); + InternalNode newRoot = root.Clone(); + newRoot.InternalCommitment.AddPoint(rootDelta); + SetInternalNode(RootKey, newRoot); + } + + private InternalNode? GetInternalNode(ReadOnlySpan nodeKey) + { + return _treeCache.GetInternalNode(nodeKey, out InternalNode? value) + ? value + : _verkleStateStore.GetInternalNode(nodeKey); + } + + private void SetInternalNode(byte[] nodeKey, InternalNode node, bool replace = true) + { + if (replace || !_treeCache.InternalTable.TryGetValue(nodeKey, out InternalNode? prevNode)) + { + _treeCache.SetInternalNode(nodeKey, node); + } + else + { + prevNode!.C1 ??= node.C1; + prevNode.C2 ??= node.C2; + _treeCache.SetInternalNode(nodeKey, prevNode); + } + } + + public void Reset() + { + _leafUpdateCache.Clear(); + ProofBranchPolynomialCache.Clear(); + ProofStemPolynomialCache.Clear(); + _treeCache = new VerkleMemoryDb(); + _verkleStateStore.Reset(); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTreeDumper.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTreeDumper.cs new file mode 100644 index 00000000000..5da8d6b9881 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTreeDumper.cs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Trie; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; + +namespace Nethermind.Verkle.Tree; + +public class VerkleTreeDumper : IVerkleTreeVisitor +{ + + private readonly StringBuilder _builder = new StringBuilder(); + + public void Reset() + { + _builder.Clear(); + } + + public bool ShouldVisit(byte[] nextNode) + { + return true; + } + public void VisitTree(VerkleCommitment rootHash, TrieVisitContext trieVisitContext) + { + if (rootHash.Equals(VerkleCommitment.Zero)) + { + _builder.AppendLine("EMPTY TREE"); + } + else + { + _builder.AppendLine("STATE TREE"); + } + } + + private string GetPrefix(TrieVisitContext context) => string.Concat($"{GetIndent(context.Level)}", context.IsStorage ? "STORAGE " : string.Empty, $"{GetChildIndex(context)}"); + private string GetIndent(int level) => new('+', level * 2); + private string GetChildIndex(TrieVisitContext context) => context.BranchChildIndex is null ? string.Empty : $"{context.BranchChildIndex:x2} "; + + public void VisitMissingNode(byte[] nodeKey, TrieVisitContext trieVisitContext) + { + _builder.AppendLine($"{GetIndent(trieVisitContext.Level)}{GetChildIndex(trieVisitContext)}MISSING {nodeKey}"); + } + public void VisitBranchNode(InternalNode node, TrieVisitContext trieVisitContext) + { + _builder.AppendLine($"{GetPrefix(trieVisitContext)}InternalNode | -> Key: {trieVisitContext.AbsolutePathIndex.ToArray().ToHexString()}"); + } + public void VisitStemNode(InternalNode node, TrieVisitContext trieVisitContext) + { + _builder.AppendLine($"{GetPrefix(trieVisitContext)}STEM | -> Key: {trieVisitContext.AbsolutePathIndex.ToArray().ToHexString()}"); + } + public void VisitLeafNode(ReadOnlySpan nodeKey, TrieVisitContext trieVisitContext, byte[]? nodeValue) + { + _builder.AppendLine($"{GetPrefix(trieVisitContext)}LEAF | -> Key: {nodeKey.ToHexString()} Value: {nodeValue.ToHexString()}"); + } + + public override string ToString() + { + return _builder.ToString(); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTrieStats.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTrieStats.cs new file mode 100644 index 00000000000..e86f37566d8 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTrieStats.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text; + +namespace Nethermind.Verkle.Tree +{ + public class VerkleTrieStats + { + private const int Levels = 128; + internal int _stateBranchCount; + internal int _stateStemCount; + internal int _stateLeafCount; + internal int _missingLeaf; + internal long _codeSize; + internal long _stateSize; + internal readonly int[] _stateLevels = new int[Levels]; + + public int StateBranchCount => _stateBranchCount; + + public int StateStemCount => _stateStemCount; + + public int StateLeafCount => _stateLeafCount; + + public int MissingLeaf => _missingLeaf; + + public int StateCount => StateLeafCount + StateStemCount + StateBranchCount; + public long CodeSize => _codeSize; + + public long StateSize => _stateSize; + + public long Size => StateSize + CodeSize; + + public int[] StateLevels => _stateLevels; + public int[] AllLevels + { + get + { + int[] result = new int[Levels]; + for (int i = 0; i < result.Length; i++) + { + result[i] = _stateLevels[i]; + } + + return result; + } + } + + public override string ToString() + { + StringBuilder builder = new(); + builder.AppendLine("TRIE STATS"); + builder.AppendLine($" SIZE {Size} (STATE {StateSize}, CODE {CodeSize})"); + builder.AppendLine($" STATE NODES {StateCount} ({StateBranchCount}|{StateStemCount}|{StateLeafCount})"); + builder.AppendLine($" MISSING {MissingLeaf}"); + builder.AppendLine($" ALL LEVELS {string.Join(" | ", AllLevels)}"); + builder.AppendLine($" STATE LEVELS {string.Join(" | ", StateLevels)}"); + return builder.ToString(); + } + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleTrieStatsCollector.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTrieStatsCollector.cs new file mode 100644 index 00000000000..94aacee3db6 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleTrieStatsCollector.cs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Verkle; +using Nethermind.Logging; +using Nethermind.Serialization.Rlp; +using Nethermind.Trie; +using Nethermind.Verkle.Tree.Interfaces; +using Nethermind.Verkle.Tree.Nodes; +using Nethermind.Verkle.Tree.Serializers; + +namespace Nethermind.Verkle.Tree; + +public class VerkleTrieStatsCollector: IVerkleTreeVisitor +{ + private static InternalNodeSerializer Serializer => InternalNodeSerializer.Instance; + private int _lastAccountNodeCount = 0; + + private readonly ILogger _logger; + public VerkleTrieStatsCollector(IKeyValueStore codeKeyValueStore, ILogManager logManager) + { + _logger = logManager.GetClassLogger(); + } + + public VerkleTrieStats Stats { get; } = new(); + + public bool IsFullDbScan => true; + + public bool ShouldVisit(byte[] nextNode) + { + return true; + } + + public void VisitTree(VerkleCommitment rootHash, TrieVisitContext trieVisitContext) { } + + public void VisitMissingNode(byte[] nodeKey, TrieVisitContext trieVisitContext) + { + Interlocked.Increment(ref Stats._missingLeaf); + Interlocked.Increment(ref Stats._stateLevels[trieVisitContext.Level]); + } + + public void VisitBranchNode(InternalNode node, TrieVisitContext trieVisitContext) + { + Interlocked.Add(ref Stats._stateSize, Serializer.GetLength(node, RlpBehaviors.None)); + Interlocked.Increment(ref Stats._stateBranchCount); + + Interlocked.Increment(ref Stats._stateLevels[trieVisitContext.Level]); + } + + public void VisitStemNode(InternalNode node, TrieVisitContext trieVisitContext) + { + Interlocked.Add(ref Stats._stateSize, Serializer.GetLength(node, RlpBehaviors.None)); + Interlocked.Increment(ref Stats._stateStemCount); + + Interlocked.Increment(ref Stats._stateLevels[trieVisitContext.Level]); + } + + public void VisitLeafNode(ReadOnlySpan nodeKey, TrieVisitContext trieVisitContext, byte[]? nodeValue) + { + if (Stats.StateCount - _lastAccountNodeCount > 1_000_000) + { + _lastAccountNodeCount = Stats.StateCount; + _logger.Warn($"Collected info from {Stats.StateCount} nodes. Missing LEAF {Stats.MissingLeaf}"); + } + + Interlocked.Add(ref Stats._stateSize, 32); + Interlocked.Increment(ref Stats._stateLeafCount); + + Interlocked.Increment(ref Stats._stateLevels[trieVisitContext.Level]); + } +} diff --git a/src/Nethermind/Nethermind.Verkle.Tree/VerkleWitness.cs b/src/Nethermind/Nethermind.Verkle.Tree/VerkleWitness.cs new file mode 100644 index 00000000000..524929450b5 --- /dev/null +++ b/src/Nethermind/Nethermind.Verkle.Tree/VerkleWitness.cs @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Core.Verkle; +using Nethermind.Int256; + +namespace Nethermind.Verkle.Tree; + +// TODO: this can be definitely optimized by caching the keys from StateProvider - because for every access we +// already calculate keys in StateProvider - or we maintain pre images? +public struct VerkleWitness : IVerkleWitness +{ + [Flags] + private enum AccountHeaderAccess + { + Version = 1, + Balance = 2, + Nonce = 4, + CodeHash = 8, + CodeSize = 16 + } + + private readonly JournalSet _accessedSubtrees; + private readonly JournalSet _accessedLeaves; + private readonly JournalSet _modifiedSubtrees; + private readonly JournalSet _modifiedLeaves; + + // TODO: add these in GasPrices List + private const long WitnessChunkRead = 200; // verkle-trie + private const long WitnessChunkWrite = 500; // verkle-trie + private const long WitnessChunkFill = 6200; // verkle-trie + private const long WitnessBranchRead = 1900; // verkle-trie + private const long WitnessBranchWrite = 3000; // verkle-trie + + private readonly Dictionary _snapshots = new Dictionary(); + private int NextSnapshot; + + public VerkleWitness() + { + _accessedSubtrees = new JournalSet(Bytes.EqualityComparer); + _accessedLeaves = new JournalSet(); + _modifiedLeaves = new JournalSet(); + _modifiedSubtrees = new JournalSet(Bytes.EqualityComparer); + } + /// + /// When a non-precompile address is the target of a CALL, CALLCODE, + /// DELEGATECALL, SELFDESTRUCT, EXTCODESIZE, or EXTCODECOPY opcode, + /// or is the target address of a contract creation whose initcode + /// starts execution. + /// + /// + /// + public long AccessForCodeOpCodes(Address caller) + { + long gas = AccessAccount(caller, AccountHeaderAccess.Version | AccountHeaderAccess.CodeSize); + // _logger.Info($"AccessForCodeOpCodes: {caller.Bytes.ToHexString()} {gas}"); + return gas; + } + + /// + /// Use this in two scenarios: + /// 1. If a call is value-bearing (ie. it transfers nonzero wei), whether + /// or not the callee is a precompile + /// 2. If the SELFDESTRUCT/SENDALL opcode is called by some caller_address + /// targeting some target_address (regardless of whether it’s value-bearing + /// or not) + /// + /// + /// + /// + public long AccessValueTransfer(Address caller, Address? callee) + { + + var gas = AccessAccount(caller, AccountHeaderAccess.Balance, true) + + // this generally happens in the case of contract creation + (callee == null ? 0 : AccessAccount(callee, AccountHeaderAccess.Balance, true)); + // _logger.Info($"AccessForCodeOpCodes: {caller.Bytes.ToHexString()} {gas}"); + return gas; + } + + /// + /// When a contract creation is initialized. + /// + /// + /// + /// + public long AccessForContractCreationInit(Address contractAddress, bool isValueTransfer) + { + long gas = isValueTransfer + ? AccessAccount(contractAddress, AccountHeaderAccess.Version | AccountHeaderAccess.Nonce | AccountHeaderAccess.Balance | AccountHeaderAccess.CodeHash, true) + : AccessAccount(contractAddress, AccountHeaderAccess.Version | AccountHeaderAccess.Nonce | AccountHeaderAccess.CodeHash, true); + // _logger.Info($"AccessForContractCreationInit: {contractAddress.Bytes.ToHexString()} {isValueTransfer} {gas}"); + + return gas; + } + + /// + /// When a contract is created. + /// + /// + /// + public long AccessContractCreated(Address contractAddress) + { + + var gas = AccessCompleteAccount(contractAddress, true); + // _logger.Info($"AccessContractCreated: {contractAddress.Bytes.ToHexString()} {gas}"); + return gas; + } + + /// + /// If the BALANCE opcode is called targeting some address. + /// + /// + /// + public long AccessBalance(Address address) + { + + var gas = AccessAccount(address, AccountHeaderAccess.Balance); + // _logger.Info($"AccessBalance: {address.Bytes.ToHexString()} {gas}"); + return gas; + } + + /// + /// If the EXTCODEHASH opcode is called targeting some address. + /// + /// + /// + public long AccessCodeHash(Address address) + { + + var gas = AccessAccount(address, AccountHeaderAccess.CodeHash); + // _logger.Info($"AccessCodeHash: {address.Bytes.ToHexString()} {gas}"); + return gas; + } + + /// + /// When SLOAD and SSTORE opcodes are called with a given address + /// and key. + /// + /// + /// + /// + /// + public long AccessStorage(Address address, UInt256 key, bool isWrite) + { + var gas = AccessKey(AccountHeader.GetTreeKeyForStorageSlot(address.Bytes, key), isWrite); + // _logger.Info($"AccessStorage: {address.Bytes.ToHexString()} {key.ToBigEndian().ToHexString()} {isWrite} {gas}"); + return gas; + } + + /// + /// When the code chunk chunk_id is accessed is accessed + /// + /// + /// + /// + /// + public long AccessCodeChunk(Address address, byte chunkId, bool isWrite) + { + var key = AccountHeader.GetTreeKeyForCodeChunk(address.Bytes, chunkId); + // _logger.Info($"AccessCodeChunkKey: {EnumerableExtensions.ToString(key)}"); + var gas = AccessKey(key, isWrite); + // _logger.Info($"AccessCodeChunk: {address.Bytes.ToHexString()} {chunkId} {isWrite} {gas}"); + return gas; + } + + /// + /// When you are starting to execute a transaction. + /// + /// + /// + /// + /// + public long AccessForTransaction(Address originAddress, Address? destinationAddress, bool isValueTransfer) + { + + // TODO: does not seem right - not upto spec + long gasCost = AccessAccount(originAddress, AccountHeaderAccess.Version | AccountHeaderAccess.Balance | AccountHeaderAccess.Nonce | AccountHeaderAccess.CodeHash | AccountHeaderAccess.CodeSize) + + (destinationAddress == null ? 0 : AccessCompleteAccount(destinationAddress)); + + // when you are executing a transaction, you are writing to the nonce of the origin address + gasCost += AccessAccount(originAddress, AccountHeaderAccess.Nonce, true); + if (isValueTransfer) + { + // when you are executing a transaction with value transfer, + // you are writing to the balance of the origin and destination address + gasCost += AccessValueTransfer(originAddress, destinationAddress); + } + else + { + gasCost += AccessAccount(originAddress, AccountHeaderAccess.Balance, true); + } + // _logger.Info($"AccessForTransaction: {originAddress.Bytes.ToHexString()} {destinationAddress?.Bytes.ToHexString()} {isValueTransfer} {gasCost}"); + return gasCost; + } + + /// + /// Proof of Absence + /// + /// + /// + public long AccessForProofOfAbsence(Address address) + { + long gas = AccessCompleteAccount(address); + // _logger.Info($"AccessForProofOfAbsence: {address.Bytes.ToHexString()} {gas}"); + return gas; + } + + /// + /// Call for the gas beneficiary. + /// + /// + /// + public long AccessForGasBeneficiary(Address gasBeneficiary) + { + long gas = AccessAccount(gasBeneficiary, + AccountHeaderAccess.Version | AccountHeaderAccess.Balance | AccountHeaderAccess.Nonce | + AccountHeaderAccess.CodeHash | AccountHeaderAccess.CodeSize); + // _logger.Info($"AccessCompleteAccount: {address.Bytes.ToHexString()} {isWrite} {gas}"); + return gas; + } + + /// + /// When you have to access the complete account + /// + /// + /// + /// + public long AccessCompleteAccount(Address address, bool isWrite = false) + { + + var gas = AccessAccount(address, + AccountHeaderAccess.Version | AccountHeaderAccess.Balance | AccountHeaderAccess.Nonce | AccountHeaderAccess.CodeHash | AccountHeaderAccess.CodeSize, + isWrite); + // _logger.Info($"AccessCompleteAccount: {address.Bytes.ToHexString()} {isWrite} {gas}"); + return gas; + } + + /// + /// When you have to access the certain keys for the account + /// you can specify the keys you want to access using the AccountHeaderAccess. + /// + /// + /// + /// + /// + private long AccessAccount(Address address, AccountHeaderAccess accessOptions, bool isWrite = false) + { + + long gasUsed = 0; + if ((accessOptions & AccountHeaderAccess.Version) == AccountHeaderAccess.Version) gasUsed += AccessKey(AccountHeader.GetTreeKey(address.Bytes, UInt256.Zero, AccountHeader.Version), isWrite); + if ((accessOptions & AccountHeaderAccess.Balance) == AccountHeaderAccess.Balance) gasUsed += AccessKey(AccountHeader.GetTreeKey(address.Bytes, UInt256.Zero, AccountHeader.Balance), isWrite); + if ((accessOptions & AccountHeaderAccess.Nonce) == AccountHeaderAccess.Nonce) gasUsed += AccessKey(AccountHeader.GetTreeKey(address.Bytes, UInt256.Zero, AccountHeader.Nonce), isWrite); + if ((accessOptions & AccountHeaderAccess.CodeHash) == AccountHeaderAccess.CodeHash) gasUsed += AccessKey(AccountHeader.GetTreeKey(address.Bytes, UInt256.Zero, AccountHeader.CodeHash), isWrite); + if ((accessOptions & AccountHeaderAccess.CodeSize) == AccountHeaderAccess.CodeSize) gasUsed += AccessKey(AccountHeader.GetTreeKey(address.Bytes, UInt256.Zero, AccountHeader.CodeSize), isWrite); + // _logger.Info($"AccessAccount: {address.Bytes.ToHexString()} {accessOptions} {isWrite} {gasUsed}"); + return gasUsed; + } + + private long AccessKey(Pedersen key, bool isWrite = false, bool leafExist = false) + { + bool newSubTreeAccess = false; + bool newLeafAccess = false; + + bool newSubTreeUpdate = false; + bool newLeafUpdate = false; + + bool newLeafFill = false; + + + if (_accessedLeaves.Add(key)) + { + newLeafAccess = true; + } + + if (_accessedSubtrees.Add(key.StemAsSpan.ToArray())) + { + newSubTreeAccess = true; + } + + long accessCost = + (newLeafAccess ? WitnessChunkRead : 0) + + (newSubTreeAccess ? WitnessBranchRead : 0); + if (!isWrite) + return accessCost; + + if (_modifiedLeaves.Add((key))) + { + // newLeafFill = !leafExist; + newLeafUpdate = true; + } + + if (_modifiedSubtrees.Add(key.StemAsSpan.ToArray())) + { + newSubTreeUpdate = true; + } + long writeCost = + (newLeafUpdate ? WitnessChunkWrite : 0) + + (newLeafFill ? WitnessChunkFill : 0) + + (newSubTreeUpdate ? WitnessBranchWrite : 0); + + return writeCost + accessCost; + } + + public byte[][] GetAccessedKeys() + { + return _accessedLeaves.ToArray().Select(x => x.Bytes).ToArray(); + } + + public int TakeSnapshot() + { + int[] snapshot = new int[2]; + snapshot[0] = _accessedSubtrees.TakeSnapshot(); + snapshot[1] = _accessedLeaves.TakeSnapshot(); + _snapshots.Add(NextSnapshot, snapshot); + return NextSnapshot++; + } + + public void Restore(int snapshot) + { + int[] witnessSnapshot = _snapshots[snapshot]; + _accessedSubtrees.Restore(witnessSnapshot[0]); + _accessedLeaves.Restore(witnessSnapshot[1]); + } +} diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index adb3d4d21a8..d4f298d44ee 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -215,6 +215,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.Serialization.Ss EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.Network.Contract", "Nethermind.Network.Contract\Nethermind.Network.Contract.csproj", "{AD151E35-4BBC-4A83-8F57-CC8665F6E007}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Verkle.Tree", "Nethermind.Verkle.Tree\Nethermind.Verkle.Tree.csproj", "{CAAC2B71-2EC5-4900-AF8F-434CD5846B13}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethermind.Verkle.Tree.Test", "Nethermind.Verkle.Tree.Test\Nethermind.Verkle.Tree.Test.csproj", "{AF8CCC79-EE1C-4A05-B6ED-465B08409ABC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -597,6 +601,14 @@ Global {AD151E35-4BBC-4A83-8F57-CC8665F6E007}.Debug|Any CPU.Build.0 = Debug|Any CPU {AD151E35-4BBC-4A83-8F57-CC8665F6E007}.Release|Any CPU.ActiveCfg = Release|Any CPU {AD151E35-4BBC-4A83-8F57-CC8665F6E007}.Release|Any CPU.Build.0 = Release|Any CPU + {CAAC2B71-2EC5-4900-AF8F-434CD5846B13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CAAC2B71-2EC5-4900-AF8F-434CD5846B13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CAAC2B71-2EC5-4900-AF8F-434CD5846B13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CAAC2B71-2EC5-4900-AF8F-434CD5846B13}.Release|Any CPU.Build.0 = Release|Any CPU + {AF8CCC79-EE1C-4A05-B6ED-465B08409ABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AF8CCC79-EE1C-4A05-B6ED-465B08409ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AF8CCC79-EE1C-4A05-B6ED-465B08409ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AF8CCC79-EE1C-4A05-B6ED-465B08409ABC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -661,6 +673,7 @@ Global {2EDE2554-59C0-4E78-92F7-EB8077AA597D} = {78BED57D-720E-4E6C-ABA2-397B73B494F9} {48E50409-26FE-4FD8-AF6E-2A0E79F794CE} = {2EDE2554-59C0-4E78-92F7-EB8077AA597D} {E1E7BEFC-52C0-49ED-B0A7-CB8C3250D120} = {4019B82F-1104-4D2C-9F96-05FD7D3575E8} + {AF8CCC79-EE1C-4A05-B6ED-465B08409ABC} = {4019B82F-1104-4D2C-9F96-05FD7D3575E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {092CA5E3-6180-4ED7-A3CB-9B57FAC2AA85} diff --git a/src/Nethermind/nuget.config b/src/Nethermind/nuget.config index 84c8ccf735e..dfef6a0d533 100644 --- a/src/Nethermind/nuget.config +++ b/src/Nethermind/nuget.config @@ -3,14 +3,15 @@ - + - +