From 34e8976546f85f6c6cfbec8ea44f4ec3a5bd3bfb Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 8 Mar 2024 12:43:21 +0700 Subject: [PATCH 1/2] feat: implement ListUintNum64Type --- .../persistent-merkle-tree/src/packedNode.ts | 34 ++++ .../test/unit/packedNode.test.ts | 126 ++++++++++++++- packages/ssz/src/type/listUintNum64.ts | 49 ++++++ .../test/unit/byType/listBasic/tree.test.ts | 153 +++++++++--------- 4 files changed, 286 insertions(+), 76 deletions(-) create mode 100644 packages/ssz/src/type/listUintNum64.ts diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index 928013b5..3c6413b8 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -1,11 +1,45 @@ import {subtreeFillToContents} from "./subtree"; import {Node, LeafNode, getNodeH, setNodeH} from "./node"; +const NUMBER_2_POW_32 = 2 ** 32; + export function packedRootsBytesToNode(depth: number, dataView: DataView, start: number, end: number): Node { const leafNodes = packedRootsBytesToLeafNodes(dataView, start, end); return subtreeFillToContents(leafNodes, depth); } +/** + * Pack a list of uint64 numbers into a list of LeafNodes. + * Each value is UintNum64, which is 8 bytes long, which is 2 h values. + * Each 4 of them forms a LeafNode. + * + * v0 v1 v2 v3 + * |-------------|-------------|-------------|-------------| + * + * h0 h1 h2 h3 h4 h5 h6 h7 + * |------|------|------|------|------|------|------|------| + */ +export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { + const leafNodes = new Array(Math.ceil(values.length / 4)); + for (let i = 0; i < values.length; i++) { + const nodeIndex = Math.floor(i / 4); + const leafNode = leafNodes[nodeIndex] ?? new LeafNode(0, 0, 0, 0, 0, 0, 0, 0); + const vIndex = i % 4; + const hIndex = 2 * vIndex; + const value = values[i]; + // same logic to UintNumberType.value_serializeToBytes() for 8 bytes + if (value === Infinity) { + setNodeH(leafNode, hIndex, 0xffffffff); + setNodeH(leafNode, hIndex + 1, 0xffffffff); + } else { + setNodeH(leafNode, hIndex, value & 0xffffffff); + setNodeH(leafNode, hIndex + 1, (value / NUMBER_2_POW_32) & 0xffffffff); + } + leafNodes[nodeIndex] = leafNode; + } + return leafNodes; +} + /** * Optimized deserialization of linear bytes to consecutive leaf nodes */ diff --git a/packages/persistent-merkle-tree/test/unit/packedNode.test.ts b/packages/persistent-merkle-tree/test/unit/packedNode.test.ts index 38a3b046..0fb53ac9 100644 --- a/packages/persistent-merkle-tree/test/unit/packedNode.test.ts +++ b/packages/persistent-merkle-tree/test/unit/packedNode.test.ts @@ -1,7 +1,7 @@ import {HashObject} from "@chainsafe/as-sha256"; import {expect} from "chai"; import {LeafNode, Node} from "../../src"; -import {packedNodeRootsToBytes, packedRootsBytesToLeafNodes} from "../../src/packedNode"; +import {packedNodeRootsToBytes, packedRootsBytesToLeafNodes, packedUintNum64sToLeafNodes} from "../../src/packedNode"; describe("subtree / packedNode single node", () => { const testCases: { @@ -9,6 +9,7 @@ describe("subtree / packedNode single node", () => { size: number; nodes: Node[]; outStr: string; + testPackedNumbers?: boolean; }[] = [ { id: "One byte", @@ -48,11 +49,37 @@ describe("subtree / packedNode single node", () => { nodes: [LeafNode.fromHashObject({h0: 0x0708090a, h1: 0x01020304, h2: 0, h3: 0, h4: 0, h5: 0, h6: 0, h7: 0})], outStr: "0x0a09080704030201", }, + { + id: "2 h values fits uint64 number", + size: 8, + nodes: [LeafNode.fromHashObject({h0: 0x0708090a, h1: 0x0102030, h2: 0, h3: 0, h4: 0, h5: 0, h6: 0, h7: 0})], + outStr: "0x0a09080730201000", + testPackedNumbers: true, + }, { id: "32 bytes zero", size: 32, nodes: [LeafNode.fromHashObject({h0: 0, h1: 0, h2: 0, h3: 0, h4: 0, h5: 0, h6: 0, h7: 0})], outStr: "0x0000000000000000000000000000000000000000000000000000000000000000", + testPackedNumbers: true, + }, + { + id: "32 bytes max", + size: 32, + nodes: [ + LeafNode.fromHashObject({ + h0: 0xffffffff, + h1: 0xffffffff, + h2: 0xffffffff, + h3: 0xffffffff, + h4: 0xffffffff, + h5: 0xffffffff, + h6: 0xffffffff, + h7: 0xffffffff, + }), + ], + outStr: "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + testPackedNumbers: true, }, { id: "32 bytes same", @@ -77,9 +104,82 @@ describe("subtree / packedNode single node", () => { ], outStr: "0x18735c375d8e7b2922ef64f165d10b9ace103f326627f0a21d0fe8a0a573f83f", }, + // same to random tests but trim h1, h3, h5, h7 to make h{2*i} + h{2*i + 1} fits uint64 + { + id: "8 bytes random fits unit64 number", + size: 32, + nodes: [ + LeafNode.fromHashObject({ + h0: 928805656, + h1: 6959632, + h2: 0, + h3: 0, + h4: 0, + h5: 0, + h6: 0, + h7: 0, + }), + ], + outStr: "0x18735c3710326a00000000000000000000000000000000000000000000000000", + testPackedNumbers: true, + }, + { + id: "16 bytes random fits unit64 number", + size: 32, + nodes: [ + LeafNode.fromHashObject({ + h0: 928805656, + h1: 6959632, + h2: 4049923874, + h3: 258446, + h4: 0, + h5: 0, + h6: 0, + h7: 0, + }), + ], + outStr: "0x18735c3710326a0022ef64f18ef1030000000000000000000000000000000000", + testPackedNumbers: true, + }, + { + id: "24 bytes random fits unit64 number", + size: 32, + nodes: [ + LeafNode.fromHashObject({ + h0: 928805656, + h1: 6959632, + h2: 4049923874, + h3: 258446, + h4: 842993870, + h5: 273364, + h6: 0, + h7: 0, + }), + ], + outStr: "0x18735c3710326a0022ef64f18ef10300ce103f32d42b04000000000000000000", + testPackedNumbers: true, + }, + { + id: "32 bytes random fits unit64 number", + size: 32, + nodes: [ + LeafNode.fromHashObject({ + h0: 928805656, + h1: 6959632, + h2: 4049923874, + h3: 258446, + h4: 842993870, + h5: 273364, + h6: 2699562781, + h7: 107324, + }), + ], + outStr: "0x18735c3710326a0022ef64f18ef10300ce103f32d42b04001d0fe8a03ca30100", + testPackedNumbers: true, + }, ]; - for (const {id, size, nodes, outStr} of testCases) { + for (const {id, size, nodes, outStr, testPackedNumbers} of testCases) { it(`${id} - packedNodeRootsToBytes`, () => { const uint8Array = new Uint8Array(size); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); @@ -93,6 +193,28 @@ describe("subtree / packedNode single node", () => { const nodesRes = packedRootsBytesToLeafNodes(dataView, 0, size); expect(onlyHashObject(nodesRes[0].rootHashObject)).to.deep.equal(onlyHashObject(nodes[0].rootHashObject)); }); + + // 1 UintNum64 = 8 bytes + if (testPackedNumbers) { + const NUMBER_2_POW_32 = 2 ** 32; + it(`${id} - packedUintNum64sToLeafNodes, value size=${Math.floor(size / 8)}`, () => { + const values: number[] = []; + const uint8Array = new Uint8Array(Buffer.from(outStr.replace("0x", ""), "hex")); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + for (let i = 0; i < size; i += 8) { + const a = dataView.getUint32(i, true); + const b = dataView.getUint32(i + 4, true); + if (a === 0xffffffff && b === 0xffffffff) { + values.push(Infinity); + } else { + values.push(b * NUMBER_2_POW_32 + a); + } + } + + const nodesRes = packedUintNum64sToLeafNodes(values); + expect(onlyHashObject(nodesRes[0].rootHashObject)).to.deep.equal(onlyHashObject(nodes[0].rootHashObject)); + }); + } } }); diff --git a/packages/ssz/src/type/listUintNum64.ts b/packages/ssz/src/type/listUintNum64.ts new file mode 100644 index 00000000..663e637d --- /dev/null +++ b/packages/ssz/src/type/listUintNum64.ts @@ -0,0 +1,49 @@ +import {LeafNode, Node, packedUintNum64sToLeafNodes, subtreeFillToContents} from "@chainsafe/persistent-merkle-tree"; + +import {ListBasicTreeViewDU} from "../viewDU/listBasic"; +import {ListBasicOpts, ListBasicType} from "./listBasic"; +import {UintNumberType} from "./uint"; +import {addLengthNode} from "./arrayBasic"; + +/** + * Specific implementation of ListBasicType for UintNumberType with some optimizations. + */ +export class ListUintNum64Type extends ListBasicType { + constructor(limit: number, opts?: ListBasicOpts) { + super(new UintNumberType(8), limit, opts); + } + + /** + * Return a ListBasicTreeViewDU with nodes populated + */ + toViewDU(value: number[]): ListBasicTreeViewDU { + // no need to serialize and deserialize like in the abstract class + const {treeNode, leafNodes} = this.packedUintNum64sToNode(value); + // cache leaf nodes in the ViewDU + return this.getViewDU(treeNode, { + nodes: leafNodes, + length: value.length, + nodesPopulated: true, + }); + } + + /** + * No need to serialize and deserialize like in the abstract class + */ + value_toTree(value: number[]): Node { + const {treeNode} = this.packedUintNum64sToNode(value); + return treeNode; + } + + private packedUintNum64sToNode(value: number[]): {treeNode: Node; leafNodes: LeafNode[]} { + if (value.length > this.limit) { + throw new Error(`Exceeds limit: ${value.length} > ${this.limit}`); + } + + const leafNodes = packedUintNum64sToLeafNodes(value); + // subtreeFillToContents mutates the leafNodes array + const rootNode = subtreeFillToContents([...leafNodes], this.chunkDepth); + const treeNode = addLengthNode(rootNode, value.length); + return {treeNode, leafNodes}; + } +} diff --git a/packages/ssz/test/unit/byType/listBasic/tree.test.ts b/packages/ssz/test/unit/byType/listBasic/tree.test.ts index 4d99d795..d7f56b4d 100644 --- a/packages/ssz/test/unit/byType/listBasic/tree.test.ts +++ b/packages/ssz/test/unit/byType/listBasic/tree.test.ts @@ -1,15 +1,18 @@ import {expect} from "chai"; import {ListBasicType, toHexString, UintNumberType} from "../../../../src"; import {runViewTestMutation, TreeMutation} from "../runViewTestMutation"; +import {ListUintNum64Type} from "../../../../src/type/listUintNum64"; const limit = 100; const uint64NumInf = new UintNumberType(8, {clipInfinity: true}); const ListN64Uint64NumberType = new ListBasicType(uint64NumInf, limit); +const ListUintNum64 = new ListUintNum64Type(limit); for (const listBasicUintType of [ ListN64Uint64NumberType, new ListBasicType(new UintNumberType(8), limit), new ListBasicType(new UintNumberType(1), limit), + ListUintNum64, ]) { runViewTestMutation({ type: listBasicUintType, @@ -95,82 +98,84 @@ runViewTestMutation({ const deltaValues = [1_000_000_000_000, 999, 0, -1_000_000]; const initBalance = 31217089836; // Must be greater than the negative delta -runViewTestMutation({ - type: ListN64Uint64NumberType, - mutations: [ - // For each delta test mutating a single item of a List - ...deltaValues.map((delta): TreeMutation => { - const valueBefore = Array.from({length: limit}, () => initBalance); - const valueAfter = [...valueBefore]; - const i = Math.floor(limit / 2); // Mutate the middle value - valueAfter[i] = valueBefore[i] + delta; - - return { - id: `applyDeltaAtIndex ${delta}`, - valueBefore, - valueAfter, - fn: (tv) => { - tv.set(i, tv.get(i) + delta); - }, - }; - }), - - // For each delta test mutating half of the List items in batch - ...deltaValues.map((delta): TreeMutation => { - const valueBefore = Array.from({length: limit}, () => initBalance); - const valueAfter = [...valueBefore]; - - // same operation for BalancesList64 using tree_applyDeltaInBatch - const deltaByIndex = new Map(); - // `i += 2` to only apply the delta to half the values - for (let i = 0; i < limit; i += 2) { - valueAfter[i] += delta; - deltaByIndex.set(i, delta); - } +for (const listBasicUintType of [ListN64Uint64NumberType, ListUintNum64]) { + runViewTestMutation({ + type: listBasicUintType, + mutations: [ + // For each delta test mutating a single item of a List + ...deltaValues.map((delta): TreeMutation => { + const valueBefore = Array.from({length: limit}, () => initBalance); + const valueAfter = [...valueBefore]; + const i = Math.floor(limit / 2); // Mutate the middle value + valueAfter[i] = valueBefore[i] + delta; + + return { + id: `applyDeltaAtIndex ${delta}`, + valueBefore, + valueAfter, + fn: (tv) => { + tv.set(i, tv.get(i) + delta); + }, + }; + }), + + // For each delta test mutating half of the List items in batch + ...deltaValues.map((delta): TreeMutation => { + const valueBefore = Array.from({length: limit}, () => initBalance); + const valueAfter = [...valueBefore]; + + // same operation for BalancesList64 using tree_applyDeltaInBatch + const deltaByIndex = new Map(); + // `i += 2` to only apply the delta to half the values + for (let i = 0; i < limit; i += 2) { + valueAfter[i] += delta; + deltaByIndex.set(i, delta); + } - return { - id: `applyDeltaInBatch ${delta}`, - valueBefore, - valueAfter, - fn: (tv) => { - for (const [i, _delta] of deltaByIndex) { - tv.set(i, tv.get(i) + _delta); - } - }, - }; - }), - - // For each delta create a new tree applying half of the List items in batch - // Since the tree is re-created `fn()` returns a new tree that is used to compare `valueAfter` - ...deltaValues.map((delta): TreeMutation => { - const valueBefore = Array.from({length: limit}, () => initBalance); - const valueAfter: number[] = []; - const deltasByIdx: number[] = []; - - for (let i = 0; i < limit; i++) { - // `i % 2 === 0` to only apply the delta to half the values - const d = i % 2 === 0 ? delta : 0; - valueAfter[i] = valueBefore[i] + d; - deltasByIdx[i] = d; - } + return { + id: `applyDeltaInBatch ${delta}`, + valueBefore, + valueAfter, + fn: (tv) => { + for (const [i, _delta] of deltaByIndex) { + tv.set(i, tv.get(i) + _delta); + } + }, + }; + }), + + // For each delta create a new tree applying half of the List items in batch + // Since the tree is re-created `fn()` returns a new tree that is used to compare `valueAfter` + ...deltaValues.map((delta): TreeMutation => { + const valueBefore = Array.from({length: limit}, () => initBalance); + const valueAfter: number[] = []; + const deltasByIdx: number[] = []; + + for (let i = 0; i < limit; i++) { + // `i % 2 === 0` to only apply the delta to half the values + const d = i % 2 === 0 ? delta : 0; + valueAfter[i] = valueBefore[i] + d; + deltasByIdx[i] = d; + } - return { - id: `newTreeFromDeltas ${delta}`, - valueBefore, - valueAfter, - // Skip since the returned viewDU is already committed, can't drop changes - skipCloneMutabilityViewDU: true, - fn: (tv) => { - const values = tv.getAll(); - for (let i = 0; i < values.length; i++) { - values[i] += deltasByIdx[i]; - } - return ListN64Uint64NumberType.toViewDU(values as number[]); - }, - }; - }), - ], -}); + return { + id: `newTreeFromDeltas ${delta}`, + valueBefore, + valueAfter, + // Skip since the returned viewDU is already committed, can't drop changes + skipCloneMutabilityViewDU: true, + fn: (tv) => { + const values = tv.getAll(); + for (let i = 0; i < values.length; i++) { + values[i] += deltasByIdx[i]; + } + return ListN64Uint64NumberType.toViewDU(values as number[]); + }, + }; + }), + ], + }); +} describe("ListBasicType tree reads", () => { for (const [id, view] of Object.entries({ From 9a80d598d4d4ee3c58a6e6cf0f8ef725ac7787ec Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 8 Mar 2024 13:02:23 +0700 Subject: [PATCH 2/2] feat: use ListUintNum64Type for lodestar Balances type --- packages/ssz/test/lodestarTypes/phase0/sszTypes.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts b/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts index 8700f597..c236be7e 100644 --- a/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts +++ b/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts @@ -8,6 +8,7 @@ import { VectorBasicType, VectorCompositeType, } from "../../../src"; +import {ListUintNum64Type} from "../../../src/type/listUintNum64"; import { preset, MAX_REQUEST_BLOCKS, @@ -250,7 +251,7 @@ export const Validator = ValidatorNodeStruct; // Export as stand-alone for direct tree optimizations export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT); -export const Balances = new ListBasicType(UintNum64, VALIDATOR_REGISTRY_LIMIT); +export const Balances = new ListUintNum64Type(VALIDATOR_REGISTRY_LIMIT); export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICAL_VECTOR); export const Slashings = new VectorBasicType(Gwei, EPOCHS_PER_SLASHINGS_VECTOR); export const JustificationBits = new BitVectorType(JUSTIFICATION_BITS_LENGTH);