diff --git a/bridgesync/bridgesync.go b/bridgesync/bridgesync.go index c9dceb77..10d79dbc 100644 --- a/bridgesync/bridgesync.go +++ b/bridgesync/bridgesync.go @@ -95,7 +95,7 @@ func new( return nil, err } if lastProcessedBlock < initialBlock { - err = processor.ProcessBlock(sync.Block{ + err = processor.ProcessBlock(ctx, sync.Block{ Num: initialBlock, }) if err != nil { diff --git a/bridgesync/processor.go b/bridgesync/processor.go index a0db15a5..72fc5a01 100644 --- a/bridgesync/processor.go +++ b/bridgesync/processor.go @@ -5,11 +5,12 @@ import ( "encoding/binary" "encoding/json" "errors" - "log" "math/big" + "path" dbCommon "github.com/0xPolygon/cdk/common" "github.com/0xPolygon/cdk/sync" + "github.com/0xPolygon/cdk/tree" "github.com/ethereum/go-ethereum/common" "github.com/iden3/go-iden3-crypto/keccak256" "github.com/ledgerwatch/erigon-lib/kv" @@ -19,8 +20,6 @@ import ( const ( eventsTableSufix = "-events" lastBlockTableSufix = "-lastBlock" - rootTableSufix = "-root" - rhtTableSufix = "-rht" ) var ( @@ -84,30 +83,26 @@ type processor struct { db kv.RwDB eventsTable string lastBlockTable string - tree *tree + exitTree *tree.AppendOnlyTree } func newProcessor(ctx context.Context, dbPath, dbPrefix string) (*processor, error) { eventsTable := dbPrefix + eventsTableSufix lastBlockTable := dbPrefix + lastBlockTableSufix - rootTable := dbPrefix + rootTableSufix - rhtTable := dbPrefix + rhtTableSufix db, err := mdbx.NewMDBX(nil). Path(dbPath). WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.TableCfg{ eventsTable: {}, lastBlockTable: {}, - rootTable: {}, - rhtTable: {}, } }). Open() if err != nil { return nil, err } - - tree, err := newTree(ctx, rhtTable, rootTable, db) + exitTreeDBPath := path.Join(dbPath, "exittree") + exitTree, err := tree.NewAppendOnly(ctx, exitTreeDBPath, dbPrefix) if err != nil { return nil, err } @@ -115,7 +110,7 @@ func newProcessor(ctx context.Context, dbPath, dbPrefix string) (*processor, err db: db, eventsTable: eventsTable, lastBlockTable: lastBlockTable, - tree: tree, + exitTree: exitTree, }, nil } @@ -181,8 +176,8 @@ func (p *processor) getLastProcessedBlockWithTx(tx kv.Tx) (uint64, error) { } } -func (p *processor) Reorg(firstReorgedBlock uint64) error { - tx, err := p.db.BeginRw(context.Background()) +func (p *processor) Reorg(ctx context.Context, firstReorgedBlock uint64) error { + tx, err := p.db.BeginRw(ctx) if err != nil { return err } @@ -221,13 +216,7 @@ func (p *processor) Reorg(firstReorgedBlock uint64) error { return err } if firstDepositCountReorged != -1 { - var lastValidDepositCount uint32 - if firstDepositCountReorged == 0 { - lastValidDepositCount = 0 - } else { - lastValidDepositCount = uint32(firstDepositCountReorged) - 1 - } - if err := p.tree.reorg(tx, lastValidDepositCount); err != nil { + if err := p.exitTree.Reorg(ctx, uint32(firstDepositCountReorged)); err != nil { tx.Rollback() return err } @@ -235,8 +224,7 @@ func (p *processor) Reorg(firstReorgedBlock uint64) error { return tx.Commit() } -func (p *processor) ProcessBlock(block sync.Block) error { - ctx := context.Background() +func (p *processor) ProcessBlock(ctx context.Context, block sync.Block) error { tx, err := p.db.BeginRw(ctx) if err != nil { return err @@ -267,19 +255,16 @@ func (p *processor) ProcessBlock(block sync.Block) error { return err } - for i, bridge := range bridges { - if err := p.tree.addLeaf(tx, bridge.DepositCount, bridge.Hash()); err != nil { - if i != 0 { - tx.Rollback() - if err2 := p.tree.initLastLeftCacheAndLastDepositCount(ctx); err2 != nil { - log.Fatalf( - "after failing to add a leaf to the tree with error: %v, error initializing the cache with error: %v", - err, err2, - ) - } - return err - } - } + leaves := []tree.Leaf{} + for _, bridge := range bridges { + leaves = append(leaves, tree.Leaf{ + Index: bridge.DepositCount, + Hash: bridge.Hash(), + }) + } + if err := p.exitTree.AddLeaves(ctx, leaves); err != nil { + tx.Rollback() + return err } return tx.Commit() } diff --git a/bridgesync/processor_test.go b/bridgesync/processor_test.go index c1535d70..7d337d08 100644 --- a/bridgesync/processor_test.go +++ b/bridgesync/processor_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/0xPolygon/cdk/sync" + "github.com/0xPolygon/cdk/tree/testvectors" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) @@ -391,7 +392,7 @@ func (a *reorgAction) desc() string { } func (a *reorgAction) execute(t *testing.T) { - actualErr := a.p.Reorg(a.firstReorgedBlock) + actualErr := a.p.Reorg(context.Background(), a.firstReorgedBlock) require.Equal(t, a.expectedErr, actualErr) } @@ -413,7 +414,7 @@ func (a *processBlockAction) desc() string { } func (a *processBlockAction) execute(t *testing.T) { - actualErr := a.p.ProcessBlock(a.block) + actualErr := a.p.ProcessBlock(context.Background(), a.block) require.Equal(t, a.expectedErr, actualErr) } @@ -425,23 +426,11 @@ func eventsToBridgeEvents(events []interface{}) []Event { return bridgeEvents } -// DepositVectorRaw represents the deposit vector -type DepositVectorRaw struct { - OriginalNetwork uint32 `json:"originNetwork"` - TokenAddress string `json:"tokenAddress"` - Amount string `json:"amount"` - DestinationNetwork uint32 `json:"destinationNetwork"` - DestinationAddress string `json:"destinationAddress"` - ExpectedHash string `json:"leafValue"` - CurrentHash string `json:"currentLeafValue"` - Metadata string `json:"metadata"` -} - func TestHashBridge(t *testing.T) { - data, err := os.ReadFile("testvectors/leaf-vectors.json") + data, err := os.ReadFile("../tree/testvectors/leaf-vectors.json") require.NoError(t, err) - var leafVectors []DepositVectorRaw + var leafVectors []testvectors.DepositVectorRaw err = json.Unmarshal(data, &leafVectors) require.NoError(t, err) diff --git a/bridgesync/tree.go b/bridgesync/tree.go deleted file mode 100644 index 9972af64..00000000 --- a/bridgesync/tree.go +++ /dev/null @@ -1,325 +0,0 @@ -package bridgesync - -import ( - "context" - "fmt" - "math" - - dbCommon "github.com/0xPolygon/cdk/common" - "github.com/ethereum/go-ethereum/common" - "github.com/ledgerwatch/erigon-lib/kv" - "golang.org/x/crypto/sha3" -) - -const ( - defaultHeight uint8 = 32 -) - -type tree struct { - db kv.RwDB - rhtTable string - rootTable string - height uint8 - lastDepositCount int64 - lastLeftCache []common.Hash - zeroHashes []common.Hash -} - -type treeNode struct { - left common.Hash - right common.Hash -} - -func (n *treeNode) hash() common.Hash { - var hash common.Hash - hasher := sha3.NewLegacyKeccak256() - hasher.Write(n.left[:]) - hasher.Write(n.right[:]) - copy(hash[:], hasher.Sum(nil)) - return hash -} - -func (n *treeNode) MarshalBinary() ([]byte, error) { - return append(n.left[:], n.right[:]...), nil -} - -func (n *treeNode) UnmarshalBinary(data []byte) error { - if len(data) != 64 { - return fmt.Errorf("expected len %d, actual len %d", 64, len(data)) - } - n.left = common.Hash(data[:32]) - n.right = common.Hash(data[32:]) - return nil -} - -func newTree(ctx context.Context, rhtTable, rootTable string, db kv.RwDB) (*tree, error) { - t := &tree{ - rhtTable: rhtTable, - rootTable: rootTable, - db: db, - height: defaultHeight, - zeroHashes: generateZeroHashes(defaultHeight), - } - - if err := t.initLastLeftCacheAndLastDepositCount(ctx); err != nil { - return nil, err - } - - return t, nil -} - -// getProof returns the merkle proof for a given deposit count and root. -func (t *tree) getProof(ctx context.Context, depositCount uint32, root common.Hash) ([]common.Hash, error) { - tx, err := t.db.BeginRw(ctx) - if err != nil { - return nil, err - } - defer tx.Rollback() - siblings := make([]common.Hash, int(t.height)) - - currentNodeHash := root - // It starts in height-1 because 0 is the level of the leafs - for h := int(t.height - 1); h >= 0; h-- { - currentNode, err := t.getRHTNode(tx, currentNodeHash) - if err != nil { - return nil, fmt.Errorf( - "height: %d, currentNode: %s, error: %v", - h, currentNodeHash.Hex(), err, - ) - } - /* - * Root (level h=3 => height=4) - * / \ - * O5 O6 (level h=2) - * / \ / \ - * O1 O2 O3 O4 (level h=1) - * /\ /\ /\ /\ - * 0 1 2 3 4 5 6 7 Leafs (level h=0) - * Example 1: - * Choose index = 3 => 011 binary - * Assuming we are in level 1 => h=1; 1< 011&010=010 which is higher than 0 so we need the left sibling (O1) - * Example 2: - * Choose index = 4 => 100 binary - * Assuming we are in level 1 => h=1; 1< 100&010=000 which is not higher than 0 so we need the right sibling (O4) - * Example 3: - * Choose index = 4 => 100 binary - * Assuming we are in level 2 => h=2; 1< 100&100=100 which is higher than 0 so we need the left sibling (O5) - */ - if depositCount&(1< 0 { - siblings = append(siblings, currentNode.left) - currentNodeHash = currentNode.right - } else { - siblings = append(siblings, currentNode.right) - currentNodeHash = currentNode.left - } - } - - // Reverse siblings to go from leafs to root - for i, j := 0, len(siblings)-1; i < j; i, j = i+1, j-1 { - siblings[i], siblings[j] = siblings[j], siblings[i] - } - - return siblings, nil -} - -func (t *tree) addLeaf(tx kv.RwTx, depositCount uint32, hash common.Hash) error { - // Sanity check - if int64(depositCount) != t.lastDepositCount+1 { - return fmt.Errorf( - "mismatched index. Expected: %d, actual: %d", - t.lastDepositCount+1, depositCount, - ) - } - - // Calculate new tree nodes - currentChildHash := hash - newNodes := []treeNode{} - for h := uint8(0); h < t.height; h++ { - var parent treeNode - if depositCount&(1< 0 { - // Add child to the right - parent = treeNode{ - left: t.lastLeftCache[h], - right: currentChildHash, - } - } else { - // Add child to the left - parent = treeNode{ - left: currentChildHash, - right: t.zeroHashes[h], - } - // Update cache - // TODO: review this part of the logic, skipping ?optimizaton? - // from OG implementation - t.lastLeftCache[h] = currentChildHash - } - currentChildHash = parent.hash() - newNodes = append(newNodes, parent) - } - - // store root - root := currentChildHash - if err := tx.Put(t.rootTable, dbCommon.Uint32ToBytes(depositCount), root[:]); err != nil { - return err - } - - // store nodes - for _, node := range newNodes { - value, err := node.MarshalBinary() - if err != nil { - return err - } - if err := tx.Put(t.rhtTable, node.hash().Bytes(), value); err != nil { - return err - } - } - - t.lastDepositCount++ - return nil -} - -func (t *tree) initLastLeftCacheAndLastDepositCount(ctx context.Context) error { - tx, err := t.db.BeginRw(ctx) - if err != nil { - return err - } - defer tx.Rollback() - - root, err := t.initLastDepositCount(tx) - if err != nil { - return err - } - return t.initLastLeftCache(tx, t.lastDepositCount, root) -} - -// getLastDepositCountAndRoot return the deposit count and the root associated to the last deposit. -// If deposit count == -1, it means no deposit added yet -func (t *tree) getLastDepositCountAndRoot(tx kv.Tx) (int64, common.Hash, error) { - iter, err := tx.RangeDescend( - t.rootTable, - dbCommon.Uint32ToBytes(math.MaxUint32), - dbCommon.Uint32ToBytes(0), - 1, - ) - if err != nil { - return 0, common.Hash{}, err - } - - lastDepositCountBytes, rootBytes, err := iter.Next() - if err != nil { - return 0, common.Hash{}, err - } - if lastDepositCountBytes == nil { - return -1, common.Hash{}, nil - } - return int64(dbCommon.BytesToUint32(lastDepositCountBytes)), common.Hash(rootBytes), nil -} - -func (t *tree) initLastDepositCount(tx kv.Tx) (common.Hash, error) { - ldc, root, err := t.getLastDepositCountAndRoot(tx) - if err != nil { - return common.Hash{}, err - } - t.lastDepositCount = ldc - return root, nil -} - -func (t *tree) initLastLeftCache(tx kv.Tx, lastDepositCount int64, lastRoot common.Hash) error { - siblings := make([]common.Hash, t.height, t.height) - if lastDepositCount == -1 { - t.lastLeftCache = siblings - return nil - } - index := lastDepositCount - - currentNodeHash := lastRoot - // It starts in height-1 because 0 is the level of the leafs - for h := int(t.height - 1); h >= 0; h-- { - currentNode, err := t.getRHTNode(tx, currentNodeHash) - if err != nil { - return fmt.Errorf( - "error getting node %s from the RHT at height %d with root %s: %v", - currentNodeHash.Hex(), h, lastRoot.Hex(), err, - ) - } - if currentNode == nil { - return ErrNotFound - } - siblings = append(siblings, currentNode.left) - if index&(1< 0 { - currentNodeHash = currentNode.right - } else { - currentNodeHash = currentNode.left - } - } - - // Reverse the siblings to go from leafs to root - for i, j := 0, len(siblings)-1; i < j; i, j = i+1, j-1 { - siblings[i], siblings[j] = siblings[j], siblings[i] - } - - t.lastLeftCache = siblings - return nil -} - -func (t *tree) getRHTNode(tx kv.Tx, nodeHash common.Hash) (*treeNode, error) { - nodeBytes, err := tx.GetOne(t.rhtTable, nodeHash[:]) - if err != nil { - return nil, err - } - if nodeBytes == nil { - return nil, ErrNotFound - } - node := &treeNode{} - err = node.UnmarshalBinary(nodeBytes) - return node, err -} - -func (t *tree) reorg(tx kv.RwTx, lastValidDepositCount uint32) error { - if t.lastDepositCount == -1 { - return nil - } - // Clean root table - for i := lastValidDepositCount + 1; i <= uint32(t.lastDepositCount); i++ { - if err := tx.Delete(t.rootTable, dbCommon.Uint32ToBytes(i)); err != nil { - return err - } - } - - // Reset cache - rootBytes, err := tx.GetOne(t.rootTable, dbCommon.Uint32ToBytes(lastValidDepositCount)) - if err != nil { - return err - } - if rootBytes == nil { - return ErrNotFound - } - err = t.initLastLeftCache(tx, int64(lastValidDepositCount), common.Hash(rootBytes)) // 0x619a9fedbe029225288d32e39e06fb868ed0d8f20db26047cf0ef8d3582b5f6e - if err != nil { - return err - } - - // Note: not cleaning RHT, not worth it - t.lastDepositCount = int64(lastValidDepositCount) - return nil -} - -func generateZeroHashes(height uint8) []common.Hash { - var zeroHashes = []common.Hash{ - {}, - } - // This generates a leaf = HashZero in position 0. In the rest of the positions that are equivalent to the ascending levels, - // we set the hashes of the nodes. So all nodes from level i=5 will have the same value and same children nodes. - for i := 1; i <= int(height); i++ { - hasher := sha3.NewLegacyKeccak256() - hasher.Write(zeroHashes[i-1][:]) - hasher.Write(zeroHashes[i-1][:]) - thisHeightHash := common.Hash{} - copy(thisHeightHash[:], hasher.Sum(nil)) - zeroHashes = append(zeroHashes, thisHeightHash) - } - return zeroHashes -} diff --git a/bridgesync/tree_test.go b/bridgesync/tree_test.go deleted file mode 100644 index 5624ad47..00000000 --- a/bridgesync/tree_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package bridgesync - -import ( - "context" - "encoding/json" - "fmt" - "math/big" - "os" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/require" -) - -// MTRootVectorRaw represents the root of Merkle Tree -type MTRootVectorRaw struct { - ExistingLeaves []string `json:"previousLeafsValues"` - CurrentRoot string `json:"currentRoot"` - NewLeaf DepositVectorRaw `json:"newLeaf"` - NewRoot string `json:"newRoot"` -} - -func TestMTAddLeaf(t *testing.T) { - data, err := os.ReadFile("testvectors/root-vectors.json") - require.NoError(t, err) - - var mtTestVectors []MTRootVectorRaw - err = json.Unmarshal(data, &mtTestVectors) - require.NoError(t, err) - ctx := context.Background() - - for ti, testVector := range mtTestVectors { - t.Run(fmt.Sprintf("Test vector %d", ti), func(t *testing.T) { - - path := t.TempDir() - p, err := newProcessor(context.Background(), path, "foo") - require.NoError(t, err) - - // Add exisiting leaves - for i, leaf := range testVector.ExistingLeaves { - tx, err := p.db.BeginRw(ctx) - require.NoError(t, err) - err = p.tree.addLeaf(tx, uint32(i), common.HexToHash(leaf)) - require.NoError(t, err) - err = tx.Commit() - require.NoError(t, err) - } - if len(testVector.ExistingLeaves) > 0 { - txRo, err := p.db.BeginRo(ctx) - require.NoError(t, err) - _, actualRoot, err := p.tree.getLastDepositCountAndRoot(txRo) - txRo.Rollback() - require.NoError(t, err) - require.Equal(t, common.HexToHash(testVector.CurrentRoot), actualRoot) - } - - // Add new bridge - amount, result := big.NewInt(0).SetString(testVector.NewLeaf.Amount, 0) - require.True(t, result) - bridge := Bridge{ - OriginNetwork: testVector.NewLeaf.OriginalNetwork, - OriginAddress: common.HexToAddress(testVector.NewLeaf.TokenAddress), - Amount: amount, - DestinationNetwork: testVector.NewLeaf.DestinationNetwork, - DestinationAddress: common.HexToAddress(testVector.NewLeaf.DestinationAddress), - DepositCount: uint32(len(testVector.ExistingLeaves)), - Metadata: common.FromHex(testVector.NewLeaf.Metadata), - } - tx, err := p.db.BeginRw(ctx) - require.NoError(t, err) - require.Equal(t, common.HexToHash(testVector.NewLeaf.CurrentHash), bridge.Hash()) - err = p.tree.addLeaf(tx, bridge.DepositCount, bridge.Hash()) - require.NoError(t, err) - err = tx.Commit() - txRo, err := p.db.BeginRo(ctx) - require.NoError(t, err) - _, actualRoot, err := p.tree.getLastDepositCountAndRoot(txRo) - txRo.Rollback() - require.NoError(t, err) - require.Equal(t, common.HexToHash(testVector.NewRoot), actualRoot) - }) - } -} - -// MTClaimVectorRaw represents the merkle proof -type MTClaimVectorRaw struct { - Deposits []DepositVectorRaw `json:"leafs"` - Index uint32 `json:"index"` - MerkleProof []string `json:"proof"` - ExpectedRoot string `json:"root"` -} - -func TestMTGetProof(t *testing.T) { - data, err := os.ReadFile("testvectors/claim-vectors.json") - require.NoError(t, err) - - var mtTestVectors []MTClaimVectorRaw - err = json.Unmarshal(data, &mtTestVectors) - require.NoError(t, err) - ctx := context.Background() - - for ti, testVector := range mtTestVectors { - t.Run(fmt.Sprintf("Test vector %d", ti), func(t *testing.T) { - path := t.TempDir() - p, err := newProcessor(context.Background(), path, "foo") - require.NoError(t, err) - - for li, leaf := range testVector.Deposits { - amount, result := big.NewInt(0).SetString(leaf.Amount, 0) - require.True(t, result) - bridge := &Bridge{ - OriginNetwork: leaf.OriginalNetwork, - OriginAddress: common.HexToAddress(leaf.TokenAddress), - Amount: amount, - DestinationNetwork: leaf.DestinationNetwork, - DestinationAddress: common.HexToAddress(leaf.DestinationAddress), - DepositCount: uint32(li), - Metadata: common.FromHex(leaf.Metadata), - } - tx, err := p.db.BeginRw(ctx) - require.NoError(t, err) - err = p.tree.addLeaf(tx, bridge.DepositCount, bridge.Hash()) - require.NoError(t, err) - err = tx.Commit() - require.NoError(t, err) - } - txRo, err := p.db.BeginRo(ctx) - require.NoError(t, err) - _, actualRoot, err := p.tree.getLastDepositCountAndRoot(txRo) - txRo.Rollback() - expectedRoot := common.HexToHash(testVector.ExpectedRoot) - require.Equal(t, expectedRoot, actualRoot) - - proof, err := p.tree.getProof(ctx, testVector.Index, expectedRoot) - require.NoError(t, err) - for i, sibling := range testVector.MerkleProof { - require.Equal(t, common.HexToHash(sibling), proof[i]) - } - }) - } -} diff --git a/l1infotreesync/downloader.go b/l1infotreesync/downloader.go index 02f4f14d..dbe6950a 100644 --- a/l1infotreesync/downloader.go +++ b/l1infotreesync/downloader.go @@ -3,8 +3,8 @@ package l1infotreesync import ( "fmt" - "github.com/0xPolygon/cdk-contracts-tooling/contracts/elderberry/polygonrollupmanager" "github.com/0xPolygon/cdk-contracts-tooling/contracts/elderberry/polygonzkevmglobalexitrootv2" + "github.com/0xPolygon/cdk-contracts-tooling/contracts/etrog/polygonrollupmanager" "github.com/0xPolygon/cdk/sync" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -28,6 +28,9 @@ type EthClienter interface { func buildAppender(client EthClienter, globalExitRoot, rollupManager common.Address) (sync.LogAppenderMap, error) { ger, err := polygonzkevmglobalexitrootv2.NewPolygonzkevmglobalexitrootv2(globalExitRoot, client) + if err != nil { + return nil, err + } rm, err := polygonrollupmanager.NewPolygonrollupmanager(rollupManager, client) if err != nil { return nil, err @@ -67,6 +70,24 @@ func buildAppender(client EthClienter, globalExitRoot, rollupManager common.Addr }}) return nil } + appender[verifyBatchesTrustedAggregatorSignature] = func(b *sync.EVMBlock, l types.Log) error { + verifyBatches, err := rm.ParseVerifyBatchesTrustedAggregator(l) + if err != nil { + return fmt.Errorf( + "error parsing log %+v using rm.ParseVerifyBatches: %v", + l, err, + ) + } + fmt.Println(verifyBatches) + b.Events = append(b.Events, Event{VerifyBatches: &VerifyBatches{ + RollupID: verifyBatches.RollupID, + NumBatch: verifyBatches.NumBatch, + StateRoot: verifyBatches.StateRoot, + ExitRoot: verifyBatches.ExitRoot, + Aggregator: verifyBatches.Aggregator, + }}) + return nil + } return appender, nil } diff --git a/l1infotreesync/l1infotreesync.go b/l1infotreesync/l1infotreesync.go index 7c45356c..67342613 100644 --- a/l1infotreesync/l1infotreesync.go +++ b/l1infotreesync/l1infotreesync.go @@ -58,7 +58,7 @@ func New( return nil, err } if lastProcessedBlock < initialBlock { - err = processor.ProcessBlock(sync.Block{ + err = processor.ProcessBlock(ctx, sync.Block{ Num: initialBlock, }) if err != nil { @@ -76,7 +76,7 @@ func New( blockFinalityType, waitForNewBlocksPeriod, appender, - []common.Address{globalExitRoot}, + []common.Address{globalExitRoot, rollupManager}, ) if err != nil { return nil, err diff --git a/l1infotreesync/processor.go b/l1infotreesync/processor.go index 4821e151..927eff82 100644 --- a/l1infotreesync/processor.go +++ b/l1infotreesync/processor.go @@ -5,11 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "path" "github.com/0xPolygon/cdk/common" "github.com/0xPolygon/cdk/l1infotree" - "github.com/0xPolygon/cdk/log" "github.com/0xPolygon/cdk/sync" + "github.com/0xPolygon/cdk/tree" ethCommon "github.com/ethereum/go-ethereum/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" @@ -17,30 +18,31 @@ import ( ) const ( + dbPrefix = "l1infotreesync" // rootTable stores the L1 info tree roots // Key: root (common.Hash) // Value: hash of the leaf that caused the update (common.Hash) - rootTable = "l1infotreesync-root" + rootTable = dbPrefix + "-root" // indexTable stores the L1 info tree indexes // Key: index (uint32 converted to bytes) // Value: hash of the leaf that caused the update (common.Hash) - indexTable = "l1infotreesync-index" + indexTable = dbPrefix + "-index" // infoTable stores the information of the tree (the leaves). Note that the value // of rootTable and indexTable references the key of the infoTable // Key: hash of the leaf that caused the update (common.Hash) // Value: JSON of storeLeaf struct - infoTable = "l1infotreesync-info" + infoTable = dbPrefix + "-info" // blockTable stores the first and last index of L1 Info Tree that have been updated on // a block. This is useful in case there are blocks with multiple updates and a reorg is needed. // Or for when querying by block number // Key: block number (uint64 converted to bytes) // Value: JSON of blockWithLeafs - blockTable = "l1infotreesync-block" + blockTable = dbPrefix + "-block" // lastBlockTable used to store the last block processed. This is needed to know the last processed blcok // when it doesn't have events that make other tables get populated // Key: it's always lastBlockKey // Value: block number (uint64 converted to bytes) - lastBlockTable = "l1infotreesync-lastBlock" + lastBlockTable = dbPrefix + "-lastBlock" treeHeight uint8 = 32 ) @@ -55,7 +57,7 @@ var ( type processor struct { db kv.RwDB l1InfoTree *l1infotree.L1InfoTree - rollupExitTree *rollupExitTree + rollupExitTree *tree.UpdatableTree } type UpdateL1InfoTree struct { @@ -140,12 +142,16 @@ func newProcessor(ctx context.Context, dbPath string) (*processor, error) { if err != nil { return nil, err } - tree, err := l1infotree.NewL1InfoTree(treeHeight, leaves) + l1InfoTree, err := l1infotree.NewL1InfoTree(treeHeight, leaves) + if err != nil { + return nil, err + } + p.l1InfoTree = l1InfoTree + rollupExitTreeDBPath := path.Join(dbPath, "rollupExitTree") + rollupExitTree, err := tree.NewUpdatable(ctx, rollupExitTreeDBPath, dbPrefix) if err != nil { return nil, err } - p.l1InfoTree = tree - rollupExitTree := newRollupExitTree() p.rollupExitTree = rollupExitTree return p, nil } @@ -338,9 +344,9 @@ func (p *processor) getLastProcessedBlockWithTx(tx kv.Tx) (uint64, error) { return common.BytesToUint64(blockNumBytes), nil } -func (p *processor) Reorg(firstReorgedBlock uint64) error { +func (p *processor) Reorg(ctx context.Context, firstReorgedBlock uint64) error { // TODO: Does tree need to be reorged? - tx, err := p.db.BeginRw(context.Background()) + tx, err := p.db.BeginRw(ctx) if err != nil { return err } @@ -415,8 +421,8 @@ func (p *processor) deleteLeaf(tx kv.RwTx, index uint32) error { // ProcessBlock process the leafs of the L1 info tree found on a block // this function can be called without leafs with the intention to track the last processed block -func (p *processor) ProcessBlock(b sync.Block) error { - tx, err := p.db.BeginRw(context.Background()) +func (p *processor) ProcessBlock(ctx context.Context, b sync.Block) error { + tx, err := p.db.BeginRw(ctx) if err != nil { return err } @@ -455,8 +461,9 @@ func (p *processor) ProcessBlock(b sync.Block) error { // Since the previous event include the rollup exit root, this can use it to assert // that the computation of the tree is correct. However, there are some execution paths // on the contract that don't follow this (verifyBatches + pendingStateTimeout != 0) - if err := p.rollupExitTree.addLeaf( - tx, event.VerifyBatches.RollupID, + if err := p.rollupExitTree.UpsertLeaf( + ctx, + event.VerifyBatches.RollupID, event.VerifyBatches.ExitRoot, nextExpectedRollupExitTreeRoot, ); err != nil { @@ -484,7 +491,6 @@ func (p *processor) ProcessBlock(b sync.Block) error { tx.Rollback() return err } - log.Debugf("block %d processed with events: %+v", b.Num, b.Events) return tx.Commit() } diff --git a/l1infotreesync/rollupexittree.go b/l1infotreesync/rollupexittree.go deleted file mode 100644 index 2004fcdd..00000000 --- a/l1infotreesync/rollupexittree.go +++ /dev/null @@ -1,119 +0,0 @@ -package l1infotreesync - -import ( - "fmt" - - "github.com/ethereum/go-ethereum/common" - "github.com/ledgerwatch/erigon-lib/kv" - "golang.org/x/crypto/sha3" -) - -type treeNode struct { - left common.Hash - right common.Hash -} - -func (n *treeNode) hash() common.Hash { - var hash common.Hash - hasher := sha3.NewLegacyKeccak256() - hasher.Write(n.left[:]) - hasher.Write(n.right[:]) - copy(hash[:], hasher.Sum(nil)) - return hash -} - -func (n *treeNode) MarshalBinary() ([]byte, error) { - return append(n.left[:], n.right[:]...), nil -} - -func (n *treeNode) UnmarshalBinary(data []byte) error { - if len(data) != 64 { - return fmt.Errorf("expected len %d, actual len %d", 64, len(data)) - } - n.left = common.Hash(data[:32]) - n.right = common.Hash(data[32:]) - return nil -} - -type rollupExitTree struct { - height uint8 - rhtTable string - lastExitTreeRoot common.Hash -} - -func newRollupExitTree() *rollupExitTree { - return &rollupExitTree{} -} - -func (t *rollupExitTree) addLeaf( - tx kv.RwTx, - rollupID uint32, - rollupExitRoot common.Hash, - expectedRollupExitRoot *common.Hash, -) error { - siblings, err := t.getProof(tx, rollupID, t.lastExitTreeRoot) - if err != nil { - return err - } - if expectedRollupExitRoot != nil && *expectedRollupExitRoot != t.lastExitTreeRoot { - return fmt.Errorf( - "expectedRollupExitRoot: %s, actual: %s", - expectedRollupExitRoot.Hex(), t.lastExitTreeRoot.Hex(), - ) - } - return nil -} - -func (t *rollupExitTree) getSiblings(tx kv.RwTx, rollupID uint32, root common.Hash) (bool, []common.Hash, error) { - siblings := make([]common.Hash, int(t.height)) - - currentNodeHash := root - // It starts in height-1 because 0 is the level of the leafs - for h := int(t.height - 1); h >= 0; h-- { - currentNode, err := t.getRHTNode(tx, currentNodeHash) - if err != nil { - // handle not found for inserts and shit - return false, nil, fmt.Errorf( - "height: %d, currentNode: %s, error: %v", - h, currentNodeHash.Hex(), err, - ) - } - if rollupID&(1< 0 { - siblings = append(siblings, currentNode.left) - currentNodeHash = currentNode.right - } else { - siblings = append(siblings, currentNode.right) - currentNodeHash = currentNode.left - } - } - - // Reverse siblings to go from leafs to root - for i, j := 0, len(siblings)-1; i < j; i, j = i+1, j-1 { - siblings[i], siblings[j] = siblings[j], siblings[i] - } - - return false, siblings, nil - -} - -// getProof returns the merkle proof for a given deposit count and root. -func (t *rollupExitTree) getProof(tx kv.RwTx, rollupID uint32, root common.Hash) ([]common.Hash, error) { - usedEmptyTree, siblings, err := t.getSiblings(tx, rollupID, root) - if usedEmptyTree { - return nil, ErrNotFound - } - return siblings, err -} - -func (t *rollupExitTree) getRHTNode(tx kv.Tx, nodeHash common.Hash) (*treeNode, error) { - nodeBytes, err := tx.GetOne(t.rhtTable, nodeHash[:]) - if err != nil { - return nil, err - } - if nodeBytes == nil { - return nil, ErrNotFound - } - node := &treeNode{} - err = node.UnmarshalBinary(nodeBytes) - return node, err -} diff --git a/sync/evmdownloader.go b/sync/evmdownloader.go index ebdde880..9a5efd5f 100644 --- a/sync/evmdownloader.go +++ b/sync/evmdownloader.go @@ -46,9 +46,9 @@ func NewEVMDownloader( if err != nil { return nil, err } - topicsToQuery := [][]common.Hash{} + topicsToQuery := []common.Hash{} for topic := range appender { - topicsToQuery = append(topicsToQuery, []common.Hash{topic}) + topicsToQuery = append(topicsToQuery, topic) } return &EVMDownloader{ syncBlockChunkSize: syncBlockChunkSize, @@ -57,7 +57,7 @@ func NewEVMDownloader( blockFinality: finality, waitForNewBlocksPeriod: waitForNewBlocksPeriod, appender: appender, - topicsToQuery: topicsToQuery, + topicsToQuery: [][]common.Hash{topicsToQuery}, adressessToQuery: adressessToQuery, }, }, nil @@ -90,7 +90,7 @@ func (d *EVMDownloader) download(ctx context.Context, fromBlock uint64, download } if len(blocks) == 0 || blocks[len(blocks)-1].Num < toBlock { // Indicate the last downloaded block if there are not events on it - log.Debugf("sending block %d to the driver (without evvents)", toBlock) + log.Debugf("sending block %d to the driver (without events)", toBlock) downloadedCh <- EVMBlock{ EVMBlockHeader: d.getBlockHeader(ctx, toBlock), } diff --git a/sync/evmdriver.go b/sync/evmdriver.go index a30b96d6..9eabe644 100644 --- a/sync/evmdriver.go +++ b/sync/evmdriver.go @@ -24,8 +24,8 @@ type EVMDriver struct { type processorInterface interface { GetLastProcessedBlock(ctx context.Context) (uint64, error) - ProcessBlock(block Block) error - Reorg(firstReorgedBlock uint64) error + ProcessBlock(ctx context.Context, block Block) error + Reorg(ctx context.Context, firstReorgedBlock uint64) error } type ReorgDetector interface { @@ -85,7 +85,7 @@ reset: d.handleNewBlock(ctx, b) case firstReorgedBlock := <-d.reorgSub.FirstReorgedBlock: log.Debug("handleReorg") - d.handleReorg(cancel, downloadCh, firstReorgedBlock) + d.handleReorg(ctx, cancel, downloadCh, firstReorgedBlock) goto reset } } @@ -109,7 +109,7 @@ func (d *EVMDriver) handleNewBlock(ctx context.Context, b EVMBlock) { Num: b.Num, Events: b.Events, } - err := d.processor.ProcessBlock(blockToProcess) + err := d.processor.ProcessBlock(ctx, blockToProcess) if err != nil { attempts++ log.Errorf("error processing events for blcok %d, err: ", b.Num, err) @@ -121,7 +121,7 @@ func (d *EVMDriver) handleNewBlock(ctx context.Context, b EVMBlock) { } func (d *EVMDriver) handleReorg( - cancel context.CancelFunc, downloadCh chan EVMBlock, firstReorgedBlock uint64, + ctx context.Context, cancel context.CancelFunc, downloadCh chan EVMBlock, firstReorgedBlock uint64, ) { // stop downloader cancel() @@ -132,7 +132,7 @@ func (d *EVMDriver) handleReorg( // handle reorg attempts := 0 for { - err := d.processor.Reorg(firstReorgedBlock) + err := d.processor.Reorg(ctx, firstReorgedBlock) if err != nil { attempts++ log.Errorf( diff --git a/sync/evmdriver_test.go b/sync/evmdriver_test.go index 502722f6..a09b94a8 100644 --- a/sync/evmdriver_test.go +++ b/sync/evmdriver_test.go @@ -78,18 +78,18 @@ func TestSync(t *testing.T) { Return(uint64(3), nil) rdm.On("AddBlockToTrack", ctx, reorgDetectorID, expectedBlock1.Num, expectedBlock1.Hash). Return(nil) - pm.On("ProcessBlock", Block{Num: expectedBlock1.Num, Events: expectedBlock1.Events}). + pm.On("ProcessBlock", ctx, Block{Num: expectedBlock1.Num, Events: expectedBlock1.Events}). Return(nil) rdm.On("AddBlockToTrack", ctx, reorgDetectorID, expectedBlock2.Num, expectedBlock2.Hash). Return(nil) - pm.On("ProcessBlock", Block{Num: expectedBlock2.Num, Events: expectedBlock2.Events}). + pm.On("ProcessBlock", ctx, Block{Num: expectedBlock2.Num, Events: expectedBlock2.Events}). Return(nil) go driver.Sync(ctx) time.Sleep(time.Millisecond * 200) // time to download expectedBlock1 // Trigger reorg 1 reorgedBlock1 := uint64(5) - pm.On("Reorg", reorgedBlock1).Return(nil) + pm.On("Reorg", ctx, reorgedBlock1).Return(nil) firstReorgedBlock <- reorgedBlock1 ok := <-reorgProcessed require.True(t, ok) @@ -100,7 +100,7 @@ func TestSync(t *testing.T) { // Trigger reorg 2: syncer restarts the porcess reorgedBlock2 := uint64(7) - pm.On("Reorg", reorgedBlock2).Return(nil) + pm.On("Reorg", ctx, reorgedBlock2).Return(nil) firstReorgedBlock <- reorgedBlock2 ok = <-reorgProcessed require.True(t, ok) @@ -126,7 +126,7 @@ func TestHandleNewBlock(t *testing.T) { rdm. On("AddBlockToTrack", ctx, reorgDetectorID, b1.Num, b1.Hash). Return(nil) - pm.On("ProcessBlock", Block{Num: b1.Num, Events: b1.Events}). + pm.On("ProcessBlock", ctx, Block{Num: b1.Num, Events: b1.Events}). Return(nil) driver.handleNewBlock(ctx, b1) @@ -143,7 +143,7 @@ func TestHandleNewBlock(t *testing.T) { rdm. On("AddBlockToTrack", ctx, reorgDetectorID, b2.Num, b2.Hash). Return(nil).Once() - pm.On("ProcessBlock", Block{Num: b2.Num, Events: b2.Events}). + pm.On("ProcessBlock", ctx, Block{Num: b2.Num, Events: b2.Events}). Return(nil) driver.handleNewBlock(ctx, b2) @@ -157,9 +157,9 @@ func TestHandleNewBlock(t *testing.T) { rdm. On("AddBlockToTrack", ctx, reorgDetectorID, b3.Num, b3.Hash). Return(nil) - pm.On("ProcessBlock", Block{Num: b3.Num, Events: b3.Events}). + pm.On("ProcessBlock", ctx, Block{Num: b3.Num, Events: b3.Events}). Return(errors.New("foo")).Once() - pm.On("ProcessBlock", Block{Num: b3.Num, Events: b3.Events}). + pm.On("ProcessBlock", ctx, Block{Num: b3.Num, Events: b3.Events}). Return(nil).Once() driver.handleNewBlock(ctx, b3) @@ -182,8 +182,8 @@ func TestHandleReorg(t *testing.T) { _, cancel := context.WithCancel(ctx) downloadCh := make(chan EVMBlock) firstReorgedBlock := uint64(5) - pm.On("Reorg", firstReorgedBlock).Return(nil) - go driver.handleReorg(cancel, downloadCh, firstReorgedBlock) + pm.On("Reorg", ctx, firstReorgedBlock).Return(nil) + go driver.handleReorg(ctx, cancel, downloadCh, firstReorgedBlock) close(downloadCh) done := <-reorgProcessed require.True(t, done) @@ -192,8 +192,8 @@ func TestHandleReorg(t *testing.T) { _, cancel = context.WithCancel(ctx) downloadCh = make(chan EVMBlock) firstReorgedBlock = uint64(6) - pm.On("Reorg", firstReorgedBlock).Return(nil) - go driver.handleReorg(cancel, downloadCh, firstReorgedBlock) + pm.On("Reorg", ctx, firstReorgedBlock).Return(nil) + go driver.handleReorg(ctx, cancel, downloadCh, firstReorgedBlock) downloadCh <- EVMBlock{} downloadCh <- EVMBlock{} downloadCh <- EVMBlock{} @@ -205,10 +205,10 @@ func TestHandleReorg(t *testing.T) { _, cancel = context.WithCancel(ctx) downloadCh = make(chan EVMBlock) firstReorgedBlock = uint64(7) - pm.On("Reorg", firstReorgedBlock).Return(errors.New("foo")).Once() - pm.On("Reorg", firstReorgedBlock).Return(errors.New("foo")).Once() - pm.On("Reorg", firstReorgedBlock).Return(nil).Once() - go driver.handleReorg(cancel, downloadCh, firstReorgedBlock) + pm.On("Reorg", ctx, firstReorgedBlock).Return(errors.New("foo")).Once() + pm.On("Reorg", ctx, firstReorgedBlock).Return(errors.New("foo")).Once() + pm.On("Reorg", ctx, firstReorgedBlock).Return(nil).Once() + go driver.handleReorg(ctx, cancel, downloadCh, firstReorgedBlock) close(downloadCh) done = <-reorgProcessed require.True(t, done) diff --git a/sync/mock_processor_test.go b/sync/mock_processor_test.go index d2c3e299..19738ef5 100644 --- a/sync/mock_processor_test.go +++ b/sync/mock_processor_test.go @@ -37,13 +37,13 @@ func (_m *ProcessorMock) GetLastProcessedBlock(ctx context.Context) (uint64, err return r0, r1 } -// ProcessBlock provides a mock function with given fields: block -func (_m *ProcessorMock) ProcessBlock(block Block) error { - ret := _m.Called(block) +// ProcessBlock provides a mock function with given fields: ctx, block +func (_m *ProcessorMock) ProcessBlock(ctx context.Context, block Block) error { + ret := _m.Called(ctx, block) var r0 error - if rf, ok := ret.Get(0).(func(Block) error); ok { - r0 = rf(block) + if rf, ok := ret.Get(0).(func(context.Context, Block) error); ok { + r0 = rf(ctx, block) } else { r0 = ret.Error(0) } @@ -51,13 +51,13 @@ func (_m *ProcessorMock) ProcessBlock(block Block) error { return r0 } -// Reorg provides a mock function with given fields: firstReorgedBlock -func (_m *ProcessorMock) Reorg(firstReorgedBlock uint64) error { - ret := _m.Called(firstReorgedBlock) +// Reorg provides a mock function with given fields: ctx, firstReorgedBlock +func (_m *ProcessorMock) Reorg(ctx context.Context, firstReorgedBlock uint64) error { + ret := _m.Called(ctx, firstReorgedBlock) var r0 error - if rf, ok := ret.Get(0).(func(uint64) error); ok { - r0 = rf(firstReorgedBlock) + if rf, ok := ret.Get(0).(func(context.Context, uint64) error); ok { + r0 = rf(ctx, firstReorgedBlock) } else { r0 = ret.Error(0) } diff --git a/tree/appendonlytree.go b/tree/appendonlytree.go new file mode 100644 index 00000000..39719028 --- /dev/null +++ b/tree/appendonlytree.go @@ -0,0 +1,243 @@ +package tree + +import ( + "context" + "fmt" + "math" + + dbCommon "github.com/0xPolygon/cdk/common" + "github.com/ethereum/go-ethereum/common" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type AppendOnlyTree struct { + *Tree + lastLeftCache []common.Hash + lastIndex int64 +} + +func NewAppendOnly(ctx context.Context, dbPath, dbPrefix string) (*AppendOnlyTree, error) { + t, err := newTree(dbPath, dbPrefix) + if err != nil { + return nil, err + } + at := &AppendOnlyTree{Tree: t} + if err := at.initLastLeftCacheAndLastDepositCount(ctx); err != nil { + return nil, err + } + return at, nil +} + +// AddLeaves adds a list leaves into the tree +func (t *AppendOnlyTree) AddLeaves(ctx context.Context, leaves []Leaf) error { + // Sanity check + if len(leaves) == 0 { + return nil + } + if int64(leaves[0].Index) != t.lastIndex+1 { + return fmt.Errorf( + "mismatched index. Expected: %d, actual: %d", + t.lastIndex+1, leaves[0].Index, + ) + } + tx, err := t.db.BeginRw(ctx) + if err != nil { + return err + } + backupIndx := t.lastIndex + backupCache := make([]common.Hash, len(t.lastLeftCache)) + copy(backupCache, t.lastLeftCache) + + for _, leaf := range leaves { + if err := t.addLeaf(tx, leaf); err != nil { + tx.Rollback() + t.lastIndex = backupIndx + t.lastLeftCache = backupCache + return err + } + } + + if err := tx.Commit(); err != nil { + t.lastIndex = backupIndx + t.lastLeftCache = backupCache + return err + } + return nil +} + +func (t *AppendOnlyTree) addLeaf(tx kv.RwTx, leaf Leaf) error { + // Calculate new tree nodes + currentChildHash := leaf.Hash + newNodes := []treeNode{} + for h := uint8(0); h < t.height; h++ { + var parent treeNode + if leaf.Index&(1< 0 { + // Add child to the right + parent = treeNode{ + left: t.lastLeftCache[h], + right: currentChildHash, + } + } else { + // Add child to the left + parent = treeNode{ + left: currentChildHash, + right: t.zeroHashes[h], + } + // Update cache + // TODO: review this part of the logic, skipping ?optimizaton? + // from OG implementation + t.lastLeftCache[h] = currentChildHash + } + currentChildHash = parent.hash() + newNodes = append(newNodes, parent) + } + + // store root + root := currentChildHash + if err := tx.Put(t.rootTable, dbCommon.Uint32ToBytes(leaf.Index), root[:]); err != nil { + return err + } + + // store nodes + for _, node := range newNodes { + value, err := node.MarshalBinary() + if err != nil { + return err + } + if err := tx.Put(t.rhtTable, node.hash().Bytes(), value); err != nil { + return err + } + } + + t.lastIndex++ + return nil +} + +func (t *AppendOnlyTree) initLastLeftCacheAndLastDepositCount(ctx context.Context) error { + tx, err := t.db.BeginRw(ctx) + if err != nil { + return err + } + defer tx.Rollback() + + root, err := t.initLastIndex(tx) + if err != nil { + return err + } + return t.initLastLeftCache(tx, t.lastIndex, root) +} + +// getLastIndexAndRoot return the index and the root associated to the last leaf inserted. +// If index == -1, it means no leaf added yet +func (t *AppendOnlyTree) getLastIndexAndRoot(tx kv.Tx) (int64, common.Hash, error) { + iter, err := tx.RangeDescend( + t.rootTable, + dbCommon.Uint32ToBytes(math.MaxUint32), + dbCommon.Uint32ToBytes(0), + 1, + ) + if err != nil { + return 0, common.Hash{}, err + } + + lastIndexBytes, rootBytes, err := iter.Next() + if err != nil { + return 0, common.Hash{}, err + } + if lastIndexBytes == nil { + return -1, common.Hash{}, nil + } + return int64(dbCommon.BytesToUint32(lastIndexBytes)), common.Hash(rootBytes), nil +} + +func (t *AppendOnlyTree) initLastIndex(tx kv.Tx) (common.Hash, error) { + ldc, root, err := t.getLastIndexAndRoot(tx) + if err != nil { + return common.Hash{}, err + } + t.lastIndex = ldc + return root, nil +} +func (t *AppendOnlyTree) initLastLeftCache(tx kv.Tx, lastIndex int64, lastRoot common.Hash) error { + siblings := make([]common.Hash, t.height, t.height) + if lastIndex == -1 { + t.lastLeftCache = siblings + return nil + } + index := lastIndex + + currentNodeHash := lastRoot + // It starts in height-1 because 0 is the level of the leafs + for h := int(t.height - 1); h >= 0; h-- { + currentNode, err := t.getRHTNode(tx, currentNodeHash) + if err != nil { + return fmt.Errorf( + "error getting node %s from the RHT at height %d with root %s: %v", + currentNodeHash.Hex(), h, lastRoot.Hex(), err, + ) + } + if currentNode == nil { + return ErrNotFound + } + siblings = append(siblings, currentNode.left) + if index&(1< 0 { + currentNodeHash = currentNode.right + } else { + currentNodeHash = currentNode.left + } + } + + // Reverse the siblings to go from leafs to root + for i, j := 0, len(siblings)-1; i < j; i, j = i+1, j-1 { + siblings[i], siblings[j] = siblings[j], siblings[i] + } + + t.lastLeftCache = siblings + return nil +} + +// Reorg deletes all the data relevant from firstReorgedIndex (includded) and onwards +// and prepares the tree tfor being used as it was at firstReorgedIndex-1 +func (t *AppendOnlyTree) Reorg(ctx context.Context, firstReorgedIndex uint32) error { + if t.lastIndex == -1 { + return nil + } + tx, err := t.db.BeginRw(ctx) + if err != nil { + return err + } + // Clean root table + for i := firstReorgedIndex; i <= uint32(t.lastIndex); i++ { + if err := tx.Delete(t.rootTable, dbCommon.Uint32ToBytes(i)); err != nil { + tx.Rollback() + return err + } + } + + // Reset + root := common.Hash{} + if firstReorgedIndex > 0 { + rootBytes, err := tx.GetOne(t.rootTable, dbCommon.Uint32ToBytes(firstReorgedIndex-1)) + if err != nil { + tx.Rollback() + return err + } + if rootBytes == nil { + tx.Rollback() + return ErrNotFound + } + root = common.Hash(rootBytes) + } + err = t.initLastLeftCache(tx, int64(firstReorgedIndex)-1, root) + if err != nil { + tx.Rollback() + return err + } + + // Note: not cleaning RHT, not worth it + if err := tx.Commit(); err != nil { + return err + } + t.lastIndex = int64(firstReorgedIndex) - 1 + return nil +} diff --git a/bridgesync/testvectors/claim-vectors.json b/tree/testvectors/claim-vectors.json similarity index 100% rename from bridgesync/testvectors/claim-vectors.json rename to tree/testvectors/claim-vectors.json diff --git a/bridgesync/testvectors/leaf-vectors.json b/tree/testvectors/leaf-vectors.json similarity index 100% rename from bridgesync/testvectors/leaf-vectors.json rename to tree/testvectors/leaf-vectors.json diff --git a/bridgesync/testvectors/root-vectors.json b/tree/testvectors/root-vectors.json similarity index 100% rename from bridgesync/testvectors/root-vectors.json rename to tree/testvectors/root-vectors.json diff --git a/tree/testvectors/types.go b/tree/testvectors/types.go new file mode 100644 index 00000000..f005b6ea --- /dev/null +++ b/tree/testvectors/types.go @@ -0,0 +1,64 @@ +package testvectors + +import ( + "encoding/binary" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/iden3/go-iden3-crypto/keccak256" +) + +// DepositVectorRaw represents the deposit vector +type DepositVectorRaw struct { + OriginalNetwork uint32 `json:"originNetwork"` + TokenAddress string `json:"tokenAddress"` + Amount string `json:"amount"` + DestinationNetwork uint32 `json:"destinationNetwork"` + DestinationAddress string `json:"destinationAddress"` + ExpectedHash string `json:"leafValue"` + CurrentHash string `json:"currentLeafValue"` + Metadata string `json:"metadata"` +} + +func (d *DepositVectorRaw) Hash() common.Hash { + origNet := make([]byte, 4) //nolint:gomnd + binary.BigEndian.PutUint32(origNet, uint32(d.OriginalNetwork)) + destNet := make([]byte, 4) //nolint:gomnd + binary.BigEndian.PutUint32(destNet, uint32(d.DestinationNetwork)) + + metaHash := keccak256.Hash(common.FromHex(d.Metadata)) + hash := common.Hash{} + var buf [32]byte //nolint:gomnd + amount, _ := big.NewInt(0).SetString(d.Amount, 0) + origAddrBytes := common.HexToAddress(d.TokenAddress) + destAddrBytes := common.HexToAddress(d.DestinationAddress) + copy( + hash[:], + keccak256.Hash( + []byte{0}, + origNet, + origAddrBytes[:], + destNet, + destAddrBytes[:], + amount.FillBytes(buf[:]), + metaHash, + ), + ) + return hash +} + +// MTClaimVectorRaw represents the merkle proof +type MTClaimVectorRaw struct { + Deposits []DepositVectorRaw `json:"leafs"` + Index uint32 `json:"index"` + MerkleProof []string `json:"proof"` + ExpectedRoot string `json:"root"` +} + +// MTRootVectorRaw represents the root of Merkle Tree +type MTRootVectorRaw struct { + ExistingLeaves []string `json:"previousLeafsValues"` + CurrentRoot string `json:"currentRoot"` + NewLeaf DepositVectorRaw `json:"newLeaf"` + NewRoot string `json:"newRoot"` +} diff --git a/tree/tree.go b/tree/tree.go new file mode 100644 index 00000000..6271f16e --- /dev/null +++ b/tree/tree.go @@ -0,0 +1,175 @@ +package tree + +import ( + "context" + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "golang.org/x/crypto/sha3" +) + +const ( + defaultHeight uint8 = 32 + rootTableSufix = "-root" + rhtTableSufix = "-rht" +) + +var ( + ErrNotFound = errors.New("not found") +) + +type Leaf struct { + Index uint32 + Hash common.Hash +} + +type Tree struct { + db kv.RwDB + rhtTable string + rootTable string + height uint8 + zeroHashes []common.Hash +} + +type treeNode struct { + left common.Hash + right common.Hash +} + +func (n *treeNode) hash() common.Hash { + var hash common.Hash + hasher := sha3.NewLegacyKeccak256() + hasher.Write(n.left[:]) + hasher.Write(n.right[:]) + copy(hash[:], hasher.Sum(nil)) + return hash +} + +func (n *treeNode) MarshalBinary() ([]byte, error) { + return append(n.left[:], n.right[:]...), nil +} + +func (n *treeNode) UnmarshalBinary(data []byte) error { + if len(data) != 64 { + return fmt.Errorf("expected len %d, actual len %d", 64, len(data)) + } + n.left = common.Hash(data[:32]) + n.right = common.Hash(data[32:]) + return nil +} + +func newTree(dbPath, dbPrefix string) (*Tree, error) { + rootTable := dbPrefix + rootTableSufix + rhtTable := dbPrefix + rhtTableSufix + db, err := mdbx.NewMDBX(nil). + Path(dbPath). + WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { + return kv.TableCfg{ + rootTable: {}, + rhtTable: {}, + } + }). + Open() + if err != nil { + return nil, err + } + t := &Tree{ + rhtTable: rhtTable, + rootTable: rootTable, + db: db, + height: defaultHeight, + zeroHashes: generateZeroHashes(defaultHeight), + } + + return t, nil +} + +// GetProof returns the merkle proof for a given index and root. +func (t *Tree) GetProof(ctx context.Context, index uint32, root common.Hash) ([]common.Hash, error) { + tx, err := t.db.BeginRw(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + siblings := make([]common.Hash, int(t.height)) + + currentNodeHash := root + // It starts in height-1 because 0 is the level of the leafs + for h := int(t.height - 1); h >= 0; h-- { + currentNode, err := t.getRHTNode(tx, currentNodeHash) + if err != nil { + return nil, fmt.Errorf( + "height: %d, currentNode: %s, error: %v", + h, currentNodeHash.Hex(), err, + ) + } + /* + * Root (level h=3 => height=4) + * / \ + * O5 O6 (level h=2) + * / \ / \ + * O1 O2 O3 O4 (level h=1) + * /\ /\ /\ /\ + * 0 1 2 3 4 5 6 7 Leafs (level h=0) + * Example 1: + * Choose index = 3 => 011 binary + * Assuming we are in level 1 => h=1; 1< 011&010=010 which is higher than 0 so we need the left sibling (O1) + * Example 2: + * Choose index = 4 => 100 binary + * Assuming we are in level 1 => h=1; 1< 100&010=000 which is not higher than 0 so we need the right sibling (O4) + * Example 3: + * Choose index = 4 => 100 binary + * Assuming we are in level 2 => h=2; 1< 100&100=100 which is higher than 0 so we need the left sibling (O5) + */ + if index&(1< 0 { + siblings = append(siblings, currentNode.left) + currentNodeHash = currentNode.right + } else { + siblings = append(siblings, currentNode.right) + currentNodeHash = currentNode.left + } + } + + // Reverse siblings to go from leafs to root + for i, j := 0, len(siblings)-1; i < j; i, j = i+1, j-1 { + siblings[i], siblings[j] = siblings[j], siblings[i] + } + + return siblings, nil +} + +func (t *Tree) getRHTNode(tx kv.Tx, nodeHash common.Hash) (*treeNode, error) { + nodeBytes, err := tx.GetOne(t.rhtTable, nodeHash[:]) + if err != nil { + return nil, err + } + if nodeBytes == nil { + return nil, ErrNotFound + } + node := &treeNode{} + err = node.UnmarshalBinary(nodeBytes) + return node, err +} + +func generateZeroHashes(height uint8) []common.Hash { + var zeroHashes = []common.Hash{ + {}, + } + // This generates a leaf = HashZero in position 0. In the rest of the positions that are equivalent to the ascending levels, + // we set the hashes of the nodes. So all nodes from level i=5 will have the same value and same children nodes. + for i := 1; i <= int(height); i++ { + hasher := sha3.NewLegacyKeccak256() + hasher.Write(zeroHashes[i-1][:]) + hasher.Write(zeroHashes[i-1][:]) + thisHeightHash := common.Hash{} + copy(thisHeightHash[:], hasher.Sum(nil)) + zeroHashes = append(zeroHashes, thisHeightHash) + } + return zeroHashes +} diff --git a/tree/tree_test.go b/tree/tree_test.go new file mode 100644 index 00000000..47129aee --- /dev/null +++ b/tree/tree_test.go @@ -0,0 +1,103 @@ +package tree + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/0xPolygon/cdk/tree/testvectors" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +func TestMTAddLeaf(t *testing.T) { + data, err := os.ReadFile("testvectors/root-vectors.json") + require.NoError(t, err) + + var mtTestVectors []testvectors.MTRootVectorRaw + err = json.Unmarshal(data, &mtTestVectors) + require.NoError(t, err) + ctx := context.Background() + + for ti, testVector := range mtTestVectors { + t.Run(fmt.Sprintf("Test vector %d", ti), func(t *testing.T) { + + path := t.TempDir() + tree, err := NewAppendOnly(context.Background(), path, "foo") + require.NoError(t, err) + + // Add exisiting leaves + leaves := []Leaf{} + for i, leaf := range testVector.ExistingLeaves { + leaves = append(leaves, Leaf{ + Index: uint32(i), + Hash: common.HexToHash(leaf), + }) + } + err = tree.AddLeaves(ctx, leaves) + require.NoError(t, err) + if len(testVector.ExistingLeaves) > 0 { + txRo, err := tree.db.BeginRo(ctx) + require.NoError(t, err) + _, actualRoot, err := tree.getLastIndexAndRoot(txRo) + txRo.Rollback() + require.NoError(t, err) + require.Equal(t, common.HexToHash(testVector.CurrentRoot), actualRoot) + } + + // Add new bridge + err = tree.AddLeaves(ctx, []Leaf{{ + Index: uint32(len(testVector.ExistingLeaves)), + Hash: common.HexToHash(testVector.NewLeaf.CurrentHash), + }}) + require.NoError(t, err) + txRo, err := tree.db.BeginRo(ctx) + require.NoError(t, err) + _, actualRoot, err := tree.getLastIndexAndRoot(txRo) + txRo.Rollback() + require.NoError(t, err) + require.Equal(t, common.HexToHash(testVector.NewRoot), actualRoot) + }) + } +} + +func TestMTGetProof(t *testing.T) { + data, err := os.ReadFile("testvectors/claim-vectors.json") + require.NoError(t, err) + + var mtTestVectors []testvectors.MTClaimVectorRaw + err = json.Unmarshal(data, &mtTestVectors) + require.NoError(t, err) + ctx := context.Background() + + for ti, testVector := range mtTestVectors { + t.Run(fmt.Sprintf("Test vector %d", ti), func(t *testing.T) { + path := t.TempDir() + tree, err := NewAppendOnly(context.Background(), path, "foo") + require.NoError(t, err) + leaves := []Leaf{} + for li, leaf := range testVector.Deposits { + leaves = append(leaves, Leaf{ + Index: uint32(li), + Hash: leaf.Hash(), + }) + } + err = tree.AddLeaves(ctx, leaves) + require.NoError(t, err) + txRo, err := tree.db.BeginRo(ctx) + require.NoError(t, err) + _, actualRoot, err := tree.getLastIndexAndRoot(txRo) + txRo.Rollback() + expectedRoot := common.HexToHash(testVector.ExpectedRoot) + require.Equal(t, expectedRoot, actualRoot) + + proof, err := tree.GetProof(ctx, testVector.Index, expectedRoot) + require.NoError(t, err) + for i, sibling := range testVector.MerkleProof { + require.Equal(t, common.HexToHash(sibling), proof[i]) + } + }) + } +} diff --git a/tree/updatabletree.go b/tree/updatabletree.go new file mode 100644 index 00000000..74812b26 --- /dev/null +++ b/tree/updatabletree.go @@ -0,0 +1,25 @@ +package tree + +import ( + "context" + "errors" + + "github.com/ethereum/go-ethereum/common" +) + +type UpdatableTree struct { + *Tree +} + +func NewUpdatable(ctx context.Context, dbPath, dbPrefix string) (*UpdatableTree, error) { + t, err := newTree(dbPath, dbPrefix) + if err != nil { + return nil, err + } + ut := &UpdatableTree{Tree: t} + return ut, nil +} + +func (t *UpdatableTree) UpsertLeaf(ctx context.Context, index uint32, leafHash common.Hash, expectedRoot *common.Hash) error { + return errors.New("not implemented") +}