From c79d1ad46cb855c4b2cf7c2dba57b94869fb45c9 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 24 Mar 2023 11:24:06 +0100 Subject: [PATCH 1/9] dump+sort reimpl --- cmd/geth/verkle.go | 270 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 57a6fd3096ee..1bc99d0b68e4 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -22,8 +22,10 @@ import ( "encoding/hex" "errors" "fmt" + "io/ioutil" "os" "runtime" + "sort" "time" "github.com/ethereum/go-ethereum/cmd/utils" @@ -84,6 +86,28 @@ This command takes a root commitment and attempts to rebuild the tree. geth verkle dump [ ...] This command will produce a dot file representing the tree, rooted at . in which key1, key2, ... are expanded. + `, + }, + { + Name: "dump-keys", + Usage: "Dump the converted keys of a verkle tree to a series of flat binary files", + ArgsUsage: "", + Action: dumpKeys, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth verkle dump-keys +Dump all converted (key, value) tuples in binary files, for later processing by sort-files. + `, + }, + { + Name: "sort-keys", + Usage: "Dump the converted keys of a verkle tree to a series of flat binary files", + ArgsUsage: "", + Action: sortKeys, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth verkle dump-keys +Dump all converted (key, value) tuples in binary files, for later processing by sort-files. `, }, }, @@ -456,3 +480,249 @@ func expandVerkle(ctx *cli.Context) error { } return nil } +func dumpKeys(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chaindb := utils.MakeChainDatabase(ctx, stack, false) + if chaindb == nil { + return errors.New("nil chaindb") + } + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + log.Error("Failed to load head block") + return errors.New("no head block") + } + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + var ( + root common.Hash + err error + ) + if ctx.NArg() == 1 { + root, err = parseRoot(ctx.Args().First()) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + log.Info("Start traversing the state", "root", root) + } else { + root = headBlock.Root() + log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) + } + + var ( + accounts int + lastReport time.Time + start = time.Now() + // Create map to hold file descriptors for each first byte + files = make(map[byte]*os.File) + ) + + snaptree, err := snapshot.New(snapshot.Config{CacheSize: 256}, chaindb, trie.NewDatabase(chaindb), root) + if err != nil { + return err + } + accIt, err := snaptree.AccountIterator(root, common.Hash{}) + if err != nil { + return err + } + defer accIt.Release() + + // root.FlushAtDepth(depth, saveverkle) + + // Process all accounts sequentially + for accIt.Next() { + accounts += 1 + acc, err := snapshot.FullAccount(accIt.Account()) + if err != nil { + log.Error("Invalid account encountered during traversal", "error", err) + return err + } + + // Store the basic account data + var ( + nonce, balance, version, size [32]byte + newValues = make([][]byte, 256) + ) + newValues[0] = version[:] + newValues[1] = balance[:] + newValues[2] = nonce[:] + newValues[4] = version[:] // memory-saving trick: by default, an account has 0 size + binary.LittleEndian.PutUint64(nonce[:8], acc.Nonce) + for i, b := range acc.Balance.Bytes() { + balance[len(acc.Balance.Bytes())-1-i] = b + } + addr := rawdb.ReadPreimage(chaindb, accIt.Hash()) + if addr == nil { + return fmt.Errorf("could not find preimage for address %x %v %v", accIt.Hash(), acc, accIt.Error()) + } + addrPoint := tutils.EvaluateAddressPoint(addr) + stem := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint) + + // Get first byte of key + firstByte := stem[0] + + // Open or create file for this first byte + file, ok := files[firstByte] + if !ok { + file, _ = os.Create(fmt.Sprintf("%02x.bin", firstByte)) + files[firstByte] = file + } + + // Write tuple to file + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, version) + stem[31] = 1 + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, balance) + stem[31] = 2 + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, nonce) + + // Store the account code if present + if !bytes.Equal(acc.CodeHash, emptyCode) { + code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) + chunks := trie.ChunkifyCode(code) + + for i := 0; i < 128 && i < len(chunks)/32; i++ { + stem[31] = byte(i + 128) + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, chunks[32*i:32*(i+1)]) + } + + for i := 128; i < len(chunks)/32; { + chunkkey := tutils.GetTreeKeyCodeChunkWithEvaluatedAddress(addrPoint, uint256.NewInt(uint64(i))) + j := i + for ; (j-i) < 256 && j < len(chunks)/32; j++ { + chunkkey[31] = byte(j - 128) + binary.Write(file, binary.LittleEndian, chunkkey) + binary.Write(file, binary.LittleEndian, chunks[32*j:32*(j+1)]) + } + i = j + } + + // Write the code size in the account header group + binary.LittleEndian.PutUint64(size[:8], uint64(len(code))) + } + stem[31] = 3 + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, acc.CodeHash[:]) + stem[31] = 4 + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, size) + + // Save every slot into the tree + if !bytes.Equal(acc.Root, emptyRoot[:]) { + storageIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) + if err != nil { + log.Error("Failed to open storage trie", "root", acc.Root, "error", err) + return err + } + defer storageIt.Release() + + for storageIt.Next() { + // The value is RLP-encoded, decode it + var ( + value []byte // slot value after RLP decoding + safeValue [32]byte // 32-byte aligned value + ) + if err := rlp.DecodeBytes(storageIt.Slot(), &value); err != nil { + return fmt.Errorf("error decoding bytes %x: %w", storageIt.Slot(), err) + } + copy(safeValue[32-len(value):], value) + + slotnr := rawdb.ReadPreimage(chaindb, storageIt.Hash()) + if slotnr == nil { + return fmt.Errorf("could not find preimage for slot %x", storageIt.Hash()) + } + + // if the slot belongs to the header group, store it there - and skip + // calculating the slot key. + slotnrbig := uint256.NewInt(0).SetBytes(slotnr) + if slotnrbig.Cmp(uint256.NewInt(64)) < 0 { + stem[31] = 64 + slotnr[31] + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, safeValue[:]) + continue + } + + // Slot not in the header group, get its tree key + slotkey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotnrbig) + // TODO cache du slotkey + + binary.Write(file, binary.LittleEndian, slotkey) + binary.Write(file, binary.LittleEndian, safeValue[:]) + } + if storageIt.Error() != nil { + log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIt.Error()) + return storageIt.Error() + } + } + + if time.Since(lastReport) > time.Second*8 { + log.Info("Traversing state", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + if accIt.Error() != nil { + log.Error("Failed to compute commitment", "root", root, "error", accIt.Error()) + return accIt.Error() + } + log.Info("Wrote all leaves", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) + + // Close all files + for _, file := range files { + file.Close() + } + + return nil +} + +func sortKeys(ctx *cli.Context) error { + // Get list of files + files, _ := ioutil.ReadDir(".") + + // Iterate over files + for _, file := range files { + // Check if file is a binary file + if !bytes.HasSuffix([]byte(file.Name()), []byte(".bin")) { + continue + } + data, _ := ioutil.ReadFile(file.Name()) + numTuples := len(data) / 64 + tuples := make([][64]byte, 0, numTuples) + binary.Read(bytes.NewReader(data), binary.LittleEndian, &tuples) + + // Sort tuples by key + sort.Slice(tuples, func(i, j int) bool { + return bytes.Compare(tuples[i][:32], tuples[j][:32]) < 0 + }) + + // Merge the values + type mergedTuple struct { + stem [31]byte + values [256][]byte + } + mergedtuples := make([]mergedTuple, 0, numTuples) + var last [31]byte + for i := range tuples { + if !bytes.Equal(tuples[i][:31], last[:]) { + mergedtuples = append(mergedtuples, mergedTuple{}) + copy(mergedtuples[len(mergedtuples)-1].stem[:], tuples[i][:]) + + copy(last[:], tuples[i][:31]) + } + + mergedtuples[len(mergedtuples)-1].values[tuples[i][31]] = tuples[i][32:] + } + + // Write sorted tuples back to file + file, _ := os.Create("sorted-" + file.Name()) + binary.Write(file, binary.LittleEndian, &mergedtuples) + file.Close() + } + return nil +} From c72e030bc463c2154a6bffb75dc568270d127068 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 24 Mar 2023 19:29:51 +0100 Subject: [PATCH 2/9] fixes for merging --- cmd/geth/verkle.go | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 1bc99d0b68e4..a92c952115a8 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -22,6 +22,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "io/ioutil" "os" "runtime" @@ -688,40 +689,54 @@ func sortKeys(ctx *cli.Context) error { // Iterate over files for _, file := range files { // Check if file is a binary file - if !bytes.HasSuffix([]byte(file.Name()), []byte(".bin")) { + if !bytes.HasSuffix([]byte(file.Name()), []byte(".bin")) || bytes.HasPrefix([]byte(file.Name()), []byte("sorted-")) { continue } + log.Info("Processing file", "name", file.Name()) data, _ := ioutil.ReadFile(file.Name()) numTuples := len(data) / 64 tuples := make([][64]byte, 0, numTuples) - binary.Read(bytes.NewReader(data), binary.LittleEndian, &tuples) + reader := bytes.NewReader(data) + for { + var tuple [64]byte + err := binary.Read(reader, binary.LittleEndian, &tuple) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } + tuples = append(tuples, tuple) + } // Sort tuples by key + log.Info("Sorting file", "name", file.Name()) sort.Slice(tuples, func(i, j int) bool { return bytes.Compare(tuples[i][:32], tuples[j][:32]) < 0 }) // Merge the values - type mergedTuple struct { + log.Info("Merging file", "name", file.Name()) + file, _ := os.Create("sorted-" + file.Name()) + var ( stem [31]byte values [256][]byte - } - mergedtuples := make([]mergedTuple, 0, numTuples) - var last [31]byte + last [31]byte + ) for i := range tuples { - if !bytes.Equal(tuples[i][:31], last[:]) { - mergedtuples = append(mergedtuples, mergedTuple{}) - copy(mergedtuples[len(mergedtuples)-1].stem[:], tuples[i][:]) + copy(stem[:], tuples[i][:31]) + if stem != last { + binary.Write(file, binary.LittleEndian, last) + binary.Write(file, binary.LittleEndian, values) copy(last[:], tuples[i][:31]) } - mergedtuples[len(mergedtuples)-1].values[tuples[i][31]] = tuples[i][32:] + values[tuples[i][31]] = tuples[i][32:] } // Write sorted tuples back to file - file, _ := os.Create("sorted-" + file.Name()) - binary.Write(file, binary.LittleEndian, &mergedtuples) + log.Info("Writing file", "name", file.Name()) file.Close() } return nil From 0ec99c0db39474ba3d3fda35f3dd0ee97e52f128 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 28 Mar 2023 12:08:25 +0200 Subject: [PATCH 3/9] reuse preimage file --- cmd/geth/verkle.go | 100 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 80 insertions(+), 20 deletions(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index a92c952115a8..c633158d1aec 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -94,7 +94,9 @@ in which key1, key2, ... are expanded. Usage: "Dump the converted keys of a verkle tree to a series of flat binary files", ArgsUsage: "", Action: dumpKeys, - Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{ + &cli.BoolFlag{Name: "dump-preimages", Usage: "Dump the preimage in reading orger"}, + &cli.StringFlag{Name: "preimage-file", Usage: "Name of the preimage file for which values are in order"}}), Description: ` geth verkle dump-keys Dump all converted (key, value) tuples in binary files, for later processing by sort-files. @@ -481,6 +483,18 @@ func expandVerkle(ctx *cli.Context) error { } return nil } + +func getFile(files map[byte]*os.File, stem []byte) *os.File { + firstByte := stem[0] + + // Open or create file for this first byte + file, ok := files[firstByte] + if !ok { + file, _ = os.Create(fmt.Sprintf("%02x.bin", firstByte)) + files[firstByte] = file + } + return file +} func dumpKeys(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -499,8 +513,10 @@ func dumpKeys(ctx *cli.Context) error { return errors.New("too many arguments") } var ( - root common.Hash - err error + root common.Hash + err error + dumppreimages bool + preimagefile *os.File ) if ctx.NArg() == 1 { root, err = parseRoot(ctx.Args().First()) @@ -513,6 +529,7 @@ func dumpKeys(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } + dumppreimages = ctx.Bool("dump-preimages") var ( accounts int @@ -532,7 +549,17 @@ func dumpKeys(ctx *cli.Context) error { } defer accIt.Release() - // root.FlushAtDepth(depth, saveverkle) + if dumppreimages { + preimagefile, _ = os.Create("preimages.bin") + defer preimagefile.Close() + } else { + if filename := ctx.String("preimage-file"); filename != "" { + preimagefile, err = os.Open(filename) + if err != nil { + panic(err) + } + } + } // Process all accounts sequentially for accIt.Next() { @@ -556,23 +583,34 @@ func dumpKeys(ctx *cli.Context) error { for i, b := range acc.Balance.Bytes() { balance[len(acc.Balance.Bytes())-1-i] = b } - addr := rawdb.ReadPreimage(chaindb, accIt.Hash()) + var addr []byte + if !dumppreimages && preimagefile != nil { + var h [32]byte + _, err = preimagefile.Read(h[:]) + if err != nil { + panic(err) + } + if !bytes.Equal(h[:], accIt.Hash().Bytes()) { + panic("differing hashes") + } + var a [20]byte + _, err = preimagefile.Read(a[:]) + addr = a[:] + } else { + addr = rawdb.ReadPreimage(chaindb, accIt.Hash()) + } if addr == nil { return fmt.Errorf("could not find preimage for address %x %v %v", accIt.Hash(), acc, accIt.Error()) } + if dumppreimages { + binary.Write(preimagefile, binary.LittleEndian, accIt.Hash()) + binary.Write(preimagefile, binary.LittleEndian, 20) + binary.Write(preimagefile, binary.LittleEndian, addr) + } addrPoint := tutils.EvaluateAddressPoint(addr) stem := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint) - // Get first byte of key - firstByte := stem[0] - - // Open or create file for this first byte - file, ok := files[firstByte] - if !ok { - file, _ = os.Create(fmt.Sprintf("%02x.bin", firstByte)) - files[firstByte] = file - } - + file := getFile(files, stem) // Write tuple to file binary.Write(file, binary.LittleEndian, stem) binary.Write(file, binary.LittleEndian, version) @@ -597,10 +635,11 @@ func dumpKeys(ctx *cli.Context) error { for i := 128; i < len(chunks)/32; { chunkkey := tutils.GetTreeKeyCodeChunkWithEvaluatedAddress(addrPoint, uint256.NewInt(uint64(i))) j := i + chunkFile := getFile(files, chunkkey) for ; (j-i) < 256 && j < len(chunks)/32; j++ { chunkkey[31] = byte(j - 128) - binary.Write(file, binary.LittleEndian, chunkkey) - binary.Write(file, binary.LittleEndian, chunks[32*j:32*(j+1)]) + binary.Write(chunkFile, binary.LittleEndian, chunkkey) + binary.Write(chunkFile, binary.LittleEndian, chunks[32*j:32*(j+1)]) } i = j } @@ -635,10 +674,30 @@ func dumpKeys(ctx *cli.Context) error { } copy(safeValue[32-len(value):], value) - slotnr := rawdb.ReadPreimage(chaindb, storageIt.Hash()) + var slotnr []byte + if !dumppreimages && preimagefile != nil { + var h [32]byte + _, err = preimagefile.Read(h[:]) + if err != nil { + panic(err) + } + if !bytes.Equal(h[:], storageIt.Hash().Bytes()) { + panic("differing hashes") + } + var s [32]byte + _, err = preimagefile.Read(s[:]) + slotnr = s[:] + } else { + slotnr = rawdb.ReadPreimage(chaindb, storageIt.Hash()) + } if slotnr == nil { return fmt.Errorf("could not find preimage for slot %x", storageIt.Hash()) } + if dumppreimages { + binary.Write(preimagefile, binary.LittleEndian, storageIt.Hash()) + binary.Write(preimagefile, binary.LittleEndian, 32) + binary.Write(preimagefile, binary.LittleEndian, slotnr) + } // if the slot belongs to the header group, store it there - and skip // calculating the slot key. @@ -654,8 +713,9 @@ func dumpKeys(ctx *cli.Context) error { slotkey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotnrbig) // TODO cache du slotkey - binary.Write(file, binary.LittleEndian, slotkey) - binary.Write(file, binary.LittleEndian, safeValue[:]) + slotfile := getFile(files, slotkey) + binary.Write(slotfile, binary.LittleEndian, slotkey) + binary.Write(slotfile, binary.LittleEndian, safeValue[:]) } if storageIt.Error() != nil { log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIt.Error()) From f4d66703ca583d5b302ff22cc6944d0eeb4e4ab0 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 28 Mar 2023 12:09:08 +0200 Subject: [PATCH 4/9] remove invalid arg management --- cmd/geth/verkle.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index c633158d1aec..df9b5a5e3458 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -508,27 +508,15 @@ func dumpKeys(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - if ctx.NArg() > 1 { - log.Error("Too many arguments given") - return errors.New("too many arguments") - } var ( root common.Hash err error dumppreimages bool preimagefile *os.File ) - if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - log.Info("Start traversing the state", "root", root) - } else { - root = headBlock.Root() - log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) - } + root = headBlock.Root() + log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) + dumppreimages = ctx.Bool("dump-preimages") var ( From eadba89c69d1bcaf606bb5d12da5c889521b9693 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 28 Mar 2023 12:10:29 +0200 Subject: [PATCH 5/9] fix:step2: only take step1 output --- cmd/geth/verkle.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index df9b5a5e3458..086dfb5dc020 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -737,7 +737,8 @@ func sortKeys(ctx *cli.Context) error { // Iterate over files for _, file := range files { // Check if file is a binary file - if !bytes.HasSuffix([]byte(file.Name()), []byte(".bin")) || bytes.HasPrefix([]byte(file.Name()), []byte("sorted-")) { + fname := file.Name() + if !bytes.HasSuffix([]byte(fname), []byte(".bin")) || bytes.HasPrefix([]byte(fname), []byte("sorted-")) || len(fname) != 6 { continue } log.Info("Processing file", "name", file.Name()) From 1520b9948d7961aba88d35cf6d4b1e206eaa8c5a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 28 Mar 2023 16:07:36 +0200 Subject: [PATCH 6/9] rebuild the tree after sorting the files --- cmd/geth/verkle.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 086dfb5dc020..4c1b66ad99b0 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -733,6 +733,7 @@ func dumpKeys(ctx *cli.Context) error { func sortKeys(ctx *cli.Context) error { // Get list of files files, _ := ioutil.ReadDir(".") + root := verkle.New() // Iterate over files for _, file := range files { @@ -778,15 +779,24 @@ func sortKeys(ctx *cli.Context) error { binary.Write(file, binary.LittleEndian, last) binary.Write(file, binary.LittleEndian, values) + var istem [31]byte + istem = stem + root.(*verkle.InternalNode).InsertStem(istem[:], values[:], nil) copy(last[:], tuples[i][:31]) } - values[tuples[i][31]] = tuples[i][32:] + values[tuples[i][31]] = make([]byte, 32) + copy(values[tuples[i][31]], tuples[i][32:]) } + // Committing file + log.Info("Committing file", "name", file.Name()) + root.Commit() + // Write sorted tuples back to file log.Info("Writing file", "name", file.Name()) file.Close() + root.(*verkle.InternalNode).FlushAtDepth(0, nil) } return nil } From c12b0f11e710c3d396691de39c7372708a302db4 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:23:09 +0200 Subject: [PATCH 7/9] fix: match base branch --- cmd/geth/verkle.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 4c1b66ad99b0..771c1da7a412 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -770,9 +770,12 @@ func sortKeys(ctx *cli.Context) error { file, _ := os.Create("sorted-" + file.Name()) var ( stem [31]byte - values [256][]byte + values = make([][]byte, 256) last [31]byte ) + if len(tuples) > 0 { + copy(last[:], tuples[0][:31]) + } for i := range tuples { copy(stem[:], tuples[i][:31]) if stem != last { @@ -780,15 +783,27 @@ func sortKeys(ctx *cli.Context) error { binary.Write(file, binary.LittleEndian, values) var istem [31]byte - istem = stem - root.(*verkle.InternalNode).InsertStem(istem[:], values[:], nil) - copy(last[:], tuples[i][:31]) + istem = last + err := root.(*verkle.InternalNode).InsertStem(istem[:], values, nil) + if err != nil { + panic(err) + } + copy(last[:], stem[:]) + values = make([][]byte, 256) } values[tuples[i][31]] = make([]byte, 32) copy(values[tuples[i][31]], tuples[i][32:]) } + // dump the last group + binary.Write(file, binary.LittleEndian, stem) + binary.Write(file, binary.LittleEndian, values) + err := root.(*verkle.InternalNode).InsertStem(stem[:], values, nil) + if err != nil { + panic(err) + } + // Committing file log.Info("Committing file", "name", file.Name()) root.Commit() @@ -796,7 +811,7 @@ func sortKeys(ctx *cli.Context) error { // Write sorted tuples back to file log.Info("Writing file", "name", file.Name()) file.Close() - root.(*verkle.InternalNode).FlushAtDepth(0, nil) } + log.Info("Done", "root", root.Commit().Bytes()) return nil } From 6de0257cc938ae6156807a9c42edc56469b3e76e Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Thu, 30 Mar 2023 09:43:30 -0300 Subject: [PATCH 8/9] fix compilation error (#195) * fix compilation error Signed-off-by: Ignacio Hagopian * print root in hex Signed-off-by: Ignacio Hagopian * verkle/stage2: print elapsed time Signed-off-by: Ignacio Hagopian * formatting Signed-off-by: Ignacio Hagopian * Remove note-to-self comments in Frenglish --------- Signed-off-by: Ignacio Hagopian Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- cmd/geth/verkle.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 771c1da7a412..a16a716fe014 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -96,7 +96,8 @@ in which key1, key2, ... are expanded. Action: dumpKeys, Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{ &cli.BoolFlag{Name: "dump-preimages", Usage: "Dump the preimage in reading orger"}, - &cli.StringFlag{Name: "preimage-file", Usage: "Name of the preimage file for which values are in order"}}), + &cli.StringFlag{Name: "preimage-file", Usage: "Name of the preimage file for which values are in order"}, + }), Description: ` geth verkle dump-keys Dump all converted (key, value) tuples in binary files, for later processing by sort-files. @@ -241,7 +242,7 @@ func convertToVerkle(ctx *cli.Context) error { // Save every slot into the tree if !bytes.Equal(acc.Root, emptyRoot[:]) { - var translatedStorage = map[string][][]byte{} + translatedStorage := map[string][][]byte{} storageIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) if err != nil { @@ -495,6 +496,7 @@ func getFile(files map[byte]*os.File, stem []byte) *os.File { } return file } + func dumpKeys(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -698,8 +700,7 @@ func dumpKeys(ctx *cli.Context) error { } // Slot not in the header group, get its tree key - slotkey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotnrbig) - // TODO cache du slotkey + slotkey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotnr) slotfile := getFile(files, slotkey) binary.Write(slotfile, binary.LittleEndian, slotkey) @@ -733,6 +734,7 @@ func dumpKeys(ctx *cli.Context) error { func sortKeys(ctx *cli.Context) error { // Get list of files files, _ := ioutil.ReadDir(".") + start := time.Now() root := verkle.New() // Iterate over files @@ -812,6 +814,7 @@ func sortKeys(ctx *cli.Context) error { log.Info("Writing file", "name", file.Name()) file.Close() } - log.Info("Done", "root", root.Commit().Bytes()) + log.Info("Done", "root", fmt.Sprintf("%x", root.Commit().Bytes())) + log.Info("Finished", "elapsed", common.PrettyDuration(time.Since(start))) return nil } From 32afdac07a9ba13519d865f6e69465cf30918b15 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 13 Apr 2023 20:55:00 +0200 Subject: [PATCH 9/9] fixes to use the latest go-verkle@master --- core/chain_makers.go | 6 +++--- go.mod | 6 +++--- go.sum | 6 ++++++ trie/verkle.go | 37 +++++++++++++++++++++++++------------ 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index 531079ccaeb2..b7113bbad5a7 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -305,12 +305,12 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, return db, blocks, receipts } -func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, [][]byte, [][]verkle.KeyValuePair) { +func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { if config == nil { config = params.TestChainConfig } - proofs := make([][]byte, 0, n) - keyvals := make([][]verkle.KeyValuePair, 0, n) + proofs := make([]*verkle.VerkleProof, 0, n) + keyvals := make([]verkle.StateDiff, 0, n) blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) chainreader := &fakeChainReader{config: config} genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { diff --git a/go.mod b/go.mod index 94f82043bf54..1369c43c71c3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.14.0 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f - github.com/crate-crypto/go-ipa v0.0.0-20230315201338-1643fdc2ead8 + github.com/crate-crypto/go-ipa v0.0.0-20230410135559-ce4a96995014 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/docker/docker v1.6.2 @@ -23,7 +23,7 @@ require ( github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20230317174103-141354da6b11 + github.com/gballet/go-verkle v0.0.0-20230413135631-4bea2763ed0f github.com/go-stack/stack v1.8.0 github.com/golang-jwt/jwt/v4 v4.3.0 github.com/golang/protobuf v1.5.2 @@ -61,7 +61,7 @@ require ( golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.6.0 + golang.org/x/sys v0.7.0 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba diff --git a/go.sum b/go.sum index 4a3b92eff6df..728030c6ffcb 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20230315201338-1643fdc2ead8 h1:2EBbIwPDRqlCD2K34Eojyy0x9d3RhOuHAZfbQm508X8= github.com/crate-crypto/go-ipa v0.0.0-20230315201338-1643fdc2ead8/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= +github.com/crate-crypto/go-ipa v0.0.0-20230410135559-ce4a96995014 h1:bbyTlFQ12wkFA6aVL+9HrBZwVl85AN0VS/Bwam7o93U= +github.com/crate-crypto/go-ipa v0.0.0-20230410135559-ce4a96995014/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -137,6 +139,8 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-verkle v0.0.0-20230317174103-141354da6b11 h1:x4hiQFgr1SlqR4IoAZiXLFZK4L7KbibqkORqa1fwKp8= github.com/gballet/go-verkle v0.0.0-20230317174103-141354da6b11/go.mod h1:IyOnn1kujMWaT+wet/6Ix1BtvYwateOBy9puuWH/8sw= +github.com/gballet/go-verkle v0.0.0-20230413135631-4bea2763ed0f h1:gP4uR2/1qx6hsIzbRI28JWcsVuP7xyjyj6SpLnoFobc= +github.com/gballet/go-verkle v0.0.0-20230413135631-4bea2763ed0f/go.mod h1:P3bwGrLhsUNIsUDlq2yzMPvO1c/15oiB3JS85P+hNfw= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -556,6 +560,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= diff --git a/trie/verkle.go b/trie/verkle.go index a6175f5e4789..8ed8c0b31636 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -273,7 +273,7 @@ func (trie *VerkleTrie) IsVerkle() bool { return true } -func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[string][]byte) ([]byte, []verkle.KeyValuePair, error) { +func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[string][]byte) (*verkle.VerkleProof, verkle.StateDiff, error) { proof, _, _, _, err := verkle.MakeVerkleMultiProof(trie.root, keys, kv) if err != nil { return nil, nil, err @@ -293,10 +293,10 @@ func addKey(s set, key []byte) { s[string(key)] = struct{}{} } -func DeserializeAndVerifyVerkleProof(serialized []byte, root []byte, keyvals []verkle.KeyValuePair) error { +func DeserializeAndVerifyVerkleProof(vp *verkle.VerkleProof, root []byte, statediff verkle.StateDiff) error { rootC := new(verkle.Point) rootC.SetBytesTrusted(root) - proof, cis, indices, yis, err := deserializeVerkleProof(serialized, rootC, keyvals) + proof, cis, indices, yis, err := deserializeVerkleProof(vp, rootC, statediff) if err != nil { return fmt.Errorf("could not deserialize proof: %w", err) } @@ -308,10 +308,10 @@ func DeserializeAndVerifyVerkleProof(serialized []byte, root []byte, keyvals []v return nil } -func deserializeVerkleProof(serialized []byte, rootC *verkle.Point, keyvals []verkle.KeyValuePair) (*verkle.Proof, []*verkle.Point, []byte, []*verkle.Fr, error) { +func deserializeVerkleProof(vp *verkle.VerkleProof, rootC *verkle.Point, statediff verkle.StateDiff) (*verkle.Proof, []*verkle.Point, []byte, []*verkle.Fr, error) { var others set = set{} // Mark when an "other" stem has been seen - proof, err := verkle.DeserializeProof(serialized, keyvals) + proof, err := verkle.DeserializeProof(vp, statediff) if err != nil { return nil, nil, nil, nil, fmt.Errorf("verkle proof deserialization error: %w", err) } @@ -328,14 +328,27 @@ func deserializeVerkleProof(serialized []byte, rootC *verkle.Point, keyvals []ve if err != nil { return nil, nil, nil, nil, fmt.Errorf("error rebuilding the tree from proof: %w", err) } - for _, kv := range keyvals { - val, err := tree.Get(kv.Key, nil) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf("could not find key %x in tree rebuilt from proof: %w", kv.Key, err) - } + for _, stemdiff := range statediff { + for _, suffixdiff := range stemdiff.SuffixDiffs { + + var key [32]byte + copy(key[:31], stemdiff.Stem[:]) + key[31] = suffixdiff.Suffix + + val, err := tree.Get(key[:], nil) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("could not find key %x in tree rebuilt from proof: %w", key, err) + } + if len(val) > 0 { + if !bytes.Equal(val, suffixdiff.CurrentValue[:]) { + return nil, nil, nil, nil, fmt.Errorf("could not find correct value at %x in tree rebuilt from proof: %x != %x", key, val, *suffixdiff.CurrentValue) + } + } else { + if suffixdiff.CurrentValue != nil && len(suffixdiff.CurrentValue) != 0 { + return nil, nil, nil, nil, fmt.Errorf("could not find correct value at %x in tree rebuilt from proof: %x != %x", key, val, *suffixdiff.CurrentValue) + } + } - if !bytes.Equal(val, kv.Value) { - return nil, nil, nil, nil, fmt.Errorf("could not find correct value at %x in tree rebuilt from proof: %x != %x", kv.Key, val, kv.Value) } }