From b8e0efda0b3e8724164cd1458f3e53608721a4c1 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 24 Jan 2025 12:07:10 +0530 Subject: [PATCH 01/10] paralellise pruning inside accounts --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 5a7010a057..7f088de5cb 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 5a7010a057c6539a3bb154d15a47c015a96b1ef8 +Subproject commit 7f088de5cb854850d9d3fdb0edc9e2f70783de32 From 2d3f708c893f660e76a8eac2a82c244116fffe5c Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 24 Jan 2025 12:27:31 +0530 Subject: [PATCH 02/10] fix --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7f088de5cb..43b8488a1b 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7f088de5cb854850d9d3fdb0edc9e2f70783de32 +Subproject commit 43b8488a1b88d41a8048a5323d8bfbdf2dd41d0e From 54dd8c4fd8b0b99cd7426bbf684f8cfe448ed21f Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 24 Jan 2025 16:13:27 +0530 Subject: [PATCH 03/10] fix --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 43b8488a1b..d2f04ddfb6 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 43b8488a1b88d41a8048a5323d8bfbdf2dd41d0e +Subproject commit d2f04ddfb6c1bd213b348e51cf62986114404151 From 1910db9308767f0ebe246e3346eed86460035dc6 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 24 Jan 2025 16:16:47 +0530 Subject: [PATCH 04/10] fix --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index d2f04ddfb6..ef26f973fd 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit d2f04ddfb6c1bd213b348e51cf62986114404151 +Subproject commit ef26f973fd3f6e0916f300e9504c9f9e25010782 From 651bd58966cd2e3553a9f271cd4bda4c1ed4465d Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 24 Jan 2025 21:46:00 +0530 Subject: [PATCH 05/10] fix race --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index ef26f973fd..b192b19038 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit ef26f973fd3f6e0916f300e9504c9f9e25010782 +Subproject commit b192b1903884ebe6343fda38eaea8f08f78af0df From 52112404c6d4194b826fe7315b28e03b14eb6814 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Fri, 24 Jan 2025 22:10:29 +0530 Subject: [PATCH 06/10] add option to enable paralle prune --- cmd/conf/init.go | 3 +++ cmd/pruning/pruning.go | 2 +- go-ethereum | 2 +- system_tests/pruning_test.go | 9 +++++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index 97b31958d5..3c64c198f9 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -31,6 +31,7 @@ type InitConfig struct { GenesisJsonFile string `koanf:"genesis-json-file"` ThenQuit bool `koanf:"then-quit"` Prune string `koanf:"prune"` + PruneParallel bool `koanf:"prune-parallel"` PruneBloomSize uint64 `koanf:"prune-bloom-size"` PruneThreads int `koanf:"prune-threads"` PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` @@ -60,6 +61,7 @@ var InitConfigDefault = InitConfig{ AccountsPerSync: 100000, ThenQuit: false, Prune: "", + PruneParallel: false, PruneBloomSize: 2048, PruneThreads: runtime.NumCPU(), PruneTrieCleanCache: 600, @@ -89,6 +91,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".genesis-json-file", InitConfigDefault.GenesisJsonFile, "path for genesis json file") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") f.String(prefix+".prune", InitConfigDefault.Prune, "pruning for a given use: \"full\" for full nodes serving RPC requests, or \"validator\" for validators") + f.Bool(prefix+".prune-parallel", InitConfigDefault.PruneParallel, "if true: use parallel pruning per account") f.Uint64(prefix+".prune-bloom-size", InitConfigDefault.PruneBloomSize, "the amount of memory in megabytes to use for the pruning bloom filter (higher values prune better)") f.Int(prefix+".prune-threads", InitConfigDefault.PruneThreads, "the number of threads to use when pruning") f.Int(prefix+".prune-trie-clean-cache", InitConfigDefault.PruneTrieCleanCache, "amount of memory in megabytes to cache unchanged state trie nodes with when traversing state database during pruning") diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index e89c79bc89..7bddcfbad5 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -248,7 +248,7 @@ func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, return fmt.Errorf("failed to find root to retain for pruning: %w", err) } - pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize, Threads: initConfig.PruneThreads, CleanCacheSize: initConfig.PruneTrieCleanCache}) + pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize, Threads: initConfig.PruneThreads, CleanCacheSize: initConfig.PruneTrieCleanCache, ParallelPrune: initConfig.PruneParallel}) if err != nil { return err } diff --git a/go-ethereum b/go-ethereum index b192b19038..3dfa815772 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit b192b1903884ebe6343fda38eaea8f08f78af0df +Subproject commit 3dfa8157722274ad39d1b0cff8146333b3503db3 diff --git a/system_tests/pruning_test.go b/system_tests/pruning_test.go index f49ed8ddcf..54ef9b35ae 100644 --- a/system_tests/pruning_test.go +++ b/system_tests/pruning_test.go @@ -32,6 +32,14 @@ func countStateEntries(db ethdb.Iteratee) int { } func TestPruning(t *testing.T) { + testPruning(t, false) +} + +func TestPruningParallel(t *testing.T) { + testPruning(t, true) +} + +func testPruning(t *testing.T, parallelPruning bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -91,6 +99,7 @@ func TestPruning(t *testing.T) { initConfig := conf.InitConfigDefault initConfig.Prune = "full" + initConfig.PruneParallel = parallelPruning coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &builder.execConfig.Caching) persistentConfig := conf.PersistentConfigDefault err = pruning.PruneChainDb(ctx, chainDb, stack, &initConfig, coreCacheConfig, &persistentConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false) From 7ae86f8ed9b5837920623493630dba80f38cea66 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 27 Jan 2025 19:35:22 +0530 Subject: [PATCH 07/10] Changes based on PR comments and fix race --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 3dfa815772..ed1277cbc8 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 3dfa8157722274ad39d1b0cff8146333b3503db3 +Subproject commit ed1277cbc867dd4f1621eabd21d784f1d8222ffc From ea4c138f912d416d495546759b83e46a7534be24 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Mon, 27 Jan 2025 19:46:35 +0530 Subject: [PATCH 08/10] fix --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index ed1277cbc8..92db07287d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit ed1277cbc867dd4f1621eabd21d784f1d8222ffc +Subproject commit 92db07287d788e2f992a6617966504572c85cc98 From 9a58c88621c3e82c7d78c75c9a3068f9bd747b48 Mon Sep 17 00:00:00 2001 From: Aman Sanghi Date: Thu, 30 Jan 2025 18:06:51 +0530 Subject: [PATCH 09/10] Changes based on PR comments --- cmd/conf/init.go | 110 +++++++++++++++++------------------ cmd/pruning/pruning.go | 2 +- go-ethereum | 2 +- system_tests/pruning_test.go | 6 +- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index 3c64c198f9..c490013bf6 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -13,63 +13,63 @@ import ( ) type InitConfig struct { - Force bool `koanf:"force"` - Url string `koanf:"url"` - Latest string `koanf:"latest"` - LatestBase string `koanf:"latest-base"` - ValidateChecksum bool `koanf:"validate-checksum"` - DownloadPath string `koanf:"download-path"` - DownloadPoll time.Duration `koanf:"download-poll"` - DevInit bool `koanf:"dev-init"` - DevInitAddress string `koanf:"dev-init-address"` - DevMaxCodeSize uint64 `koanf:"dev-max-code-size"` - DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` - Empty bool `koanf:"empty"` - ImportWasm bool `koanf:"import-wasm"` - AccountsPerSync uint `koanf:"accounts-per-sync"` - ImportFile string `koanf:"import-file"` - GenesisJsonFile string `koanf:"genesis-json-file"` - ThenQuit bool `koanf:"then-quit"` - Prune string `koanf:"prune"` - PruneParallel bool `koanf:"prune-parallel"` - PruneBloomSize uint64 `koanf:"prune-bloom-size"` - PruneThreads int `koanf:"prune-threads"` - PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` - RecreateMissingStateFrom uint64 `koanf:"recreate-missing-state-from"` - RebuildLocalWasm string `koanf:"rebuild-local-wasm"` - ReorgToBatch int64 `koanf:"reorg-to-batch"` - ReorgToMessageBatch int64 `koanf:"reorg-to-message-batch"` - ReorgToBlockBatch int64 `koanf:"reorg-to-block-batch"` + Force bool `koanf:"force"` + Url string `koanf:"url"` + Latest string `koanf:"latest"` + LatestBase string `koanf:"latest-base"` + ValidateChecksum bool `koanf:"validate-checksum"` + DownloadPath string `koanf:"download-path"` + DownloadPoll time.Duration `koanf:"download-poll"` + DevInit bool `koanf:"dev-init"` + DevInitAddress string `koanf:"dev-init-address"` + DevMaxCodeSize uint64 `koanf:"dev-max-code-size"` + DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` + Empty bool `koanf:"empty"` + ImportWasm bool `koanf:"import-wasm"` + AccountsPerSync uint `koanf:"accounts-per-sync"` + ImportFile string `koanf:"import-file"` + GenesisJsonFile string `koanf:"genesis-json-file"` + ThenQuit bool `koanf:"then-quit"` + Prune string `koanf:"prune"` + PruneParallelStorageTraversal bool `koanf:"prune-parallel-storage-traversal"` + PruneBloomSize uint64 `koanf:"prune-bloom-size"` + PruneThreads int `koanf:"prune-threads"` + PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` + RecreateMissingStateFrom uint64 `koanf:"recreate-missing-state-from"` + RebuildLocalWasm string `koanf:"rebuild-local-wasm"` + ReorgToBatch int64 `koanf:"reorg-to-batch"` + ReorgToMessageBatch int64 `koanf:"reorg-to-message-batch"` + ReorgToBlockBatch int64 `koanf:"reorg-to-block-batch"` } var InitConfigDefault = InitConfig{ - Force: false, - Url: "", - Latest: "", - LatestBase: "https://snapshot.arbitrum.foundation/", - ValidateChecksum: true, - DownloadPath: "/tmp/", - DownloadPoll: time.Minute, - DevInit: false, - DevInitAddress: "", - DevMaxCodeSize: 0, - DevInitBlockNum: 0, - Empty: false, - ImportWasm: false, - ImportFile: "", - GenesisJsonFile: "", - AccountsPerSync: 100000, - ThenQuit: false, - Prune: "", - PruneParallel: false, - PruneBloomSize: 2048, - PruneThreads: runtime.NumCPU(), - PruneTrieCleanCache: 600, - RecreateMissingStateFrom: 0, // 0 = disabled - RebuildLocalWasm: "auto", - ReorgToBatch: -1, - ReorgToMessageBatch: -1, - ReorgToBlockBatch: -1, + Force: false, + Url: "", + Latest: "", + LatestBase: "https://snapshot.arbitrum.foundation/", + ValidateChecksum: true, + DownloadPath: "/tmp/", + DownloadPoll: time.Minute, + DevInit: false, + DevInitAddress: "", + DevMaxCodeSize: 0, + DevInitBlockNum: 0, + Empty: false, + ImportWasm: false, + ImportFile: "", + GenesisJsonFile: "", + AccountsPerSync: 100000, + ThenQuit: false, + Prune: "", + PruneParallelStorageTraversal: false, + PruneBloomSize: 2048, + PruneThreads: runtime.NumCPU(), + PruneTrieCleanCache: 600, + RecreateMissingStateFrom: 0, // 0 = disabled + RebuildLocalWasm: "auto", + ReorgToBatch: -1, + ReorgToMessageBatch: -1, + ReorgToBlockBatch: -1, } func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -91,7 +91,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".genesis-json-file", InitConfigDefault.GenesisJsonFile, "path for genesis json file") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") f.String(prefix+".prune", InitConfigDefault.Prune, "pruning for a given use: \"full\" for full nodes serving RPC requests, or \"validator\" for validators") - f.Bool(prefix+".prune-parallel", InitConfigDefault.PruneParallel, "if true: use parallel pruning per account") + f.Bool(prefix+".prune-parallel-storage-traversal", InitConfigDefault.PruneParallelStorageTraversal, "if true: use parallel pruning per account") f.Uint64(prefix+".prune-bloom-size", InitConfigDefault.PruneBloomSize, "the amount of memory in megabytes to use for the pruning bloom filter (higher values prune better)") f.Int(prefix+".prune-threads", InitConfigDefault.PruneThreads, "the number of threads to use when pruning") f.Int(prefix+".prune-trie-clean-cache", InitConfigDefault.PruneTrieCleanCache, "amount of memory in megabytes to cache unchanged state trie nodes with when traversing state database during pruning") diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index 7bddcfbad5..0d65ee496f 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -248,7 +248,7 @@ func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, return fmt.Errorf("failed to find root to retain for pruning: %w", err) } - pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize, Threads: initConfig.PruneThreads, CleanCacheSize: initConfig.PruneTrieCleanCache, ParallelPrune: initConfig.PruneParallel}) + pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize, Threads: initConfig.PruneThreads, CleanCacheSize: initConfig.PruneTrieCleanCache, ParallelStorageTraversal: initConfig.PruneParallelStorageTraversal}) if err != nil { return err } diff --git a/go-ethereum b/go-ethereum index 92db07287d..2e15121672 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 92db07287d788e2f992a6617966504572c85cc98 +Subproject commit 2e15121672fa28867057dfd82ad7a4dcca0cd45d diff --git a/system_tests/pruning_test.go b/system_tests/pruning_test.go index 54ef9b35ae..4236a4d567 100644 --- a/system_tests/pruning_test.go +++ b/system_tests/pruning_test.go @@ -35,11 +35,11 @@ func TestPruning(t *testing.T) { testPruning(t, false) } -func TestPruningParallel(t *testing.T) { +func TestPruningPruneParallelStorageTraversal(t *testing.T) { testPruning(t, true) } -func testPruning(t *testing.T, parallelPruning bool) { +func testPruning(t *testing.T, pruneParallelStorageTraversal bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -99,7 +99,7 @@ func testPruning(t *testing.T, parallelPruning bool) { initConfig := conf.InitConfigDefault initConfig.Prune = "full" - initConfig.PruneParallel = parallelPruning + initConfig.PruneParallelStorageTraversal = pruneParallelStorageTraversal coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &builder.execConfig.Caching) persistentConfig := conf.PersistentConfigDefault err = pruning.PruneChainDb(ctx, chainDb, stack, &initConfig, coreCacheConfig, &persistentConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false) From c935483a9f814bcadae12f1af5b2350d821216d6 Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Mon, 3 Feb 2025 22:11:59 -0700 Subject: [PATCH 10/10] update go-ethereum pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index ac98f8adea..a0f1ea5727 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit ac98f8adea779b104289869c51dc783dda2b25ef +Subproject commit a0f1ea572742f8d10223001da05be4c4f4e28007