Generalise prune mode option 4 different db models (#2139)

* Update README * Nimbus-main: replaced `PruneMode` options by `ChainDbMode` options details: For the legacy database, this changes the phrase - `conf.pruneMode == PruneMode.Full` to the expression + `conf.chainDbMode == ChainDbMode.Prune`. * Fix issues moaned about by NIM compiler * Fix copyright year
status-im · Apr 17, 2024 · e8eb326 · e8eb326
1 parent c610053
commit e8eb326
Show file tree

Hide file tree

Showing 20 changed files with 71 additions and 62 deletions.
diff --git a/README.md b/README.md
@@ -230,9 +230,6 @@ available.)
    Disable legacy chunked RLPx messages which are enabled by default for
    synchronising against `Nethermind` nodes
 
- * ENABLE_ETH_VERSION=66<br>
-   Enable legacy protocol `eth66` (or whatever other protocol version.)
-
  * ENABLE_EVMC=1<br>
    Enable mostly EVMC compliant wrapper around the native Nim VM
 
@@ -244,6 +241,12 @@ available.)
 For these variables, using &lt;variable&gt;=0 is ignored and &lt;variable&gt;=2
 has the same effect as &lt;variable&gt;=1 (ditto for other numbers.)
 
+Other settings where the non-zero value matters:
+
+ * ENABLE_ETH_VERSION=66<br>
+   Enable legacy protocol `eth66` (or another available protocol version.)
+
+
 ### <a name="devel-tips"></a>Development tips
 
 Interesting Make variables and targets are documented in the [nimbus-build-system](https://github.com/status-im/nimbus-build-system) repo.

diff --git a/fluffy/docs/the_fluffy_book/mkdocs.yml b/fluffy/docs/the_fluffy_book/mkdocs.yml
@@ -1,5 +1,5 @@
 # Nimbus Fluffy book
-# Copyright (c) 2023 Status Research & Development GmbH
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).

diff --git a/hive_integration/nodocker/engine/engine_env.nim b/hive_integration/nodocker/engine/engine_env.nim
@@ -59,7 +59,7 @@ const
 proc makeCom*(conf: NimbusConf): CommonRef =
   CommonRef.new(
     newCoreDbRef LegacyDbMemory,
-    conf.pruneMode == PruneMode.Full,
+    conf.chainDbMode == ChainDbMode.Prune,
     conf.networkId,
     conf.networkParams
   )

diff --git a/hive_integration/nodocker/pyspec/test_env.nim b/hive_integration/nodocker/pyspec/test_env.nim
@@ -53,11 +53,11 @@ proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
   t.com = CommonRef.new(
       memDB,
       conf,
-      t.conf.pruneMode == PruneMode.Full
+      t.conf.chainDbMode == ChainDbMode.Prune
     )
   t.chainRef = newChain(t.com, extraValidation = true)
   let
-    stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.pruneMode == PruneMode.Full)
+    stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.chainDbMode == ChainDbMode.Prune)
     genesisHeader = node.genesisHeader
 
   setupStateDB(node["pre"], stateDB)

diff --git a/hive_integration/nodocker/rpc/test_env.nim b/hive_integration/nodocker/rpc/test_env.nim
@@ -61,7 +61,7 @@ proc stopRpcHttpServer(srv: RpcServer) =
 
 proc setupEnv*(): TestEnv =
   let conf = makeConfig(@[
-    "--prune-mode:archive",
+    "--chaindb:archive",
     # "--nat:extip:0.0.0.0",
     "--network:7",
     "--import-key:" & initPath / "private-key",
@@ -77,7 +77,7 @@ proc setupEnv*(): TestEnv =
     ethCtx  = newEthContext()
     ethNode = setupEthNode(conf, ethCtx, eth)
     com     = CommonRef.new(newCoreDbRef LegacyDbMemory,
-      conf.pruneMode == PruneMode.Full,
+      conf.chainDbMode == ChainDbMode.Prune,
       conf.networkId,
       conf.networkParams
     )

diff --git a/nimbus/config.nim b/nimbus/config.nim
@@ -105,9 +105,10 @@ const sharedLibText = if defined(linux): " (*.so, *.so.N)"
                       else: ""
 
 type
-  PruneMode* {.pure.} = enum
-    Full
+  ChainDbMode* {.pure.} = enum
+    Prune
     Archive
+    Aristo
 
   NimbusCmd* {.pure.} = enum
     noCommand
@@ -154,12 +155,17 @@ type
       abbr: "k"
       name: "key-store" }: OutDir
 
-    pruneMode* {.
-      desc: "Blockchain prune mode (Full or Archive)"
-      defaultValue: PruneMode.Full
-      defaultValueDesc: $PruneMode.Full
+    chainDbMode* {.
+      desc: "Blockchain database"
+      longDesc:
+        "- Prune   -- Legacy/reference database, full pruning\n" &
+        "- Archive -- Legacy/reference database without pruning\n" &
+        "- Aristo  -- Experimental single state DB\n" &
+        ""
+      defaultValue: ChainDbMode.Prune
+      defaultValueDesc: $ChainDbMode.Prune
       abbr : "p"
-      name: "prune-mode" }: PruneMode
+      name: "chaindb" }: ChainDbMode
 
     syncMode* {.
       desc: "Specify particular blockchain sync mode."

diff --git a/nimbus/db/aristo/aristo_init/rocks_db.nim b/nimbus/db/aristo/aristo_init/rocks_db.nim
@@ -60,10 +60,6 @@ when extraTraceMessages:
 # Private helpers
 # ------------------------------------------------------------------------------
 
-template logTxt(info: static[string]): static[string] =
-  "RocksDB " & info
-
-
 proc newSession(db: RdbBackendRef): RdbPutHdlRef =
   new result
   result.TypedPutHdlRef.beginSession db

diff --git a/nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim b/nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim
@@ -15,19 +15,12 @@
 
 import
   eth/common,
-  rocksdb/lib/librocksdb,
   rocksdb,
   results,
   ../../aristo_desc,
   ../init_common,
   ./rdb_desc
 
-type
-  RdbPutSession = object
-    writer: ptr rocksdb_sstfilewriter_t
-    sstPath: string
-    nRecords: int
-
 const
   extraTraceMessages = false
     ## Enable additional logging noise

diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim b/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim
@@ -141,15 +141,12 @@ when EnableDebugLog:
 
 # -------------------------
 
-func getOrVoid(tab: TableRef[Blob,Blob]; w: openArray[byte]): Blob =
-  tab.getOrDefault(@w, EmptyBlob)
-
 func leafTie(
     root: VertexID;
     path: openArray[byte];
       ): Result[LeafTie,(VertexID,AristoError)] =
   let tag = path.pathToTag.valueOr:
-    return err((VertexID(root), error))
+    return err((root, error))
   ok LeafTie(root: root, path: tag)
 
 proc blobify(

diff --git a/nimbus/db/kvstore_rocksdb.nim b/nimbus/db/kvstore_rocksdb.nim
@@ -54,8 +54,10 @@ proc contains*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
 
 proc del*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
 
-  let exists = ? store.db.keyExists(key)
-  if not exists:
+  let rc = store.db.keyExists(key)
+  if rc.isErr:
+    return rc
+  if not rc.value:
     return ok(false)
 
   let res = store.db.delete(key)

diff --git a/nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim b/nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim
@@ -39,7 +39,7 @@ func dataDir*(rdb: RdbInst): string =
   rdb.baseDir / DataFolder
 
 
-template logTxt(info: static[string]): static[string] =
+template logTxt*(info: static[string]): static[string] =
    "RocksDB/" & info
 
 # ------------------------------------------------------------------------------

diff --git a/nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim b/nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim
@@ -15,7 +15,6 @@
 
 import
   eth/common,
-  stew/byteutils,
   rocksdb,
   results,
   ../../kvt_desc,
@@ -26,7 +25,9 @@ const
     ## Enable additional logging noise
 
 when extraTraceMessages:
-  import chronicles
+  import
+    chronicles,
+    stew/byteutils
 
   logScope:
     topics = "kvt-rocksdb"
@@ -39,9 +40,10 @@ proc disposeSession(rdb: var RdbInst) =
   rdb.session.close()
   rdb.session = WriteBatchRef(nil)
 
-proc `$`(a: Blob): string =
-  a.toHex
-
+when extraTraceMessages:
+  proc `$`(a: Blob): string =
+    a.toHex
+
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------

diff --git a/nimbus/db/ledger/distinct_ledgers.nim b/nimbus/db/ledger/distinct_ledgers.nim
@@ -206,7 +206,6 @@ proc init*(
   ## https://github.com/status-im/nimbus-eth1/issues/932.)
   const
     info = "StorageLedger/init(): "
-    noisy = true
   let
     db = al.distinctBase.parent
     stt = account.stoTrie

diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim
@@ -280,12 +280,16 @@ proc start(nimbus: NimbusNode, conf: NimbusConf) =
     evmcSetLibraryPath(conf.evm)
 
   createDir(string conf.dataDir)
+  let coreDB =
+    # Resolve statically for database type
+    case conf.chainDbMode:
+    of Prune,Archive: LegacyDbPersistent.newCoreDbRef(string conf.dataDir)
+    of Aristo: AristoDbRocks.newCoreDbRef(string conf.dataDir)
   let com = CommonRef.new(
-    newCoreDbRef(LegacyDbPersistent, string conf.dataDir),
-    conf.pruneMode == PruneMode.Full,
-    conf.networkId,
-    conf.networkParams
-    )
+    db = coreDB,
+    pruneTrie = (conf.chainDbMode == ChainDbMode.Prune),
+    networkId = conf.networkId,
+    params = conf.networkParams)
 
   com.initializeEmptyDb()
   com.db.compensateLegacySetup()

diff --git a/tests/test_beacon/setup_env.nim b/tests/test_beacon/setup_env.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2023 Status Research & Development GmbH
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at
 #     https://opensource.org/licenses/MIT).
@@ -64,7 +64,7 @@ proc setupEnv*(extraValidation: bool = false, ccm: CCModify = nil): TestEnv =
   let
     com = CommonRef.new(
       newCoreDbRef LegacyDbMemory,
-      conf.pruneMode == PruneMode.Full,
+      conf.chainDbMode == ChainDbMode.Prune,
       conf.networkId,
       conf.networkParams
     )

diff --git a/tests/test_configuration.nim b/tests/test_configuration.nim
@@ -39,18 +39,21 @@ proc configurationMain*() =
       check dd.dataDir.string == "apple\\bin"
       check dd.keyStore.string == "banana/bin"
 
-    test "prune-mode":
+    test "chaindb-mode":
       let aa = makeTestConfig()
-      check aa.pruneMode == PruneMode.Full
+      check aa.chainDbMode == ChainDbMode.Prune
 
-      let bb = makeConfig(@["--prune-mode:full"])
-      check bb.pruneMode == PruneMode.Full
+      let bb = makeConfig(@["--chaindb:prune"])
+      check bb.chainDbMode == ChainDbMode.Prune
 
-      let cc = makeConfig(@["--prune-mode:archive"])
-      check cc.pruneMode == PruneMode.Archive
+      let cc = makeConfig(@["--chaindb:archive"])
+      check cc.chainDbMode == ChainDbMode.Archive
 
       let dd = makeConfig(@["-p:archive"])
-      check dd.pruneMode == PruneMode.Archive
+      check dd.chainDbMode == ChainDbMode.Archive
+
+      let ee = makeConfig(@["--chaindb:aristo"])
+      check ee.chainDbMode == ChainDbMode.Aristo
 
     test "import":
       let aa = makeTestConfig()

diff --git a/tests/test_merge.nim b/tests/test_merge.nim
@@ -66,7 +66,7 @@ proc runTest(steps: Steps) =
     ethNode = setupEthNode(conf, ctx, eth)
     com = CommonRef.new(
       newCoreDbRef LegacyDbMemory,
-      conf.pruneMode == PruneMode.Full,
+      conf.chainDbMode == ChainDbMode.Prune,
       conf.networkId,
       conf.networkParams
     )

diff --git a/tests/test_rpc.nim b/tests/test_rpc.nim
@@ -209,7 +209,7 @@ proc rpcMain*() =
       ethNode = setupEthNode(conf, ctx, eth)
       com = CommonRef.new(
         newCoreDbRef LegacyDbMemory,
-        conf.pruneMode == PruneMode.Full,
+        conf.chainDbMode == ChainDbMode.Prune,
         conf.networkId,
         conf.networkParams
       )

diff --git a/tests/test_txpool/helpers.nim b/tests/test_txpool/helpers.nim
@@ -15,7 +15,7 @@ import
   ../replay/[pp, undump_blocks],
   chronicles,
   eth/[common, keys],
-  stew/[keyed_queue, sorted_set],
+  stew/[byteutils,keyed_queue, sorted_set],
   stint
 
 # Make sure that the runner can stay on public view without the need
@@ -76,6 +76,10 @@ const
 # Helpers
 # ------------------------------------------------------------------------------
 
+proc toHex2(a: byte): string =
+  # Avoids compiler warning with `toHex(2)`
+  [a].toHex
+
 proc joinXX(s: string): string =
   if s.len <= 30:
     return s
@@ -92,13 +96,13 @@ proc toXX[T](s: T): string =
   s.toHex.strip(leading=true,chars={'0'}).toLowerAscii
 
 proc toXX(q: Blob): string =
-  q.mapIt(it.toHex(2)).join(":")
+  q.mapIt(it.toHex2).join(":")
 
 proc toXX(a: EthAddress): string =
-  a.mapIt(it.toHex(2)).joinXX
+  a.mapIt(it.toHex2).joinXX
 
 proc toXX(h: Hash256): string =
-  h.data.mapIt(it.toHex(2)).joinXX
+  h.data.mapIt(it.toHex2).joinXX
 
 proc toXX(v: int64; r,s: UInt256): string =
   v.toXX & ":" & ($r).joinXX & ":" & ($s).joinXX
@@ -184,7 +188,7 @@ proc isOk*(rc: ValidationResult): bool =
   rc == ValidationResult.OK
 
 proc toHex*(acc: EthAddress): string =
-  acc.toSeq.mapIt(it.toHex(2)).join
+  acc.toHex
 
 proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) =
   if noisy:

diff --git a/tests/test_txpool2.nim b/tests/test_txpool2.nim
@@ -106,7 +106,7 @@ proc initEnv(envFork: HardFork): TestEnv =
   let
     com = CommonRef.new(
       newCoreDbRef LegacyDbMemory,
-      conf.pruneMode == PruneMode.Full,
+      conf.chainDbMode == ChainDbMode.Prune,
       conf.networkId,
       conf.networkParams
     )