Merge pull request #146 from eosnetworkfoundation/elmato/fix-serializ…

…ation-consensus-params

[1.0] Store block extra data in a separate table

eosevm/block_extra_data.cpp

@@ -0,0 +1,50 @@
+#include <silkworm/core/rlp/encode.hpp>
+#include <silkworm/core/rlp/decode.hpp>
+
+#include "block_extra_data.hpp"
+
+using namespace silkworm;
+
+namespace eosevm {
+    bool operator==(const eosevm::block_extra_data& a, const eosevm::block_extra_data& b) {
+        return a.consensus_parameter_index == b.consensus_parameter_index;
+    }
+}
+
+namespace silkworm { namespace rlp {
+
+    template <typename T>
+    void encode(Bytes& to, const std::optional<T>& v) noexcept {
+        auto has_value = v.has_value();
+        rlp::encode(to, has_value);
+        if(has_value) {
+            rlp::encode(to, v.value());
+        }
+    }
+
+    template <typename T>
+    DecodingResult decode(ByteView& from, std::optional<T>& out) noexcept {
+        bool has_value = false;
+        if (DecodingResult res{decode(from, has_value, Leftover::kAllow)}; !res) {
+            return res;
+        }
+        if(has_value) {
+            out = T{};
+            if (DecodingResult res{decode(from, *out, Leftover::kAllow)}; !res) {
+                return res;
+            }
+        }
+        return {};
+    }
+
+    silkworm::Bytes encode(const eosevm::block_extra_data& out) {
+        silkworm::Bytes to;
+        encode(to, out.consensus_parameter_index);
+        return to;
+    }
+
+    DecodingResult decode(silkworm::ByteView& from, eosevm::block_extra_data& to) noexcept{
+        decode(from, to.consensus_parameter_index);
+        return {};
+    }
+} }  //namespace silkworm::rlp

eosevm/block_extra_data.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <optional>
+
+namespace eosevm {
+
+struct block_extra_data {
+    std::optional<evmc::bytes32> consensus_parameter_index;
+};
+
+} // namespace eosevm
+
+namespace silkworm { namespace rlp {
+    silkworm::Bytes encode(const eosevm::block_extra_data&);
+    DecodingResult decode(silkworm::ByteView& from, eosevm::block_extra_data& to) noexcept;
+}}  // namespace silkworm::rlp

silkworm/core/types/block.cpp

@@ -33,7 +33,7 @@ bool operator==(const BlockHeader& a, const BlockHeader& b) {
 }
 
 bool operator==(const BlockBody& a, const BlockBody& b) {
-    return a.transactions == b.transactions && a.ommers == b.ommers && a.consensus_parameter_index == b.consensus_parameter_index;
+    return a.transactions == b.transactions && a.ommers == b.ommers;
 }
 
 BlockNum height(const BlockId& b) { return b.number; }
@@ -241,9 +241,6 @@ namespace rlp {
         Header rlp_head{.list = true};
         rlp_head.payload_length += length(b.transactions);
         rlp_head.payload_length += length(b.ommers);
-        if (b.consensus_parameter_index) {
-            rlp_head.payload_length += length(*b.consensus_parameter_index);
-        }
         if (b.withdrawals) {
             rlp_head.payload_length += length(*b.withdrawals);
         }
@@ -259,9 +256,6 @@ namespace rlp {
         encode_header(to, rlp_header_body(block_body));
         encode(to, block_body.transactions);
         encode(to, block_body.ommers);
-        if (block_body.consensus_parameter_index) {
-            encode(to, *block_body.consensus_parameter_index);
-        }
         if (block_body.withdrawals) {
             encode(to, *block_body.withdrawals);
         }
@@ -284,15 +278,6 @@ namespace rlp {
             return res;
         }
 
-        to.consensus_parameter_index = std::nullopt;
-        if (from.length() > leftover) {
-            evmc::bytes32 consensus_parameter_index;
-            if (DecodingResult res{decode(from, consensus_parameter_index, Leftover::kAllow)}; !res) {
-                return res;
-            }
-            to.consensus_parameter_index = consensus_parameter_index;
-        }
-
         to.withdrawals = std::nullopt;
         if (from.length() > leftover) {
             std::vector<Withdrawal> withdrawals;
@@ -325,15 +310,6 @@ namespace rlp {
             return res;
         }
 
-        to.consensus_parameter_index = std::nullopt;
-        if (from.length() > leftover) {
-            evmc::bytes32 consensus_parameter_index;
-            if (DecodingResult res{decode(from, consensus_parameter_index, Leftover::kAllow)}; !res) {
-                return res;
-            }
-            to.consensus_parameter_index = consensus_parameter_index;
-        }
-
         to.withdrawals = std::nullopt;
         if (from.length() > leftover) {
             std::vector<Withdrawal> withdrawals;
@@ -365,9 +341,6 @@ namespace rlp {
         encode(to, block.header);
         encode(to, block.transactions);
         encode(to, block.ommers);
-        if (block.consensus_parameter_index) {
-            encode(to, *block.consensus_parameter_index);
-        }
         if (block.withdrawals) {
             encode(to, *block.withdrawals);
         }

silkworm/core/types/block.hpp

@@ -32,7 +32,7 @@
 #include <silkworm/core/types/transaction.hpp>
 #include <silkworm/core/types/withdrawal.hpp>
 
-#include <eosevm/consensus_parameters.hpp>
+#include <eosevm/block_extra_data.hpp>
 
 namespace silkworm {
 
@@ -96,16 +96,32 @@ struct BlockBody {
     std::vector<BlockHeader> ommers;
     std::optional<std::vector<Withdrawal>> withdrawals{std::nullopt};
 
-    // EOS-EVM 
-    std::optional<evmc::bytes32> consensus_parameter_index{std::nullopt};
+    // EOS-EVM
+    std::optional<eosevm::block_extra_data> eosevm_extra_data{std::nullopt};
+
+    bool has_consensus_parameter_index() {
+        return eosevm_extra_data.has_value() && eosevm_extra_data->consensus_parameter_index.has_value();
+    }
+
+    void set_consensus_parameter_index(const std::optional<evmc::bytes32>& index) {
+        if(!eosevm_extra_data.has_value()) eosevm_extra_data=eosevm::block_extra_data{};
+        eosevm_extra_data->consensus_parameter_index = index;
+    }
+
+    std::optional<evmc::bytes32> get_consensus_parameter_index() const {
+        if(!eosevm_extra_data.has_value()) return {};
+        return eosevm_extra_data.value().consensus_parameter_index;
+    }
 
     friend bool operator==(const BlockBody&, const BlockBody&);
 };
 
 struct Block : public BlockBody {
     BlockHeader header;
 
+    // EOS-EVM
     bool irreversible{false};
+
     void recover_senders();
 };
 

silkworm/core/types/block_test.cpp

@@ -104,8 +104,6 @@ TEST_CASE("BlockBody RLP 2") {
     body.ommers[0].prev_randao = 0xf0a53dfdd6c2f2a661e718ef29092de60d81d45f84044bec7bf4b36630b2bc08_bytes32;
     body.ommers[0].nonce[7] = 35;
 
-    body.consensus_parameter_index = evmc::bytes32(1234);
-
     Bytes rlp{};
     rlp::encode(rlp, body);
 

silkworm/node/db/access_layer.cpp

@@ -25,6 +25,7 @@
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/node/db/bitmap.hpp>
 #include <silkworm/node/db/tables.hpp>
+#include <eosevm/block_extra_data.hpp>
 
 namespace silkworm::db {
 
@@ -411,6 +412,9 @@ size_t process_blocks_at_height(ROTxn& txn, BlockNum height, std::function<void(
             auto [block_num, hash] = split_block_key(key);
             bool present = read_header(txn, hash, block_num, block.header);
             if (!present) throw std::logic_error("header not found for body number= " + std::to_string(block_num) + ", hash= " + to_hex(hash));
+            // ...extra data
+            const Bytes ekey{block_key(block_num, hash.bytes)};
+            read_extra_block_data(txn, ekey, block);
             // invoke handler
             process_func(block);
         },
@@ -429,6 +433,19 @@ bool read_body(ROTxn& txn, BlockNum block_number, const uint8_t (&hash)[kHashLen
     return read_body(txn, key, read_senders, out);
 }
 
+void read_extra_block_data(ROTxn& txn, const Bytes& key, BlockBody& out) {
+    auto extra_cursor = txn.ro_cursor(table::kExtraBlockData);
+    auto extra_data{extra_cursor->find(to_slice(key), false)};
+    if (!extra_data) {
+        return;
+    }
+    ByteView extra_data_view{from_slice(extra_data.value)};
+    if(extra_data_view.length()) {
+        out.eosevm_extra_data = eosevm::block_extra_data{};
+        rlp::decode(extra_data_view, *out.eosevm_extra_data);
+    }
+}
+
 bool read_body(ROTxn& txn, const Bytes& key, bool read_senders, BlockBody& out) {
     auto cursor = txn.ro_cursor(table::kBlockBodies);
     auto data{cursor->find(to_slice(key), false)};
@@ -443,7 +460,9 @@ bool read_body(ROTxn& txn, const Bytes& key, bool read_senders, BlockBody& out)
     if (!out.transactions.empty() && read_senders) {
         parse_senders(txn, key, out.transactions);
     }
-    out.consensus_parameter_index = body.consensus_parameter_index;
+
+    // Read block extra data
+    read_extra_block_data(txn, key, out);
     return true;
 }
 
@@ -499,14 +518,21 @@ void write_body(RWTxn& txn, const BlockBody& body, const uint8_t (&hash)[kHashLe
     body_for_storage.txn_count = body.transactions.size();
     body_for_storage.base_txn_id =
         increment_map_sequence(txn, table::kBlockTransactions.name, body_for_storage.txn_count);
-    body_for_storage.consensus_parameter_index = body.consensus_parameter_index;
     Bytes value{body_for_storage.encode()};
     auto key{db::block_key(number, hash)};
 
     auto target = txn.rw_cursor(table::kBlockBodies);
     target->upsert(to_slice(key), to_slice(value));
 
     write_transactions(txn, body.transactions, body_for_storage.base_txn_id);
+
+    // Write block extra data
+    if(!body.eosevm_extra_data.has_value()) return;
+    auto extra_value = rlp::encode(*body.eosevm_extra_data);
+
+    key = db::block_key(number, hash);
+    auto extra = txn.rw_cursor(table::kExtraBlockData);
+    extra->upsert(to_slice(key), to_slice(extra_value));
 }
 
 static ByteView read_senders_raw(ROTxn& txn, const Bytes& key) {
@@ -1230,6 +1256,17 @@ std::optional<eosevm::ConsensusParameters> read_consensus_parameters(ROTxn& txn,
     return eosevm::ConsensusParameters::decode(encoded);
 }
 
+std::optional<eosevm::ConsensusParameters> read_consensus_parameters(ROTxn& txn, const Block& block) {
+    std::optional<eosevm::ConsensusParameters> consensus_parameter;
+    auto cpi = block.get_consensus_parameter_index();
+    if(cpi.has_value()) {
+        consensus_parameter = read_consensus_parameters(txn, cpi.value());
+    }
+    return consensus_parameter;
+}
+
+
+
 void update_consensus_parameters(RWTxn& txn, const evmc::bytes32& index, const eosevm::ConsensusParameters& config) {
     auto cursor = txn.rw_cursor(table::kConsensusParameters);
     auto key{db::block_key(index.bytes)};

silkworm/node/db/access_layer.hpp

@@ -120,6 +120,8 @@ size_t process_blocks_at_height(ROTxn& txn, BlockNum height, std::function<void(
 void write_body(RWTxn& txn, const BlockBody& body, const evmc::bytes32& hash, BlockNum bn);
 void write_body(RWTxn& txn, const BlockBody& body, const uint8_t (&hash)[kHashLength], BlockNum number);
 
+void read_extra_block_data(ROTxn& txn, const Bytes& key, BlockBody& out);
+
 // See Erigon ReadTd
 std::optional<intx::uint256> read_total_difficulty(ROTxn& txn, BlockNum, const evmc::bytes32& hash);
 std::optional<intx::uint256> read_total_difficulty(ROTxn& txn, BlockNum, const uint8_t (&hash)[kHashLength]);
@@ -254,6 +256,9 @@ void write_runtime_states_u64(RWTxn& txn, RuntimeState runtime_state, uint64_t n
 //! \brief  Read ConsensusParameters indexed by blocknum that it is added.
 std::optional<eosevm::ConsensusParameters> read_consensus_parameters(ROTxn& txn, const evmc::bytes32& index);
 
+//! \brief  Read ConsensusParameters for block.
+std::optional<eosevm::ConsensusParameters> read_consensus_parameters(ROTxn& txn, const Block& block);
+
 //! \brief  Write ConsensusParameters indexed by blocknum that it is added. Can overwrite during forks.
 void update_consensus_parameters(RWTxn& txn, const evmc::bytes32&, const eosevm::ConsensusParameters& config);
 

silkworm/node/db/access_layer_test.cpp

@@ -35,7 +35,7 @@
 
 namespace silkworm {
 
-static BlockBody sample_block_body() {
+static BlockBody sample_block_body(bool with_block_extra_data=true) {
     BlockBody body;
     body.transactions.resize(2);
 
@@ -81,7 +81,12 @@ static BlockBody sample_block_body() {
     body.ommers[0].prev_randao = 0xf0a53dfdd6c2f2a661e718ef29092de60d81d45f84044bec7bf4b36630b2bc08_bytes32;
     body.ommers[0].nonce[7] = 35;
 
-    body.consensus_parameter_index = evmc::bytes32(1234);
+    if(with_block_extra_data) {
+        body.eosevm_extra_data = {
+            .consensus_parameter_index = 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3_bytes32
+        };
+    }
+
     return body;
 }
 
@@ -502,6 +507,9 @@ TEST_CASE("Headers and bodies") {
         CHECK(block.header == header);
         CHECK(block.ommers == body.ommers);
         CHECK(block.transactions == body.transactions);
+        CHECK(block.eosevm_extra_data.has_value() == true);
+        CHECK(block.eosevm_extra_data.value().consensus_parameter_index.has_value() == true);
+        CHECK(block.eosevm_extra_data->consensus_parameter_index == 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3_bytes32);
 
         CHECK(!block.transactions[0].from);
         CHECK(!block.transactions[1].from);
@@ -528,8 +536,15 @@ TEST_CASE("Headers and bodies") {
         auto [b, h] = split_block_key(key);
         REQUIRE(b == header.number);
         REQUIRE(h == header.hash());
+    }
+
+    SECTION("empty eosevm_extra_data") {
+        BlockBody body{sample_block_body(false)};
+        CHECK_NOTHROW(write_body(txn, body, header.hash(), header.number));
 
-        CHECK(block.consensus_parameter_index == evmc::bytes32(1234));
+        Block block;
+        REQUIRE(read_block_by_number(txn, block_num, false, block));
+        CHECK(block.eosevm_extra_data.has_value() == false);
     }
 
     SECTION("process_blocks_at_height") {
@@ -545,6 +560,9 @@ TEST_CASE("Headers and bodies") {
             [&count, &height](const Block& block) {
                 REQUIRE(block.header.number == height);
                 count++;
+                CHECK(block.eosevm_extra_data.has_value() == true);
+                CHECK(block.eosevm_extra_data.value().consensus_parameter_index.has_value() == true);
+                CHECK(block.eosevm_extra_data->consensus_parameter_index == 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3_bytes32);
             });
         REQUIRE(processed == 1);
         REQUIRE(processed == count);
@@ -564,6 +582,9 @@ TEST_CASE("Headers and bodies") {
             height,
             [&count, &height](const Block& block) {
                 REQUIRE(block.header.number == height);
+                CHECK(block.eosevm_extra_data.has_value() == true);
+                CHECK(block.eosevm_extra_data.value().consensus_parameter_index.has_value() == true);
+                CHECK(block.eosevm_extra_data->consensus_parameter_index == 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3_bytes32);
                 count++;
             });
         REQUIRE(processed == 2);

silkworm/node/db/tables.hpp

@@ -69,6 +69,15 @@ inline constexpr db::MapConfig kAccountHistory{kAccountHistoryName};
 inline constexpr const char* kBlockBodiesName{"BlockBody"};
 inline constexpr db::MapConfig kBlockBodies{kBlockBodiesName};
 
+//! \details Holds extra block data
+//! \struct
+//! \verbatim
+//!   key   : block number (BE 8 bytes) + block header hash (32 bytes)
+//!   value : extra block data RLP encoded
+//! \endverbatim
+inline constexpr const char* kExtraBlockDataName{"ExtraBlockData"};
+inline constexpr db::MapConfig kExtraBlockData{kExtraBlockDataName};
+
 //! \details Stores the binding of *canonical* block number with header hash
 //! \struct
 //! \verbatim
@@ -426,6 +435,7 @@ inline constexpr db::MapConfig kChainDataTables[]{
     kTxLookup,
     kRuntimeStates,
     kConsensusParameters,
+    kExtraBlockData
 };
 
 //! \brief Ensures all defined tables are present in db with consistent flags. Should a table not exist it gets created