Skip to content

Commit

Permalink
SKALED=1870 Split OverlayDB and tune tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@dimalit
dimalit committed Oct 31, 2024
1 parent 75322fb commit 14a0c16
Show file tree
Hide file tree
Showing 10 changed files with 286 additions and 148 deletions.
6 changes: 3 additions & 3 deletions libethereum/Account.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,14 +222,14 @@ AccountMap dev::eth::jsonToAccountMap( std::string const& _json, u256 const& _de
}


u256 Account::originalStorageValue( u256 const& _key, skale::OverlayDB const& _db ) const {
u256 Account::originalStorageValue( u256 const& _key, skale::ClassicOverlayDB const& _db ) const {
auto it = m_storageOriginal.find( _key );
if ( it != m_storageOriginal.end() )
return it->second;

// Not in the original values cache - go to the DB.
SecureTrieDB< h256, skale::OverlayDB > const memdb(
const_cast< skale::OverlayDB* >( &_db ), m_storageRoot );
SecureTrieDB< h256, skale::ClassicOverlayDB > const memdb(
const_cast< skale::ClassicOverlayDB* >( &_db ), m_storageRoot );
std::string const payload = memdb.at( _key );
auto const value = payload.size() ? RLP( payload ).toInt< u256 >() : 0;
m_storageOriginal[_key] = value;
Expand Down
4 changes: 2 additions & 2 deletions libethereum/Account.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,11 +244,11 @@ class Account {

/// @returns account's original storage value corresponding to the @_key
/// not taking into account overlayed modifications
u256 originalStorageValue( u256 const& _key, skale::OverlayDB const& _db ) const;
u256 originalStorageValue( u256 const& _key, skale::ClassicOverlayDB const& _db ) const;

/// @returns account's storage value corresponding to the @_key
/// taking into account overlayed modifications
u256 storageValue( u256 const& _key, skale::OverlayDB const& _db ) const {
u256 storageValue( u256 const& _key, skale::ClassicOverlayDB const& _db ) const {
auto mit = m_storageOverlay.find( _key );
if ( mit != m_storageOverlay.end() )
return mit->second;
Expand Down
2 changes: 1 addition & 1 deletion libethereum/Client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ void Client::populateNewChainStateFromGenesis() {
m_state.populateFrom( bc().chainParams().genesisState );
m_state.mutableHistoricState().saveRootForBlockNumber( 0 ); // TODO is it safe to assume
// it's 0?
m_state.mutableHistoricState().db().commit( "0", true );
m_state.mutableHistoricState().db().commit( "0" );
m_state.releaseWriteLock();
#else
m_state.createStateModifyCopy().populateFrom( bc().chainParams().genesisState );
Expand Down
27 changes: 14 additions & 13 deletions libhistoric/HistoricState.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ using namespace dev::eth;
namespace fs = boost::filesystem;

HistoricState::HistoricState( u256 const& _accountStartNonce, s256 _maxHistoricStateDbSize,
std::pair< skale::OverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > > _db,
std::pair< skale::OverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
std::pair< skale::ClassicOverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > > _db,
std::pair< skale::ClassicOverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
_blockToStateRootDB,
skale::BaseState _bs )
: m_db( _db.first ),
Expand Down Expand Up @@ -60,7 +60,7 @@ HistoricState::HistoricState( HistoricState const& _s )
m_maxHistoricStateDbSize( _s.m_maxHistoricStateDbSize ),
m_storageUsage( _s.storageUsedTotal().convert_to< uint64_t >() ) {}

std::pair< skale::OverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
std::pair< skale::ClassicOverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
HistoricState::openDB( fs::path const& _basePath, h256 const& _genesisHash, WithExisting _we ) {
DatabasePaths const dbPaths{ _basePath, _genesisHash };
if ( db::isDiskDatabase() ) {
Expand Down Expand Up @@ -91,7 +91,7 @@ HistoricState::openDB( fs::path const& _basePath, h256 const& _genesisHash, With
auto rotatingDB = std::make_shared< dev::db::RotatingHistoricState >( rotator );
auto bdb = std::make_unique< batched_io::batched_db >();
bdb->open( rotatingDB );
return { skale::OverlayDB( std::move( bdb ) ), rotatingDB };
return { skale::ClassicOverlayDB( std::move( bdb ) ), rotatingDB };
} catch ( boost::exception const& ex ) {
if ( db::isDiskDatabase() ) {
clog( VerbosityError, "statedb" )
Expand Down Expand Up @@ -243,7 +243,7 @@ void HistoricState::commitExternalChanges( AccountMap const& _accountMap, uint64
auto newDataSize = calculateNewDataSize( _accountMap );
commitExternalChangesIntoTrieDB( _accountMap, m_state );
updateStorageUsage( newDataSize );
m_state.db()->commit( std::to_string( _blockNumber ), true );
m_state.db()->commit( std::to_string( _blockNumber ) );
m_changeLog.clear();
m_cache.clear();
m_unchangedCacheEntries.clear();
Expand Down Expand Up @@ -348,7 +348,7 @@ void HistoricState::saveRootForBlockNumber( uint64_t _blockNumber ) {
// record the latest block number
auto bnk = sha3( "latest" );
m_blockToStateRootDB.insert( bnk, &str );
m_blockToStateRootDB.commit( "0", true );
m_blockToStateRootDB.commit( "0" );
}

void HistoricState::setRootFromDB() {
Expand Down Expand Up @@ -505,7 +505,8 @@ map< h256, pair< u256, u256 > > HistoricState::storage( Address const& _id ) con
if ( HistoricAccount const* a = account( _id ) ) {
// Pull out all values from trie storage.
if ( h256 root = a->originalStorageRoot() ) {
SecureTrieDB< h256, skale::OverlayDB > memdb( const_cast< skale::OverlayDB* >( &m_db ),
SecureTrieDB< h256, skale::ClassicOverlayDB > memdb(
const_cast< skale::ClassicOverlayDB* >( &m_db ),
root ); // promise we won't alter the overlay! :)

for ( auto it = memdb.hashedBegin(); it != memdb.hashedEnd(); ++it ) {
Expand Down Expand Up @@ -701,8 +702,8 @@ std::ostream& dev::eth::operator<<( std::ostream& _out, HistoricState const& _s
_out << "--- " << _s.globalRoot() << std::endl;
std::set< Address > d;
std::set< Address > dtr;
auto trie = SecureTrieDB< Address, skale::OverlayDB >(
const_cast< skale::OverlayDB* >( &_s.m_db ), _s.globalRoot() );
auto trie = SecureTrieDB< Address, skale::ClassicOverlayDB >(
const_cast< skale::ClassicOverlayDB* >( &_s.m_db ), _s.globalRoot() );
for ( auto i : trie )
d.insert( i.first ), dtr.insert( i.first );
for ( auto i : _s.m_cache )
Expand Down Expand Up @@ -732,8 +733,8 @@ std::ostream& dev::eth::operator<<( std::ostream& _out, HistoricState const& _s
std::set< u256 > delta;
std::set< u256 > cached;
if ( r ) {
SecureTrieDB< h256, skale::OverlayDB > memdb(
const_cast< skale::OverlayDB* >( &_s.m_db ),
SecureTrieDB< h256, skale::ClassicOverlayDB > memdb(
const_cast< skale::ClassicOverlayDB* >( &_s.m_db ),
r[2].toHash< h256 >() ); // promise we won't alter the overlay! :)
for ( auto const& j : memdb )
mem[j.first] = RLP( j.second ).toInt< u256 >(), back.insert( j.first );
Expand Down Expand Up @@ -799,7 +800,7 @@ _txIndex, BlockChain const &_bc) {
*/

AddressHash HistoricState::commitExternalChangesIntoTrieDB(
const AccountMap& _cache, SecureTrieDB< Address, skale::OverlayDB >& _state ) {
const AccountMap& _cache, SecureTrieDB< Address, skale::ClassicOverlayDB >& _state ) {
AddressHash ret;
for ( auto const& i : _cache )
if ( i.second.isDirty() ) {
Expand All @@ -822,7 +823,7 @@ AddressHash HistoricState::commitExternalChangesIntoTrieDB(
storageRoot = existingAccount->originalStorageRoot();
}

SecureTrieDB< h256, skale::OverlayDB > storageDB( _state.db(), storageRoot );
SecureTrieDB< h256, skale::ClassicOverlayDB > storageDB( _state.db(), storageRoot );

for ( auto const& j : i.second.storageOverlay() ) {
if ( j.second ) {
Expand Down
24 changes: 12 additions & 12 deletions libhistoric/HistoricState.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@ class HistoricState {
/// which uses it. If you have no preexisting database then set BaseState to something other
/// than BaseState::PreExisting in order to prepopulate the Trie.
explicit HistoricState( u256 const& _accountStartNonce, s256 _maxHistoricStateDbSize,
std::pair< skale::OverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > > _db,
std::pair< skale::OverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
std::pair< skale::ClassicOverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > > _db,
std::pair< skale::ClassicOverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
_blockToStateRootDB,
skale::BaseState _bs = skale::BaseState::PreExisting );

Expand All @@ -141,11 +141,11 @@ class HistoricState {

/// Open a DB - useful for passing into the constructor & keeping for other states that are
/// necessary.
static std::pair< skale::OverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > > openDB(
boost::filesystem::path const& _path, h256 const& _genesisHash,
static std::pair< skale::ClassicOverlayDB, std::shared_ptr< dev::db::RotatingHistoricState > >
openDB( boost::filesystem::path const& _path, h256 const& _genesisHash,
WithExisting _we = WithExisting::Trust );
skale::OverlayDB const& db() const { return m_db; }
skale::OverlayDB& db() { return m_db; }
skale::ClassicOverlayDB const& db() const { return m_db; }
skale::ClassicOverlayDB& db() { return m_db; }

/// @returns the set containing all addresses currently in use in Ethereum.
/// @warning This is slowslowslow. Don't use it unless you want to lock the object for seconds
Expand Down Expand Up @@ -324,16 +324,16 @@ class HistoricState {
bool executeTransaction( AlethExecutive& _e, Transaction const& _t, OnOpFunc const& _onOp );

/// Our overlay for the state tree.
skale::OverlayDB m_db;
skale::ClassicOverlayDB m_db;
/// Interface for rotating db for the state tree
std::shared_ptr< dev::db::RotatingHistoricState > m_rotatingTreeDb;
/// Overlay DB for the block id state root mapping
skale::OverlayDB m_blockToStateRootDB;
/// ClassicOverlayDB for the block id state root mapping
skale::ClassicOverlayDB m_blockToStateRootDB;
/// Interface for rotating db for the state root mapping
std::shared_ptr< dev::db::RotatingHistoricState > m_rotatingRootsDb;

/// Our state tree, as an OverlayDB DB.
SecureTrieDB< Address, skale::OverlayDB > m_state;
/// Our state tree, as an ClassicOverlayDB DB.
SecureTrieDB< Address, skale::ClassicOverlayDB > m_state;
/// Our address cache. This stores the states of each address that has (or at least might have)
/// been changed.
mutable std::unordered_map< Address, HistoricAccount > m_cache;
Expand All @@ -353,7 +353,7 @@ class HistoricState {
uint64_t readLatestBlock();

AddressHash commitExternalChangesIntoTrieDB(
AccountMap const& _cache, SecureTrieDB< Address, skale::OverlayDB >& _state );
AccountMap const& _cache, SecureTrieDB< Address, skale::ClassicOverlayDB >& _state );

uint64_t calculateNewDataSize( AccountMap const& _cache ) const;

Expand Down
Loading

0 comments on commit 14a0c16

Please sign in to comment.