#1870 rotate historic state dbs before block execution

libbatched-io/BatchedRotatingHistoricDbIO.cpp

@@ -26,7 +26,11 @@ BatchedRotatingHistoricDbIO::BatchedRotatingHistoricDbIO( const boost::filesyste
 void BatchedRotatingHistoricDbIO::rotate( uint64_t timestamp ) {
     piecesByTimestamp.push_back( timestamp );
 
+    auto storageUsed = current->lookup( dev::db::Slice( "storageUsed" ) );
+    current->kill( dev::db::Slice( "storageUsed" ) );
+
     current.reset( new LevelDB( basePath / std::to_string( timestamp ) ) );
+    current->insert( dev::db::Slice( "storageUsed" ), storageUsed );
 
     test_crash_before_commit( "after_open_leveldb" );
 }

libdevcore/RotatingHistoricState.cpp

@@ -30,6 +30,9 @@ void RotatingHistoricState::rotate( uint64_t timestamp ) {
 std::string RotatingHistoricState::lookup( Slice _key ) const {
     std::shared_lock< std::shared_mutex > lock( m_mutex );
 
+    if ( _key.toString() == std::string( "storageUsed" ) )
+        return currentPiece()->lookup( _key );
+
     for ( auto timestamp : getPiecesByTimestamp() ) {
         auto db = ioBackend->getPieceByTimestamp( timestamp );
         auto v = db->lookup( _key );

libethereum/Block.cpp

@@ -507,7 +507,7 @@ tuple< TransactionReceipts, unsigned > Block::syncEveryone(
             }
 
             ExecutionResult res =
-                execute( _bc.lastBlockHashes(), tr, Permanence::Committed, OnOpFunc(), i == 0 );
+                execute( _bc.lastBlockHashes(), tr, Permanence::Committed, OnOpFunc() );
 
             if ( !SkipInvalidTransactionsPatch::isEnabledInWorkingBlock() ||
                  res.excepted != TransactionException::WouldNotBeInBlock ) {
@@ -854,8 +854,8 @@ ExecutionResult Block::executeHistoricCall( LastBlockHashesFace const& _lh, Tran
 #endif
 
 
-ExecutionResult Block::execute( LastBlockHashesFace const& _lh, Transaction const& _t,
-    Permanence _p, OnOpFunc const& _onOp, bool _isFirstTxnInBlock ) {
+ExecutionResult Block::execute(
+    LastBlockHashesFace const& _lh, Transaction const& _t, Permanence _p, OnOpFunc const& _onOp ) {
     MICROPROFILE_SCOPEI( "Block", "execute transaction", MP_CORNFLOWERBLUE );
     if ( isSealed() )
         BOOST_THROW_EXCEPTION( InvalidOperationOnSealedBlock() );
@@ -886,8 +886,8 @@ ExecutionResult Block::execute( LastBlockHashesFace const& _lh, Transaction cons
         if ( _t.isInvalid() )
             throw -1;  // will catch below
 
-        resultReceipt = stateSnapshot.execute(
-            envInfo, m_sealEngine->chainParams(), _t, _p, _onOp, _isFirstTxnInBlock );
+        resultReceipt =
+            stateSnapshot.execute( envInfo, m_sealEngine->chainParams(), _t, _p, _onOp );
 
         // use fake receipt created above if execution throws!!
     } catch ( const TransactionException& ex ) {

libethereum/Block.h

@@ -217,8 +217,7 @@ class Block {
     /// Execute a given transaction.
     /// This will append @a _t to the transaction list and change the state accordingly.
     ExecutionResult execute( LastBlockHashesFace const& _lh, Transaction const& _t,
-        skale::Permanence _p = skale::Permanence::Committed, OnOpFunc const& _onOp = OnOpFunc(),
-        bool _isFirstTxnInBlock = false );
+        skale::Permanence _p = skale::Permanence::Committed, OnOpFunc const& _onOp = OnOpFunc() );
 
 #ifdef HISTORIC_STATE
     ExecutionResult executeHistoricCall( LastBlockHashesFace const& _lh, Transaction const& _t,

libethereum/Client.cpp

@@ -655,6 +655,8 @@ size_t Client::syncTransactions(
     DEV_WRITE_GUARDED( x_working ) {
         assert( !m_working.isSealed() );
 
+        m_state.mutableHistoricState().rotateDbsIfNeeded( _timestamp );
+
         // assert(m_state.m_db_write_lock.has_value());
         tie( newPendingReceipts, goodReceipts ) =
             m_working.syncEveryone( bc(), _transactions, _timestamp, _gasPrice, vecMissing );

libhistoric/HistoricState.cpp

@@ -239,12 +239,9 @@ uint64_t HistoricState::calculateNewDataSize( const AccountMap& _cache ) const {
 }
 
 void HistoricState::commitExternalChanges(
-    AccountMap const& _accountMap, uint64_t _blockTimestamp, bool _isFirstTxnInBlock ) {
+    AccountMap const& _accountMap, uint64_t _blockTimestamp ) {
     auto historicStateStart = dev::db::LevelDB::getCurrentTimeMs();
     auto newDataSize = calculateNewDataSize( _accountMap );
-    if ( _isFirstTxnInBlock && isRotationNeeded( newDataSize ) ) {
-        m_rotatingTreeDb->rotate( _blockTimestamp );
-    }
     commitExternalChangesIntoTrieDB( _accountMap, m_state );
     updateStorageUsage( newDataSize );
     m_state.db()->commit( std::to_string( _blockTimestamp ), true );
@@ -857,3 +854,10 @@ AddressHash HistoricState::commitExternalChangesIntoTrieDB(
         }
     return ret;
 }
+
+void HistoricState::rotateDbsIfNeeded( uint64_t _timestamp ) {
+    if ( m_maxHistoricStateDbSize < 1 )
+        return;
+    if ( m_db.storageUsed() > m_maxHistoricStateDbSize )
+        m_rotatingTreeDb->rotate( _timestamp );
+}

libhistoric/HistoricState.h

@@ -266,8 +266,7 @@ class HistoricState {
     /// The hash of the root of our state tree.
     GlobalRoot globalRoot() const { return GlobalRoot( m_state.root() ); }
 
-    void commitExternalChanges(
-        AccountMap const& _cache, uint64_t _blockTimestamp = -1, bool _isFirstTxnInBlock = false );
+    void commitExternalChanges( AccountMap const& _cache, uint64_t _blockTimestamp = -1 );
 
     /// Resets any uncommitted changes to the cache.
     void setRoot( GlobalRoot const& _root );
@@ -299,6 +298,8 @@ class HistoricState {
 
     uint64_t getAndResetBlockCommitTime();
 
+    void rotateDbsIfNeeded( uint64_t _timestamp );
+
 private:
     /// Turns all "touched" empty accounts into non-alive accounts.
     void removeEmptyAccounts();
@@ -354,20 +355,12 @@ class HistoricState {
 
     uint64_t calculateNewDataSize( AccountMap const& _cache ) const;
 
-    bool isRotationNeeded( uint64_t bytes ) const {
-        if ( m_maxHistoricStateDbSize < 1 )
-            return false;
-        return m_storageUsage + bytes > m_maxHistoricStateDbSize;
-    }
-
     void updateStorageUsage( uint64_t bytes ) {
         m_storageUsage += bytes;
         m_db.updateStorageUsage( m_storageUsage );
     }
 
-    dev::s256 storageUsedTotal() const {
-        return m_db.storageUsed();
-    }
+    dev::s256 storageUsedTotal() const { return m_db.storageUsed(); }
 
     uint64_t m_totalTimeSpentInStateCommitsPerBlock = 0;
     dev::s256 m_maxHistoricStateDbSize = -1;

libskale/State.cpp

@@ -490,8 +490,7 @@ void State::clearCacheIfTooLarge() const {
     }
 }
 
-void State::commit(
-    dev::eth::CommitBehaviour _commitBehaviour, uint64_t _timestamp, bool _isFirstTxnInBlock ) {
+void State::commit( dev::eth::CommitBehaviour _commitBehaviour, uint64_t _timestamp ) {
     if ( _commitBehaviour == dev::eth::CommitBehaviour::RemoveEmptyAccounts )
         removeEmptyAccounts();
 
@@ -547,7 +546,7 @@ void State::commit(
 
 
 #ifdef HISTORIC_STATE
-    m_historicState.commitExternalChanges( m_cache, _timestamp, _isFirstTxnInBlock );
+    m_historicState.commitExternalChanges( m_cache, _timestamp );
 #endif
 
     m_changeLog.clear();
@@ -1010,7 +1009,7 @@ bool State::empty() const {
 
 std::pair< ExecutionResult, TransactionReceipt > State::execute( EnvInfo const& _envInfo,
     eth::ChainOperationParams const& _chainParams, Transaction const& _t, Permanence _p,
-    OnOpFunc const& _onOp, bool _isFirstTxnInBlock ) {
+    OnOpFunc const& _onOp ) {
     // Create and initialize the executive. This will throw fairly cheaply and quickly if the
     // transaction is bad in any way.
     // HACK 0 here is for gasPrice
@@ -1070,7 +1069,7 @@ std::pair< ExecutionResult, TransactionReceipt > State::execute( EnvInfo const&
         removeEmptyAccounts = _envInfo.number() >= _chainParams.EIP158ForkBlock;
         commit( removeEmptyAccounts ? dev::eth::CommitBehaviour::RemoveEmptyAccounts :
                                       dev::eth::CommitBehaviour::KeepEmptyAccounts,
-            _envInfo.timestamp(), _isFirstTxnInBlock );
+            _envInfo.timestamp() );
 
         break;
     }

libskale/State.h

@@ -341,14 +341,14 @@ class State {
 
     void commit(
         dev::eth::CommitBehaviour _commitBehaviour = dev::eth::CommitBehaviour::RemoveEmptyAccounts,
-        uint64_t timestamp = -1, bool _isFirstTxnInBlock = false );
+        uint64_t timestamp = -1 );
 
     /// Execute a given transaction.
     /// This will change the state accordingly.
     std::pair< dev::eth::ExecutionResult, dev::eth::TransactionReceipt > execute(
         dev::eth::EnvInfo const& _envInfo, dev::eth::ChainOperationParams const& _chainParams,
         dev::eth::Transaction const& _t, Permanence _p = Permanence::Committed,
-        dev::eth::OnOpFunc const& _onOp = dev::eth::OnOpFunc(), bool _isFirstTxnInBlock = false );
+        dev::eth::OnOpFunc const& _onOp = dev::eth::OnOpFunc() );
 
     /// Get the account start nonce. May be required.
     dev::u256 const& accountStartNonce() const { return m_accountStartNonce; }