#1870 optimized search in hisstoric dbs

libbatched-io/BatchedRotatingHistoricDbIO.cpp

@@ -79,6 +79,28 @@ std::shared_ptr< dev::db::DatabaseFace > BatchedRotatingHistoricDbIO::getPieceBy
     return dbsInUse[path];
 }
 
+uint64_t BatchedRotatingHistoricDbIO::getTimestampFromKey( Slice& _key ) {
+    auto keyStr = _key.toString();
+    size_t pos = keyStr.find( ':' );
+    if ( pos == std::string::npos )
+        return -1;
+    uint64_t timestamp = std::stoull( keyStr.substr( pos ) );
+    _key = Slice( _key.begin(), pos + 1 );
+    return timestamp;
+}
+
+std::pair< std::vector< uint64_t >::const_iterator, std::vector< uint64_t >::const_iterator >
+BatchedRotatingHistoricDbIO::getRangeForKey( dev::db::Slice& _key ) {
+    auto timestamps = getTimestamps();
+    auto timestampForCall = getTimestampFromKey( _key );
+    auto it = timestampForCall == uint64_t( -1 ) ?
+                  timestamps.begin() :
+                  std::lower_bound( timestamps.begin(), timestamps.end(), timestampForCall );
+    if ( it != timestamps.begin() )
+        --it;
+    return { it, timestamps.end() };
+}
+
 BatchedRotatingHistoricDbIO::~BatchedRotatingHistoricDbIO() {}
 
 }  // namespace batched_io

libbatched-io/BatchedRotatingHistoricDbIO.h

@@ -30,7 +30,9 @@ class BatchedRotatingHistoricDbIO : public batched_face {
     BatchedRotatingHistoricDbIO( const boost::filesystem::path& _path );
     std::shared_ptr< dev::db::DatabaseFace > currentPiece() const { return current; }
     std::shared_ptr< dev::db::DatabaseFace > getPieceByTimestamp( uint64_t timestamp );
-    std::vector< uint64_t > getPieces() const { return timestamps; }
+    std::vector< uint64_t > getTimestamps() const { return timestamps; }
+    std::pair< std::vector< uint64_t >::const_iterator, std::vector< uint64_t >::const_iterator >
+    getRangeForKey( dev::db::Slice& _key );
     void rotate( uint64_t timestamp );
     void checkOpenedDbsAndCloseIfNeeded();
     virtual void revert() { /* no need - as all write is in rotate() */
@@ -43,6 +45,9 @@ class BatchedRotatingHistoricDbIO : public batched_face {
 
 protected:
     virtual void recover();
+
+private:
+    static uint64_t getTimestampFromKey( dev::db::Slice& _key );
 };
 
 }  // namespace batched_io

libdevcore/RotatingHistoricState.cpp

@@ -7,23 +7,15 @@ namespace db {
 
 using namespace batched_io;
 
-uint64_t getTimestampFromKey( Slice _key ) {
-    auto keyStr = _key.toString();
-    auto pos = keyStr.find( ':' );
-    if ( pos == std::string::npos )
-        return -1;
-    uint64_t timestamp = std::stoull( keyStr.substr( pos ) );
-    _key = Slice( _key.begin(), pos + 1 );
-    return timestamp;
-}
-
 RotatingHistoricState::RotatingHistoricState(
     std::shared_ptr< BatchedRotatingHistoricDbIO > ioBackend_ )
     : ioBackend( ioBackend_ ) {}
 
 void RotatingHistoricState::rotate( uint64_t timestamp ) {
     std::unique_lock< std::shared_mutex > lock( m_mutex );
+
     assert( this->batch_cache.empty() );
+
     ioBackend->rotate( timestamp );
 }
 
@@ -35,12 +27,15 @@ std::string RotatingHistoricState::lookup( Slice _key ) const {
     if ( _key.toString() == std::string( "storageUsed" ) )
         return currentPiece()->lookup( _key );
 
-    for ( auto timestamp : ioBackend->getPieces() ) {
-        auto db = ioBackend->getPieceByTimestamp( timestamp );
+    auto range = ioBackend->getRangeForKey( _key );
+
+    for ( auto it = range.first; it != range.second; ++it ) {
+        auto db = ioBackend->getPieceByTimestamp( *it );
         auto v = db->lookup( _key );
         if ( !v.empty() )
             return v;
     }
+
     return std::string();
 }
 
@@ -49,8 +44,10 @@ bool RotatingHistoricState::exists( Slice _key ) const {
 
     ioBackend->checkOpenedDbsAndCloseIfNeeded();
 
-    for ( auto timestamp : ioBackend->getPieces() ) {
-        auto db = ioBackend->getPieceByTimestamp( timestamp );
+    auto range = ioBackend->getRangeForKey( _key );
+
+    for ( auto it = range.first; it != range.second; ++it ) {
+        auto db = ioBackend->getPieceByTimestamp( *it );
         if ( db->exists( _key ) )
             return true;
     }
@@ -71,29 +68,34 @@ void RotatingHistoricState::kill( Slice _key ) {
 
     ioBackend->checkOpenedDbsAndCloseIfNeeded();
 
-    for ( auto timestamp : ioBackend->getPieces() ) {
-        auto db = ioBackend->getPieceByTimestamp( timestamp );
+    auto range = ioBackend->getRangeForKey( _key );
+
+    for ( auto it = range.first; it != range.second; ++it ) {
+        auto db = ioBackend->getPieceByTimestamp( *it );
         db->kill( _key );
     }
 }
 
 std::unique_ptr< WriteBatchFace > RotatingHistoricState::createWriteBatch() const {
     std::shared_lock< std::shared_mutex > lock( m_mutex );
+
     std::unique_ptr< WriteBatchFace > wbf = currentPiece()->createWriteBatch();
     batch_cache.insert( wbf.get() );
+
     return wbf;
 }
 
 void RotatingHistoricState::commit( std::unique_ptr< WriteBatchFace > _batch ) {
     std::shared_lock< std::shared_mutex > lock( m_mutex );
+
     batch_cache.erase( _batch.get() );
     currentPiece()->commit( std::move( _batch ) );
 }
 
 void RotatingHistoricState::forEach( std::function< bool( Slice, Slice ) > f ) const {
     std::shared_lock< std::shared_mutex > lock( m_mutex );
 
-    for ( auto timestamp : ioBackend->getPieces() ) {
+    for ( auto timestamp : ioBackend->getTimestamps() ) {
         auto db = ioBackend->getPieceByTimestamp( timestamp );
         db->forEach( f );
     }
@@ -103,7 +105,7 @@ void RotatingHistoricState::forEachWithPrefix(
     std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
     std::shared_lock< std::shared_mutex > lock( m_mutex );
 
-    for ( auto timestamp : ioBackend->getPieces() ) {
+    for ( auto timestamp : ioBackend->getTimestamps() ) {
         auto db = ioBackend->getPieceByTimestamp( timestamp );
         db->forEachWithPrefix( _prefix, f );
     }
@@ -114,14 +116,15 @@ h256 RotatingHistoricState::hashBase() const {
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
 
-    for ( auto timestamp : ioBackend->getPieces() ) {
+    for ( auto timestamp : ioBackend->getTimestamps() ) {
         auto db = ioBackend->getPieceByTimestamp( timestamp );
         h256 h = db->hashBase();
         secp256k1_sha256_write( &ctx, h.data(), h.size );
     }
 
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
+
     return hash;
 }
 

libhistoric/HistoricState.cpp

@@ -331,7 +331,6 @@ void HistoricState::setRoot( GlobalRoot const& _r ) {
     m_state.setRoot( _r );
 }
 
-
 void HistoricState::setRootByBlockNumber( uint64_t _blockNumber ) {
     auto key = h256( _blockNumber );
     if ( !m_blockToStateRootDB.exists( key ) ) {
@@ -342,7 +341,6 @@ void HistoricState::setRootByBlockNumber( uint64_t _blockNumber ) {
     setRoot( GlobalRoot( root ) );
 }
 
-
 void HistoricState::saveRootForBlock( uint64_t _blockNumber ) {
     auto key = h256( _blockNumber );
     m_blockToStateRootDB.insert( key, m_state.root().ref() );