Bug/1588 change snapshot hash computation

libdevcore/LevelDB.cpp

@@ -22,13 +22,14 @@
 #include "Assertions.h"
 #include "Log.h"
 #include <libdevcore/microprofile.h>
-#include <secp256k1_sha256.h>
 
 namespace dev {
 namespace db {
 
 unsigned c_maxOpenLeveldbFiles = 25;
 
+const size_t LevelDB::BATCH_CHUNK_SIZE = 10000;
+
 namespace {
 inline leveldb::Slice toLDBSlice( Slice _slice ) {
     return leveldb::Slice( _slice.data(), _slice.size() );
@@ -213,7 +214,6 @@
     }
 }
 
-
 void LevelDB::forEachWithPrefix(
     std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
     cnote << "Iterating over the LevelDB prefix: " << _prefix;
@@ -238,21 +238,23 @@
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
-        std::string key_ = it->key().ToString();
-        std::string value_ = it->value().ToString();
+        std::string keyTmp = it->key().ToString();
+        std::string valueTmp = it->value().ToString();
         // HACK! For backward compatibility! When snapshot could happen between update of two nodes
         // - it would lead to stateRoot mismatch
         // TODO Move this logic to separate "compatiliblity layer"!
-        if ( key_ == "pieceUsageBytes" )
+        if ( keyTmp == "pieceUsageBytes" )
             continue;
-        std::string key_value = key_ + value_;
-        const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-        bytesConstRef str_key_value( usc.data(), usc.size() );
-        secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
+        std::string keyValue = keyTmp + valueTmp;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
     }
+
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
     return hash;
@@ -263,23 +265,57 @@
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
         if ( it->key()[0] == _prefix ) {
-            std::string key_ = it->key().ToString();
-            std::string value_ = it->value().ToString();
-            std::string key_value = key_ + value_;
-            const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-            bytesConstRef str_key_value( usc.data(), usc.size() );
-            secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
+            std::string keyTmp = it->key().ToString();
+            std::string valueTmp = it->value().ToString();
+            std::string keyValue = keyTmp + valueTmp;
+            const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+            bytesConstRef strKeyValue( usc.data(), usc.size() );
+            secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
         }
     }
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
     return hash;
 }
 
+bool LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const {
+    std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
+    if ( it == nullptr ) {
+        BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
+    }
+
+    if ( lastHashedKey != "start" )
+        it->Seek( lastHashedKey );
+    else
+        it->SeekToFirst();
+
+    for ( size_t counter = 0; it->Valid() && counter < BATCH_CHUNK_SIZE; it->Next() ) {
+        std::string keyTmp = it->key().ToString();
+        std::string valueTmp = it->value().ToString();
+        // HACK! For backward compatibility! When snapshot could happen between update of two nodes
+        // - it would lead to stateRoot mismatch
+        // TODO Move this logic to separate "compatiliblity layer"!
+        if ( keyTmp == "pieceUsageBytes" )
+            continue;
+        std::string keyValue = keyTmp + valueTmp;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
+        ++counter;
+    }
+
+    if ( it->Valid() ) {
+        lastHashedKey = it->key().ToString();
+        return true;
+    } else
+        return false;
+}
+
 void LevelDB::doCompaction() const {
     m_db->CompactRange( nullptr, nullptr );
 }

libdevcore/LevelDB.h

@@ -26,6 +26,8 @@
 #include <leveldb/write_batch.h>
 #include <boost/filesystem.hpp>
 
+#include <secp256k1_sha256.h>
+
 namespace dev {
 namespace db {
 class LevelDB : public DatabaseFace {
@@ -59,6 +61,8 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
+    bool hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const;
+
     void doCompaction() const;
 
     // Return the total count of key deletes  since the start
@@ -74,6 +78,8 @@ class LevelDB : public DatabaseFace {
     leveldb::WriteOptions const m_writeOptions;
     leveldb::Options m_options;
     boost::filesystem::path const m_path;
+
+    static const size_t BATCH_CHUNK_SIZE;
 };
 
 }  // namespace db

libskale/SnapshotManager.cpp

@@ -445,13 +445,25 @@ void SnapshotManager::computeDatabaseHash(
         BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
     }
 
-    std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
-        dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
-        dev::db::LevelDB::defaultSnapshotDBOptions() ) );
-    dev::h256 hash_volume = m_db->hashBase();
-    cnote << _dbDir << " hash is: " << hash_volume << std::endl;
+    secp256k1_sha256_t dbCtx;
+    secp256k1_sha256_initialize( &dbCtx );
 
-    secp256k1_sha256_write( ctx, hash_volume.data(), hash_volume.size );
+    std::string lastHashedKey = "start";
+    bool isContinue = true;
+
+    while ( isContinue ) {
+        std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
+            dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
+            dev::db::LevelDB::defaultSnapshotDBOptions() ) );
+
+        isContinue = m_db->hashBasePartially( &dbCtx, lastHashedKey );
+    }
+
+    dev::h256 dbHash;
+    secp256k1_sha256_finalize( &dbCtx, dbHash.data() );
+    cnote << _dbDir << " hash is: " << dbHash << std::endl;
+
+    secp256k1_sha256_write( ctx, dbHash.data(), dbHash.size );
 } catch ( const fs::filesystem_error& ex ) {
     std::throw_with_nested( CannotRead( ex.path1() ) );
 }

skaled/main.cpp

@@ -438,9 +438,9 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
             try {
                 snapshotManager->computeSnapshotHash( blockNumber, true );
             } catch ( const std::exception& ) {
-                std::throw_with_nested(
-                    std::runtime_error( cc::fatal( "FATAL:" ) + " " +
-                                        cc::error( "Exception while computing snapshot hash " ) ) );
+                std::throw_with_nested( std::runtime_error(
+                    std::string( "FATAL:" ) +
+                    std::string( " Exception while computing snapshot hash " ) ) );
             }
 
             dev::h256 calculated_hash = snapshotManager->getSnapshotHash( blockNumber );
@@ -449,18 +449,15 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
                 successfullDownload = true;
                 if ( isRegularSnapshot ) {
                     snapshotManager->restoreSnapshot( blockNumber );
-                    std::cout << cc::success( "Snapshot restore success for block " )
-                              << cc::u( to_string( blockNumber ) ) << std::endl;
+                    std::cout << "Snapshot restore success for block " << to_string( blockNumber )
+                              << std::endl;
                 }
                 return successfullDownload;
             } else {
                 clog( VerbosityWarning, "tryDownloadSnapshot" )
-                    << cc::notice(
-                           "Downloaded snapshot with incorrect hash! Incoming "
-                           "hash " )
-                    << cc::notice( votedHash.first.hex() )
-                    << cc::notice( " is not equal to calculated hash " )
-                    << cc::notice( calculated_hash.hex() ) << cc::notice( "Will try again" );
+                    << "Downloaded snapshot with incorrect hash! Incoming hash "
+                    << votedHash.first.hex() << " is not equal to calculated hash "
+                    << calculated_hash.hex() << " Will try again";
                 if ( isRegularSnapshot )
                     snapshotManager->cleanup();
                 else

test/unittests/libweb3core/LevelDBHash.cpp

@@ -9,44 +9,96 @@ BOOST_AUTO_TEST_SUITE( LevelDBHashBase )
 BOOST_AUTO_TEST_CASE( hash ) {
     dev::TransientDirectory td;
 
+    std::vector< std::pair< std::string, std::string > > randomKeysValues(123);
+
     dev::h256 hash;
     {
         std::unique_ptr< dev::db::LevelDB > db( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db );
 
+        db->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        db->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 43 + i );
-            std::string value = std::to_string( i );
+            std::string key = dev::h256::random().hex();
+            std::string value = dev::h256::random().hex();
             db->insert( dev::db::Slice(key), dev::db::Slice(value) );
+
+            randomKeysValues[i] = { key, value };
         }
 
         hash = db->hashBase();
     }
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hash_same;
     {
         std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db_copy );
 
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 43 + i );
-            std::string value = std::to_string( i );
+            std::string key = randomKeysValues[i].first;
+            std::string value = randomKeysValues[i].second;
             db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
         }
+        db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
 
         hash_same = db_copy->hashBase();
     }
 
     BOOST_REQUIRE( hash == hash_same );
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
+    dev::h256 hashPartially;
+    {
+        {
+            std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
+            BOOST_REQUIRE( db_copy );
+
+            for ( size_t i = 0; i < 123; ++i ) {
+                std::string key = randomKeysValues[i].first;
+                std::string value = randomKeysValues[i].second;
+                db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
+            }
+
+            db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+            db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        }
+
+        secp256k1_sha256_t dbCtx;
+        secp256k1_sha256_initialize( &dbCtx );
+
+        std::string lastHashedKey = "start";
+        bool isContinue = true;
+        while ( isContinue ) {
+            std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( td.path(),
+                dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
+                dev::db::LevelDB::defaultSnapshotDBOptions() ) );
+
+            isContinue = m_db->hashBasePartially( &dbCtx, lastHashedKey );
+        }
+
+        secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );
+    }
+
+    BOOST_REQUIRE( hash == hashPartially );
+
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hash_diff;
     {
         std::unique_ptr< dev::db::LevelDB > db_diff( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db_diff );
 
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 42 + i );
-            std::string value = std::to_string( i );
+            std::string key = dev::h256::random().hex();
+            std::string value = dev::h256::random().hex();
             db_diff->insert( dev::db::Slice(key), dev::db::Slice(value) );
         }