Bug/1588 change snapshot hash computation

libdevcore/LevelDB.cpp

@@ -22,7 +22,6 @@
 #include "Assertions.h"
 #include "Log.h"
 #include <libdevcore/microprofile.h>
-#include <secp256k1_sha256.h>
 
 namespace dev {
 namespace db {
@@ -213,7 +212,6 @@
     }
 }
 
-
 void LevelDB::forEachWithPrefix(
     std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
     cnote << "Iterating over the LevelDB prefix: " << _prefix;
@@ -238,6 +236,7 @@
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
@@ -253,6 +252,7 @@
         bytesConstRef str_key_value( usc.data(), usc.size() );
         secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
     }
+
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
     return hash;
@@ -263,6 +263,7 @@
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
@@ -280,6 +281,39 @@
     return hash;
 }
 
+void LevelDB::hashBasePartially(
+    secp256k1_sha256_t* ctx, std::string& lastHashedKey, size_t batchSize ) const {
+    std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
+    if ( it == nullptr ) {
+        BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
+    }
+
+    if ( lastHashedKey != "start" )
+        it->Seek( lastHashedKey );
+    else
+        it->SeekToFirst();
+
+    for ( size_t counter = 0; it->Valid() && counter < batchSize; it->Next() ) {
+        std::string key_ = it->key().ToString();
+        std::string value_ = it->value().ToString();
+        // HACK! For backward compatibility! When snapshot could happen between update of two nodes
+        // - it would lead to stateRoot mismatch
+        // TODO Move this logic to separate "compatiliblity layer"!
+        if ( key_ == "pieceUsageBytes" )
+            continue;
+        std::string keyValue = key_ + value_;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
+        ++counter;
+    }
+
+    if ( it->Valid() )
+        lastHashedKey = it->key().ToString();
+    else
+        lastHashedKey = "stop";
+}
+
 void LevelDB::doCompaction() const {
     m_db->CompactRange( nullptr, nullptr );
 }

libdevcore/LevelDB.h

@@ -26,6 +26,8 @@
 #include <leveldb/write_batch.h>
 #include <boost/filesystem.hpp>
 
+#include <secp256k1_sha256.h>
+
 namespace dev {
 namespace db {
 class LevelDB : public DatabaseFace {
@@ -59,6 +61,9 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
+    void hashBasePartially(
+        secp256k1_sha256_t* ctx, std::string& lastHashedKey, size_t batchSize = 10000 ) const;
+
     void doCompaction() const;
 
     // Return the total count of key deletes  since the start

libskale/SnapshotManager.cpp

@@ -445,13 +445,24 @@ void SnapshotManager::computeDatabaseHash(
         BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
     }
 
-    std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
-        dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
-        dev::db::LevelDB::defaultSnapshotDBOptions() ) );
-    dev::h256 hash_volume = m_db->hashBase();
-    cnote << _dbDir << " hash is: " << hash_volume << std::endl;
+    secp256k1_sha256_t dbCtx;
+    secp256k1_sha256_initialize( &dbCtx );
 
-    secp256k1_sha256_write( ctx, hash_volume.data(), hash_volume.size );
+    std::string finishKey = "start";
+
+    while ( finishKey != "stop" ) {
+        std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
+            dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
+            dev::db::LevelDB::defaultSnapshotDBOptions() ) );
+
+        m_db->hashBasePartially( &dbCtx, finishKey );
+    }
+
+    dev::h256 dbHash;
+    secp256k1_sha256_finalize( &dbCtx, dbHash.data() );
+    cnote << _dbDir << " hash is: " << dbHash << std::endl;
+
+    secp256k1_sha256_write( ctx, dbHash.data(), dbHash.size );
 } catch ( const fs::filesystem_error& ex ) {
     std::throw_with_nested( CannotRead( ex.path1() ) );
 }

skaled/main.cpp

@@ -460,7 +460,7 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
                            "hash " )
                     << cc::notice( votedHash.first.hex() )
                     << cc::notice( " is not equal to calculated hash " )
-                    << cc::notice( calculated_hash.hex() ) << cc::notice( "Will try again" );
+                    << cc::notice( calculated_hash.hex() ) << cc::notice( " Will try again" );
                 if ( isRegularSnapshot )
                     snapshotManager->cleanup();
                 else

test/unittests/libweb3core/LevelDBHash.cpp

@@ -9,44 +9,95 @@ BOOST_AUTO_TEST_SUITE( LevelDBHashBase )
 BOOST_AUTO_TEST_CASE( hash ) {
     dev::TransientDirectory td;
 
+    std::vector< std::pair< std::string, std::string > > randomKeysValues(123);
+
     dev::h256 hash;
     {
         std::unique_ptr< dev::db::LevelDB > db( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db );
 
+        db->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        db->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 43 + i );
-            std::string value = std::to_string( i );
+            std::string key = dev::h256::random().hex();
+            std::string value = dev::h256::random().hex();
             db->insert( dev::db::Slice(key), dev::db::Slice(value) );
+
+            randomKeysValues[i] = { key, value };
         }
 
         hash = db->hashBase();
     }
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hash_same;
     {
         std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db_copy );
 
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 43 + i );
-            std::string value = std::to_string( i );
+            std::string key = randomKeysValues[i].first;
+            std::string value = randomKeysValues[i].second;
             db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
         }
+        db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
 
         hash_same = db_copy->hashBase();
     }
 
     BOOST_REQUIRE( hash == hash_same );
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
+    dev::h256 hashPartially;
+    {
+        {
+            std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
+            BOOST_REQUIRE( db_copy );
+
+            for ( size_t i = 0; i < 123; ++i ) {
+                std::string key = randomKeysValues[i].first;
+                std::string value = randomKeysValues[i].second;
+                db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
+            }
+
+            db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+            db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        }
+
+        secp256k1_sha256_t dbCtx;
+        secp256k1_sha256_initialize( &dbCtx );
+
+        std::string finishKey = "start";
+        while ( finishKey != "stop" ) {
+            std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( td.path(),
+                dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
+                dev::db::LevelDB::defaultSnapshotDBOptions() ) );
+
+            m_db->hashBasePartially( &dbCtx, finishKey, 10 );
+        }
+
+        secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );
+    }
+
+    BOOST_REQUIRE( hash == hashPartially );
+
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hash_diff;
     {
         std::unique_ptr< dev::db::LevelDB > db_diff( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db_diff );
 
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 42 + i );
-            std::string value = std::to_string( i );
+            std::string key = dev::h256::random().hex();
+            std::string value = dev::h256::random().hex();
             db_diff->insert( dev::db::Slice(key), dev::db::Slice(value) );
         }