Issue #2242 load checkpoints as stream (#2243)

* (feat) Read compressed persistence in line

* (fix) Remove spurious function

* (fix) Fix benchmarks so they run again

* (fix) Resize decompression buffer

zilliqa/benches/it.rs

@@ -68,6 +68,11 @@ fn process_empty(c: &mut Criterion) {
                 consensus.minimum_stake = "1"
                 consensus.eth_block_gas_limit = 1000000000
                 consensus.gas_price = "1"
+                consensus.genesis_fork.at_height = 0
+                consensus.genesis_fork.failed_scilla_call_from_gas_exempt_caller_causes_revert = true
+                consensus.genesis_fork.call_mode_1_sets_caller_to_parent_caller = true
+                consensus.genesis_fork.scilla_messages_can_call_evm_contracts = true
+                consensus.genesis_fork.scilla_contract_creation_increments_account_balance = true
                 consensus.genesis_accounts = [
                     [
                         "0x0000000000000000000000000000000000000000",
@@ -188,6 +193,11 @@ fn consensus(
             consensus.minimum_stake = "1"
             consensus.eth_block_gas_limit = 84000000
             consensus.gas_price = "1"
+            consensus.genesis_fork.at_height = 0
+            consensus.genesis_fork.failed_scilla_call_from_gas_exempt_caller_causes_revert = true
+            consensus.genesis_fork.call_mode_1_sets_caller_to_parent_caller = true
+            consensus.genesis_fork.scilla_messages_can_call_evm_contracts = true
+            consensus.genesis_fork.scilla_contract_creation_increments_account_balance = true
         "#,
     )
     .unwrap();

zilliqa/src/db.rs

@@ -519,14 +519,9 @@ impl Db {
         const SUPPORTED_VERSION: u32 = 3;
 
         // Decompress file and write to temp file
-        let input_filename = path.as_ref();
-        let temp_filename = input_filename.with_extension("part");
-        decompress_file(input_filename, &temp_filename)?;
-
-        // Read decompressed file
-        let input = File::open(&temp_filename)?;
-
-        let mut reader = BufReader::with_capacity(128 * 1024 * 1024, input); // 128 MiB read chunks
+        let input_file = File::open(path.as_ref())?;
+        let buf_reader: BufReader<File> = BufReader::with_capacity(128 * 1024 * 1024, input_file);
+        let mut reader = Decoder::new(buf_reader)?;
         let trie_storage = Arc::new(self.state_trie()?);
         let mut state_trie = EthTrie::new(trie_storage.clone());
 
@@ -712,8 +707,6 @@ impl Db {
             Ok(())
         })?;
 
-        fs::remove_file(temp_filename)?;
-
         Ok(Some((block, transactions, parent)))
     }
 
@@ -1328,26 +1321,6 @@ fn compress_file<P: AsRef<Path> + Debug>(input_file_path: P, output_file_path: P
     Ok(())
 }
 
-/// Read lz4 compressed file and write into output file
-fn decompress_file<P: AsRef<Path> + Debug>(input_file_path: P, output_file_path: P) -> Result<()> {
-    let reader: BufReader<File> = BufReader::new(File::open(input_file_path)?);
-    let mut decoder = Decoder::new(reader)?;
-
-    let mut writer = BufWriter::new(File::create(output_file_path)?);
-    let mut buffer = [0u8; 1024 * 64]; // read 64KB chunks at a time
-    loop {
-        let bytes_read = decoder.read(&mut buffer)?; // Read a chunk of decompressed data
-        if bytes_read == 0 {
-            break; // End of file
-        }
-        writer.write_all(&buffer[..bytes_read])?;
-    }
-
-    writer.flush()?;
-
-    Ok(())
-}
-
 /// An implementor of [eth_trie::DB] which uses a [Connection] to persist data.
 #[derive(Debug, Clone)]
 pub struct TrieStorage {