diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 70cf8b869..b7410840f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -272,3 +272,20 @@ jobs: if: ${{runner.environment == 'self-hosted'}} working-directory: ./tools/rescue run: RUST_MIN_STACK=104857600 cargo test --no-fail-fast -- --test-threads=1 + storage: + timeout-minutes: 60 + needs: [build-framework] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: setup env + uses: ./.github/actions/build_env + + - uses: Swatinem/rust-cache@v2.7.3 + with: + shared-key: "libra-framework" + cache-all-crates: true + - name: storage + working-directory: ./tools/storage + # TODO: not testing the V6 file parsing + run: cargo test --no-fail-fast e2e_ diff --git a/.vscode/launch-orig-.json b/.vscode/launch-orig-.json deleted file mode 100644 index 224af64df..000000000 --- a/.vscode/launch-orig-.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "lldb", - "request": "launch", - "name": "Debug", - "program": "${workspaceFolder}/", - "args": [], - "cwd": "${workspaceFolder}" - } - ] -} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index a4f5e8201..f538b5ac5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,35 +5,6 @@ "version": "0.2.0", // target/debug/ol-genesis-tools --fork --output-path /opt/genesis_from_snapshot.blob --snapshot-path /Users/gsimsek/code/libra-main/ol/devnet/snapshot/state_ver_267.54ab` "configurations": [ - { - "name": "restore-bootstrap", - "type": "lldb", - "request": "launch", - "cargo": { - "args": [ - "build", - "-p", - "storage" - ], - }, - // cd ${ARCHIVE_PATH} && ${BIN_PATH}/${BIN_FILE} restore bootstrap-db --target-db-dir ${DB_PATH} --metadata-cache-dir ${REPO_PATH}/metacache --command-adapter-config ${REPO_PATH}/epoch-archive.yaml - - "args": [ - "restore", - "bootstrap-db", - "epoch-ending", - "--metadata-cache-dir", - "${workspaceFolder}/tools/storage/fixtures/epoch_ending.manifest", - "--command-adapter-config", - "${workspaceFolder}/tools/storage/fixtures", - "--target-db-dir", - "${workspaceFolder}/test_db", - ], - "env": { - "RUST_LOG": "warn" - }, - "cwd": "${workspaceFolder}" - }, { "name": "restore-epoch", "type": "lldb", @@ -42,24 +13,21 @@ "args": [ "build", "-p", - "storage" + "libra-storage" ], }, "args": [ + "db", "restore", "oneoff", "epoch-ending", "--epoch-ending-manifest", - "${workspaceFolder}/tools/storage/fixtures/epoch_ending_79-.166d/epoch_ending.manifest.unzip", + "${workspaceFolder}/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest", "--local-fs-dir", - "${workspaceFolder}/test_db", + "${workspaceFolder}/tools/storage/fixtures/v7", "--target-db-dir", "${workspaceFolder}/test_db", ], - "env": { - "RUST_LOG": "warn" - }, - "cwd": "${workspaceFolder}" }, { "name": "restore-state", @@ -69,27 +37,76 @@ "args": [ "build", "-p", - "storage" + "libra-storage" ], }, "args": [ + "db", "restore", "oneoff", "state-snapshot", "--state-manifest", - "${workspaceFolder}/tools/storage/fixtures/state.manifest", + "${workspaceFolder}/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest", "--restore-mode", "default", "--state-into-version", "1", "--local-fs-dir", + "${workspaceFolder}/tools/storage/fixtures/v7/", + "--target-db-dir", "${workspaceFolder}/test_db", + ], + "cwd": "${workspaceFolder}" + }, + { + "name": "restore-from-cli", + "type": "lldb", + "request": "launch", + "cargo": { + "args": [ + "build", + "-p", + "libra" + ], + }, + "args": [ + "ops", + "storage", + "db", + "restore", + "oneoff", + "epoch-ending", + "--epoch-ending-manifest", + "${workspaceFolder}/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest", + "--local-fs-dir", + "${workspaceFolder}/tools/storage/fixtures/v7", "--target-db-dir", "${workspaceFolder}/test_db", ], - "env": { - "RUST_LOG": "warn" + }, + { + "name": "restore-bootstrap", + "type": "lldb", + "request": "launch", + "cargo": { + "args": [ + "build", + "-p", + "storage" + ], }, + // cd ${ARCHIVE_PATH} && ${BIN_PATH}/${BIN_FILE} restore bootstrap-db --target-db-dir ${DB_PATH} --metadata-cache-dir ${REPO_PATH}/metacache --command-adapter-config ${REPO_PATH}/epoch-archive.yaml + "args": [ + "restore", + "bootstrap-db", + "epoch-ending", + "--metadata-cache-dir", + "${workspaceFolder}/tools/storage/fixtures/epoch_ending.manifest", + "--command-adapter-config", + "${workspaceFolder}/tools/storage/fixtures", + "--target-db-dir", + "${workspaceFolder}/test_db", + ], "cwd": "${workspaceFolder}" }, { diff --git a/Cargo.lock b/Cargo.lock index 9bc5e5dda..bf2f6edb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5655,6 +5655,8 @@ dependencies = [ "libra-framework", "libra-genesis-tools", "libra-query", + "libra-rescue", + "libra-storage", "libra-txs", "libra-types", "libra-wallet", @@ -5771,6 +5773,49 @@ dependencies = [ "tokio", ] +[[package]] +name = "libra-rescue" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bcs 0.1.4", + "clap 4.5.4", + "diem-api-types", + "diem-config", + "diem-crypto", + "diem-db", + "diem-executor", + "diem-forge", + "diem-gas", + "diem-genesis", + "diem-logger", + "diem-storage-interface", + "diem-temppath", + "diem-types", + "diem-vm", + "diem-vm-types", + "fs_extra", + "futures-util", + "hex", + "libra-config", + "libra-framework", + "libra-query", + "libra-smoke-tests", + "libra-txs", + "libra-types", + "libra-wallet", + "move-core-types", + "move-vm-runtime", + "move-vm-types", + "regex", + "serde 1.0.199", + "serde_json", + "serde_yaml 0.8.26", + "smoke-test", + "tokio", +] + [[package]] name = "libra-smoke-tests" version = "0.1.0" @@ -5792,6 +5837,24 @@ dependencies = [ "url", ] +[[package]] +name = "libra-storage" +version = "7.0.2" +dependencies = [ + "anyhow", + "bcs 0.1.4", + "clap 4.5.4", + "diem-backup-cli", + "diem-db-tool", + "diem-logger", + "diem-push-metrics", + "diem-temppath", + "diem-types", + "libra-types", + "serde_json", + "tokio", +] + [[package]] name = "libra-txs" version = "7.0.2" @@ -8314,49 +8377,6 @@ dependencies = [ "wasm-timer", ] -[[package]] -name = "rescue" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bcs 0.1.4", - "clap 4.5.4", - "diem-api-types", - "diem-config", - "diem-crypto", - "diem-db", - "diem-executor", - "diem-forge", - "diem-gas", - "diem-genesis", - "diem-logger", - "diem-storage-interface", - "diem-temppath", - "diem-types", - "diem-vm", - "diem-vm-types", - "fs_extra", - "futures-util", - "hex", - "libra-config", - "libra-framework", - "libra-query", - "libra-smoke-tests", - "libra-txs", - "libra-types", - "libra-wallet", - "move-core-types", - "move-vm-runtime", - "move-vm-types", - "regex", - "serde 1.0.199", - "serde_json", - "serde_yaml 0.8.26", - "smoke-test", - "tokio", -] - [[package]] name = "retain_mut" version = "0.1.9" @@ -9199,23 +9219,6 @@ dependencies = [ "atty", ] -[[package]] -name = "storage" -version = "7.0.2" -dependencies = [ - "anyhow", - "bcs 0.1.4", - "clap 4.5.4", - "diem-backup-cli", - "diem-db-tool", - "diem-logger", - "diem-push-metrics", - "diem-types", - "libra-types", - "serde_json", - "tokio", -] - [[package]] name = "str_stack" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 50b362c46..567849e2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,8 @@ libra-config = { path = "tools/config" } libra-framework = { path = "framework" } libra-genesis-tools = { path = "tools/genesis" } libra-query = { path = "tools/query" } +libra-rescue = { path = "tools/rescue" } +libra-storage = { path = "tools/storage" } libra-smoke-tests = { path = "smoke-tests" } libra-tower = { path = "tools/tower" } libra-types = { path = "types" } diff --git a/tools/cli/Cargo.toml b/tools/cli/Cargo.toml index c63e34e28..5846c703b 100644 --- a/tools/cli/Cargo.toml +++ b/tools/cli/Cargo.toml @@ -21,6 +21,8 @@ libra-config = { workspace = true } libra-framework = { workspace = true } libra-genesis-tools = { workspace = true } libra-query = { workspace = true } +libra-rescue = { workspace = true } +libra-storage = { workspace = true } libra-txs = { workspace = true } libra-types = { workspace = true } libra-wallet = { workspace = true } diff --git a/tools/cli/src/main.rs b/tools/cli/src/main.rs index f2af91cc6..fd8a8201d 100644 --- a/tools/cli/src/main.rs +++ b/tools/cli/src/main.rs @@ -1,10 +1,10 @@ mod move_cli; mod node_cli; +mod ops_cli; use anyhow::anyhow; use clap::{Parser, Subcommand}; use libra_config::config_cli::ConfigCli; -use libra_genesis_tools::cli::GenesisCli; use libra_query::query_cli::QueryCli; use libra_txs::txs_cli::TxsCli; use libra_wallet::wallet_cli::WalletCli; @@ -26,7 +26,8 @@ enum Sub { Query(QueryCli), Txs(TxsCli), Wallet(WalletCli), - Genesis(GenesisCli), + #[clap(subcommand)] + Ops(ops_cli::OpsTool), Version, } @@ -81,13 +82,11 @@ fn main() -> anyhow::Result<()> { } } - // Execute Genesis CLI subcommand - Some(Sub::Genesis(genesis_cli)) => { - if let Err(e) = genesis_cli.execute().await { - eprintln!("Failed to execute genesis tool, message: {}", &e); + Some(Sub::Ops(tool)) => { + if let Err(e) = tool.run().await { + eprintln!("Failed to execute ops tool, message: {}", &e); } } - // Display version information Some(Sub::Version) => { println!("LIBRA VERSION {}", env!("CARGO_PKG_VERSION")); diff --git a/tools/cli/src/ops_cli.rs b/tools/cli/src/ops_cli.rs new file mode 100644 index 000000000..32af6de02 --- /dev/null +++ b/tools/cli/src/ops_cli.rs @@ -0,0 +1,39 @@ +use clap::Subcommand; +use libra_genesis_tools::cli::GenesisCli; +use libra_rescue::rescue_cli::RescueCli; +use libra_storage::storage_cli::StorageCli; + +#[derive(Subcommand)] +/// Node and DB operations tools +pub enum OpsTool { + Genesis(GenesisCli), + // #[clap(subcommand)] + Storage(StorageCli), + Rescue(RescueCli), +} + +impl OpsTool { + // TODO: note that run here is consuming the self, and not borrowing. + // this is because downstream StorageCli::Db::DbTool, cannot be copied. + pub async fn run(self) -> anyhow::Result<()> { + match self { + // Execute Genesis CLI subcommand + Self::Genesis(genesis_cli) => { + if let Err(e) = genesis_cli.run().await { + eprintln!("Failed to execute genesis tool, message: {}", &e); + } + } + OpsTool::Storage(storage_cli) => { + if let Err(e) = storage_cli.run().await { + eprintln!("Failed to execute genesis tool, message: {}", &e); + } + } + OpsTool::Rescue(rescue_cli) => { + if let Err(e) = rescue_cli.run() { + eprintln!("Failed to execute genesis tool, message: {}", &e); + } + } + }; + Ok(()) + } +} diff --git a/tools/genesis/src/cli.rs b/tools/genesis/src/cli.rs index 1a5fb1212..00b9ede29 100644 --- a/tools/genesis/src/cli.rs +++ b/tools/genesis/src/cli.rs @@ -22,7 +22,7 @@ pub struct GenesisCli { } impl GenesisCli { - pub async fn execute(&self) -> anyhow::Result<()> { + pub async fn run(&self) -> anyhow::Result<()> { let data_path = self.home_dir.clone().unwrap_or_else(global_config_dir); let chain_name = self.chain.unwrap_or(NamedChain::TESTNET); // chain_id = 2 diff --git a/tools/genesis/src/main.rs b/tools/genesis/src/main.rs index cde95ebda..d1e46a139 100644 --- a/tools/genesis/src/main.rs +++ b/tools/genesis/src/main.rs @@ -3,5 +3,5 @@ use clap::Parser; #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = libra_genesis_tools::cli::GenesisCli::parse(); - cli.execute().await + cli.run().await } diff --git a/tools/rescue/Cargo.toml b/tools/rescue/Cargo.toml index 8c8c01ed2..4768e22ac 100644 --- a/tools/rescue/Cargo.toml +++ b/tools/rescue/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "rescue" +name = "libra-rescue" version = "0.1.0" # Workspace inherited keys diff --git a/tools/rescue/src/lib.rs b/tools/rescue/src/lib.rs index bbabab808..1659695d5 100644 --- a/tools/rescue/src/lib.rs +++ b/tools/rescue/src/lib.rs @@ -1,4 +1,5 @@ pub mod diem_db_bootstrapper; +pub mod rescue_cli; pub mod rescue_tx; pub mod session_tools; pub mod twin; diff --git a/tools/rescue/src/main.rs b/tools/rescue/src/main.rs index d953a5ce1..2f64e381d 100644 --- a/tools/rescue/src/main.rs +++ b/tools/rescue/src/main.rs @@ -1,51 +1,7 @@ -//! CLI tool for rescue operations in Diem, providing commands for transaction rescue, -//! database bootstrapping, and debugging twin states. +//! entry point -use clap::{Parser, Subcommand}; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts, twin::TwinOpts}; -use std::time::Duration; - -#[derive(Parser)] -#[clap(author, version, about, long_about = None)] -#[clap(arg_required_else_help(true))] -struct RescueCli { - #[clap(subcommand)] - command: Option, -} - -#[derive(Subcommand)] -enum Sub { - RescueTx(RescueTxOpts), - Bootstrap(BootstrapOpts), - Debug(TwinOpts), -} +use clap::Parser; fn main() -> anyhow::Result<()> { - let cli = RescueCli::parse(); - match cli.command { - Some(Sub::RescueTx(mission)) => { - let blob_path = mission.run()?; - - let b = BootstrapOpts { - db_dir: mission.data_path, - genesis_txn_file: blob_path, - waypoint_to_verify: None, - commit: false, - info: false, - }; - let _ = b.run()?; - } - Some(Sub::Bootstrap(bootstrap)) => { - bootstrap.run()?; - } - Some(Sub::Debug(twin)) => { - twin.run()?; - } - _ => {} // prints help - } - println!("done"); - // hack. let the DB close before exiting - // TODO: fix in Diem or place in thread - std::thread::sleep(Duration::from_millis(10)); - Ok(()) + libra_rescue::rescue_cli::RescueCli::parse().run() } diff --git a/tools/rescue/src/rescue_cli.rs b/tools/rescue/src/rescue_cli.rs new file mode 100644 index 000000000..47e7e7208 --- /dev/null +++ b/tools/rescue/src/rescue_cli.rs @@ -0,0 +1,53 @@ +//! CLI tool for rescue operations in Diem, providing commands for transaction rescue, +//! database bootstrapping, and debugging twin states. +use crate::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts, twin::TwinOpts}; + +use clap::{Parser, Subcommand}; +use std::time::Duration; + +#[derive(Parser)] +#[clap(author, version, about, long_about = None)] +#[clap(arg_required_else_help(true))] +/// Apply writeset transactions to DB at rest +pub struct RescueCli { + #[clap(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Sub { + RescueTx(RescueTxOpts), + Bootstrap(BootstrapOpts), + Debug(TwinOpts), +} + +impl RescueCli { + pub fn run(&self) -> anyhow::Result<()> { + match &self.command { + Some(Sub::RescueTx(mission)) => { + let blob_path = mission.run()?; + + let b = BootstrapOpts { + db_dir: mission.data_path.clone(), + genesis_txn_file: blob_path, + waypoint_to_verify: None, + commit: false, + info: false, + }; + let _ = b.run()?; + } + Some(Sub::Bootstrap(bootstrap)) => { + bootstrap.run()?; + } + Some(Sub::Debug(twin)) => { + twin.run()?; + } + _ => {} // prints help + } + println!("done"); + // hack. let the DB close before exiting + // TODO: fix in Diem or place in thread + std::thread::sleep(Duration::from_millis(10)); + Ok(()) + } +} diff --git a/tools/rescue/tests/rescue_cli_creates_blob.rs b/tools/rescue/tests/rescue_cli_creates_blob.rs index 81b1d8bb9..c8fd7c94a 100644 --- a/tools/rescue/tests/rescue_cli_creates_blob.rs +++ b/tools/rescue/tests/rescue_cli_creates_blob.rs @@ -1,7 +1,7 @@ mod support; +use libra_rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use libra_smoke_tests::libra_smoke::LibraSmoke; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; #[tokio::test] async fn test_valid_genesis() -> anyhow::Result<()> { diff --git a/tools/rescue/tests/rescue_cli_framework_upgrade_valid.rs b/tools/rescue/tests/rescue_cli_framework_upgrade_valid.rs index 4a73bd30f..c97bd7517 100644 --- a/tools/rescue/tests/rescue_cli_framework_upgrade_valid.rs +++ b/tools/rescue/tests/rescue_cli_framework_upgrade_valid.rs @@ -1,7 +1,7 @@ mod support; +use libra_rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use libra_smoke_tests::libra_smoke::LibraSmoke; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; #[tokio::test] async fn test_framework_upgrade_writeset() -> anyhow::Result<()> { diff --git a/tools/rescue/tests/rescue_e2e_can_create_genesis.rs b/tools/rescue/tests/rescue_e2e_can_create_genesis.rs index 08d89b039..515de297e 100644 --- a/tools/rescue/tests/rescue_e2e_can_create_genesis.rs +++ b/tools/rescue/tests/rescue_e2e_can_create_genesis.rs @@ -4,8 +4,8 @@ use anyhow::Context; use diem_forge::SwarmExt; use diem_temppath::TempPath; use diem_types::transaction::Transaction; +use libra_rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use libra_smoke_tests::libra_smoke::LibraSmoke; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use smoke_test::test_utils::MAX_CATCH_UP_WAIT_SECS; use std::{fs, time::Duration}; diff --git a/tools/rescue/tests/rescue_e2e_can_restart.rs b/tools/rescue/tests/rescue_e2e_can_restart.rs index 9efe9a80a..371246146 100644 --- a/tools/rescue/tests/rescue_e2e_can_restart.rs +++ b/tools/rescue/tests/rescue_e2e_can_restart.rs @@ -7,8 +7,8 @@ use diem_forge::{NodeExt, SwarmExt}; use diem_temppath::TempPath; use diem_types::transaction::Transaction; use futures_util::future::try_join_all; +use libra_rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use libra_smoke_tests::{helpers::get_libra_balance, libra_smoke::LibraSmoke}; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use smoke_test::test_utils::{swarm_utils::insert_waypoint, MAX_CATCH_UP_WAIT_SECS}; use std::{fs, time::Duration}; diff --git a/tools/rescue/tests/rescue_e2e_full.rs b/tools/rescue/tests/rescue_e2e_full.rs index 138e15af2..86b5311ad 100644 --- a/tools/rescue/tests/rescue_e2e_full.rs +++ b/tools/rescue/tests/rescue_e2e_full.rs @@ -7,8 +7,8 @@ use diem_forge::{NodeExt, SwarmExt}; use diem_temppath::TempPath; use diem_types::transaction::Transaction; use futures_util::future::try_join_all; +use libra_rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use libra_smoke_tests::{helpers::get_libra_balance, libra_smoke::LibraSmoke}; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use smoke_test::test_utils::{swarm_utils::insert_waypoint, MAX_CATCH_UP_WAIT_SECS}; use std::{fs, time::Duration}; diff --git a/tools/rescue/tests/twin.rs b/tools/rescue/tests/twin.rs index 7679a7c91..46f6a63d8 100644 --- a/tools/rescue/tests/twin.rs +++ b/tools/rescue/tests/twin.rs @@ -4,8 +4,8 @@ use crate::support::{deadline_secs, update_node_config_restart}; use diem_config::config::InitialSafetyRulesConfig; use diem_forge::SwarmExt; use diem_types::transaction::Transaction; +use libra_rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use libra_smoke_tests::libra_smoke::LibraSmoke; -use rescue::{diem_db_bootstrapper::BootstrapOpts, rescue_tx::RescueTxOpts}; use smoke_test::test_utils::swarm_utils::insert_waypoint; #[tokio::test] diff --git a/tools/rescue/tests/upgrade_multi_action.rs b/tools/rescue/tests/upgrade_multi_action.rs index e5fd69e41..41fbe557a 100644 --- a/tools/rescue/tests/upgrade_multi_action.rs +++ b/tools/rescue/tests/upgrade_multi_action.rs @@ -2,12 +2,12 @@ mod support; use diem_crypto::ValidCryptoMaterialStringExt; use libra_query::query_view; +use libra_rescue::twin::{Twin, TwinSetup}; use libra_txs::{ txs_cli::{TxsCli, TxsSub, TxsSub::Transfer}, txs_cli_community::{CageTx, ClaimTx, CommunityTxs, InitTx}, }; use libra_types::core_types::app_cfg::TxCost; -use rescue::twin::{Twin, TwinSetup}; use std::{env, path::PathBuf}; // TODO: Remove after offer structure is migrated diff --git a/tools/storage/Cargo.toml b/tools/storage/Cargo.toml index ef07d382d..8526c4c88 100644 --- a/tools/storage/Cargo.toml +++ b/tools/storage/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "storage" +name = "libra-storage" version = { workspace = true } # Workspace inherited keys @@ -24,3 +24,6 @@ diem-types = { workspace = true } libra-types = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } + +[dev-dependencies] +diem-temppath = { workspace = true } diff --git a/tools/storage/example.makefile b/tools/storage/example.makefile new file mode 100644 index 000000000..9441f8e93 --- /dev/null +++ b/tools/storage/example.makefile @@ -0,0 +1,28 @@ +# Examples of how to use the storage tool. +# uses example fixture files located here in ./tools/storage/fixtures +# try this: `make -f example.makefile restore` + +# you need to use the absolute path of the snapshot files. +ABS_PATH?=`cd "$1"; pwd` +BACKUP_DIR= ${ABS_PATH}/fixtures/v7/ +OUTPUT_DB_PATH = ${ABS_PATH}/test_db + +# For the epoch restore +EPOCH_MANIFEST = ${BACKUP_DIR}epoch_ending_116-.be9b/epoch_ending.manifest + +# For the snapshot restore +STATE_MANIFEST = ${BACKUP_DIR}state_epoch_116_ver_38180075.05af/state.manifest + +restore: restore-epoch restore-snapshot + +restore-epoch: + libra ops storage db restore oneoff epoch-ending --epoch-ending-manifest ${EPOCH_MANIFEST} --target-db-dir ${OUTPUT_DB_PATH} --local-fs-dir ${BACKUP_DIR} + +restore-snapshot: + libra ops storage db restore oneoff state-snapshot --state-manifest ${STATE_MANIFEST} --target-db-dir ${OUTPUT_DB_PATH} --local-fs-dir ${BACKUP_DIR} --restore-mode default --state-into-version 1 + +echo: + @echo ${ABS_PATH} + @echo ${BACKUP_DIR} + @echo ${EPOCH_MANIFEST} + @echo ${STATE_MANIFEST} diff --git a/tools/storage/fixtures/epoch_ending_79-.166d/79-.chunk b/tools/storage/fixtures/v6/epoch_ending_79-.166d/79-.chunk similarity index 100% rename from tools/storage/fixtures/epoch_ending_79-.166d/79-.chunk rename to tools/storage/fixtures/v6/epoch_ending_79-.166d/79-.chunk diff --git a/tools/storage/fixtures/epoch_ending_79-.166d/epoch_ending.manifest b/tools/storage/fixtures/v6/epoch_ending_79-.166d/epoch_ending.manifest similarity index 100% rename from tools/storage/fixtures/epoch_ending_79-.166d/epoch_ending.manifest rename to tools/storage/fixtures/v6/epoch_ending_79-.166d/epoch_ending.manifest diff --git a/tools/storage/fixtures/epoch_ending_79-.166d/epoch_ending.manifest.unzip b/tools/storage/fixtures/v6/epoch_ending_79-.166d/epoch_ending.manifest.unzip similarity index 100% rename from tools/storage/fixtures/epoch_ending_79-.166d/epoch_ending.manifest.unzip rename to tools/storage/fixtures/v6/epoch_ending_79-.166d/epoch_ending.manifest.unzip diff --git a/tools/storage/fixtures/state_epoch_79_ver_33217173.795d/0-.chunk b/tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/0-.chunk similarity index 100% rename from tools/storage/fixtures/state_epoch_79_ver_33217173.795d/0-.chunk rename to tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/0-.chunk diff --git a/tools/storage/fixtures/state_epoch_79_ver_33217173.795d/0-141533.proof b/tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/0-141533.proof similarity index 100% rename from tools/storage/fixtures/state_epoch_79_ver_33217173.795d/0-141533.proof rename to tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/0-141533.proof diff --git a/tools/storage/fixtures/state_epoch_79_ver_33217173.795d/state.manifest b/tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/state.manifest similarity index 100% rename from tools/storage/fixtures/state_epoch_79_ver_33217173.795d/state.manifest rename to tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/state.manifest diff --git a/tools/storage/fixtures/state_epoch_79_ver_33217173.795d/state.proof b/tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/state.proof similarity index 100% rename from tools/storage/fixtures/state_epoch_79_ver_33217173.795d/state.proof rename to tools/storage/fixtures/v6/state_epoch_79_ver_33217173.795d/state.proof diff --git a/tools/storage/fixtures/v7/epoch_ending_116-.be9b/116-.chunk b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/116-.chunk new file mode 100644 index 000000000..276808fa0 Binary files /dev/null and b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/116-.chunk differ diff --git a/tools/storage/fixtures/v7/epoch_ending_116-.be9b/116-.chunk.gz b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/116-.chunk.gz new file mode 100644 index 000000000..8243313fd Binary files /dev/null and b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/116-.chunk.gz differ diff --git a/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest new file mode 100644 index 000000000..32bea6f90 --- /dev/null +++ b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest @@ -0,0 +1 @@ +{"first_epoch":116,"last_epoch":116,"waypoints":["38180075:98d7eb16747c1441350c4a1a6aa8b2d504c30ce2520807f2adc87d5a06d4f70f"],"chunks":[{"first_epoch":116,"last_epoch":116,"ledger_infos":"epoch_ending_116-.be9b/116-.chunk"}]} \ No newline at end of file diff --git a/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest.gz b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest.gz new file mode 100644 index 000000000..e3b989eb1 Binary files /dev/null and b/tools/storage/fixtures/v7/epoch_ending_116-.be9b/epoch_ending.manifest.gz differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-.chunk b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-.chunk new file mode 100644 index 000000000..629d3ccb4 Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-.chunk differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-.chunk.gz b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-.chunk.gz new file mode 100644 index 000000000..53b69499f Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-.chunk.gz differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-130241.proof b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-130241.proof new file mode 100644 index 000000000..f76dd238a Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-130241.proof differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-130241.proof.gz b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-130241.proof.gz new file mode 100644 index 000000000..cdbc4ca63 Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/0-130241.proof.gz differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest new file mode 100644 index 000000000..76cb5f271 --- /dev/null +++ b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest @@ -0,0 +1 @@ +{"version":38180075,"epoch":116,"root_hash":"b4c9918ddb62469cc3e7e7b2a01b43aeac803470913b3a89afdcc44078df8d8a","chunks":[{"first_idx":0,"last_idx":130241,"first_key":"0000acadcb83f9bdcffcbb63d132be250f8018a731137db7e3aab5af2e3bc8bd","last_key":"fffff7e8a7aff9e730227f0decbef95c00ccdd60744654b8c4799a80bec23e5a","blobs":"state_epoch_116_ver_38180075.05af/0-.chunk","proof":"state_epoch_116_ver_38180075.05af/0-130241.proof"}],"proof":"state_epoch_116_ver_38180075.05af/state.proof"} diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest.gz b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest.gz new file mode 100644 index 000000000..29027f8a1 Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.manifest.gz differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.proof b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.proof new file mode 100644 index 000000000..76e1274b0 Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.proof differ diff --git a/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.proof.gz b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.proof.gz new file mode 100644 index 000000000..35c6e0bb5 Binary files /dev/null and b/tools/storage/fixtures/v7/state_epoch_116_ver_38180075.05af/state.proof.gz differ diff --git a/tools/storage/src/lib.rs b/tools/storage/src/lib.rs index d08e90809..818ec46e1 100644 --- a/tools/storage/src/lib.rs +++ b/tools/storage/src/lib.rs @@ -1 +1,2 @@ pub mod read_snapshot; +pub mod storage_cli; diff --git a/tools/storage/src/main.rs b/tools/storage/src/main.rs index dfc8a6299..be58926dc 100644 --- a/tools/storage/src/main.rs +++ b/tools/storage/src/main.rs @@ -1,43 +1,7 @@ -use anyhow::Result; +//! entry point use clap::Parser; -use diem_db_tool::DBTool; -use diem_logger::{Level, Logger}; -use diem_push_metrics::MetricsPusher; -use std::path::PathBuf; -use storage::read_snapshot::manifest_to_json; - -#[derive(Parser)] -#[clap(name = "libra storage", author, version)] -#[allow(clippy::large_enum_variant)] -enum StorageCli { - #[clap(subcommand)] - Db(DBTool), - ExportSnapshot { - #[clap(short, long)] - manifest_path: PathBuf, - #[clap(short, long)] - out_path: Option, - }, -} #[tokio::main] -async fn main() -> Result<()> { - Logger::new().level(Level::Info).init(); - let _mp = MetricsPusher::start(vec![]); - - match StorageCli::parse() { - StorageCli::Db(tool) => { - tool.run().await?; - } - StorageCli::ExportSnapshot { - manifest_path, - out_path, - } => { - manifest_to_json(manifest_path, out_path).await; - } - } - - DBTool::parse().run().await?; - - Ok(()) +async fn main() -> anyhow::Result<()> { + libra_storage::storage_cli::StorageCli::parse().run().await } diff --git a/tools/storage/src/storage_cli.rs b/tools/storage/src/storage_cli.rs new file mode 100644 index 000000000..be68599ca --- /dev/null +++ b/tools/storage/src/storage_cli.rs @@ -0,0 +1,56 @@ +use anyhow::Result; +use clap::{Parser, Subcommand}; +use diem_db_tool::DBTool; +use diem_logger::{Level, Logger}; +use diem_push_metrics::MetricsPusher; +use std::path::PathBuf; + +use crate::read_snapshot; + +#[derive(Parser)] +#[clap(author, version, about, long_about = None)] +#[clap(arg_required_else_help(true))] +/// DB tools e.g.: backup, restore, export to json +pub struct StorageCli { + #[clap(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +#[allow(clippy::large_enum_variant)] +pub enum Sub { + #[clap(subcommand)] + /// DB tools for backup, restore, verify, etc. + Db(DBTool), + /// Read a snapshot, parse and export to JSON + ExportSnapshot { + #[clap(short, long)] + manifest_path: PathBuf, + #[clap(short, long)] + out_path: Option, + }, +} + +impl StorageCli { + // Note: using owned self since DBTool::run uses an owned self. + pub async fn run(self) -> Result<()> { + Logger::new().level(Level::Info).init(); + let _mp = MetricsPusher::start(vec![]); + + match self.command { + Some(Sub::Db(tool)) => { + tool.run().await?; + } + Some(Sub::ExportSnapshot { + manifest_path, + out_path, + }) => { + read_snapshot::manifest_to_json(manifest_path.to_owned(), out_path.to_owned()) + .await; + } + _ => {} // prints help + } + + Ok(()) + } +} diff --git a/tools/storage/tests/test_e2e.rs b/tools/storage/tests/test_e2e.rs new file mode 100644 index 000000000..7e70f4823 --- /dev/null +++ b/tools/storage/tests/test_e2e.rs @@ -0,0 +1,41 @@ +use clap::Parser; +use std::path::PathBuf; + +use diem_temppath::TempPath; +use libra_storage::storage_cli::StorageCli; + +#[tokio::test] +async fn e2e_epoch() -> anyhow::Result<()> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let fixtures = dir.join("fixtures/v7"); + let db_temp = TempPath::new(); + db_temp.create_as_dir()?; + + let cmd = format!("storage db restore oneoff epoch-ending --epoch-ending-manifest {manifest} --target-db-dir {db} --local-fs-dir {fs}", + manifest = fixtures.join("epoch_ending_116-.be9b/epoch_ending.manifest").display(), + db = db_temp.path().display(), + fs = fixtures.display() + ); + + let to_vec: Vec<_> = cmd.split_whitespace().collect(); + let s = StorageCli::try_parse_from(to_vec)?; + s.run().await +} + +#[tokio::test] +async fn e2e_snapshot() -> anyhow::Result<()> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let fixtures = dir.join("fixtures/v7"); + let db_temp = TempPath::new(); + db_temp.create_as_dir()?; + + let cmd = format!("storage db restore oneoff state-snapshot --state-manifest {manifest} --target-db-dir {db} --local-fs-dir {fs} --restore-mode default --state-into-version 1", + manifest = fixtures.join("state_epoch_116_ver_38180075.05af/state.manifest").display(), + db = db_temp.path().display(), + fs = fixtures.display() + ); + + let to_vec: Vec<_> = cmd.split_whitespace().collect(); + let s = StorageCli::try_parse_from(to_vec)?; + s.run().await +}