implement first test with testcontainers

Cargo.toml

@@ -279,6 +279,7 @@ syn = { version = "1.0.92", features = ["derive", "extra-traits"] }
 sysinfo = "0.28.4"
 tempfile = "3.3.0"
 termcolor = "1.1.2"
+testcontainers = { version = "0.15" }
 textwrap = "0.15.0"
 thiserror = "1.0.37"
 tiny-bip39 = "0.8.2"

compatibility/src/version_five/state_snapshot_v5.rs

@@ -107,10 +107,12 @@ pub async fn v5_accounts_from_snapshot_backup(
 }
 
 /// one step extraction of account state blobs from a manifest path
-pub async fn v5_accounts_from_manifest_path(manifest_file: &Path) -> Result<Vec<AccountStateBlob>>{
-  let archive_path = manifest_file.parent().context("could not get archive path from manifest file")?;
-  let manifest = v5_read_from_snapshot_manifest(manifest_file)?;
-  v5_accounts_from_snapshot_backup(manifest, archive_path).await
+pub async fn v5_accounts_from_manifest_path(manifest_file: &Path) -> Result<Vec<AccountStateBlob>> {
+    let archive_path = manifest_file
+        .parent()
+        .context("could not get archive path from manifest file")?;
+    let manifest = v5_read_from_snapshot_manifest(manifest_file)?;
+    v5_accounts_from_snapshot_backup(manifest, archive_path).await
 }
 
 #[test]
@@ -152,7 +154,6 @@ fn decode_record_from_string() {
 
     let address = ar.address();
     assert!(address.len() > 0);
-
 }
 
 #[test]

warehouse/Cargo.toml

@@ -16,5 +16,9 @@ glob = { workspace = true }
 libra-backwards-compatibility = { workspace = true }
 libra-storage = { workspace =  true }
 libra-types = { workspace =  true }
+once_cell = { workspace =  true }
 sqlx = { workspace = true }
 tokio = { workspace = true }
+
+[dev-dependencies]
+testcontainers = { workspace = true }

warehouse/src/extract.rs

@@ -7,7 +7,7 @@ use libra_types::exports::AccountAddress;
 use crate::table_structs::{WarehouseAccount, WarehouseRecord, WarehouseTime};
 
 pub async fn extract_v5_snapshot(v5_manifest_path: &Path) -> Result<Vec<WarehouseRecord>> {
-    let account_blobs = v5_accounts_from_manifest_path(&v5_manifest_path).await?;
+    let account_blobs = v5_accounts_from_manifest_path(v5_manifest_path).await?;
     dbg!(&account_blobs.len());
     let mut warehouse_state = vec![];
     for el in account_blobs.iter() {

warehouse/src/lib.rs

@@ -1,8 +1,8 @@
-pub mod migrate;
-pub mod scan;
-pub mod warehouse_cli;
 pub mod extract;
-pub mod table_structs;
 pub mod load_account;
 pub mod load_coin;
+pub mod migrate;
 pub mod query_balance;
+pub mod scan;
+pub mod table_structs;
+pub mod warehouse_cli;

warehouse/src/load_account.rs

@@ -2,7 +2,7 @@ use crate::table_structs::{WarehouseAccount, WarehouseRecord};
 use anyhow::Result;
 use sqlx::{sqlite::SqliteQueryResult, QueryBuilder, Sqlite, SqlitePool};
 
-pub async fn load_account_state(pool: &SqlitePool, accounts: &Vec<WarehouseRecord>) -> Result<i64> {
+pub async fn load_account_state(pool: &SqlitePool, accounts: &[WarehouseRecord]) -> Result<i64> {
     let mut rows = 0;
     // insert missing accounts
     for ws in accounts.iter() {
@@ -48,14 +48,18 @@ pub async fn batch_insert_account(
 }
 
 // TODO: return specific commit errors for this batch
-pub async fn impl_batch_insert(pool: &SqlitePool, batch_accounts: &[WarehouseRecord]) -> Result<SqliteQueryResult> {
+pub async fn impl_batch_insert(
+    pool: &SqlitePool,
+    batch_accounts: &[WarehouseRecord],
+) -> Result<SqliteQueryResult> {
     let mut query_builder: QueryBuilder<Sqlite> = QueryBuilder::new(
         // Note the trailing space; most calls to `QueryBuilder` don't automatically insert
         "INSERT INTO users (account_address, is_legacy) ",
     );
 
     query_builder.push_values(batch_accounts, |mut b, acc| {
-        b.push_bind(acc.account.address.to_hex_literal()).push_bind(true);
+        b.push_bind(acc.account.address.to_hex_literal())
+            .push_bind(true);
     });
 
     // makes sure the txs don't fail on repeated attempts to add users

warehouse/src/query_balance.rs

@@ -21,8 +21,9 @@ pub async fn query_last_balance(
         "#
     );
 
-    let row  = sqlx::query_as::<_, WarehouseBalance>(&query_template)
-    .fetch_one(pool).await?;
+    let row = sqlx::query_as::<_, WarehouseBalance>(&query_template)
+        .fetch_one(pool)
+        .await?;
 
     Ok(row)
 }

warehouse/src/table_structs.rs

@@ -3,46 +3,44 @@ use libra_types::exports::AccountAddress;
 #[derive(Debug, Clone)]
 /// The basic information for an account
 pub struct WarehouseRecord {
-  pub account: WarehouseAccount,
-  pub time: WarehouseTime,
-  pub balance: Option<WarehouseBalance>,
+    pub account: WarehouseAccount,
+    pub time: WarehouseTime,
+    pub balance: Option<WarehouseBalance>,
 }
 
 impl WarehouseRecord {
-  pub fn new(address: AccountAddress) -> Self {
-    Self {
-      account: WarehouseAccount {
-        address,
-      },
-      time: WarehouseTime::default(),
-      balance: Some(WarehouseBalance::default()),
+    pub fn new(address: AccountAddress) -> Self {
+        Self {
+            account: WarehouseAccount { address },
+            time: WarehouseTime::default(),
+            balance: Some(WarehouseBalance::default()),
+        }
+    }
+    pub fn set_time(&mut self, timestamp: u64, version: u64, epoch: u64) {
+        self.time.timestamp = timestamp;
+        self.time.version = version;
+        self.time.epoch = epoch;
     }
-  }
-  pub fn set_time(&mut self, timestamp: u64, version: u64, epoch: u64) {
-    self.time.timestamp = timestamp;
-    self.time.version = version;
-    self.time.epoch = epoch;
-  }
 }
 
 // holds timestamp, chain height, and epoch
 #[derive(Debug, Clone, Default)]
 pub struct WarehouseTime {
-  pub timestamp: u64,
-  pub version: u64,
-  pub epoch: u64,
+    pub timestamp: u64,
+    pub version: u64,
+    pub epoch: u64,
 }
 #[derive(Debug, Clone)]
 pub struct WarehouseAccount {
-  pub address: AccountAddress,
+    pub address: AccountAddress,
 }
 
 #[derive(Debug, Default, Clone, sqlx::FromRow)]
 pub struct WarehouseBalance {
-  // balances in v6+ terms
-  #[sqlx(try_from = "i64")]
-  pub balance: u64,
-  // the balance pre v6 recast
-  #[sqlx(default)]
-  pub legacy_balance: Option<u64>,
+    // balances in v6+ terms
+    #[sqlx(try_from = "i64")]
+    pub balance: u64,
+    // the balance pre v6 recast
+    #[sqlx(default)]
+    pub legacy_balance: Option<u64>,
 }

warehouse/tests/lib.rs

@@ -0,0 +1 @@
+pub(crate) mod support;

warehouse/tests/mock_migrations/1_mock_init.sql

@@ -1,5 +1,5 @@
 CREATE TABLE
   foo (
-    contact_id INTEGER PRIMARY KEY,
-    first_name TEXT NOT NULL
+    contact_id int,
+    first_name varchar(80) NOT NULL
   );

warehouse/tests/sqlx_meta.rs

@@ -1,8 +1,7 @@
-// in cargo.toml...
-// sqlx = { version = "0.8", features = [ "runtime-tokio", "tls-native-tls", "sqlite", "migrate"] }
-
+mod support;
 use sqlx::{Pool, Row, Sqlite};
 
+// NOTE: for reference, this is the sqlx test framework runtime, which can setup sqlite dbs. Left here for reference
 #[sqlx::test]
 async fn sql_insert_test(pool: Pool<Sqlite>) -> anyhow::Result<()> {
     let mut conn = pool.acquire().await?;
@@ -54,31 +53,62 @@ async fn sql_insert_test(pool: Pool<Sqlite>) -> anyhow::Result<()> {
     Ok(())
 }
 
-#[sqlx::test]
-async fn test_migrate_from_file(pool: Pool<Sqlite>) -> anyhow::Result<()> {
+// #[sqlx::test]
+// async fn test_migrate_from_file(pool: Pool<Sqlite>) -> anyhow::Result<()> {
+//     // The directory must be relative to the project root (the directory containing Cargo.toml), unlike include_str!() which uses compiler internals to get the path of the file where it was invoked.
+//     sqlx::migrate!("tests/mock_migrations").run(&pool).await?;
+
+//     let mut conn = pool.acquire().await?;
+
+//     let id = sqlx::query(
+//         r#"
+//           INSERT INTO foo (contact_id, first_name)
+//           VALUES
+//             (1, "hello");
+//         "#,
+//     )
+//     .execute(&mut *conn)
+//     .await?
+//     .last_insert_rowid();
+
+//     assert!(id == 1);
+
+//     let a = sqlx::query("SELECT * FROM foo")
+//         .fetch_all(&mut *conn)
+//         .await?;
+
+//     let q = a.first().unwrap().get_unchecked::<u64, _>(0);
+//     assert!(q == 1);
+
+//     Ok(())
+// }
+
+#[tokio::test]
+
+async fn test_migrate_from_file_pg() -> anyhow::Result<()> {
+    let (pool, _c) = support::pg_testcontainer::get_test_pool().await?;
     // The directory must be relative to the project root (the directory containing Cargo.toml), unlike include_str!() which uses compiler internals to get the path of the file where it was invoked.
     sqlx::migrate!("tests/mock_migrations").run(&pool).await?;
 
     let mut conn = pool.acquire().await?;
 
-    let id = sqlx::query(
+    let rows = sqlx::query(
         r#"
           INSERT INTO foo (contact_id, first_name)
-          VALUES
-            (1, "hello");
+          VALUES (1, 'hello');
         "#,
     )
     .execute(&mut *conn)
     .await?
-    .last_insert_rowid();
+    .rows_affected();
 
-    assert!(id == 1);
+    assert!(rows == 1);
 
     let a = sqlx::query("SELECT * FROM foo")
         .fetch_all(&mut *conn)
         .await?;
 
-    let q = a.first().unwrap().get_unchecked::<u64, _>(0);
+    let q = a.first().unwrap().get_unchecked::<i64, _>(0);
     assert!(q == 1);
 
     Ok(())

warehouse/tests/support/mod.rs

@@ -0,0 +1 @@
+pub mod pg_testcontainer;

warehouse/tests/support/pg_testcontainer.rs

@@ -0,0 +1,60 @@
+use once_cell::sync::Lazy;
+use sqlx::PgPool;
+use std::{thread, time::Duration};
+use testcontainers::{clients::Cli, core::WaitFor, Container, GenericImage, RunnableImage};
+
+// need to wrap the docker cli in a once_cell so that the borrow does not cause issues when container is passed along
+// copy pasta from: https://www.reddit.com/r/rust/comments/1294pfy/help_testcontainers_client_does_not_live_long/?rdt=54538
+static CLI: Lazy<Cli> = Lazy::new(Cli::default);
+
+// Note: testcontainers drops the container once it's out of scope. So for each
+// test we should pass it along, even if we don't reference it.
+// Otherwise, the docker contain will stop before you run the test
+pub async fn get_test_pool<'a>() -> anyhow::Result<(PgPool, Container<'a, GenericImage>)> {
+    let container = start_container();
+    // prepare connection string
+    let connection_string = &format!(
+        "postgres://postgres:[email protected]:{}/postgres",
+        container.get_host_port_ipv4(5432)
+    );
+
+    let sqlx_pool = PgPool::connect(connection_string).await?;
+    println!("database container started at: {}\n", &connection_string);
+
+    Ok((sqlx_pool, container))
+}
+
+pub fn start_container<'a>() -> Container<'a, GenericImage> {
+    let container = GenericImage::new("postgres", "17.0-alpine3.20")
+        .with_env_var("POSTGRES_PASSWORD", "testing")
+        .with_env_var("POSTGRES_HOST_AUTH_METHOD".to_owned(), "trust".to_owned())
+        .with_env_var("POSTGRES_DB".to_owned(), "postgres".to_owned())
+        .with_env_var("POSTGRES_USER".to_owned(), "postgres".to_owned())
+        .with_env_var("POSTGRES_PASSWORD".to_owned(), "postgres".to_owned())
+        .with_wait_for(WaitFor::message_on_stdout(
+            "database system is ready to accept connections",
+        ));
+
+    let image = RunnableImage::from(container);
+    // need to wrap the docker cli in a once_cell so that the borrow does not cause issues when container is passed along
+    let container = CLI.run(image);
+    container.start();
+    // TODO: not sure why we need a bit of a wait since we have the WaitFor above
+    // will otherwise get: "unexpected response from SSLRequest: 0x00 (sqlx_postgres::connection::tls:97)"
+    thread::sleep(Duration::from_millis(500));
+
+    container
+}
+
+#[tokio::test]
+async fn test_meta_setup() {
+    let (db, _c) = get_test_pool()
+        .await
+        .expect("test could not start container");
+
+    let query = sqlx::query("SELECT 'hello world!'")
+        .execute(&db)
+        .await
+        .unwrap();
+    assert!(query.rows_affected() == 1);
+}

warehouse/tests/test_extract.rs

@@ -8,7 +8,6 @@ fn v5_state_manifest_fixtures_path() -> PathBuf {
     project_root.join("compatibility/fixtures/v5/state_ver_119757649.17a8/state.manifest")
 }
 
-
 #[tokio::test]
 
 async fn test_extract_v5_manifest() -> Result<()> {

warehouse/tests/test_load.rs

@@ -172,7 +172,6 @@ async fn increment_coin_noop(pool: SqlitePool) -> anyhow::Result<()> {
     Ok(())
 }
 
-
 // Increment the balance table when there balance changes.
 #[sqlx::test]
 async fn increment_coin(pool: SqlitePool) -> anyhow::Result<()> {

warehouse/tests/test_migrate.rs

@@ -17,7 +17,6 @@ async fn can_init(pool: SqlitePool) -> anyhow::Result<()> {
 
     assert!(id == 1);
 
-
     let id = sqlx::query(
         r#"
       INSERT INTO balance (account_address, balance, chain_timestamp, db_version, epoch_number)

warehouse/tests/test_scan_dirs.rs

@@ -36,8 +36,8 @@ fn test_scan_dir_for_v7_manifests() -> Result<()> {
     assert!(archives.len() == 3);
 
     for (p, a) in archives.iter() {
-      dbg!(&p);
-      dbg!(&a);
+        dbg!(&p);
+        dbg!(&a);
     }
 
     Ok(())