fix: enable retries for integration tests

.github/workflows/test.yml

@@ -212,10 +212,11 @@ jobs:
         version: [02,99]
         include:
           - version: 02
-            compose: "-f process-compose.yaml"
+            test-name: test_native_demo_basic
 
           - version: 99
-            compose: "-f process-compose.yaml -f process-compose-mp.yml"
+            test-name: test_native_demo_upgrade
+
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
@@ -242,17 +243,14 @@ jobs:
       - name: Install process-compose
         run: nix profile install nixpkgs#process-compose
 
-      - name: Run Demo-Native ${{matrix.version}}
-        run: bash -x scripts/demo-native ${{matrix.compose}} --tui=false > ${{ env.PC_LOGS }} 2>&1 &
-
       - name: Test Integration
         env:
           RUST_LOG: debug
           NEXTEST_PROFILE: integration
           INTEGRATION_TEST_SEQUENCER_VERSION: ${{ matrix.version }}
         run: |
           cargo nextest run --archive-file nextest-archive-postgres.tar.zst --verbose --no-fail-fast --nocapture \
-          --workspace-remap $PWD $(if [ "${{ matrix.version }}" == "2" ]; then echo " smoke"; fi)
+          --workspace-remap $PWD ${{ matrix.test-name }}
         timeout-minutes: 10
 
       - name: Show end of logs

justfile

@@ -75,11 +75,10 @@ test-all:
     cargo nextest run --locked --release --workspace --verbose --profile all
 
 test-integration:
-	@echo 'NOTE that demo-native must be running for this test to succeed.'
-	INTEGRATION_TEST_SEQUENCER_VERSION=2 cargo nextest run --all-features --nocapture --profile integration smoke
+	INTEGRATION_TEST_SEQUENCER_VERSION=2 cargo nextest run -p tests --nocapture --profile integration test_native_demo_basic
+
 test-integration-mp:
-    @echo 'NOTE that demo-native-mp must be running for this test to succeed.'
-    INTEGRATION_TEST_SEQUENCER_VERSION=99 cargo nextest run --all-features --nocapture --profile integration
+    INTEGRATION_TEST_SEQUENCER_VERSION=99 cargo nextest run -p tests --nocapture --profile integration test_native_demo_upgrade
 
 clippy:
     @echo 'features: "embedded-db"'

tests/Cargo.toml

@@ -18,5 +18,6 @@ ethers = { workspace = true }
 futures = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }
 surf-disco = { workspace = true }
+tempfile = { workspace = true }
 tokio = { workspace = true }
 vbs = { workspace = true }

tests/common/mod.rs

@@ -1,6 +1,14 @@
-use std::{fmt, str::FromStr, time::Duration};
-
-use anyhow::{anyhow, Result};
+use std::{
+    fmt,
+    fs::File,
+    io::{stderr, stdout},
+    path::PathBuf,
+    process::{Child, Command},
+    str::FromStr,
+    time::Duration,
+};
+
+use anyhow::{anyhow, Context, Result};
 use client::SequencerClient;
 use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion};
 use ethers::prelude::*;
@@ -278,3 +286,77 @@ async fn wait_for_service(url: Url, interval: u64, timeout_duration: u64) -> Res
     .await
     .map_err(|e| anyhow!("Wait for service, timeout: ({}) {}", url, e))?
 }
+
+pub struct NativeDemo {
+    _child: Child,
+}
+
+impl Drop for NativeDemo {
+    fn drop(&mut self) {
+        // It would be preferable to send a SIGINT or similar to the process that we started
+        // originally but despite quite some effort this never worked for the process-compose
+        // process started from within the scripts/demo-native script.
+        //
+        // Using `process-compose down` seems to pretty reliably stop the process-compose process
+        // and all the services it started.
+        println!("Terminating process compose");
+        let res = Command::new("process-compose")
+            .arg("down")
+            .stdout(stdout())
+            .stderr(stderr())
+            .spawn()
+            .expect("process-compose runs")
+            .wait()
+            .unwrap();
+        println!("process-compose down exited with: {}", res);
+    }
+}
+
+impl NativeDemo {
+    pub(crate) fn run(process_compose_extra_args: Option<String>) -> anyhow::Result<Self> {
+        // Because we use nextest with the archive feature on CI we need to use the **runtime**
+        // value of CARGO_MANIFEST_DIR.
+        let crate_dir = PathBuf::from(
+            std::env::var("CARGO_MANIFEST_DIR")
+                .expect("CARGO_MANIFEST_DIR is set")
+                .clone(),
+        );
+        let workspace_dir = crate_dir.parent().expect("crate_dir has a parent");
+
+        let mut cmd = Command::new("bash");
+        cmd.arg("scripts/demo-native")
+            .current_dir(workspace_dir)
+            .arg("--tui=false");
+
+        if let Some(args) = process_compose_extra_args {
+            cmd.args(args.split(' '));
+        }
+
+        // Save output to file if PC_LOGS if that's set.
+        let log_path = std::env::var("PC_LOGS").unwrap_or_else(|_| {
+            tempfile::NamedTempFile::new()
+                .expect("tempfile creation succeeds")
+                .into_temp_path()
+                .to_string_lossy()
+                .to_string()
+        });
+
+        println!("Writing native demo logs to file: {}", log_path);
+        let outputs = File::create(log_path).context("unable to create log file")?;
+        cmd.stdout(outputs);
+
+        println!("Spawning: {:?}", cmd);
+        let mut child = cmd.spawn().context("failed to spawn command")?;
+
+        // Wait for three seconds and check if process has already exited so we don't waste time
+        // waiting for results later.
+        std::thread::sleep(Duration::from_secs(3));
+        if let Some(exit_code) = child.try_wait()? {
+            return Err(anyhow!("process-compose exited early with: {}", exit_code));
+        }
+
+        println!("process-compose started ...");
+
+        Ok(Self { _child: child })
+    }
+}

tests/smoke.rs

@@ -3,15 +3,14 @@ use std::time::Instant;
 use anyhow::Result;
 use futures::StreamExt;
 
-use crate::common::TestConfig;
+use crate::common::{NativeDemo, TestConfig};
 
 /// We allow for no change in state across this many consecutive iterations.
 const MAX_STATE_NOT_INCREMENTING: u8 = 1;
 /// We allow for no new transactions across this many consecutive iterations.
 const MAX_TXNS_NOT_INCREMENTING: u8 = 5;
 
-#[tokio::test(flavor = "multi_thread")]
-async fn test_smoke() -> Result<()> {
+pub async fn assert_native_demo_works() -> Result<()> {
     let start = Instant::now();
     dotenvy::dotenv()?;
 
@@ -21,7 +20,7 @@ async fn test_smoke() -> Result<()> {
     let _ = testing.readiness().await?;
 
     let initial = testing.test_state().await;
-    println!("Initial State:{}", initial);
+    println!("Initial State: {}", initial);
 
     let mut sub = testing
         .espresso
@@ -47,7 +46,9 @@ async fn test_smoke() -> Result<()> {
         }
 
         // test that we progress EXPECTED_BLOCK_HEIGHT blocks from where we started
-        if new.block_height.unwrap() >= testing.expected_block_height() + testing.initial_height {
+        if new.block_height.unwrap()
+            >= testing.expected_block_height() + initial.block_height.unwrap()
+        {
             println!("Reached {} block(s)!", testing.expected_block_height());
             if new.txn_count - initial.txn_count < 1 {
                 panic!("Did not receive transactions");
@@ -81,3 +82,9 @@ async fn test_smoke() -> Result<()> {
     }
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_native_demo_basic() -> Result<()> {
+    let _child = NativeDemo::run(None);
+    assert_native_demo_works().await
+}

tests/upgrades.rs

@@ -3,12 +3,21 @@ use espresso_types::{FeeVersion, MarketplaceVersion};
 use futures::{future::join_all, StreamExt};
 use vbs::version::StaticVersionType;
 
-use crate::common::TestConfig;
+use crate::{
+    common::{NativeDemo, TestConfig},
+    smoke::assert_native_demo_works,
+};
 
 const SEQUENCER_BLOCKS_TIMEOUT: u64 = 200;
 
 #[tokio::test(flavor = "multi_thread")]
-async fn test_upgrade() -> Result<()> {
+async fn test_native_demo_upgrade() -> Result<()> {
+    let _demo = NativeDemo::run(Some(
+        "-f process-compose.yaml -f process-compose-mp.yml".to_string(),
+    ))?;
+
+    assert_native_demo_works().await?;
+
     dotenvy::dotenv()?;
 
     let testing = TestConfig::new().await.unwrap();