Separate HotShot CI (#2620)

.config/nextest.toml

@@ -1,25 +1,9 @@
 [profile.default]
 # Kill tests after 3 periods of 2m, because they are probably hanging
 slow-timeout = { period = "2m", terminate-after = 3 }
-default-filter = 'not (test(slow_) | package(tests))'
+default-filter = 'not (deps(hotshot-testing) | test(slow_) | package(tests))'
 retries = 2
 
-# More flaky tests
-[[profile.default.overrides]]
-filter = """
-test(tests_2::catchup::test_all_restart) |
-test(tests_4::test_marketplace) |
-test(tests_6::test_epochs)
-"""
-retries = 10
-
-# Sometimes fails persistently in CI
-[[profile.default.overrides]]
-filter = """
-test(tests_4::test_with_builder_failures::test_with_builder_failures)
-"""
-threads-required = 2
-
 [[profile.default.overrides]]
 # These tests are fast if they work (usually about 150ms) but sometimes they
 # hang, especially when running Concurrently with other tests. Kill after a
@@ -36,6 +20,14 @@ retries  = 10
 filter = 'test(slow_test_restart)'
 threads-required = 'num-cpus'
 
+# HotShot integration tests
+[profile.hotshot]
+retries = 2
+slow-timeout = { period = "1m", terminate-after = 3 }
+default-filter = 'deps(hotshot-testing)'
+final-status-level = "flaky"
+threads-required = "num-test-threads"
+
 [profile.slow]
 slow-timeout = "2m"
 default-filter = 'test(slow_)'

.github/workflows/doc-rust.yml

@@ -20,8 +20,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Install Just
-        run: sudo apt-get install -y just
+      - uses: taiki-e/install-action@just
 
       - uses: Swatinem/rust-cache@v2
         name: Enable Rust Caching

.github/workflows/hotshot.yml

@@ -0,0 +1,83 @@
+name: HotShot integration tests
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  schedule:
+    - cron: "0 0 * * 1"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        test_suites:
+          - test-ci-1
+          - test-ci-2
+          - test-ci-3
+          - test-ci-4
+          - test-ci-5
+          - test-ci-6-1
+          - test-ci-6-2
+          - test-ci-6-3
+          - test-ci-6-4
+          - test-ci-6-5
+          - test-ci-6-6
+          - test-ci-rest
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Install Rust
+        uses: mkroening/rust-toolchain-toml@main
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+        with:
+          shared-key: "hotshot-tests"
+          cache-on-failure: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' && matrix.test_suites == 'test-ci-rest' }}
+
+      - uses: taiki-e/install-action@just
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Unit and integration tests for all crates in workspace
+        run: |
+          just hotshot::${{ matrix.test_suites }}
+        timeout-minutes: 60
+        env:
+          RUST_BACKTRACE: full
+
+  test-examples:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Install Rust
+        uses: mkroening/rust-toolchain-toml@main
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+        with:
+          shared-key: "hotshot-examples"
+          cache-on-failure: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - uses: taiki-e/install-action@just
+
+      - name: Test examples
+        run: |
+          just hotshot::example all-push-cdn -- --config_file ./hotshot-orchestrator/run-config.toml
+        timeout-minutes: 20

hotshot.just

@@ -0,0 +1,61 @@
+test-ci-rest *ARGS:
+  echo Running unit tests
+  RUST_LOG=error cargo nextest run -E 'not (test(tests_1) | test(tests_2) | test(tests_3) | test(tests_4) | test(tests_5) | test(tests_6))' --profile hotshot --lib --bins --tests --benches --workspace --no-fail-fast {{ARGS}}
+
+test-ci-1:
+  echo Running integration test group 1
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_1 --no-fail-fast
+
+test-ci-2:
+  echo Running integration test group 2
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_2 --no-fail-fast
+
+test-ci-3:
+  echo Running integration test group 3
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_3 --no-fail-fast
+
+test-ci-4:
+  echo Running integration test group 4
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_4 --no-fail-fast
+
+test-ci-5:
+  echo Running integration test group 5
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_5 --no-fail-fast
+
+test-ci-6-1:
+  echo Running integration test group 6
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_6 --no-fail-fast --partition hash:1/6
+
+test-ci-6-2:
+  echo Running integration test group 6
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_6 --no-fail-fast --partition hash:2/6
+
+test-ci-6-3:
+  echo Running integration test group 6
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_6 --no-fail-fast --partition hash:3/6
+
+test-ci-6-4:
+  echo Running integration test group 6
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_6 --no-fail-fast --partition hash:4/6
+
+test-ci-6-5:
+  echo Running integration test group 6
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_6 --no-fail-fast --partition hash:5/6
+
+test-ci-6-6:
+  echo Running integration test group 6
+  RUST_LOG=error cargo nextest run --profile hotshot --test tests_6 --no-fail-fast --partition hash:6/6
+
+# Usage:
+#
+#   just test memoryimpl_::test_success
+#
+# To display logs from a test run:
+#
+#   just test memoryimpl_::test_success --nocapture
+test *ARGS:
+  echo Running test {{ARGS}}
+  cargo nextest run --profile hotshot {{ARGS}}
+
+example *ARGS:
+  cargo run --package hotshot-examples --example {{ARGS}}

justfile

@@ -1,3 +1,5 @@
+mod hotshot
+
 default:
     just --list