nightly.yml

name: Nightly -- Full Network Tests

# on:
#   schedule:
#     - cron:  '0 0 * * *'
#   workflow_dispatch:

on:
  # tests must run for a PR to be valid and pass merge queue muster
  # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors
  # the merge run checks should show on master and enable this clear test/passing history
  merge_group:
    branches: [main]
  pull_request:
    branches: ["*"]
    

env:
  CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI.
  WORKFLOW_URL: https://github.com/maidsafe/stableset_net/actions/runs

jobs:
  e2e:
    name: E2E tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true
          
      - name: Build safe
        run: cargo build --release --bin safe
        timeout-minutes: 30

      - name: Start a WAN network
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: create
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          security-group-id: sg-0d47df5b3f0d01e2a
          subnet-id: subnet-018f2ab26755df7f9
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyE2E
          # if we were to run on a PR, use the below
          # custom-node-bin-org-name: ${{ github.actor }}"
          # custom-node-bin-branch-name: ${{ github.event.pull_request.head.ref }}
          # Prevent deployer from fetching the latest release. It contains the `network-contacts` feature turned on.
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Check env variables
        shell: bash
        run: |
          echo "Peer is $SAFE_PEERS"
          echo "Deployment inventory is $SN_INVENTORY"

      - name: Obtain the funds from the faucet
        run: |
          # read the inventory file
          inventory_path=/home/runner/.local/share/safe/testnet-deploy/NightlyE2E-inventory.json
          echo "Inventory Path: $inventory_path"
          faucet_address=$(jq -r '.faucet_address' $inventory_path)
          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to carry out chunk actions
        run: cargo run --bin safe --release -- --log-output-dest=data-dir files upload "./resources" -r 0
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to create a register
        run: cargo run --bin safe --release -- --log-output-dest=data-dir register create -n baobao
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to get a register
        run: cargo run --bin safe --release -- --log-output-dest=data-dir register get -n baobao
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to edit a register
        run: cargo run --bin safe --release -- --log-output-dest=data-dir register edit -n baobao wood
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Fetch network logs
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: logs
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyE2E
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Upload local logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local_logs_NightlyE2E
          path: |
            ~/.local/share/safe/node/*/logs/*.log*
            ~/.local/share/safe/*/*/*.log*
            ~/.local/share/safe/client/logs/*/*.log*

      - name: Stop the WAN network
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: destroy
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyE2E
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      # - name: post notification to slack on failure
      #   if: ${{ failure() }}
      #   uses: bryannice/gitactions-slack-notification@2.0.0
      #   env:
      #     SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
      #     SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
      #     SLACK_TITLE: "Nightly E2E Test Run Failed"

  # full_unit:
  #   name: Full Unit Tests (including proptests)
  #   runs-on: ${{ matrix.os }}
  #   strategy:
  #     matrix:
  #       os: [ubuntu-latest, windows-latest, macos-latest]
  #   steps:
  #     - uses: actions/checkout@v4

  #     - name: Install Rust
  #       uses: dtolnay/rust-toolchain@stable

  #     - uses: Swatinem/rust-cache@v2
  #       continue-on-error: true

  #     - name: Build unit tests before running
  #       run: cargo test --release --lib --bins --no-run 
  #       timeout-minutes: 30

  #     - name: Run CLI tests
  #       timeout-minutes: 25
  #       run: cargo test --release --package sn_cli

  #     - name: Run testnet tests
  #       timeout-minutes: 25
  #       run: cargo test --release --package sn_testnet

  #     - name: Run client tests
  #       timeout-minutes: 25
  #       run: cargo test --release --package sn_client

  #     - name: Run network tests
  #       timeout-minutes: 25
  #       run: cargo test --release -p sn_networking

  #     - name: Run protocol tests
  #       timeout-minutes: 25
  #       run: cargo test --release -p sn_protocol

  #     - name: Run transfers tests
  #       timeout-minutes: 25
  #       run: cargo test --release --package sn_transfers

  #     - name: Run register tests
  #       shell: bash
  #       timeout-minutes: 50
  #       env:
  #         PROPTEST_CASES: 512
  #       run: cargo test --release -p sn_registers

  #     - name: post notification to slack on failure
  #       if: ${{ failure() }}
  #       uses: bryannice/gitactions-slack-notification@2.0.0
  #       env:
  #         SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
  #         SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
  #         SLACK_TITLE: "Nightly Unit Test Run Failed"

  gossipsub:
    if: "!startsWith(github.event.head_commit.message, 'chore(release):')" 
    name: Gossipsub E2E tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Build gossipsub testing executable
        run: cargo test --release -p sn_node --test msgs_over_gossipsub --no-run
        timeout-minutes: 30

      - name: Start a WAN network
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: create
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          security-group-id: sg-0d47df5b3f0d01e2a
          subnet-id: subnet-018f2ab26755df7f9
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyGossipSubE2E
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Check env variables
        shell: bash
        run: |
          echo "Peer is $SAFE_PEERS"
          echo "Deployment inventory is $SN_INVENTORY"

      - name: Gossipsub - nodes to subscribe to topics, and publish messages 
        run: cargo test --release -p sn_node --test msgs_over_gossipsub -- --nocapture
        timeout-minutes: 20

      - name: Fetch network logs
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: logs
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyGossipSubE2E
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Upload local logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local_logs_NightlyGossipSubE2E
          path: |
            ~/.local/share/safe/node/*/logs/*.log*
            ~/.local/share/safe/*/*/*.log*
            ~/.local/share/safe/client/logs/*/*.log*

      - name: Stop the WAN network
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: destroy
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyGossipSubE2E
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly    

  spend_test:
    name: Spend tests against network
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build testing executable
        run: cargo test --release -p sn_node --test sequential_transfers --test storage_payments --test nodes_rewards --no-run
        timeout-minutes: 30
        env:
          CARGO_TARGET_DIR: "./transfer-target"

      - name: Start a WAN network
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: create
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          security-group-id: sg-0d47df5b3f0d01e2a
          subnet-id: subnet-018f2ab26755df7f9
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlySpendTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Check env variables
        shell: bash
        run: |
          echo "Peer is $SAFE_PEERS"
          echo "Deployment inventory is $SN_INVENTORY"

      # This should be first to avoid slow reward acceptance etc
      - name: execute the nodes rewards tests
        run: cargo test --release -p sn_node --test nodes_rewards -- --nocapture --test-threads=1
        env:
          CARGO_TARGET_DIR: "./transfer-target"
          SN_LOG: "all"
        timeout-minutes: 25

      - name: execute the spend test
        run: cargo test --release -p sn_node --test sequential_transfers -- --nocapture --test-threads=1
        env:
          CARGO_TARGET_DIR: "./transfer-target"
          SN_LOG: "all"
        timeout-minutes: 10

      - name: execute the storage payment tests
        run: cargo test --release -p sn_node --test storage_payments -- --nocapture --test-threads=1          
        env:
          CARGO_TARGET_DIR: "./transfer-target"
          SN_LOG: "all"
        timeout-minutes: 10

      - name: Small wait to allow reward receipt
        run: sleep 30
        timeout-minutes: 1
      
      - name: Fetch network logs
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: logs
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlySpendTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Upload local logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local_logs_NightlySpendTest
          path: |
            ~/.local/share/safe/node/*/logs/*.log*
            ~/.local/share/safe/*/*/*.log*
            ~/.local/share/safe/client/logs/*/*.log*

      - name: Stop the WAN network
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: destroy
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlySpendTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      # - name: post notification to slack on failure
      #   if: ${{ failure() }}
      #   uses: bryannice/gitactions-slack-notification@2.0.0
      #   env:
      #     SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
      #     SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
      #     SLACK_TITLE: "Nightly Spend Test Run Failed"

  churn:
    name: Network churning tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            wan_logs_path: /home/runner/sn-testnet-deploy/logs
            local_safe_path: /home/runner/.local/share/safe
          # - os: windows-latest
          #   node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node
          #   safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe
          # - os: macos-latest
          #   node_data_path: /Users/runner/Library/Application Support/safe/node
          #   safe_path: /Users/runner/Library/Application Support/safe
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: install ripgrep
        run: sudo apt-get -y install ripgrep

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build churn tests 
        run: cargo test --release -p sn_node --test data_with_churn --no-run
        timeout-minutes: 30

      - name: Start a WAN network
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: create
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          security-group-id: sg-0d47df5b3f0d01e2a
          subnet-id: subnet-018f2ab26755df7f9
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyChurnTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Check env variables
        shell: bash
        run: |
          echo "Peer is $SAFE_PEERS"
          echo "Deployment inventory is $SN_INVENTORY"

      - name: Chunks data integrity during nodes churn
        run: cargo test --release -p sn_node --test data_with_churn -- --nocapture
        env:
          # TEST_DURATION_MINS: 60
          # TEST_CHURN_CYCLES: 6
          # SN_LOG: "all"
          # todo: lower time for testing
          TEST_DURATION_MINS: 10
          TEST_CHURN_CYCLES: 2
          SN_LOG: "all"
        timeout-minutes: 90

      - name: Fetch network logs
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: logs
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyChurnTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Upload local logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local_logs_NightlyChurnTest
          path: |
            ~/.local/share/safe/node/*/logs/*.log*
            ~/.local/share/safe/*/*/*.log*
            ~/.local/share/safe/client/logs/*/*.log*

      - name: Stop the WAN network
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: destroy
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyChurnTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly
      
      - name: Verify restart of nodes using rg
        shell: bash
        timeout-minutes: 1
        # get the counts, then the specific line, and then the digit count only
        # then check we have an expected level of restarts
        # TODO: make this use an env var, or relate to testnet size
        run : |
          restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "Restart $restart_count nodes"
          peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "PeerRemovedFromRoutingTable $peer_removed times"
          if [ $peer_removed -lt $restart_count ]; then
            echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count"
            exit 1
          fi
          node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l)
          echo "Node dir count is $node_count"
        # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here

        # if [ $restart_count -lt $node_count ]; then
        #   echo "Restart count of: $restart_count is less than the node count of: $node_count"
        #   exit 1
        # fi

      - name: Verify data replication using rg
        shell: bash
        timeout-minutes: 1
        # get the counts, then the specific line, and then the digit count only
        # then check we have an expected level of replication
        # TODO: make this use an env var, or relate to testnet size
        run : |
          fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.wan_logs_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "Carried out $fetching_attempt_count fetching attempts"
          node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l)
          if [ $fetching_attempt_count -lt $node_count ]; then
            echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count"
            exit 1
          fi

      # Only error out after uploading the logs
      - name: Don't log raw data
        if: always() && matrix.os != 'windows-latest' # causes error
        shell: bash
        timeout-minutes: 10
        run: |
          if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }'
          then
            echo "We are logging an extremely large data"
            exit 1
          fi
          # the wan log dir contains: testnet/machine/node(or faucet)/node_name/logs
          if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }'
          then
            echo "We are logging an extremely large data"
            exit 1
          fi

          # sanity check
          if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }'
          then
            echo "Sanity check pass for local safe path"
          fi
          # the wan log dir contains: testnet/machine/node(or faucet)/node_name/logs
          if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/*/logs | awk 'length($0) > 1000 { print; exit 1 }'
          then
            echo "Sanity check pass for wan logs"
          fi

      # - name: post notification to slack on failure
      #   if: ${{ failure() }}
      #   uses: bryannice/gitactions-slack-notification@2.0.0
      #   env:
      #     SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
      #     SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
      #     SLACK_TITLE: "Nightly Churn Test Run Failed"


  verify_data_location_routing_table:
    name: Verify data location and Routing Table
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            wan_logs_path: /home/runner/sn-testnet-deploy/logs
            local_safe_path: /home/runner/.local/share/safe
          # - os: windows-latest
          #   node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node
          #   safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe
          # - os: macos-latest
          #   node_data_path: /Users/runner/Library/Application Support/safe/node
          #   safe_path: /Users/runner/Library/Application Support/safe
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: install ripgrep
        run: sudo apt-get -y install ripgrep

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build data location and routing table tests
        run: cargo test --release -p sn_node --test verify_data_location --test verify_routing_table --no-run
        timeout-minutes: 30

      - name: Start a WAN network
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: create
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          security-group-id: sg-0d47df5b3f0d01e2a
          subnet-id: subnet-018f2ab26755df7f9
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyDataLocationTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Check env variables
        shell: bash
        run: |
          echo "Peer is $SAFE_PEERS"
          echo "Deployment inventory is $SN_INVENTORY"

      - name: Verify the Routing table of the nodes
        run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture
        timeout-minutes: 5

      - name: Verify the location of the data on the network
        run: cargo test --release -p sn_node --test verify_data_location -- --nocapture
        env:
          SN_LOG: "all"
        timeout-minutes: 90

      - name: Verify the routing tables of the nodes
        run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture 
        timeout-minutes: 5

      - name: Fetch network logs
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: logs
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyDataLocationTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Upload local logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local_logs_NightlyDataLocationTest
          path: |
            ~/.local/share/safe/node/*/logs/*.log*
            ~/.local/share/safe/*/*/*.log*
            ~/.local/share/safe/client/logs/*/*.log*

      - name: Stop the WAN network
        if: always()
        uses: RolandSherwin/sn-testnet-action@wan_nightly
        with:
          action: destroy
          rust-log: debug
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
          node-count: 20
          vm-count: 1
          provider: digital-ocean
          testnet-name: NightlyDataLocationTest
          custom-node-bin-org-name: maidsafe
          custom-node-bin-branch-name: wan_nightly
          testnet-tool-repo-user: RolandSherwin
          testnet-tool-repo-branch: wan_nightly

      - name: Verify restart of nodes using rg
        shell: bash
        timeout-minutes: 1
        # get the counts, then the specific line, and then the digit count only
        # then check we have an expected level of restarts
        # TODO: make this use an env var, or relate to testnet size
        run : |
          restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "Restart $restart_count nodes"
          peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "PeerRemovedFromRoutingTable $peer_removed times"
          if [ $peer_removed -lt $restart_count ]; then
            echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count"
            exit 1
          fi
          node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l)
          echo "Node dir count is $node_count"
        # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here

        # if [ $restart_count -lt $node_count ]; then
        #   echo "Restart count of: $restart_count is less than the node count of: $node_count"
        #   exit 1
        # fi

      # Only error out after uploading the logs
      - name: Don't log raw data
        if: always() && matrix.os != 'windows-latest' # causes error
        shell: bash
        timeout-minutes: 10
        run: |
          if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }'
          then
            echo "We are logging an extremely large data"
            exit 1
          fi
          # the wan log dir contains: testnet/machine/node(or faucet)/node_name/logs
          if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }'
          then
            echo "We are logging an extremely large data"
            exit 1
          fi

          # sanity check
          if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }'
          then
            echo "Sanity check pass for local safe path"
          fi
          # the wan log dir contains: testnet/machine/node(or faucet)/node_name/logs
          if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/*/logs | awk 'length($0) > 1000 { print; exit 1 }'
          then
            echo "Sanity check pass for wan logs"
          fi

      # - name: post notification to slack on failure
      #   if: ${{ failure() }}
      #   uses: bryannice/gitactions-slack-notification@2.0.0
      #   env:
      #     SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
      #     SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
      #     SLACK_TITLE: "Nightly Data Location Test Run Failed"