feat: Run monitoring follower with acceptance testing

.github/workflows/integration.yml

@@ -301,7 +301,11 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: a3p-integration
-          path: /tmp/export/a3p-integration
+          path: |
+            /tmp/before-test-run-hook-logs
+            /tmp/export/a3p-integration
+            /tmp/loadgen-follower-logs
+            /tmp/loadgen-output
       - name: archive slogfile
         if: (success() || failure())
         uses: actions/upload-artifact@v4

a3p-integration/package.json

@@ -16,7 +16,7 @@
     "doctor": "yarn synthetic-chain doctor"
   },
   "dependencies": {
-    "@agoric/synthetic-chain": "^0.5.2",
+    "@agoric/synthetic-chain": "^0.5.3",
     "@types/better-sqlite3": "^7.6.11"
   },
   "devDependencies": {

a3p-integration/proposals/z:acceptance/host/before-test-run.sh

@@ -0,0 +1,94 @@
+#! /bin/bash
+# shellcheck disable=SC2155
+
+set -o errexit -o errtrace -o pipefail
+
+DIRECTORY_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+FOLLOWER_LOGS_FILE="/tmp/loadgen-follower-logs"
+GITHUB_HOST_NAME="https://github.com"
+GO_VERSION="1.22.12"
+LOADGEN_REPOSITORY_NAME="testnet-load-generator"
+LOGS_FILE="/tmp/before-test-run-hook-logs"
+ORGANIZATION_NAME="agoric"
+SDK_REPOSITORY_NAME="agoric-sdk"
+TIMESTAMP="$(date '+%s')"
+
+LOADGEN_REPOSITORY_LINK="$GITHUB_HOST_NAME/$ORGANIZATION_NAME/$LOADGEN_REPOSITORY_NAME.git"
+NETWORK_CONFIG="/tmp/network-config-$TIMESTAMP"
+OUTPUT_DIRECTORY="/tmp/loadgen-output"
+SDK_REPOSITORY_LINK="$GITHUB_HOST_NAME/$ORGANIZATION_NAME/$SDK_REPOSITORY_NAME.git"
+
+get_branch_name() {
+  if ! test -n "$GITHUB_HEAD_REF"; then
+    if test -n "$GITHUB_REF"; then
+      GITHUB_HEAD_REF="${GITHUB_REF#refs/heads/}"
+    else
+      GITHUB_HEAD_REF="master"
+    fi
+  fi
+
+  echo "$GITHUB_HEAD_REF"
+}
+
+install_go() {
+  if ! which go > /dev/null; then
+    local go_tar="/tmp/go.tar.gz"
+
+    curl --location --output "$go_tar" --silent "https://go.dev/dl/go$GO_VERSION.linux-amd64.tar.gz"
+    tar --directory "$HOME" --extract --file "$go_tar" --gzip
+    rm --force "$go_tar"
+    export PATH="$HOME/go/bin:$PATH"
+  fi
+}
+
+main() {
+  install_go
+  setup_sdk
+  setup_loadgen_runner
+  mkdir --parents "$NETWORK_CONFIG" "$OUTPUT_DIRECTORY"
+  wait_for_network_config
+  start_follower > "$FOLLOWER_LOGS_FILE" 2>&1
+}
+
+setup_loadgen_runner() {
+  cd "$HOME"
+  git clone "$LOADGEN_REPOSITORY_LINK"
+  cd "$LOADGEN_REPOSITORY_NAME/runner"
+  yarn install
+}
+
+setup_sdk() {
+  local branch_name="$(get_branch_name)"
+  echo "Checking out branch '$branch_name' of sdk"
+
+  cd "$HOME"
+  git clone "$SDK_REPOSITORY_LINK" --branch "$branch_name"
+  cd "$SDK_REPOSITORY_NAME"
+  yarn install
+  make --directory "packages/cosmic-swingset" all
+}
+
+start_follower() {
+  AG_CHAIN_COSMOS_HOME="$HOME/.agoric" \
+    SDK_SRC="$HOME/$SDK_REPOSITORY_NAME" \
+    "$HOME/$LOADGEN_REPOSITORY_NAME/runner/bin/loadgen-runner" \
+    --acceptance-integration-message-file "$MESSAGE_FILE_PATH" \
+    --chain-only \
+    --custom-bootstrap \
+    --no-stage.save-storage \
+    --output-dir "$OUTPUT_DIRECTORY" \
+    --profile "testnet" \
+    --stages "3" \
+    --testnet-origin "file://$NETWORK_CONFIG" \
+    --use-state-sync
+
+  echo -n "exit code $?" > "$MESSAGE_FILE_PATH"
+}
+
+wait_for_network_config() {
+  local network_config=$(node "$DIRECTORY_PATH/../wait-for-follower.mjs" "^{.*")
+  echo "Got network config: $network_config"
+  echo "$network_config" > "$NETWORK_CONFIG/network-config"
+}
+
+main > "$LOGS_FILE" 2>&1 &

a3p-integration/proposals/z:acceptance/setup-test.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o errexit -o pipefail
+
+# shellcheck disable=SC1091
+source /usr/src/upgrade-test-scripts/env_setup.sh
+
+set -o xtrace
+
+AGORIC_HOME="$HOME/.agoric"
+
+CHAIN_ID="$(jq --raw-output '.chain_id' < "$AGORIC_HOME/config/genesis.json")"
+IP_ADDRESS="$(hostname --ip-address)"
+NODE_ID="$(agd tendermint show-node-id)"
+
+echo -n "{\"chainName\": \"$CHAIN_ID\", \"rpcAddrs\": [\"$IP_ADDRESS:26657\"], \"gci\": \"$IP_ADDRESS:26657/genesis\", \"peers\":[\"$NODE_ID@$IP_ADDRESS:26656\"], \"seeds\":[]}" > "$MESSAGE_FILE_PATH"
+
+SNAPSHOT_INTERVAL="$(($(jq --raw-output '.SyncInfo.latest_block_height' < "$STATUS_FILE") + 2))"
+sed "s/^snapshot-interval\s*=.*/snapshot-interval = $SNAPSHOT_INTERVAL/" \
+  "$AGORIC_HOME/config/app.toml" \
+  --in-place

a3p-integration/proposals/z:acceptance/test.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
-set -ueo pipefail
+set -o errexit -o nounset -o pipefail
+
+DIRECTORY_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+
+if ! test -z "$MESSAGE_FILE_PATH"; then
+  node "$DIRECTORY_PATH/wait-for-follower.mjs" '^ready$'
+fi
 
 # Place here any test that should be executed using the executed proposal.
 # The effects of this step are not persisted in further proposal layers.
@@ -47,6 +53,16 @@ echo "VALIDATOR_ADDRESS: $VALIDATOR_ADDRESS from delegator $DELEGATOR_ADDRRESS (
 
 yarn ava stakeBld.test.js
 
+if ! test -z "$MESSAGE_FILE_PATH"; then
+  if [[ "$(cat "$MESSAGE_FILE_PATH")" == "ready" ]]; then
+    echo -n "stop" > "$MESSAGE_FILE_PATH"
+  fi
+
+  exit_message="$(node "$DIRECTORY_PATH/wait-for-follower.mjs" "^exit code \d+$")"
+  echo "follower test result: $exit_message"
+  echo "$exit_message" | grep --extended-regexp --silent "^exit code 0$"
+fi
+
 echo ACCEPTANCE TESTING state sync
 ./state-sync-snapshots-test.sh
 ./genesis-test.sh

a3p-integration/proposals/z:acceptance/wait-for-follower.mjs

@@ -0,0 +1,40 @@
+/* eslint-env node */
+
+import { readFile, watch } from 'fs/promises';
+
+const FILE_ENCODING = 'utf-8';
+const FILE_PATH = process.env.MESSAGE_FILE_PATH;
+
+/**
+ * @param {string} filePath
+ * @param {string} message
+ */
+const checkFileContent = async (filePath, message) => {
+  const fileContent = (await readFile(filePath, FILE_ENCODING)).trim();
+  if (!new RegExp(message).test(fileContent)) return '';
+  return fileContent;
+};
+
+/**
+ * @param {string} filePath
+ */
+const watchSharedFile = async filePath => {
+  const [, , message] = process.argv;
+
+  let possibleContent = await checkFileContent(filePath, message);
+  if (possibleContent) return possibleContent;
+
+  for await (const { eventType } of watch(filePath)) {
+    if (eventType === 'change') {
+      possibleContent = await checkFileContent(filePath, message);
+      if (possibleContent) return possibleContent;
+    }
+  }
+
+  return undefined;
+};
+
+FILE_PATH &&
+  watchSharedFile(FILE_PATH).then(
+    fileContent => fileContent && console.log(fileContent),
+  );

a3p-integration/yarn.lock

@@ -5,19 +5,20 @@ __metadata:
   version: 8
   cacheKey: 10c0
 
-"@agoric/synthetic-chain@npm:^0.5.2":
-  version: 0.5.2
-  resolution: "@agoric/synthetic-chain@npm:0.5.2"
+"@agoric/synthetic-chain@npm:^0.5.3":
+  version: 0.5.3
+  resolution: "@agoric/synthetic-chain@npm:0.5.3"
   dependencies:
     "@endo/zip": "npm:^1.0.9"
     better-sqlite3: "npm:^11.8.1"
     chalk: "npm:^5.4.1"
     cosmjs-types: "npm:^0.9.0"
     execa: "npm:^9.5.2"
     glob: "npm:^11.0.1"
+    tmp: "npm:0.2.3"
   bin:
     synthetic-chain: dist/cli/cli.js
-  checksum: 10c0/190939c33f47a1d71558356f42d4b4b10a4caf3176d31281a250877f97d25d5df5b5d261fd507489fd481e24e6ba5ec12e8dc267a0461b67e9f02a8f8769700c
+  checksum: 10c0/47e984c05a64fd520c1c7063b3f0a63d231f0da431bd169e0dece1cba5535696c3c582d32afc7771d869ecb6aa9f2945991c47e1ae015ba321d43a465e37573b
   languageName: node
   linkType: hard
 
@@ -2671,7 +2672,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "root-workspace-0b6124@workspace:."
   dependencies:
-    "@agoric/synthetic-chain": "npm:^0.5.2"
+    "@agoric/synthetic-chain": "npm:^0.5.3"
     "@types/better-sqlite3": "npm:^7.6.11"
     eslint: "npm:^8.57.1"
     npm-run-all: "npm:^4.1.5"
@@ -3173,6 +3174,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tmp@npm:0.2.3":
+  version: 0.2.3
+  resolution: "tmp@npm:0.2.3"
+  checksum: 10c0/3e809d9c2f46817475b452725c2aaa5d11985cf18d32a7a970ff25b568438e2c076c2e8609224feef3b7923fa9749b74428e3e634f6b8e520c534eef2fd24125
+  languageName: node
+  linkType: hard
+
 "tunnel-agent@npm:^0.6.0":
   version: 0.6.0
   resolution: "tunnel-agent@npm:0.6.0"