From b22db156e7ece321cafbc6063a4f93dd1028ab5e Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sat, 1 Feb 2025 09:56:09 -0800 Subject: [PATCH] [testing] Replace script-based tool installation with nix Previously, binary tools like promtail and prometheus (enabling log and metrics collection) and kube and kind (enabling local kube clusters) were installed using bash scripts. A script-based approach, while simple, requires tedious and error-prone copying and committing to other repos (e.g. subnet-evm and hypersdk) that need the same functionality. Possible to rewrite the scripts to golang which allows usage via `go run` or `go install`. Simple enough if the versions never change, but still requires retrieving binaries, verifying their hashes for security, and potentially dealing with tar archives. Still more complicated if it becomes necessary to ensure the desired version is the one in the path. Switching to [nix](https://nixos.org/), on the other hand, is an established way of solving the problem that doesn't involve maintaining code. Installing nix is a one-time operation not too different from homebrew, except that the result has more guarantees of reproducibility across linux and macos. --- .envrc | 2 + .../run-monitored-tmpnet-cmd/action.yml | 6 +- .github/workflows/ci.yml | 17 +++++ .gitignore | 2 + flake.lock | 25 +++++++ flake.nix | 75 +++++++++++++++++++ scripts/run_prometheus.sh | 41 ++-------- scripts/run_promtail.sh | 35 ++------- scripts/tests.e2e.bootstrap_monitor.sh | 42 ++--------- 9 files changed, 149 insertions(+), 96 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc index 65b6f223990f..e0523a20e3d2 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,5 @@ +use flake + # Repo-local commands like ginkgo and tmpnetctl PATH_add bin diff --git a/.github/actions/run-monitored-tmpnet-cmd/action.yml b/.github/actions/run-monitored-tmpnet-cmd/action.yml index 6f070544ec7c..3a3bc911efbc 100644 --- a/.github/actions/run-monitored-tmpnet-cmd/action.yml +++ b/.github/actions/run-monitored-tmpnet-cmd/action.yml @@ -40,14 +40,16 @@ runs: # Only run for the original repo; a forked repo won't have access to the monitoring credentials if: (inputs.prometheus_username != '') shell: bash - run: bash -x ./scripts/run_prometheus.sh + # Assumes calling project has a nix flake that ensures a compatible prometheus + run: nix develop --impure --command bash -x ./scripts/run_prometheus.sh env: PROMETHEUS_USERNAME: ${{ inputs.prometheus_username }} PROMETHEUS_PASSWORD: ${{ inputs.prometheus_password }} - name: Start promtail if: (inputs.prometheus_username != '') shell: bash - run: bash -x ./scripts/run_promtail.sh + # Assumes calling project has a nix flake that ensures a compatible promtail + run: nix develop --impure --command bash -x ./scripts/run_promtail.sh env: LOKI_USERNAME: ${{ inputs.loki_username }} LOKI_PASSWORD: ${{ inputs.loki_password }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 619547b0eb4a..0c170d017535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project + - uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + # TODO(marun) Maybe figure out how to use `nix build` somehow i.e. make the default shell double as the default package + - run: nix develop --command echo "dependencies installed" - name: Build AvalancheGo Binary shell: bash run: ./scripts/build.sh -r @@ -72,6 +77,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project + - uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - run: nix develop --command echo "dependencies installed" - name: Build AvalancheGo Binary shell: bash run: ./scripts/build.sh -r @@ -93,6 +102,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project + - uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - run: nix develop --command echo "dependencies installed" - name: Build AvalancheGo Binary shell: bash run: ./scripts/build.sh @@ -230,6 +243,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project + - uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - run: nix develop --command echo "dependencies installed" - name: Run e2e tests shell: bash run: bash -x ./scripts/tests.e2e.bootstrap_monitor.sh diff --git a/.gitignore b/.gitignore index c5a3cad32b13..bacc6f26c318 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,5 @@ tests/upgrade/upgrade.test vendor **/testdata + +.direnv diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000000..a8e1ac35b230 --- /dev/null +++ b/flake.lock @@ -0,0 +1,25 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1721949857, + "narHash": "sha256-DID446r8KsmJhbCzx4el8d9SnPiE8qa6+eEQOJ40vR0=", + "rev": "a1cc729dcbc31d9b0d11d86dc7436163548a9665", + "revCount": 633481, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2405.633481%2Brev-a1cc729dcbc31d9b0d11d86dc7436163548a9665/0190ef32-8438-79be-9d83-2e97dc792840/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2405.%2A.tar.gz" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000000..44943b747445 --- /dev/null +++ b/flake.nix @@ -0,0 +1,75 @@ +{ + # To use: + # - install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix + # - run `nix develop` or use direnv (https://direnv.net/) + # - for quieter direnv output, set `export DIRENV_LOG_FORMAT=` + + description = "AvalancheGo development environment"; + + # Flake inputs + inputs = { + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2405.*.tar.gz"; + }; + + # Flake outputs + outputs = { self, nixpkgs }: + let + # Systems supported + allSystems = [ + "x86_64-linux" # 64-bit Intel/AMD Linux + "aarch64-linux" # 64-bit ARM Linux + "x86_64-darwin" # 64-bit Intel macOS + "aarch64-darwin" # 64-bit ARM macOS + ]; + + # Helper to provide system-specific attributes + forAllSystems = f: nixpkgs.lib.genAttrs allSystems (system: f { + pkgs = import nixpkgs { inherit system; }; + }); + in + { + # Development environment output + devShells = forAllSystems ({ pkgs }: { + default = pkgs.mkShell { + # The Nix packages provided in the environment + packages = with pkgs; [ + # Monitoring tools + promtail # Loki log shipper + prometheus # Metrics collector + + # Kube tools + kubectl # Kubernetes CLI + kind # Kubernetes-in-Docker + kubernetes-helm # Helm CLI (Kubernetes package manager) + self.packages.${system}.kind-with-registry # Script installing kind configured with a local registry + ]; + }; + }); + + # Package to install the kind-with-registry script + packages = forAllSystems ({ pkgs }: { + kind-with-registry = pkgs.stdenv.mkDerivation { + pname = "kind-with-registry"; + version = "1.0.0"; + + src = pkgs.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/kind/7cb9e6be25b48a0e248097eef29d496ab1a044d0/site/static/examples/kind-with-registry.sh"; + sha256 = "0gri0x0ygcwmz8l4h6zzsvydw8rsh7qa8p5218d4hncm363i81hv"; + }; + + phases = [ "installPhase" ]; + + installPhase = '' + mkdir -p $out/bin + install -m755 $src $out/bin/kind-with-registry.sh + ''; + + meta = with pkgs.lib; { + description = "Script to set up kind with a local registry"; + license = licenses.mit; + maintainers = with maintainers; [ "maru-ava" ]; + }; + }; + }); + }; +} diff --git a/scripts/run_prometheus.sh b/scripts/run_prometheus.sh index 43c1084f8755..15e02637fc4c 100755 --- a/scripts/run_prometheus.sh +++ b/scripts/run_prometheus.sh @@ -20,6 +20,14 @@ if ! [[ "$0" =~ scripts/run_prometheus.sh ]]; then exit 255 fi +CMD=prometheus + +if ! command -v "${CMD}" &> /dev/null; then + echo "prometheus not found, have you run `nix develop`?" + echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix" + exit 1 +fi + PROMETHEUS_WORKING_DIR="${HOME}/.tmpnet/prometheus" PIDFILE="${PROMETHEUS_WORKING_DIR}"/run.pid @@ -48,39 +56,6 @@ if [[ -z "${PROMETHEUS_PASSWORD}" ]]; then exit 1 fi -# This was the LTS version when this script was written. Probably not -# much reason to update it unless something breaks since the usage -# here is only to collect metrics from temporary networks. -VERSION="2.45.3" - -# Ensure the prometheus command is locally available -CMD=prometheus -if ! command -v "${CMD}" &> /dev/null; then - # Try to use a local version - CMD="${PWD}/bin/prometheus" - if ! command -v "${CMD}" &> /dev/null; then - echo "prometheus not found, attempting to install..." - - GOOS="$(go env GOOS)" - GOARCH="$(go env GOARCH)" - if [[ "${GOOS}" == "darwin" && "${GOARCH}" == "arm64" ]]; then - echo "On macos, only amd64 binaries are available so rosetta is required on apple silicon machines." - echo "To avoid using rosetta, install via homebrew: brew install prometheus" - fi - if [[ "${GOOS}" == "linux" && "${GOARCH}" != "amd64" ]]; then - echo "On linux, only amd64 binaries are available. Manual installation of prometheus is required." - exit 1 - fi - - # Install the specified release - PROMETHEUS_FILE="prometheus-${VERSION}.${GOOS}-amd64" - URL="https://github.com/prometheus/prometheus/releases/download/v${VERSION}/${PROMETHEUS_FILE}.tar.gz" - curl -s -L "${URL}" | tar zxv -C /tmp > /dev/null - mkdir -p "$(dirname "${CMD}")" - cp /tmp/"${PROMETHEUS_FILE}/prometheus" "${CMD}" - fi -fi - # Configure prometheus FILE_SD_PATH="${PROMETHEUS_WORKING_DIR}/file_sd_configs" mkdir -p "${FILE_SD_PATH}" diff --git a/scripts/run_promtail.sh b/scripts/run_promtail.sh index 077ad9c02616..090625055410 100755 --- a/scripts/run_promtail.sh +++ b/scripts/run_promtail.sh @@ -19,6 +19,14 @@ if ! [[ "$0" =~ scripts/run_promtail.sh ]]; then exit 255 fi +CMD=promtail + +if ! command -v "${CMD}" &> /dev/null; then + echo "promtail not found, have you run `nix develop`?" + echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix" + exit 1 +fi + PROMTAIL_WORKING_DIR="${HOME}/.tmpnet/promtail" PIDFILE="${PROMTAIL_WORKING_DIR}"/run.pid @@ -47,33 +55,6 @@ if [[ -z "${LOKI_PASSWORD}" ]]; then exit 1 fi -# Version as of this writing -VERSION="v2.9.5" - -# Ensure the promtail command is locally available -CMD=promtail -if ! command -v "${CMD}" &> /dev/null; then - # Try to use a local version - CMD="${PWD}/bin/promtail" - if ! command -v "${CMD}" &> /dev/null; then - echo "promtail not found, attempting to install..." - - # Determine the platform - GOOS="$(go env GOOS)" - GOARCH="$(go env GOARCH)" - DIST="${GOOS}-${GOARCH}" - - # Install the specified release - PROMTAIL_FILE="promtail-${DIST}" - ZIP_PATH="/tmp/${PROMTAIL_FILE}.zip" - BIN_DIR="$(dirname "${CMD}")" - URL="https://github.com/grafana/loki/releases/download/${VERSION}/promtail-${DIST}.zip" - curl -L -o "${ZIP_PATH}" "${URL}" - unzip "${ZIP_PATH}" -d "${BIN_DIR}" - mv "${BIN_DIR}/${PROMTAIL_FILE}" "${CMD}" - fi -fi - # Configure promtail FILE_SD_PATH="${PROMTAIL_WORKING_DIR}/file_sd_configs" mkdir -p "${FILE_SD_PATH}" diff --git a/scripts/tests.e2e.bootstrap_monitor.sh b/scripts/tests.e2e.bootstrap_monitor.sh index 95e2a4273f5a..fc365e316422 100755 --- a/scripts/tests.e2e.bootstrap_monitor.sh +++ b/scripts/tests.e2e.bootstrap_monitor.sh @@ -9,40 +9,14 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then exit 255 fi -GOOS="$(go env GOOS)" -GOARCH="$(go env GOARCH)" +CMD=kind-with-registry.sh -function ensure_command { - local cmd=$1 - local install_uri=$2 - - if ! command -v "${cmd}" &> /dev/null; then - # Try to use a local version - local local_cmd="${PWD}/bin/${cmd}" - mkdir -p "${PWD}/bin" - if ! command -v "${local_cmd}" &> /dev/null; then - echo "${cmd} not found, attempting to install..." - curl -L -o "${local_cmd}" "${install_uri}" - # TODO(marun) Optionally validate the binary against published checksum - chmod +x "${local_cmd}" - fi - fi -} - -# Ensure the kubectl command is available -KUBECTL_VERSION=v1.30.2 -ensure_command kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${GOOS}/${GOARCH}/kubectl" - -# Ensure the kind command is available -KIND_VERSION=v0.23.0 -ensure_command kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-${GOOS}-${GOARCH}" - -# Ensure the kind-with-registry command is available -ensure_command "kind-with-registry.sh" "https://raw.githubusercontent.com/kubernetes-sigs/kind/7cb9e6be25b48a0e248097eef29d496ab1a044d0/site/static/examples/kind-with-registry.sh" +if ! command -v "${CMD}" &> /dev/null; then + echo "kind-with-registry.sh not found, have you run `nix develop`?" + echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix" + exit 1 +fi -# Deploy a kind cluster with a local registry. Include the local bin in the path to -# ensure locally installed kind and kubectl are available since the script expects to -# call them without a qualifying path. -PATH="${PWD}/bin:$PATH" bash -x "${PWD}/bin/kind-with-registry.sh" +"${CMD}" -KUBECONFIG="$HOME/.kube/config" PATH="${PWD}/bin:$PATH" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e +KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e