From 934d00b21d32265eb240dcc196eae8426882ec70 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sat, 21 Dec 2024 04:59:50 -0800 Subject: [PATCH 1/3] [testing] Always use the go.mod version of ginkgo Previously, ginkgo was installed with an explicit version which made it easy for the ginkgo command and the test code to be using different versions. Using `go run` without a tag ensures that the go.mod version of ginkgo is always used for the command. --- scripts/ginkgo.sh | 6 ++++++ scripts/tests.e2e.bootstrap_monitor.sh | 6 ++---- scripts/tests.e2e.existing.sh | 2 +- scripts/tests.e2e.sh | 9 +-------- scripts/tests.upgrade.sh | 10 +--------- tests/e2e/README.md | 26 +++++++++----------------- 6 files changed, 20 insertions(+), 39 deletions(-) create mode 100755 scripts/ginkgo.sh diff --git a/scripts/ginkgo.sh b/scripts/ginkgo.sh new file mode 100755 index 000000000000..9beef8d73cec --- /dev/null +++ b/scripts/ginkgo.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Run the ginkgo version from go.mod +go run github.com/onsi/ginkgo/v2/ginkgo "${@}" diff --git a/scripts/tests.e2e.bootstrap_monitor.sh b/scripts/tests.e2e.bootstrap_monitor.sh index e0742f9760ac..67da11c373e7 100755 --- a/scripts/tests.e2e.bootstrap_monitor.sh +++ b/scripts/tests.e2e.bootstrap_monitor.sh @@ -63,7 +63,5 @@ ensure_command "kind-with-registry.sh" "https://raw.githubusercontent.com/kubern # call them without a qualifying path. PATH="${PWD}/bin:$PATH" bash -x "${PWD}/bin/kind-with-registry.sh" -# TODO(marun) Factor out ginkgo installation to avoid duplicating it across test scripts -go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.13.1 - -KUBECONFIG="$HOME/.kube/config" PATH="${PWD}/bin:$PATH" ginkgo -v ./tests/fixture/bootstrapmonitor/e2e +KUBECONFIG="$HOME/.kube/config" PATH="${PWD}/bin:$PATH" \ + ./scripts/ginkgo.sh -v ./tests/fixture/bootstrapmonitor/e2e diff --git a/scripts/tests.e2e.existing.sh b/scripts/tests.e2e.existing.sh index 4b28fc1ad271..6cc01a9b44eb 100755 --- a/scripts/tests.e2e.existing.sh +++ b/scripts/tests.e2e.existing.sh @@ -22,7 +22,7 @@ function print_separator { function cleanup { print_separator echo "cleaning up reusable network" - ginkgo -v ./tests/e2e/e2e.test -- --stop-network + ./scripts/ginkgo.sh -v ./tests/e2e -- --stop-network } trap cleanup EXIT diff --git a/scripts/tests.e2e.sh b/scripts/tests.e2e.sh index 86de18359e8b..e1af3694e90c 100755 --- a/scripts/tests.e2e.sh +++ b/scripts/tests.e2e.sh @@ -20,13 +20,6 @@ fi # the instructions to build non-portable BLST. source ./scripts/constants.sh -################################# -echo "building e2e.test" -# to install the ginkgo binary (required for test build and run) -go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.13.1 -ACK_GINKGO_RC=true ginkgo build ./tests/e2e -./tests/e2e/e2e.test --help - # Enable subnet testing by building xsvm ./scripts/build_xsvm.sh echo "" @@ -66,4 +59,4 @@ fi ################################# # shellcheck disable=SC2086 -ginkgo ${GINKGO_ARGS} -v ./tests/e2e/e2e.test -- "${E2E_ARGS[@]}" "${@}" +./scripts/ginkgo.sh ${GINKGO_ARGS} -v ./tests/e2e -- "${E2E_ARGS[@]}" "${@}" diff --git a/scripts/tests.upgrade.sh b/scripts/tests.upgrade.sh index 64e53c6537ad..1f94d30f3121 100755 --- a/scripts/tests.upgrade.sh +++ b/scripts/tests.upgrade.sh @@ -62,17 +62,9 @@ find "/tmp/avalanchego-v${VERSION}" # the instructions to build non-portable BLST. source ./scripts/constants.sh -################################# -echo "building upgrade.test" -# to install the ginkgo binary (required for test build and run) -go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.13.1 -ACK_GINKGO_RC=true ginkgo build ./tests/upgrade -./tests/upgrade/upgrade.test --help - ################################# # By default, it runs all upgrade test cases! echo "running upgrade tests against the local cluster with ${AVALANCHEGO_PATH}" -./tests/upgrade/upgrade.test \ - --ginkgo.v \ +./scripts/ginkgo.sh -v ./tests/upgrade -- \ --avalanchego-path="/tmp/avalanchego-v${VERSION}/avalanchego" \ --avalanchego-path-to-upgrade-to="${AVALANCHEGO_PATH}" diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 187623893681..6b0770961276 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -6,12 +6,9 @@ ## Running tests ```bash -go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.0.0 -ACK_GINKGO_RC=true ginkgo build ./tests/e2e -./tests/e2e/e2e.test --help - -./tests/e2e/e2e.test \ ---avalanchego-path=./build/avalanchego +./scripts/build.sh # Builds avalanchego for use in deploying a test network +./scripts/build_xsvm.sh # Builds xsvm for use in deploying a test network with a subnet +./scripts/ginkgo.sh -v ./tests/e2e -- --avalanchego-path=./build/avalanchego ``` See [`tests.e2e.sh`](../../scripts/tests.e2e.sh) for an example. @@ -27,9 +24,7 @@ primarily target the X-Chain: ```bash -./tests/e2e/e2e.test \ - --avalanchego-path=./build/avalanchego \ - --ginkgo.label-filter=x +./scripts/ginkgo.sh -v --label-filter=x ./tests/e2e -- --avalanchego-path=./build/avalanchego ``` The ginkgo docs provide further detail on [how to compose label @@ -45,17 +40,14 @@ Create a new package to implement feature-specific tests, or add tests to an exi tests └── e2e ├── README.md - ├── e2e.go ├── e2e_test.go └── x └── transfer.go └── virtuous.go ``` -`e2e.go` defines common configuration for other test -packages. `x/transfer/virtuous.go` defines X-Chain transfer tests, -labeled with `x`, which can be selected by `./tests/e2e/e2e.test ---ginkgo.label-filter "x"`. +`x/transfer/virtuous.go` defines X-Chain transfer tests, +labeled with `x`, which can be selected by `--label-filter=x`. ## Reusing temporary networks @@ -70,7 +62,7 @@ To enable network reuse across test runs, pass `--reuse-network` as an argument to the test suite: ```bash -ginkgo -v ./tests/e2e -- --avalanchego-path=/path/to/avalanchego --reuse-network +./scripts/gingko.sh -v ./tests/e2e -- --avalanchego-path=/path/to/avalanchego --reuse-network ``` If a network is not already running the first time the suite runs with @@ -93,7 +85,7 @@ To stop a network configured for reuse, invoke the test suite with the immediately without executing any tests: ```bash -ginkgo -v ./tests/e2e -- --stop-network +./scripts/gingko.sh -v ./tests/e2e -- --stop-network ``` ## Skipping bootstrap checks @@ -105,5 +97,5 @@ these bootstrap checks during development, set the `E2E_SKIP_BOOTSTRAP_CHECKS` env var to a non-empty value: ```bash -E2E_SKIP_BOOTSTRAP_CHECKS=1 ginkgo -v ./tests/e2e ... +E2E_SKIP_BOOTSTRAP_CHECKS=1 ./scripts/ginkgo.sh -v ./tests/e2e ... ``` From 766bb37f7198f58e45fa76e6d48f94d9811bb047 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sun, 22 Dec 2024 19:16:14 -0800 Subject: [PATCH 2/3] [testing] Add `av` command to simplify cross-repo tool maintenance --- .envrc | 8 ++ .gitignore | 2 + scripts/ginkgo.sh | 3 +- tests/av/cmd/main.go | 172 +++++++++++++++++++++++++++++++++++++++++++ tools/av | 17 +++++ tools/ginkgo | 6 ++ tools/kubectl | 6 ++ tools/tmpnetctl | 6 ++ 8 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 .envrc create mode 100644 tests/av/cmd/main.go create mode 100755 tools/av create mode 100755 tools/ginkgo create mode 100755 tools/kubectl create mode 100755 tools/tmpnetctl diff --git a/.envrc b/.envrc new file mode 100644 index 000000000000..4a59461f0cbe --- /dev/null +++ b/.envrc @@ -0,0 +1,8 @@ +# TODO(marun) Document usage of direnv + +# contains manually-installed tooling +PATH_add bin +# contains avalanchego and xsvm binaries +PATH_add build +# contains tool wrappers +PATH_add tools diff --git a/.gitignore b/.gitignore index c5a3cad32b13..9fd544a5af0e 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,5 @@ tests/upgrade/upgrade.test vendor **/testdata + +.tool-downloads diff --git a/scripts/ginkgo.sh b/scripts/ginkgo.sh index 9beef8d73cec..02aade791234 100755 --- a/scripts/ginkgo.sh +++ b/scripts/ginkgo.sh @@ -3,4 +3,5 @@ set -euo pipefail # Run the ginkgo version from go.mod -go run github.com/onsi/ginkgo/v2/ginkgo "${@}" +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) +${AVALANCHE_PATH}/scripts/av.sh tool ginkgo "$@" diff --git a/tests/av/cmd/main.go b/tests/av/cmd/main.go new file mode 100644 index 000000000000..f52558a97619 --- /dev/null +++ b/tests/av/cmd/main.go @@ -0,0 +1,172 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/spf13/cobra" + "go.uber.org/zap" + + "github.com/ava-labs/avalanchego/tests" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/perms" +) + +const ( + cmdName = "av" + kubectlVersion = "v1.30.2" +) + +func main() { + var rawLogFormat string + rootCmd := &cobra.Command{ + Use: cmdName, + Short: cmdName + " commands", + } + rootCmd.PersistentFlags().StringVar(&rawLogFormat, "log-format", logging.AutoString, logging.FormatDescription) + + toolCmd := &cobra.Command{ + Use: "tool", + Short: "configured cli tools", + RunE: func(*cobra.Command, []string) error { + return fmt.Errorf("please specify a valid tool name") + }, + } + rootCmd.AddCommand(toolCmd) + + ginkgoCmd := &cobra.Command{ + Use: "ginkgo", + Short: "cli for building and running e2e tests", + RunE: func(c *cobra.Command, args []string) error { + cmdArgs := []string{ + "run", + "github.com/onsi/ginkgo/v2/ginkgo", + } + cmdArgs = append(cmdArgs, args...) + cmd := exec.Command("go", cmdArgs...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + c.SilenceUsage = true + c.SilenceErrors = true + // TODO(marun) Suppress the duplicated 'exit status X' + // caused by passing through the exit code from the + // `go run` subcommand + return cmd.Run() + }, + } + toolCmd.AddCommand(ginkgoCmd) + + tmpnetctlCmd := &cobra.Command{ + Use: "tmpnetctl", + Short: "cli for managing temporary networks", + RunE: func(c *cobra.Command, args []string) error { + cmdArgs := []string{ + "run", + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet/cmd", + } + cmdArgs = append(cmdArgs, args...) + cmd := exec.Command("go", cmdArgs...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + c.SilenceUsage = true + c.SilenceErrors = true + // TODO(marun) Suppress the duplicated 'exit status X' + // caused by passing through the exit code from the + // `go run` subcommand + return cmd.Run() + }, + } + toolCmd.AddCommand(tmpnetctlCmd) + + kubectlCmd := &cobra.Command{ + Use: "kubectl", + Short: "cli for interacting with a kubernetes cluster", + RunE: func(c *cobra.Command, args []string) error { + log, err := tests.LoggerForFormat("", rawLogFormat) + if err != nil { + return err + } + + downloadDir, err := filepath.Abs(".tool-downloads") + if err != nil { + return err + } + + // Ensure the download directory exists + if info, err := os.Stat(downloadDir); errors.Is(err, os.ErrNotExist) { + log.Info("creating tool download directory", + zap.String("dir", downloadDir), + ) + if err := os.MkdirAll(downloadDir, perms.ReadWriteExecute); err != nil { + return err + } + } else if err != nil { + return err + } else if !info.IsDir() { + return fmt.Errorf("download path is not a directory: %s", downloadDir) + } + + var ( + kubectlPath = downloadDir + "/kubectl-" + kubectlVersion + // TODO(marun) Make these dynamic + kubeOS = "linux" + kubeArch = "arm64" + ) + + if _, err := os.Stat(downloadDir); errors.Is(err, os.ErrNotExist) { + // TODO(marun) Maybe use a library to download the binary? + curlArgs := []string{ + "curl -L -o " + kubectlPath + " https://dl.k8s.io/release/" + kubectlVersion + "/bin/" + kubeOS + "/" + kubeArch + "/kubectl", + } + log.Info("downloading kubectl with curl", + zap.Strings("curlArgs", curlArgs), + ) + // Run in a subshell to ensure -L redirects work + curl := exec.Command("bash", append([]string{"-x", "-c"}, curlArgs...)...) + curl.Stdin = os.Stdin + curl.Stdout = os.Stdout + curl.Stderr = os.Stderr + c.SilenceUsage = true + c.SilenceErrors = true + err = curl.Run() + if err != nil { + return err + } + + if err := os.Chmod(kubectlPath, perms.ReadWriteExecute); err != nil { + return err + } + + log.Info("running kubectl for the first time", + zap.String("path", kubectlPath), + ) + } + + kubectl := exec.Command(kubectlPath, args...) + kubectl.Stdin = os.Stdin + kubectl.Stdout = os.Stdout + kubectl.Stderr = os.Stderr + c.SilenceUsage = true + c.SilenceErrors = true + return kubectl.Run() + }, + } + toolCmd.AddCommand(kubectlCmd) + + if err := rootCmd.Execute(); err != nil { + var exitCode int = 1 + if exitError, ok := err.(*exec.ExitError); ok { + exitCode = exitError.ExitCode() + } + os.Exit(exitCode) + } + os.Exit(0) +} diff --git a/tools/av b/tools/av new file mode 100755 index 000000000000..d515b0693ba1 --- /dev/null +++ b/tools/av @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) +cd ${AVALANCHE_PATH} + +# Set the CGO flags to use the portable version of BLST +# +# We use "export" here instead of just setting a bash variable because we need +# to pass this flag to all child processes spawned by the shell. +export CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" +# While CGO_ENABLED doesn't need to be explicitly set, it produces a much more +# clear error due to the default value change in go1.20. +export CGO_ENABLED=1 + +go run ./tests/av/cmd "${@}" diff --git a/tools/ginkgo b/tools/ginkgo new file mode 100755 index 000000000000..c466d2525c04 --- /dev/null +++ b/tools/ginkgo @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TOOLS_DIR="$( dirname "${BASH_SOURCE[0]}" )" +${TOOLS_DIR}/av tool ginkgo "$@" diff --git a/tools/kubectl b/tools/kubectl new file mode 100755 index 000000000000..7c3c59016c88 --- /dev/null +++ b/tools/kubectl @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TOOLS_DIR="$( dirname "${BASH_SOURCE[0]}" )" +${TOOLS_DIR}/av tool kubectl "$@" diff --git a/tools/tmpnetctl b/tools/tmpnetctl new file mode 100755 index 000000000000..4184dba3c81e --- /dev/null +++ b/tools/tmpnetctl @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TOOLS_DIR="$( dirname "${BASH_SOURCE[0]}" )" +${TOOLS_DIR}/av tool tmpnetctl "$@" From 147a44bbbe1b99e6cf30331e742223190471a1cc Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sun, 22 Dec 2024 22:08:06 -0800 Subject: [PATCH 3/3] fixup: Wrapping cli's is trickier than expected - not sure it is ideal to have to wrap commands - how does `go tool` do it? - maybe stick to installing and using the installed version - `go run` makes sense for local commands, but why for ones in different repos? --- scripts/ginkgo.sh | 7 ------- scripts/tests.e2e.bootstrap_monitor.sh | 8 ++------ scripts/tests.e2e.existing.sh | 2 +- scripts/tests.e2e.sh | 10 +++++----- scripts/tests.upgrade.sh | 2 +- tests/e2e/README.md | 6 +++--- 6 files changed, 12 insertions(+), 23 deletions(-) delete mode 100755 scripts/ginkgo.sh diff --git a/scripts/ginkgo.sh b/scripts/ginkgo.sh deleted file mode 100755 index 02aade791234..000000000000 --- a/scripts/ginkgo.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Run the ginkgo version from go.mod -AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) -${AVALANCHE_PATH}/scripts/av.sh tool ginkgo "$@" diff --git a/scripts/tests.e2e.bootstrap_monitor.sh b/scripts/tests.e2e.bootstrap_monitor.sh index 67da11c373e7..e81a3b036f41 100755 --- a/scripts/tests.e2e.bootstrap_monitor.sh +++ b/scripts/tests.e2e.bootstrap_monitor.sh @@ -47,10 +47,6 @@ function ensure_command { fi } -# Ensure the kubectl command is available -KUBECTL_VERSION=v1.30.2 -ensure_command kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl" - # Ensure the kind command is available KIND_VERSION=v0.23.0 ensure_command kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-${OS}-${ARCH}" @@ -61,7 +57,7 @@ ensure_command "kind-with-registry.sh" "https://raw.githubusercontent.com/kubern # Deploy a kind cluster with a local registry. Include the local bin in the path to # ensure locally installed kind and kubectl are available since the script expects to # call them without a qualifying path. -PATH="${PWD}/bin:$PATH" bash -x "${PWD}/bin/kind-with-registry.sh" +PATH="${PWD}/tools:${PWD}/bin:$PATH" bash -x "${PWD}/bin/kind-with-registry.sh" KUBECONFIG="$HOME/.kube/config" PATH="${PWD}/bin:$PATH" \ - ./scripts/ginkgo.sh -v ./tests/fixture/bootstrapmonitor/e2e + ./tools/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e diff --git a/scripts/tests.e2e.existing.sh b/scripts/tests.e2e.existing.sh index 6cc01a9b44eb..188c2bc8c521 100755 --- a/scripts/tests.e2e.existing.sh +++ b/scripts/tests.e2e.existing.sh @@ -22,7 +22,7 @@ function print_separator { function cleanup { print_separator echo "cleaning up reusable network" - ./scripts/ginkgo.sh -v ./tests/e2e -- --stop-network + ./tools/ginkgo -v ./tests/e2e -- --stop-network } trap cleanup EXIT diff --git a/scripts/tests.e2e.sh b/scripts/tests.e2e.sh index e1af3694e90c..8683e615911c 100755 --- a/scripts/tests.e2e.sh +++ b/scripts/tests.e2e.sh @@ -31,7 +31,7 @@ E2E_ARGS="--avalanchego-path=${AVALANCHEGO_PATH}" ################################# # Determine ginkgo args -GINKGO_ARGS="" +GINKGO_ARGS="--ginkgo.v" if [[ -n "${E2E_SERIAL:-}" ]]; then # Specs will be executed serially. This supports running e2e tests in CI # where parallel execution of tests that start new nodes beyond the @@ -46,17 +46,17 @@ else # since the test binary isn't capable of executing specs in # parallel. echo "tests will be executed in parallel" - GINKGO_ARGS="-p" + GINKGO_ARGS=+" --ginkgo.p" fi # Reference: https://onsi.github.io/ginkgo/#spec-randomization if [[ -n "${E2E_RANDOM_SEED:-}" ]]; then # Supply a specific seed to simplify reproduction of test failures - GINKGO_ARGS+=" --seed=${E2E_RANDOM_SEED}" + GINKGO_ARGS+=" --ginkgo.seed=${E2E_RANDOM_SEED}" else # Execute in random order to identify unwanted dependency - GINKGO_ARGS+=" --randomize-all" + GINKGO_ARGS+=" --ginkgo.randomize-all" fi ################################# # shellcheck disable=SC2086 -./scripts/ginkgo.sh ${GINKGO_ARGS} -v ./tests/e2e -- "${E2E_ARGS[@]}" "${@}" +./tools/ginkgo ./tests/e2e -- ${GINKGO_ARGS} "${E2E_ARGS[@]}" "${@}" diff --git a/scripts/tests.upgrade.sh b/scripts/tests.upgrade.sh index 1f94d30f3121..829d187cd294 100755 --- a/scripts/tests.upgrade.sh +++ b/scripts/tests.upgrade.sh @@ -65,6 +65,6 @@ source ./scripts/constants.sh ################################# # By default, it runs all upgrade test cases! echo "running upgrade tests against the local cluster with ${AVALANCHEGO_PATH}" -./scripts/ginkgo.sh -v ./tests/upgrade -- \ +./tools/ginkgo -v ./tests/upgrade -- \ --avalanchego-path="/tmp/avalanchego-v${VERSION}/avalanchego" \ --avalanchego-path-to-upgrade-to="${AVALANCHEGO_PATH}" diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 6b0770961276..ae8006a6af3d 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -8,7 +8,7 @@ ```bash ./scripts/build.sh # Builds avalanchego for use in deploying a test network ./scripts/build_xsvm.sh # Builds xsvm for use in deploying a test network with a subnet -./scripts/ginkgo.sh -v ./tests/e2e -- --avalanchego-path=./build/avalanchego +./tools/ginkgo ./tests/e2e -- --ginkgo.v --avalanchego-path=./build/avalanchego ``` See [`tests.e2e.sh`](../../scripts/tests.e2e.sh) for an example. @@ -24,7 +24,7 @@ primarily target the X-Chain: ```bash -./scripts/ginkgo.sh -v --label-filter=x ./tests/e2e -- --avalanchego-path=./build/avalanchego +./tools/ginkgo ./tests/e2e -- --ginkgo.v --ginkgo.label-filter=x --avalanchego-path=./build/avalanchego ``` The ginkgo docs provide further detail on [how to compose label @@ -97,5 +97,5 @@ these bootstrap checks during development, set the `E2E_SKIP_BOOTSTRAP_CHECKS` env var to a non-empty value: ```bash -E2E_SKIP_BOOTSTRAP_CHECKS=1 ./scripts/ginkgo.sh -v ./tests/e2e ... +E2E_SKIP_BOOTSTRAP_CHECKS=1 ./tools/ginkgo ./tests/e2e ... ```