From 8aaa13f9004b9e873a31cb9f2d609502cf32dad8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 25 Jul 2024 18:01:48 -0600 Subject: [PATCH] DRIVERS-2882 Add EKS OIDC Support (#462) Co-authored-by: Noah Stapp --- .evergreen/auth_oidc/k8s/README.md | 116 ++++++++++++++++-- .../k8s/remote-scripts/run-self-test.sh | 10 +- .../k8s/remote-scripts/setup_oidc.js | 20 +++ .../k8s/remote-scripts/start-server.sh | 47 +++++++ .evergreen/auth_oidc/k8s/run-driver-test.sh | 26 +--- .evergreen/auth_oidc/k8s/run-self-test.sh | 22 ++++ .evergreen/auth_oidc/k8s/setup-pod.sh | 20 +++ .evergreen/auth_oidc/k8s/setup.sh | 20 ++- .evergreen/auth_oidc/k8s/start-server.sh | 27 ++++ .evergreen/auth_oidc/k8s/teardown-pod.sh | 17 +++ .evergreen/auth_oidc/k8s/teardown.sh | 12 +- .evergreen/config.yml | 93 ++++++++++++-- .evergreen/ensure-binary.sh | 9 +- .evergreen/k8s/aks/README.md | 20 ++- .evergreen/k8s/aks/setup.sh | 5 +- .evergreen/k8s/configure-pod.sh | 15 ++- .evergreen/k8s/eks/README.md | 20 +++ .evergreen/k8s/eks/setup-cluster.sh | 25 ++++ .evergreen/k8s/eks/setup-secrets.sh | 8 ++ .evergreen/k8s/eks/setup.sh | 51 ++++++++ .evergreen/k8s/eks/teardown-cluster.sh | 17 +++ .evergreen/k8s/eks/teardown.sh | 13 ++ .evergreen/k8s/gke/README.md | 22 +++- .evergreen/k8s/gke/setup.sh | 15 ++- .evergreen/k8s/remote-scripts/setup-pod.sh | 4 +- 25 files changed, 573 insertions(+), 81 deletions(-) create mode 100644 .evergreen/auth_oidc/k8s/remote-scripts/setup_oidc.js create mode 100755 .evergreen/auth_oidc/k8s/remote-scripts/start-server.sh create mode 100755 .evergreen/auth_oidc/k8s/run-self-test.sh create mode 100755 .evergreen/auth_oidc/k8s/setup-pod.sh create mode 100755 .evergreen/auth_oidc/k8s/start-server.sh create mode 100755 .evergreen/auth_oidc/k8s/teardown-pod.sh create mode 100644 .evergreen/k8s/eks/README.md create mode 100755 .evergreen/k8s/eks/setup-cluster.sh create mode 100755 .evergreen/k8s/eks/setup-secrets.sh create mode 100755 .evergreen/k8s/eks/setup.sh create mode 100755 .evergreen/k8s/eks/teardown-cluster.sh create mode 100755 .evergreen/k8s/eks/teardown.sh diff --git a/.evergreen/auth_oidc/k8s/README.md b/.evergreen/auth_oidc/k8s/README.md index 8971240e..86b76b90 100644 --- a/.evergreen/auth_oidc/k8s/README.md +++ b/.evergreen/auth_oidc/k8s/README.md @@ -1,9 +1,111 @@ -## OIDC on K8S +# OIDC on K8S -- Launch an Atlas cluster -- Wait for the Atlas cluster -- Setup a pod and self-test -- Teardown pod -- Run test on pod +Scripts to manage OIDC integration tests on kubernetes: eks, aks, and gke. -- Repeat pod steps for other variants if desired +## Background + +Uses the scripts in `$DRIVERS_TOOLS/.evergreen/k8s` to launch and configure +a kubernetes pod in the given cloud provider, and allows you to run a driver +test on the pod. + +## Local Usage + +The scripts can be run locally as follows: + +```bash +bash setup.sh local # needs to be done once to set up variables +bash setup-pod.sh aks # or gke, or eks +bash start-server.sh +bash run-self-test.sh +``` + +Or if running tests for a specific driver: + +```bash +bash setup.sh local # needs to be done once to set up variables +bash setup-pod.sh aks # or gke, or eks +bash start-server.sh +pushd $PROJECT_HOME +export K8S_DRIVERS_TAR_FILE=/tmp/driver.tgz +git archive -o $K8S_DRIVERS_TAR_FILE HEAD +export K8S_TEST_CMD="OIDC_PROVIDER_NAME=k8s ./.evergreen/run-mongodb-oidc-test.sh" +popd +bash run-driver-test.sh +``` + +### Local EKS Testing + +Local EKS testing requires assuming a role to interact with the EKS cluster. +See the [Wiki](https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets) for details. + + +## EVG Usage + +The test should use a task group to ensure the resources are cleaned up properly. + +```yaml + - name: test_oidc_k8s_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: prepare resources + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh + teardown_group: + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh + - func: "teardown assets" + - func: "upload logs" + - func: "upload test results" + - func: "cleanup" + tasks: + - test-oidc-k8s +``` + +And should be run for all three variants: + +```yaml +- name: "test-oidc-k8s" + tags: ["latest", "oidc", "pr"] + commands: + - func: "run oidc k8s test" + vars: + VARIANT: eks + - func: "run oidc k8s test" + vars: + VARIANT: gke + - func: "run oidc k8s test" + vars: + VARIANT: aks +``` + +Where the test looks something like: + +```yaml +"run oidc k8s test": +- command: ec2.assume_role +params: + role_arn: ${drivers_test_secrets_role} +- command: shell.exec + type: test + params: + shell: bash + working-directory: "src" + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + script: |- + set -o errexit + export K8S_VARIANT=${VARIANT} + export K8S_DRIVERS_TAR_FILE=/tmp/driver.tgz + git archive -o $K8S_DRIVERS_TAR_FILE HEAD + export K8S_TEST_CMD="OIDC_PROVIDER_NAME=k8s ./.evergreen/run-mongodb-oidc-test.sh" + bash ./.evergreen/auth_oidc/k8s/run-driver-test.sh +``` diff --git a/.evergreen/auth_oidc/k8s/remote-scripts/run-self-test.sh b/.evergreen/auth_oidc/k8s/remote-scripts/run-self-test.sh index da52d5ce..24770fe0 100755 --- a/.evergreen/auth_oidc/k8s/remote-scripts/run-self-test.sh +++ b/.evergreen/auth_oidc/k8s/remote-scripts/run-self-test.sh @@ -2,16 +2,14 @@ set -eu echo "Installing dependencies ... begin" -git clone https://github.com/mongodb/mongo-python-driver -pushd mongo-python-driver +rm -rf .venv python3 -m venv .venv source .venv/bin/activate -pip install -U -q pip -pip install . -popd +pip install -U -q pip pymongo echo "Installing dependencies ... end" # Run the Python Driver Self Test -cd /tmp +SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]})) +pushd $SCRIPT_DIR source secrets-export.sh python test.py diff --git a/.evergreen/auth_oidc/k8s/remote-scripts/setup_oidc.js b/.evergreen/auth_oidc/k8s/remote-scripts/setup_oidc.js new file mode 100644 index 00000000..d2725709 --- /dev/null +++ b/.evergreen/auth_oidc/k8s/remote-scripts/setup_oidc.js @@ -0,0 +1,20 @@ +/** + * Set up OIDC auth roles. + */ +(function() { + "use strict"; + + console.log("Setting up Admin User"); + const admin = Mongo().getDB("admin"); + assert(admin.auth("bob", "pwd123")); + + console.log("Setting up OIDC User"); + const authPrefix = process.env['K8S_OIDC_AUTHPREFIX']; + const authClaim = process.env['K8S_OIDC_CLIENT']; + const username = `${authPrefix}/${authClaim}` + + const external = admin.getSiblingDB("$external"); + console.log('Adding user:', username); + external.runCommand({createUser: username, roles:[{role: 'readWrite', db: 'test'}]}); + + }()); diff --git a/.evergreen/auth_oidc/k8s/remote-scripts/start-server.sh b/.evergreen/auth_oidc/k8s/remote-scripts/start-server.sh new file mode 100755 index 00000000..4e9daf77 --- /dev/null +++ b/.evergreen/auth_oidc/k8s/remote-scripts/start-server.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eu + +# Start an OIDC-enabled server. +SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]})) +cd $SCRIPT_DIR +source secrets-export.sh + +mkdir drivers-tools +tar -xf drivers-tools.tgz -C drivers-tools + +export ORCHESTRATION_FILE=auth-oidc.json +export DRIVERS_TOOLS=$SCRIPT_DIR/drivers-tools +export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration +export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration +export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin +export K8S_OIDC_AUTHPREFIX=auth_prefix +export MONGODB_VERSION=8.0 + +OIDC_IDENTITY_PROVIDERS="[{\\\"issuer\\\":\\\"$K8S_OIDC_ISSUER\\\",\\\"audience\\\":\\\"$K8S_OIDC_AUDIENCE\\\",\\\"authNamePrefix\\\":\\\"$K8S_OIDC_AUTHPREFIX\\\",\\\"principalName\\\":\\\"sub\\\",\\\"logClaims\\\":[\\\"aud\\\",\\\"sub\\\"],\\\"JWKSPollSecs\\\":86400,\\\"useAuthorizationClaim\\\":false,\\\"supportsHumanFlows\\\":false}]" + +cat <> $MONGO_ORCHESTRATION_HOME/configs/servers/$ORCHESTRATION_FILE +{ + "id": "oidc", + "auth_key": "secret", + "login": "bob", + "name": "mongod", + "password": "pwd123", + "procParams": { + "ipv6": false, + "bind_ip": "0.0.0.0,::1", + "logappend": true, + "port": 27017, + "setParameter": { + "enableTestCommands": 1, + "authenticationMechanisms": "SCRAM-SHA-1,SCRAM-SHA-256,MONGODB-OIDC", + "oidcIdentityProviders": "$OIDC_IDENTITY_PROVIDERS" + } + } +} +EOF + +cd $DRIVERS_TOOLS/.evergreen +apt-get -yqq install curl lsof +bash run-orchestration.sh + +$MONGODB_BINARIES/mongosh -f $SCRIPT_DIR/setup_oidc.js "mongodb://127.0.0.1:27017/directConnection=true&serverSelectionTimeoutMS=10000" diff --git a/.evergreen/auth_oidc/k8s/run-driver-test.sh b/.evergreen/auth_oidc/k8s/run-driver-test.sh index 04724de6..015fc1bf 100755 --- a/.evergreen/auth_oidc/k8s/run-driver-test.sh +++ b/.evergreen/auth_oidc/k8s/run-driver-test.sh @@ -9,30 +9,17 @@ pushd $SCRIPT_DIR VARLIST=( K8S_DRIVERS_TAR_FILE -K8S_VARIANT K8S_TEST_CMD ) -# Ensure that all variables required to run the test are set, otherwise throw -# an error. -for VARNAME in ${VARLIST[*]}; do -[[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; -done - -# Set the current K8S_VARIANT. -echo "K8S_VARIANT=$K8S_VARIANT" >> secrets-export.sh -VARIANT=$(echo "$K8S_VARIANT" | tr '[:upper:]' '[:lower:]') -VARIANT_DIR=$DRIVERS_TOOLS/.evergreen/k8s/$VARIANT - -# Set up the pod. -echo "Setting up $VARIANT pod..." -. $VARIANT_DIR/setup.sh -echo "Setting up $VARIANT pod... done." +source secrets-export.sh +source $K8S_VARIANT_DIR/secrets-export.sh # Extract the tar file to the /tmp/test directory. echo "Setting up driver test files..." kubectl exec ${K8S_POD_NAME} -- bash -c "rm -rf /tmp/test && mkdir /tmp/test" -tar cf - ${K8S_DRIVERS_TAR_FILE} | kubectl exec -i ${K8S_POD_NAME} -- /bin/sh -c 'tar xf - -C /tmp/test' +kubectl cp ${K8S_DRIVERS_TAR_FILE} ${K8S_POD_NAME}:/tmp/drivers-test.tgz +kubectl exec ${K8S_POD_NAME} -- bash -c "cd /tmp && tar -xf drivers-test.tgz -C test" echo "Setting up driver test files... done." # Run the command. @@ -40,9 +27,4 @@ echo "Running the driver test command..." kubectl exec ${K8S_POD_NAME} -- bash -c "cd /tmp/test && ${K8S_TEST_CMD}" echo "Running the driver test command... done." -# Tear down the pod. -echo "Tearding down $VARIANT pod..." -. $VARIANT_DIR/teardown.sh -echo "Tearding down $VARIANT pod... done." - popd diff --git a/.evergreen/auth_oidc/k8s/run-self-test.sh b/.evergreen/auth_oidc/k8s/run-self-test.sh new file mode 100755 index 00000000..19ecbc87 --- /dev/null +++ b/.evergreen/auth_oidc/k8s/run-self-test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail +set -o nounset + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../../handle-paths.sh +pushd $SCRIPT_DIR + +source secrets-export.sh +source $K8S_VARIANT_DIR/secrets-export.sh + +# Run the self test. +echo "Running self test on $K8S_VARIANT..." +kubectl exec ${K8S_POD_NAME} -- bash -c "rm -rf /tmp/test && mkdir /tmp/test" +kubectl cp ./remote-scripts/run-self-test.sh ${K8S_POD_NAME}:/tmp/test/run-self-test.sh +kubectl cp ./remote-scripts/test.py ${K8S_POD_NAME}:/tmp/test/test.py +kubectl cp ./secrets-export.sh ${K8S_POD_NAME}:/tmp/test/secrets-export.sh +kubectl exec ${K8S_POD_NAME} -- /tmp/test/run-self-test.sh +echo "Running self test on $K8S_VARIANT... done." + +popd diff --git a/.evergreen/auth_oidc/k8s/setup-pod.sh b/.evergreen/auth_oidc/k8s/setup-pod.sh new file mode 100755 index 00000000..6a878782 --- /dev/null +++ b/.evergreen/auth_oidc/k8s/setup-pod.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eu + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../../handle-paths.sh +pushd $SCRIPT_DIR + +# Set the current K8S_VARIANT. +K8S_VARIANT=${K8S_VARIANT:-$1} +echo "export K8S_VARIANT=$K8S_VARIANT" >> secrets-export.sh +VARIANT=$(echo "$K8S_VARIANT" | tr '[:upper:]' '[:lower:]') +VARIANT_DIR=$DRIVERS_TOOLS/.evergreen/k8s/$VARIANT +echo "export K8S_VARIANT_DIR=$VARIANT_DIR" >> secrets-export.sh + +# Set up the pod. +echo "Setting up $VARIANT pod..." +. $VARIANT_DIR/setup.sh +echo "Setting up $VARIANT pod... done." + +popd diff --git a/.evergreen/auth_oidc/k8s/setup.sh b/.evergreen/auth_oidc/k8s/setup.sh index 3bfbb14d..9757fb86 100755 --- a/.evergreen/auth_oidc/k8s/setup.sh +++ b/.evergreen/auth_oidc/k8s/setup.sh @@ -4,10 +4,25 @@ set -o errexit SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../../handle-paths.sh + +rm -f $SCRIPT_DIR/secrets-export.sh + +# If running locally, just set up the variables and exit. +if [ "$1" == "local" ]; then + URI="mongodb://127.0.0.1" + cat <> "$SCRIPT_DIR/secrets-export.sh" +export OIDC_SERVER_TYPE=local +export MONGODB_URI="$URI" +export MONGODB_URI_SINGLE="$URI/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s" +export OIDC_ADMIN_USER=bob +export OIDC_ADMIN_PWD=pwd123 +EOF + exit 0 +fi + pushd $SCRIPT_DIR # Handle secrets from vault. -rm -f secrets-export.sh . ./setup-secrets.sh ######################## @@ -40,7 +55,7 @@ export DEPLOYMENT_DATA=$(cat <> "secrets-export.sh" +export OIDC_SERVER_TYPE=local export MONGODB_URI="$URI" export MONGODB_URI_SINGLE="$URI/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s" export OIDC_ADMIN_USER=$OIDC_ATLAS_USER diff --git a/.evergreen/auth_oidc/k8s/start-server.sh b/.evergreen/auth_oidc/k8s/start-server.sh new file mode 100755 index 00000000..98065ccd --- /dev/null +++ b/.evergreen/auth_oidc/k8s/start-server.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Start an OIDC-enabled server on a kubernetes pod. +set -eu + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../../handle-paths.sh +pushd $SCRIPT_DIR + +source secrets-export.sh +source $K8S_VARIANT_DIR/secrets-export.sh + +# Extract the tar file to the /tmp/test directory. +echo "Setting up server on ${K8S_POD_NAME}..." +kubectl exec ${K8S_POD_NAME} -- bash -c "rm -rf /tmp/server && mkdir /tmp/server/" +K8S_TAR_FILE=/tmp/drivers-tools.tgz +pushd $DRIVERS_TOOLS +git archive -o $K8S_TAR_FILE HEAD +popd +kubectl cp ${K8S_TAR_FILE} ${K8S_POD_NAME}:/tmp/server/drivers-tools.tgz +kubectl cp $K8S_VARIANT_DIR/secrets-export.sh ${K8S_POD_NAME}:/tmp/server/secrets-export.sh +kubectl cp ./remote-scripts/start-server.sh ${K8S_POD_NAME}:/tmp/server/start-server.sh +kubectl cp ./remote-scripts/setup_oidc.js ${K8S_POD_NAME}:/tmp/server/setup_oidc.js +kubectl exec ${K8S_POD_NAME} -- /tmp/server/start-server.sh +echo "Setting up server on ${K8S_POD_NAME}... done." + +popd diff --git a/.evergreen/auth_oidc/k8s/teardown-pod.sh b/.evergreen/auth_oidc/k8s/teardown-pod.sh new file mode 100755 index 00000000..034b183f --- /dev/null +++ b/.evergreen/auth_oidc/k8s/teardown-pod.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail +set -o nounset + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../../handle-paths.sh +pushd $SCRIPT_DIR + +source secrets-export.sh + +# Tear down the pod. +echo "Tearding down $K8S_VARIANT pod..." +. $K8S_VARIANT_DIR/teardown.sh +echo "Tearding down $K8S_VARIANT pod... done." + +popd diff --git a/.evergreen/auth_oidc/k8s/teardown.sh b/.evergreen/auth_oidc/k8s/teardown.sh index 86d08313..a065f82e 100755 --- a/.evergreen/auth_oidc/k8s/teardown.sh +++ b/.evergreen/auth_oidc/k8s/teardown.sh @@ -4,15 +4,17 @@ set -eu SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../../handle-paths.sh -pushd $SCRIPT_DIR # Source the secrets. -source ./secrets-export.sh +source $SCRIPT_DIR/secrets-export.sh + +# If running locally, just exit +if [ "$OIDC_SERVER_TYPE" == "local" ]; then + exit 0 +fi # Tear down the Atlas Cluster export DRIVERS_ATLAS_PUBLIC_API_KEY=$OIDC_ATLAS_PUBLIC_API_KEY export DRIVERS_ATLAS_PRIVATE_API_KEY=$OIDC_ATLAS_PRIVATE_API_KEY export DRIVERS_ATLAS_GROUP_ID=$OIDC_ATLAS_GROUP_ID -bash ../../atlas/teardown-atlas-cluster.sh - -popd +bash $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1335e67c..fb85a9d6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -441,19 +441,47 @@ functions: cd ${PROJECT_DIRECTORY} make test - "run oidc k8s test": + "run oidc k8s atlas test": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: shell.exec type: test params: shell: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: |- set -o errexit cd ${DRIVERS_TOOLS} export K8S_VARIANT=${VARIANT} + bash ./.evergreen/auth_oidc/k8s/setup-pod.sh + bash ./.evergreen/auth_oidc/k8s/run-self-test.sh export K8S_DRIVERS_TAR_FILE=/tmp/drivers-tools.tgz git archive -o $K8S_DRIVERS_TAR_FILE HEAD export K8S_TEST_CMD="echo 'hello'" bash ./.evergreen/auth_oidc/k8s/run-driver-test.sh + bash ./.evergreen/auth_oidc/k8s/teardown-pod.sh + # Generate a test results file + cd ${PROJECT_DIRECTORY} + make test + + "run oidc k8s local test": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + type: test + params: + shell: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + script: |- + set -o errexit + cd ${DRIVERS_TOOLS} + export K8S_VARIANT=${VARIANT} + bash ./.evergreen/auth_oidc/k8s/setup-pod.sh + bash ./.evergreen/auth_oidc/k8s/start-server.sh + bash ./.evergreen/auth_oidc/k8s/run-self-test.sh + bash ./.evergreen/auth_oidc/k8s/teardown-pod.sh # Generate a test results file cd ${PROJECT_DIRECTORY} make test @@ -819,12 +847,12 @@ tasks: - func: "run tests" - name: "test-serverless" - tags: ["serverless", "pr"] + tags: ["serverless"] commands: - func: "run tests" - name: "test-atlas" - tags: ["atlas", "pr"] + tags: ["atlas"] commands: - func: "run tests" @@ -853,24 +881,40 @@ tasks: - func: "run oidc docker test" - name: "test-oidc-azure" - tags: ["oidc", "pr"] + tags: ["latest", "oidc", "pr"] commands: - func: "run oidc azure test" - name: "test-oidc-gcp" - tags: ["latest", "oidc", "pr"] + tags: ["latest", "oidc"] commands: - func: "run oidc gcp test" - - name: "test-oidc-k8s" - tags: ["latest", "oidc", "pr"] + - name: "test-oidc-k8s-atlas" + tags: ["latest", "oidc"] commands: - - func: "run oidc k8s test" + - func: "run oidc k8s atlas test" + vars: + VARIANT: eks + - func: "run oidc k8s atlas test" vars: VARIANT: gke - - func: "run oidc k8s test" + # - func: "run oidc k8s atlas test" + # vars: + # VARIANT: aks + + - name: "test-oidc-k8s-local" + tags: ["latest", "oidc", "pr"] + commands: + - func: "run oidc k8s local test" vars: - VARIANT: aks + VARIANT: eks + - func: "run oidc k8s local test" + vars: + VARIANT: gke + # - func: "run oidc k8s local test" + # vars: + # VARIANT: aks # }}} @@ -1000,7 +1044,7 @@ task_groups: tasks: - test-oidc-gcp - - name: testk8soidc_task_group + - name: test_oidc_k8s_atlas_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true @@ -1024,7 +1068,29 @@ task_groups: - func: "upload test results" - func: "cleanup" tasks: - - test-oidc-k8s + - "test-oidc-k8s-atlas" + + - name: test_oidc_k8s_local_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: prepare resources + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh + - local + teardown_group: + - func: "teardown assets" + - func: "upload logs" + - func: "upload test results" + - func: "cleanup" + tasks: + - "test-oidc-k8s-local" axes: - id: versions @@ -1209,7 +1275,8 @@ buildvariants: - "test-oidc-docker" - "testgcpoidc_task_group" - "testazureoidc_task_group" - - "testk8soidc_task_group" + - "test_oidc_k8s_atlas_task_group" + - "test_oidc_k8s_local_task_group" - matrix_name: "tests-all" matrix_spec: {"os-fully-featured": "*", auth: "*", ssl: "*" } diff --git a/.evergreen/ensure-binary.sh b/.evergreen/ensure-binary.sh index 0d1da7c4..ceccd920 100755 --- a/.evergreen/ensure-binary.sh +++ b/.evergreen/ensure-binary.sh @@ -15,11 +15,6 @@ if [ -z "$DRIVERS_TOOLS" ]; then return 1 fi -# Google cloud gets special handling. -if [ "$NAME" == "gcloud" ]; then - PATH="$PATH:/tmp/google-cloud-sdk/bin" -fi - if command -v $NAME &> /dev/null; then echo "$NAME found in PATH!" return 0 @@ -80,14 +75,14 @@ if [ "$NAME" != "gcloud" ]; then chmod +x $TARGET else - # Google Cloud needs special handling: the bin dir must be added to PATH. + # Google Cloud needs special handling: we need a symlink to the source location. pushd /tmp rm -rf google-cloud-sdk FNAME=/tmp/google-cloud-sdk.tgz curl -L -s $URL -o $FNAME || curl -L $URL -o $FNAME tar xfz $FNAME - PATH="$PATH:/tmp/google-cloud-sdk/bin" popd + ln -s /tmp/google-cloud-sdk/bin/gcloud $DRIVERS_TOOLS/.bin/gcloud fi echo "Installing $NAME... done." diff --git a/.evergreen/k8s/aks/README.md b/.evergreen/k8s/aks/README.md index cfc8c695..58fb203f 100644 --- a/.evergreen/k8s/aks/README.md +++ b/.evergreen/k8s/aks/README.md @@ -1,7 +1,19 @@ -# +# AKS Cluster Management -## AKS Cluster Setup +Scripts to manage a drivers test cluster on Azure. + +## Cluster Management + +These steps must be done by an account with admin access (one time): 1. Create an App Registration -2. Store the secrets in the vault -3. Run setup-cluster.sh +2. Store the secrets in the AWS vault +3. Run `setup-cluster.sh` + +## Usage + +These steps can be run using the drivers test secrets role: + +1. Run `setup.sh` +2. Run the desired tests in the pod. +3. Run `teardown.sh` diff --git a/.evergreen/k8s/aks/setup.sh b/.evergreen/k8s/aks/setup.sh index c30d135e..5155e976 100755 --- a/.evergreen/k8s/aks/setup.sh +++ b/.evergreen/k8s/aks/setup.sh @@ -20,7 +20,7 @@ fi az aks get-credentials --overwrite-existing -n "${AKS_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" # Create the pod with a random name. -POD_NAME="test-$RANDOM" +POD_NAME="test-aks-$RANDOM" echo "export K8S_POD_NAME=$POD_NAME" >> ./secrets-export.sh export K8S_POD_NAME=$POD_NAME @@ -33,11 +33,12 @@ metadata: namespace: ${AKS_SERVICE_ACCOUNT_NAMESPACE} labels: azure.workload.identity/use: "true" + app: test-pod spec: serviceAccountName: ${AKS_SERVICE_ACCOUNT_NAME} containers: - name: debian - image: debian:11 + image: debian:12 command: ["/bin/sleep", "3650d"] imagePullPolicy: IfNotPresent diff --git a/.evergreen/k8s/configure-pod.sh b/.evergreen/k8s/configure-pod.sh index 5196619f..106214f7 100755 --- a/.evergreen/k8s/configure-pod.sh +++ b/.evergreen/k8s/configure-pod.sh @@ -13,15 +13,26 @@ fi . $DRIVERS_TOOLS/.evergreen/ensure-binary.sh kubectl # Delete pods over one hour old in case they were not torn down. -kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' | awk '$2 <= "'$(date -d'now-1 hours' -Ins --utc | sed 's/+0000/Z/')'" { print $1 }' | xargs --no-run-if-empty kubectl delete pod +echo "Deleting old pods..." +# Delete successful pods more than an hour old. +kubectl get pods -l app=test-pod -o go-template --template '{{range .items}}{{.metadata.name}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' | awk '$2 <= "'$(date -d'now-1 hours' -Ins --utc | sed 's/+0000/Z/')'" { print $1 }' | xargs --no-run-if-empty kubectl delete pod +# Delete pending (stuck) pods more than 5 minutes old. +kubectl get pods --all-namespaces -l app=test-pod --field-selector=status.phase=Pending -o json | jq '.items[] | select((now - (.metadata.creationTimestamp | fromdateiso8601)) > 600) | .metadata.name' | xargs -I{} kubectl delete pod {} --force --grace-period=0 +echo "Deleting old pods... done." # Wait for the new pod to be ready. -kubectl wait --for=condition=Ready pod/${POD_NAME} --timeout=600s +echo "Waiting for pod to be ready..." +kubectl wait --for=condition=Ready pod/${POD_NAME} --timeout=2000s +kubectl get pods +echo "Waiting for pod to be ready... done." # Run the setup script and ensure git was installed. +echo "Configuring pod $POD_NAME..." set -x kubectl cp ./remote-scripts/setup-pod.sh ${POD_NAME}:/tmp/setup-pod.sh kubectl exec ${POD_NAME} -- /tmp/setup-pod.sh kubectl exec ${POD_NAME} -- git --version +set +x +echo "Configuring pod $POD_NAME... done." popd diff --git a/.evergreen/k8s/eks/README.md b/.evergreen/k8s/eks/README.md new file mode 100644 index 00000000..cbb1c973 --- /dev/null +++ b/.evergreen/k8s/eks/README.md @@ -0,0 +1,20 @@ +# EKS Cluster Management + +Scripts to manage a drivers test cluster on AWS. + +## Cluster Management + +These steps must be done by an account with admin access (one time): + +1. Run `setup-cluster.sh` +2. Set up an [access entry](https://docs.aws.amazon.com/eks/latest/userguide/access-entries.html) for the + drivers test secrets role. +3. Store the secrets in the AWS vault. + +## Usage + +These steps can be run using the drivers test secrets role: + +1. Run `setup.sh` +2. Run the desired tests in the pod. +3. Run `teardown.sh` diff --git a/.evergreen/k8s/eks/setup-cluster.sh b/.evergreen/k8s/eks/setup-cluster.sh new file mode 100755 index 00000000..629db8c3 --- /dev/null +++ b/.evergreen/k8s/eks/setup-cluster.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eux + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../../handle-paths.sh + +if [ -f ./secrets-export.sh ]; then + echo "Sourcing secrets" + source ./secrets-export.sh +fi +if [ -z "${EKS_CLUSTER_NAME:-}" ]; then + . ./../../secrets_handling/setup-secrets.sh drivers/eks +fi + +eksctl create cluster --name $EKS_CLUSTER_NAME --zones "${EKS_REGION}a,${EKS_REGION}b" +kubectl config set-context --current +eksctl utils associate-iam-oidc-provider --cluster $EKS_CLUSTER_NAME --approve +eksctl create iamserviceaccount \ + --name $EKS_SERVICE_ACCOUNT_NAME \ + --cluster $EKS_CLUSTER_NAME \ + --role-name $EKS_IAM_ROLE_NAME \ + --attach-policy-arn $EKS_POLICY_ARN \ + --approve \ + --override-existing-serviceaccounts diff --git a/.evergreen/k8s/eks/setup-secrets.sh b/.evergreen/k8s/eks/setup-secrets.sh new file mode 100755 index 00000000..f3bb8790 --- /dev/null +++ b/.evergreen/k8s/eks/setup-secrets.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../handle-paths.sh +pushd $SCRIPT_DIR +. $SCRIPT_DIR/../secrets_handling/setup-secrets.sh drivers/oidc +popd diff --git a/.evergreen/k8s/eks/setup.sh b/.evergreen/k8s/eks/setup.sh new file mode 100755 index 00000000..2c9403bb --- /dev/null +++ b/.evergreen/k8s/eks/setup.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eux + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../../handle-paths.sh +pushd $SCRIPT_DIR + +# Handle secrets from vault. +if [ -f ./secrets-export.sh ]; then + echo "Sourcing secrets" + source ./secrets-export.sh +fi +if [ -z "${EKS_CLUSTER_NAME:-}" ]; then + . ./../../secrets_handling/setup-secrets.sh drivers/eks +fi + +# Set up kubectl creds. +. $DRIVERS_TOOLS/.evergreen/ensure-binary.sh kubectl +aws eks update-kubeconfig --region $EKS_REGION --name $EKS_CLUSTER_NAME + +# Create the pod with a random name. +POD_NAME="test-eks-$RANDOM" +echo "export K8S_POD_NAME=$POD_NAME" >> ./secrets-export.sh +export K8S_POD_NAME=$POD_NAME + +. $DRIVERS_TOOLS/.evergreen/ensure-binary.sh kubectl +cat < $GKE_KEYFILE chmod 600 $GKE_KEYFILE gcloud auth activate-service-account --key-file $GKE_KEYFILE gcloud components install --quiet gke-gcloud-auth-plugin -gcloud container clusters get-credentials $GKE_CLUSTER_NAME --region $GKE_REGION --project $GKE_PROJECT +gcloud container clusters get-credentials $GKE_CLUSTER_NAME --region ${GKE_REGION} --project $GKE_PROJECT # Create the pod with a random name. -POD_NAME="test-$RANDOM" +POD_NAME="test-gke-$RANDOM" echo "export K8S_POD_NAME=$POD_NAME" >> ./secrets-export.sh export K8S_POD_NAME=$POD_NAME @@ -39,17 +39,24 @@ kind: Pod metadata: name: ${POD_NAME} namespace: default + labels: + app: test-pod spec: containers: - name: debian - image: debian:11 + image: debian:12 + resources: + limits: + memory: "2Gi" + cpu: "1" + ephemeral-storage: "2Gi" command: ["/bin/sleep", "3650d"] imagePullPolicy: IfNotPresent nodeSelector: kubernetes.io/os: linux EOF -# Set up the pod. +# Set up the pod - run directly so PATH is passed in. bash $DRIVERS_TOOLS/.evergreen/k8s/configure-pod.sh ${POD_NAME} popd diff --git a/.evergreen/k8s/remote-scripts/setup-pod.sh b/.evergreen/k8s/remote-scripts/setup-pod.sh index ffbdc900..ba36529e 100755 --- a/.evergreen/k8s/remote-scripts/setup-pod.sh +++ b/.evergreen/k8s/remote-scripts/setup-pod.sh @@ -5,7 +5,7 @@ echo "Installing dependencies ... begin" apt-get -qq update # Same dependencies used in KMS testing. export DEBIAN_FRONTEND=noninteractive -apt-get -qq -y -o DPkg::Lock::Timeout=-1 install libcurl4 libgssapi-krb5-2 libldap-2.4-2 libwrap0 libsasl2-2 \ +apt-get -qq -y -o DPkg::Lock::Timeout=-1 install libcurl4 libgssapi-krb5-2 libldap-common libwrap0 libsasl2-2 \ libsasl2-modules-gssapi-mit snmp openssl liblzma5 \ - python3-pip python3.9-venv git sudo < /dev/null > /dev/null + python3-pip python3.11-venv git sudo < /dev/null > /dev/null echo "Installing dependencies ... end"