DRIVERS-2882 Add EKS OIDC Support (mongodb-labs#462)

Co-authored-by: Noah Stapp <[email protected]>

.evergreen/auth_oidc/k8s/README.md

@@ -1,9 +1,111 @@
-## OIDC on K8S
+# OIDC on K8S
 
-- Launch an Atlas cluster
-- Wait for the Atlas cluster
-- Setup a pod and self-test
-- Teardown pod
-- Run test on pod
+Scripts to manage OIDC integration tests on kubernetes: eks, aks, and gke.
 
-- Repeat pod steps for other variants if desired
+## Background
+
+Uses the scripts in `$DRIVERS_TOOLS/.evergreen/k8s` to launch and configure
+a kubernetes pod in the given cloud provider, and allows you to run a driver
+test on the pod.
+
+## Local Usage
+
+The scripts can be run locally as follows:
+
+```bash
+bash setup.sh local  # needs to be done once to set up variables
+bash setup-pod.sh aks  # or gke, or eks
+bash start-server.sh
+bash run-self-test.sh
+```
+
+Or if running tests for a specific driver:
+
+```bash
+bash setup.sh local  # needs to be done once to set up variables
+bash setup-pod.sh aks  # or gke, or eks
+bash start-server.sh
+pushd $PROJECT_HOME
+export K8S_DRIVERS_TAR_FILE=/tmp/driver.tgz
+git archive -o $K8S_DRIVERS_TAR_FILE HEAD
+export K8S_TEST_CMD="OIDC_PROVIDER_NAME=k8s ./.evergreen/run-mongodb-oidc-test.sh"
+popd
+bash run-driver-test.sh
+```
+
+### Local EKS Testing
+
+Local EKS testing requires assuming a role to interact with the EKS cluster.
+See the [Wiki](https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets) for details.
+
+
+## EVG Usage
+
+The test should use a task group to ensure the resources are cleaned up properly.
+
+```yaml
+  - name: test_oidc_k8s_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    teardown_task_can_fail_task: true
+    teardown_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh
+    teardown_group:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh
+      - func: "teardown assets"
+      - func: "upload logs"
+      - func: "upload test results"
+      - func: "cleanup"
+    tasks:
+      - test-oidc-k8s
+```
+
+And should be run for all three variants:
+
+```yaml
+- name: "test-oidc-k8s"
+    tags: ["latest", "oidc", "pr"]
+    commands:
+    - func: "run oidc k8s test"
+        vars:
+        VARIANT: eks
+    - func: "run oidc k8s test"
+        vars:
+        VARIANT: gke
+    - func: "run oidc k8s test"
+        vars:
+        VARIANT: aks
+```
+
+Where the test looks something like:
+
+```yaml
+"run oidc k8s test":
+- command: ec2.assume_role
+params:
+    role_arn: ${drivers_test_secrets_role}
+- command: shell.exec
+    type: test
+    params:
+    shell: bash
+    working-directory: "src"
+    include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+    script: |-
+        set -o errexit
+        export K8S_VARIANT=${VARIANT}
+        export K8S_DRIVERS_TAR_FILE=/tmp/driver.tgz
+        git archive -o $K8S_DRIVERS_TAR_FILE HEAD
+        export K8S_TEST_CMD="OIDC_PROVIDER_NAME=k8s ./.evergreen/run-mongodb-oidc-test.sh"
+        bash ./.evergreen/auth_oidc/k8s/run-driver-test.sh
+```

.evergreen/auth_oidc/k8s/remote-scripts/run-self-test.sh

@@ -2,16 +2,14 @@
 set -eu
 
 echo "Installing dependencies ... begin"
-git clone https://github.com/mongodb/mongo-python-driver
-pushd mongo-python-driver
+rm -rf .venv
 python3 -m venv .venv
 source .venv/bin/activate
-pip install -U -q pip
-pip install .
-popd
+pip install -U -q pip pymongo
 echo "Installing dependencies ... end"
 
 # Run the Python Driver Self Test
-cd /tmp
+SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]}))
+pushd $SCRIPT_DIR
 source secrets-export.sh
 python test.py

.evergreen/auth_oidc/k8s/remote-scripts/setup_oidc.js

@@ -0,0 +1,20 @@
+/**
+ * Set up OIDC auth roles.
+ */
+(function() {
+    "use strict";
+
+    console.log("Setting up Admin User");
+    const admin = Mongo().getDB("admin");
+    assert(admin.auth("bob", "pwd123"));
+
+    console.log("Setting up OIDC User");
+    const authPrefix = process.env['K8S_OIDC_AUTHPREFIX'];
+    const authClaim = process.env['K8S_OIDC_CLIENT'];
+    const username = `${authPrefix}/${authClaim}`
+
+    const external = admin.getSiblingDB("$external");
+    console.log('Adding user:', username);
+    external.runCommand({createUser: username, roles:[{role: 'readWrite', db: 'test'}]});
+
+    }());

.evergreen/auth_oidc/k8s/remote-scripts/start-server.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -eu
+
+# Start an OIDC-enabled server.
+SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]}))
+cd $SCRIPT_DIR
+source secrets-export.sh
+
+mkdir drivers-tools
+tar -xf drivers-tools.tgz -C drivers-tools
+
+export ORCHESTRATION_FILE=auth-oidc.json
+export DRIVERS_TOOLS=$SCRIPT_DIR/drivers-tools
+export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
+export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
+export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin
+export K8S_OIDC_AUTHPREFIX=auth_prefix
+export MONGODB_VERSION=8.0
+
+OIDC_IDENTITY_PROVIDERS="[{\\\"issuer\\\":\\\"$K8S_OIDC_ISSUER\\\",\\\"audience\\\":\\\"$K8S_OIDC_AUDIENCE\\\",\\\"authNamePrefix\\\":\\\"$K8S_OIDC_AUTHPREFIX\\\",\\\"principalName\\\":\\\"sub\\\",\\\"logClaims\\\":[\\\"aud\\\",\\\"sub\\\"],\\\"JWKSPollSecs\\\":86400,\\\"useAuthorizationClaim\\\":false,\\\"supportsHumanFlows\\\":false}]"
+
+cat <<EOF >> $MONGO_ORCHESTRATION_HOME/configs/servers/$ORCHESTRATION_FILE
+{
+    "id": "oidc",
+    "auth_key": "secret",
+    "login": "bob",
+    "name": "mongod",
+    "password": "pwd123",
+    "procParams": {
+        "ipv6": false,
+        "bind_ip": "0.0.0.0,::1",
+        "logappend": true,
+        "port": 27017,
+        "setParameter": {
+            "enableTestCommands": 1,
+            "authenticationMechanisms": "SCRAM-SHA-1,SCRAM-SHA-256,MONGODB-OIDC",
+            "oidcIdentityProviders": "$OIDC_IDENTITY_PROVIDERS"
+        }
+    }
+}
+EOF
+
+cd $DRIVERS_TOOLS/.evergreen
+apt-get -yqq install curl lsof
+bash run-orchestration.sh
+
+$MONGODB_BINARIES/mongosh -f $SCRIPT_DIR/setup_oidc.js "mongodb://127.0.0.1:27017/directConnection=true&serverSelectionTimeoutMS=10000"

.evergreen/auth_oidc/k8s/run-driver-test.sh

@@ -9,40 +9,22 @@ pushd $SCRIPT_DIR
 
 VARLIST=(
 K8S_DRIVERS_TAR_FILE
-K8S_VARIANT
 K8S_TEST_CMD
 )
 
-# Ensure that all variables required to run the test are set, otherwise throw
-# an error.
-for VARNAME in ${VARLIST[*]}; do
-[[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1;
-done
-
-# Set the current K8S_VARIANT.
-echo "K8S_VARIANT=$K8S_VARIANT" >> secrets-export.sh
-VARIANT=$(echo "$K8S_VARIANT" | tr '[:upper:]' '[:lower:]')
-VARIANT_DIR=$DRIVERS_TOOLS/.evergreen/k8s/$VARIANT
-
-# Set up the pod.
-echo "Setting up $VARIANT pod..."
-. $VARIANT_DIR/setup.sh
-echo "Setting up $VARIANT pod... done."
+source secrets-export.sh
+source $K8S_VARIANT_DIR/secrets-export.sh
 
 # Extract the tar file to the /tmp/test directory.
 echo "Setting up driver test files..."
 kubectl exec ${K8S_POD_NAME} -- bash -c "rm -rf /tmp/test && mkdir /tmp/test"
-tar cf - ${K8S_DRIVERS_TAR_FILE} | kubectl exec -i ${K8S_POD_NAME} -- /bin/sh -c 'tar xf - -C /tmp/test'
+kubectl cp ${K8S_DRIVERS_TAR_FILE} ${K8S_POD_NAME}:/tmp/drivers-test.tgz
+kubectl exec ${K8S_POD_NAME} -- bash -c "cd /tmp && tar -xf drivers-test.tgz -C test"
 echo "Setting up driver test files... done."
 
 # Run the command.
 echo "Running the driver test command..."
 kubectl exec ${K8S_POD_NAME} -- bash -c "cd /tmp/test && ${K8S_TEST_CMD}"
 echo "Running the driver test command... done."
 
-# Tear down the pod.
-echo "Tearding down $VARIANT pod..."
-. $VARIANT_DIR/teardown.sh
-echo "Tearding down $VARIANT pod... done."
-
 popd

.evergreen/auth_oidc/k8s/run-self-test.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../../handle-paths.sh
+pushd $SCRIPT_DIR
+
+source secrets-export.sh
+source $K8S_VARIANT_DIR/secrets-export.sh
+
+# Run the self test.
+echo "Running self test on $K8S_VARIANT..."
+kubectl exec ${K8S_POD_NAME} -- bash -c "rm -rf /tmp/test && mkdir /tmp/test"
+kubectl cp ./remote-scripts/run-self-test.sh ${K8S_POD_NAME}:/tmp/test/run-self-test.sh
+kubectl cp ./remote-scripts/test.py ${K8S_POD_NAME}:/tmp/test/test.py
+kubectl cp ./secrets-export.sh ${K8S_POD_NAME}:/tmp/test/secrets-export.sh
+kubectl exec ${K8S_POD_NAME} -- /tmp/test/run-self-test.sh
+echo "Running self test on $K8S_VARIANT... done."
+
+popd

.evergreen/auth_oidc/k8s/setup-pod.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../../handle-paths.sh
+pushd $SCRIPT_DIR
+
+# Set the current K8S_VARIANT.
+K8S_VARIANT=${K8S_VARIANT:-$1}
+echo "export K8S_VARIANT=$K8S_VARIANT" >> secrets-export.sh
+VARIANT=$(echo "$K8S_VARIANT" | tr '[:upper:]' '[:lower:]')
+VARIANT_DIR=$DRIVERS_TOOLS/.evergreen/k8s/$VARIANT
+echo "export K8S_VARIANT_DIR=$VARIANT_DIR" >> secrets-export.sh
+
+# Set up the pod.
+echo "Setting up $VARIANT pod..."
+. $VARIANT_DIR/setup.sh
+echo "Setting up $VARIANT pod... done."
+
+popd

.evergreen/auth_oidc/k8s/setup.sh

@@ -4,10 +4,25 @@ set -o errexit
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
 . $SCRIPT_DIR/../../handle-paths.sh
+
+rm -f $SCRIPT_DIR/secrets-export.sh
+
+# If running locally, just set up the variables and exit.
+if [ "$1" == "local" ]; then
+  URI="mongodb://127.0.0.1"
+  cat <<EOF >> "$SCRIPT_DIR/secrets-export.sh"
+export OIDC_SERVER_TYPE=local
+export MONGODB_URI="$URI"
+export MONGODB_URI_SINGLE="$URI/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s"
+export OIDC_ADMIN_USER=bob
+export OIDC_ADMIN_PWD=pwd123
+EOF
+  exit 0
+fi
+
 pushd $SCRIPT_DIR
 
 # Handle secrets from vault.
-rm -f secrets-export.sh
 . ./setup-secrets.sh
 
 ########################
@@ -40,7 +55,7 @@ export DEPLOYMENT_DATA=$(cat <<EOF
   "clusterType" : "REPLICASET",
   "diskSizeGB" : 10.0,
   "encryptionAtRestProvider" : "NONE",
-  "mongoDBMajorVersion" : "7.0",
+  "mongoDBMajorVersion" : "8.0",
   "name" : "${DEPLOYMENT_NAME}",
   "numShards" : 1,
   "paused" : false,
@@ -79,6 +94,7 @@ create_deployment
 URI=$(check_deployment)
 
 cat <<EOF >> "secrets-export.sh"
+export OIDC_SERVER_TYPE=local
 export MONGODB_URI="$URI"
 export MONGODB_URI_SINGLE="$URI/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s"
 export OIDC_ADMIN_USER=$OIDC_ATLAS_USER

.evergreen/auth_oidc/k8s/start-server.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Start an OIDC-enabled server on a kubernetes pod.
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../../handle-paths.sh
+pushd $SCRIPT_DIR
+
+source secrets-export.sh
+source $K8S_VARIANT_DIR/secrets-export.sh
+
+# Extract the tar file to the /tmp/test directory.
+echo "Setting up server on ${K8S_POD_NAME}..."
+kubectl exec ${K8S_POD_NAME} -- bash -c "rm -rf /tmp/server && mkdir /tmp/server/"
+K8S_TAR_FILE=/tmp/drivers-tools.tgz
+pushd $DRIVERS_TOOLS
+git archive -o $K8S_TAR_FILE HEAD
+popd
+kubectl cp ${K8S_TAR_FILE}  ${K8S_POD_NAME}:/tmp/server/drivers-tools.tgz
+kubectl cp $K8S_VARIANT_DIR/secrets-export.sh ${K8S_POD_NAME}:/tmp/server/secrets-export.sh
+kubectl cp ./remote-scripts/start-server.sh ${K8S_POD_NAME}:/tmp/server/start-server.sh
+kubectl cp ./remote-scripts/setup_oidc.js ${K8S_POD_NAME}:/tmp/server/setup_oidc.js
+kubectl exec ${K8S_POD_NAME} -- /tmp/server/start-server.sh
+echo "Setting up server on ${K8S_POD_NAME}... done."
+
+popd

.evergreen/auth_oidc/k8s/teardown-pod.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../../handle-paths.sh
+pushd $SCRIPT_DIR
+
+source secrets-export.sh
+
+# Tear down the pod.
+echo "Tearding down $K8S_VARIANT pod..."
+. $K8S_VARIANT_DIR/teardown.sh
+echo "Tearding down $K8S_VARIANT pod... done."
+
+popd