PYTHON-4975 Use justfile as the task runner (#2057)

Co-authored-by: Noah Stapp <[email protected]>

.evergreen/combine-coverage.sh

@@ -8,7 +8,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 . .evergreen/utils.sh
 
-if [ -z "$PYTHON_BINARY" ]; then
+if [ -z "${PYTHON_BINARY:-}" ]; then
     PYTHON_BINARY=$(find_python3)
 fi
 

.evergreen/install-dependencies.sh

@@ -13,6 +13,33 @@ else
   SUDO="sudo"
 fi
 
+# Install just.
+# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
+if [ "${CI:-}" == "true" ]; then
+  BIN_DIR=$DRIVERS_TOOLS_BINARIES
+else
+  BIN_DIR=$HOME/.local/bin
+fi
+if [ ! -f $BIN_DIR/just ]; then
+  if [ "Windows_NT" = "${OS:-}" ]; then
+    TARGET="--target x86_64-pc-windows-msvc"
+  else
+    TARGET=""
+  fi
+  curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $TARGET --to "$BIN_DIR" || {
+    # CARGO_HOME is defined in configure-env.sh
+    export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/}
+    export RUSTUP_HOME="${CARGO_HOME}/.rustup"
+    . ${DRIVERS_TOOLS}/.evergreen/install-rust.sh
+    cargo install just
+    if [ "Windows_NT" = "${OS:-}" ]; then
+      mv $CARGO_HOME/just.exe $BIN_DIR/just
+    else
+      mv $CARGO_HOME/just $BIN_DIR
+    fi
+  }
+fi
+
 # Add 'server' and 'hostname_not_in_cert' as a hostnames
 echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts
 echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts

.evergreen/just.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eu
+
+. .evergreen/scripts/setup-dev-env.sh
+just "$@"

.evergreen/run-azurekms-fail-test.sh

@@ -8,5 +8,5 @@ PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
     KEY_NAME="${AZUREKMS_KEYNAME}" \
     KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
     SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
-    $HERE/hatch.sh test:test-eg
+    $HERE/just.sh test-eg
 bash $HERE/teardown-encryption.sh

.evergreen/run-azurekms-test.sh

@@ -18,7 +18,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \
+AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/just.sh test-eg" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Running test ... end"
 bash $HERE/teardown-encryption.sh

.evergreen/run-gcpkms-test.sh

@@ -17,6 +17,6 @@ echo "Untarring file ... begin"
 GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Running test ... end"
 bash $HERE/teardown-encryption.sh

.evergreen/run-import-time-test.sh

@@ -5,7 +5,7 @@ set -x
 
 . .evergreen/utils.sh
 
-if [ -z "$PYTHON_BINARY" ]; then
+if [ -z "${PYTHON_BINARY:-}" ]; then
     PYTHON_BINARY=$(find_python3)
 fi
 

.evergreen/run-mongodb-aws-ecs-test.sh

@@ -31,4 +31,4 @@ export AUTH="auth"
 export SET_XTRACE_ON=1
 cd src
 rm -rf .venv
-bash .evergreen/hatch.sh test:test-eg
+bash .evergreen/just.sh test-eg

.evergreen/run-mongodb-oidc-test.sh

@@ -32,4 +32,4 @@ fi
 export TEST_AUTH_OIDC=1
 export COVERAGE=1
 export AUTH="auth"
-bash ./.evergreen/hatch.sh test:test-eg -- "${@:1}"
+bash ./.evergreen/just.sh test-eg "${@:1}"

.evergreen/run-perf-tests.sh

@@ -16,4 +16,4 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
 export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
 export PERF_TEST=1
 
-bash ./.evergreen/hatch.sh test:test-eg
+bash ./.evergreen/just.sh test-eg

.evergreen/scripts/cleanup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
-  . .evergreen/hatch.sh encryption:teardown
+  bash .evergreen/teardown-encryption.sh
 fi
 rm -rf "${DRIVERS_TOOLS}" || true
 rm -f ./secrets-export.sh || true

.evergreen/scripts/configure-env.sh

@@ -14,12 +14,14 @@ fi
 PROJECT_DIRECTORY="$(pwd)"
 DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
+HATCH_CONFIG=$PROJECT_DIRECTORY/hatch_config.toml
 
 # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
 if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
     DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
     PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
     CARGO_HOME=$(cygpath -m $CARGO_HOME)
+    HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
 fi
 
 SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
@@ -32,15 +34,16 @@ fi
 
 export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
 export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
+export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
 
 cat <<EOT > "$SCRIPT_DIR"/env.sh
-set -o errexit
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 export CURRENT_VERSION="$CURRENT_VERSION"
 export SKIP_LEGACY_SHELL=1
 export DRIVERS_TOOLS="$DRIVERS_TOOLS"
 export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
 export MONGODB_BINARIES="$MONGODB_BINARIES"
+export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES"
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}"
 export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}"
@@ -59,7 +62,8 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
 
 export CARGO_HOME="$CARGO_HOME"
 export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-export PATH="$MONGODB_BINARIES:$PATH"
+export HATCH_CONFIG="$HATCH_CONFIG"
+export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH"
 # shellcheck disable=SC2154
 export PROJECT="${project:-mongo-python-driver}"
 export PIP_QUIET=1

.evergreen/scripts/install-dependencies.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -o xtrace
+set -eu
 file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh"
 # Don't use ${file} syntax here because evergreen treats it as an empty expansion.
 [ -f "$file" ] && bash "$file" || echo "$file not available, skipping"

.evergreen/scripts/run-atlas-tests.sh

@@ -4,4 +4,4 @@
 set +x
 set -o errexit
 bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
-TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
+TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg

.evergreen/scripts/run-doctests.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 set -o xtrace
-PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test
+PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh docs-test

.evergreen/scripts/run-enterprise-auth-tests.sh

@@ -5,4 +5,4 @@ set -eu
 set +x
 # Use the default python to bootstrap secrets.
 bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth
-TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
+TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg

.evergreen/scripts/run-gcpkms-fail-test.sh

@@ -4,4 +4,4 @@
 export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
 export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
-SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg
+SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg

.evergreen/scripts/run-mockupdb-tests.sh

@@ -2,4 +2,4 @@
 
 set -o xtrace
 export PYTHON_BINARY=${PYTHON_BINARY}
-bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb
+bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-mockupdb

.evergreen/scripts/run-mongodb-aws-test.sh

@@ -30,4 +30,4 @@ set -x
 export TEST_AUTH_AWS=1
 export AUTH="auth"
 export SET_XTRACE_ON=1
-bash ./.evergreen/hatch.sh test:test-eg
+bash ./.evergreen/just.sh test-eg

.evergreen/scripts/run-ocsp-test.sh

@@ -4,5 +4,5 @@ TEST_OCSP=1 \
 PYTHON_BINARY="${PYTHON_BINARY}" \
 CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
 OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
-bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
+bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg
 bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh

.evergreen/scripts/run-tests.sh

@@ -51,4 +51,4 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
   TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \
   TEST_SUITES=${TEST_SUITES:-} \
   MONGODB_API_VERSION=${MONGODB_API_VERSION} \
-  bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
+  bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg

.evergreen/scripts/setup-dev-env.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -eu
+
+HERE=$(dirname ${BASH_SOURCE:-$0})
+pushd "$(dirname "$(dirname $HERE)")" > /dev/null
+
+# Source the env file to pick up common variables.
+if [ -f $HERE/scripts/env.sh ]; then
+  source $HERE/scripts/env.sh
+fi
+
+# Set the location of the python bin dir.
+if [ "Windows_NT" = "${OS:-}" ]; then
+  BIN_DIR=.venv/Scripts
+else
+  BIN_DIR=.venv/bin
+fi
+
+# Ensure there is a python venv.
+if [ ! -d $BIN_DIR ]; then
+  . .evergreen/utils.sh
+
+  if [ -z "${PYTHON_BINARY:-}" ]; then
+      PYTHON_BINARY=$(find_python3)
+  fi
+
+  echo "Creating virtual environment..."
+  createvirtualenv "$PYTHON_BINARY" .venv
+  echo "Creating virtual environment... done."
+fi
+
+# Activate the virtual env.
+. $BIN_DIR/activate
+
+# Ensure there is a local hatch.
+if [ ! -f $BIN_DIR/hatch ]; then
+  echo "Installing hatch..."
+  python -m pip install hatch || {
+    # CARGO_HOME is defined in configure-env.sh
+    export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/}
+    export RUSTUP_HOME="${CARGO_HOME}/.rustup"
+    ${DRIVERS_TOOLS}/.evergreen/install-rust.sh
+    source "${CARGO_HOME}/env"
+    python -m pip install hatch
+  }
+  echo "Installing hatch... done."
+fi
+
+# Ensure hatch does not write to user or global locations.
+HATCH_CONFIG=${HATCH_CONFIG:-hatch_config.toml}
+if [ ! -f ${HATCH_CONFIG} ]; then
+  touch hatch_config.toml
+  hatch config restore
+  hatch config set dirs.data "$(pwd)/.hatch/data"
+  hatch config set dirs.cache "$(pwd)/.hatch/cache"
+fi
+
+# Ensure there is a local pre-commit if there is a git checkout.
+if [ -d .git ]; then
+  if [ ! -f $BIN_DIR/pre-commit ]; then
+    python -m pip install pre-commit
+  fi
+
+  # Ensure the pre-commit hook is installed.
+  if [ ! -f .git/hooks/pre-commit ]; then
+    pre-commit install
+  fi
+fi
+
+# Install pymongo and its test deps.
+python -m pip install ".[test]"

.evergreen/scripts/setup-encryption.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
 if [ -n "${test_encryption}" ]; then
-  ./.evergreen/hatch.sh encryption:setup
+  bash .evergreen/setup-encryption.sh
 fi

.evergreen/setup-spawn-host.sh

@@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_
 echo "Copying files to $target... done"
 
 ssh $target $remote_dir/.evergreen/scripts/setup-system.sh
-ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/ensure-hatch.sh"
+ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} just install"

.evergreen/utils.sh

@@ -1,6 +1,6 @@
-#!/bin/bash -ex
+#!/bin/bash
 
-set -o xtrace
+set -eu
 
 find_python3() {
     PYTHON=""