diff --git a/.evergreen/combine-coverage.sh b/.evergreen/combine-coverage.sh index 92d2f1f1f8..c31f755bd9 100755 --- a/.evergreen/combine-coverage.sh +++ b/.evergreen/combine-coverage.sh @@ -8,7 +8,7 @@ set -o errexit # Exit the script with error if any of the commands fail . .evergreen/utils.sh -if [ -z "$PYTHON_BINARY" ]; then +if [ -z "${PYTHON_BINARY:-}" ]; then PYTHON_BINARY=$(find_python3) fi diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh deleted file mode 100755 index c01dfcd19e..0000000000 --- a/.evergreen/hatch.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -eu - -. .evergreen/scripts/ensure-hatch.sh -hatch run "$@" diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index e6dceb33fc..8773fa2c6d 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -13,6 +13,33 @@ else SUDO="sudo" fi +# Install just. +# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. +if [ "${CI:-}" == "true" ]; then + BIN_DIR=$DRIVERS_TOOLS_BINARIES +else + BIN_DIR=$HOME/.local/bin +fi +if [ ! -f $BIN_DIR/just ]; then + if [ "Windows_NT" = "${OS:-}" ]; then + TARGET="--target x86_64-pc-windows-msvc" + else + TARGET="" + fi + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $TARGET --to "$BIN_DIR" || { + # CARGO_HOME is defined in configure-env.sh + export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/} + export RUSTUP_HOME="${CARGO_HOME}/.rustup" + . ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + cargo install just + if [ "Windows_NT" = "${OS:-}" ]; then + mv $CARGO_HOME/just.exe $BIN_DIR/just + else + mv $CARGO_HOME/just $BIN_DIR + fi + } +fi + # Add 'server' and 'hostname_not_in_cert' as a hostnames echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts diff --git a/.evergreen/just.sh b/.evergreen/just.sh new file mode 100755 index 0000000000..bebbca8282 --- /dev/null +++ b/.evergreen/just.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -eu + +. .evergreen/scripts/setup-dev-env.sh +just "$@" diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh index d99c178fb9..d1117dcb32 100755 --- a/.evergreen/run-azurekms-fail-test.sh +++ b/.evergreen/run-azurekms-fail-test.sh @@ -8,5 +8,5 @@ PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ KEY_NAME="${AZUREKMS_KEYNAME}" \ KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ - $HERE/hatch.sh test:test-eg + $HERE/just.sh test-eg bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh index bb515a9386..d5c332fa8d 100755 --- a/.evergreen/run-azurekms-test.sh +++ b/.evergreen/run-azurekms-test.sh @@ -18,7 +18,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \ +AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/just.sh test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Running test ... end" bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh index 7ccc74b453..4c953584b2 100755 --- a/.evergreen/run-gcpkms-test.sh +++ b/.evergreen/run-gcpkms-test.sh @@ -17,6 +17,6 @@ echo "Untarring file ... begin" GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Running test ... end" bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-import-time-test.sh b/.evergreen/run-import-time-test.sh index e9f6161bcc..95e3c93d25 100755 --- a/.evergreen/run-import-time-test.sh +++ b/.evergreen/run-import-time-test.sh @@ -5,7 +5,7 @@ set -x . .evergreen/utils.sh -if [ -z "$PYTHON_BINARY" ]; then +if [ -z "${PYTHON_BINARY:-}" ]; then PYTHON_BINARY=$(find_python3) fi diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 3189a6cc6c..91777be226 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -31,4 +31,4 @@ export AUTH="auth" export SET_XTRACE_ON=1 cd src rm -rf .venv -bash .evergreen/hatch.sh test:test-eg +bash .evergreen/just.sh test-eg diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 22864528c0..46c4f24969 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -32,4 +32,4 @@ fi export TEST_AUTH_OIDC=1 export COVERAGE=1 export AUTH="auth" -bash ./.evergreen/hatch.sh test:test-eg -- "${@:1}" +bash ./.evergreen/just.sh test-eg "${@:1}" diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index ff8d81a837..e6a51b3297 100755 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -16,4 +16,4 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 export PERF_TEST=1 -bash ./.evergreen/hatch.sh test:test-eg +bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index 9e583e4f1e..a1fd92f04d 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then - . .evergreen/hatch.sh encryption:teardown + bash .evergreen/teardown-encryption.sh fi rm -rf "${DRIVERS_TOOLS}" || true rm -f ./secrets-export.sh || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index e0c845a333..ebbffcf1db 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -14,12 +14,14 @@ fi PROJECT_DIRECTORY="$(pwd)" DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} +HATCH_CONFIG=$PROJECT_DIRECTORY/hatch_config.toml # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) CARGO_HOME=$(cygpath -m $CARGO_HOME) + HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" @@ -32,15 +34,16 @@ fi export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" +export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" cat < "$SCRIPT_DIR"/env.sh -set -o errexit export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export CURRENT_VERSION="$CURRENT_VERSION" export SKIP_LEGACY_SHELL=1 export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" +export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" @@ -59,7 +62,8 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export CARGO_HOME="$CARGO_HOME" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" -export PATH="$MONGODB_BINARIES:$PATH" +export HATCH_CONFIG="$HATCH_CONFIG" +export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH" # shellcheck disable=SC2154 export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 diff --git a/.evergreen/scripts/ensure-hatch.sh b/.evergreen/scripts/ensure-hatch.sh deleted file mode 100755 index e63d98bb6d..0000000000 --- a/.evergreen/scripts/ensure-hatch.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -set -eu - -HERE=$(dirname ${BASH_SOURCE:-$0}) -pushd "$(dirname "$(dirname $HERE)")" > /dev/null - -# Ensure hatch is available. -if [ ! -x "$(command -v hatch)" ]; then - # Install a virtual env with "hatch" - # Ensure there is a python venv. - . .evergreen/utils.sh - - if [ -z "${PYTHON_BINARY:-}" ]; then - PYTHON_BINARY=$(find_python3) - fi - VENV_DIR=.venv - if [ ! -d $VENV_DIR ]; then - echo "Creating virtual environment..." - createvirtualenv "$PYTHON_BINARY" .venv - echo "Creating virtual environment... done." - fi - if [ -f $VENV_DIR/Scripts/activate ]; then - . $VENV_DIR/Scripts/activate - else - . $VENV_DIR/bin/activate - fi - - python --version - - echo "Installing hatch..." - python -m pip install -U pip - python -m pip install hatch || { - # Install rust and try again. - CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} - # Handle paths on Windows. - if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - CARGO_HOME=$(cygpath -m $CARGO_HOME) - fi - export RUSTUP_HOME="${CARGO_HOME}/.rustup" - ${DRIVERS_TOOLS}/.evergreen/install-rust.sh - source "${CARGO_HOME}/env" - python -m pip install hatch - } - # Ensure hatch does not write to user or global locations. - touch hatch_config.toml - HATCH_CONFIG=$(pwd)/hatch_config.toml - if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") - fi - export HATCH_CONFIG - hatch config restore - hatch config set dirs.data "$(pwd)/.hatch/data" - hatch config set dirs.cache "$(pwd)/.hatch/cache" - - echo "Installing hatch... done." -fi -hatch --version -popd > /dev/null diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index ebcc8f3069..bbbfc745ec 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -o xtrace +set -eu file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. [ -f "$file" ] && bash "$file" || echo "$file not available, skipping" diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index 98a19f047f..30b8d5a615 100755 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -4,4 +4,4 @@ set +x set -o errexit bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect -TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-doctests.sh b/.evergreen/scripts/run-doctests.sh index f7215ad347..5950e2c107 100755 --- a/.evergreen/scripts/run-doctests.sh +++ b/.evergreen/scripts/run-doctests.sh @@ -1,4 +1,4 @@ #!/bin/bash set -o xtrace -PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test +PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh docs-test diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh index 7f936b1955..e015a34ca4 100755 --- a/.evergreen/scripts/run-enterprise-auth-tests.sh +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -5,4 +5,4 @@ set -eu set +x # Use the default python to bootstrap secrets. bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth -TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh index dd9d522c8a..594a2984fa 100755 --- a/.evergreen/scripts/run-gcpkms-fail-test.sh +++ b/.evergreen/scripts/run-gcpkms-fail-test.sh @@ -4,4 +4,4 @@ export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh -SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg +SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-mockupdb-tests.sh b/.evergreen/scripts/run-mockupdb-tests.sh index 8825a0237d..32594f05d3 100755 --- a/.evergreen/scripts/run-mockupdb-tests.sh +++ b/.evergreen/scripts/run-mockupdb-tests.sh @@ -2,4 +2,4 @@ set -o xtrace export PYTHON_BINARY=${PYTHON_BINARY} -bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-mockupdb diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index ec20bfd06b..88c3236b3f 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -30,4 +30,4 @@ set -x export TEST_AUTH_AWS=1 export AUTH="auth" export SET_XTRACE_ON=1 -bash ./.evergreen/hatch.sh test:test-eg +bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh index 3c6d3b2b3b..328bd2f203 100755 --- a/.evergreen/scripts/run-ocsp-test.sh +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -4,5 +4,5 @@ TEST_OCSP=1 \ PYTHON_BINARY="${PYTHON_BINARY}" \ CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ -bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 6986a0bbee..ea923b3f5e 100755 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -51,4 +51,4 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ TEST_SUITES=${TEST_SUITES:-} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg + bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh new file mode 100755 index 0000000000..7042871942 --- /dev/null +++ b/.evergreen/scripts/setup-dev-env.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -eu + +HERE=$(dirname ${BASH_SOURCE:-$0}) +pushd "$(dirname "$(dirname $HERE)")" > /dev/null + +# Source the env file to pick up common variables. +if [ -f $HERE/scripts/env.sh ]; then + source $HERE/scripts/env.sh +fi + +# Set the location of the python bin dir. +if [ "Windows_NT" = "${OS:-}" ]; then + BIN_DIR=.venv/Scripts +else + BIN_DIR=.venv/bin +fi + +# Ensure there is a python venv. +if [ ! -d $BIN_DIR ]; then + . .evergreen/utils.sh + + if [ -z "${PYTHON_BINARY:-}" ]; then + PYTHON_BINARY=$(find_python3) + fi + + echo "Creating virtual environment..." + createvirtualenv "$PYTHON_BINARY" .venv + echo "Creating virtual environment... done." +fi + +# Activate the virtual env. +. $BIN_DIR/activate + +# Ensure there is a local hatch. +if [ ! -f $BIN_DIR/hatch ]; then + echo "Installing hatch..." + python -m pip install hatch || { + # CARGO_HOME is defined in configure-env.sh + export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/} + export RUSTUP_HOME="${CARGO_HOME}/.rustup" + ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + source "${CARGO_HOME}/env" + python -m pip install hatch + } + echo "Installing hatch... done." +fi + +# Ensure hatch does not write to user or global locations. +HATCH_CONFIG=${HATCH_CONFIG:-hatch_config.toml} +if [ ! -f ${HATCH_CONFIG} ]; then + touch hatch_config.toml + hatch config restore + hatch config set dirs.data "$(pwd)/.hatch/data" + hatch config set dirs.cache "$(pwd)/.hatch/cache" +fi + +# Ensure there is a local pre-commit if there is a git checkout. +if [ -d .git ]; then + if [ ! -f $BIN_DIR/pre-commit ]; then + python -m pip install pre-commit + fi + + # Ensure the pre-commit hook is installed. + if [ ! -f .git/hooks/pre-commit ]; then + pre-commit install + fi +fi + +# Install pymongo and its test deps. +python -m pip install ".[test]" diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh index 2f167cd20b..5b73240205 100755 --- a/.evergreen/scripts/setup-encryption.sh +++ b/.evergreen/scripts/setup-encryption.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup + bash .evergreen/setup-encryption.sh fi diff --git a/.evergreen/setup-spawn-host.sh b/.evergreen/setup-spawn-host.sh index 4109e59183..c20e1c756e 100755 --- a/.evergreen/setup-spawn-host.sh +++ b/.evergreen/setup-spawn-host.sh @@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_ echo "Copying files to $target... done" ssh $target $remote_dir/.evergreen/scripts/setup-system.sh -ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/ensure-hatch.sh" +ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} just install" diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index d3af2dcc7a..e044b3d766 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -1,6 +1,6 @@ -#!/bin/bash -ex +#!/bin/bash -set -o xtrace +set -eu find_python3() { PYTHON="" diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 2310b7698d..a41daaabb1 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -27,12 +27,14 @@ jobs: python-version: "3.9" cache: 'pip' cache-dependency-path: 'pyproject.toml' + - name: Install just + uses: extractions/setup-just@v2 - name: Install Python dependencies run: | - python -m pip install -U pip hatch + just install - name: Run linters run: | - hatch run lint:run-manual + just lint-manual - name: Run compilation run: | export PYMONGO_C_EXT_MUST_BUILD=1 @@ -40,7 +42,7 @@ jobs: python tools/fail_if_no_c.py - name: Run typecheck run: | - hatch run typing:check + just typing - run: | sudo apt-get install -y cppcheck - run: | @@ -73,18 +75,16 @@ jobs: cache: 'pip' cache-dependency-path: 'pyproject.toml' allow-prereleases: true + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip - if [[ "${{ matrix.python-version }}" == "3.13" ]]; then - pip install --pre cffi setuptools - pip install --no-build-isolation hatch - elif [[ "${{ matrix.python-version }}" == "3.13t" ]]; then - # Hatch can't be installed on 3.13t, use pytest directly. + if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then + # Just can't be installed on 3.13t, use pytest directly. pip install . pip install -r requirements/test.txt else - pip install hatch + just install fi - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 @@ -95,7 +95,7 @@ jobs: if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then pytest -v --durations=5 --maxfail=10 else - hatch run test:test + just test fi doctest: @@ -111,16 +111,18 @@ jobs: python-version: "3.9" cache: 'pip' cache-dependency-path: 'pyproject.toml' - - name: Install dependencies - run: | - pip install -U hatch pip + - name: Install just + uses: extractions/setup-just@v2 - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 with: mongodb-version: '8.0.0-rc4' + - name: Install dependencies + run: | + just install - name: Run tests run: | - hatch run doctest:test + just docs-test docs: name: Docs Checks @@ -135,12 +137,14 @@ jobs: cache-dependency-path: 'pyproject.toml' # Build docs on lowest supported Python for furo python-version: '3.9' + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip hatch + just install - name: Build docs run: | - hatch run doc:build + just docs linkcheck: name: Link Check @@ -155,12 +159,14 @@ jobs: cache-dependency-path: 'pyproject.toml' # Build docs on lowest supported Python for furo python-version: '3.9' + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip hatch + just install - name: Build docs run: | - hatch run doc:linkcheck + just docs-linkcheck typing: name: Typing Tests @@ -177,12 +183,14 @@ jobs: python-version: "${{matrix.python}}" cache: 'pip' cache-dependency-path: 'pyproject.toml' + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip hatch + just install - name: Run typecheck run: | - hatch run typing:check + just typing make_sdist: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index e4587125e8..01f896d316 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ libmongocrypt/ libmongocrypt_git/ hatch_config.toml .venv +expansion.yml +.evergreen/scripts/env.sh # Lambda temp files test/lambda/.aws-sam diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 814e040048..5a46151760 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,9 +28,10 @@ including 4 space indents and 79 character line limits. - Avoid backward breaking changes if at all possible. - Write inline documentation for new classes and methods. -- We use [hatch](https://hatch.pypa.io/dev/) for our script runner and packaging tool. +- We use [hatch](https://hatch.pypa.io/dev/) for python environment management and packaging. +- We use [just](https://just.systems/man/en/) as our task runner. - Write tests and make sure they pass (make sure you have a mongod - running on the default port, then execute `hatch run test:test` from the cmd + running on the default port, then execute `just test` from the cmd line to run the test suite). - Add yourself to doc/contributors.rst `:)` @@ -148,17 +149,18 @@ To run `pre-commit` manually, run: pre-commit run --all-files ``` -To run a manual hook like `mypy` manually, run: +To run a manual hook like `ruff` manually, run: ```bash -pre-commit run --all-files --hook-stage manual mypy +pre-commit run --all-files --hook-stage manual ruff ``` -Typically we use `hatch` to run the linters, e.g. +Typically we use `just` to run the linters, e.g. ```bash -hatch run typing:check-mypy -hatch run lint:build-manual +just install # this will install a venv with pre-commit installed, and install the pre-commit hook. +just typing-mypy +just run lint-manual ``` ## Documentation @@ -176,13 +178,13 @@ documentation including narrative docs, and the [Sphinx docstring format](https: You can build the documentation locally by running: ```bash -hatch run doc:build +just docs-build ``` When updating docs, it can be helpful to run the live docs server as: ```bash -hatch run doc:serve +just docs-serve ``` Browse to the link provided, and then as you make changes to docstrings or narrative docs, @@ -192,13 +194,14 @@ the pages will re-render and the browser will automatically refresh. ## Running Tests Locally - Ensure you have started the appropriate Mongo Server(s). -- Run `pip install hatch` to use `hatch` for testing or run - `pip install -e ".[test]"` to run `pytest` directly. -- Run `hatch run test:test` or `pytest` to run all of the tests. +- Run `just install` to set up `hatch` in a local virtual environment, or you can manually + create a virtual environment and run `pytest` directly. If you want to use a specific + version of Python, remove the `.venv` folder and set `PYTHON_BINARY` before running `just install`. +- Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run specific tests. You can omit the `` to test a full class and the `` to test a full module. For example: - `hatch run test:test -- test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`. + `just test test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`. - Use the `-k` argument to select tests by pattern. ## Running Load Balancer Tests Locally @@ -211,15 +214,15 @@ the pages will re-render and the browser will automatically refresh. - Start the load balancer using: `MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`. - Run the tests from the `pymongo` checkout directory using: - `TEST_LOADBALANCER=1 hatch run test:test-eg`. + `TEST_LOADBALANCER=1 just test-eg`. ## Running Encryption Tests Locally - Clone `drivers-evergreen-tools`: `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. - Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools` -- Run `AWS_PROFILE= hatch run encryption:setup` after setting up your AWS profile with `aws configure sso`. -- Run the tests with `TEST_ENCRYPTION=1 hatch run test:test-eg`. -- When done, run `hatch run encryption:teardown` to clean up. +- Run `AWS_PROFILE= just setup-encryption` after setting up your AWS profile with `aws configure sso`. +- Run the tests with `TEST_ENCRYPTION=1 just test-eg`. +- When done, run `just teardown-encryption` to clean up. ## Re-sync Spec Tests diff --git a/hatch.toml b/hatch.toml index 60bd0af014..15d0f25f07 100644 --- a/hatch.toml +++ b/hatch.toml @@ -30,13 +30,6 @@ check-strict-pyright = [ ] check = ["check-mypy", "check-pyright", "check-strict-pyright"] -[envs.lint] -skip-install = true -dependencies = ["pre-commit"] -[envs.lint.scripts] -run = "pre-commit run --all-files" -run-manual = "pre-commit run --all-files --hook-stage manual" - [envs.test] features = ["test"] [envs.test.scripts] @@ -44,9 +37,3 @@ test = "pytest -v --durations=5 --maxfail=10 {args}" test-eg = "bash ./.evergreen/run-tests.sh {args}" test-async = "pytest -v --durations=5 --maxfail=10 -m default_async {args}" test-mockupdb = ["pip install -U git+https://github.com/mongodb-labs/mongo-mockup-db@master", "test -m mockupdb"] - -[envs.encryption] -skip-install = true -[envs.encryption.scripts] -setup = "bash .evergreen/setup-encryption.sh" -teardown = "bash .evergreen/teardown-encryption.sh" diff --git a/justfile b/justfile new file mode 100644 index 0000000000..23f0993c6b --- /dev/null +++ b/justfile @@ -0,0 +1,69 @@ +# See https://just.systems/man/en/ for instructions +set shell := ["bash", "-c"] +set dotenv-load +set dotenv-filename := "./.evergreen/scripts/env.sh" + +# Handle cross-platform paths to local python cli tools. +python_bin_dir := if os_family() == "windows" { "./.venv/Scripts" } else { "./.venv/bin" } +hatch_bin := python_bin_dir + "/hatch" +pre_commit_bin := python_bin_dir + "/pre-commit" + +# Make the default recipe private so it doesn't show up in the list. +[private] +default: + @just --list + +install: + bash .evergreen/scripts/setup-dev-env.sh + +[group('docs')] +docs: + {{hatch_bin}} run doc:build + +[group('docs')] +docs-serve: + {{hatch_bin}} run doc:serve + +[group('docs')] +docs-linkcheck: + {{hatch_bin}} run doc:linkcheck + +[group('docs')] +docs-test: + {{hatch_bin}} run doctest:test + +[group('typing')] +typing: + {{hatch_bin}} run typing:check + +[group('typing')] +typing-mypy: + {{hatch_bin}} run typing:mypy + +[group('lint')] +lint: + {{pre_commit_bin}} run --all-files + +[group('lint')] +lint-manual: + {{pre_commit_bin}} run --all-files --hook-stage manual + +[group('test')] +test *args: + {{hatch_bin}} run test:test {{args}} + +[group('test')] +test-mockupdb: + {{hatch_bin}} run test:test-mockupdb + +[group('test')] +test-eg *args: + {{hatch_bin}} run test:test-eg {{args}} + +[group('encryption')] +setup-encryption: + bash .evergreen/setup-encryption.sh + +[group('encryption')] +teardown-encryption: + bash .evergreen/teardown-encryption.sh