From 81c6b49dd61e833229d750084f9f3c71b31acd8d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 3 Jan 2025 08:20:05 -0600 Subject: [PATCH 1/3] DRIVERS-3032 Improve usability of mongodl (#566) --- .evergreen/download-mongodb.sh | 15 +------- .evergreen/mongodl.py | 65 ++++++++++++++++++++-------------- .evergreen/mongosh_dl.py | 22 ++++++------ 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/.evergreen/download-mongodb.sh b/.evergreen/download-mongodb.sh index 43a06f62..1887ea6f 100755 --- a/.evergreen/download-mongodb.sh +++ b/.evergreen/download-mongodb.sh @@ -101,20 +101,7 @@ get_mongodb_download_url_for () exit 1 fi - # Get the download URL for crypt_shared. - # The crypt_shared package is available on server 6.0 and newer. - # Try to download a version of crypt_shared matching the server version. - # If no matching version is available, try to download the latest Major release of crypt_shared. - case "$_VERSION" in - 5.0 | 4.4 | 4.2 | 4.0 | 3.6) - # Default to using the latest Major release. Major releases are expected yearly. - # MONGODB_60 may be empty if there is no 6.0 download available for this platform. - _crypt_shared_version=latest - ;; - *) - _crypt_shared_version=$_VERSION - esac - MONGO_CRYPT_SHARED_DOWNLOAD_URL=$($_python3 "${_script_dir}/mongodl.py" --version $_crypt_shared_version --component crypt_shared --no-download | tr -d '\r') + MONGO_CRYPT_SHARED_DOWNLOAD_URL=$($_python3 "${_script_dir}/mongodl.py" --version $_VERSION --component crypt_shared --no-download | tr -d '\r') echo "$MONGODB_DOWNLOAD_URL" } diff --git a/.evergreen/mongodl.py b/.evergreen/mongodl.py index 1cba5b07..e3185150 100755 --- a/.evergreen/mongodl.py +++ b/.evergreen/mongodl.py @@ -29,6 +29,7 @@ import textwrap import urllib.error import urllib.request +import warnings import zipfile from collections import namedtuple from contextlib import contextmanager @@ -822,7 +823,23 @@ def _dl_component( target, arch, edition, component, latest_build_branch ) else: - dl_url = _published_build_url(cache, version, target, arch, edition, component) + try: + dl_url = _published_build_url( + cache, version, target, arch, edition, component + ) + except ValueError: + if component == "crypt_shared" and version != "latest": + warnings.warn( + "No matching version of crypt_shared found, using 'latest'", + stacklevel=2, + ) + version = "latest" + else: + raise + dl_url = _published_build_url( + cache, version, target, arch, edition, component + ) + if no_download: print(dl_url) return None @@ -999,7 +1016,7 @@ def _maybe_extract_member( return 1 -def main(): +def main(argv=None): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) @@ -1019,32 +1036,37 @@ def main(): dl_grp = parser.add_argument_group( "Download arguments", description="Select what to download and extract. " - "Non-required arguments will be inferred " + "Some arguments will be inferred " "based on the host system.", ) dl_grp.add_argument( "--target", "-T", + default="auto", help="The target platform for which to download. " 'Use "--list" to list available targets.', ) - dl_grp.add_argument("--arch", "-A", help="The architecture for which to download") + dl_grp.add_argument( + "--arch", "-A", default="auto", help="The architecture for which to download" + ) dl_grp.add_argument( "--edition", "-E", + default="enterprise", help='The edition of the product to download (Default is "enterprise"). ' 'Use "--list" to list available editions.', ) dl_grp.add_argument( "--out", "-o", - help="The directory in which to download components. (Required)", + help="The directory in which to download components.", type=Path, ) dl_grp.add_argument( "--version", "-V", - help='The product version to download (Required). Use "latest" to download ' + default="latest", + help='The product version to download. Use "latest" to download ' "the newest available version (including release candidates). Use " '"latest-stable" to download the newest version, excluding release ' 'candidates. Use "rapid" to download the latest rapid release. ' @@ -1054,8 +1076,8 @@ def main(): dl_grp.add_argument( "--component", "-C", - help="The component to download (Required). " - 'Use "--list" to list available components.', + default="archive", + help="The component to download. " 'Use "--list" to list available components.', ) dl_grp.add_argument( "--only", @@ -1103,33 +1125,24 @@ def main(): 'download the with "--version=latest-build"', metavar="BRANCH_NAME", ) - args = parser.parse_args() + args = parser.parse_args(argv) cache = Cache.open_in(args.cache_dir) cache.refresh_full_json() - if args.list: - _print_list( - cache.db, args.version, args.target, args.arch, args.edition, args.component - ) - return - - if args.version is None: - raise argparse.ArgumentError(None, 'A "--version" is required') - if args.component is None: - raise argparse.ArgumentError(None, 'A "--component" name should be provided') - if args.out is None and args.test is None and args.no_download is None: - raise argparse.ArgumentError(None, 'A "--out" directory should be provided') - version = args.version if version in PERF_VERSIONS: version = PERF_VERSIONS[version] target = args.target - if target in (None, "auto"): + if target == "auto": target = infer_target(version) arch = args.arch - if arch in (None, "auto"): + if arch == "auto": arch = infer_arch() - edition = args.edition or "enterprise" + + if args.list: + _print_list(cache.db, version, target, arch, args.edition, args.component) + return + out = args.out or Path.cwd() out = out.absolute() @@ -1139,7 +1152,7 @@ def main(): version=version, target=target, arch=arch, - edition=edition, + edition=args.edition, component=args.component, pattern=args.only, strip_components=args.strip_components, diff --git a/.evergreen/mongosh_dl.py b/.evergreen/mongosh_dl.py index 6e853b1a..54deeeb4 100755 --- a/.evergreen/mongosh_dl.py +++ b/.evergreen/mongosh_dl.py @@ -110,34 +110,37 @@ def _download( return resp -def main(): +def main(argv=None): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) dl_grp = parser.add_argument_group( "Download arguments", description="Select what to download and extract. " - "Non-required arguments will be inferred " + "Some arguments will be inferred " "based on the host system.", ) dl_grp.add_argument( "--target", "-T", + default="auto", help="The target platform for which to download. " 'Use "--list" to list available targets.', ) - dl_grp.add_argument("--arch", "-A", help="The architecture for which to download") + dl_grp.add_argument( + "--arch", "-A", default="auto", help="The architecture for which to download" + ) dl_grp.add_argument( "--out", "-o", - help="The directory in which to download components. (Required)", + help="The directory in which to download components.", type=Path, ) dl_grp.add_argument( "--version", "-V", default="latest", - help='The product version to download (Required). Use "latest" to download ' + help='The product version to download. Use "latest" to download ' "the newest available stable version.", ) dl_grp.add_argument( @@ -174,16 +177,13 @@ def main(): help="Do not extract or place any files/directories. " "Only print what will be extracted without placing any files.", ) - args = parser.parse_args() - - if args.out is None and args.test is None and args.no_download is None: - raise argparse.ArgumentError(None, 'A "--out" directory should be provided') + args = parser.parse_args(argv) target = args.target - if target in (None, "auto"): + if target == "auto": target = sys.platform arch = args.arch - if arch in (None, "auto"): + if arch == "auto": arch = infer_arch() out = args.out or Path.cwd() out = out.absolute() From 50f50f9c039b4f13932e4d26c3cea1e1a732f881 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 3 Jan 2025 08:46:52 -0600 Subject: [PATCH 2/3] DRIVERS-3077 Drop support for Python 3.8 (#567) Co-authored-by: Noah Stapp --- .evergreen/config.yml | 1 - .evergreen/docker/README.md | 12 ++-- .evergreen/docker/rhel8/Dockerfile | 38 +++++++++++++ .../{ubuntu18.04 => rhel8}/base-entrypoint.sh | 0 .../local-entrypoint.sh | 0 .../{ubuntu18.04 => rhel8}/test-entrypoint.sh | 0 .evergreen/docker/ubuntu18.04/Dockerfile | 55 ------------------- .evergreen/docker/ubuntu20.04/Dockerfile | 11 +++- .evergreen/find-python3.sh | 3 +- .evergreen/pyproject.toml | 2 +- .evergreen/tests/test-csfle.sh | 2 +- 11 files changed, 55 insertions(+), 69 deletions(-) create mode 100644 .evergreen/docker/rhel8/Dockerfile rename .evergreen/docker/{ubuntu18.04 => rhel8}/base-entrypoint.sh (100%) rename .evergreen/docker/{ubuntu18.04 => rhel8}/local-entrypoint.sh (100%) rename .evergreen/docker/{ubuntu18.04 => rhel8}/test-entrypoint.sh (100%) delete mode 100644 .evergreen/docker/ubuntu18.04/Dockerfile diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 46898bff..26059567 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1381,7 +1381,6 @@ buildvariants: display_name: Docker run_on: - ubuntu2204-small - - macos-14 tasks: - ".docker" # Run all tasks with the "docker" tag diff --git a/.evergreen/docker/README.md b/.evergreen/docker/README.md index b610b516..55581466 100644 --- a/.evergreen/docker/README.md +++ b/.evergreen/docker/README.md @@ -45,17 +45,13 @@ Note that the default `TOPOLOGY` is [`servers`](https://github.com/mongodb-labs/ TOPOLOGY=replica_set ORCHESTRATION_FILE=auth.json bash ./run-server.sh ``` -If you want to test server versions older than 4.4, you can use the 18.04 image, e.g.: +If you want to test server versions older than 4.4, you can use the rhel8 image, e.g.: -```bash -TOPOLOGY=sharded_cluster MONGODB_VERSION=4.2 TARGET_IMAGE=ubuntu18.04 ./run-server.sh -``` - -You must also specify `ARCH=amd64` in order to run MongoDB 3.6 and 4.0, since `aarch` support -is not available in Ubuntu 18.04 for those versions. +You must also specify `ARCH=amd64` in order to run MongoDB versions older than 4.4 on RHEL8, since `aarch` support +is not available in RHEL8 for those versions. ```bash -ARCH=amd64 TOPOLOGY=sharded_cluster MONGODB_VERSION=3.6 TARGET_IMAGE=ubuntu18.04 ./run-server.sh +ARCH=amd64 TOPOLOGY=sharded_cluster MONGODB_VERSION=4.2 TARGET_IMAGE=rhel8 ./run-server.sh ``` ### Get Logs diff --git a/.evergreen/docker/rhel8/Dockerfile b/.evergreen/docker/rhel8/Dockerfile new file mode 100644 index 00000000..a0c6d764 --- /dev/null +++ b/.evergreen/docker/rhel8/Dockerfile @@ -0,0 +1,38 @@ +FROM registry.access.redhat.com/ubi8/ubi:8.4 + +# Install mongodb 4.4 server to get older deps. +RUN echo -e "\ +[mongodb-enterprise-4.4] \n\ +name=MongoDB Enterprise Repository \n\ +baseurl=https://repo.mongodb.com/yum/redhat/8/mongodb-enterprise/4.4/$(uname -m)/ \n\ +enabled = 1 \n\ +gpgkey=https://pgp.mongodb.com/server-4.4.asc \n\ +gpgcheck = 1\ +" > /etc/yum.repos.d/mongodb-enterprise-4.4.repo + +RUN dnf update -y && \ + dnf install -y \ + git \ + ca-certificates \ + curl \ + wget \ + sudo \ + make \ + gnupg \ + python3.9 \ + mongodb-enterprise \ + lsof < /dev/null > /dev/null + +# Alias python3 -> python3.9 +RUN ln -f -s /usr/bin/python3.9 /usr/bin/python3 + +ARG USER_ID +ARG GROUP_ID + +ENV DRIVERS_TOOLS=/root/drivers-tools +ENV MONGO_ORCHESTRATION_HOME=/root/drivers-tools/.evergreen/orchestration +ENV MONGODB_BINARIES=/root/drivers-tools/mongodb/bin +ENV DOCKER_RUNNING=true + +COPY . /root/drivers-tools +COPY .evergreen/docker/rhel8/*.sh /root diff --git a/.evergreen/docker/ubuntu18.04/base-entrypoint.sh b/.evergreen/docker/rhel8/base-entrypoint.sh similarity index 100% rename from .evergreen/docker/ubuntu18.04/base-entrypoint.sh rename to .evergreen/docker/rhel8/base-entrypoint.sh diff --git a/.evergreen/docker/ubuntu18.04/local-entrypoint.sh b/.evergreen/docker/rhel8/local-entrypoint.sh similarity index 100% rename from .evergreen/docker/ubuntu18.04/local-entrypoint.sh rename to .evergreen/docker/rhel8/local-entrypoint.sh diff --git a/.evergreen/docker/ubuntu18.04/test-entrypoint.sh b/.evergreen/docker/rhel8/test-entrypoint.sh similarity index 100% rename from .evergreen/docker/ubuntu18.04/test-entrypoint.sh rename to .evergreen/docker/rhel8/test-entrypoint.sh diff --git a/.evergreen/docker/ubuntu18.04/Dockerfile b/.evergreen/docker/ubuntu18.04/Dockerfile deleted file mode 100644 index 1cfed659..00000000 --- a/.evergreen/docker/ubuntu18.04/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -FROM ubuntu:18.04 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get -qq update && apt-get -qq -y install --no-install-recommends \ - git \ - ca-certificates \ - curl \ - wget \ - sudo \ - make \ - gnupg \ - python \ - lsof \ - software-properties-common \ - # python3 deps - build-essential \ - zlib1g-dev \ - libncurses5-dev \ - libgdbm-dev \ - libnss3-dev \ - libssl-dev \ - libsqlite3-dev \ - libreadline-dev \ - libffi-dev wget \ - libbz2-dev < /dev/null > /dev/null \ - && rm -rf /var/lib/apt/lists/* - -# Install mongodb server to get deps -RUN export DEBIAN_FRONTEND=noninteractive \ - && export TZ=Etc/UTC \ - && curl -fsSL https://pgp.mongodb.com/server-4.4.asc | gpg -o /usr/share/keyrings/mongodb-server-4.4.gpg --dearmor \ - && echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-4.4.gpg ] https://repo.mongodb.com/apt/ubuntu bionic/mongodb-enterprise/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-enterprise-4.4.list \ - && apt-get -qq update \ - && apt-get -qq -y install mongodb-enterprise < /dev/null > /dev/null \ - && rm -rf /var/lib/apt/lists/* - -# Install python 3.8 from ppa -RUN export DEBIAN_FRONTEND=noninteractive \ - && export TZ=Etc/UTC \ - && add-apt-repository ppa:deadsnakes/ppa \ - && apt-get -qq update \ - && apt-get -qq -y install python3.8 python3.8-venv python3-venv < /dev/null > /dev/null \ - && ln -sf /usr/bin/python3.8 /usr/bin/python3 \ - && rm -rf /var/lib/apt/lists/* - -ARG USER_ID -ARG GROUP_ID - -ENV DRIVERS_TOOLS=/root/drivers-tools -ENV MONGO_ORCHESTRATION_HOME=/root/drivers-tools/.evergreen/orchestration -ENV MONGODB_BINARIES=/root/drivers-tools/mongodb/bin -ENV DOCKER_RUNNING=true - -COPY . /root/drivers-tools -COPY .evergreen/docker/ubuntu18.04/*.sh /root diff --git a/.evergreen/docker/ubuntu20.04/Dockerfile b/.evergreen/docker/ubuntu20.04/Dockerfile index 7ab5fcee..cc966ef8 100644 --- a/.evergreen/docker/ubuntu20.04/Dockerfile +++ b/.evergreen/docker/ubuntu20.04/Dockerfile @@ -10,14 +10,21 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sudo \ gnupg \ python \ - python3 \ - python3.8-venv \ lsof \ software-properties-common \ libsnmp35 \ net-tools < /dev/null > /dev/null \ && rm -rf /var/lib/apt/lists/* +# Install python 3.9 from ppa +RUN export DEBIAN_FRONTEND=noninteractive \ +&& export TZ=Etc/UTC \ +&& add-apt-repository ppa:deadsnakes/ppa \ +&& apt-get -qq update \ +&& apt-get -qq -y install python3.9 python3.9-venv python3-venv < /dev/null > /dev/null \ +&& ln -sf /usr/bin/python3.9 /usr/bin/python3 \ +&& rm -rf /var/lib/apt/lists/* + ARG USER_ID ARG GROUP_ID diff --git a/.evergreen/find-python3.sh b/.evergreen/find-python3.sh index 71863884..f2f0fb8e 100755 --- a/.evergreen/find-python3.sh +++ b/.evergreen/find-python3.sh @@ -57,8 +57,9 @@ is_python3() ( return 1 fi + # The minimum version we support is Python 3.9. All other versions are EOL. # shellcheck disable=SC2091 - if ! $("$bin" -c "import sys; exit(sys.version_info[0] == 3 and sys.version_info[1] < 8)"); then + if ! $("$bin" -c "import sys; exit(sys.version_info[0] == 3 and sys.version_info[1] < 9)"); then version=$($bin --version) echo "Detected EOL Python ${version}, skipping." return 1 diff --git a/.evergreen/pyproject.toml b/.evergreen/pyproject.toml index ad7e46ef..cae40917 100644 --- a/.evergreen/pyproject.toml +++ b/.evergreen/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "drivers-evergreen-tools" version = "0.1.0" description = 'Scripts for drivers-evergreen-tools' -requires-python = ">=3.8" +requires-python = ">=3.9" license = "MIT" keywords = [] authors = [ diff --git a/.evergreen/tests/test-csfle.sh b/.evergreen/tests/test-csfle.sh index ba49e1dc..ee3195fc 100755 --- a/.evergreen/tests/test-csfle.sh +++ b/.evergreen/tests/test-csfle.sh @@ -29,7 +29,7 @@ if [[ "$(uname -s)" == CYGWIN* ]]; then fi # Test with supported pythons -pythons="3.8 3.9 3.10 3.11 3.12 3.13" +pythons="3.9 3.10 3.11 3.12 3.13" for python in $pythons; do if [ "$(uname -s)" = "Darwin" ]; then PYTHON_BINARY="/Library/Frameworks/Python.Framework/Versions/$python/bin/python3" From eadf5c21e411d5582e52b43cfe9e2ffd26274fb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Jan 2025 10:23:59 -0600 Subject: [PATCH 3/3] Bump jinja2 from 3.1.4 to 3.1.5 in /.evergreen/ocsp (#568) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .evergreen/ocsp/mock-ocsp-responder-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/ocsp/mock-ocsp-responder-requirements.txt b/.evergreen/ocsp/mock-ocsp-responder-requirements.txt index c12b77ea..b898e671 100644 --- a/.evergreen/ocsp/mock-ocsp-responder-requirements.txt +++ b/.evergreen/ocsp/mock-ocsp-responder-requirements.txt @@ -2,7 +2,7 @@ asn1crypto==1.5.1 click==8.1.7 flask==2.2.5 itsdangerous==2.1.2 -Jinja2==3.1.4 +Jinja2==3.1.5 MarkupSafe==2.1.4 git+https://github.com/wbond/oscrypto.git@d5f3437 Werkzeug==3.0.6