Merge branch 'master' of github.com:mongodb-labs/drivers-evergreen-to…

…ols into DRIVERS-3032-dec

.evergreen/config.yml

@@ -1381,7 +1381,6 @@ buildvariants:
   display_name: Docker
   run_on:
     - ubuntu2204-small
-    - macos-14
   tasks:
     - ".docker"  # Run all tasks with the "docker" tag
 

.evergreen/docker/README.md

@@ -45,17 +45,13 @@ Note that the default `TOPOLOGY` is [`servers`](https://github.com/mongodb-labs/
 TOPOLOGY=replica_set ORCHESTRATION_FILE=auth.json bash ./run-server.sh
 ```
 
-If you want to test server versions older than 4.4, you can use the 18.04 image, e.g.:
+If you want to test server versions older than 4.4, you can use the rhel8 image, e.g.:
 
-```bash
-TOPOLOGY=sharded_cluster MONGODB_VERSION=4.2 TARGET_IMAGE=ubuntu18.04 ./run-server.sh
-```
-
-You must also specify `ARCH=amd64` in order to run MongoDB 3.6 and 4.0, since `aarch` support
-is not available in Ubuntu 18.04 for those versions.
+You must also specify `ARCH=amd64` in order to run MongoDB versions older than 4.4 on RHEL8, since `aarch` support
+is not available in RHEL8 for those versions.
 
 ```bash
-ARCH=amd64 TOPOLOGY=sharded_cluster MONGODB_VERSION=3.6 TARGET_IMAGE=ubuntu18.04 ./run-server.sh
+ARCH=amd64 TOPOLOGY=sharded_cluster MONGODB_VERSION=4.2 TARGET_IMAGE=rhel8 ./run-server.sh
 ```
 
 ### Get Logs

.evergreen/docker/rhel8/Dockerfile

@@ -0,0 +1,38 @@
+FROM registry.access.redhat.com/ubi8/ubi:8.4
+
+# Install mongodb 4.4 server to get older deps.
+RUN echo -e "\
+[mongodb-enterprise-4.4] \n\
+name=MongoDB Enterprise Repository \n\
+baseurl=https://repo.mongodb.com/yum/redhat/8/mongodb-enterprise/4.4/$(uname -m)/ \n\
+enabled         = 1 \n\
+gpgkey=https://pgp.mongodb.com/server-4.4.asc \n\
+gpgcheck        = 1\
+" > /etc/yum.repos.d/mongodb-enterprise-4.4.repo
+
+RUN dnf update -y && \
+  dnf install -y \
+    git \
+    ca-certificates \
+    curl \
+    wget \
+    sudo \
+    make \
+    gnupg \
+    python3.9 \
+    mongodb-enterprise \
+    lsof < /dev/null > /dev/null
+
+# Alias python3 -> python3.9
+RUN ln -f -s /usr/bin/python3.9 /usr/bin/python3
+
+ARG USER_ID
+ARG GROUP_ID
+
+ENV DRIVERS_TOOLS=/root/drivers-tools
+ENV MONGO_ORCHESTRATION_HOME=/root/drivers-tools/.evergreen/orchestration
+ENV MONGODB_BINARIES=/root/drivers-tools/mongodb/bin
+ENV DOCKER_RUNNING=true
+
+COPY . /root/drivers-tools
+COPY .evergreen/docker/rhel8/*.sh /root

.evergreen/docker/ubuntu20.04/Dockerfile

@@ -10,14 +10,21 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
   sudo \
   gnupg \
   python \
-  python3 \
-  python3.8-venv \
   lsof \
   software-properties-common \
   libsnmp35 \
   net-tools < /dev/null > /dev/null \
  && rm -rf /var/lib/apt/lists/*
 
+# Install python 3.9 from ppa
+RUN export DEBIAN_FRONTEND=noninteractive \
+&& export TZ=Etc/UTC \
+&& add-apt-repository ppa:deadsnakes/ppa \
+&& apt-get -qq update \
+&& apt-get -qq -y install python3.9 python3.9-venv python3-venv < /dev/null > /dev/null \
+&& ln -sf /usr/bin/python3.9 /usr/bin/python3 \
+&& rm -rf /var/lib/apt/lists/*
+
 ARG USER_ID
 ARG GROUP_ID
 

.evergreen/download-mongodb.sh

@@ -68,20 +68,7 @@ get_mongodb_download_url_for ()
      exit 1
    fi
 
-   # Get the download URL for crypt_shared.
-   # The crypt_shared package is available on server 6.0 and newer.
-   # Try to download a version of crypt_shared matching the server version.
-   # If no matching version is available, try to download the latest Major release of crypt_shared.
-   case "$_VERSION" in
-      5.0 | 4.4 | 4.2 | 4.0 | 3.6)
-         # Default to using the latest Major release. Major releases are expected yearly.
-         # MONGODB_60 may be empty if there is no 6.0 download available for this platform.
-         _crypt_shared_version=latest
-         ;;
-      *)
-         _crypt_shared_version=$_VERSION
-   esac
-   MONGO_CRYPT_SHARED_DOWNLOAD_URL=$($_python3 "${_script_dir}/mongodl.py" --version $_crypt_shared_version --component crypt_shared --no-download | tr -d '\r')
+   MONGO_CRYPT_SHARED_DOWNLOAD_URL=$($_python3 "${_script_dir}/mongodl.py" --version $_VERSION --component crypt_shared --no-download | tr -d '\r')
 
    echo "$MONGODB_DOWNLOAD_URL"
 }

.evergreen/find-python3.sh

@@ -57,8 +57,9 @@ is_python3() (
     return 1
   fi
 
+  # The minimum version we support is Python 3.9.  All other versions are EOL.
   # shellcheck disable=SC2091
-  if ! $("$bin" -c "import sys; exit(sys.version_info[0] == 3 and sys.version_info[1] < 8)"); then
+  if ! $("$bin" -c "import sys; exit(sys.version_info[0] == 3 and sys.version_info[1] < 9)"); then
     version=$($bin --version)
     echo "Detected EOL Python ${version}, skipping."
     return 1

.evergreen/mongodl.py

@@ -29,6 +29,7 @@
 import textwrap
 import urllib.error
 import urllib.request
+import warnings
 import zipfile
 from collections import namedtuple
 from contextlib import contextmanager
@@ -822,7 +823,23 @@ def _dl_component(
             target, arch, edition, component, latest_build_branch
         )
     else:
-        dl_url = _published_build_url(cache, version, target, arch, edition, component)
+        try:
+            dl_url = _published_build_url(
+                cache, version, target, arch, edition, component
+            )
+        except ValueError:
+            if component == "crypt_shared" and version != "latest":
+                warnings.warn(
+                    "No matching version of crypt_shared found, using 'latest'",
+                    stacklevel=2,
+                )
+                version = "latest"
+            else:
+                raise
+            dl_url = _published_build_url(
+                cache, version, target, arch, edition, component
+            )
+
     if no_download:
         print(dl_url)
         return None
@@ -999,7 +1016,7 @@ def _maybe_extract_member(
     return 1
 
 
-def main():
+def main(argv=None):
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
@@ -1019,32 +1036,37 @@ def main():
     dl_grp = parser.add_argument_group(
         "Download arguments",
         description="Select what to download and extract. "
-        "Non-required arguments will be inferred "
+        "Some arguments will be inferred "
         "based on the host system.",
     )
     dl_grp.add_argument(
         "--target",
         "-T",
+        default="auto",
         help="The target platform for which to download. "
         'Use "--list" to list available targets.',
     )
-    dl_grp.add_argument("--arch", "-A", help="The architecture for which to download")
+    dl_grp.add_argument(
+        "--arch", "-A", default="auto", help="The architecture for which to download"
+    )
     dl_grp.add_argument(
         "--edition",
         "-E",
+        default="enterprise",
         help='The edition of the product to download (Default is "enterprise"). '
         'Use "--list" to list available editions.',
     )
     dl_grp.add_argument(
         "--out",
         "-o",
-        help="The directory in which to download components. (Required)",
+        help="The directory in which to download components.",
         type=Path,
     )
     dl_grp.add_argument(
         "--version",
         "-V",
-        help='The product version to download (Required). Use "latest" to download '
+        default="latest",
+        help='The product version to download. Use "latest" to download '
         "the newest available version (including release candidates). Use "
         '"latest-stable" to download the newest version, excluding release '
         'candidates. Use "rapid" to download the latest rapid release. '
@@ -1054,8 +1076,8 @@ def main():
     dl_grp.add_argument(
         "--component",
         "-C",
-        help="The component to download (Required). "
-        'Use "--list" to list available components.',
+        default="archive",
+        help="The component to download. " 'Use "--list" to list available components.',
     )
     dl_grp.add_argument(
         "--only",
@@ -1103,33 +1125,24 @@ def main():
         'download the with "--version=latest-build"',
         metavar="BRANCH_NAME",
     )
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     cache = Cache.open_in(args.cache_dir)
     cache.refresh_full_json()
 
-    if args.list:
-        _print_list(
-            cache.db, args.version, args.target, args.arch, args.edition, args.component
-        )
-        return
-
-    if args.version is None:
-        raise argparse.ArgumentError(None, 'A "--version" is required')
-    if args.component is None:
-        raise argparse.ArgumentError(None, 'A "--component" name should be provided')
-    if args.out is None and args.test is None and args.no_download is None:
-        raise argparse.ArgumentError(None, 'A "--out" directory should be provided')
-
     version = args.version
     if version in PERF_VERSIONS:
         version = PERF_VERSIONS[version]
     target = args.target
-    if target in (None, "auto"):
+    if target == "auto":
         target = infer_target(version)
     arch = args.arch
-    if arch in (None, "auto"):
+    if arch == "auto":
         arch = infer_arch()
-    edition = args.edition or "enterprise"
+
+    if args.list:
+        _print_list(cache.db, version, target, arch, args.edition, args.component)
+        return
+
     out = args.out or Path.cwd()
     out = out.absolute()
 
@@ -1139,7 +1152,7 @@ def main():
         version=version,
         target=target,
         arch=arch,
-        edition=edition,
+        edition=args.edition,
         component=args.component,
         pattern=args.only,
         strip_components=args.strip_components,

.evergreen/mongosh_dl.py

@@ -110,34 +110,37 @@ def _download(
     return resp
 
 
-def main():
+def main(argv=None):
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
     dl_grp = parser.add_argument_group(
         "Download arguments",
         description="Select what to download and extract. "
-        "Non-required arguments will be inferred "
+        "Some arguments will be inferred "
         "based on the host system.",
     )
     dl_grp.add_argument(
         "--target",
         "-T",
+        default="auto",
         help="The target platform for which to download. "
         'Use "--list" to list available targets.',
     )
-    dl_grp.add_argument("--arch", "-A", help="The architecture for which to download")
+    dl_grp.add_argument(
+        "--arch", "-A", default="auto", help="The architecture for which to download"
+    )
     dl_grp.add_argument(
         "--out",
         "-o",
-        help="The directory in which to download components. (Required)",
+        help="The directory in which to download components.",
         type=Path,
     )
     dl_grp.add_argument(
         "--version",
         "-V",
         default="latest",
-        help='The product version to download (Required). Use "latest" to download '
+        help='The product version to download. Use "latest" to download '
         "the newest available stable version.",
     )
     dl_grp.add_argument(
@@ -174,16 +177,13 @@ def main():
         help="Do not extract or place any files/directories. "
         "Only print what will be extracted without placing any files.",
     )
-    args = parser.parse_args()
-
-    if args.out is None and args.test is None and args.no_download is None:
-        raise argparse.ArgumentError(None, 'A "--out" directory should be provided')
+    args = parser.parse_args(argv)
 
     target = args.target
-    if target in (None, "auto"):
+    if target == "auto":
         target = sys.platform
     arch = args.arch
-    if arch in (None, "auto"):
+    if arch == "auto":
         arch = infer_arch()
     out = args.out or Path.cwd()
     out = out.absolute()

.evergreen/ocsp/mock-ocsp-responder-requirements.txt

@@ -2,7 +2,7 @@ asn1crypto==1.5.1
 click==8.1.7
 flask==2.2.5
 itsdangerous==2.1.2
-Jinja2==3.1.4
+Jinja2==3.1.5
 MarkupSafe==2.1.4
 git+https://github.com/wbond/oscrypto.git@d5f3437
 Werkzeug==3.0.6