diff --git a/CHANGELOG.md b/CHANGELOG.md index a45b267478b..9b0a6cebe72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ ### Internals * Added a CombinedTests target that runs tests from object-store, sync, and core together. ([PR #6964](https://github.com/realm/realm-core/pull/6964)) +* The version of mongod we test against in evergreen was updated to 7.0.3 ([PR #7187](https://github.com/realm/realm-core/pull/7187)). ---------------------------------------------- diff --git a/evergreen/config.yml b/evergreen/config.yml index a771e207fc5..5fb69f7fc7d 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -580,7 +580,7 @@ functions: # Run the setup_baas_host_local.sh script to configure and run baas on the remote host # Add -v to this command for verbose script logging ./evergreen/setup_baas_host_local.sh -w ./baas-work-dir -u $BAAS_USER $OPT_BAAS_BRANCH $OPT_BAAS_PROXY \ - $OPT_BAAS_DIRECT ./baas_host_vars.sh ./.baas_ssh_key 2>&1 | tee install_baas_output.log + $OPT_BAAS_DIRECT -i ./.baas_ssh_key ./baas_host_vars.sh 2>&1 | tee install_baas_output.log "wait for baas to start": - command: shell.exec @@ -600,6 +600,24 @@ functions: echo "Baas is started!" + "wait for remote baas to start": + - command: shell.exec + params: + working_dir: realm-core + shell: bash + script: |- + set -o errexit + set -o verbose + + if [[ -n "${disable_tests_against_baas|}" ]]; then + exit 0 + fi + + # Don't print out the tail of the log file + ./evergreen/wait_for_remote_baas.sh -i ./.baas_ssh_key ./baas_host_vars.sh ./.baas_ssh_key + + echo "Baas is started!" + "setup proxy parameters": - command: shell.exec params: @@ -962,7 +980,7 @@ tasks: - func: "compile" vars: target_to_build: ObjectStoreTests - - func: "wait for baas to start" + - func: "wait for remote baas to start" - func: "run tests" vars: test_label: objstore-baas diff --git a/evergreen/install_baas.sh b/evergreen/install_baas.sh index 0119098e124..8c7249011e8 100755 --- a/evergreen/install_baas.sh +++ b/evergreen/install_baas.sh @@ -67,11 +67,11 @@ function setup_baas_dependencies() { Darwin) if [[ "$(uname -m)" == "arm64" ]]; then export GOARCH=arm64 - MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-arm64-enterprise-6.0.0-rc13.tgz" - MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-1.5.0-darwin-arm64.zip" + MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-arm64-enterprise-7.0.3.tgz" + MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-2.1.1-darwin-arm64.zip" # <-- Remove after enabling constants.sh STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-6.1.0-alpha-527-g796351f.tgz" - ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg" + ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_c5880b9bd6039908a2fd85d1c95f457c36b5d33b_6542b80ae3c331e8d3788186_23_11_01_20_41_47/assisted_agg" GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-arm64.tar.gz" # --> @@ -87,10 +87,11 @@ function setup_baas_dependencies() { BAAS_PLATFORM="Darwin_arm64" else export GOARCH=amd64 - MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-x86_64-enterprise-5.0.3.tgz" + MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-x86_64-enterprise-7.0.3.tgz" + MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-2.1.1-darwin-x64.zip" # <-- Remove after enabling constants.sh STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz" - ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg" + ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_c5880b9bd6039908a2fd85d1c95f457c36b5d33b_6542b80ae3c331e8d3788186_23_11_01_20_41_47/assisted_agg" GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-amd64.tar.gz" # --> BAAS_PLATFORM="Darwin_x86_64" @@ -113,9 +114,10 @@ function setup_baas_dependencies() { DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "${DISTRO_VERSION}")" fi platform_string="${BAAS_PLATFORM} - ${DISTRO_NAME} ${DISTRO_VERSION}" + MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-2.1.1-linux-x64.tgz" case "${DISTRO_NAME}" in ubuntu | linuxmint) - MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-5.0.3.tgz" + MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-7.0.3.tgz" # <-- Remove after enabling constants.sh LIBMONGO_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_ubuntu2004_x86_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo-ubuntu2004-x86_64.so" STITCH_SUPPORT_LIB_URL="https://s3.amazonaws.com/static.realm.io/stitch-support/stitch-support-ubuntu2004-4.4.17-rc1-2-g85de0cc.tgz" @@ -125,7 +127,7 @@ function setup_baas_dependencies() { rhel) case "${DISTRO_VERSION_MAJOR}" in 7) - MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-5.0.3.tgz" + MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-7.0.3.tgz" # <-- Remove after enabling constants.sh LIBMONGO_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_linux_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo.so" STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/linux-x64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz" @@ -459,18 +461,24 @@ if [ ! -x "${MONGO_BINARIES_DIR}/bin/mongod" ]; then fi echo "mongod version: $("${MONGO_BINARIES_DIR}/bin/mongod" --version --quiet | sed 1q)" -if [[ -n "${MONGOSH_DOWNLOAD_URL}" ]]; then - if [[ ! -x "${MONGO_BINARIES_DIR}/bin/mongosh" ]]; then - echo "Downloading mongosh" +if [[ ! -x "${MONGO_BINARIES_DIR}/bin/mongosh" ]]; then + MONGOSH_DOWNLOAD_FILENAME=$(basename "${MONGOSH_DOWNLOAD_URL}") + MONGOSH_DOWNLOAD_EXTENSION="${MONGOSH_DOWNLOAD_FILENAME##*.}" + echo "Downloading ${MONGOSH_DOWNLOAD_URL}" + if [[ "${MONGOSH_DOWNLOAD_EXTENSION}" == "zip" ]]; then ${CURL} -sLS "${MONGOSH_DOWNLOAD_URL}" --output mongosh-binaries.zip unzip -jnqq mongosh-binaries.zip '*/bin/*' -d "${MONGO_BINARIES_DIR}/bin/" rm mongosh-binaries.zip + elif [[ ${MONGOSH_DOWNLOAD_EXTENSION} == "tgz" ]]; then + ${CURL} -sLS "${MONGOSH_DOWNLOAD_URL}" --output mongosh-binaries.tgz + tar -xzf mongosh-binaries.tgz --strip-components=1 -C "${MONGO_BINARIES_DIR}" + rm mongosh-binaries.tgz + else + echo "Unsupported mongosh format $MONGOSH_DOWNLOAD_EXTENSION" + exit 1 fi - MONGOSH="mongosh" -else - # Use the mongo shell provided with mongod - MONGOSH="mongo" fi +MONGOSH="mongosh" chmod +x "${MONGO_BINARIES_DIR}/bin"/* echo "${MONGOSH} version: $("${MONGO_BINARIES_DIR}/bin/${MONGOSH}" --version)" diff --git a/evergreen/setup_baas_host_local.sh b/evergreen/setup_baas_host_local.sh index 4c49fcbf9f7..f40eaf03d82 100755 --- a/evergreen/setup_baas_host_local.sh +++ b/evergreen/setup_baas_host_local.sh @@ -20,12 +20,13 @@ DIRECT_PORT=9098 LISTEN_PORT=9092 CONFIG_PORT=8474 BAAS_PORT=9090 +BAAS_HOST_KEY= function usage() { - echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] HOST_VARS SSH_KEY" + echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] [-i SSH_KEY] HOST_VARS" echo -e "\tHOST_VARS\tPath to baas host vars script file" - echo -e "\tSSH_KEY\t\tPath to baas host private key file" + echo -e "\t -i SSH_KEY\t\tPath to baas host private key file" echo "Options:" echo -e "\t-w PATH\t\tPath to local baas server working directory (default ${BAAS_WORK_PATH})" echo -e "\t-u USER\t\tUsername to connect to baas host (default ${BAAS_USER})" @@ -42,7 +43,7 @@ function usage() exit "${1:0}" } -while getopts "w:u:b:ta:d:l:c:vh" opt; do +while getopts "w:u:b:ta:d:l:c:vhi:" opt; do case "${opt}" in w) BAAS_WORK_PATH="${OPTARG}";; u) BAAS_USER="${OPTARG}";; @@ -51,6 +52,7 @@ while getopts "w:u:b:ta:d:l:c:vh" opt; do d) DIRECT_PORT="${OPTARG}";; l) LISTEN_PORT="${OPTARG}";; c) CONFIG_PORT="${OPTARG}";; + i) BAAS_HOST_KEY="${OPTARG}";; v) VERBOSE="yes";; h) usage 0;; *) usage 1;; @@ -73,16 +75,7 @@ elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then usage 1 fi -if [[ $# -lt 1 ]]; then - echo "Error: Baas host private key not provided" - usage 1 -fi -BAAS_HOST_KEY="${1}"; shift; - -if [[ -z "${BAAS_HOST_KEY}" ]]; then - echo "Error: Baas host private key value was empty" - usage 1 -elif [[ ! -f "${BAAS_HOST_KEY}" ]]; then +if [[ -n "${BAAS_HOST_KEY}" && ! -f "${BAAS_HOST_KEY}" ]]; then echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}" usage 1 fi @@ -185,16 +178,20 @@ fi SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")" -ssh-agent > ssh_agent_commands.sh +SSH_OPTIONS=(-o ForwardAgent=yes -o StrictHostKeyChecking=no ) +if [[ -n "${BAAS_HOST_KEY}" ]]; then + ssh-agent > ssh_agent_commands.sh + + # shellcheck disable=SC1091 + source ssh_agent_commands.sh -# shellcheck disable=SC1091 -source ssh_agent_commands.sh + if [[ -f ~/.ssh/id_rsa ]]; then + ssh-add ~/.ssh/id_rsa + fi -if [[ -f ~/.ssh/id_rsa ]]; then - ssh-add ~/.ssh/id_rsa + ssh-add "${BAAS_HOST_KEY}" + SSH_OPTIONS+=(-o IdentitiesOnly=yes -i "${BAAS_HOST_KEY}") fi -ssh-add "${BAAS_HOST_KEY}" -SSH_OPTIONS=(-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i "${BAAS_HOST_KEY}") echo "running ssh with ${SSH_OPTIONS[*]}" diff --git a/evergreen/wait_for_remote_baas.sh b/evergreen/wait_for_remote_baas.sh new file mode 100755 index 00000000000..3eaf3da5915 --- /dev/null +++ b/evergreen/wait_for_remote_baas.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Wait for baas to be setup on a remote host + +set -o errexit +set -o pipefail + +EVERGREEN_PATH=./evergreen +BAAS_WORK_PATH=./baas-work-dir +BAAS_HOST_NAME= +BAAS_USER=ubuntu +VERBOSE= +BAAS_HOST_KEY= + +function usage() +{ + echo "Usage: wait_for_remote_baas.sh [-v] [-h] [-i SSH_KEY] HOST_VARS" + echo -e "\tHOST_VARS\tPath to baas host vars script file" + echo -e "\t-i SSH_KEY\t\tPath to baas host private key file" + echo "Options:" + echo -e "\t-v\t\tEnable verbose script debugging" + echo -e "\t-h\t\tShow this usage summary and exit" + echo "If an SSH_KEY is not provided, the script will assume an ssh agent is already running with" + echo "an appropriate key" + exit "${1:0}" +} + +while getopts "vhi:" opt; do + case "${opt}" in + v) VERBOSE="yes";; + i) BAAS_HOST_KEY="${OPTARG}";; + h) usage 0;; + *) usage 1;; + esac +done + +shift $((OPTIND - 1)) + +if [[ $# -lt 1 ]]; then + echo "Error: Baas host vars script not provided" + usage 1 +fi +BAAS_HOST_VARS="${1}"; shift; + +if [[ -z "${BAAS_HOST_VARS}" ]]; then + echo "Error: Baas host vars script value was empty" + usage 1 +elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then + echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}" + usage 1 +fi + +if [[ -n "${BAAS_HOST_KEY}" && ! -f "${BAAS_HOST_KEY}" ]]; then + echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}" + usage 1 +fi + +if [[ "${BAAS_USER}" = "root" ]]; then + FILE_DEST_DIR="/root/remote-baas" +else + FILE_DEST_DIR="/home/${BAAS_USER}/remote-baas" +fi +EVERGREEN_DEST_DIR="${FILE_DEST_DIR}/evergreen" + +# shellcheck disable=SC1090 +source "${BAAS_HOST_VARS}" + +# Wait until after the BAAS_HOST_VARS file is loaded to enable verbose tracing +if [[ -n "${VERBOSE}" ]]; then + set -o verbose + set -o xtrace +fi + +if [[ -z "${BAAS_HOST_NAME}" ]]; then + echo "Baas remote hostname (BAAS_HOST_NAME) not provided in baas host vars script" + usage 1 +fi + +if [[ -z "${BAAS_USER}" ]]; then + echo "Error: Baas host username was empty" + usage 1 +fi + +if [[ ! -d "${EVERGREEN_PATH}/" ]]; then + echo "This script must be run from the realm-core directory for accessing files in '${EVERGREEN_PATH}/'" + exit 1 +fi + +SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")" +SSH_OPTIONS=(-o ForwardAgent=yes -o StrictHostKeyChecking=no) + +if [[ -n "${BAAS_HOST_KEY}" ]]; then + ssh-agent > ssh_agent_commands.sh + + # shellcheck disable=SC1091 + source ssh_agent_commands.sh + + ssh-add "${BAAS_HOST_KEY}" + SSH_OPTIONS+=(-o IdentitiesOnly=yes -i "${BAAS_HOST_KEY}") +fi + +echo "running ssh with ${SSH_OPTIONS[*]}" +RETRY_COUNT=25 +WAIT_COUNTER=0 +WAIT_START=$(date -u +'%s') +CONNECT_COUNT=2 +TEST_COMMAND="[[ -f /data/baas-remote/baas-work-dir/baas_ready ]]" + +# Check for remote connectivity - try to connect twice to verify server is "really" ready +# The tests failed one time due to this ssh command passing, but the next scp command failed +while [[ ${CONNECT_COUNT} -gt 0 ]]; do + until ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=10 "${SSH_USER}" "${TEST_COMMAND}" ; do + if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]] ; then + secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) + echo "Timed out after waiting ${secs_spent_waiting} seconds for host ${BAAS_HOST_NAME} to start" + exit 1 + fi + + ((++WAIT_COUNTER)) + printf "SSH connection attempt %d/%d failed. Retrying...\n" "${WAIT_COUNTER}" "${RETRY_COUNT}" + sleep 10 + done + + ((CONNECT_COUNT--)) +done + +echo "Detected remote baas server ready"