Skip to content

Commit

Permalink
Update mongod used by baas in testing to 7.0.3 (#7187)
Browse files Browse the repository at this point in the history
  • Loading branch information
jbreams authored Dec 7, 2023
1 parent 9686dff commit c41cfd0
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 36 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

### Internals
* Added a CombinedTests target that runs tests from object-store, sync, and core together. ([PR #6964](https://github.com/realm/realm-core/pull/6964))
* The version of mongod we test against in evergreen was updated to 7.0.3 ([PR #7187](https://github.com/realm/realm-core/pull/7187)).

----------------------------------------------

Expand Down
22 changes: 20 additions & 2 deletions evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -580,7 +580,7 @@ functions:
# Run the setup_baas_host_local.sh script to configure and run baas on the remote host
# Add -v to this command for verbose script logging
./evergreen/setup_baas_host_local.sh -w ./baas-work-dir -u $BAAS_USER $OPT_BAAS_BRANCH $OPT_BAAS_PROXY \
$OPT_BAAS_DIRECT ./baas_host_vars.sh ./.baas_ssh_key 2>&1 | tee install_baas_output.log
$OPT_BAAS_DIRECT -i ./.baas_ssh_key ./baas_host_vars.sh 2>&1 | tee install_baas_output.log
"wait for baas to start":
- command: shell.exec
Expand All @@ -600,6 +600,24 @@ functions:
echo "Baas is started!"
"wait for remote baas to start":
- command: shell.exec
params:
working_dir: realm-core
shell: bash
script: |-
set -o errexit
set -o verbose
if [[ -n "${disable_tests_against_baas|}" ]]; then
exit 0
fi
# Don't print out the tail of the log file
./evergreen/wait_for_remote_baas.sh -i ./.baas_ssh_key ./baas_host_vars.sh ./.baas_ssh_key
echo "Baas is started!"
"setup proxy parameters":
- command: shell.exec
params:
Expand Down Expand Up @@ -962,7 +980,7 @@ tasks:
- func: "compile"
vars:
target_to_build: ObjectStoreTests
- func: "wait for baas to start"
- func: "wait for remote baas to start"
- func: "run tests"
vars:
test_label: objstore-baas
Expand Down
36 changes: 22 additions & 14 deletions evergreen/install_baas.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ function setup_baas_dependencies() {
Darwin)
if [[ "$(uname -m)" == "arm64" ]]; then
export GOARCH=arm64
MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-arm64-enterprise-6.0.0-rc13.tgz"
MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-1.5.0-darwin-arm64.zip"
MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-arm64-enterprise-7.0.3.tgz"
MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-2.1.1-darwin-arm64.zip"
# <-- Remove after enabling constants.sh
STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-6.1.0-alpha-527-g796351f.tgz"
ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg"
ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_c5880b9bd6039908a2fd85d1c95f457c36b5d33b_6542b80ae3c331e8d3788186_23_11_01_20_41_47/assisted_agg"
GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-arm64.tar.gz"
# -->

Expand All @@ -87,10 +87,11 @@ function setup_baas_dependencies() {
BAAS_PLATFORM="Darwin_arm64"
else
export GOARCH=amd64
MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-x86_64-enterprise-5.0.3.tgz"
MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-x86_64-enterprise-7.0.3.tgz"
MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-2.1.1-darwin-x64.zip"
# <-- Remove after enabling constants.sh
STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz"
ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg"
ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_c5880b9bd6039908a2fd85d1c95f457c36b5d33b_6542b80ae3c331e8d3788186_23_11_01_20_41_47/assisted_agg"
GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-amd64.tar.gz"
# -->
BAAS_PLATFORM="Darwin_x86_64"
Expand All @@ -113,9 +114,10 @@ function setup_baas_dependencies() {
DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "${DISTRO_VERSION}")"
fi
platform_string="${BAAS_PLATFORM} - ${DISTRO_NAME} ${DISTRO_VERSION}"
MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-2.1.1-linux-x64.tgz"
case "${DISTRO_NAME}" in
ubuntu | linuxmint)
MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-5.0.3.tgz"
MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-7.0.3.tgz"
# <-- Remove after enabling constants.sh
LIBMONGO_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_ubuntu2004_x86_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo-ubuntu2004-x86_64.so"
STITCH_SUPPORT_LIB_URL="https://s3.amazonaws.com/static.realm.io/stitch-support/stitch-support-ubuntu2004-4.4.17-rc1-2-g85de0cc.tgz"
Expand All @@ -125,7 +127,7 @@ function setup_baas_dependencies() {
rhel)
case "${DISTRO_VERSION_MAJOR}" in
7)
MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-5.0.3.tgz"
MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-7.0.3.tgz"
# <-- Remove after enabling constants.sh
LIBMONGO_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_linux_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo.so"
STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/linux-x64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz"
Expand Down Expand Up @@ -459,18 +461,24 @@ if [ ! -x "${MONGO_BINARIES_DIR}/bin/mongod" ]; then
fi
echo "mongod version: $("${MONGO_BINARIES_DIR}/bin/mongod" --version --quiet | sed 1q)"

if [[ -n "${MONGOSH_DOWNLOAD_URL}" ]]; then
if [[ ! -x "${MONGO_BINARIES_DIR}/bin/mongosh" ]]; then
echo "Downloading mongosh"
if [[ ! -x "${MONGO_BINARIES_DIR}/bin/mongosh" ]]; then
MONGOSH_DOWNLOAD_FILENAME=$(basename "${MONGOSH_DOWNLOAD_URL}")
MONGOSH_DOWNLOAD_EXTENSION="${MONGOSH_DOWNLOAD_FILENAME##*.}"
echo "Downloading ${MONGOSH_DOWNLOAD_URL}"
if [[ "${MONGOSH_DOWNLOAD_EXTENSION}" == "zip" ]]; then
${CURL} -sLS "${MONGOSH_DOWNLOAD_URL}" --output mongosh-binaries.zip
unzip -jnqq mongosh-binaries.zip '*/bin/*' -d "${MONGO_BINARIES_DIR}/bin/"
rm mongosh-binaries.zip
elif [[ ${MONGOSH_DOWNLOAD_EXTENSION} == "tgz" ]]; then
${CURL} -sLS "${MONGOSH_DOWNLOAD_URL}" --output mongosh-binaries.tgz
tar -xzf mongosh-binaries.tgz --strip-components=1 -C "${MONGO_BINARIES_DIR}"
rm mongosh-binaries.tgz
else
echo "Unsupported mongosh format $MONGOSH_DOWNLOAD_EXTENSION"
exit 1
fi
MONGOSH="mongosh"
else
# Use the mongo shell provided with mongod
MONGOSH="mongo"
fi
MONGOSH="mongosh"
chmod +x "${MONGO_BINARIES_DIR}/bin"/*
echo "${MONGOSH} version: $("${MONGO_BINARIES_DIR}/bin/${MONGOSH}" --version)"

Expand Down
37 changes: 17 additions & 20 deletions evergreen/setup_baas_host_local.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,13 @@ DIRECT_PORT=9098
LISTEN_PORT=9092
CONFIG_PORT=8474
BAAS_PORT=9090
BAAS_HOST_KEY=

function usage()
{
echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] HOST_VARS SSH_KEY"
echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] [-i SSH_KEY] HOST_VARS"
echo -e "\tHOST_VARS\tPath to baas host vars script file"
echo -e "\tSSH_KEY\t\tPath to baas host private key file"
echo -e "\t -i SSH_KEY\t\tPath to baas host private key file"
echo "Options:"
echo -e "\t-w PATH\t\tPath to local baas server working directory (default ${BAAS_WORK_PATH})"
echo -e "\t-u USER\t\tUsername to connect to baas host (default ${BAAS_USER})"
Expand All @@ -42,7 +43,7 @@ function usage()
exit "${1:0}"
}

while getopts "w:u:b:ta:d:l:c:vh" opt; do
while getopts "w:u:b:ta:d:l:c:vhi:" opt; do
case "${opt}" in
w) BAAS_WORK_PATH="${OPTARG}";;
u) BAAS_USER="${OPTARG}";;
Expand All @@ -51,6 +52,7 @@ while getopts "w:u:b:ta:d:l:c:vh" opt; do
d) DIRECT_PORT="${OPTARG}";;
l) LISTEN_PORT="${OPTARG}";;
c) CONFIG_PORT="${OPTARG}";;
i) BAAS_HOST_KEY="${OPTARG}";;
v) VERBOSE="yes";;
h) usage 0;;
*) usage 1;;
Expand All @@ -73,16 +75,7 @@ elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then
usage 1
fi

if [[ $# -lt 1 ]]; then
echo "Error: Baas host private key not provided"
usage 1
fi
BAAS_HOST_KEY="${1}"; shift;

if [[ -z "${BAAS_HOST_KEY}" ]]; then
echo "Error: Baas host private key value was empty"
usage 1
elif [[ ! -f "${BAAS_HOST_KEY}" ]]; then
if [[ -n "${BAAS_HOST_KEY}" && ! -f "${BAAS_HOST_KEY}" ]]; then
echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}"
usage 1
fi
Expand Down Expand Up @@ -185,16 +178,20 @@ fi

SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")"

ssh-agent > ssh_agent_commands.sh
SSH_OPTIONS=(-o ForwardAgent=yes -o StrictHostKeyChecking=no )
if [[ -n "${BAAS_HOST_KEY}" ]]; then
ssh-agent > ssh_agent_commands.sh

# shellcheck disable=SC1091
source ssh_agent_commands.sh

# shellcheck disable=SC1091
source ssh_agent_commands.sh
if [[ -f ~/.ssh/id_rsa ]]; then
ssh-add ~/.ssh/id_rsa
fi

if [[ -f ~/.ssh/id_rsa ]]; then
ssh-add ~/.ssh/id_rsa
ssh-add "${BAAS_HOST_KEY}"
SSH_OPTIONS+=(-o IdentitiesOnly=yes -i "${BAAS_HOST_KEY}")
fi
ssh-add "${BAAS_HOST_KEY}"
SSH_OPTIONS=(-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -i "${BAAS_HOST_KEY}")

echo "running ssh with ${SSH_OPTIONS[*]}"

Expand Down
126 changes: 126 additions & 0 deletions evergreen/wait_for_remote_baas.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#!/usr/bin/env bash
# Wait for baas to be setup on a remote host

set -o errexit
set -o pipefail

EVERGREEN_PATH=./evergreen
BAAS_WORK_PATH=./baas-work-dir
BAAS_HOST_NAME=
BAAS_USER=ubuntu
VERBOSE=
BAAS_HOST_KEY=

function usage()
{
echo "Usage: wait_for_remote_baas.sh [-v] [-h] [-i SSH_KEY] HOST_VARS"
echo -e "\tHOST_VARS\tPath to baas host vars script file"
echo -e "\t-i SSH_KEY\t\tPath to baas host private key file"
echo "Options:"
echo -e "\t-v\t\tEnable verbose script debugging"
echo -e "\t-h\t\tShow this usage summary and exit"
echo "If an SSH_KEY is not provided, the script will assume an ssh agent is already running with"
echo "an appropriate key"
exit "${1:0}"
}

while getopts "vhi:" opt; do
case "${opt}" in
v) VERBOSE="yes";;
i) BAAS_HOST_KEY="${OPTARG}";;
h) usage 0;;
*) usage 1;;
esac
done

shift $((OPTIND - 1))

if [[ $# -lt 1 ]]; then
echo "Error: Baas host vars script not provided"
usage 1
fi
BAAS_HOST_VARS="${1}"; shift;

if [[ -z "${BAAS_HOST_VARS}" ]]; then
echo "Error: Baas host vars script value was empty"
usage 1
elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then
echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}"
usage 1
fi

if [[ -n "${BAAS_HOST_KEY}" && ! -f "${BAAS_HOST_KEY}" ]]; then
echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}"
usage 1
fi

if [[ "${BAAS_USER}" = "root" ]]; then
FILE_DEST_DIR="/root/remote-baas"
else
FILE_DEST_DIR="/home/${BAAS_USER}/remote-baas"
fi
EVERGREEN_DEST_DIR="${FILE_DEST_DIR}/evergreen"

# shellcheck disable=SC1090
source "${BAAS_HOST_VARS}"

# Wait until after the BAAS_HOST_VARS file is loaded to enable verbose tracing
if [[ -n "${VERBOSE}" ]]; then
set -o verbose
set -o xtrace
fi

if [[ -z "${BAAS_HOST_NAME}" ]]; then
echo "Baas remote hostname (BAAS_HOST_NAME) not provided in baas host vars script"
usage 1
fi

if [[ -z "${BAAS_USER}" ]]; then
echo "Error: Baas host username was empty"
usage 1
fi

if [[ ! -d "${EVERGREEN_PATH}/" ]]; then
echo "This script must be run from the realm-core directory for accessing files in '${EVERGREEN_PATH}/'"
exit 1
fi

SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")"
SSH_OPTIONS=(-o ForwardAgent=yes -o StrictHostKeyChecking=no)

if [[ -n "${BAAS_HOST_KEY}" ]]; then
ssh-agent > ssh_agent_commands.sh

# shellcheck disable=SC1091
source ssh_agent_commands.sh

ssh-add "${BAAS_HOST_KEY}"
SSH_OPTIONS+=(-o IdentitiesOnly=yes -i "${BAAS_HOST_KEY}")
fi

echo "running ssh with ${SSH_OPTIONS[*]}"
RETRY_COUNT=25
WAIT_COUNTER=0
WAIT_START=$(date -u +'%s')
CONNECT_COUNT=2
TEST_COMMAND="[[ -f /data/baas-remote/baas-work-dir/baas_ready ]]"

# Check for remote connectivity - try to connect twice to verify server is "really" ready
# The tests failed one time due to this ssh command passing, but the next scp command failed
while [[ ${CONNECT_COUNT} -gt 0 ]]; do
until ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=10 "${SSH_USER}" "${TEST_COMMAND}" ; do
if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]] ; then
secs_spent_waiting=$(($(date -u +'%s') - WAIT_START))
echo "Timed out after waiting ${secs_spent_waiting} seconds for host ${BAAS_HOST_NAME} to start"
exit 1
fi

((++WAIT_COUNTER))
printf "SSH connection attempt %d/%d failed. Retrying...\n" "${WAIT_COUNTER}" "${RETRY_COUNT}"
sleep 10
done

((CONNECT_COUNT--))
done

echo "Detected remote baas server ready"

0 comments on commit c41cfd0

Please sign in to comment.