Add non-ideal transfer and network fault tests with non-ideal network conditions

CHANGELOG.md

@@ -17,7 +17,8 @@
 -----------
 
 ### Internals
-* None.
+* Add baas-network-tests nightly task for testing sync client operation with non-ideal network conditions. ([PR #6852](https://github.com/realm/realm-core/pull/6852))
+* Added non-ideal network conditions and network fault tests to the evergreen nightly test runs. ([PR #7063](https://github.com/realm/realm-core/pull/7063))
 
 ----------------------------------------------
 
@@ -180,7 +181,6 @@
 
 ### Internals
 * Add a fake app id to the baas server's schema change history store to prevent server drop optimization from running during integration tests. ([PR #6927](https://github.com/realm/realm-core/pull/6927))
-* Add baas-network-tests nightly task for (future) testing sync client operation with non-ideal network conditions. ([PR #6852](https://github.com/realm/realm-core/pull/6852))
 
 ----------------------------------------------
 

evergreen/config.yml

@@ -138,6 +138,10 @@ functions:
           set_cmake_var realm_vars REALM_TEST_LOGGING BOOL On
           set_cmake_var realm_vars REALM_TEST_LOGGING_LEVEL STRING "${test_logging_level|debug}"
 
+          if [[ -n "${test_timeout_extra|}" ]]; then
+              set_cmake_var realm_vars REALM_TEST_TIMEOUT_EXTRA ${test_timeout_extra}
+          fi
+
           GENERATOR="${cmake_generator}"
           if [ -z "${cmake_generator|}" ]; then
               GENERATOR="Ninja Multi-Config"
@@ -594,6 +598,35 @@ functions:
 
         echo "Baas is started!"
 
+  "setup proxy parameters":
+  - command: shell.exec
+    params:
+      working_dir: realm-core
+      shell: bash
+      script: |-
+        set -o errexit
+        set -o pipefail
+        set -o verbose
+
+        if [[ -n "${disable_tests_against_baas|}" ]]; then
+          echo "Error: Bass is disabled for network tests"
+          exit 1
+        fi
+
+        if [[ -z "${proxy_toxics_file|}" ]]; then
+          echo "Error: Baas proxy toxics config file was not provided"
+          exit 1
+        fi
+
+        if [[ -n "${proxy_toxics_randoms|}" ]]; then
+          PROXY_RANDOMS="-r ${proxy_toxics_randoms}"
+        fi
+
+        # Configure the toxics for the baas proxy
+        evergreen/configure_baas_proxy.sh $PROXY_RANDOMS "${proxy_toxics_file}"
+        # Display the list of configured toxics
+        curl --silent http://localhost:8474/proxies/baas_proxy/toxics
+
   "check branch state":
   - command: shell.exec
     type: setup
@@ -843,11 +876,11 @@ tasks:
 - name: long-running-core-tests
   tags: [ "for_nightly_tests" ]
   allowed_requesters: [ "ad_hoc", "patch" ]
+  # The long-running tests can take a really long time on Windows, so we give the test up to 4
+  # hours to complete
+  exec_timeout_secs: 14400
   commands:
   - func: "run tests"
-    # The long-running tests can take a really long time on Windows, so we give the test up to 4
-    # hours to complete
-    timeout_secs: 14400
     vars:
       test_filter: CoreTests
       report_test_progress: On
@@ -922,7 +955,7 @@ tasks:
   commands:
   - func: "launch remote baas"
     vars:
-      baas_branch: master
+      baas_branch: 3f31617aacfe5d31b9057fc298b735b60acd6424
   - func: "compile"
     vars:
       target_to_build: ObjectStoreTests
@@ -936,16 +969,20 @@ tasks:
 
 - name: baas-network-tests
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch", "github_pr" ]
+  # The network tests can take a really long time, so we give the test up to 4
+  # hours to complete
   exec_timeout_secs: 14400
   commands:
   - func: "launch remote baas"
     vars:
-      baas_branch: master
+      baas_branch: 3f31617aacfe5d31b9057fc298b735b60acd6424
       baas_proxy: On
   - func: "compile"
     vars:
       target_to_build: ObjectStoreTests
   - func: "wait for baas to start"
+  - func: "setup proxy parameters"
   - func: "run tests"
     vars:
       test_label: objstore-baas
@@ -1398,8 +1435,28 @@ buildvariants:
   tasks:
   - name: fuzzer-tests
 
-- name: ubuntu2004-network
-  display_name: "Ubuntu 20.04 x86_64 (Clang 11 Baas network tests)"
+- name: ubuntu2004-network-nonideal
+  display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 Baas network tests - nonideal transfer)"
+  run_on: ubuntu2004-large
+  expansions:
+    clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz"
+    cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-linux-x86_64.tar.gz"
+    cmake_bindir: "./cmake_binaries/bin"
+    fetch_missing_dependencies: On
+    c_compiler: "./clang_binaries/bin/clang"
+    cxx_compiler: "./clang_binaries/bin/clang++"
+    cmake_build_type: RelWithDebInfo
+    run_with_encryption: On
+    baas_admin_port: 9098
+    test_logging_level: trace
+    test_timeout_extra: 60
+    proxy_toxics_file: evergreen/proxy-nonideal-transfer.toxics
+    proxy_toxics_randoms: "10:50|10:50"
+  tasks:
+  - name: network_tests
+
+- name: ubuntu2004-network-faulty
+  display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 Baas network tests - network faults)"
   run_on: ubuntu2004-large
   expansions:
     clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz"
@@ -1412,9 +1469,10 @@ buildvariants:
     run_with_encryption: On
     baas_admin_port: 9098
     test_logging_level: trace
+    proxy_toxics_file: evergreen/proxy-network-faults.toxics
+    proxy_toxics_randoms: "1000:3000|1000:3000|1000:1500|50:200"
   tasks:
   - name: network_tests
-    activate: true
 
 - name: rhel70
   display_name: "RHEL 7 x86_64"

evergreen/configure_baas_proxy.sh

@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+# The script to send and execute the configuration to set up the baas proxy toxics.
+#
+# Usage:
+# ./evergreen/configure_baas_proxy.sh [-c PORT] [-r NUM] [-p PATH] [-v] [-h] CONFIG_JSON
+#
+
+set -o errexit
+set -o errtrace
+set -o pipefail
+
+CONFIG_PORT=8474
+PROXY_NAME="baas_proxy"
+RND_STRING=
+RND_MIN=
+RND_MAX=
+RND_DIFF=32767
+CURL=/usr/bin/curl
+VERBOSE=
+CONFIG_TMP_DIR=
+
+function usage()
+{
+    echo "Usage: configure_baas_proxy.sh [-c PORT] [-r MIN:MAX] [-p PATH] [-t NAME] [-v] [-h] CONFIG_JSON"
+    echo -e "\tCONFIG_JSON\tPath to baas proxy toxics config file (one toxic config JSON object per line)"
+    echo "Options:"
+    echo -e "\t-c PORT\t\tLocal configuration port for proxy HTTP API (default ${CONFIG_PORT})"
+    echo -e "\t-r MIN:MAX\tString containing one or more sets of min:max values to replace %RANDOM#% in toxics"
+    echo -e "\t-p PATH\t\tPath to the curl executable (default ${CURL})"
+    echo -e "\t-t NAME\t\tName of the proxy to be configured (default ${PROXY_NAME})"
+    echo -e "\t-v\t\tEnable verbose script debugging"
+    echo -e "\t-h\t\tShow this usage summary and exit"
+    # Default to 0 if exit code not provided
+    exit "${1:0}"
+}
+
+while getopts "c:r:p:vh" opt; do
+    case "${opt}" in
+        c) CONFIG_PORT="${OPTARG}";;
+        r) RND_STRING="${OPTARG}";;
+        p) CURL="${OPTARG}";;
+        v) VERBOSE="yes";;
+        h) usage 0;;
+        *) usage 1;;
+    esac
+done
+
+TOXIPROXY_URL="http://localhost:${CONFIG_PORT}"
+
+shift $((OPTIND - 1))
+
+if [[ $# -lt 1 ]]; then
+    echo "Error: Baas proxy toxics config file not provided"
+    usage 1
+fi
+PROXY_JSON_FILE="${1}"; shift;
+
+if [[ -z "${PROXY_JSON_FILE}" ]]; then
+    echo "Error: Baas proxy toxics config file value was empty"
+    usage 1
+elif [[ ! -f "${PROXY_JSON_FILE}" ]]; then
+    echo "Error: Baas proxy toxics config file not found: ${PROXY_JSON_FILE}"
+    usage 1
+fi
+
+if [[ -z "${CURL}" ]]; then
+    echo "Error: curl path is empty"
+    usage 1
+elif [[ ! -x "${CURL}" ]]; then
+    echo "Error: curl path is not valid: ${CURL}"
+    usage 1
+fi
+
+trap 'catch $? ${LINENO}' ERR
+trap 'on_exit' INT TERM EXIT
+
+# Set up catch function that runs when an error occurs
+function catch()
+{
+    # Usage: catch EXIT_CODE LINE_NUM
+    echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while configuring baas proxy"
+}
+
+function on_exit()
+{
+    # Usage: on_exit
+    if [[ -n "${CONFIG_TMP_DIR}" && -d "${CONFIG_TMP_DIR}" ]]; then
+        rm -rf "${CONFIG_TMP_DIR}"
+    fi
+}
+
+function check_port()
+{
+    # Usage check_port PORT
+    port_num="${1}"
+    if [[ -n "${port_num}" && ${port_num} -gt 0 && ${port_num} -lt 65536 ]]; then
+        return 0
+    fi
+    return 1
+}
+
+function check_port_ready()
+{
+    # Usage: check_port_active PORT PORT_NAME
+    port_num="${1}"
+    port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true)
+    if [[ -z "${port_check}" ]]; then
+        echo "Error: ${2} port (${port_num}) is not ready - is the Baas proxy running?"
+        exit 1
+    fi
+    if ! curl "${TOXIPROXY_URL}/version" --silent --fail --connect-timeout 10 > /dev/null; then
+        echo "Error: No response from ${2} (${port_num}) - is the Baas proxy running?"
+        exit 1
+    fi
+}
+
+function parse_random()
+{
+    # Usage: parse_random RANDOM_STRING => RND_MIN, RND_MAX
+    random_string="${1}"
+    old_ifs="${IFS}"
+
+    RND_MIN=()
+    RND_MAX=()
+
+    if [[ "${random_string}" =~ .*|.* ]]; then
+        IFS='|'
+        read -ra random_list <<< "${random_string}"
+    else
+        random_list=("${random_string}")
+    fi
+    for random in "${random_list[@]}"
+    do
+        if [[ ! "${random}" =~ .*:.* ]]; then
+            IFS="${old_ifs}"
+            return 1
+        fi
+
+        # Setting IFS (input field separator) value as ":" and read the split string into array
+        IFS=':'
+        read -ra rnd_arr <<< "${random}"
+
+        if [[ ${#rnd_arr[@]} -ne 2 ]]; then
+            IFS="${old_ifs}"
+            return 1
+        elif [[ -z "${rnd_arr[0]}" || -z "${rnd_arr[0]}" ]]; then
+            IFS="${old_ifs}"
+            return 1
+        fi
+
+        if [[ ${rnd_arr[0]} -le ${rnd_arr[1]} ]]; then
+            RND_MIN+=("${rnd_arr[0]}")
+            RND_MAX+=("${rnd_arr[1]}")
+        else
+            RND_MIN+=("${rnd_arr[1]}")
+            RND_MAX+=("${rnd_arr[0]}")
+        fi
+    done
+    IFS="${old_ifs}"
+    return 0
+}
+
+function generate_random()
+{
+    # Usage: generate_random MINVAL MAXVAL => RAND_VAL
+    minval="${1}"
+    maxval="${2}"
+    diff=$(( "${maxval}" - "${minval}" ))
+    if [[ ${diff} -gt ${RND_DIFF} ]]; then
+        return 1
+    fi
+    RAND_VAL=$(( "$minval" + $(("$RANDOM" % "$diff")) ))
+}
+
+# Wait until after the functions are configured before enabling verbose tracing
+if [[ -n "${VERBOSE}" ]]; then
+    set -o verbose
+    set -o xtrace
+fi
+
+if ! check_port "${CONFIG_PORT}"; then
+    echo "Error: Baas proxy HTTP API config port was invalid: '${CONFIG_PORT}'"
+    usage 1
+fi
+
+# Parse and verify the random string, if provided
+if [[ -n "${RND_STRING}" ]]; then
+    if ! parse_random "${RND_STRING}"; then
+        echo "Error: Malformed random string: ${random_string} - format 'MIN:MAX[|MIN:MAX[|...]]"
+        usage 1
+    fi
+fi
+
+# Verify the Baas proxy is ready to roll
+check_port_ready "${CONFIG_PORT}" "Baas proxy HTTP API config"
+
+# Create a temp directory for constructing the updated config file
+CONFIG_TMP_DIR=$(mktemp -d -t "proxy-config.XXXXXX")
+cp "${PROXY_JSON_FILE}" "${CONFIG_TMP_DIR}"
+json_file="$(basename "${PROXY_JSON_FILE}")"
+TMP_CONFIG="${CONFIG_TMP_DIR}/${json_file}"
+
+if [[ ${#RND_MIN[@]} -gt 0 ]]; then
+    cnt=0
+    while [[ cnt -lt ${#RND_MIN[@]} ]]; do
+        rndmin=${RND_MIN[cnt]}
+        rndmax=${RND_MAX[cnt]}
+        if ! generate_random "${rndmin}" "${rndmax}"; then
+            echo "Error: MAX - MIN cannot be more than ${RND_DIFF}"
+            exit 1
+        fi
+
+        cnt=$((cnt + 1))
+        printf "Generated random value #%d from %d to %d: %d\n" "${cnt}" "${rndmin}" "${rndmax}" "${RAND_VAL}"
+        sed_pattern=$(printf "s/%%RANDOM%d%%/%d/g" "${cnt}" "${RAND_VAL}")
+        sed -i.bak "${sed_pattern}" "${TMP_CONFIG}"
+    done
+fi
+
+# Get the current list of configured toxics for the baas_proxy proxy
+TOXICS=$(${CURL} --silent "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics")
+if [[ "${TOXICS}" != "[]" ]]; then
+    # Extract the toxic names from the toxics list JSON
+    # Steps: Remove brackets, split into lines, extract "name" value
+    mapfile -t TOXIC_LIST < <(echo "${TOXICS}" | sed 's/\[\(.*\)\]/\1/g' | sed  's/},{/}\n{/g' | sed 's/.*"name":"\([^"]*\).*/\1/g')
+    echo "Clearing existing set of toxics (${#TOXIC_LIST[@]}) for ${PROXY_NAME} proxy"
+    for toxic in "${TOXIC_LIST[@]}"
+    do
+        ${CURL} -X DELETE "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics/${toxic}"
+    done
+fi
+
+# Configure the new set of toxics for the baas_proxy proxy
+echo "Configuring toxics for ${PROXY_NAME} proxy with file: ${json_file}"
+while IFS= read -r line; do
+    ${CURL} -X POST -H "Content-Type: application/json" --silent -d "${line}" "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics" > /dev/null
+done < "${TMP_CONFIG}"

evergreen/proxy-network-faults.toxics

@@ -0,0 +1,4 @@
+{"name": "limit-data-upstream","type": "limit_data","stream": "upstream","toxicity": 0.6,"attributes": {"bytes": %RANDOM1%}}
+{"name": "limit-data-downstream","type": "limit_data","stream": "downstream","toxicity": 0.6,"attributes": {"bytes": %RANDOM2%}}
+{"name": "slow-close-upstream","type": "slow_close","stream": "upstream","toxicity": 0.6,"attributes": {"delay": %RANDOM3%}}
+{"name": "reset-peer-upstream","type": "reset_peer","stream": "upstream","toxicity": 0.6,"attributes": {"timeout": %RANDOM4%}}