RCORE-1928 Use baasaas for baas integration tests in CI (#7423)

CHANGELOG.md

@@ -16,7 +16,8 @@
 -----------
 
 ### Internals
-* None.
+* The CMake option `REALM_MONGODB_ENDPOINT` for running the object-store-tests against baas has been deprecated in favor of an environment variable of the same name ([PR #7423](https://github.com/realm/realm-core/pull/7423)).
+* The object-store-tests test suite can now launch baas containers on its own by specifying a `BAASAAS_API_KEY` in the environment ([PR #7423](https://github.com/realm/realm-core/pull/7423)).
 
 ----------------------------------------------
 

dependencies.yml

@@ -3,5 +3,5 @@ VERSION: 14.2.0
 OPENSSL_VERSION: 3.2.0
 ZLIB_VERSION: 1.2.13
 # https://github.com/10gen/baas/commits
-# dd016 is 2024 Feb 22
-BAAS_VERSION: dd01629d83b86292af9c59ebe2a28673c2e559cf
+# acb71d0 is 2024 Mar 12
+BAAS_VERSION: acb71d0183b33eb304bb496390567efcfb8a6e60

evergreen/config.yml

@@ -78,7 +78,6 @@ functions:
           if [ -z "${disable_tests_against_baas|}" ]; then
               scheme="http"
               set_cmake_var baas_vars REALM_ENABLE_AUTH_TESTS BOOL On
-              set_cmake_var baas_vars REALM_MONGODB_ENDPOINT STRING "$scheme://localhost:9090"
               if [ -n "${baas_admin_port|}" ]; then
                 set_cmake_var baas_vars REALM_ADMIN_ENDPOINT STRING "$scheme://localhost:${baas_admin_port}"
               fi
@@ -156,7 +155,7 @@ functions:
           if [ -n "${curl_base|}" ]; then
               set_cmake_var curl_vars CURL_LIBRARY PATH "$(./evergreen/abspath.sh ${curl_base}/lib/libcurl.dll.a)"
               set_cmake_var curl_vars CURL_INCLUDE_DIR PATH "$(./evergreen/abspath.sh ${curl_base}/include)"
-              set_cmake_var baas_vars REALM_CURL_CACERTS PATH "$(./evergreen/abspath.sh "${curl_base}/bin/cacert.pem")"
+              set_cmake_var baas_vars REALM_CURL_CACERTS PATH "$(./evergreen/abspath.sh "${curl_base}/bin/curl-ca-bundle.crt")"
           fi
 
           set_cmake_var realm_vars REALM_NO_TESTS BOOL ${no_tests|Off}
@@ -208,10 +207,17 @@ functions:
         file: './realm-core/benchmark_results/results.latest.json'
 
   "run tests":
+    - command: expansions.update
+      params:
+        file: realm-core/dependencies.yml
     - command: shell.exec
       params:
         working_dir: realm-core
         shell: bash
+        env:
+          BAASAAS_API_KEY: "${baasaas_api_key}"
+          BAASAAS_REF_SPEC: "${BAAS_VERSION}"
+          BAASAAS_START_MODE: "githash"
         script: |-
           set -o errexit
           set -o verbose
@@ -244,6 +250,12 @@ functions:
 
           TEST_FLAGS="--no-tests=error $TEST_FLAGS ${test_flags|}"
 
+          if [[ -n "${disable_tests_against_baas|}" ]]; then
+              unset BAASAAS_API_KEY
+              unset BAASAAS_REF_SPEC
+              unset BAASAAS_START_MODE
+          fi
+
           if [[ -n "${llvm_symbolizer}" ]]; then
               export ASAN_SYMBOLIZER_PATH="$(./evergreen/abspath.sh ${llvm_symbolizer})"
           fi
@@ -977,18 +989,17 @@ tasks:
       test_label: objstore-local
       test_executable_name: "realm-object-store-tests"
       verbose_test_output: true
+      disable_tests_against_baas: true
   - func: "check branch state"
 
 # These are baas object store tests that run against baas running on a remote host
 - name: baas-integration-tests
   tags: [ "test_suite", "for_pull_requests", "requires_baas" ]
   exec_timeout_secs: 3600
   commands:
-  - func: "launch remote baas"
   - func: "compile"
     vars:
       target_to_build: ObjectStoreTests
-  - func: "wait for remote baas to start"
   - func: "run tests"
     vars:
       test_label: objstore-baas

how-to-build.md

@@ -145,39 +145,41 @@ These are the available variables:
    testing process as soon as a check fails or an unexpected exception is thrown
    in a test.
 
-## Running [app] tests against a local MongoDB BAAS
-
-Due to MongoDB security policies, running baas requires company issued AWS account credentials.
-These are for MongoDB employees only, if you do not have these, reach out to #realm-core.
-Once you have them, they need to be set in the shell environment.
-
-First, log in to aws using their command line tool. On mac this requries `brew install awscli`.
-Then login using `aws configure` and input your access key and secret acess key. The other 
-configuration options can be left as none. This creates a correctly formatted file locally at
-`~/.aws/credentials` which we will use later.
-
-If you do not want to install the aws command line tools, you can also create the aws file
-manually in the correct location (`~/.aws/credentials`) with the following contents:
+## Running [app] tests against a local BAAS instance
 
+If you already have a baas instance running, you can specify that directly via the
+`BAAS_BASE_URL` environment variable. You can run baas in a local docker container using
+instructions from [the wiki](https://wiki.corp.mongodb.com/display/10GEN/%28Device+Sync%29+Using+Docker+to+run+a+BAAS+server+instance).
 ```
-AWS_ACCESS_KEY_ID = <your-key-id>
-AWS_SECRET_ACCESS_KEY = <your-secret-key>
+export BAAS_BASE_URL=http://localhost:9090
+mkdir build.sync.ninja
+cmake -B build.sync.ninja -G Ninja -DREALM_ENABLE_AUTH_TESTS=1
+cmake --build build.sync.ninja --target realm-object-store-tests
+./build.sync.ninja/test/object-store/realm-object-store-tests -d=1
 ```
 
-We use a script to fetch the dependencies for and run baas locally. Use the `-b sha` to use a particular version from https://github.com/10gen/baas/
-The script uses the configuration from https://github.com/10gen/baas/blob/master/etc/configs/test_rcore_config.json
+## Running [app] tests against an on-demand BAASAAS container
 
+Due to MongoDB security policies, running baas requires company issued credentials.
+These are for MongoDB employees only, if you do not have these, reach out to
+#appx-device-sync-internal. Once you have a baasaas API key, it needs to be set
+in the shell environment.
 ```
-./evergreen/install_baas.sh -w baas
-```
-
-To run the [app] tests against the local baas, you need to configure a build with some cmake options to tell the tests where to point to.
-```
+export BAASAAS_API_KEY=<your API key here>
 mkdir build.sync.ninja
-cmake -B build.sync.ninja -G Ninja -DREALM_ENABLE_AUTH_TESTS=1 -DREALM_MONGODB_ENDPOINT=http://localhost:9090
+cmake -B build.sync.ninja -G Ninja -DREALM_ENABLE_AUTH_TESTS=1
 cmake --build build.sync.ninja --target realm-object-store-tests
 ./build.sync.ninja/test/object-store/realm-object-store-tests -d=1
 ```
+You can tell the object-store tests to use a specific version of baas with the
+`BAASAAS_START_MODE` environment variable, which can either be `githash`, `patchid`,
+or `branch`. If you specify a start mode, you need to tell it which githash or
+branch name to start with via the `BAASAAS_REF_SPEC` environment variable. Omitting
+these will use the latest available commit from the main branch of baas.
+
+If you've started a baasaas container already via the baasaas CLI, you can tell
+the object-store tests to use that with the `BAASAAS_INSTANCE_ID` environment variable.
+
 
 ### Developing inside a container
 

test/object-store/CMakeLists.txt

@@ -92,6 +92,12 @@ endif()
 target_link_libraries(ObjectStoreTestLib Catch2::Catch2 ObjectStore RealmFFIStatic TestUtil)
 enable_stdfilesystem(ObjectStoreTestLib)
 
+if(REALM_CURL_CACERTS)
+  target_compile_definitions(ObjectStoreTestLib PRIVATE
+    REALM_CURL_CACERTS="${REALM_CURL_CACERTS}"
+  )
+endif()
+
 add_executable(ObjectStoreTests main.cpp ${RESOURCES})
 set_target_properties(ObjectStoreTests PROPERTIES OUTPUT_NAME realm-object-store-tests)
 target_link_libraries(ObjectStoreTests ObjectStoreTestLib TestUtil)
@@ -131,16 +137,17 @@ if(REALM_ENABLE_SYNC)
     target_link_libraries(ObjectStoreTestLib SyncServer)
     option(REALM_ENABLE_AUTH_TESTS "" OFF)
     if(REALM_ENABLE_AUTH_TESTS)
-        if(NOT REALM_MONGODB_ENDPOINT)
-            message(FATAL_ERROR "REALM_MONGODB_ENDPOINT must be set when specifying REALM_ENABLE_AUTH_TESTS.")
-        endif()
-
-        message(STATUS "Auth tests enabled: ${REALM_MONGODB_ENDPOINT}")
         target_compile_definitions(ObjectStoreTestLib PRIVATE
             REALM_ENABLE_AUTH_TESTS=1
-            REALM_MONGODB_ENDPOINT="${REALM_MONGODB_ENDPOINT}"
         )
 
+        if(REALM_MONGODB_ENDPOINT)
+          message(STATUS "Auth tests enabled: ${REALM_MONGODB_ENDPOINT}")
+          target_compile_definitions(ObjectStoreTestLib PRIVATE
+              REALM_MONGODB_ENDPOINT="${REALM_MONGODB_ENDPOINT}"
+          )
+        endif()
+
         if(REALM_ADMIN_ENDPOINT)
             message(STATUS "BAAS admin endpoint: ${REALM_ADMIN_ENDPOINT}")
             target_compile_definitions(ObjectStoreTests PRIVATE

test/object-store/sync/app.cpp

@@ -3924,7 +3924,6 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") {
     std::unique_ptr<realm::AppSession> app_session;
     auto redir_transport = std::make_shared<BaseUrlTransport>();
     AutoVerifiedEmailCredentials creds;
-    util::LogCategory::realm.set_default_level_threshold(realm::util::Logger::Level::TEST_LOGGING_LEVEL);
     auto logger = util::Logger::get_default_logger();
 
     App::Config app_config = {"fake-app-id"};

test/object-store/test_runner.cpp

@@ -20,6 +20,7 @@
 #include <util/test_path.hpp>
 
 #include <realm/util/features.h>
+#include <realm/util/logger.hpp>
 #include <realm/util/to_string.hpp>
 
 #if TEST_SCHEDULER_UV
@@ -38,6 +39,76 @@
 #include <iostream>
 #include <limits.h>
 
+#ifndef TEST_ENABLE_LOGGING
+#define TEST_ENABLE_LOGGING 0 // change to 1 to enable trace-level logging
+#endif
+
+#ifndef TEST_LOGGING_LEVEL
+#if TEST_ENABLE_LOGGING
+#define TEST_LOGGING_LEVEL all
+#else
+#define TEST_LOGGING_LEVEL off
+#endif // TEST_ENABLE_LOGGING
+#endif // TEST_LOGGING_LEVEL
+
+#define TEST_LOGGING_LEVEL_STORAGE off
+#define TEST_LOGGING_LEVEL_SERVER off
+/*
+#define TEST_LOGGING_LEVEL_SYNC off
+#define TEST_LOGGING_LEVEL_RESET trace
+#define TEST_LOGGING_LEVEL_APP off
+*/
+
+static std::vector<std::pair<std::string_view, realm::util::Logger::Level>> default_log_levels = {
+    {"Realm", realm::util::Logger::Level::TEST_LOGGING_LEVEL},
+#ifdef TEST_LOGGING_LEVEL_STORAGE
+    {"Realm.Storage", realm::util::Logger::Level::TEST_LOGGING_LEVEL_STORAGE},
+#endif
+#ifdef TEST_LOGGING_LEVEL_TRANSACTION
+    {"Realm.Storage.Transaction", realm::util::Logger::Level::TEST_LOGGING_LEVEL_TRANSACTION},
+#endif
+#ifdef TEST_LOGGING_LEVEL_QUERY
+    {"Realm.Storage.Query", realm::util::Logger::Level::TEST_LOGGING_LEVEL_QUERY},
+#endif
+#ifdef TEST_LOGGING_LEVEL_OBJECT
+    {"Realm.Storage.Object", realm::util::Logger::Level::TEST_LOGGING_LEVEL_OBJECT},
+#endif
+#ifdef TEST_LOGGING_LEVEL_NOTIFICATION
+    {"Realm.Storage.Notification", realm::util::Logger::Level::TEST_LOGGING_LEVEL_NOTIFICATION},
+#endif
+#ifdef TEST_LOGGING_LEVEL_SYNC
+    {"Realm.Sync", realm::util::Logger::Level::TEST_LOGGING_LEVEL_SYNC},
+#endif
+#ifdef TEST_LOGGING_LEVEL_CLIENT
+    {"Realm.Sync.Client", realm::util::Logger::Level::TEST_LOGGING_LEVEL_CLIENT},
+#endif
+#ifdef TEST_LOGGING_LEVEL_SESSION
+    {"Realm.Sync.Client.Session", realm::util::Logger::Level::TEST_LOGGING_LEVEL_SESSION},
+#endif
+#ifdef TEST_LOGGING_LEVEL_CHANGESET
+    {"Realm.Sync.Client.Changeset", realm::util::Logger::Level::TEST_LOGGING_LEVEL_CHANGESET},
+#endif
+#ifdef TEST_LOGGING_LEVEL_NETWORK
+    {"Realm.Sync.Client.Network", realm::util::Logger::Level::TEST_LOGGING_LEVEL_NETWORK},
+#endif
+#ifdef TEST_LOGGING_LEVEL_RESET
+    {"Realm.Sync.Client.Reset", realm::util::Logger::Level::TEST_LOGGING_LEVEL_RESET},
+#endif
+#ifdef TEST_LOGGING_LEVEL_SERVER
+    {"Realm.Sync.Server", realm::util::Logger::Level::TEST_LOGGING_LEVEL_SERVER},
+#endif
+#ifdef TEST_LOGGING_LEVEL_APP
+    {"Realm.App", realm::util::Logger::Level::TEST_LOGGING_LEVEL_APP},
+#endif
+};
+
+static void set_default_level_thresholds()
+{
+    for (auto [cat, level] : default_log_levels) {
+        realm::util::LogCategory::get_category(cat).set_default_level_threshold(level);
+    }
+}
+
 
 int run_object_store_tests(int argc, const char** argv);
 
@@ -91,6 +162,8 @@ int run_object_store_tests(int argc, const char** argv)
     });
 #endif
 
+    set_default_level_thresholds();
+
     Catch::Session session;
     session.useConfigData(config);
     int result = session.run(argc, argv);