From 7c472b4c454da52bf15b56efab20d53779b1f214 Mon Sep 17 00:00:00 2001 From: Kenneth Geisshirt Date: Fri, 10 Nov 2023 13:37:57 +0100 Subject: [PATCH 01/16] Add bindgen to PR template --- .github/pull_request_template.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a0ba3bab203..da162fc438c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,4 +5,5 @@ ## ☑️ ToDos * [ ] 📝 Changelog update * [ ] 🚦 Tests (or not relevant) -* [ ] C-API, if public C++ API changed. +* [ ] C-API, if public C++ API changed +* [ ] `bindgen/spec.yml`, if public C++ API changed From 9dba4d2de8b9d318a7c9a6326fbbda887743c4e1 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Sat, 4 Nov 2023 19:47:06 -0700 Subject: [PATCH 02/16] Replace ClientResetOperation with a function --- src/realm/object-store/sync/sync_session.cpp | 7 +- src/realm/sync/client.cpp | 16 +- src/realm/sync/noinst/client_impl_base.cpp | 201 ++++++++---------- src/realm/sync/noinst/client_impl_base.hpp | 77 +++---- src/realm/sync/noinst/client_reset.cpp | 10 +- src/realm/sync/noinst/client_reset.hpp | 2 +- .../sync/noinst/client_reset_operation.cpp | 124 ++++------- .../sync/noinst/client_reset_operation.hpp | 62 ++---- test/object-store/sync/client_reset.cpp | 10 +- test/object-store/sync/flx_migration.cpp | 4 +- test/object-store/sync/flx_sync.cpp | 4 +- .../util/sync/sync_test_utils.cpp | 2 +- 12 files changed, 198 insertions(+), 321 deletions(-) diff --git a/src/realm/object-store/sync/sync_session.cpp b/src/realm/object-store/sync/sync_session.cpp index a6772cea074..98336d6a5e0 100644 --- a/src/realm/object-store/sync/sync_session.cpp +++ b/src/realm/object-store/sync/sync_session.cpp @@ -460,7 +460,7 @@ void SyncSession::download_fresh_realm(sync::ProtocolErrorInfo::Action server_re options.encryption_key = encryption_key.data(); DBRef db; - auto fresh_path = ClientResetOperation::get_fresh_path_for(m_db->get_path()); + auto fresh_path = client_reset::get_fresh_path_for(m_db->get_path()); try { // We want to attempt to use a pre-existing file to reduce the chance of // downloading the first part of the file only to then delete it over @@ -876,9 +876,8 @@ void SyncSession::create_sync_session() session_config.proxy_config = sync_config.proxy_config; session_config.simulate_integration_error = sync_config.simulate_integration_error; session_config.flx_bootstrap_batch_size_bytes = sync_config.flx_bootstrap_batch_size_bytes; - session_config.session_reason = ClientResetOperation::is_fresh_path(m_config.path) - ? sync::SessionReason::ClientReset - : sync::SessionReason::Sync; + session_config.session_reason = + client_reset::is_fresh_path(m_config.path) ? sync::SessionReason::ClientReset : sync::SessionReason::Sync; if (sync_config.on_sync_client_event_hook) { session_config.on_sync_client_event_hook = [hook = sync_config.on_sync_client_event_hook, diff --git a/src/realm/sync/client.cpp b/src/realm/sync/client.cpp index 33cda5df313..38df01a559c 100644 --- a/src/realm/sync/client.cpp +++ b/src/realm/sync/client.cpp @@ -688,13 +688,18 @@ DBRef SessionImpl::get_db() const noexcept return m_wrapper.m_db; } -ClientReplication& SessionImpl::access_realm() +ClientReplication& SessionImpl::get_repl() const noexcept { // Can only be called if the session is active or being activated REALM_ASSERT_EX(m_state == State::Active || m_state == State::Unactivated, m_state); return m_wrapper.get_replication(); } +ClientHistory& SessionImpl::get_history() const noexcept +{ + return get_repl().get_history(); +} + util::Optional& SessionImpl::get_client_reset_config() noexcept { // Can only be called if the session is active or being activated @@ -725,9 +730,7 @@ void SessionImpl::initiate_integrate_changesets(std::uint_fast64_t downloadable_ version_type client_version; if (REALM_LIKELY(!get_client().is_dry_run())) { VersionInfo version_info; - ClientReplication& repl = access_realm(); // Throws - integrate_changesets(repl, progress, downloadable_bytes, changesets, version_info, - batch_state); // Throws + integrate_changesets(progress, downloadable_bytes, changesets, version_info, batch_state); // Throws client_version = version_info.realm_version; } else { @@ -873,7 +876,7 @@ void SessionImpl::process_pending_flx_bootstrap() "changeset size: %3)", pending_batch_stats.query_version, pending_batch_stats.pending_changesets, pending_batch_stats.pending_changeset_bytes); - auto& history = access_realm().get_history(); + auto& history = get_repl().get_history(); VersionInfo new_version; SyncProgress progress; int64_t query_version = -1; @@ -1792,8 +1795,7 @@ void SessionWrapper::handle_pending_client_reset_acknowledgement() REALM_ASSERT(pending_reset); m_sess->logger.info("Tracking pending client reset of type \"%1\" from %2", pending_reset->type, pending_reset->time); - util::bind_ptr self(this); - async_wait_for(true, true, [self = std::move(self), pending_reset = *pending_reset](Status status) { + async_wait_for(true, true, [self = util::bind_ptr(this), pending_reset = *pending_reset](Status status) { if (status == ErrorCodes::OperationAborted) { return; } diff --git a/src/realm/sync/noinst/client_impl_base.cpp b/src/realm/sync/noinst/client_impl_base.cpp index 11e48d7cc6d..d07d874c269 100644 --- a/src/realm/sync/noinst/client_impl_base.cpp +++ b/src/realm/sync/noinst/client_impl_base.cpp @@ -1,28 +1,30 @@ -#include -#include +#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include #include #include -#include #include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include #include // Only for websocket::Error TODO remove +#include +#include + // NOTE: The protocol specification is in `/doc/protocol.md` using namespace realm; @@ -1522,12 +1524,11 @@ void Session::gather_pending_compensating_writes(util::Span changeset } -void Session::integrate_changesets(ClientReplication& repl, const SyncProgress& progress, - std::uint_fast64_t downloadable_bytes, +void Session::integrate_changesets(const SyncProgress& progress, std::uint_fast64_t downloadable_bytes, const ReceivedChangesets& received_changesets, VersionInfo& version_info, DownloadBatchState download_batch_state) { - auto& history = repl.get_history(); + auto& history = get_history(); if (received_changesets.empty()) { if (download_batch_state == DownloadBatchState::MoreToCome) { throw IntegrationException(ErrorCodes::SyncProtocolInvariantFailed, @@ -1650,34 +1651,13 @@ void Session::activate() bool has_pending_client_reset = false; if (REALM_LIKELY(!get_client().is_dry_run())) { - // The reason we need a mutable reference from get_client_reset_config() is because we - // don't want the session to keep a strong reference to the client_reset_config->fresh_copy - // DB. If it did, then the fresh DB would stay alive for the duration of this sync session - // and we want to clean it up once the reset is finished. Additionally, the fresh copy will - // be set to a new copy on every reset so there is no reason to keep a reference to it. - // The modification to the client reset config happens via std::move(client_reset_config->fresh_copy). - // If the client reset config were a `const &` then this std::move would create another strong - // reference which we don't want to happen. - util::Optional& client_reset_config = get_client_reset_config(); - bool file_exists = util::File::exists(get_realm_path()); + m_performing_client_reset = get_client_reset_config().has_value(); - logger.info("client_reset_config = %1, Realm exists = %2, " - "client reset = %3", - client_reset_config ? "true" : "false", file_exists ? "true" : "false", - (client_reset_config && file_exists) ? "true" : "false"); // Throws - if (client_reset_config && !m_client_reset_operation) { - m_client_reset_operation = std::make_unique<_impl::ClientResetOperation>( - logger, get_db(), std::move(client_reset_config->fresh_copy), client_reset_config->mode, - std::move(client_reset_config->notify_before_client_reset), - std::move(client_reset_config->notify_after_client_reset), - client_reset_config->recovery_is_allowed); // Throws - } - - if (!m_client_reset_operation) { - const ClientReplication& repl = access_realm(); // Throws - repl.get_history().get_status(m_last_version_available, m_client_file_ident, m_progress, - &has_pending_client_reset); // Throws + logger.info("client_reset_config = %1, Realm exists = %2 ", m_performing_client_reset, file_exists); + if (!m_performing_client_reset) { + get_history().get_status(m_last_version_available, m_client_file_ident, m_progress, + &has_pending_client_reset); // Throws } } logger.debug("client_file_ident = %1, client_file_ident_salt = %2", m_client_file_ident.ident, @@ -2006,12 +1986,10 @@ void Session::send_upload_message() target_upload_version = m_pending_flx_sub_set->snapshot_version; } - const ClientReplication& repl = access_realm(); // Throws - std::vector uploadable_changesets; version_type locked_server_version = 0; - repl.get_history().find_uploadable_changesets(m_upload_progress, target_upload_version, uploadable_changesets, - locked_server_version); // Throws + get_history().find_uploadable_changesets(m_upload_progress, target_upload_version, uploadable_changesets, + locked_server_version); // Throws if (uploadable_changesets.empty()) { // Nothing more to upload right now @@ -2218,6 +2196,68 @@ void Session::send_test_command_message() enlist_to_send(); } +bool Session::client_reset_if_needed() +{ + // Regardless of what happens, once we return from this function we will + // no longer be in the middle of a client reset + m_performing_client_reset = false; + + // Even if we end up not actually performing a client reset, consume the + // config to ensure that the resources it holds are released + auto client_reset_config = std::exchange(get_client_reset_config(), std::nullopt); + if (!client_reset_config) { + return false; + } + + auto on_flx_version_complete = [this](int64_t version) { + this->on_flx_sync_version_complete(version); + }; + bool did_reset = client_reset::perform_client_reset( + logger, *get_db(), *client_reset_config->fresh_copy, client_reset_config->mode, + std::move(client_reset_config->notify_before_client_reset), + std::move(client_reset_config->notify_after_client_reset), m_client_file_ident, get_flx_subscription_store(), + on_flx_version_complete, client_reset_config->recovery_is_allowed); + if (!did_reset) { + return false; + } + + // The fresh Realm has been used to reset the state + logger.debug("Client reset is completed, path=%1", get_realm_path()); // Throws + + SaltedFileIdent client_file_ident; + bool has_pending_client_reset = false; + get_history().get_status(m_last_version_available, client_file_ident, m_progress, + &has_pending_client_reset); // Throws + REALM_ASSERT_3(m_client_file_ident.ident, ==, client_file_ident.ident); + REALM_ASSERT_3(m_client_file_ident.salt, ==, client_file_ident.salt); + REALM_ASSERT_EX(m_progress.download.last_integrated_client_version == 0, + m_progress.download.last_integrated_client_version); + REALM_ASSERT_EX(m_progress.upload.client_version == 0, m_progress.upload.client_version); + REALM_ASSERT_EX(m_progress.upload.last_integrated_server_version == 0, + m_progress.upload.last_integrated_server_version); + logger.trace("last_version_available = %1", m_last_version_available); // Throws + + m_upload_progress = m_progress.upload; + m_download_progress = m_progress.download; + // In recovery mode, there may be new changesets to upload and nothing left to download. + // In FLX DiscardLocal mode, there may be new commits due to subscription handling. + // For both, we want to allow uploads again without needing external changes to download first. + m_allow_upload = true; + REALM_ASSERT_EX(m_last_version_selected_for_upload == 0, m_last_version_selected_for_upload); + + if (has_pending_client_reset) { + handle_pending_client_reset_acknowledgement(); + } + + update_subscription_version_info(); + + // If a migration or rollback is in progress, mark it complete when client reset is completed. + if (auto migration_store = get_migration_store()) { + migration_store->complete_migration_or_rollback(); + } + + return true; +} Status Session::receive_ident_message(SaltedFileIdent client_file_ident) { @@ -2250,67 +2290,6 @@ Status Session::receive_ident_message(SaltedFileIdent client_file_ident) return Status::OK(); // Success } - // access before the client reset (if applicable) because - // the reset can take a while and the sync session might have died - // by the time the reset finishes. - ClientReplication& repl = access_realm(); // Throws - - auto client_reset_if_needed = [&]() -> bool { - if (!m_client_reset_operation) { - return false; - } - - // ClientResetOperation::finalize() will return true only if the operation actually did - // a client reset. It may choose not to do a reset if the local Realm does not exist - // at this point (in that case there is nothing to reset). But in any case, we must - // clean up m_client_reset_operation at this point as sync should be able to continue from - // this point forward. - auto client_reset_operation = std::move(m_client_reset_operation); - util::UniqueFunction on_flx_subscription_complete = [this](int64_t version) { - this->on_flx_sync_version_complete(version); - }; - if (!client_reset_operation->finalize(client_file_ident, get_flx_subscription_store(), - std::move(on_flx_subscription_complete))) { - return false; - } - - // The fresh Realm has been used to reset the state - logger.debug("Client reset is completed, path=%1", get_realm_path()); // Throws - - SaltedFileIdent client_file_ident; - bool has_pending_client_reset = false; - repl.get_history().get_status(m_last_version_available, client_file_ident, m_progress, - &has_pending_client_reset); // Throws - REALM_ASSERT_3(m_client_file_ident.ident, ==, client_file_ident.ident); - REALM_ASSERT_3(m_client_file_ident.salt, ==, client_file_ident.salt); - REALM_ASSERT_EX(m_progress.download.last_integrated_client_version == 0, - m_progress.download.last_integrated_client_version); - REALM_ASSERT_EX(m_progress.upload.client_version == 0, m_progress.upload.client_version); - REALM_ASSERT_EX(m_progress.upload.last_integrated_server_version == 0, - m_progress.upload.last_integrated_server_version); - logger.trace("last_version_available = %1", m_last_version_available); // Throws - - m_upload_progress = m_progress.upload; - m_download_progress = m_progress.download; - // In recovery mode, there may be new changesets to upload and nothing left to download. - // In FLX DiscardLocal mode, there may be new commits due to subscription handling. - // For both, we want to allow uploads again without needing external changes to download first. - m_allow_upload = true; - REALM_ASSERT_EX(m_last_version_selected_for_upload == 0, m_last_version_selected_for_upload); - - if (has_pending_client_reset) { - handle_pending_client_reset_acknowledgement(); - } - - update_subscription_version_info(); - - // If a migration or rollback is in progress, mark it complete when client reset is completed. - if (auto migration_store = get_migration_store()) { - migration_store->complete_migration_or_rollback(); - } - - return true; - }; // if a client reset happens, it will take care of setting the file ident // and if not, we do it here bool did_client_reset = false; @@ -2325,8 +2304,8 @@ Status Session::receive_ident_message(SaltedFileIdent client_file_ident) return Status::OK(); } if (!did_client_reset) { - repl.get_history().set_client_file_ident(client_file_ident, - m_fix_up_object_ids); // Throws + get_history().set_client_file_ident(client_file_ident, + m_fix_up_object_ids); // Throws m_progress.download.last_integrated_client_version = 0; m_progress.upload.client_version = 0; m_last_version_selected_for_upload = 0; @@ -2716,7 +2695,7 @@ void Session::check_for_upload_completion() } // during an ongoing client reset operation, we never upload anything - if (m_client_reset_operation) + if (m_performing_client_reset) return; // Upload process must have reached end of history diff --git a/src/realm/sync/noinst/client_impl_base.hpp b/src/realm/sync/noinst/client_impl_base.hpp index 3137ca761ae..a335dea4196 100644 --- a/src/realm/sync/noinst/client_impl_base.hpp +++ b/src/realm/sync/noinst/client_impl_base.hpp @@ -2,37 +2,34 @@ #ifndef REALM_NOINST_CLIENT_IMPL_BASE_HPP #define REALM_NOINST_CLIENT_IMPL_BASE_HPP -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include +#include #include -#include +#include #include -#include +#include #include #include -#include -#include #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -namespace realm { -namespace sync { +namespace realm::sync { // (protocol, address, port, session_multiplex_ident) // @@ -162,7 +159,6 @@ class ClientImpl { using port_type = network::Endpoint::port_type; using OutputBuffer = util::ResettableExpandableBufferOutputStream; using ClientProtocol = _impl::ClientProtocol; - using ClientResetOperation = _impl::ClientResetOperation; using RandomEngine = std::mt19937_64; /// Per-server endpoint information used to determine reconnect delays. @@ -426,7 +422,7 @@ class ClientImpl::Connection { /// /// Prior to being activated, no messages will be sent or received on behalf /// of this session, and the associated Realm file will not be accessed, - /// i.e., Session::access_realm() will not be called. + /// i.e., `Session::get_db()` will not be called. /// /// If activation is successful, the connection keeps the session alive /// until the application calls initiated_session_deactivation() or until @@ -449,7 +445,7 @@ class ClientImpl::Connection { /// initiate_session_deactivation(). /// /// After the initiation of the deactivation process, the associated Realm - /// file will no longer be accessed, i.e., access_realm() will not be called + /// file will no longer be accessed, i.e., `get_db()` will not be called /// again, and a previously returned reference will also not be accessed /// again. /// @@ -844,12 +840,8 @@ class ClientImpl::Session { /// To be used in connection with implementations of /// initiate_integrate_changesets(). - /// - /// This function is thread-safe, but if called from a thread other than the - /// event loop thread of the associated client object, the specified history - /// accessor must **not** be the one made available by access_realm(). - void integrate_changesets(ClientReplication&, const SyncProgress&, std::uint_fast64_t downloadable_bytes, - const ReceivedChangesets&, VersionInfo&, DownloadBatchState last_in_batch); + void integrate_changesets(const SyncProgress&, std::uint_fast64_t downloadable_bytes, const ReceivedChangesets&, + VersionInfo&, DownloadBatchState last_in_batch); /// To be used in connection with implementations of /// initiate_integrate_changesets(). @@ -900,18 +892,11 @@ class ClientImpl::Session { const std::string& get_virt_path() const noexcept; const std::string& get_realm_path() const noexcept; - DBRef get_db() const noexcept; - /// The implementation need only ensure that the returned reference stays valid - /// until the next invocation of access_realm() on one of the session - /// objects associated with the same client object. - /// - /// This function is always called by the event loop thread of the - /// associated client object. - /// - /// This function is guaranteed to not be called before activation, and also - /// not after initiation of deactivation. - ClientReplication& access_realm(); + // Can only be called if the session is active or being activated + DBRef get_db() const noexcept; + ClientReplication& get_repl() const noexcept; + ClientHistory& get_history() const noexcept; // client_reset_config() returns the config for client // reset. If it returns none, ordinary sync is used. If it returns a @@ -935,10 +920,6 @@ class ClientImpl::Session { /// on_changesets_integrated() to be called without unnecessary delay, /// although never after initiation of session deactivation. /// - /// The integration of the specified changesets must happen by means of an - /// invocation of integrate_changesets(), but not necessarily using the - /// history accessor made available by access_realm(). - /// /// The implementation is allowed, but not obliged to aggregate changesets /// from multiple invocations of initiate_integrate_changesets() and pass /// them to ClientReplication::integrate_server_changesets() at once. @@ -1011,6 +992,7 @@ class ClientImpl::Session { // Processes any pending FLX bootstraps, if one exists. Otherwise this is a noop. void process_pending_flx_bootstrap(); + bool client_reset_if_needed(); void handle_pending_client_reset_acknowledgement(); void update_subscription_version_info(); @@ -1070,8 +1052,8 @@ class ClientImpl::Session { // `ident == 0` means unassigned. SaltedFileIdent m_client_file_ident = {0, 0}; - // m_client_reset_operation stores state for the lifetime of a client reset - std::unique_ptr m_client_reset_operation; + // True while this session is in the process of performing a client reset. + bool m_performing_client_reset = false; // The latest sync progress reported by the server via a DOWNLOAD // message. See struct SyncProgress for a description. The values stored in @@ -1617,7 +1599,6 @@ inline void ClientImpl::Session::enlist_to_send() m_conn.enlist_to_send(this); // Throws } -} // namespace sync -} // namespace realm +} // namespace realm::sync #endif // REALM_NOINST_CLIENT_IMPL_BASE_HPP diff --git a/src/realm/sync/noinst/client_reset.cpp b/src/realm/sync/noinst/client_reset.cpp index 87cad3f8a22..0ff45279c1e 100644 --- a/src/realm/sync/noinst/client_reset.cpp +++ b/src/realm/sync/noinst/client_reset.cpp @@ -554,7 +554,7 @@ static ClientResyncMode reset_precheck_guard(Transaction& wt, ClientResyncMode m LocalVersionIDs perform_client_reset_diff(DB& db_local, DB& db_remote, sync::SaltedFileIdent client_file_ident, util::Logger& logger, ClientResyncMode mode, bool recovery_is_allowed, bool* did_recover_out, sync::SubscriptionStore* sub_store, - util::UniqueFunction on_flx_version_complete) + util::FunctionRef on_flx_version_complete) { auto wt_local = db_local.start_write(); auto actual_mode = reset_precheck_guard(*wt_local, mode, recovery_is_allowed, logger); @@ -605,9 +605,7 @@ LocalVersionIDs perform_client_reset_diff(DB& db_local, DB& db_remote, sync::Sal if (did_recover_out) { *did_recover_out = false; } - if (on_flx_version_complete) { - on_flx_version_complete(subscription_version); - } + on_flx_version_complete(subscription_version); VersionID new_version_local = wt_local->get_version_of_current_transaction(); logger.info("perform_client_reset_diff is done: old_version = (version: %1, index: %2), " @@ -626,9 +624,7 @@ LocalVersionIDs perform_client_reset_diff(DB& db_local, DB& db_remote, sync::Sal auto mut_subs = subs.make_mutable_copy(); mut_subs.update_state(sync::SubscriptionSet::State::Complete); auto sub = std::move(mut_subs).commit(); - if (on_flx_version_complete) { - on_flx_version_complete(sub.version()); - } + on_flx_version_complete(sub.version()); logger.info("Recreated the active subscription set in the complete state (%1 -> %2)", before_version, sub.version()); }; diff --git a/src/realm/sync/noinst/client_reset.hpp b/src/realm/sync/noinst/client_reset.hpp index 9df29a13d74..fd113bc3d54 100644 --- a/src/realm/sync/noinst/client_reset.hpp +++ b/src/realm/sync/noinst/client_reset.hpp @@ -89,7 +89,7 @@ struct LocalVersionIDs { LocalVersionIDs perform_client_reset_diff(DB& db, DB& db_remote, sync::SaltedFileIdent client_file_ident, util::Logger& logger, ClientResyncMode mode, bool recovery_is_allowed, bool* did_recover_out, sync::SubscriptionStore* sub_store, - util::UniqueFunction on_flx_version_complete); + util::FunctionRef on_flx_version_complete); } // namespace _impl::client_reset } // namespace realm diff --git a/src/realm/sync/noinst/client_reset_operation.cpp b/src/realm/sync/noinst/client_reset_operation.cpp index 31509135efd..10780bba52d 100644 --- a/src/realm/sync/noinst/client_reset_operation.cpp +++ b/src/realm/sync/noinst/client_reset_operation.cpp @@ -16,14 +16,15 @@ * **************************************************************************/ -#include +#include + #include #include #include -#include +#include #include -namespace realm::_impl { +namespace realm::_impl::client_reset { namespace { @@ -31,24 +32,7 @@ constexpr static std::string_view c_fresh_suffix(".fresh"); } // namespace -ClientResetOperation::ClientResetOperation(util::Logger& logger, DBRef db, DBRef db_fresh, ClientResyncMode mode, - CallbackBeforeType notify_before, CallbackAfterType notify_after, - bool recovery_is_allowed) - : m_logger{logger} - , m_db{db} - , m_db_fresh(std::move(db_fresh)) - , m_mode(mode) - , m_notify_before(std::move(notify_before)) - , m_notify_after(std::move(notify_after)) - , m_recovery_is_allowed(recovery_is_allowed) -{ - REALM_ASSERT(m_db); - REALM_ASSERT_RELEASE(m_mode != ClientResyncMode::Manual); - m_logger.debug("Create ClientResetOperation, realm_path = %1, mode = %2, recovery_allowed = %3", m_db->get_path(), - m_mode, m_recovery_is_allowed); -} - -std::string ClientResetOperation::get_fresh_path_for(const std::string& path) +std::string get_fresh_path_for(const std::string& path) { const size_t suffix_len = c_fresh_suffix.size(); REALM_ASSERT(path.length()); @@ -57,7 +41,7 @@ std::string ClientResetOperation::get_fresh_path_for(const std::string& path) return path + c_fresh_suffix.data(); } -bool ClientResetOperation::is_fresh_path(const std::string& path) +bool is_fresh_path(const std::string& path) { const size_t suffix_len = c_fresh_suffix.size(); REALM_ASSERT(path.length()); @@ -67,83 +51,57 @@ bool ClientResetOperation::is_fresh_path(const std::string& path) return path.substr(path.size() - suffix_len, suffix_len) == c_fresh_suffix; } -bool ClientResetOperation::finalize(sync::SaltedFileIdent salted_file_ident, sync::SubscriptionStore* sub_store, - util::UniqueFunction on_flx_version_complete) +bool perform_client_reset(util::Logger& logger, DB& db, DB& fresh_db, ClientResyncMode mode, + CallbackBeforeType notify_before, CallbackAfterType notify_after, + sync::SaltedFileIdent new_file_ident, sync::SubscriptionStore* sub_store, + util::FunctionRef on_flx_version, bool recovery_is_allowed) { - m_salted_file_ident = salted_file_ident; + REALM_ASSERT(mode != ClientResyncMode::Manual); + logger.debug("Possibly beginning client reset operation: realm_path = %1, mode = %2, recovery_allowed = %3", + db.get_path(), mode, recovery_is_allowed); + + auto always_try_clean_up = util::make_scope_exit([&]() noexcept { + std::string path_to_clean = fresh_db.get_path(); + try { + fresh_db.close(); + constexpr bool delete_lockfile = true; + DB::delete_files(path_to_clean, nullptr, delete_lockfile); + } + catch (const std::exception& err) { + logger.warn("In ClientResetOperation::finalize, the fresh copy '%1' could not be cleaned up due to " + "an exception: '%2'", + path_to_clean, err.what()); + // ignored, this is just a best effort + } + }); + // only do the reset if there is data to reset // if there is nothing in this Realm, then there is nothing to reset and // sync should be able to continue as normal - auto latest_version = m_db->get_version_id_of_latest_snapshot(); - - bool local_realm_exists = latest_version.version != 0; - m_logger.debug("ClientResetOperation::finalize, realm_path = %1, local_realm_exists = %2, mode = %3", - m_db->get_path(), local_realm_exists, m_mode); + auto latest_version = db.get_version_id_of_latest_snapshot(); + bool local_realm_exists = latest_version.version > 1; if (!local_realm_exists) { + logger.debug("Local Realm file has never been written to, so skipping client reset."); return false; } - REALM_ASSERT_EX(m_db_fresh, m_db->get_path(), m_mode); - - client_reset::LocalVersionIDs local_version_ids; - auto always_try_clean_up = util::make_scope_exit([&]() noexcept { - clean_up_state(); - }); - - VersionID frozen_before_state_version = m_notify_before ? m_notify_before() : latest_version; + VersionID frozen_before_state_version = notify_before ? notify_before() : latest_version; // If m_notify_after is set, pin the previous state to keep it around. TransactionRef previous_state; - if (m_notify_after) { - previous_state = m_db->start_frozen(frozen_before_state_version); + if (notify_after) { + previous_state = db.start_frozen(frozen_before_state_version); } bool did_recover_out = false; - local_version_ids = client_reset::perform_client_reset_diff( - *m_db, *m_db_fresh, m_salted_file_ident, m_logger, m_mode, m_recovery_is_allowed, &did_recover_out, sub_store, - std::move(on_flx_version_complete)); // throws + client_reset::perform_client_reset_diff(db, fresh_db, new_file_ident, logger, mode, recovery_is_allowed, + &did_recover_out, sub_store, + on_flx_version); // throws - if (m_notify_after) { - m_notify_after(previous_state->get_version_of_current_transaction(), did_recover_out); + if (notify_after) { + notify_after(previous_state->get_version_of_current_transaction(), did_recover_out); } - m_client_reset_old_version = local_version_ids.old_version; - m_client_reset_new_version = local_version_ids.new_version; - return true; } -void ClientResetOperation::clean_up_state() noexcept -{ - if (m_db_fresh) { - std::string path_to_clean = m_db_fresh->get_path(); - try { - // In order to obtain the lock and delete the realm, we first have to close - // the Realm. This requires that we are the only remaining ref holder, and - // this is expected. Releasing the last ref should release the hold on the - // lock file and allow us to clean up. - long use_count = m_db_fresh.use_count(); - REALM_ASSERT_DEBUG_EX(use_count == 1, use_count, path_to_clean); - m_db_fresh.reset(); - // clean up the fresh Realm - // we don't mind leaving the fresh lock file around because trying to delete it - // here could cause a race if there are multiple resets ongoing - bool did_lock = DB::call_with_lock(path_to_clean, [&](const std::string& path) { - constexpr bool delete_lockfile = false; - DB::delete_files(path, nullptr, delete_lockfile); - }); - if (!did_lock) { - m_logger.warn("In ClientResetOperation::finalize, the fresh copy '%1' could not be cleaned up. " - "There were %2 refs remaining.", - path_to_clean, use_count); - } - } - catch (const std::exception& err) { - m_logger.warn("In ClientResetOperation::finalize, the fresh copy '%1' could not be cleaned up due to " - "an exception: '%2'", - path_to_clean, err.what()); - // ignored, this is just a best effort - } - } -} - -} // namespace realm::_impl +} // namespace realm::_impl::client_reset diff --git a/src/realm/sync/noinst/client_reset_operation.hpp b/src/realm/sync/noinst/client_reset_operation.hpp index 450dca74684..5c2ce5bec7f 100644 --- a/src/realm/sync/noinst/client_reset_operation.hpp +++ b/src/realm/sync/noinst/client_reset_operation.hpp @@ -21,65 +21,27 @@ #include #include +#include #include +#include #include namespace realm::sync { class SubscriptionStore; } -namespace realm::_impl { +namespace realm::_impl::client_reset { +using CallbackBeforeType = util::UniqueFunction; +using CallbackAfterType = util::UniqueFunction; -// A ClientResetOperation object is used per client session to keep track of -// state Realm download. -class ClientResetOperation { -public: - using CallbackBeforeType = util::UniqueFunction; - using CallbackAfterType = util::UniqueFunction; +std::string get_fresh_path_for(const std::string& realm_path); +bool is_fresh_path(const std::string& realm_path); - ClientResetOperation(util::Logger& logger, DBRef db, DBRef db_fresh, ClientResyncMode mode, - CallbackBeforeType notify_before, CallbackAfterType notify_after, bool recovery_is_allowed); +bool perform_client_reset(util::Logger& logger, DB& target_db, DB& fresh_db, ClientResyncMode mode, + CallbackBeforeType notify_before, CallbackAfterType notify_after, + sync::SaltedFileIdent new_file_ident, sync::SubscriptionStore*, + util::FunctionRef on_flx_version, bool recovery_is_allowed); - // When the client has received the salted file ident from the server, it - // should deliver the ident to the ClientResetOperation object. The ident - // will be inserted in the Realm after download. - bool finalize(sync::SaltedFileIdent salted_file_ident, sync::SubscriptionStore*, - util::UniqueFunction); // throws - - static std::string get_fresh_path_for(const std::string& realm_path); - static bool is_fresh_path(const std::string& realm_path); - - realm::VersionID get_client_reset_old_version() const noexcept; - realm::VersionID get_client_reset_new_version() const noexcept; - -private: - void clean_up_state() noexcept; - - // The lifetime of this class is within a Session, so no need for a shared_ptr - util::Logger& m_logger; - DBRef m_db; - DBRef m_db_fresh; - ClientResyncMode m_mode; - sync::SaltedFileIdent m_salted_file_ident = {0, 0}; - realm::VersionID m_client_reset_old_version; - realm::VersionID m_client_reset_new_version; - CallbackBeforeType m_notify_before; - CallbackAfterType m_notify_after; - bool m_recovery_is_allowed; -}; - -// Implementation - -inline realm::VersionID ClientResetOperation::get_client_reset_old_version() const noexcept -{ - return m_client_reset_old_version; -} - -inline realm::VersionID ClientResetOperation::get_client_reset_new_version() const noexcept -{ - return m_client_reset_new_version; -} - -} // namespace realm::_impl +} // namespace realm::_impl::client_reset #endif // REALM_NOINST_CLIENT_RESET_OPERATION_HPP diff --git a/test/object-store/sync/client_reset.cpp b/test/object-store/sync/client_reset.cpp index 140d4d89fc8..0a3abd28bb0 100644 --- a/test/object-store/sync/client_reset.cpp +++ b/test/object-store/sync/client_reset.cpp @@ -185,7 +185,7 @@ TEST_CASE("sync: large reset with recovery is restartable", "[sync][pbs][client realm->sync_session()->resume(); timed_wait_for([&] { - return util::File::exists(_impl::ClientResetOperation::get_fresh_path_for(realm_config.path)); + return util::File::exists(_impl::client_reset::get_fresh_path_for(realm_config.path)); }); realm->sync_session()->pause(); realm->sync_session()->resume(); @@ -371,7 +371,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { local_config.cache = false; local_config.automatic_change_notifications = false; - const std::string fresh_path = realm::_impl::ClientResetOperation::get_fresh_path_for(local_config.path); + const std::string fresh_path = realm::_impl::client_reset::get_fresh_path_for(local_config.path); size_t before_callback_invocations = 0; size_t after_callback_invocations = 0; std::mutex mtx; @@ -968,7 +968,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { ->run(); timed_wait_for([&] { - return util::File::exists(_impl::ClientResetOperation::get_fresh_path_for(local_config.path)); + return util::File::exists(_impl::client_reset::get_fresh_path_for(local_config.path)); }); // Restart the session before the client reset finishes. @@ -996,7 +996,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { local_config.sync_config->error_handler = [&](std::shared_ptr, SyncError error) { err = error; }; - std::string fresh_path = realm::_impl::ClientResetOperation::get_fresh_path_for(local_config.path); + std::string fresh_path = realm::_impl::client_reset::get_fresh_path_for(local_config.path); util::File f(fresh_path, util::File::Mode::mode_Write); f.write("a non empty file"); f.sync(); @@ -1013,7 +1013,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { local_config.sync_config->error_handler = [&](std::shared_ptr, SyncError error) { err = error; }; - std::string fresh_path = realm::_impl::ClientResetOperation::get_fresh_path_for(local_config.path); + std::string fresh_path = realm::_impl::client_reset::get_fresh_path_for(local_config.path); // create a non-empty directory that we'll fail to delete util::make_dir(fresh_path); util::File(util::File::resolve("file", fresh_path), util::File::mode_Write); diff --git a/test/object-store/sync/flx_migration.cpp b/test/object-store/sync/flx_migration.cpp index 886e5fbfd7a..c504252d104 100644 --- a/test/object-store/sync/flx_migration.cpp +++ b/test/object-store/sync/flx_migration.cpp @@ -539,7 +539,7 @@ TEST_CASE("An interrupted migration or rollback can recover on the next session" auto realm = Realm::get_shared_realm(config); timed_wait_for([&] { - return util::File::exists(_impl::ClientResetOperation::get_fresh_path_for(config.path)); + return util::File::exists(_impl::client_reset::get_fresh_path_for(config.path)); }); // Pause then resume the session. This triggers the server to send a new client reset request. @@ -567,7 +567,7 @@ TEST_CASE("An interrupted migration or rollback can recover on the next session" auto realm = Realm::get_shared_realm(config); timed_wait_for([&] { - return util::File::exists(_impl::ClientResetOperation::get_fresh_path_for(config.path)); + return util::File::exists(_impl::client_reset::get_fresh_path_for(config.path)); }); // Pause then resume the session. This triggers the server to send a new client reset request. diff --git a/test/object-store/sync/flx_sync.cpp b/test/object-store/sync/flx_sync.cpp index 5f41c7066ff..1f2189112c9 100644 --- a/test/object-store/sync/flx_sync.cpp +++ b/test/object-store/sync/flx_sync.cpp @@ -579,7 +579,7 @@ TEST_CASE("flx: client reset", "[sync][flx][client reset][baas]") { auto&& [error_future, error_handler] = make_error_handler(); config_local.sync_config->error_handler = error_handler; - std::string fresh_path = realm::_impl::ClientResetOperation::get_fresh_path_for(config_local.path); + std::string fresh_path = realm::_impl::client_reset::get_fresh_path_for(config_local.path); // create a non-empty directory that we'll fail to delete util::make_dir(fresh_path); util::File(util::File::resolve("file", fresh_path), util::File::mode_Write); @@ -1005,7 +1005,7 @@ TEST_CASE("flx: client reset", "[sync][flx][client reset][baas]") { auto&& [error_future, error_handler] = make_error_handler(); config_local.sync_config->error_handler = error_handler; - std::string fresh_path = realm::_impl::ClientResetOperation::get_fresh_path_for(config_local.path); + std::string fresh_path = realm::_impl::client_reset::get_fresh_path_for(config_local.path); // create a non-empty directory that we'll fail to delete util::make_dir(fresh_path); util::File(util::File::resolve("file", fresh_path), util::File::mode_Write); diff --git a/test/object-store/util/sync/sync_test_utils.cpp b/test/object-store/util/sync/sync_test_utils.cpp index b2342aabab8..806359d96b8 100644 --- a/test/object-store/util/sync/sync_test_utils.cpp +++ b/test/object-store/util/sync/sync_test_utils.cpp @@ -408,7 +408,7 @@ struct FakeLocalClientReset : public TestClientReset { using _impl::client_reset::perform_client_reset_diff; constexpr bool recovery_is_allowed = true; perform_client_reset_diff(*local_db, *remote_db, fake_ident, *logger, m_mode, recovery_is_allowed, - nullptr, nullptr, nullptr); + nullptr, nullptr, [](int64_t) {}); remote_realm->close(); if (m_on_post_reset) { From ccfb22d3a5d6b1afb2d5b46bf8f04e6ff8355d83 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 6 Nov 2023 11:46:55 -0800 Subject: [PATCH 03/16] Remove an unnecessary ad-hoc semaphore --- src/realm/sync/subscriptions.cpp | 65 +++++++++----------------------- src/realm/sync/subscriptions.hpp | 40 ++++++++++---------- 2 files changed, 39 insertions(+), 66 deletions(-) diff --git a/src/realm/sync/subscriptions.cpp b/src/realm/sync/subscriptions.cpp index 855cd4a479f..91230cda258 100644 --- a/src/realm/sync/subscriptions.cpp +++ b/src/realm/sync/subscriptions.cpp @@ -438,10 +438,8 @@ void MutableSubscriptionSet::update_state(State new_state, util::Optional SubscriptionSet::get_state_change_notificat { auto mgr = get_flx_subscription_store(); // Throws - std::unique_lock lk(mgr->m_pending_notifications_mutex); + util::CheckedLockGuard lk(mgr->m_pending_notifications_mutex); // If we've already been superceded by another version getting completed, then we should skip registering // a notification because it may never fire. if (mgr->m_min_outstanding_version > version()) { return util::Future::make_ready(State::Superseded); } - // Begin by blocking process_notifications from starting to fill futures. No matter the outcome, we'll - // unblock process_notifications() at the end of this function via the guard we construct below. - mgr->m_outstanding_requests++; - auto guard = util::make_scope_exit([&]() noexcept { - if (!lk.owns_lock()) { - lk.lock(); - } - --mgr->m_outstanding_requests; - mgr->m_pending_notifications_cv.notify_one(); - }); - lk.unlock(); - State cur_state = state(); StringData err_str = error_str(); @@ -501,9 +487,6 @@ util::Future SubscriptionSet::get_state_change_notificat return util::Future::make_ready(cur_state); } - // Otherwise put in a new request to be filled in by process_notifications(). - lk.lock(); - // Otherwise, make a promise/future pair and add it to the list of pending notifications. auto [promise, future] = util::make_promise_future(); mgr->m_pending_notifications.emplace_back(version(), std::move(promise), notify_when); @@ -530,28 +513,24 @@ void MutableSubscriptionSet::process_notifications() auto my_version = version(); std::list to_finish; - std::unique_lock lk(mgr->m_pending_notifications_mutex); - mgr->m_pending_notifications_cv.wait(lk, [&] { - return mgr->m_outstanding_requests == 0; - }); - - for (auto it = mgr->m_pending_notifications.begin(); it != mgr->m_pending_notifications.end();) { - if ((it->version == my_version && - (new_state == State::Error || state_to_order(new_state) >= state_to_order(it->notify_when))) || - (new_state == State::Complete && it->version < my_version)) { - to_finish.splice(to_finish.end(), mgr->m_pending_notifications, it++); - } - else { - ++it; + { + util::CheckedLockGuard lk(mgr->m_pending_notifications_mutex); + for (auto it = mgr->m_pending_notifications.begin(); it != mgr->m_pending_notifications.end();) { + if ((it->version == my_version && + (new_state == State::Error || state_to_order(new_state) >= state_to_order(it->notify_when))) || + (new_state == State::Complete && it->version < my_version)) { + to_finish.splice(to_finish.end(), mgr->m_pending_notifications, it++); + } + else { + ++it; + } } - } - if (new_state == State::Complete) { - mgr->m_min_outstanding_version = my_version; + if (new_state == State::Complete) { + mgr->m_min_outstanding_version = my_version; + } } - lk.unlock(); - for (auto& req : to_finish) { if (new_state == State::Error && req.version == my_version) { req.promise.set_error({ErrorCodes::SubscriptionFailed, std::string_view(error_str())}); @@ -843,11 +822,7 @@ std::vector SubscriptionStore::get_pending_subscriptions() cons void SubscriptionStore::notify_all_state_change_notifications(Status status) { - std::unique_lock lk(m_pending_notifications_mutex); - m_pending_notifications_cv.wait(lk, [&] { - return m_outstanding_requests == 0; - }); - + util::CheckedUniqueLock lk(m_pending_notifications_mutex); auto to_finish = std::move(m_pending_notifications); lk.unlock(); @@ -863,13 +838,9 @@ void SubscriptionStore::terminate() // Clear out and initialize the subscription store initialize_subscriptions_table(m_db->start_read(), true); - std::unique_lock lk(m_pending_notifications_mutex); - m_pending_notifications_cv.wait(lk, [&] { - return m_outstanding_requests == 0; - }); + util::CheckedUniqueLock lk(m_pending_notifications_mutex); auto to_finish = std::move(m_pending_notifications); m_min_outstanding_version = 0; - lk.unlock(); for (auto& req : to_finish) { @@ -896,7 +867,7 @@ SubscriptionSet SubscriptionStore::get_by_version(int64_t version_id) const return SubscriptionSet(weak_from_this(), *tr, obj); } - std::lock_guard lk(m_pending_notifications_mutex); + util::CheckedLockGuard lk(m_pending_notifications_mutex); if (version_id < m_min_outstanding_version) { return SubscriptionSet(weak_from_this(), version_id, SubscriptionSet::SupersededTag{}); } diff --git a/src/realm/sync/subscriptions.hpp b/src/realm/sync/subscriptions.hpp index 8e074723018..b09c30052da 100644 --- a/src/realm/sync/subscriptions.hpp +++ b/src/realm/sync/subscriptions.hpp @@ -16,16 +16,18 @@ * **************************************************************************/ -#pragma once - -#include "realm/db.hpp" -#include "realm/obj.hpp" -#include "realm/query.hpp" -#include "realm/timestamp.hpp" -#include "realm/util/future.hpp" -#include "realm/util/functional.hpp" -#include "realm/util/optional.hpp" -#include "realm/util/tagged_bool.hpp" +#ifndef REALM_SYNC_SUBSCRIPTIONS_HPP +#define REALM_SYNC_SUBSCRIPTIONS_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -332,7 +334,7 @@ class SubscriptionStore : public std::enable_shared_from_this // To be used internally by the sync client. This returns a read-only view of a subscription set by its // version ID. If there is no SubscriptionSet with that version ID, this throws KeyNotFound. - SubscriptionSet get_by_version(int64_t version_id) const; + SubscriptionSet get_by_version(int64_t version_id) const REQUIRES(!m_pending_notifications_mutex); // Returns true if there have been commits to the DB since the given version bool would_refresh(DB::version_type version) const noexcept; @@ -346,17 +348,17 @@ class SubscriptionStore : public std::enable_shared_from_this }; util::Optional get_next_pending_version(int64_t last_query_version) const; - std::vector get_pending_subscriptions() const; + std::vector get_pending_subscriptions() const REQUIRES(!m_pending_notifications_mutex); // Notify all subscription state change notification handlers on this subscription store with the // provided Status - this does not change the state of any pending subscriptions. // Does not necessarily need to be called from the event loop thread. - void notify_all_state_change_notifications(Status status); + void notify_all_state_change_notifications(Status status) REQUIRES(!m_pending_notifications_mutex); // Reset SubscriptionStore and erase all current subscriptions and supersede any pending // subscriptions. Must be called from the event loop thread to prevent data race issues // with the subscription store. - void terminate(); + void terminate() REQUIRES(!m_pending_notifications_mutex); // Recreate the active subscription set, marking any newer pending ones as // superseded. This is a no-op if there are no pending subscription sets. @@ -413,11 +415,11 @@ class SubscriptionStore : public std::enable_shared_from_this ColKey m_sub_set_error_str; ColKey m_sub_set_subscriptions; - mutable std::mutex m_pending_notifications_mutex; - mutable std::condition_variable m_pending_notifications_cv; - mutable int64_t m_outstanding_requests = 0; - mutable int64_t m_min_outstanding_version = 0; - mutable std::list m_pending_notifications; + mutable util::CheckedMutex m_pending_notifications_mutex; + mutable int64_t m_min_outstanding_version GUARDED_BY(m_pending_notifications_mutex) = 0; + mutable std::list m_pending_notifications GUARDED_BY(m_pending_notifications_mutex); }; } // namespace realm::sync + +#endif // REALM_SYNC_SUBSCRIPTIONS_HPP From 9e5d133c79946b399a13df55d82f2d51cc583af6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 6 Nov 2023 12:53:39 -0800 Subject: [PATCH 04/16] Deliver appropriate subscription state change notifications in DiscardLocal client resets --- src/realm/sync/subscriptions.cpp | 55 +++-- src/realm/sync/subscriptions.hpp | 9 +- test/test_client_reset.cpp | 393 ++++++++++++++++++++++++++++++- test/util/test_path.hpp | 9 + 4 files changed, 440 insertions(+), 26 deletions(-) diff --git a/src/realm/sync/subscriptions.cpp b/src/realm/sync/subscriptions.cpp index 91230cda258..6c65e9de27b 100644 --- a/src/realm/sync/subscriptions.cpp +++ b/src/realm/sync/subscriptions.cpp @@ -130,6 +130,19 @@ size_t state_to_order(SubscriptionSet::State needle) REALM_UNREACHABLE(); } +template +void splice_if(std::list& src, std::list& dst, Predicate pred) +{ + for (auto it = src.begin(); it != src.end();) { + if (pred(*it)) { + dst.splice(dst.end(), src, it++); + } + else { + ++it; + } + } +} + } // namespace Subscription::Subscription(const SubscriptionStore* parent, Obj obj) @@ -510,32 +523,26 @@ void MutableSubscriptionSet::process_notifications() { auto mgr = get_flx_subscription_store(); // Throws auto new_state = state(); - auto my_version = version(); std::list to_finish; { util::CheckedLockGuard lk(mgr->m_pending_notifications_mutex); - for (auto it = mgr->m_pending_notifications.begin(); it != mgr->m_pending_notifications.end();) { - if ((it->version == my_version && - (new_state == State::Error || state_to_order(new_state) >= state_to_order(it->notify_when))) || - (new_state == State::Complete && it->version < my_version)) { - to_finish.splice(to_finish.end(), mgr->m_pending_notifications, it++); - } - else { - ++it; - } - } + splice_if(mgr->m_pending_notifications, to_finish, [&](auto& req) { + return (req.version == m_version && + (new_state == State::Error || state_to_order(new_state) >= state_to_order(req.notify_when))) || + (new_state == State::Complete && req.version < m_version); + }); if (new_state == State::Complete) { - mgr->m_min_outstanding_version = my_version; + mgr->m_min_outstanding_version = m_version; } } for (auto& req : to_finish) { - if (new_state == State::Error && req.version == my_version) { + if (new_state == State::Error && req.version == m_version) { req.promise.set_error({ErrorCodes::SubscriptionFailed, std::string_view(error_str())}); } - else if (req.version < my_version) { + else if (req.version < m_version) { req.promise.emplace_value(State::Superseded); } else { @@ -942,8 +949,24 @@ int64_t SubscriptionStore::set_active_as_latest(Transaction& wt) sub_sets->where().greater(sub_sets->get_primary_key_column(), active.get_primary_key().get_int()).remove(); // Mark the active set as complete even if it was previously WaitingForMark // as we've completed rebootstrapping before calling this. - active.set(m_sub_set_state, state_to_storage(SubscriptionSet::State::Complete)); - return active.get_primary_key().get_int(); + active.set(m_sub_set_state, state_to_storage(State::Complete)); + auto version = active.get_primary_key().get_int(); + + std::list to_finish; + { + util::CheckedLockGuard lock(m_pending_notifications_mutex); + splice_if(m_pending_notifications, to_finish, [&](auto& req) { + if (req.version == version && state_to_order(req.notify_when) <= state_to_order(State::Complete)) + return true; + return req.version != version; + }); + } + + for (auto& req : to_finish) { + req.promise.emplace_value(req.version == version ? State::Complete : State::Superseded); + } + + return version; } } // namespace realm::sync diff --git a/src/realm/sync/subscriptions.hpp b/src/realm/sync/subscriptions.hpp index b09c30052da..c1ce7f7344d 100644 --- a/src/realm/sync/subscriptions.hpp +++ b/src/realm/sync/subscriptions.hpp @@ -362,15 +362,16 @@ class SubscriptionStore : public std::enable_shared_from_this // Recreate the active subscription set, marking any newer pending ones as // superseded. This is a no-op if there are no pending subscription sets. - int64_t set_active_as_latest(Transaction& wt); + int64_t set_active_as_latest(Transaction& wt) REQUIRES(!m_pending_notifications_mutex); + +protected: + explicit SubscriptionStore(DBRef db); private: + using State = SubscriptionSet::State; using std::enable_shared_from_this::weak_from_this; DBRef m_db; -protected: - explicit SubscriptionStore(DBRef db); - struct NotificationRequest { NotificationRequest(int64_t version, util::Promise promise, SubscriptionSet::State notify_when) diff --git a/test/test_client_reset.cpp b/test/test_client_reset.cpp index 1cebdc746f6..663d2a78969 100644 --- a/test/test_client_reset.cpp +++ b/test/test_client_reset.cpp @@ -1,19 +1,20 @@ -#include -#include - -#include #include #include -#include - #include #include +#include +#include +#include +#include #include "test.hpp" #include "sync_fixtures.hpp" #include "util/semaphore.hpp" #include "util/compare_groups.hpp" +#include +#include + using namespace realm; using namespace realm::sync; using namespace realm::test_util; @@ -838,4 +839,384 @@ TEST(ClientReset_PinnedVersion) } } +void mark_as_synchronized(DB& db) +{ + auto& history = static_cast(db.get_replication())->get_history(); + sync::version_type current_version; + sync::SaltedFileIdent file_ident; + sync::SyncProgress progress; + history.get_status(current_version, file_ident, progress); + progress.download.last_integrated_client_version = current_version; + progress.upload.client_version = current_version; + progress.upload.last_integrated_server_version = current_version; + sync::VersionInfo info_out; + history.set_sync_progress(progress, nullptr, info_out); +} + +void expect_reset(unit_test::TestContext& test_context, DB& target, DB& fresh, ClientResyncMode mode, + bool allow_recovery = true) +{ + auto db_version = target.get_version_of_latest_snapshot(); + auto fresh_path = fresh.get_path(); + bool did_reset = _impl::client_reset::perform_client_reset( + *test_context.logger, target, fresh, mode, nullptr, nullptr, {100, 200}, nullptr, [](int64_t) {}, + allow_recovery); + CHECK(did_reset); + + // Should have closed and deleted the fresh realm + CHECK_NOT(fresh.is_attached()); + CHECK_NOT(util::File::exists(fresh_path)); + + // Should have performed exactly one write on the target DB + CHECK_EQUAL(target.get_version_of_latest_snapshot(), db_version + 1); + + // Should have set the client file ident + CHECK_EQUAL(target.start_read()->get_sync_file_id(), 100); + + // Client resets aren't marked as complete until the server has acknowledged + // sync completion to avoid reset cycles + { + auto wt = target.start_write(); + _impl::client_reset::remove_pending_client_resets(*wt); + wt->commit(); + } +} + +void expect_reset(unit_test::TestContext& test_context, DB& target, DB& fresh, ClientResyncMode mode, + SubscriptionStore* sub_store) +{ + auto db_version = target.get_version_of_latest_snapshot(); + auto fresh_path = fresh.get_path(); + bool did_reset = _impl::client_reset::perform_client_reset( + *test_context.logger, target, fresh, mode, nullptr, nullptr, {100, 200}, sub_store, [](int64_t) {}, true); + CHECK(did_reset); + + // Should have closed and deleted the fresh realm + CHECK_NOT(fresh.is_attached()); + CHECK_NOT(util::File::exists(fresh_path)); + + // Should have performed exactly one write on the target DB + CHECK_EQUAL(target.get_version_of_latest_snapshot(), db_version + 1); + + // Should have set the client file ident + CHECK_EQUAL(target.start_read()->get_sync_file_id(), 100); + + // Client resets aren't marked as complete until the server has acknowledged + // sync completion to avoid reset cycles + { + auto wt = target.start_write(); + _impl::client_reset::remove_pending_client_resets(*wt); + wt->commit(); + } +} + +std::pair prepare_db(const std::string& path, const std::string& copy_path, + util::FunctionRef fn) +{ + DBRef db = DB::create(make_client_replication(), path); + { + auto wt = db->start_write(); + fn(*wt); + wt->commit(); + } + mark_as_synchronized(*db); + db->write_copy(copy_path, nullptr); + auto db_2 = DB::create(make_client_replication(), copy_path); + return {db, db_2}; +} + +TEST(ClientReset_UninitializedFile) +{ + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + SHARED_GROUP_TEST_PATH(path_3); + + auto [db, db_fresh] = prepare_db(path_1, path_2, [](Transaction& tr) { + tr.add_table_with_primary_key("class_table", type_Int, "pk"); + }); + + auto db_empty = DB::create(make_client_replication(), path_3); + // Should not perform a client reset because the target file has never been + // written to + bool did_reset = _impl::client_reset::perform_client_reset( + *test_context.logger, *db_empty, *db_fresh, ClientResyncMode::Recover, nullptr, nullptr, {100, 200}, nullptr, + [](int64_t) {}, true); + CHECK_NOT(did_reset); + + // Should still have closed and deleted the fresh realm + CHECK_NOT(db_fresh->is_attached()); + CHECK_NOT(util::File::exists(path_2)); +} + +TEST(ClientReset_NoChanges) +{ + SHARED_GROUP_TEST_PATH(path); + SHARED_GROUP_TEST_PATH(path_fresh); + SHARED_GROUP_TEST_PATH(path_backup); + + DBRef db = DB::create(make_client_replication(), path); + { + auto wt = db->start_write(); + auto table = wt->add_table_with_primary_key("class_table", type_Int, "pk"); + table->create_object_with_primary_key(1); + table->create_object_with_primary_key(2); + table->create_object_with_primary_key(3); + wt->commit(); + } + mark_as_synchronized(*db); + + // Write a copy of the pre-reset state to compare against + db->write_copy(path_backup, nullptr); + DBOptions options; + options.is_immutable = true; + auto backup_db = DB::create(path_backup, true, options); + + const ClientResyncMode modes[] = {ClientResyncMode::Recover, ClientResyncMode::DiscardLocal, + ClientResyncMode::RecoverOrDiscard}; + for (auto mode : modes) { + // Perform a reset with a fresh Realm that exactly matches the current + // one, which shouldn't result in any changes regardless of mode + db->write_copy(path_fresh, nullptr); + expect_reset(test_context, *db, *DB::create(make_client_replication(), path_fresh), mode); + + // End state should exactly match the pre-reset state + CHECK_OR_RETURN(compare_groups(*db->start_read(), *backup_db->start_read())); + } +} + +TEST(ClientReset_SimpleNonconflictingChanges) +{ + const std::pair modes[] = { + {ClientResyncMode::Recover, true}, + {ClientResyncMode::RecoverOrDiscard, true}, + {ClientResyncMode::RecoverOrDiscard, false}, + {ClientResyncMode::DiscardLocal, false}, + }; + for (auto [mode, allow_recovery] : modes) { + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + + auto [db, db_fresh] = prepare_db(path_1, path_2, [](Transaction& tr) { + auto table = tr.add_table_with_primary_key("class_table", type_Int, "pk"); + table->create_object_with_primary_key(1); + table->create_object_with_primary_key(2); + table->create_object_with_primary_key(3); + }); + + for (int i = 0; i < 5; ++i) { + auto wt = db->start_write(); + auto table = wt->get_table("class_table"); + table->create_object_with_primary_key(4 + i); + wt->commit(); + } + + { + auto wt = db_fresh->start_write(); + auto table = wt->get_table("class_table"); + for (int i = 0; i < 5; ++i) { + table->create_object_with_primary_key(10 + i); + } + wt->commit(); + } + + expect_reset(test_context, *db, *db_fresh, mode, allow_recovery); + + if (allow_recovery) { + // Should have both the objects created locally and from the reset realm + auto tr = db->start_read(); + auto table = tr->get_table("class_table"); + CHECK_EQUAL(table->size(), 13); + } + else { + // Should only have the objects from the fresh realm + auto tr = db->start_read(); + auto table = tr->get_table("class_table"); + CHECK_EQUAL(table->size(), 8); + CHECK(table->get_object_with_primary_key(10)); + CHECK_NOT(table->get_object_with_primary_key(4)); + } + } +} + +TEST(ClientReset_SimpleConflictingWrites) +{ + const std::pair modes[] = { + {ClientResyncMode::Recover, true}, + {ClientResyncMode::RecoverOrDiscard, true}, + {ClientResyncMode::RecoverOrDiscard, false}, + {ClientResyncMode::DiscardLocal, false}, + }; + for (auto [mode, allow_recovery] : modes) { + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + + auto [db, db_fresh] = prepare_db(path_1, path_2, [](Transaction& tr) { + auto table = tr.add_table_with_primary_key("class_table", type_Int, "pk"); + table->add_column(type_Int, "value"); + table->create_object_with_primary_key(1).set_all(1); + table->create_object_with_primary_key(2).set_all(2); + table->create_object_with_primary_key(3).set_all(3); + }); + + { + auto wt = db->start_write(); + auto table = wt->get_table("class_table"); + for (auto&& obj : *table) { + obj.set_all(obj.get("value") + 10); + } + wt->commit(); + } + + { + auto wt = db_fresh->start_write(); + auto table = wt->get_table("class_table"); + for (auto&& obj : *table) { + obj.set_all(0); + } + wt->commit(); + } + + expect_reset(test_context, *db, *db_fresh, mode, allow_recovery); + + auto tr = db->start_read(); + auto table = tr->get_table("class_table"); + CHECK_EQUAL(table->size(), 3); + if (allow_recovery) { + CHECK_EQUAL(table->get_object_with_primary_key(1).get("value"), 11); + CHECK_EQUAL(table->get_object_with_primary_key(2).get("value"), 12); + CHECK_EQUAL(table->get_object_with_primary_key(3).get("value"), 13); + } + else { + CHECK_EQUAL(table->get_object_with_primary_key(1).get("value"), 0); + CHECK_EQUAL(table->get_object_with_primary_key(2).get("value"), 0); + CHECK_EQUAL(table->get_object_with_primary_key(3).get("value"), 0); + } + } +} + +TEST(ClientReset_Recover_RecoveryDisabled) +{ + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + + auto dbs = prepare_db(path_1, path_2, [](Transaction& tr) { + tr.add_table_with_primary_key("class_table", type_Int, "pk"); + }); + CHECK_THROW((_impl::client_reset::perform_client_reset( + *test_context.logger, *dbs.first, *dbs.second, ClientResyncMode::Recover, nullptr, nullptr, + {100, 200}, nullptr, [](int64_t) {}, false)), + _impl::client_reset::ClientResetFailed); + CHECK_NOT(_impl::client_reset::has_pending_reset(*dbs.first->start_read())); +} + +TEST(ClientReset_Recover_ModificationsOnDeletedObject) +{ + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + + ColKey col; + auto [db, db_fresh] = prepare_db(path_1, path_2, [&](Transaction& tr) { + auto table = tr.add_table_with_primary_key("class_table", type_Int, "pk"); + col = table->add_column(type_Int, "value"); + table->create_object_with_primary_key(1).set_all(1); + table->create_object_with_primary_key(2).set_all(2); + table->create_object_with_primary_key(3).set_all(3); + }); + + { + auto wt = db->start_write(); + auto table = wt->get_table("class_table"); + table->get_object(0).set(col, 11); + table->get_object(1).add_int(col, 10); + table->get_object(2).set(col, 13); + wt->commit(); + } + { + auto wt = db_fresh->start_write(); + auto table = wt->get_table("class_table"); + table->get_object(0).remove(); + table->get_object(0).remove(); + wt->commit(); + } + + expect_reset(test_context, *db, *db_fresh, ClientResyncMode::Recover); + + auto tr = db->start_read(); + auto table = tr->get_table("class_table"); + CHECK_EQUAL(table->size(), 1); + CHECK_EQUAL(table->get_object_with_primary_key(3).get("value"), 13); +} + +SubscriptionSet add_subscription(SubscriptionStore& sub_store, const std::string& name, const Query& q, + std::optional state = std::nullopt) +{ + auto mut = sub_store.get_latest().make_mutable_copy(); + mut.insert_or_assign(name, q); + if (state) { + mut.update_state(*state); + } + return mut.commit(); +} + +TEST(ClientReset_DiscardLocal_DiscardsPendingSubscriptions) +{ + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + auto [db, db_fresh] = prepare_db(path_1, path_2, [&](Transaction& tr) { + tr.add_table_with_primary_key("class_table", type_Int, "pk"); + }); + + auto tr = db->start_read(); + Query query = tr->get_table("class_table")->where(); + auto sub_store = SubscriptionStore::create(db); + add_subscription(*sub_store, "complete", query, SubscriptionSet::State::Complete); + + std::vector pending_sets; + std::vector> futures; + for (int i = 0; i < 3; ++i) { + auto set = add_subscription(*sub_store, util::format("pending %1", i), query); + futures.push_back(set.get_state_change_notification(SubscriptionSet::State::Complete)); + pending_sets.push_back(std::move(set)); + } + + expect_reset(test_context, *db, *db_fresh, ClientResyncMode::DiscardLocal, sub_store.get()); + + CHECK(sub_store->get_pending_subscriptions().empty()); + auto subs = sub_store->get_latest(); + CHECK_EQUAL(subs.state(), SubscriptionSet::State::Complete); + CHECK_EQUAL(subs.size(), 1); + CHECK_EQUAL(subs.at(0).name, "complete"); + + for (auto& fut : futures) { + CHECK_EQUAL(fut.get(), SubscriptionSet::State::Superseded); + } + for (auto& set : pending_sets) { + CHECK_EQUAL(set.state(), SubscriptionSet::State::Pending); + set.refresh(); + CHECK_EQUAL(set.state(), SubscriptionSet::State::Superseded); + } +} + +TEST(ClientReset_DiscardLocal_MakesAwaitingMarkActiveSubscriptionsComplete) +{ + SHARED_GROUP_TEST_PATH(path_1); + SHARED_GROUP_TEST_PATH(path_2); + auto [db, db_fresh] = prepare_db(path_1, path_2, [&](Transaction& tr) { + tr.add_table_with_primary_key("class_table", type_Int, "pk"); + }); + + auto tr = db->start_read(); + Query query = tr->get_table("class_table")->where(); + auto sub_store = SubscriptionStore::create(db); + auto set = add_subscription(*sub_store, "complete", query, SubscriptionSet::State::AwaitingMark); + auto future = set.get_state_change_notification(SubscriptionSet::State::Complete); + + expect_reset(test_context, *db, *db_fresh, ClientResyncMode::DiscardLocal, sub_store.get()); + + CHECK_EQUAL(future.get(), SubscriptionSet::State::Complete); + CHECK_EQUAL(set.state(), SubscriptionSet::State::AwaitingMark); + set.refresh(); + CHECK_EQUAL(set.state(), SubscriptionSet::State::Complete); +} + } // unnamed namespace diff --git a/test/util/test_path.hpp b/test/util/test_path.hpp index 651ae2d4bfe..b4df71f3f77 100644 --- a/test/util/test_path.hpp +++ b/test/util/test_path.hpp @@ -22,6 +22,7 @@ #include #include +#include #include #define TEST_PATH_HELPER(class_name, var_name, suffix) \ @@ -101,6 +102,10 @@ class TestPathGuard { { return m_path; } + operator StringData() const + { + return m_path; + } const char* c_str() const noexcept { return m_path.c_str(); @@ -126,6 +131,10 @@ class TestDirGuard { { return m_path; } + operator StringData() const + { + return m_path; + } const char* c_str() const { return m_path.c_str(); From 9fa759ec5522e157a9f8a1771fe46a28a2efaff2 Mon Sep 17 00:00:00 2001 From: Kenneth Geisshirt Date: Mon, 13 Nov 2023 15:03:01 +0100 Subject: [PATCH 05/16] [bindgen] Expose Obj::set_embedded() (#7126) * [bindgen] Expose Obj::set_embedded() --------- Co-authored-by: LJ <81748770+elle-j@users.noreply.github.com> --- bindgen/spec.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bindgen/spec.yml b/bindgen/spec.yml index 291bb12e28b..1fc7ddc7eed 100644 --- a/bindgen/spec.yml +++ b/bindgen/spec.yml @@ -998,6 +998,7 @@ classes: insert_any: '(list_ndx: count_t, value: Mixed)' insert_embedded: '(ndx: count_t) -> Obj' set_any: '(list_ndx: count_t, value: Mixed)' + set_embedded: '(list_ndx: count_t) -> Obj' filter: '(q: Query) const -> Results' freeze: '(frozen_realm: SharedRealm const&) const -> List' From 55b0c8b03406f3796ce26e852cb5928a67156a15 Mon Sep 17 00:00:00 2001 From: Michael Wilkerson-Barker Date: Mon, 13 Nov 2023 22:52:11 -0500 Subject: [PATCH 06/16] Merge feature/network-faultiness branch into master (#7120) * Add baas-network-tests nightly task for testing sync client operation with non-ideal network conditions (#6852) * Added support for starting baas proxy * Fixed some issues when running the scripts * minor updates to install_baas.sh * Updates to scripts to run on evergreen spawn host * Added total time output to object store tests * Increased initial ssh connect attempts; renamed proxy to 'baas_proxy' * Minor updates to help output * Added baas network test to run bass with the proxy * Added support for separate baas admin api url value * Minor port check adjustments * Removed 'compile' task from network_tests * Disable development mode by default * Added baas remote host and network tests instructions doc * Minor updates to the instructions * Minor updates to documentation * Add non-ideal transfer and network fault tests with non-ideal network conditions (#7063) * Added non-ideal transfer and network fault tests * Added baas-network-tests to be allowed for PR runs * Updated changelog * Updated curl command to be silent * Updated changelog * Reverted some changes and added descriptions to config.yml * Fixed test extra delay * Added some evergreen script updates from another branch * Disabled baas network faults tests * Update flx migration test to enable dev mode * Fix baas branch for network tests * add more time to 'too large sync message error handling' test * Added network faults test and pulled nonideal out of nightly tests until fixes are added --- .gitignore | 4 + CHANGELOG.md | 4 +- .../how-to-use-remote-baas-host.md | 295 ++++++++++ evergreen/config.yml | 201 ++++++- evergreen/configure_baas_proxy.sh | 237 ++++++++ evergreen/install_baas.sh | 516 ++++++++++-------- evergreen/proxy-network-faults.toxics | 4 + evergreen/proxy-nonideal-transfer.toxics | 5 + evergreen/setup_baas_host.sh | 190 ++++++- evergreen/setup_baas_host_local.sh | 204 +++++-- evergreen/setup_baas_proxy.sh | 263 +++++++++ evergreen/wait_for_baas.sh | 32 +- test/object-store/CMakeLists.txt | 16 + test/object-store/audit.cpp | 2 +- test/object-store/c_api/c_api.cpp | 9 +- test/object-store/main.cpp | 12 + test/object-store/sync/app.cpp | 50 +- test/object-store/sync/client_reset.cpp | 24 +- test/object-store/sync/flx_migration.cpp | 43 +- test/object-store/sync/flx_sync.cpp | 9 +- .../object-store/util/sync/baas_admin_api.cpp | 150 ++--- .../object-store/util/sync/baas_admin_api.hpp | 33 +- .../util/sync/flx_sync_harness.hpp | 2 +- .../util/sync/sync_test_utils.cpp | 72 ++- .../util/sync/sync_test_utils.hpp | 15 +- test/object-store/util/test_file.cpp | 7 +- test/object-store/util/test_file.hpp | 3 + 27 files changed, 1887 insertions(+), 515 deletions(-) create mode 100644 doc/development/how-to-use-remote-baas-host.md create mode 100755 evergreen/configure_baas_proxy.sh create mode 100644 evergreen/proxy-network-faults.toxics create mode 100644 evergreen/proxy-nonideal-transfer.toxics create mode 100755 evergreen/setup_baas_proxy.sh diff --git a/.gitignore b/.gitignore index 1b3badb3ec5..8fd4a65df13 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ compile_commands.json /bindgen/generated/ node_modules/ tsconfig.tsbuildinfo + +# Baas remote host artifacts +baas-work-dir/ +ssh_agent_commands.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 888b15012d4..32aabb1c524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ ----------- ### Internals -* None. +* Add baas-network-tests nightly task for testing sync client operation with non-ideal network conditions. ([PR #6852](https://github.com/realm/realm-core/pull/6852)) +* Added non-ideal network conditions and network fault tests to the evergreen nightly test runs. ([PR #7063](https://github.com/realm/realm-core/pull/7063)) +* Updated baas tests to run with dev mode disabled by default. ([PR #6852](https://github.com/realm/realm-core/pull/6852)) ---------------------------------------------- diff --git a/doc/development/how-to-use-remote-baas-host.md b/doc/development/how-to-use-remote-baas-host.md new file mode 100644 index 00000000000..59327b78310 --- /dev/null +++ b/doc/development/how-to-use-remote-baas-host.md @@ -0,0 +1,295 @@ +# How to Run remote-baas, the baas-proxy and Network Tests Locally + +The following instructions demonstrate how to set up and run the remote baas and remote proxy + +## Running remote baas on an Evergreen Spawn Host + +1. Spawn and/or start an `ubuntu2004-medium` host in Evergreen and select an SSH key to use + for the connection. Make sure you have a local copy of the private and public key selected. +2. Create a `baas_host_vars.sh` file with the following contents: + +```bash +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export BAAS_HOST_NAME="" +export REALM_CORE_REVISION="" +export GITHUB_KNOWN_HOSTS="github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" +``` + +Here are descriptions of the different parameters in the `baas_host_vars.sh` file: + +* `AWS_ACCESS_KEY_ID` - The AWS Access Key ID used to run the baas server. +* `AWS_SECRET_ACCESS_KEY` - The AWS Secret Key used to run the baas server. +* `BAAS_HOST_NAME` - The hostname of the Linux host to download and run the baas server. +* `REALM_CORE_REVISION` - The commit/branch of Realm Core to use for other files from the + _evergreen/_ directory when running the baas server. +* `GITHUB_KNOWN_HOSTS` - The public server key to authenticate ssh with Github for downloading + files from Github. This is a fixed value that is provided by Github. + +**NOTE**: The `baas_host_vars.sh` and the `install_baas.sh` and `setup_baas_host.sh` files from the +local _evergreen/_ directory will be transferred to the remote host. The `setup_baas_host.sh` script +will check out Realm Core using the branch/commit provided to use for other auxillary files needed +when starting the baas_server. + +3. CD into the _realm-core/_ directory. +4. Run the _evergreen/setup_baas_host_local.sh_ script with the following arguments: + +```bash +$ evergreen/setup_baas_host_local.sh +. . . +Running setup script (with forward tunnel on :9090 to 127.0.0.1:9090) +. . . +Starting baas app server +Adding roles to admin user +--------------------------------------------- +Baas server ready +--------------------------------------------- + +# To enable verbose logging, add the -v argument before the baas_host_vars.sh file argurment +# Use the '-b ' to specify a specific version of the baas server to download/use +``` + +**NOTES:** + +* You must be connected to the MongoDB network either directly or via VPN in order to communicate +with the spawn host. If you get this error, check your network connection. It could also be due to +an incorrect host name for the `BAAS_HOST_NAME` setting in `baas_host_vars.sh`. + +```bash +SSH connection attempt 1/25 failed. Retrying... +ssh: connect to host port 22: Operation timed out +``` + +* If any of the local ports are in use, an error message will be displayed when the + script is run. Close any programs that are using these ports. + +```bash +Error: Local baas server port (9090) is already in use +baas_serv 66287 ... 12u IPv6 0x20837f52a108aa91 0t0 TCP *:9090 (LISTEN) +``` + +5. The required script files will be uploaded to the spawn host and the baas server will be + downloaded and started. The following local tunnels will be created over the SSH + connection to forward traffic to the remote host: + * **localhost:9090 -> baas server** - any traffic to local port 9090 will be forwarded to + the baas server running on the remote host. +6. Use CTRL-C to cancel the baas remote host script and stop the baas server running on the + spawn host. + +## Running remote baas and proxy on an Evergreen Spawn Host + +1. Spawn and/or start an `ubuntu2004-medium` host in Evergreen and select an SSH key to use + for the connection. Make sure you have a local copy of the private and public key selected. +2. Create a `baas_host_vars.sh` file with the following contents: + +```bash +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export BAAS_HOST_NAME="" +export REALM_CORE_REVISION="" +export GITHUB_KNOWN_HOSTS="github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" +``` + +Here are descriptions of the different parameters in the `baas_host_vars.sh` file: + +* `AWS_ACCESS_KEY_ID` - The AWS Access Key ID used to run the baas server. +* `AWS_SECRET_ACCESS_KEY` - The AWS Secret Key used to run the baas server. +* `BAAS_HOST_NAME` - The hostname of the Linux host to download and run the baas server and proxy. +* `REALM_CORE_REVISION` - The commit/branch of Realm Core to use for other files from the + _evergreen/_ directory when running the baas server. +* `GITHUB_KNOWN_HOSTS` - The public server key to authenticate ssh with Github for downloading + files from Github. This is a fixed value that is provided by Github. + +**NOTE**: The `baas_host_vars.sh` and the `install_baas.sh`, `setup_baas_host.sh` and +`setup_baas_proxy.sh` files from the local _evergreen/_ directory will be transferred to the +remote host. The `setup_baas_host.sh` script will check out Realm Core using the branch/commit +provided to use for other auxillary files needed when starting the baas_server. + +3. CD into the _realm-core/_ directory. +4. Run the _evergreen/setup_baas_host_local.sh_ script with the following arguments: + +```bash +$ evergreen/setup_baas_host_local.sh -t +. . . +Running setup script (with forward tunnel on :9090 to 127.0.0.1:9092) +- Baas proxy enabled - local HTTP API config port on :8474 +- Baas direct connection on port :9098 +. . . +Starting baas app server +Starting baas proxy: 127.0.0.1:9092 => 127.0.0.1:9090 +--------------------------------------------- +Baas proxy ready +--------------------------------------------- +Adding roles to admin user +--------------------------------------------- +Baas server ready +--------------------------------------------- + +# To enable verbose logging, add the -v argument before the baas_host_vars.sh file argurment +# Use the '-b ' to specify a specific version of the baas server to download/use +# Use the '-d ' to change the local port number for the baas server direct connection +# Use the '-c ' to change the local port number for the baas proxy configuration connection +# Use the '-l ' to change the port number for the baas proxy server listen port for new +# connections if there is a conflict with the default port (9092) on the remote host. The local +# baas server port 9090 forwards traffic to this port. +``` + +**NOTES:** + +* You must be connected to the MongoDB network either directly or via VPN in order to communicate +with the spawn host. If you get this error, check your network connection. It could also be due to +an incorrect host name for the `BAAS_HOST_NAME` setting in `baas_host_vars.sh`. + +```bash +SSH connection attempt 1/25 failed. Retrying... +ssh: connect to host port 22: Operation timed out +``` + +* If any of the local ports are in use, an error message will be displayed when the script is + run. Close any programs that are using these ports. + +```bash +Error: Local baas server port (9090) is already in use +baas_serv 66287 ... 12u IPv6 0x20837f52a108aa91 0t0 TCP *:9090 (LISTEN) +``` + +5. The required script files will be uploaded to the spawn host and the baas server will be + downloaded and started. The following local tunnels will be created over the SSH + connection to forward traffic to the remote host: + * **localhost:9090 -> baas proxy** - any traffic intended for the baas server will first go + through the baas proxy so network "faults" can be applied to the connection or packet. + * **localhost:9098 -> baas server** - any baas Admin API configuration or test setup should + be sent directly to the baas server via this port, since this traffic is not part of + the test. The local port value can be changed using the `-d ` command line option. + * **localhost:8474 -> baas proxy configuration** - the Toxiproxy server providing the baas + proxy operation can be configured through this port. The `baas_proxy` proxy for routing + traffic to the baas server is automatically configured when the baas proxy is started. The + local port value can be changed using the `-c ` command line option. +6. Use CTRL-C to cancel the baas remote host script and stop the baas server running on the + spawn host. + +## Running network fault tests + +### Building Realm Core for testing + +The Realm Core object store test executable needs to be built with the following options +provided to `cmake` when configuring the build: + +* REALM_ENABLE_SYNC: `On` +* REALM_ENABLE_AUTH_TESTS: `On` +* REALM_MONGODB_ENDPOINT: `"https://localhost1:9090"` +* REALM_ADMIN_ENDPOINT: `"https://localhost:9098"` + +When the object store tests executable it compiled, the baas Admin API commands will be +sent directly to the baas server via local port 9098 and all other baas server network +traffic will be sent to the baas server via the baas proxy using local port 9090. + +### Starting the baas proxy and server + +Refer to the +**[Running remote baas and proxy on an Evergreen Spawn Host](#running-remote-baas-and-proxy-on-an-evergreen-spawn-host)** +section for instructions on starting the baas proxy and server on the remote host. + +### Configuring network "faults" in baas proxy + +The complete list of network faults/conditions suppported by Toxiproxy can be found in the +_README.md_ file in the [Shopify/toxiproxy](https://github.com/Shopify/toxiproxy) github repo. +Any fault or condition added to a proxy is called a "toxic" and multiple toxics can be applied +to a proxy with a toxicity value that specifies the probability of applying that toxic to the +connection when it is opened. + +#### View current toxics applied to the proxy + +Query the `baas_proxy` proxy on the /proxies endpoint to view the current list of toxics +configured for the baas proxy. + +```bash +$ curl localhost:8474/proxies/baas_proxy/toxics +[]% + +$ curl localhost:8474/proxies/baas_proxy/toxics +[{"attributes":{"rate":20},"name":"bandwidth_limit","type":"bandwidth","stream":"downstream","toxicity":1}]% +``` + +#### Add a new toxic parameter to a proxy + +To create a new toxic send a message via the POST HTTP method with a JSON payload containing +details about the toxic and the attributes for the toxic type. A toxicity value is supported +to specify the probability that the toxic will be applied to a connection when it is opened. + +**NOTE:** A name is recommended when adding a toxic to make referencing it easier. Otherwise, +the name defaults to `_` (e.g. `bandwidth_downstream`). Using a unique name for +each toxic entry allows multiple toxics of the same type and stream to be configured for the +proxy. + +The following parameters can be provided or are required when adding a toxic: + +* `name`: toxic name (string, defaults to `_`) +* `type`: toxic type (string). See **[Available Toxics](#available-toxics)** +* `stream`: link direction to affect (string, defaults to `downstream`) +* `toxicity`: probability of the toxic being applied to a link (numeric, defaults to 1.0, 100%) +* `attributes`: a map (JSON object) of toxic-specific attributes + +```bash +curl --data "{\"name\":\"bandwidth_limit\", \"type\":\"bandwidth\", \"stream\": \"downstream\", \"toxicity\": 1.0, \"attributes\": {\"rate\": 20}}" localhost:8474/proxies/baas_proxy/toxics +{"attributes":{"rate":20},"name":"bandwidth_limit","type":"bandwidth","stream":"downstream","toxicity":1}% + +$ curl localhost:8474/proxies/baas_proxy/toxics +[{"attributes":{"rate":20},"name":"bandwidth_limit","type":"bandwidth","stream":"downstream","toxicity":1}]% +``` + +#### Available toxics + +The following list of toxics can be configured for the proxy: + +* `bandwidth` - Limit a connection to a maximum number of kilobytes per second. + * `rate` - rate in KB/s +* `down` - The proxy can be taken down, which will close all existing connections and not + allow new connections, by setting the `enabled` field for the proxy to `false`. +* `latency` - Add a delay to all data going through proxy. The delay is equal to latency + +/- jitter. + * `latency` - time in milliseconds + * `jitter` - time in milliseconds +* `limit_data` - Close the connection after a certain number of bytes has been transmited. + * `bytes` - number of bytes to be transmitted before the connection is closed +* `reset_peer` - Simulate a TCP RESET (connection closed by peer) on the connections + immediately or after a timeout. + * `timeout` - time in milliseconds +* `slicer` - Slice (split) data up into smaller bits, optionally adding a delay between each + slice packet. + * `average_size` - size in bytes of an average packet + * `size_variation` - variation of bytes of an average packet (should be smaller than `average_size`) + * `delay` - time in microseconds to delay each packet +* `slow_close` - Delay the TCP socket from closing until delay has elapsed. + * `delay` - time in milliseconds +* `timeout` - Stops all data from getting through and closes the connection after a timeout. If + timeout is 0, the connection won't close, but the data will be delayed until the toxic is removed. + * `timeout` - time in milliseconds + +#### Delete a current toxic from the proxy + +Use the DELETE HTTP method with the toxic name to delete an existing toxic applied to the proxy. + +```bash +$ curl localhost:8474/proxies/baas_proxy/toxics +[{"attributes":{"rate":20},"name":"bandwidth_limit","type":"bandwidth","stream":"downstream","toxicity":1}]% + +$ curl -X DELETE localhost:8474/proxies/baas_proxy/toxics/bandwidth_limit + +$ curl localhost:8474/proxies/baas_proxy/toxics +[]% +``` + +### Running the tests + +Only the object store tests that are performed against the baas server are needed to be run. +These can be run by running the following command. Depending on the toxics specified for the +baas proxy, these tests may take a long time. + +```bash +$ cd /test/object-store/realm-object-store-tests.app/Contents/MacOS +$ ./realm-object-store-tests "[baas]" +Filters: [baas] +Randomness seeded to: 1962875058 +. . . +``` diff --git a/evergreen/config.yml b/evergreen/config.yml index 4bd5a4ef054..940df4b78e1 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -75,8 +75,12 @@ functions: fi if [ -z "${disable_tests_against_baas|}" ]; then + scheme="http" set_cmake_var baas_vars REALM_ENABLE_AUTH_TESTS BOOL On - set_cmake_var baas_vars REALM_MONGODB_ENDPOINT STRING "http://localhost:9090" + set_cmake_var baas_vars REALM_MONGODB_ENDPOINT STRING "$scheme://localhost:9090" + if [ -n "${baas_admin_port|}" ]; then + set_cmake_var baas_vars REALM_ADMIN_ENDPOINT STRING "$scheme://localhost:${baas_admin_port}" + fi fi if [ -n "${enable_asan|}" ]; then @@ -132,7 +136,11 @@ functions: fi set_cmake_var realm_vars REALM_TEST_LOGGING BOOL On - set_cmake_var realm_vars REALM_TEST_LOGGING_LEVEL STRING "debug" + set_cmake_var realm_vars REALM_TEST_LOGGING_LEVEL STRING "${test_logging_level|debug}" + + if [[ -n "${test_timeout_extra|}" ]]; then + set_cmake_var realm_vars REALM_TEST_TIMEOUT_EXTRA STRING ${test_timeout_extra} + fi GENERATOR="${cmake_generator}" if [ -z "${cmake_generator|}" ]; then @@ -274,6 +282,8 @@ functions: - command: attach.results params: file_location: realm-core/${task_name}_results.json + + "upload baas artifacts": - command: shell.exec params: working_dir: realm-core @@ -289,7 +299,7 @@ functions: # Copy the baas_server log from the remote baas host if it exists if [[ ! -f baas_host.yml || ! -f .baas_ssh_key ]]; then - echo "No remote baas host or remote baas host definitions not found" + echo "Skipping - no remote baas host or remote baas host definitions not found" exit 0 fi @@ -299,14 +309,22 @@ functions: ssh_user="$(printf "ubuntu@%s" "$BAAS_HOST_NAME")" ssh_options="-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o ConnectTimeout=60 -i .baas_ssh_key" - # Copy the baas_server.log and mongod.log files from the remote baas host - REMOTE_PATH=/data/baas-remote/baas-work-dir - LOCAL_PATH=./baas-work-dir + # Copy the baas_server.log, mongod.log and (optionally) baas_proxy.log files from the remote baas host + REMOTE_PATH=/data/baas-remote + REMOTE_BAAS_PATH="$REMOTE_PATH/baas-work-dir" + REMOTE_BAAS_DB_PATH="$REMOTE_BAAS_PATH/mongodb-dbpath" + REMOTE_PROXY_PATH="$REMOTE_PATH/baas-proxy-dir" + + LOCAL_BAAS_PATH=./baas-work-dir + LOCAL_BAAS_DB_PATH="$LOCAL_BAAS_PATH/mongodb-dbpath" + LOCAL_PROXY_PATH=./baas-proxy-dir - mkdir -p "$LOCAL_PATH/" - mkdir -p "$LOCAL_PATH/mongodb-dbpath/" - scp $ssh_options $ssh_user:"$REMOTE_PATH/baas_server.log" "$LOCAL_PATH/baas_server.log" || true - scp $ssh_options $ssh_user:"$REMOTE_PATH/mongodb-dbpath/mongod.log" "$LOCAL_PATH/mongodb-dbpath/mongod.log" || true + mkdir -p "$LOCAL_BAAS_PATH/" + mkdir -p "$LOCAL_BAAS_DB_PATH/" + mkdir -p "$LOCAL_PROXY_PATH/" + scp $ssh_options $ssh_user:"$REMOTE_BAAS_PATH/baas_server.log" "$LOCAL_BAAS_PATH/baas_server.log" || true + scp $ssh_options $ssh_user:"$REMOTE_BAAS_DB_PATH/mongod.log" "$LOCAL_BAAS_DB_PATH/mongod.log" || true + scp $ssh_options $ssh_user:"$REMOTE_PROXY_PATH/baas_proxy.log" "$LOCAL_PROXY_PATH/baas_proxy.log" || true - command: s3.put params: @@ -341,6 +359,17 @@ functions: content_type: text/text display_name: mongod logs optional: true + - command: s3.put + params: + aws_key: '${artifacts_aws_access_key}' + aws_secret: '${artifacts_aws_secret_key}' + local_file: 'realm-core/baas-proxy-dir/baas_proxy.log' + remote_file: 'realm-core-stable/${branch_name}/${task_id}/${execution}/baas_proxy.log' + bucket: mciuploads + permissions: public-read + content_type: text/text + display_name: baas proxy logs + optional: true "upload fuzzer results": - command: shell.exec @@ -533,13 +562,23 @@ functions: BAAS_USER=ubuntu OPT_BAAS_BRANCH= + OPT_BAAS_PROXY= + OPT_BAAS_DIRECT= + if [ -n "${baas_branch}" ]; then OPT_BAAS_BRANCH="-b ${baas_branch}" fi + if [ -n "${baas_proxy}" ]; then + OPT_BAAS_PROXY="-t" + if [ -n "${baas_admin_port}" ]; then + OPT_BAAS_DIRECT="-d ${baas_admin_port}" + fi + fi # Run the setup_baas_host_local.sh script to configure and run baas on the remote host # Add -v to this command for verbose script logging - ./evergreen/setup_baas_host_local.sh -w ./baas-work-dir -u $BAAS_USER $OPT_BAAS_BRANCH ./baas_host_vars.sh ./.baas_ssh_key 2>&1 | tee install_baas_output.log + ./evergreen/setup_baas_host_local.sh -w ./baas-work-dir -u $BAAS_USER $OPT_BAAS_BRANCH $OPT_BAAS_PROXY \ + $OPT_BAAS_DIRECT ./baas_host_vars.sh ./.baas_ssh_key 2>&1 | tee install_baas_output.log "wait for baas to start": - command: shell.exec @@ -559,6 +598,35 @@ functions: echo "Baas is started!" + "setup proxy parameters": + - command: shell.exec + params: + working_dir: realm-core + shell: bash + script: |- + set -o errexit + set -o pipefail + set -o verbose + + if [[ -n "${disable_tests_against_baas|}" ]]; then + echo "Error: Bass is disabled for network tests" + exit 1 + fi + + if [[ -z "${proxy_toxics_file|}" ]]; then + echo "Error: Baas proxy toxics config file was not provided" + exit 1 + fi + + if [[ -n "${proxy_toxics_randoms|}" ]]; then + PROXY_RANDOMS="-r ${proxy_toxics_randoms}" + fi + + # Configure the toxics for the baas proxy + evergreen/configure_baas_proxy.sh $PROXY_RANDOMS "${proxy_toxics_file}" + # Display the list of configured toxics + curl --silent http://localhost:8474/proxies/baas_proxy/toxics + "check branch state": - command: shell.exec type: setup @@ -808,11 +876,11 @@ tasks: - name: long-running-core-tests tags: [ "for_nightly_tests" ] allowed_requesters: [ "ad_hoc", "patch" ] + # The long-running tests can take a really long time on Windows, so we give the test up to 4 + # hours to complete + exec_timeout_secs: 14400 commands: - func: "run tests" - # The long-running tests can take a really long time on Windows, so we give the test up to 4 - # hours to complete - timeout_secs: 14400 vars: test_filter: CoreTests report_test_progress: On @@ -867,7 +935,7 @@ tasks: # These are local object store tests; baas is not started, however some use the sync server - name: object-store-tests - tags: [ "test_suite_remote_baas", "for_pull_requests" ] + tags: [ "object_store_test_suite", "for_pull_requests" ] exec_timeout_secs: 3600 commands: - func: "compile" @@ -882,7 +950,7 @@ tasks: # These are baas object store tests that run against baas running on a remote host - name: baas-integration-tests - tags: [ "test_suite_remote_baas", "for_pull_requests" ] + tags: [ "object_store_test_suite", "for_pull_requests" ] exec_timeout_secs: 3600 commands: - func: "launch remote baas" @@ -899,6 +967,31 @@ tasks: verbose_test_output: true - func: "check branch state" +- name: baas-network-tests + # Uncomment once tests are passing + # tags: [ "for_nightly_tests" ] + # These tests can be manually requested for patches and pull requests + allowed_requesters: [ "ad_hoc", "patch", "github_pr" ] + # The network tests can take a really long time, so we give the test up to 4 + # hours to complete + exec_timeout_secs: 14400 + commands: + - func: "launch remote baas" + vars: + baas_branch: 27f42f55a7944ed7d8ba9fad1854a4b22714cb8d + baas_proxy: On + - func: "compile" + vars: + target_to_build: ObjectStoreTests + - func: "wait for baas to start" + - func: "setup proxy parameters" + - func: "run tests" + vars: + test_label: objstore-baas + test_executable_name: "realm-object-store-tests" + verbose_test_output: true + - func: "check branch state" + - name: process_coverage_data tags: [ "for_pull_requests" ] exec_timeout_secs: 1800 @@ -1074,12 +1167,13 @@ task_groups: - func: "fetch binaries" teardown_task: - func: "upload test results" + - func: "upload baas artifacts" timeout: - func: "run hang analyzer" tasks: - compile - .test_suite - - .test_suite_remote_baas + - .object_store_test_suite - package # Runs object-store-tests against baas running on remote host @@ -1091,12 +1185,29 @@ task_groups: - func: "fetch binaries" teardown_task: - func: "upload test results" + - func: "upload baas artifacts" timeout: - func: "run hang analyzer" tasks: - compile - .test_suite - - .test_suite_remote_baas + - .object_store_test_suite + +# Runs object-store-tests against baas running on remote host and runs +# the network simulation tests as a separate task for nightly builds +- name: network_tests + max_hosts: 1 + setup_group_can_fail_task: true + setup_group: + - func: "fetch source" + - func: "fetch binaries" + teardown_task: + - func: "upload test results" + - func: "upload baas artifacts" + timeout: + - func: "run hang analyzer" + tasks: + - baas-network-tests # Runs object-store-tests against baas running on remote host - name: compile_test_coverage @@ -1107,12 +1218,13 @@ task_groups: - func: "fetch binaries" teardown_task: - func: "upload test results" + - func: "upload baas artifacts" timeout: - func: "run hang analyzer" tasks: - compile - .test_suite - - .test_suite_remote_baas + - .object_store_test_suite - process_coverage_data - name: benchmarks @@ -1331,6 +1443,51 @@ buildvariants: tasks: - name: fuzzer-tests +- name: ubuntu2004-network-nonideal + display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 - nonideal transfer)" + run_on: ubuntu2004-large + expansions: + clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" + cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-linux-x86_64.tar.gz" + cmake_bindir: "./cmake_binaries/bin" + fetch_missing_dependencies: On + c_compiler: "./clang_binaries/bin/clang" + cxx_compiler: "./clang_binaries/bin/clang++" + cmake_build_type: RelWithDebInfo + run_with_encryption: On + baas_admin_port: 9098 + test_logging_level: trace + test_timeout_extra: 60 + proxy_toxics_file: evergreen/proxy-nonideal-transfer.toxics + # RANDOM1: bandwidth-upstream limited to between 10-50 KB/s from the client to the server + # RANDOM2: bandwidth-downstream limited to between 10-50 KB/s from the server to the client + proxy_toxics_randoms: "10:50|10:50" + tasks: + - name: network_tests + +- name: ubuntu2004-network-faulty + display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 - network faults)" + run_on: ubuntu2004-large + expansions: + clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" + cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-linux-x86_64.tar.gz" + cmake_bindir: "./cmake_binaries/bin" + fetch_missing_dependencies: On + c_compiler: "./clang_binaries/bin/clang" + cxx_compiler: "./clang_binaries/bin/clang++" + cmake_build_type: RelWithDebInfo + run_with_encryption: On + baas_admin_port: 9098 + test_logging_level: trace + proxy_toxics_file: evergreen/proxy-network-faults.toxics + # RANDOM1: limit-data-upstream to close connection after between 1000-3000 bytes have been sent + # RANDOM2: limit-data-downstream to close connection after between 1000-3000 bytes have been received + # RANDOM3: slow-close-upstream to keep connection to server open after 1000-1500 milliseconds after being closed + # RANDOM4: reset-peer-upstream after 50-200 seconds to force close the connection to the server + proxy_toxics_randoms: "1000:3000|1000:3000|1000:1500|50:200" + tasks: + - name: network_tests + - name: rhel70 display_name: "RHEL 7 x86_64" run_on: rhel70-large @@ -1565,8 +1722,7 @@ buildvariants: curl_base: "/cygdrive/c/curl" python3: "/cygdrive/c/python/python37/python.exe" tasks: - - name: compile_test_and_package - - name: long-running-tests + - name: compile_test - name: windows-64-encryption display_name: "Windows x86_64 (Encryption enabled)" @@ -1598,7 +1754,8 @@ buildvariants: curl_base: "/cygdrive/c/curl" python3: "/cygdrive/c/python/python37/python.exe" tasks: - - name: compile_test + - name: compile_test_and_package + - name: long-running-tests - name: windows-64-vs2019-asan display_name: "Windows x86_64 (VS 2019 ASAN)" diff --git a/evergreen/configure_baas_proxy.sh b/evergreen/configure_baas_proxy.sh new file mode 100755 index 00000000000..fa2cc5c30ab --- /dev/null +++ b/evergreen/configure_baas_proxy.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# The script to send and execute the configuration to set up the baas proxy toxics. +# +# Usage: +# ./evergreen/configure_baas_proxy.sh [-c PORT] [-r NUM] [-p PATH] [-v] [-h] CONFIG_JSON +# + +set -o errexit +set -o errtrace +set -o pipefail + +CONFIG_PORT=8474 +PROXY_NAME="baas_proxy" +RND_STRING= +RND_MIN= +RND_MAX= +RND_DIFF=32767 +CURL=/usr/bin/curl +VERBOSE= +CONFIG_TMP_DIR= + +function usage() +{ + echo "Usage: configure_baas_proxy.sh [-c PORT] [-r MIN:MAX] [-p PATH] [-t NAME] [-v] [-h] CONFIG_JSON" + echo -e "\tCONFIG_JSON\tPath to baas proxy toxics config file (one toxic config JSON object per line)" + echo "Options:" + echo -e "\t-c PORT\t\tLocal configuration port for proxy HTTP API (default ${CONFIG_PORT})" + echo -e "\t-r MIN:MAX\tString containing one or more sets of min:max values to replace %RANDOM#% in toxics" + echo -e "\t-p PATH\t\tPath to the curl executable (default ${CURL})" + echo -e "\t-t NAME\t\tName of the proxy to be configured (default ${PROXY_NAME})" + echo -e "\t-v\t\tEnable verbose script debugging" + echo -e "\t-h\t\tShow this usage summary and exit" + # Default to 0 if exit code not provided + exit "${1:0}" +} + +while getopts "c:r:p:vh" opt; do + case "${opt}" in + c) CONFIG_PORT="${OPTARG}";; + r) RND_STRING="${OPTARG}";; + p) CURL="${OPTARG}";; + v) VERBOSE="yes";; + h) usage 0;; + *) usage 1;; + esac +done + +TOXIPROXY_URL="http://localhost:${CONFIG_PORT}" + +shift $((OPTIND - 1)) + +if [[ $# -lt 1 ]]; then + echo "Error: Baas proxy toxics config file not provided" + usage 1 +fi +PROXY_JSON_FILE="${1}"; shift; + +if [[ -z "${PROXY_JSON_FILE}" ]]; then + echo "Error: Baas proxy toxics config file value was empty" + usage 1 +elif [[ ! -f "${PROXY_JSON_FILE}" ]]; then + echo "Error: Baas proxy toxics config file not found: ${PROXY_JSON_FILE}" + usage 1 +fi + +if [[ -z "${CURL}" ]]; then + echo "Error: curl path is empty" + usage 1 +elif [[ ! -x "${CURL}" ]]; then + echo "Error: curl path is not valid: ${CURL}" + usage 1 +fi + +trap 'catch $? ${LINENO}' ERR +trap 'on_exit' INT TERM EXIT + +# Set up catch function that runs when an error occurs +function catch() +{ + # Usage: catch EXIT_CODE LINE_NUM + echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while configuring baas proxy" +} + +function on_exit() +{ + # Usage: on_exit + if [[ -n "${CONFIG_TMP_DIR}" && -d "${CONFIG_TMP_DIR}" ]]; then + rm -rf "${CONFIG_TMP_DIR}" + fi +} + +function check_port() +{ + # Usage check_port PORT + port_num="${1}" + if [[ -n "${port_num}" && ${port_num} -gt 0 && ${port_num} -lt 65536 ]]; then + return 0 + fi + return 1 +} + +function check_port_ready() +{ + # Usage: check_port_active PORT PORT_NAME + port_num="${1}" + port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) + if [[ -z "${port_check}" ]]; then + echo "Error: ${2} port (${port_num}) is not ready - is the Baas proxy running?" + exit 1 + fi + if ! curl "${TOXIPROXY_URL}/version" --silent --fail --connect-timeout 10 > /dev/null; then + echo "Error: No response from ${2} (${port_num}) - is the Baas proxy running?" + exit 1 + fi +} + +function parse_random() +{ + # Usage: parse_random RANDOM_STRING => RND_MIN, RND_MAX + random_string="${1}" + old_ifs="${IFS}" + + RND_MIN=() + RND_MAX=() + + if [[ "${random_string}" =~ .*|.* ]]; then + IFS='|' + read -ra random_list <<< "${random_string}" + else + random_list=("${random_string}") + fi + for random in "${random_list[@]}" + do + if [[ ! "${random}" =~ .*:.* ]]; then + IFS="${old_ifs}" + return 1 + fi + + # Setting IFS (input field separator) value as ":" and read the split string into array + IFS=':' + read -ra rnd_arr <<< "${random}" + + if [[ ${#rnd_arr[@]} -ne 2 ]]; then + IFS="${old_ifs}" + return 1 + elif [[ -z "${rnd_arr[0]}" || -z "${rnd_arr[0]}" ]]; then + IFS="${old_ifs}" + return 1 + fi + + if [[ ${rnd_arr[0]} -le ${rnd_arr[1]} ]]; then + RND_MIN+=("${rnd_arr[0]}") + RND_MAX+=("${rnd_arr[1]}") + else + RND_MIN+=("${rnd_arr[1]}") + RND_MAX+=("${rnd_arr[0]}") + fi + done + IFS="${old_ifs}" + return 0 +} + +function generate_random() +{ + # Usage: generate_random MINVAL MAXVAL => RAND_VAL + minval="${1}" + maxval="${2}" + diff=$(( "${maxval}" - "${minval}" )) + if [[ ${diff} -gt ${RND_DIFF} ]]; then + return 1 + fi + RAND_VAL=$(( "$minval" + $(("$RANDOM" % "$diff")) )) +} + +# Wait until after the functions are configured before enabling verbose tracing +if [[ -n "${VERBOSE}" ]]; then + set -o verbose + set -o xtrace +fi + +if ! check_port "${CONFIG_PORT}"; then + echo "Error: Baas proxy HTTP API config port was invalid: '${CONFIG_PORT}'" + usage 1 +fi + +# Parse and verify the random string, if provided +if [[ -n "${RND_STRING}" ]]; then + if ! parse_random "${RND_STRING}"; then + echo "Error: Malformed random string: ${random_string} - format 'MIN:MAX[|MIN:MAX[|...]]" + usage 1 + fi +fi + +# Verify the Baas proxy is ready to roll +check_port_ready "${CONFIG_PORT}" "Baas proxy HTTP API config" + +# Create a temp directory for constructing the updated config file +CONFIG_TMP_DIR=$(mktemp -d -t "proxy-config.XXXXXX") +cp "${PROXY_JSON_FILE}" "${CONFIG_TMP_DIR}" +json_file="$(basename "${PROXY_JSON_FILE}")" +TMP_CONFIG="${CONFIG_TMP_DIR}/${json_file}" + +if [[ ${#RND_MIN[@]} -gt 0 ]]; then + cnt=0 + while [[ cnt -lt ${#RND_MIN[@]} ]]; do + rndmin=${RND_MIN[cnt]} + rndmax=${RND_MAX[cnt]} + if ! generate_random "${rndmin}" "${rndmax}"; then + echo "Error: MAX - MIN cannot be more than ${RND_DIFF}" + exit 1 + fi + + cnt=$((cnt + 1)) + printf "Generated random value #%d from %d to %d: %d\n" "${cnt}" "${rndmin}" "${rndmax}" "${RAND_VAL}" + sed_pattern=$(printf "s/%%RANDOM%d%%/%d/g" "${cnt}" "${RAND_VAL}") + sed -i.bak "${sed_pattern}" "${TMP_CONFIG}" + done +fi + +# Get the current list of configured toxics for the baas_proxy proxy +TOXICS=$(${CURL} --silent "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics") +if [[ "${TOXICS}" != "[]" ]]; then + # Extract the toxic names from the toxics list JSON + # Steps: Remove brackets, split into lines, extract "name" value + mapfile -t TOXIC_LIST < <(echo "${TOXICS}" | sed 's/\[\(.*\)\]/\1/g' | sed 's/},{/}\n{/g' | sed 's/.*"name":"\([^"]*\).*/\1/g') + echo "Clearing existing set of toxics (${#TOXIC_LIST[@]}) for ${PROXY_NAME} proxy" + for toxic in "${TOXIC_LIST[@]}" + do + ${CURL} -X DELETE "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics/${toxic}" + done +fi + +# Configure the new set of toxics for the baas_proxy proxy +echo "Configuring toxics for ${PROXY_NAME} proxy with file: ${json_file}" +while IFS= read -r line; do + ${CURL} -X POST -H "Content-Type: application/json" --silent -d "${line}" "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics" > /dev/null +done < "${TMP_CONFIG}" diff --git a/evergreen/install_baas.sh b/evergreen/install_baas.sh index d0cb7493612..baf9ea6a4c0 100755 --- a/evergreen/install_baas.sh +++ b/evergreen/install_baas.sh @@ -3,7 +3,7 @@ # and will import a given app into it. # # Usage: -# ./evergreen/install_baas.sh -w {path to working directory} [-b git revision of baas] [-v] [-h] +# ./evergreen/install_baas.sh -w PATH [-b BRANCH] [-v] [-h] # # shellcheck disable=SC1091 @@ -13,112 +13,171 @@ set -o errexit set -o pipefail set -o functrace -case $(uname -s) in - Darwin) - if [[ "$(uname -m)" == "arm64" ]]; then - export GOARCH=arm64 - STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-6.1.0-alpha-527-g796351f.tgz" - STITCH_ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg" - GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-arm64.tar.gz" - MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-arm64-enterprise-6.0.0-rc13.tgz" - MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-1.5.0-darwin-arm64.zip" - - # Go's scheduler is not BIG.little aware, and by default will spawn - # threads until they end up getting scheduled on efficiency cores, - # which is slower than just not using them. Limiting the threads to - # the number of performance cores results in them usually not - # running on efficiency cores. Checking the performance core count - # wasn't implemented until the first CPU with a performance core - # count other than 4 was released, so if it's unavailable it's 4. - GOMAXPROCS="$(sysctl -n hw.perflevel0.logicalcpu || echo 4)" - export GOMAXPROCS - else - export GOARCH=amd64 - STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz" - STITCH_ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg" - GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-amd64.tar.gz" - MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-x86_64-enterprise-5.0.3.tgz" - fi - - NODE_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/node-v14.17.0-darwin-x64.tar.gz" - JQ_DOWNLOAD_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/jq-1.6-darwin-amd64" - ;; - Linux) - GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.linux-amd64.tar.gz" - JQ_DOWNLOAD_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/jq-1.6-linux-amd64" - NODE_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/node-v14.17.0-linux-x64.tar.gz" - - # Only x86_64 Linux machines are supported - linux_arch="$(uname -m)" - if [[ "${linux_arch}" != "x86_64" ]]; then - echo "Error: only x86_64 Linux machines are supported: ${linux_arch}" +# Set up catch function that runs when an error occurs +trap 'catch $? ${LINENO}' ERR +trap "exit" INT TERM + +function catch() +{ + # Usage: catch EXIT_CODE LINE_NUM + echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while starting baas" +} + +# Adds a string to $PATH if not already present. +function pathadd() { + if [ -d "${1}" ] && [[ ":${PATH}:" != *":${1}:"* ]]; then + PATH="${1}${PATH:+":${PATH}"}" + export PATH + fi +} + +function setup_target_dependencies() { + target="$(uname -s)" + case "${target}" in + Darwin) + NODE_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/node-v14.17.0-darwin-x64.tar.gz" + JQ_DOWNLOAD_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/jq-1.6-darwin-amd64" + ;; + Linux) + NODE_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/node-v14.17.0-linux-x64.tar.gz" + JQ_DOWNLOAD_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/jq-1.6-linux-amd64" + ;; + *) + echo "Error: unsupported platform ${target}" exit 1 - fi - - # Detect what distro/version of Linux we are running on to determine the right version of MongoDB to download - # /etc/os-release covers debian/ubuntu/suse - if [[ -e /etc/os-release ]]; then - # Amazon Linux 2 comes back as 'amzn' - DISTRO_NAME="$(. /etc/os-release ; echo "${ID}")" - DISTRO_VERSION="$(. /etc/os-release ; echo "${VERSION_ID}")" - DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "${DISTRO_VERSION}" )" - elif [[ -e /etc/redhat-release ]]; then - # /etc/redhat-release covers RHEL - DISTRO_NAME=rhel - DISTRO_VERSION="$(lsb_release -s -r)" - DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "${DISTRO_VERSION}" )" - fi - case $DISTRO_NAME in - ubuntu | linuxmint) - MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-5.0.3.tgz" - STITCH_ASSISTED_AGG_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_ubuntu2004_x86_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo-ubuntu2004-x86_64.so" - STITCH_SUPPORT_LIB_URL="https://s3.amazonaws.com/static.realm.io/stitch-support/stitch-support-ubuntu2004-4.4.17-rc1-2-g85de0cc.tgz" - ;; - rhel) - case ${DISTRO_VERSION_MAJOR} in - 7) - MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-5.0.3.tgz" - STITCH_ASSISTED_AGG_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_linux_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo.so" - STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/linux-x64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz" - ;; - *) - echo "Error: unsupported version of RHEL ${DISTRO_VERSION}" - exit 1 - ;; - esac - ;; - *) - if [[ -z "${MONGODB_DOWNLOAD_URL}" ]]; then - echo "Error: missing MONGODB_DOWNLOAD_URL env variable to download mongodb from." - exit 1 - fi - if [[ -z "${STITCH_ASSISTED_AGG_LIB_PATH}" ]]; then - echo "Error: missing STITCH_ASSISTED_AGG_LIB_PATH env variable to find assisted agg libmongo.so" - exit 1 - fi - if [[ -z "${STITCH_SUPPORT_LIB_PATH}" ]]; then - echo "Error: missing STITCH_SUPPORT_LIB_PATH env variable to find the mongo stitch support library" + ;; + esac +} + +function setup_baas_dependencies() { + # <-- Uncomment to enable constants.sh + #baas_directory="${1}" + #baas_contents_file="${baas_directory}/.evergreen/constants.sh" + # --> + BAAS_PLATFORM= + MONGODB_DOWNLOAD_URL= + MONGOSH_DOWNLOAD_URL= + GOLANG_URL= + STITCH_SUPPORT_LIB_URL= + LIBMONGO_URL= + ASSISTED_AGG_URL= + target="$(uname -s)" + platform_string="unknown" + case "${target}" in + Darwin) + if [[ "$(uname -m)" == "arm64" ]]; then + export GOARCH=arm64 + MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-arm64-enterprise-6.0.0-rc13.tgz" + MONGOSH_DOWNLOAD_URL="https://downloads.mongodb.com/compass/mongosh-1.5.0-darwin-arm64.zip" + # <-- Remove after enabling constants.sh + STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-6.1.0-alpha-527-g796351f.tgz" + ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg" + GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-arm64.tar.gz" + # --> + + # Go's scheduler is not BIG.little aware, and by default will spawn + # threads until they end up getting scheduled on efficiency cores, + # which is slower than just not using them. Limiting the threads to + # the number of performance cores results in them usually not + # running on efficiency cores. Checking the performance core count + # wasn't implemented until the first CPU with a performance core + # count other than 4 was released, so if it's unavailable it's 4. + GOMAXPROCS="$(sysctl -n hw.perflevel0.logicalcpu || echo 4)" + export GOMAXPROCS + BAAS_PLATFORM="Darwin_arm64" + else + export GOARCH=amd64 + MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/osx/mongodb-macos-x86_64-enterprise-5.0.3.tgz" + # <-- Remove after enabling constants.sh + STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/macos-arm64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz" + ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_6513254ad6d80abfffa5fbdc_23_09_26_18_39_06/assisted_agg" + GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.darwin-amd64.tar.gz" + # --> + BAAS_PLATFORM="Darwin_x86_64" + fi + platform_string="${BAAS_PLATFORM}" + ;; + Linux) + BAAS_PLATFORM="Linux_x86_64" + # Detect what distro/version of Linux we are running on to download the right version of MongoDB to download + # /etc/os-release covers debian/ubuntu/suse + if [[ -e /etc/os-release ]]; then + # Amazon Linux 2 comes back as 'amzn' + DISTRO_NAME="$(. /etc/os-release ; echo "${ID}")" + DISTRO_VERSION="$(. /etc/os-release ; echo "${VERSION_ID}")" + DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "${DISTRO_VERSION}")" + elif [[ -e /etc/redhat-release ]]; then + # /etc/redhat-release covers RHEL + DISTRO_NAME=rhel + DISTRO_VERSION="$(lsb_release -s -r)" + DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "${DISTRO_VERSION}")" + fi + platform_string="${BAAS_PLATFORM} - ${DISTRO_NAME} ${DISTRO_VERSION}" + case "${DISTRO_NAME}" in + ubuntu | linuxmint) + MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-5.0.3.tgz" + # <-- Remove after enabling constants.sh + LIBMONGO_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_ubuntu2004_x86_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo-ubuntu2004-x86_64.so" + STITCH_SUPPORT_LIB_URL="https://s3.amazonaws.com/static.realm.io/stitch-support/stitch-support-ubuntu2004-4.4.17-rc1-2-g85de0cc.tgz" + GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.linux-amd64.tar.gz" + # --> + ;; + rhel) + case "${DISTRO_VERSION_MAJOR}" in + 7) + MONGODB_DOWNLOAD_URL="https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-5.0.3.tgz" + # <-- Remove after enabling constants.sh + LIBMONGO_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_linux_64_patch_1e7861d9b7462f01ea220fad334f10e00f0f3cca_65135b432fbabe741bd24429_23_09_26_22_29_24/libmongo.so" + STITCH_SUPPORT_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-support/linux-x64/stitch-support-4.4.17-rc1-2-g85de0cc.tgz" + GOLANG_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.21.1.linux-amd64.tar.gz" + # --> + ;; + *) + echo "Error: unsupported version of RHEL ${DISTRO_VERSION}" + exit 1 + ;; + esac + ;; + *) + echo "Error: unsupported platform Linux ${DISTRO_NAME}" exit 1 - fi - ;; - esac - ;; - *) - if [[ -z "${MONGODB_DOWNLOAD_URL}" ]]; then - echo "Error: missing MONGODB_DOWNLOAD_URL env variable to download mongodb from." - exit 1 - fi - if [[ -z "${STITCH_ASSISTED_AGG_LIB_PATH}" ]]; then - echo "Error: missing STITCH_ASSISTED_AGG_LIB_PATH env variable to find assisted agg libmongo.so" - exit 1 - fi - if [[ -z "${STITCH_SUPPORT_LIB_PATH}" ]]; then - echo "Error: missing STITCH_SUPPORT_LIB_PATH env variable to find the mongo stitch support library" + ;; + esac + ;; + *) + echo "Error: unsupported platform: ${target}" exit 1 - fi + ;; + esac + export BAAS_PLATFORM + echo "Platform: ${platform_string}" + # shellcheck source=/dev/null + # <-- Uncomment to enable constants.sh + # source "${baas_contents_file}" + # --> + + exit_code=0 + + if [[ -z "${GOLANG_URL}" ]]; then + echo "Error: go download URL (GOLANG_URL) not defined for this platform" + exit_code=1 + fi + if [[ -z "${STITCH_SUPPORT_LIB_URL}" ]]; then + echo "Error: baas support library URL (STITCH_SUPPORT_LIB_URL) not defined for this platform" + exit_code=1 + fi + if [[ "${target}" == "Linux" && -z "${LIBMONGO_URL}" ]]; then + echo "Error: baas assisted agg library URL (LIBMONGO_URL) not defined for this Linux platform" + exit_code=1 + fi + if [[ "${target}" == "Darwin" && -z "${ASSISTED_AGG_URL}" ]]; then + echo "Error: baas assisted agg library URL (ASSISTED_AGG_URL) not defined for this Mac OS platform" + exit_code=1 + fi + if [[ ${exit_code} -eq 1 ]]; then exit 1 - ;; -esac + fi +} # Allow path to CURL to be overloaded by an environment variable CURL="${CURL:=$LAUNCHER curl}" @@ -130,7 +189,8 @@ REALPATH="${BASE_PATH}/abspath.sh" function usage() { echo "Usage: install_baas.sh -w PATH [-b BRANCH] [-v] [-h]" - echo -e "\t-w PATH\t\tPath to working dir" + echo -e "\t-w PATH\t\tPath to working directory" + echo "Options:" echo -e "\t-b BRANCH\tOptional branch or git spec of baas to checkout/build" echo -e "\t-v\t\tEnable verbose script debugging" echo -e "\t-h\t\tShow this usage summary and exit" @@ -146,33 +206,45 @@ while getopts "w:b:vh" opt; do case "${opt}" in w) WORK_PATH="$($REALPATH "${OPTARG}")";; b) BAAS_VERSION="${OPTARG}";; - v) VERBOSE="yes"; set -o verbose; set -o xtrace;; + v) VERBOSE="yes";; h) usage 0;; *) usage 1;; esac done if [[ -z "${WORK_PATH}" ]]; then - echo "Must specify working directory" + echo "Error: Baas working directory was empty or not provided" usage 1 fi +if [[ -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" ]]; then + echo "Error: AWS_ACCESS_KEY_ID and/or AWS_SECRET_ACCESS_KEY are not set" + exit 1 +fi + +function check_port_in_use() +{ + # Usage: check_port_in_use PORT PORT_NAME + port_num="${1}" + port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) + if [[ -n "${port_check}" ]]; then + echo "Error: ${2} port (${port_num}) is already in use" + echo -e "${port_check}" + exit 1 + fi +} + # Check the mongodb and baas_server port availability first MONGODB_PORT=26000 -BAAS_PORT=9090 +check_port_in_use "${MONGODB_PORT}" "mongodb" -MONGODB_PORT_CHECK=$(lsof -P -i:${MONGODB_PORT} | grep "LISTEN" || true) -if [[ -n "${MONGODB_PORT_CHECK}" ]]; then - echo "Error: mongodb port (${MONGODB_PORT}) is already in use" - echo -e "${MONGODB_PORT_CHECK}" - exit 1 -fi +BAAS_PORT=9090 +check_port_in_use "${BAAS_PORT}" "baas server" -BAAS_PORT_CHECK=$(lsof -P -i:${BAAS_PORT} | grep "LISTEN" || true) -if [[ -n "${BAAS_PORT_CHECK}" ]]; then - echo "Error: baas server port (${BAAS_PORT}) is already in use" - echo -e "${BAAS_PORT_CHECK}" - exit 1 +# Wait to enable verbosity logging, if enabled +if [[ -n "${VERBOSE}" ]]; then + set -o verbose + set -o xtrace fi # Create and cd into the working directory @@ -182,14 +254,6 @@ echo "Work path: ${WORK_PATH}" # Set up some directory paths BAAS_DIR="${WORK_PATH}/baas" -BAAS_DEPS_DIR="${WORK_PATH}/baas_dep_binaries" -NODE_BINARIES_DIR="${WORK_PATH}/node_binaries" -MONGO_BINARIES_DIR="${WORK_PATH}/mongodb-binaries" -MONGODB_PATH="${WORK_PATH}/mongodb-dbpath" - -DYLIB_DIR="${BAAS_DIR}/etc/dylib" -DYLIB_LIB_DIR="${DYLIB_DIR}/lib" -TRANSPILER_DIR="${BAAS_DIR}/etc/transpiler" # Define files for storing state BAAS_SERVER_LOG="${WORK_PATH}/baas_server.log" @@ -197,39 +261,36 @@ BAAS_READY_FILE="${WORK_PATH}/baas_ready" BAAS_STOPPED_FILE="${WORK_PATH}/baas_stopped" BAAS_PID_FILE="${WORK_PATH}/baas_server.pid" MONGOD_PID_FILE="${WORK_PATH}/mongod.pid" -MONGOD_LOG="${MONGODB_PATH}/mongod.log" - -# Delete the mongod working directory if it exists from a previous run -# Wait to create this directory until just before mongod is started -if [[ -d "${MONGODB_PATH}" ]]; then - rm -rf "${MONGODB_PATH}" -fi +GO_ROOT_FILE="${WORK_PATH}/go_root" # Remove some files from a previous run if they exist if [[ -f "${BAAS_SERVER_LOG}" ]]; then - rm "${BAAS_SERVER_LOG}" + rm -f "${BAAS_SERVER_LOG}" fi if [[ -f "${BAAS_READY_FILE}" ]]; then - rm "${BAAS_READY_FILE}" + rm -f "${BAAS_READY_FILE}" fi if [[ -f "${BAAS_STOPPED_FILE}" ]]; then - rm "${BAAS_STOPPED_FILE}" + rm -f "${BAAS_STOPPED_FILE}" fi if [[ -f "${BAAS_PID_FILE}" ]]; then - rm "${BAAS_PID_FILE}" + rm -f "${BAAS_PID_FILE}" fi if [[ -f "${MONGOD_PID_FILE}" ]]; then - rm "${MONGOD_PID_FILE}" + rm -f "${MONGOD_PID_FILE}" fi # Set up the cleanup function that runs at exit and stops mongod and the baas server -function cleanup() +trap 'on_exit' EXIT + +function on_exit() { + # Usage: on_exit # The baas server is being stopped (or never started), create a 'baas_stopped' file - touch "${BAAS_STOPPED_FILE}" + touch "${BAAS_STOPPED_FILE}" || true - baas_pid="" - mongod_pid="" + baas_pid= + mongod_pid= if [[ -f "${BAAS_PID_FILE}" ]]; then baas_pid="$(< "${BAAS_PID_FILE}")" fi @@ -240,54 +301,40 @@ function cleanup() if [[ -n "${baas_pid}" ]]; then echo "Stopping baas ${baas_pid}" - kill "${baas_pid}" + kill "${baas_pid}" || true echo "Waiting for baas to stop" wait "${baas_pid}" + rm -f "${BAAS_PID_FILE}" || true fi if [[ -n "${mongod_pid}" ]]; then - echo "Killing mongod ${mongod_pid}" - kill "${mongod_pid}" + echo "Stopping mongod ${mongod_pid}" + kill "${mongod_pid}" || true echo "Waiting for processes to exit" wait + rm -f "${MONGOD_PID_FILE}" || true fi } -trap "exit" INT TERM ERR -trap 'cleanup $?' EXIT - -echo "Installing node and go to build baas and its dependencies" - -# Create the /node_binaries/ directory -[[ -d "${NODE_BINARIES_DIR}" ]] || mkdir -p "${NODE_BINARIES_DIR}" -# Download node if it's not found -if [[ ! -x "${NODE_BINARIES_DIR}/bin/node" ]]; then - pushd "${NODE_BINARIES_DIR}" > /dev/null - ${CURL} -LsS "${NODE_URL}" | tar -xz --strip-components=1 - popd > /dev/null # node_binaries -fi -export PATH="${NODE_BINARIES_DIR}/bin":${PATH} -echo "Node version: $(node --version)" +setup_target_dependencies -# Download go if it's not found and set up the GOROOT for building/running baas -[[ -x ${WORK_PATH}/go/bin/go ]] || (${CURL} -sL $GO_URL | tar -xz) -export GOROOT="${WORK_PATH}/go" -export PATH="${WORK_PATH}/go/bin":${PATH} -echo "Go version: $(go version)" +BAAS_DEPS_DIR="${WORK_PATH}/baas_dep_binaries" # Create the /baas_dep_binaries/ directory [[ -d "${BAAS_DEPS_DIR}" ]] || mkdir -p "${BAAS_DEPS_DIR}" -export PATH="${BAAS_DEPS_DIR}":${PATH} +pathadd "${BAAS_DEPS_DIR}" # Download jq (used for parsing json files) if it's not found if [[ ! -x "${BAAS_DEPS_DIR}/jq" ]]; then pushd "${BAAS_DEPS_DIR}" > /dev/null - which jq || (${CURL} -LsS "${JQ_DOWNLOAD_URL}" > jq && chmod +x jq) + which jq > /dev/null || (${CURL} -LsS "${JQ_DOWNLOAD_URL}" > jq && chmod +x jq) popd > /dev/null # baas_dep_binaries fi +echo "jq version: $(jq --version)" # Fix incompatible github path that was changed in a BAAS dependency git config --global url."git@github.com:".insteadOf "https://github.com/" +#export GOPRIVATE="github.com/10gen/*" # If a baas branch or commit version was not provided, retrieve the latest release version if [[ -z "${BAAS_VERSION}" ]]; then @@ -296,7 +343,7 @@ fi # Clone the baas repo and check out the specified version if [[ ! -d "${BAAS_DIR}/.git" ]]; then - git clone git@github.com:10gen/baas.git + git clone git@github.com:10gen/baas.git "${BAAS_DIR}" pushd "${BAAS_DIR}" > /dev/null else pushd "${BAAS_DIR}" > /dev/null @@ -308,46 +355,72 @@ git checkout "${BAAS_VERSION}" echo "Using baas commit: $(git rev-parse HEAD)" popd > /dev/null # baas +setup_baas_dependencies "${BAAS_DIR}" + +echo "Installing node and go to build baas and its dependencies" + +NODE_BINARIES_DIR="${WORK_PATH}/node_binaries" + +# Create the /node_binaries/ directory +[[ -d "${NODE_BINARIES_DIR}" ]] || mkdir -p "${NODE_BINARIES_DIR}" +# Download node if it's not found +if [[ ! -x "${NODE_BINARIES_DIR}/bin/node" ]]; then + pushd "${NODE_BINARIES_DIR}" > /dev/null + ${CURL} -LsS "${NODE_URL}" | tar -xz --strip-components=1 + popd > /dev/null # node_binaries +fi +pathadd "${NODE_BINARIES_DIR}/bin" +echo "Node version: $(node --version)" + +export GOROOT="${WORK_PATH}/go" + +# Download go if it's not found and set up the GOROOT for building/running baas +[[ -x "${GOROOT}/bin/go" ]] || (${CURL} -sL "${GOLANG_URL}" | tar -xz) +pathadd "${GOROOT}/bin" +# Write the GOROOT to a file after the download completes so the baas proxy +# can use the same path. +echo "${GOROOT}" > "${GO_ROOT_FILE}" +echo "Go version: $(go version)" + +DYLIB_DIR="${BAAS_DIR}/etc/dylib" +DYLIB_LIB_DIR="${DYLIB_DIR}/lib" + # Copy or download and extract the baas support archive if it's not found if [[ ! -d "${DYLIB_DIR}" ]]; then - if [[ -n "${STITCH_SUPPORT_LIB_PATH}" ]]; then - echo "Copying baas support library from ${STITCH_SUPPORT_LIB_PATH}" - mkdir -p "${DYLIB_DIR}" - cp -rav "${STITCH_SUPPORT_LIB_PATH}"/* "${DYLIB_DIR}" - else - echo "Downloading baas support library" - mkdir -p "${DYLIB_DIR}" - pushd "${DYLIB_DIR}" > /dev/null - ${CURL} -LsS "${STITCH_SUPPORT_LIB_URL}" | tar -xz --strip-components=1 - popd > /dev/null # baas/etc/dylib - fi + echo "Downloading baas support library" + mkdir -p "${DYLIB_DIR}" + pushd "${DYLIB_DIR}" > /dev/null + ${CURL} -LsS "${STITCH_SUPPORT_LIB_URL}" | tar -xz --strip-components=1 + popd > /dev/null # baas/etc/dylib fi + export LD_LIBRARY_PATH="${DYLIB_LIB_DIR}" export DYLD_LIBRARY_PATH="${DYLIB_LIB_DIR}" +LIBMONGO_DIR="${BAAS_DIR}/etc/libmongo" + +# Create the libmongo/ directory +[[ -d "${LIBMONGO_DIR}" ]] || mkdir -p "${LIBMONGO_DIR}" +pathadd "${LIBMONGO_DIR}" + # Copy or download the assisted agg library as libmongo.so (for Linux) if it's not found -LIBMONGO_LIB="${BAAS_DEPS_DIR}/libmongo.so" -if [[ ! -x "${LIBMONGO_LIB}" ]]; then - if [[ -n "${STITCH_ASSISTED_AGG_LIB_PATH}" ]]; then - echo "Copying assisted agg library from ${STITCH_ASSISTED_AGG_LIB_PATH}" - cp -rav "${STITCH_ASSISTED_AGG_LIB_PATH}" "${LIBMONGO_LIB}" - chmod 755 "${LIBMONGO_LIB}" - elif [[ -n "${STITCH_ASSISTED_AGG_LIB_URL}" ]]; then - echo "Downloading assisted agg library (libmongo.so)" - pushd "${BAAS_DEPS_DIR}" > /dev/null - ${CURL} -LsS "${STITCH_ASSISTED_AGG_LIB_URL}" > libmongo.so - chmod 755 libmongo.so - popd > /dev/null # baas_dep_binaries - fi +LIBMONGO_LIB="${LIBMONGO_DIR}/libmongo.so" +if [[ ! -x "${LIBMONGO_LIB}" && -n "${LIBMONGO_URL}" ]]; then + echo "Downloading assisted agg library (libmongo.so)" + pushd "${LIBMONGO_DIR}" > /dev/null + ${CURL} -LsS "${LIBMONGO_URL}" > "${LIBMONGO_LIB}" + chmod 755 "${LIBMONGO_LIB}" + popd > /dev/null # etc/libmongo fi # Download the assisted agg library as assisted_agg (for MacOS) if it's not found -if [[ ! -x "${BAAS_DEPS_DIR}/assisted_agg" && -n "${STITCH_ASSISTED_AGG_URL}" ]]; then +ASSISTED_AGG_LIB="${LIBMONGO_DIR}/assisted_agg" +if [[ ! -x "${ASSISTED_AGG_LIB}" && -n "${ASSISTED_AGG_URL}" ]]; then echo "Downloading assisted agg binary (assisted_agg)" - pushd "${BAAS_DEPS_DIR}" > /dev/null - ${CURL} -LsS "${STITCH_ASSISTED_AGG_URL}" > assisted_agg - chmod 755 assisted_agg - popd > /dev/null # baas_dep_binaries + pushd "${LIBMONGO_DIR}" > /dev/null + ${CURL} -LsS "${ASSISTED_AGG_URL}" > "${ASSISTED_AGG_LIB}" + chmod 755 "${ASSISTED_AGG_LIB}" + popd > /dev/null # etc/libmongo fi # Download yarn if it's not found @@ -355,13 +428,15 @@ YARN="${WORK_PATH}/yarn/bin/yarn" if [[ ! -x "${YARN}" ]]; then echo "Getting yarn" mkdir -p yarn && pushd yarn > /dev/null - ${CURL} -LsS https://yarnpkg.com/latest.tar.gz | tar -xz --strip-components=1 + ${CURL} -LsS "https://yarnpkg.com/latest.tar.gz" | tar -xz --strip-components=1 popd > /dev/null # yarn mkdir "${WORK_PATH}/yarn_cache" fi # Use yarn to build the transpiler for the baas server +TRANSPILER_DIR="${BAAS_DIR}/etc/transpiler" BAAS_TRANSPILER="${BAAS_DEPS_DIR}/transpiler" + if [[ ! -x "${BAAS_TRANSPILER}" ]]; then echo "Building transpiler" pushd "${TRANSPILER_DIR}" > /dev/null @@ -371,6 +446,8 @@ if [[ ! -x "${BAAS_TRANSPILER}" ]]; then ln -s "${TRANSPILER_DIR}/bin/transpiler" "${BAAS_TRANSPILER}" fi +MONGO_BINARIES_DIR="${WORK_PATH}/mongodb-binaries" + # Download mongod (daemon) and mongosh (shell) binaries if [ ! -x "${MONGO_BINARIES_DIR}/bin/mongod" ]; then echo "Downloading mongodb" @@ -379,18 +456,23 @@ if [ ! -x "${MONGO_BINARIES_DIR}/bin/mongod" ]; then tar -xzf mongodb-binaries.tgz rm mongodb-binaries.tgz mv mongodb* mongodb-binaries - chmod +x "${MONGO_BINARIES_DIR}/bin"/* fi - -if [[ -n "${MONGOSH_DOWNLOAD_URL}" ]] && [[ ! -x "${MONGO_BINARIES_DIR}/bin/mongosh" ]]; then - echo "Downloading mongosh" - ${CURL} -sLS "${MONGOSH_DOWNLOAD_URL}" --output mongosh-binaries.zip - unzip -jnqq mongosh-binaries.zip '*/bin/*' -d "${MONGO_BINARIES_DIR}/bin/" - rm mongosh-binaries.zip +echo "mongod version: $("${MONGO_BINARIES_DIR}/bin/mongod" --version --quiet | sed 1q)" + +if [[ -n "${MONGOSH_DOWNLOAD_URL}" ]]; then + if [[ ! -x "${MONGO_BINARIES_DIR}/bin/mongosh" ]]; then + echo "Downloading mongosh" + ${CURL} -sLS "${MONGOSH_DOWNLOAD_URL}" --output mongosh-binaries.zip + unzip -jnqq mongosh-binaries.zip '*/bin/*' -d "${MONGO_BINARIES_DIR}/bin/" + rm mongosh-binaries.zip + MONGOSH="mongosh" + fi +else + # Use the mongo shell provided with mongod + MONGOSH="mongo" fi - -[[ -n "${MONGOSH_DOWNLOAD_URL}" ]] && MONGOSH="mongosh" || MONGOSH="mongo" - +chmod +x "${MONGO_BINARIES_DIR}/bin"/* +echo "${MONGOSH} version: $("${MONGO_BINARIES_DIR}/bin/${MONGOSH}" --version)" # Start mongod on port 26000 and listening on all network interfaces echo "Starting mongodb" @@ -398,6 +480,12 @@ echo "Starting mongodb" # Increase the maximum number of open file descriptors (needed by mongod) ulimit -n 32000 +MONGODB_PATH="${WORK_PATH}/mongodb-dbpath" +MONGOD_LOG="${MONGODB_PATH}/mongod.log" + +# Delete the mongod working directory if it exists from a previous run +[[ -d "${MONGODB_PATH}" ]] && rm -rf "${MONGODB_PATH}" + # The mongod working files will be stored in the /mongodb_dbpath directory mkdir -p "${MONGODB_PATH}" @@ -410,7 +498,6 @@ mkdir -p "${MONGODB_PATH}" --dbpath "${MONGODB_PATH}/" \ --pidfilepath "${MONGOD_PID_FILE}" & - # Wait for mongod to start (up to 40 secs) while attempting to initialize the replica set echo "Initializing replica set" @@ -423,11 +510,11 @@ do ((++WAIT_COUNTER)) if [[ -z "$(pgrep mongod)" ]]; then secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) - echo "Mongodb process has terminated after ${secs_spent_waiting} seconds" + echo "Error: mongodb process has terminated after ${secs_spent_waiting} seconds" exit 1 elif [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]]; then secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) - echo "Timed out after waiting ${secs_spent_waiting} seconds for mongod to start" + echo "Error: timed out after waiting ${secs_spent_waiting} seconds for mongod to start" exit 1 fi @@ -465,12 +552,7 @@ echo "Starting baas app server" --configFile=etc/configs/test_config.json --configFile=etc/configs/test_rcore_config.json > "${BAAS_SERVER_LOG}" 2>&1 & echo $! > "${BAAS_PID_FILE}" -WAIT_BAAS_OPTS=() -if [[ -n "${VERBOSE}" ]]; then - WAIT_BAAS_OPTS=("-v") -fi - -"${BASE_PATH}/wait_for_baas.sh" "${WAIT_BAAS_OPTS[@]}" -w "${WORK_PATH}" +"${BASE_PATH}/wait_for_baas.sh" -w "${WORK_PATH}" # Create the admin user and set up the allowed roles echo "Adding roles to admin user" diff --git a/evergreen/proxy-network-faults.toxics b/evergreen/proxy-network-faults.toxics new file mode 100644 index 00000000000..8168089606c --- /dev/null +++ b/evergreen/proxy-network-faults.toxics @@ -0,0 +1,4 @@ +{"name": "limit-data-upstream","type": "limit_data","stream": "upstream","toxicity": 0.6,"attributes": {"bytes": %RANDOM1%}} +{"name": "limit-data-downstream","type": "limit_data","stream": "downstream","toxicity": 0.6,"attributes": {"bytes": %RANDOM2%}} +{"name": "slow-close-upstream","type": "slow_close","stream": "upstream","toxicity": 0.6,"attributes": {"delay": %RANDOM3%}} +{"name": "reset-peer-upstream","type": "reset_peer","stream": "upstream","toxicity": 0.6,"attributes": {"timeout": %RANDOM4%}} diff --git a/evergreen/proxy-nonideal-transfer.toxics b/evergreen/proxy-nonideal-transfer.toxics new file mode 100644 index 00000000000..c0ff2d4b9b1 --- /dev/null +++ b/evergreen/proxy-nonideal-transfer.toxics @@ -0,0 +1,5 @@ +{"name": "latency-upstream","type": "latency","stream": "upstream","toxicity": 0.5,"attributes": {"latency": 250,"jitter": 250}} +{"name": "latency-downstream","type": "latency","stream": "downstream","toxicity": 0.5,"attributes": {"latency": 0,"jitter": 250}} +{"name": "bandwidth-upstream","type": "bandwidth","stream": "upstream","toxicity": 0.5,"attributes": {"rate": %RANDOM1%}} +{"name": "bandwidth-downstream","type": "bandwidth","stream": "downstream","toxicity": 0.5,"attributes": {"rate": %RANDOM2%}} +{"name": "slicer-downstream","type": "slicer","stream": "downstream","toxicity": 0.5,"attributes": {"average_size": 500,"size_variation": 250,"delay": 10000}} diff --git a/evergreen/setup_baas_host.sh b/evergreen/setup_baas_host.sh index 5de0e6397d6..e1d850b5e27 100755 --- a/evergreen/setup_baas_host.sh +++ b/evergreen/setup_baas_host.sh @@ -2,38 +2,55 @@ # The script to be run on the ubuntu host that will run baas for the evergreen windows tests # # Usage: -# ./evergreen/setup_baas_host.sh [-f FILE] [-b BRANCH] [-v] [-h] +# ./evergreen/setup_baas_host.sh [-b BRANCH] [-d PATH] [-t PORT] [-v] [-h] HOST_VARS # set -o errexit +set -o errtrace set -o pipefail -trap 'catch $? ${LINENO}' EXIT +trap 'catch $? ${LINENO}' ERR +trap "exit" INT TERM + +# Set up catch function that runs when an error occurs function catch() { - if [ "$1" != "0" ]; then - echo "Error $1 occurred while starting remote baas at line $2" - fi + # Usage: catch EXIT_CODE LINE_NUM + echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while starting remote baas" } function usage() { - echo "Usage: setup_baas_host.sh [-b BRANCH] [-v] [-h] HOST_VARS" - echo -e "\tHOST_VARS\t\tPath to baas host vars script file" + # Usage: usage [EXIT_CODE] + echo "Usage: setup_baas_host.sh [-b BRANCH] [-d PATH] [-t PORT] [-v] [-h] HOST_VARS" + echo -e "\tHOST_VARS\tPath to baas host vars script file" echo "Options:" echo -e "\t-b BRANCH\tOptional branch or git spec of baas to checkout/build" + echo -e "\t-d PATH\t\tSkip setting up the data device and use alternate data path" echo -e "\t-v\t\tEnable verbose script debugging" echo -e "\t-h\t\tShow this usage summary and exit" + echo "ToxiProxy Options:" + echo -e "\t-t PORT\t\tEnable Toxiproxy support (proxy between baas on :9090 and PORT)" # Default to 0 if exit code not provided exit "${1:0}" } BAAS_BRANCH= +OPT_DATA_DIR= +PROXY_PORT= VERBOSE= -while getopts "b:vh" opt; do +while getopts "b:d:t:vh" opt; do case "${opt}" in b) BAAS_BRANCH="${OPTARG}";; + d) if [[ -z "${OPTARG}" ]]; then + echo "Error: Alternate data directory was empty" + usage 1 + fi; OPT_DATA_DIR="${OPTARG}";; + t) if [[ -z "${OPTARG}" ]]; then + echo "Error: Baas proxy port was empty"; + usage 1 + fi; PROXY_PORT="${OPTARG}";; v) VERBOSE="yes";; h) usage 0;; *) usage 1;; @@ -41,40 +58,57 @@ while getopts "b:vh" opt; do done shift $((OPTIND - 1)) + +if [[ $# -lt 1 ]]; then + echo "Error: Baas host vars script not provided" + usage 1 +fi + BAAS_HOST_VARS="${1}"; shift; if [[ -z "${BAAS_HOST_VARS}" ]]; then - echo "Baas host vars script not provided" + echo "Error: Baas host vars script value was empty" usage 1 elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then - echo "Baas host vars script not found: ${BAAS_HOST_VARS}" + echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}" usage 1 fi # shellcheck disable=SC1090 source "${BAAS_HOST_VARS}" +if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then + echo "Error: AWS_ACCESS_KEY_ID was not provided by baas host vars script" + exit 1 +fi + +if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then + echo "Error: AWS_SECRET_ACCESS_KEY was not provided by baas host vars script" + exit 1 +fi + if [[ -z "${GITHUB_KNOWN_HOSTS}" ]]; then # Use a default if not defined, but this may become outdated one day... GITHUB_KNOWN_HOSTS="github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" - echo "Info: GITHUB_KNOWN_HOSTS not defined in baas host vars script" + echo "Info: GITHUB_KNOWN_HOSTS not defined in baas host vars script - using default" +fi +KNOWN_HOSTS_FILE="${HOME}/.ssh/known_hosts" +if [[ -f "${KNOWN_HOSTS_FILE}" ]] && grep "${GITHUB_KNOWN_HOSTS}" < "${KNOWN_HOSTS_FILE}"; then + echo "Github known hosts entry found - skipping known_hosts update" +else + echo "${GITHUB_KNOWN_HOSTS}" | tee -a "${KNOWN_HOSTS_FILE}" fi -echo "${GITHUB_KNOWN_HOSTS}" | tee -a "${HOME}/.ssh/known_hosts" - -DATA_DIR=/data -DATA_TEMP_DIR="${DATA_DIR}/tmp" -BAAS_REMOTE_DIR="${DATA_DIR}/baas-remote" -BAAS_WORK_DIR="${BAAS_REMOTE_DIR}/baas-work-dir" -function setup_data_dir() +function init_data_device() { + #Usage: init_data_device data_device= # Find /data ebs device to be mounted devices=$(sudo lsblk | grep disk | awk '{print $1}') for device in ${devices}; do is_data=$(sudo file -s "/dev/${device}" | awk '{print $2}') - if [ "${is_data}" == "data" ]; then + if [[ "${is_data}" == "data" ]]; then data_device="/dev/${device}" fi done @@ -97,7 +131,12 @@ function setup_data_dir() fi sudo chmod 777 "${DATA_DIR}" +} +function setup_data_dir() +{ + # Usage: setup_data_dir + # Data directory is expected to be set in DATA_DIR variable # Delete /data/baas-remote/ dir if is already exists [[ -d "${BAAS_REMOTE_DIR}" ]] && sudo rm -rf "${BAAS_REMOTE_DIR}" @@ -110,13 +149,70 @@ function setup_data_dir() mkdir -p "${BAAS_WORK_DIR}" chmod -R 755 "${BAAS_WORK_DIR}" - # Set up the temp directory - mkdir -p "${DATA_TEMP_DIR}" - chmod 1777 "${DATA_TEMP_DIR}" - echo "original TMP=${TMPDIR}" + # Set up the temp directory - it may already exist on evergreen spawn hosts + if [[ -d "${DATA_TEMP_DIR}" ]]; then + sudo chmod 1777 "${DATA_TEMP_DIR}" + else + mkdir -p "${DATA_TEMP_DIR}" + chmod 1777 "${DATA_TEMP_DIR}" + fi export TMPDIR="${DATA_TEMP_DIR}" } +function on_exit() +{ + # Usage: on_exit + baas_pid= + proxy_pid= + if [[ -f "${PROXY_PID_FILE}" ]]; then + proxy_pid="$(< "${PROXY_PID_FILE}")" + fi + + if [[ -f "${SERVER_PID_FILE}" ]]; then + baas_pid="$(< "${SERVER_PID_FILE}")" + fi + + if [[ -n "${proxy_pid}" ]]; then + echo "Stopping baas proxy ${proxy_pid}" + kill "${proxy_pid}" || true + rm -f "${PROXY_PID_FILE}" || true + fi + + if [[ -n "${baas_pid}" ]]; then + echo "Stopping baas server ${baas_pid}" + kill "${baas_pid}" || true + rm -f "${SERVER_PID_FILE}" || true + fi + + echo "Waiting for processes to exit" + wait +} + +function start_baas_proxy() +{ + # Usage: start_baas_proxy PORT + listen_port="${1}" + # Delete the toxiproxy working directory if it currently exists + if [[ -n "${BAAS_PROXY_DIR}" && -d "${BAAS_PROXY_DIR}" ]]; then + rm -rf "${BAAS_PROXY_DIR}" + fi + + if [[ -f "${HOME}/setup_baas_proxy.sh" ]]; then + cp "${HOME}/setup_baas_proxy.sh" evergreen/ + fi + + proxy_options=("-w" "${BAAS_PROXY_DIR}" "-s" "${BAAS_WORK_DIR}" "-p" "${listen_port}") + if [[ -n "${VERBOSE}" ]]; then + proxy_options=("-v") + fi + + # Pass the baas work directory to the toxiproxy script for the go executable + echo "Staring baas proxy with listen port :${listen_port}" + ./evergreen/setup_baas_proxy.sh "${proxy_options[@]}" 2>&1 & + echo $! > "${PROXY_PID_FILE}" +} + + # Wait until after the BAAS_HOST_VARS file is loaded to enable verbose tracing if [[ -n "${VERBOSE}" ]]; then set -o verbose @@ -125,25 +221,59 @@ fi sudo chmod 600 "${HOME}/.ssh"/* +# Should an alternate data directory location be used? If so, don't init the data device +if [[ -z "${OPT_DATA_DIR}" ]]; then + DATA_DIR=/data + init_data_device +else + DATA_DIR="${OPT_DATA_DIR}" +fi + +DATA_TEMP_DIR="${DATA_DIR}/tmp" +BAAS_REMOTE_DIR="${DATA_DIR}/baas-remote" +BAAS_WORK_DIR="${BAAS_REMOTE_DIR}/baas-work-dir" +SERVER_PID_FILE="${BAAS_REMOTE_DIR}/baas-server.pid" +BAAS_STOPPED_FILE="${BAAS_WORK_DIR}/baas_stopped" + +BAAS_PROXY_DIR="${BAAS_REMOTE_DIR}/baas-proxy-dir" +PROXY_PID_FILE="${BAAS_REMOTE_DIR}/baas-proxy.pid" +PROXY_STOPPED_FILE="${BAAS_PROXY_DIR}/baas_proxy_stopped" + setup_data_dir pushd "${BAAS_REMOTE_DIR}" > /dev/null -if [[ -d "${HOME}/evergreen/" ]]; then - cp -R "${HOME}/evergreen/" ./evergreen/ +if [[ -d "${HOME}/remote-baas/evergreen/" ]]; then + cp -R "${HOME}/remote-baas/evergreen/" ./evergreen/ else - echo "evergreen/ directory not found in ${HOME}" + echo "remote-baas/evergreen/ directory not found in ${HOME}" exit 1 fi -INSTALL_BAAS_OPTS=() +# Set up the cleanup function that runs at exit and stops baas server and proxy (if run) +trap 'on_exit' EXIT + +BAAS_OPTIONS=() if [[ -n "${BAAS_BRANCH}" ]]; then - INSTALL_BAAS_OPTS=("-b" "${BAAS_BRANCH}") + BAAS_OPTIONS=("-b" "${BAAS_BRANCH}") fi if [[ -n "${VERBOSE}" ]]; then - INSTALL_BAAS_OPTS+=("-v") + BAAS_OPTIONS+=("-v") fi -./evergreen/install_baas.sh -w "${BAAS_WORK_DIR}" "${INSTALL_BAAS_OPTS[@]}" 2>&1 +echo "Staring baas server..." +./evergreen/install_baas.sh "${BAAS_OPTIONS[@]}" -w "${BAAS_WORK_DIR}" 2>&1 & +echo $! > "${SERVER_PID_FILE}" + +if [[ -n "${PROXY_PORT}" ]]; then + start_baas_proxy "${PROXY_PORT}" +fi + +# Turn off verbose logging since it's so noisy +set +o verbose +set +o xtrace +until [[ -f "${BAAS_STOPPED_FILE}" || -f "${PROXY_STOPPED_FILE}" ]]; do + sleep 1 +done popd > /dev/null # /data/baas-remote diff --git a/evergreen/setup_baas_host_local.sh b/evergreen/setup_baas_host_local.sh index c25c84bce94..4c49fcbf9f7 100755 --- a/evergreen/setup_baas_host_local.sh +++ b/evergreen/setup_baas_host_local.sh @@ -2,43 +2,55 @@ # The script to be run on the ubuntu host that will run baas for the evergreen windows tests # # Usage: -# ./evergreen/setup_baas_host_local.sh -f FILE [-i FILE] [-w PATH] [-u USER] [-d PATH] [-b BRANCH] [-v] [-h] +# ./evergreen/setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] HOST_VARS SSH_KEY # set -o errexit +set -o errtrace set -o pipefail +EVERGREEN_PATH=./evergreen +BAAS_WORK_PATH=./baas-work-dir +BAAS_HOST_NAME= +BAAS_USER=ubuntu +BAAS_BRANCH= +VERBOSE= +BAAS_PROXY= +DIRECT_PORT=9098 +LISTEN_PORT=9092 +CONFIG_PORT=8474 +BAAS_PORT=9090 + function usage() { - echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-d PATH] [-b BRANCH] [-v] [-h] HOST_VARS SSH_KEY" - echo -e "\tHOST_VARS\t\tPath to baas host vars script file" + echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] HOST_VARS SSH_KEY" + echo -e "\tHOST_VARS\tPath to baas host vars script file" echo -e "\tSSH_KEY\t\tPath to baas host private key file" echo "Options:" - echo -e "\t-w PATH\t\tPath to local baas server working directory (default ./baas-work-dir)" - echo -e "\t-u USER\t\tUsername to connect to baas host (default ubuntu)" - echo -e "\t-d PATH\t\tPath on baas host to transfer files (default /home/)" + echo -e "\t-w PATH\t\tPath to local baas server working directory (default ${BAAS_WORK_PATH})" + echo -e "\t-u USER\t\tUsername to connect to baas host (default ${BAAS_USER})" echo -e "\t-b BRANCH\tOptional branch or git spec of baas to checkout/build" echo -e "\t-v\t\tEnable verbose script debugging" echo -e "\t-h\t\tShow this usage summary and exit" + echo "Baas Proxy Options:" + echo -e "\t-t\t\tEnable baas proxy support (proxy between baas on :9090 and listen port)" + echo -e "\t-d PORT\t\tPort for direct connection to baas - skips proxy (default ${DIRECT_PORT})" + echo -e "\t-l PORT\t\tBaas proxy listen port on remote host (default ${LISTEN_PORT})" + echo -e "\t-c PORT\t\tLocal configuration port for proxy HTTP API (default ${CONFIG_PORT})" echo "Note: This script must be run from a cloned realm-core/ repository directory." # Default to 0 if exit code not provided exit "${1:0}" } -EVERGREEN_PATH=./evergreen -BAAS_WORK_PATH=./baas-work-dir -BAAS_HOST_NAME= -BAAS_USER=ubuntu -BAAS_BRANCH= -FILE_DEST_DIR= -VERBOSE= - -while getopts "w:u:d:b:vh" opt; do +while getopts "w:u:b:ta:d:l:c:vh" opt; do case "${opt}" in w) BAAS_WORK_PATH="${OPTARG}";; u) BAAS_USER="${OPTARG}";; - d) FILE_DEST_DIR="${OPTARG}";; b) BAAS_BRANCH="${OPTARG}";; + t) BAAS_PROXY="yes";; + d) DIRECT_PORT="${OPTARG}";; + l) LISTEN_PORT="${OPTARG}";; + c) CONFIG_PORT="${OPTARG}";; v) VERBOSE="yes";; h) usage 0;; *) usage 1;; @@ -46,37 +58,105 @@ while getopts "w:u:d:b:vh" opt; do done shift $((OPTIND - 1)) + +if [[ $# -lt 1 ]]; then + echo "Error: Baas host vars script not provided" + usage 1 +fi BAAS_HOST_VARS="${1}"; shift; -BAAS_HOST_KEY="${1}"; shift; if [[ -z "${BAAS_HOST_VARS}" ]]; then - echo "Baas host vars script not provided" + echo "Error: Baas host vars script value was empty" usage 1 elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then - echo "Baas host vars script not found: ${BAAS_HOST_VARS}" + echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}" usage 1 fi +if [[ $# -lt 1 ]]; then + echo "Error: Baas host private key not provided" + usage 1 +fi +BAAS_HOST_KEY="${1}"; shift; + if [[ -z "${BAAS_HOST_KEY}" ]]; then - echo "Baas host private key not provided" + echo "Error: Baas host private key value was empty" usage 1 elif [[ ! -f "${BAAS_HOST_KEY}" ]]; then - echo "Baas host private key not found: ${BAAS_HOST_KEY}" + echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}" usage 1 fi -trap 'catch $? ${LINENO}' EXIT +if [[ "${BAAS_USER}" = "root" ]]; then + FILE_DEST_DIR="/root/remote-baas" +else + FILE_DEST_DIR="/home/${BAAS_USER}/remote-baas" +fi +EVERGREEN_DEST_DIR="${FILE_DEST_DIR}/evergreen" + +function check_port() +{ + # Usage check_port PORT + port_num="${1}" + if [[ -n "${port_num}" && ${port_num} -gt 0 && ${port_num} -lt 65536 ]]; then + return 0 + fi + return 1 +} + +function check_port_in_use() +{ + # Usage: check_port_in_use PORT PORT_NAME + port_num="${1}" + port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) + if [[ -n "${port_check}" ]]; then + echo "Error: ${2} port (${port_num}) is already in use" + echo -e "${port_check}" + exit 1 + fi +} + +# Check the local baas port availability +check_port_in_use "${BAAS_PORT}" "Local baas server" + +# Check the port values and local ports in use for baas proxy +if [[ -n "${BAAS_PROXY}" ]]; then + if ! check_port "${CONFIG_PORT}"; then + echo "Error: Baas proxy local HTTP API config port was invalid: '${CONFIG_PORT}'" + usage 1 + elif ! check_port "${LISTEN_PORT}"; then + echo "Error: Baas proxy listen port was invalid: '${LISTEN_PORT}'" + usage 1 + fi + check_port_in_use "${CONFIG_PORT}" "Local baas proxy config" + + if [[ -n "${DIRECT_PORT}" ]]; then + if ! check_port "${DIRECT_PORT}"; then + echo "Error: Baas direct connect port was invalid: '${DIRECT_PORT}'" + usage 1 + fi + check_port_in_use "${DIRECT_PORT}" "Local baas server direct connect" + fi +fi + +trap 'catch $? ${LINENO}' ERR +trap 'on_exit' INT TERM EXIT + +# Set up catch function that runs when an error occurs function catch() { - if [ "$1" != "0" ]; then - echo "Error $1 occurred while starting baas (local) at line $2" - fi - - if [[ -n "${BAAS_WORK_PATH}" ]]; then - # Create the baas_stopped file so wait_for_baas can exit early - [[ -d "${BAAS_WORK_PATH}" ]] || mkdir -p "${BAAS_WORK_PATH}" - touch "${BAAS_WORK_PATH}/baas_stopped" - fi + # Usage: catch EXIT_CODE LINE_NUM + echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while starting baas (local)" +} + +function on_exit() +{ + # Usage: on_exit + if [[ -n "${BAAS_WORK_PATH}" ]]; then + # Create the baas_stopped file so wait_for_baas can exit early + [[ -d "${BAAS_WORK_PATH}" ]] || mkdir -p "${BAAS_WORK_PATH}" + touch "${BAAS_WORK_PATH}/baas_stopped" + fi } # shellcheck disable=SC1090 @@ -93,15 +173,8 @@ if [[ -z "${BAAS_HOST_NAME}" ]]; then usage 1 fi -if [[ -z "${BAAS_HOST_KEY}" ]]; then - echo "Baas host private key not provided" - usage 1 -elif [[ ! -f "${BAAS_HOST_KEY}" ]]; then - echo "Baas host private key not found: ${BAAS_HOST_KEY}" - usage 1 -fi if [[ -z "${BAAS_USER}" ]]; then - echo "Baas host username not provided" + echo "Error: Baas host username was empty" usage 1 fi @@ -110,11 +183,6 @@ if [[ ! -d "${EVERGREEN_PATH}/" ]]; then exit 1 fi -if [[ -z "${FILE_DEST_DIR}" ]]; then - FILE_DEST_DIR="/home/${BAAS_USER}" -fi -EVERGREEN_DEST_DIR="${FILE_DEST_DIR}/evergreen" - SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")" ssh-agent > ssh_agent_commands.sh @@ -138,7 +206,7 @@ CONNECT_COUNT=2 # Check for remote connectivity - try to connect twice to verify server is "really" ready # The tests failed one time due to this ssh command passing, but the next scp command failed while [[ ${CONNECT_COUNT} -gt 0 ]]; do - until ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=10 "${SSH_USER}" "echo -n 'hello from '; hostname" ; do + until ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=10 "${SSH_USER}" "mkdir -p ${EVERGREEN_DEST_DIR} && echo -n 'hello from '; hostname" ; do if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]] ; then secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) echo "Timed out after waiting ${secs_spent_waiting} seconds for host ${BAAS_HOST_NAME} to start" @@ -158,19 +226,45 @@ echo "Transferring setup scripts to ${SSH_USER}:${FILE_DEST_DIR}" scp "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 "${BAAS_HOST_VARS}" "${SSH_USER}:${FILE_DEST_DIR}/" # Copy the entire evergreen/ directory from the working copy of realm-core to the remote host # This ensures the remote host the latest copy, esp when running evergreen patches +echo "Transferring evergreen scripts to ${SSH_USER}:${FILE_DEST_DIR}" scp -r "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 "${EVERGREEN_PATH}/" "${SSH_USER}:${FILE_DEST_DIR}/" -echo "Starting remote baas with branch/commit: '${BAAS_BRANCH}'" -SETUP_BAAS_OPTS=() -if [[ -n "${BAAS_BRANCH}" ]]; then - SETUP_BAAS_OPTS=("-b" "${BAAS_BRANCH}") -fi +BAAS_TUNNELS=() +SETUP_OPTIONS=() + if [[ -n "${VERBOSE}" ]]; then - SETUP_BAAS_OPTS+=("-v") + SETUP_OPTIONS+=("-v") fi +if [[ -n "${BAAS_PROXY}" ]]; then + # Add extra tunnel for baas proxy HTTP API config interface and direct connection to baas + BAAS_TUNNELS+=("-L" "${CONFIG_PORT}:127.0.0.1:8474") + if [[ -n "${DIRECT_PORT}" ]]; then + BAAS_TUNNELS+=("-L" "${DIRECT_PORT}:127.0.0.1:9090") + fi + # Enable baas proxy and use LISTEN_PORT as the proxy listen port + SETUP_OPTIONS+=("-t" "${LISTEN_PORT}") +else + # Force remote port to 9090 if baas proxy is not used - connect directly to baas + LISTEN_PORT=9090 +fi + +BAAS_TUNNELS+=("-L" "9090:127.0.0.1:${LISTEN_PORT}") + # Run the setup baas host script and provide the location of the baas host vars script -# Also sets up a forward tunnel for port 9090 through the ssh connection to the baas remote host -echo "Running setup script (with forward tunnel to 127.0.0.1:9090)" -ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 -L 9090:127.0.0.1:9090 "${SSH_USER}" \ - "${EVERGREEN_DEST_DIR}/setup_baas_host.sh" "${SETUP_BAAS_OPTS[@]}" "${FILE_DEST_DIR}/baas_host_vars.sh" +# Also sets up a forward tunnel for local port 9090 through the ssh connection to the baas remote host +# If baas proxy is enabled, a second forward tunnel is added for the HTTP API config interface +echo "Running setup script (with forward tunnel on :9090 to 127.0.0.1:${LISTEN_PORT})" +if [[ -n "${BAAS_BRANCH}" ]]; then + echo "- Starting remote baas with branch/commit: '${BAAS_BRANCH}'" + SETUP_OPTIONS+=("-b" "${BAAS_BRANCH}") +fi +if [[ -n "${BAAS_PROXY}" ]]; then + echo "- Baas proxy enabled - local HTTP API config port on :${CONFIG_PORT}" + if [[ -n "${DIRECT_PORT}" ]]; then + echo "- Baas direct connection on port :${DIRECT_PORT}" + fi +fi + +ssh -t "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 "${BAAS_TUNNELS[@]}" "${SSH_USER}" \ + "${EVERGREEN_DEST_DIR}/setup_baas_host.sh" "${SETUP_OPTIONS[@]}" "${FILE_DEST_DIR}/baas_host_vars.sh" diff --git a/evergreen/setup_baas_proxy.sh b/evergreen/setup_baas_proxy.sh new file mode 100755 index 00000000000..8a5d1ba4443 --- /dev/null +++ b/evergreen/setup_baas_proxy.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# The script to download, build and run toxiproxy as a proxy to the baas server +# for simulating network error conditions for testing. +# +# Usage: +# ./evergreen/setup_baas_proxy.sh -w PATH [-p PORT] [-s PATH] [-b BRANCH] [-d] [-v] [-h] +# + +set -o errexit +set -o errtrace +set -o pipefail + +trap 'catch $? ${LINENO}' ERR +trap "exit" INT TERM + +function catch() +{ + echo "Error $1 occurred while starting baas proxy at line $2" +} + +WORK_PATH= +BAAS_PATH= +TOXIPROXY_VERSION="v2.5.0" +LISTEN_PORT=9092 +BAAS_PORT=9090 +SKIP_BAAS_WAIT= +CONFIG_PORT=8474 + +function usage() +{ + echo "Usage: setup_baas_proxy.sh -w PATH [-p PORT] [-s PATH] [-b BRANCH] [-d] [-v] [-h]" + echo -e "\t-w PATH\t\tPath to baas proxy working directory" + echo "Options:" + echo -e "\t-p PORT\t\tListen port for proxy connected to baas (default: ${LISTEN_PORT})" + echo -e "\t-s PATH\t\tOptional path to baas server working directory (for go binary)" + echo -e "\t-b BRANCH\tOptional branch or git spec to checkout/build (default: ${TOXIPROXY_VERSION})" + echo -e "\t-d\t\tDon't wait for baas to start before starting proxy" + echo -e "\t-v\t\tEnable verbose script debugging" + echo -e "\t-h\t\tShow this usage summary and exit" + # Default to 0 if exit code not provided + exit "${1:0}" +} + +BASE_PATH="$(cd "$(dirname "$0")"; pwd)" + +# Allow path to CURL to be overloaded by an environment variable +CURL="${CURL:=$LAUNCHER curl}" + +while getopts "w:p:s:b:dvh" opt; do + case "${opt}" in + w) WORK_PATH="${OPTARG}";; + p) LISTEN_PORT="${OPTARG}";; + s) BAAS_PATH="${OPTARG}";; + b) TOXIPROXY_VERSION="${OPTARG}";; + d) SKIP_BAAS_WAIT="yes";; + v) set -o verbose; set -o xtrace;; + h) usage 0;; + *) usage 1;; + esac +done + +if [[ -z "${WORK_PATH}" ]]; then + echo "Baas proxy work path was not provided" + usage 1 +fi +if [[ -z "${LISTEN_PORT}" ]]; then + echo "Baas proxy remote port was empty" + usage 1 +fi + +function check_port_in_use() +{ + # Usage: check_port_in_use PORT PORT_NAME + port_num="${1}" + port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) + if [[ -n "${port_check}" ]]; then + echo "Error: ${2} port (${port_num}) is already in use" + echo -e "${port_check}" + exit 1 + fi +} + +# Check the baas proxy listen and Toxiproxy config port availability first +check_port_in_use "${LISTEN_PORT}" "baas proxy" +check_port_in_use "${CONFIG_PORT}" "Toxiproxy config" + +[[ -d "${WORK_PATH}" ]] || mkdir -p "${WORK_PATH}" +pushd "${WORK_PATH}" > /dev/null + +PROXY_CFG_FILE="${WORK_PATH}/baas_proxy.json" +PROXY_LOG="${WORK_PATH}/baas_proxy.log" +PROXY_PID_FILE="${WORK_PATH}/baas_proxy.pid" +PROXY_STOPPED_FILE="${WORK_PATH}/baas_proxy_stopped" +BAAS_STOPPED_FILE="${BAAS_PATH}/baas_stopped" + +# Remove some files from a previous run if they exist +if [[ -f "${CONFIG_FILE}" ]]; then + rm -f "${CONFIG_FILE}" +fi +if [[ -f "${PROXY_LOG}" ]]; then + rm -f "${PROXY_LOG}" +fi +if [[ -f "${PROXY_PID_FILE}" ]]; then + rm -f "${PROXY_PID_FILE}" +fi + +if [[ -f "${PROXY_STOPPED}" ]]; then + rm -f "${PROXY_STOPPED}" +fi + +# Set up the cleanup function that runs at exit and stops the toxiproxy server +trap 'on_exit' EXIT + +function on_exit() +{ + # Usage: on_exit + # Toxiproxy is being stopped (or never started), create a 'baas-proxy-stopped' file + touch "${PROXY_STOPPED_FILE}" || true + + proxy_pid= + if [[ -f "${PROXY_PID_FILE}" ]]; then + proxy_pid="$(< "${PROXY_PID_FILE}")" + fi + + if [[ -n "${proxy_pid}" ]]; then + echo "Stopping baas proxy ${proxy_pid}" + kill "${proxy_pid}" || true + echo "Waiting for baas proxy to stop" + wait + rm -f "${PROXY_PID_FILE}" || true + fi +} + +case $(uname -s) in + Darwin) + if [[ "$(uname -m)" == "arm64" ]]; then + export GOARCH=arm64 + GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.19.3.darwin-arm64.tar.gz" + # Go's scheduler is not BIG.little aware, and by default will spawn + # threads until they end up getting scheduled on efficiency cores, + # which is slower than just not using them. Limiting the threads to + # the number of performance cores results in them usually not + # running on efficiency cores. Checking the performance core count + # wasn't implemented until the first CPU with a performance core + # count other than 4 was released, so if it's unavailable it's 4. + GOMAXPROCS="$(sysctl -n hw.perflevel0.logicalcpu || echo 4)" + export GOMAXPROCS + else + export GOARCH=amd64 + GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.19.1.darwin-amd64.tar.gz" + fi + ;; + Linux) + GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.19.1.linux-amd64.tar.gz" + ;; +esac + +# Looking for go - first in the work path, then in the baas path (if provided), or +# download go into the work path +GOROOT= +# Was it found in the work path? +if [[ ! -x ${WORK_PATH}/go/bin/go ]]; then + # If the baas work path is set, check there first and wait + if [[ -n "${BAAS_PATH}" && -d "${BAAS_PATH}" ]]; then + WAIT_COUNTER=0 + RETRY_COUNT=10 + WAIT_START=$(date -u +'%s') + FOUND_GO="yes" + GO_ROOT_FILE="${BAAS_PATH}/go_root" + # Bass may be initializing at the same time, allow a bit of time for the two to sync + echo "Looking for go in baas work path for 20 secs in case both are starting concurrently" + until [[ -f "${GO_ROOT_FILE}" ]]; do + if [[ -n "${BAAS_STOPPED_FILE}" && -f "${BAAS_STOPPED_FILE}" ]]; then + echo "Error: Baas server failed to start (found baas_stopped file)" + exit 1 + fi + if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]]; then + FOUND_GO= + secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) + echo "Error: Stopped after waiting ${secs_spent_waiting} seconds for baas go to become available" + break + fi + ((++WAIT_COUNTER)) + sleep 2 + done + if [[ -n "${FOUND_GO}" ]]; then + GOROOT="$(cat "${GO_ROOT_FILE}")" + echo "Found go in baas working directory: ${GOROOT}" + export GOROOT + fi + fi + + # If GOROOT is not set, then baas path was nor provided or go was not found + if [[ -z "${GOROOT}" ]]; then + # Download go since it wasn't found in the working directory + if [[ -z "${GO_URL}" ]]; then + echo "Error: go url not defined for current OS architecture" + uname -a + exit 1 + fi + echo "Downloading go to baas proxy working directory" + ${CURL} -sL "${GO_URL}" | tar -xz + # Set up the GOROOT for building/running baas + export GOROOT="${WORK_PATH}/go" + fi +else + echo "Found go in baas proxy working directory" + # Set up the GOROOT for building/running baas + export GOROOT="${WORK_PATH}/go" +fi +export PATH="${GOROOT}/bin":${PATH} +echo "Go version: $(go version)" + +if [[ ! -d "toxiproxy" ]]; then + git clone git@github.com:Shopify/toxiproxy.git toxiproxy +fi + +# Clone the baas repo and check out the specified version +if [[ ! -d "toxiproxy/.git" ]]; then + git clone git@github.com:Shopify/toxiproxy.git toxiproxy + pushd toxiproxy > /dev/null +else + pushd toxiproxy > /dev/null + git fetch +fi + +echo "Checking out Toxiproxy version '${TOXIPROXY_VERSION}'" +git checkout "${TOXIPROXY_VERSION}" +echo "Using Toxiproxy commit: $(git rev-parse HEAD)" + +# Build toxiproxy +make build + +if [[ -z "${SKIP_BAAS_WAIT}" ]]; then + # Wait for baas to start before starting Toxiproxy + OPT_WAIT_BAAS=() + if [[ -n "${BAAS_PATH}" ]]; then + OPT_WAIT_BAAS=("-w" "{$BAAS_PATH}") + fi + + "${BASE_PATH}/wait_for_baas.sh" "${OPT_WAIT_BAAS[@]}" +fi + +cat >"${PROXY_CFG_FILE}" < 127.0.0.1:${BAAS_PORT}" +./dist/toxiproxy-server -config "${PROXY_CFG_FILE}" > "${PROXY_LOG}" 2>&1 & +echo $! > "${PROXY_PID_FILE}" + +echo "---------------------------------------------" +echo "Baas proxy ready" +echo "---------------------------------------------" +wait + +popd > /dev/null # toxiproxy +popd > /dev/null # / diff --git a/evergreen/wait_for_baas.sh b/evergreen/wait_for_baas.sh index f7f5605e1f1..508ec310eca 100755 --- a/evergreen/wait_for_baas.sh +++ b/evergreen/wait_for_baas.sh @@ -20,15 +20,15 @@ BAAS_STOPPED_FILE= RETRY_COUNT=120 BAAS_SERVER_LOG= STATUS_OUT= -VERBOSE= function usage() { echo "Usage: wait_for_baas.sh [-w PATH] [-p FILE] [-r COUNT] [-l FILE] [-s] [-v] [-h]" + echo "Options:" echo -e "\t-w PATH\t\tPath to baas server working directory" - echo -e "\t-p FILE\t\tPath to baas server pid file" + echo -e "\t-p FILE\t\tPath to baas server pid file (also set by -w option)" echo -e "\t-r COUNT\tNumber of attempts to check for baas server (default 120)" - echo -e "\t-l FILE\t\tPath to baas server log file" + echo -e "\t-l FILE\t\tPath to baas server log file (also set by -w option)" echo -e "\t-s\t\tDisplay a status for each attempt" echo -e "\t-v\t\tEnable verbose script debugging" echo -e "\t-h\t\tShow this usage summary and exit" @@ -52,7 +52,7 @@ while getopts "w:p:r:l:svh" opt; do r) RETRY_COUNT="${OPTARG}";; l) BAAS_SERVER_LOG="${OPTARG}";; s) STATUS_OUT="yes";; - v) VERBOSE="yes"; set -o verbose; set -o xtrace;; + v) set -o verbose; set -o xtrace;; h) usage 0;; *) usage 1;; esac @@ -61,30 +61,40 @@ done WAIT_COUNTER=0 WAIT_START=$(date -u +'%s') +function output_log_tail() +{ + if [[ -n "${BAAS_SERVER_LOG}" && -f "${BAAS_SERVER_LOG}" ]]; then + tail -n 10 "${BAAS_SERVER_LOG}" + fi +} + +echo "Waiting for baas server to start..." until $CURL --output /dev/null --head --fail http://localhost:9090 --silent ; do if [[ -n "${BAAS_STOPPED_FILE}" && -f "${BAAS_STOPPED_FILE}" ]]; then echo "Baas server failed to start (found baas_stopped file)" + output_log_tail exit 1 fi if [[ -n "${BAAS_PID_FILE}" && -f "${BAAS_PID_FILE}" ]]; then - pgrep -F "${BAAS_PID_FILE}" > /dev/null || (echo "Baas server $(< "${BAAS_PID_FILE}") is not running"; exit 1) + if ! pgrep -F "${BAAS_PID_FILE}" > /dev/null; then + echo "Baas server $(< "${BAAS_PID_FILE}") is no longer running" + output_log_tail + exit 1 + fi fi ((++WAIT_COUNTER)) + secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]]; then - echo "Timed out waiting for baas server to start" + echo "Timed out after ${secs_spent_waiting} secs waiting for baas server to start" + output_log_tail exit 1 fi if [[ -n "${STATUS_OUT}" ]]; then - secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) echo "Waiting for baas server to start... ${secs_spent_waiting} secs so far" fi - if [[ -n "${VERBOSE}" && -n "${BAAS_SERVER_LOG}" && -f "${BAAS_SERVER_LOG}" ]]; then - tail -n 5 "${BAAS_SERVER_LOG}" - fi - sleep 5 done diff --git a/test/object-store/CMakeLists.txt b/test/object-store/CMakeLists.txt index 241c5eb58da..d0a845cbabd 100644 --- a/test/object-store/CMakeLists.txt +++ b/test/object-store/CMakeLists.txt @@ -127,11 +127,19 @@ if(REALM_ENABLE_SYNC) message(FATAL_ERROR "REALM_MONGODB_ENDPOINT must be set when specifying REALM_ENABLE_AUTH_TESTS.") endif() + message(STATUS "Auth tests enabled: ${REALM_MONGODB_ENDPOINT}") target_compile_definitions(ObjectStoreTests PRIVATE REALM_ENABLE_AUTH_TESTS=1 REALM_MONGODB_ENDPOINT="${REALM_MONGODB_ENDPOINT}" ) + if(REALM_ADMIN_ENDPOINT) + message(STATUS "BAAS admin endpoint: ${REALM_ADMIN_ENDPOINT}") + target_compile_definitions(ObjectStoreTests PRIVATE + REALM_ADMIN_ENDPOINT="${REALM_ADMIN_ENDPOINT}" + ) + endif() + find_package(CURL REQUIRED) target_link_libraries(ObjectStoreTests CURL::libcurl) endif() @@ -152,6 +160,14 @@ if(REALM_TEST_LOGGING) endif() endif() +# Optional extra time to add to test timeout values +if(REALM_TEST_TIMEOUT_EXTRA) + target_compile_definitions(ObjectStoreTests PRIVATE + TEST_TIMEOUT_EXTRA=${REALM_TEST_TIMEOUT_EXTRA} + ) + message(STATUS "Test wait timeouts extended by ${REALM_TEST_TIMEOUT_EXTRA} seconds") +endif() + target_include_directories(ObjectStoreTests PRIVATE ${CATCH_INCLUDE_DIR} ${JSON_INCLUDE_DIR} diff --git a/test/object-store/audit.cpp b/test/object-store/audit.cpp index 12e1504c3d3..6e46d7b51be 100644 --- a/test/object-store/audit.cpp +++ b/test/object-store/audit.cpp @@ -1673,7 +1673,7 @@ TEST_CASE("audit integration tests", "[sync][pbs][audit][baas]") { const Schema no_audit_event_schema{ {"object", {{"_id", PropertyType::Int, Property::IsPrimary{true}}, {"value", PropertyType::Int}}}}; - auto app_create_config = default_app_config(get_base_url()); + auto app_create_config = default_app_config(); app_create_config.schema = schema; app_create_config.dev_mode_enabled = false; TestAppSession session = create_app(app_create_config); diff --git a/test/object-store/c_api/c_api.cpp b/test/object-store/c_api/c_api.cpp index 5ccb72beb4c..53a907e05d1 100644 --- a/test/object-store/c_api/c_api.cpp +++ b/test/object-store/c_api/c_api.cpp @@ -5304,9 +5304,7 @@ TEST_CASE("C API - client reset", "[sync][pbs][c_api][client reset][baas]") { }}, }; - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "c_api_client_reset_tests", schema); + auto server_app_config = minimal_app_config("c_api_client_reset_tests", schema); server_app_config.partition_key = partition_prop; TestAppSession test_app_session(create_app(server_app_config)); @@ -5544,7 +5542,7 @@ TEST_CASE("C API app: link_user integration w/c_api transport", "[sync][app][c_a // user_data will be deleted when user_data_free() is called auto user_data = new TestTransportUserData(); auto http_transport = realm_http_transport_new(send_request_to_server, user_data, user_data_free); - auto app_session = get_runtime_app_session(get_base_url()); + auto app_session = get_runtime_app_session(); TestAppSession session(app_session, *http_transport, DeleteApp{false}); realm_app app(session.app()); @@ -6039,7 +6037,8 @@ TEST_CASE("app: flx-sync basic tests", "[sync][flx][c_api][baas]") { { using namespace std::chrono_literals; std::unique_lock lock{m_mutex}; - bool completed_within_time_limit = m_cv.wait_for(lock, 5s, [this]() { + const auto delay = TEST_TIMEOUT_EXTRA > 0 ? std::chrono::seconds(5 + TEST_TIMEOUT_EXTRA) : 5s; + bool completed_within_time_limit = m_cv.wait_for(lock, delay, [this]() { return m_state == RLM_SYNC_SUBSCRIPTION_COMPLETE && m_userdata != nullptr; }); CHECK(completed_within_time_limit); diff --git a/test/object-store/main.cpp b/test/object-store/main.cpp index 540b38e095e..630eb4e4c8d 100644 --- a/test/object-store/main.cpp +++ b/test/object-store/main.cpp @@ -32,14 +32,18 @@ #include #include +#include #include #include #include #include +using namespace std::chrono; int main(int argc, const char** argv) { + auto t1 = steady_clock::now(); + realm::test_util::initialize_test_path(1, argv); Catch::ConfigData config; @@ -76,6 +80,10 @@ int main(int argc, const char** argv) } } +#ifdef TEST_TIMEOUT_EXTRA + std::cout << "Test wait timeouts extended by " << TEST_TIMEOUT_EXTRA << " seconds" << std::endl; +#endif + #if TEST_SCHEDULER_UV realm::util::Scheduler::set_default_factory([]() { return std::make_shared(); @@ -85,6 +93,10 @@ int main(int argc, const char** argv) Catch::Session session; session.useConfigData(config); int result = session.run(argc, argv); + + auto t2 = steady_clock::now(); + auto ms_int = duration_cast(t2 - t1); + std::cout << "Test time: " << (ms_int.count() / 1000.0) << "s" << std::endl << std::endl; return result < 0xff ? result : 0xff; } diff --git a/test/object-store/sync/app.cpp b/test/object-store/sync/app.cpp index c473f2440f0..87a7e8504df 100644 --- a/test/object-store/sync/app.cpp +++ b/test/object-store/sync/app.cpp @@ -982,7 +982,8 @@ TEST_CASE("app: remote mongo client", "[sync][app][mongo][baas]") { auto app = session.app(); auto remote_client = app->current_user()->mongo_client("BackingDB"); - auto db = remote_client.db(get_runtime_app_session("").config.mongo_dbname); + auto app_session = get_runtime_app_session(); + auto db = remote_client.db(app_session.config.mongo_dbname); auto dog_collection = db["Dog"]; auto cat_collection = db["Cat"]; auto person_collection = db["Person"]; @@ -1734,7 +1735,8 @@ TEST_CASE("app: token refresh", "[sync][app][token][baas]") { sync_user->update_access_token(ENCODE_FAKE_JWT("fake_access_token")); auto remote_client = app->current_user()->mongo_client("BackingDB"); - auto db = remote_client.db(get_runtime_app_session("").config.mongo_dbname); + auto app_session = get_runtime_app_session(); + auto db = remote_client.db(app_session.config.mongo_dbname); auto dog_collection = db["Dog"]; bson::BsonDocument dog_document{{"name", "fido"}, {"breed", "king charles"}}; @@ -1760,9 +1762,7 @@ TEST_CASE("app: token refresh", "[sync][app][token][baas]") { // MARK: - Sync Tests TEST_CASE("app: mixed lists with object links", "[sync][pbs][app][links][baas]") { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; - REQUIRE(!base_url.empty()); Schema schema{ {"TopLevel", @@ -1777,7 +1777,7 @@ TEST_CASE("app: mixed lists with object links", "[sync][pbs][app][links][baas]") }}, }; - auto server_app_config = minimal_app_config(base_url, "set_new_embedded_object", schema); + auto server_app_config = minimal_app_config("set_new_embedded_object", schema); auto app_session = create_app(server_app_config); auto partition = random_string(100); @@ -1836,9 +1836,7 @@ TEST_CASE("app: mixed lists with object links", "[sync][pbs][app][links][baas]") } TEST_CASE("app: roundtrip values", "[sync][pbs][app][baas]") { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; - REQUIRE(!base_url.empty()); Schema schema{ {"TopLevel", @@ -1848,7 +1846,7 @@ TEST_CASE("app: roundtrip values", "[sync][pbs][app][baas]") { }}, }; - auto server_app_config = minimal_app_config(base_url, "roundtrip_values", schema); + auto server_app_config = minimal_app_config("roundtrip_values", schema); auto app_session = create_app(server_app_config); auto partition = random_string(100); @@ -1885,9 +1883,7 @@ TEST_CASE("app: roundtrip values", "[sync][pbs][app][baas]") { } TEST_CASE("app: upgrade from local to synced realm", "[sync][pbs][app][upgrade][baas]") { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; - REQUIRE(!base_url.empty()); Schema schema{ {"origin", @@ -1930,7 +1926,7 @@ TEST_CASE("app: upgrade from local to synced realm", "[sync][pbs][app][upgrade][ } /* Create a synced realm and upload some data */ - auto server_app_config = minimal_app_config(base_url, "upgrade_from_local", schema); + auto server_app_config = minimal_app_config("upgrade_from_local", schema); TestAppSession test_session(create_app(server_app_config)); auto partition = random_string(100); auto user1 = test_session.app()->current_user(); @@ -1987,9 +1983,7 @@ TEST_CASE("app: upgrade from local to synced realm", "[sync][pbs][app][upgrade][ } TEST_CASE("app: set new embedded object", "[sync][pbs][app][baas]") { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; - REQUIRE(!base_url.empty()); Schema schema{ {"TopLevel", @@ -2017,7 +2011,7 @@ TEST_CASE("app: set new embedded object", "[sync][pbs][app][baas]") { }}, }; - auto server_app_config = minimal_app_config(base_url, "set_new_embedded_object", schema); + auto server_app_config = minimal_app_config("set_new_embedded_object", schema); TestAppSession test_session(create_app(server_app_config)); auto partition = random_string(100); @@ -2128,7 +2122,7 @@ TEST_CASE("app: make distributable client file", "[sync][pbs][app][baas]") { TestAppSession session; auto app = session.app(); - auto schema = default_app_config("").schema; + auto schema = get_default_schema(); SyncTestFile original_config(app, bson::Bson("foo"), schema); create_user_and_log_in(app); SyncTestFile target_config(app, bson::Bson("foo"), schema); @@ -2201,7 +2195,7 @@ TEST_CASE("app: make distributable client file", "[sync][pbs][app][baas]") { TEST_CASE("app: sync integration", "[sync][pbs][app][baas]") { auto logger = util::Logger::get_default_logger(); - const auto schema = default_app_config("").schema; + const auto schema = get_default_schema(); auto get_dogs = [](SharedRealm r) -> Results { wait_for_upload(*r, std::chrono::seconds(10)); @@ -2664,8 +2658,7 @@ TEST_CASE("app: sync integration", "[sync][pbs][app][baas]") { logger->trace("redirect_url: %1", redirect_url); }; - auto base_url = get_base_url(); - auto server_app_config = minimal_app_config(base_url, "websocket_redirect", schema); + auto server_app_config = minimal_app_config("websocket_redirect", schema); TestAppSession test_session(create_app(server_app_config), redir_transport, DeleteApp{true}, realm::ReconnectMode::normal, redir_provider); auto partition = random_string(100); @@ -3310,7 +3303,14 @@ TEST_CASE("app: sync integration", "[sync][pbs][app][baas]") { } r->commit_transaction(); - auto error = wait_for_future(std::move(pf.future), std::chrono::minutes(5)).get(); +#if defined(TEST_TIMEOUT_EXTRA) && TEST_TIMEOUT_EXTRA > 0 + // It may take 30 minutes to transfer 16MB at 10KB/s + auto delay = std::chrono::minutes(35); +#else + auto delay = std::chrono::minutes(5); +#endif + + auto error = wait_for_future(std::move(pf.future), delay).get(); REQUIRE(error.status == ErrorCodes::LimitExceeded); REQUIRE(error.status.reason() == "Sync websocket closed because the server received a message that was too large: " @@ -3494,9 +3494,7 @@ TEMPLATE_TEST_CASE("app: collections of links integration", "[sync][pbs][app][co cf::ListOfMixedLinks, cf::SetOfObjects, cf::SetOfMixedLinks, cf::DictionaryOfObjects, cf::DictionaryOfMixedLinks) { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; - REQUIRE(!base_url.empty()); const auto partition = random_string(100); TestType test_type("collection", "dest"); Schema schema = {{"source", @@ -3508,7 +3506,7 @@ TEMPLATE_TEST_CASE("app: collections of links integration", "[sync][pbs][app][co {valid_pk_name, PropertyType::Int | PropertyType::Nullable, true}, {"realm_id", PropertyType::String | PropertyType::Nullable}, }}}; - auto server_app_config = minimal_app_config(base_url, "collections_of_links", schema); + auto server_app_config = minimal_app_config("collections_of_links", schema); TestAppSession test_session(create_app(server_app_config)); auto wait_for_num_objects_to_equal = [](realm::SharedRealm r, const std::string& table_name, size_t count) { @@ -3647,18 +3645,16 @@ TEMPLATE_TEST_CASE("app: partition types", "[sync][pbs][app][partition][baas]", cf::UUID, cf::BoxedOptional, cf::UnboxedOptional, cf::BoxedOptional, cf::BoxedOptional) { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; const std::string partition_key_col_name = "partition_key_prop"; const std::string table_name = "class_partition_test_type"; - REQUIRE(!base_url.empty()); auto partition_property = Property(partition_key_col_name, TestType::property_type); Schema schema = {{Group::table_name_to_class_name(table_name), { {valid_pk_name, PropertyType::Int, true}, partition_property, }}}; - auto server_app_config = minimal_app_config(base_url, "partition_types_app_name", schema); + auto server_app_config = minimal_app_config("partition_types_app_name", schema); server_app_config.partition_key = partition_property; TestAppSession test_session(create_app(server_app_config)); auto app = test_session.app(); @@ -3731,9 +3727,7 @@ TEMPLATE_TEST_CASE("app: partition types", "[sync][pbs][app][partition][baas]", } TEST_CASE("app: full-text compatible with sync", "[sync][app][baas]") { - std::string base_url = get_base_url(); const std::string valid_pk_name = "_id"; - REQUIRE(!base_url.empty()); Schema schema{ {"TopLevel", @@ -3743,7 +3737,7 @@ TEST_CASE("app: full-text compatible with sync", "[sync][app][baas]") { }}, }; - auto server_app_config = minimal_app_config(base_url, "full_text", schema); + auto server_app_config = minimal_app_config("full_text", schema); auto app_session = create_app(server_app_config); const auto partition = random_string(100); TestAppSession test_session(app_session, nullptr); diff --git a/test/object-store/sync/client_reset.cpp b/test/object-store/sync/client_reset.cpp index 0a3abd28bb0..bc23a0d0efb 100644 --- a/test/object-store/sync/client_reset.cpp +++ b/test/object-store/sync/client_reset.cpp @@ -122,9 +122,7 @@ TEST_CASE("sync: large reset with recovery is restartable", "[sync][pbs][client }}, }; - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); + auto server_app_config = minimal_app_config("client_reset_tests", schema); server_app_config.partition_key = partition_prop; TestAppSession test_app_session(create_app(server_app_config)); auto app = test_app_session.app(); @@ -217,9 +215,7 @@ TEST_CASE("sync: pending client resets are cleared when downloads are complete", }}, }; - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); + auto server_app_config = minimal_app_config("client_reset_tests", schema); server_app_config.partition_key = partition_prop; TestAppSession test_app_session(create_app(server_app_config)); auto app = test_app_session.app(); @@ -293,9 +289,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { partition_prop, }}, }; - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); + auto server_app_config = minimal_app_config("client_reset_tests", schema); server_app_config.partition_key = partition_prop; TestAppSession test_app_session(create_app(server_app_config)); auto app = test_app_session.app(); @@ -604,6 +598,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { } }; make_reset(local_config, remote_config) + ->set_development_mode(true) ->setup([&](SharedRealm before) { before->update_schema( { @@ -703,6 +698,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { err = error; }; make_reset(local_config, remote_config) + ->set_development_mode(true) ->make_local_changes([&](SharedRealm local) { local->update_schema( { @@ -1180,6 +1176,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { err = error; }; make_reset(local_config, remote_config) + ->set_development_mode(true) ->make_local_changes([&](SharedRealm local) { local->update_schema( { @@ -1212,6 +1209,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { err = error; }; make_reset(local_config, remote_config) + ->set_development_mode(true) ->make_local_changes([](SharedRealm local) { local->update_schema( { @@ -1243,7 +1241,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { } SECTION("compatible schema changes in both remote and local transactions") { - test_reset + test_reset->set_development_mode(true) ->make_local_changes([](SharedRealm local) { local->update_schema( { @@ -1295,6 +1293,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { err = error; }; make_reset(local_config, remote_config) + ->set_development_mode(true) ->make_local_changes([](SharedRealm local) { local->update_schema( { @@ -1337,6 +1336,7 @@ TEST_CASE("sync: client reset", "[sync][pbs][client reset][baas]") { }; make_reset(local_config, remote_config) + ->set_development_mode(true) ->make_local_changes([](SharedRealm local) { local->update_schema( { @@ -1794,9 +1794,7 @@ TEST_CASE("sync: Client reset during async open", "[sync][pbs][client reset][baa }}, }; - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); + auto server_app_config = minimal_app_config("client_reset_tests", schema); server_app_config.partition_key = partition_prop; TestAppSession test_app_session(create_app(server_app_config)); auto app = test_app_session.app(); diff --git a/test/object-store/sync/flx_migration.cpp b/test/object-store/sync/flx_migration.cpp index c504252d104..6f2b9cc26d0 100644 --- a/test/object-store/sync/flx_migration.cpp +++ b/test/object-store/sync/flx_migration.cpp @@ -116,7 +116,6 @@ static std::vector fill_test_data(SyncTestFile& config, std::optional< TEST_CASE("Test server migration and rollback", "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition1 = "migration-test"; const std::string partition2 = "another-value"; const Schema mig_schema{ @@ -124,7 +123,7 @@ TEST_CASE("Test server migration and rollback", "[sync][flx][flx migration][baas {"string_field", PropertyType::String | PropertyType::Nullable}, {"realm_id", PropertyType::String | PropertyType::Nullable}}), }; - auto server_app_config = minimal_app_config(base_url, "server_migrate_rollback", mig_schema); + auto server_app_config = minimal_app_config("server_migrate_rollback", mig_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config1(session.app(), partition1, server_app_config.schema); SyncTestFile config2(session.app(), partition2, server_app_config.schema); @@ -266,14 +265,13 @@ TEST_CASE("Test server migration and rollback", "[sync][flx][flx migration][baas TEST_CASE("Test client migration and rollback", "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition = "migration-test"; const Schema mig_schema{ ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, {"string_field", PropertyType::String | PropertyType::Nullable}, {"realm_id", PropertyType::String | PropertyType::Nullable}}), }; - auto server_app_config = minimal_app_config(base_url, "server_migrate_rollback", mig_schema); + auto server_app_config = minimal_app_config("server_migrate_rollback", mig_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config(session.app(), partition, server_app_config.schema); config.sync_config->client_resync_mode = ClientResyncMode::DiscardLocal; @@ -322,13 +320,12 @@ TEST_CASE("Test client migration and rollback", "[sync][flx][flx migration][baas TEST_CASE("Test client migration and rollback with recovery", "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition = "migration-test"; const Schema mig_schema{ ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, {"string_field", PropertyType::String | PropertyType::Nullable}}), }; - auto server_app_config = minimal_app_config(base_url, "server_migrate_rollback", mig_schema); + auto server_app_config = minimal_app_config("server_migrate_rollback", mig_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config(session.app(), partition, server_app_config.schema); config.sync_config->client_resync_mode = ClientResyncMode::Recover; @@ -477,14 +474,13 @@ TEST_CASE("An interrupted migration or rollback can recover on the next session" "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition = "migration-test"; const Schema mig_schema{ ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, {"string_field", PropertyType::String | PropertyType::Nullable}, {"realm_id", PropertyType::String | PropertyType::Nullable}}), }; - auto server_app_config = minimal_app_config(base_url, "server_migrate_rollback", mig_schema); + auto server_app_config = minimal_app_config("server_migrate_rollback", mig_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config(session.app(), partition, server_app_config.schema); config.sync_config->client_resync_mode = ClientResyncMode::DiscardLocal; @@ -588,14 +584,13 @@ TEST_CASE("An interrupted migration or rollback can recover on the next session" TEST_CASE("Update to native FLX after migration", "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition = "migration-test"; const Schema mig_schema{ ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, {"string_field", PropertyType::String | PropertyType::Nullable}, {"realm_id", PropertyType::String | PropertyType::Nullable}}), }; - auto server_app_config = minimal_app_config(base_url, "server_migrate_rollback", mig_schema); + auto server_app_config = minimal_app_config("server_migrate_rollback", mig_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config(session.app(), partition, server_app_config.schema); config.sync_config->client_resync_mode = ClientResyncMode::DiscardLocal; @@ -707,14 +702,15 @@ TEST_CASE("Update to native FLX after migration", "[sync][flx][flx migration][ba TEST_CASE("New table is synced after migration", "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition = "migration-test"; - const Schema mig_schema{ - ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, - {"string_field", PropertyType::String | PropertyType::Nullable}, - {"realm_id", PropertyType::String | PropertyType::Nullable}}), - }; - auto server_app_config = minimal_app_config(base_url, "server_migrate_rollback", mig_schema); + const auto obj1_schema = ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, + {"string_field", PropertyType::String | PropertyType::Nullable}, + {"realm_id", PropertyType::String | PropertyType::Nullable}}); + const auto obj2_schema = ObjectSchema("Object2", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, + {"realm_id", PropertyType::String | PropertyType::Nullable}}); + const Schema mig_schema{obj1_schema}; + const Schema two_obj_schema{obj1_schema, obj2_schema}; + auto server_app_config = minimal_app_config("server_migrate_rollback", two_obj_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config(session.app(), partition, server_app_config.schema); config.sync_config->client_resync_mode = ClientResyncMode::DiscardLocal; @@ -748,14 +744,7 @@ TEST_CASE("New table is synced after migration", "[sync][flx][flx migration][baa // Open a new realm with an additional table. { - const Schema schema{ - ObjectSchema("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, - {"string_field", PropertyType::String | PropertyType::Nullable}, - {"realm_id", PropertyType::String | PropertyType::Nullable}}), - ObjectSchema("Object2", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, - {"realm_id", PropertyType::String | PropertyType::Nullable}}), - }; - SyncTestFile flx_config(session.app()->current_user(), schema, SyncConfig::FLXSyncEnabled{}); + SyncTestFile flx_config(session.app()->current_user(), two_obj_schema, SyncConfig::FLXSyncEnabled{}); auto flx_realm = Realm::get_shared_realm(flx_config); @@ -816,7 +805,6 @@ TEST_CASE("New table is synced after migration", "[sync][flx][flx migration][baa TEST_CASE("Async open + client reset", "[sync][flx][flx migration][baas]") { auto logger_ptr = util::Logger::get_default_logger(); - const std::string base_url = get_base_url(); const std::string partition = "async-open-migration-test"; ObjectSchema shared_object("Object", {{"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, {"string_field", PropertyType::String | PropertyType::Nullable}, @@ -824,7 +812,8 @@ TEST_CASE("Async open + client reset", "[sync][flx][flx migration][baas]") { const Schema mig_schema{shared_object}; size_t num_before_reset_notifications = 0; size_t num_after_reset_notifications = 0; - auto server_app_config = minimal_app_config(base_url, "async_open_during_migration", mig_schema); + auto server_app_config = minimal_app_config("async_open_during_migration", mig_schema); + server_app_config.dev_mode_enabled = true; std::optional config; // destruct this after the sessions are torn down TestAppSession session(create_app(server_app_config)); config.emplace(session.app(), partition, server_app_config.schema); diff --git a/test/object-store/sync/flx_sync.cpp b/test/object-store/sync/flx_sync.cpp index 1f2189112c9..18b02389262 100644 --- a/test/object-store/sync/flx_sync.cpp +++ b/test/object-store/sync/flx_sync.cpp @@ -2444,8 +2444,7 @@ TEST_CASE("flx: subscriptions persist after closing/reopening", "[sync][flx][baa #endif TEST_CASE("flx: no subscription store created for PBS app", "[sync][flx][baas]") { - const std::string base_url = get_base_url(); - auto server_app_config = minimal_app_config(base_url, "flx_connect_as_pbs", g_minimal_schema); + auto server_app_config = minimal_app_config("flx_connect_as_pbs", g_minimal_schema); TestAppSession session(create_app(server_app_config)); SyncTestFile config(session.app(), bson::Bson{}, g_minimal_schema); @@ -2488,9 +2487,7 @@ TEST_CASE("flx: connect to FLX with partition value returns an error", "[sync][f } TEST_CASE("flx: connect to PBS as FLX returns an error", "[sync][flx][protocol][baas]") { - const std::string base_url = get_base_url(); - - auto server_app_config = minimal_app_config(base_url, "flx_connect_as_pbs", g_minimal_schema); + auto server_app_config = minimal_app_config("flx_connect_as_pbs", g_minimal_schema); TestAppSession session(create_app(server_app_config)); auto app = session.app(); auto user = app->current_user(); @@ -3715,7 +3712,7 @@ TEST_CASE("flx: convert flx sync realm to bundled realm", "[app][flx][baas]") { create_user_and_log_in(harness->app()); SyncTestFile target_config(harness->app()->current_user(), harness->schema(), SyncConfig::FLXSyncEnabled{}); - auto pbs_app_config = minimal_app_config(harness->app()->base_url(), "pbs_to_flx_convert", harness->schema()); + auto pbs_app_config = minimal_app_config("pbs_to_flx_convert", harness->schema()); TestAppSession pbs_app_session(create_app(pbs_app_config)); SyncTestFile source_config(pbs_app_session.app()->current_user(), "54321"s, pbs_app_config.schema); diff --git a/test/object-store/util/sync/baas_admin_api.cpp b/test/object-store/util/sync/baas_admin_api.cpp index 6dd5dcbbeb0..461294aa6d8 100644 --- a/test/object-store/util/sync/baas_admin_api.cpp +++ b/test/object-store/util/sync/baas_admin_api.cpp @@ -451,17 +451,20 @@ nlohmann::json AdminAPIEndpoint::patch_json(nlohmann::json body) const return nlohmann::json::parse(resp.body.empty() ? "{}" : resp.body); } -AdminAPISession AdminAPISession::login(const std::string& base_url, const std::string& username, - const std::string& password) +AdminAPISession AdminAPISession::login(const AppCreateConfig& config) { + std::string admin_url = config.admin_url; nlohmann::json login_req_body{ {"provider", "userpass"}, - {"username", username}, - {"password", password}, + {"username", config.admin_username}, + {"password", config.admin_password}, }; + if (config.logger) { + config.logger->trace("Logging into baas admin api: %1", admin_url); + } app::Request auth_req{ app::HttpMethod::post, - util::format("%1/api/admin/v3.0/auth/providers/local-userpass/login", base_url), + util::format("%1/api/admin/v3.0/auth/providers/local-userpass/login", admin_url), 60000, // 1 minute timeout { {"Content-Type", "application/json;charset=utf-8"}, @@ -475,12 +478,12 @@ AdminAPISession AdminAPISession::login(const std::string& base_url, const std::s std::string access_token = login_resp_body["access_token"]; - AdminAPIEndpoint user_profile(util::format("%1/api/admin/v3.0/auth/profile", base_url), access_token); + AdminAPIEndpoint user_profile(util::format("%1/api/admin/v3.0/auth/profile", admin_url), access_token); auto profile_resp = user_profile.get_json(); std::string group_id = profile_resp["roles"][0]["group_id"]; - return AdminAPISession(std::move(base_url), std::move(access_token), std::move(group_id)); + return AdminAPISession(std::move(admin_url), std::move(access_token), std::move(group_id)); } void AdminAPISession::revoke_user_sessions(const std::string& user_id, const std::string& app_id) const @@ -760,10 +763,36 @@ AdminAPIEndpoint AdminAPISession::apps(APIFamily family) const REALM_UNREACHABLE(); } -AppCreateConfig default_app_config(const std::string& base_url) +realm::Schema get_default_schema() +{ + const auto dog_schema = + ObjectSchema("Dog", {realm::Property("_id", PropertyType::ObjectId | PropertyType::Nullable, true), + realm::Property("breed", PropertyType::String | PropertyType::Nullable), + realm::Property("name", PropertyType::String), + realm::Property("realm_id", PropertyType::String | PropertyType::Nullable)}); + const auto cat_schema = + ObjectSchema("Cat", {realm::Property("_id", PropertyType::String | PropertyType::Nullable, true), + realm::Property("breed", PropertyType::String | PropertyType::Nullable), + realm::Property("name", PropertyType::String), + realm::Property("realm_id", PropertyType::String | PropertyType::Nullable)}); + const auto person_schema = + ObjectSchema("Person", {realm::Property("_id", PropertyType::ObjectId | PropertyType::Nullable, true), + realm::Property("age", PropertyType::Int), + realm::Property("dogs", PropertyType::Object | PropertyType::Array, "Dog"), + realm::Property("firstName", PropertyType::String), + realm::Property("lastName", PropertyType::String), + realm::Property("realm_id", PropertyType::String | PropertyType::Nullable)}); + return realm::Schema({dog_schema, cat_schema, person_schema}); +} + +AppCreateConfig default_app_config() { ObjectId id = ObjectId::gen(); std::string db_name = util::format("test_data_%1", id.to_string()); + std::string app_url = get_base_url(); + std::string admin_url = get_admin_url(); + REALM_ASSERT(!app_url.empty()); + REALM_ASSERT(!admin_url.empty()); std::string update_user_data_func = util::format(R"( exports = async function(data) { @@ -822,25 +851,6 @@ AppCreateConfig default_app_config(const std::string& base_url) {"resetFunc", reset_func, false}, }; - const auto dog_schema = - ObjectSchema("Dog", {realm::Property("_id", PropertyType::ObjectId | PropertyType::Nullable, true), - realm::Property("breed", PropertyType::String | PropertyType::Nullable), - realm::Property("name", PropertyType::String), - realm::Property("realm_id", PropertyType::String | PropertyType::Nullable)}); - const auto cat_schema = - ObjectSchema("Cat", {realm::Property("_id", PropertyType::String | PropertyType::Nullable, true), - realm::Property("breed", PropertyType::String | PropertyType::Nullable), - realm::Property("name", PropertyType::String), - realm::Property("realm_id", PropertyType::String | PropertyType::Nullable)}); - const auto person_schema = - ObjectSchema("Person", {realm::Property("_id", PropertyType::ObjectId | PropertyType::Nullable, true), - realm::Property("age", PropertyType::Int), - realm::Property("dogs", PropertyType::Object | PropertyType::Array, "Dog"), - realm::Property("firstName", PropertyType::String), - realm::Property("lastName", PropertyType::String), - realm::Property("realm_id", PropertyType::String | PropertyType::Nullable)}); - realm::Schema default_schema({dog_schema, cat_schema, person_schema}); - Property partition_key("realm_id", PropertyType::String | PropertyType::Nullable); AppCreateConfig::UserPassAuthConfig user_pass_config{ @@ -855,27 +865,36 @@ AppCreateConfig default_app_config(const std::string& base_url) true, }; - return AppCreateConfig{"test", - base_url, - "unique_user@domain.com", - "password", - "mongodb://localhost:26000", - db_name, - std::move(default_schema), - std::move(partition_key), - true, // dev_mode_enabled - util::none, // Default to no FLX sync config - std::move(funcs), - std::move(user_pass_config), - std::string{"authFunc"}, - true, // enable_api_key_auth - true, // enable_anonymous_auth - true}; // enable_custom_token_auth + return AppCreateConfig{ + "test", + std::move(app_url), + std::move(admin_url), // BAAS Admin API URL may be different + "unique_user@domain.com", + "password", + get_mongodb_server(), + db_name, + get_default_schema(), + std::move(partition_key), + false, // Dev mode disabled + util::none, // Default to no FLX sync config + std::move(funcs), // Add default functions + std::move(user_pass_config), // enable basic user/pass auth + std::string{"authFunc"}, // custom auth function + true, // enable_api_key_auth + true, // enable_anonymous_auth + true, // enable_custom_token_auth + {}, // no service roles on the default rule + util::Logger::get_default_logger(), // provide the logger to the admin api + }; } -AppCreateConfig minimal_app_config(const std::string& base_url, const std::string& name, const Schema& schema) +AppCreateConfig minimal_app_config(const std::string& name, const Schema& schema) { Property partition_key("realm_id", PropertyType::String | PropertyType::Nullable); + std::string app_url = get_base_url(); + std::string admin_url = get_admin_url(); + REALM_ASSERT(!app_url.empty()); + REALM_ASSERT(!admin_url.empty()); AppCreateConfig::UserPassAuthConfig user_pass_config{ true, "Confirm", "", "http://example.com/confirmEmail", "", "Reset", "http://exmaple.com/resetPassword", @@ -885,27 +904,30 @@ AppCreateConfig minimal_app_config(const std::string& base_url, const std::strin ObjectId id = ObjectId::gen(); return AppCreateConfig{ name, - base_url, + std::move(app_url), + std::move(admin_url), // BAAS Admin API URL may be different "unique_user@domain.com", "password", - "mongodb://localhost:26000", + get_mongodb_server(), util::format("test_data_%1_%2", name, id.to_string()), schema, std::move(partition_key), - true, // dev_mode_enabled - util::none, // no FLX sync config - {}, // no functions - std::move(user_pass_config), // enable basic user/pass auth - util::none, // disable custom auth - true, // enable api key auth - true, // enable anonymous auth - {}, // no service roles on the default rule + false, // Dev mode disabled + util::none, // no FLX sync config + {}, // no functions + std::move(user_pass_config), // enable basic user/pass auth + util::none, // disable custom auth + true, // enable api key auth + true, // enable anonymous auth + false, // enable_custom_token_auth + {}, // no service roles on the default rule + util::Logger::get_default_logger(), // provide the logger to the admin api }; } AppSession create_app(const AppCreateConfig& config) { - auto session = AdminAPISession::login(config.base_url, config.admin_username, config.admin_password); + auto session = AdminAPISession::login(config); auto create_app_resp = session.apps().post_json(nlohmann::json{{"name", config.app_name}}); std::string app_id = create_app_resp["_id"]; std::string client_app_id = create_app_resp["client_app_id"]; @@ -1159,20 +1181,22 @@ AppSession create_app(const AppCreateConfig& config) return {client_app_id, app_id, session, config}; } -AppSession get_runtime_app_session(std::string base_url) +AppSession get_runtime_app_session() { static const AppSession cached_app_session = [&] { - auto cached_app_session = create_app(default_app_config(base_url)); + auto cached_app_session = create_app(default_app_config()); return cached_app_session; }(); return cached_app_session; } +std::string get_mongodb_server() +{ + return "mongodb://localhost:26000"; +} #ifdef REALM_MONGODB_ENDPOINT TEST_CASE("app: baas admin api", "[sync][app][admin api][baas]") { - std::string base_url = REALM_QUOTE(REALM_MONGODB_ENDPOINT); - base_url.erase(std::remove(base_url.begin(), base_url.end(), '"'), base_url.end()); SECTION("embedded objects") { Schema schema{{"top", {{"_id", PropertyType::String, true}, @@ -1181,7 +1205,7 @@ TEST_CASE("app: baas admin api", "[sync][app][admin api][baas]") { ObjectSchema::ObjectType::Embedded, {{"coordinates", PropertyType::Double | PropertyType::Array}}}}; - auto test_app_config = minimal_app_config(base_url, "test", schema); + auto test_app_config = minimal_app_config("test", schema); create_app(test_app_config); } @@ -1195,7 +1219,7 @@ TEST_CASE("app: baas admin api", "[sync][app][admin api][baas]") { {{"c_link", PropertyType::Object | PropertyType::Nullable, "c"}}}, {"c", {{"_id", PropertyType::String, true}, {"d_str", PropertyType::String}}}, }; - auto test_app_config = minimal_app_config(base_url, "test", schema); + auto test_app_config = minimal_app_config("test", schema); create_app(test_app_config); } @@ -1204,7 +1228,7 @@ TEST_CASE("app: baas admin api", "[sync][app][admin api][baas]") { {"a", {{"_id", PropertyType::String, true}, {"b_dict", PropertyType::Dictionary | PropertyType::String}}}, }; - auto test_app_config = minimal_app_config(base_url, "test", schema); + auto test_app_config = minimal_app_config("test", schema); create_app(test_app_config); } @@ -1213,7 +1237,7 @@ TEST_CASE("app: baas admin api", "[sync][app][admin api][baas]") { {"a", {{"_id", PropertyType::String, true}, {"b_dict", PropertyType::Set | PropertyType::String}}}, }; - auto test_app_config = minimal_app_config(base_url, "test", schema); + auto test_app_config = minimal_app_config("test", schema); create_app(test_app_config); } } diff --git a/test/object-store/util/sync/baas_admin_api.hpp b/test/object-store/util/sync/baas_admin_api.hpp index b50e940b9ed..fcf3d5e3ab4 100644 --- a/test/object-store/util/sync/baas_admin_api.hpp +++ b/test/object-store/util/sync/baas_admin_api.hpp @@ -30,6 +30,8 @@ #include #include +#include + #include #include @@ -65,10 +67,11 @@ class AdminAPIEndpoint { std::string m_access_token; }; +struct AppCreateConfig; + class AdminAPISession { public: - static AdminAPISession login(const std::string& base_url, const std::string& username, - const std::string& password); + static AdminAPISession login(const AppCreateConfig& config); enum class APIFamily { Admin, Private }; AdminAPIEndpoint apps(APIFamily family = APIFamily::Admin) const; @@ -133,14 +136,14 @@ class AdminAPISession { MigrationStatus get_migration_status(const std::string& app_id) const; - const std::string& base_url() const noexcept + const std::string& admin_url() const noexcept { return m_base_url; } private: - AdminAPISession(std::string base_url, std::string access_token, std::string group_id) - : m_base_url(std::move(base_url)) + AdminAPISession(std::string admin_url, std::string access_token, std::string group_id) + : m_base_url(std::move(admin_url)) , m_access_token(std::move(access_token)) , m_group_id(std::move(group_id)) { @@ -213,7 +216,8 @@ struct AppCreateConfig { }; std::string app_name; - std::string base_url; + std::string app_url; + std::string admin_url; std::string admin_username; std::string admin_password; @@ -234,10 +238,13 @@ struct AppCreateConfig { bool enable_custom_token_auth = false; std::vector service_roles; + + std::shared_ptr logger; }; -AppCreateConfig default_app_config(const std::string& base_url); -AppCreateConfig minimal_app_config(const std::string& base_url, const std::string& name, const Schema& schema); +realm::Schema get_default_schema(); +AppCreateConfig default_app_config(); +AppCreateConfig minimal_app_config(const std::string& name, const Schema& schema); struct AppSession { std::string client_app_id; @@ -271,16 +278,18 @@ class SynchronousTestTransport : public app::GenericNetworkTransport { std::mutex m_mutex; }; -// This will create a new test app in the baas server at base_url -// to be used in tests. -AppSession get_runtime_app_session(std::string base_url); +// This will create a new test app in the baas server - base_url and admin_url +// are automatically set +AppSession get_runtime_app_session(); + +std::string get_mongodb_server(); template inline app::App::Config get_config(Factory factory, const AppSession& app_session) { return {app_session.client_app_id, factory, - app_session.admin_api.base_url(), + app_session.config.app_url, util::none, {"Object Store Platform Version Blah", "An sdk version", "An sdk name", "A device name", "A device version", "A framework name", "A framework version", "A bundle id"}}; diff --git a/test/object-store/util/sync/flx_sync_harness.hpp b/test/object-store/util/sync/flx_sync_harness.hpp index c8f2736bbd1..5c4b0701268 100644 --- a/test/object-store/util/sync/flx_sync_harness.hpp +++ b/test/object-store/util/sync/flx_sync_harness.hpp @@ -52,7 +52,7 @@ class FLXSyncTestHarness { static AppSession make_app_from_server_schema(const std::string& test_name, const FLXSyncTestHarness::ServerSchema& server_schema) { - auto server_app_config = minimal_app_config(get_base_url(), test_name, server_schema.schema); + auto server_app_config = minimal_app_config(test_name, server_schema.schema); server_app_config.dev_mode_enabled = server_schema.dev_mode_enabled; AppCreateConfig::FLXSyncConfig flx_config; flx_config.queryable_fields = server_schema.queryable_fields; diff --git a/test/object-store/util/sync/sync_test_utils.cpp b/test/object-store/util/sync/sync_test_utils.cpp index 806359d96b8..4426d82adb0 100644 --- a/test/object-store/util/sync/sync_test_utils.cpp +++ b/test/object-store/util/sync/sync_test_utils.cpp @@ -16,9 +16,9 @@ // //////////////////////////////////////////////////////////////////////////// -#include +#include "util/sync/sync_test_utils.hpp" -#include +#include "util/sync/baas_admin_api.hpp" #include #include @@ -76,9 +76,11 @@ bool results_contains_original_name(SyncFileActionMetadataResults& results, cons bool ReturnsTrueWithinTimeLimit::match(util::FunctionRef condition) const { const auto wait_start = std::chrono::steady_clock::now(); + const auto delay = TEST_TIMEOUT_EXTRA > 0 ? m_max_ms + std::chrono::seconds(TEST_TIMEOUT_EXTRA) : m_max_ms; bool predicate_returned_true = false; util::EventLoop::main().run_until([&] { - if (std::chrono::steady_clock::now() - wait_start > m_max_ms) { + if (std::chrono::steady_clock::now() - wait_start > delay) { + util::format("ReturnsTrueWithinTimeLimit exceeded %1 ms", delay.count()); return true; } auto ret = condition(); @@ -94,9 +96,10 @@ bool ReturnsTrueWithinTimeLimit::match(util::FunctionRef condition) cons void timed_wait_for(util::FunctionRef condition, std::chrono::milliseconds max_ms) { const auto wait_start = std::chrono::steady_clock::now(); + const auto delay = TEST_TIMEOUT_EXTRA > 0 ? max_ms + std::chrono::seconds(TEST_TIMEOUT_EXTRA) : max_ms; util::EventLoop::main().run_until([&] { - if (std::chrono::steady_clock::now() - wait_start > max_ms) { - throw std::runtime_error(util::format("timed_wait_for exceeded %1 ms", max_ms.count())); + if (std::chrono::steady_clock::now() - wait_start > delay) { + throw std::runtime_error(util::format("timed_wait_for exceeded %1 ms", delay.count())); } return condition(); }); @@ -106,9 +109,10 @@ void timed_sleeping_wait_for(util::FunctionRef condition, std::chrono::m std::chrono::milliseconds sleep_ms) { const auto wait_start = std::chrono::steady_clock::now(); + const auto delay = TEST_TIMEOUT_EXTRA > 0 ? max_ms + std::chrono::seconds(TEST_TIMEOUT_EXTRA) : max_ms; while (!condition()) { - if (std::chrono::steady_clock::now() - wait_start > max_ms) { - throw std::runtime_error(util::format("timed_sleeping_wait_for exceeded %1 ms", max_ms.count())); + if (std::chrono::steady_clock::now() - wait_start > delay) { + throw std::runtime_error(util::format("timed_sleeping_wait_for exceeded %1 ms", delay.count())); } std::this_thread::sleep_for(sleep_ms); } @@ -192,18 +196,38 @@ void subscribe_to_all_and_bootstrap(Realm& realm) #if REALM_ENABLE_AUTH_TESTS +static std::string unquote_string(std::string_view possibly_quoted_string) +{ + if (possibly_quoted_string.size() > 0) { + auto check_char = possibly_quoted_string.front(); + if (check_char == '"' || check_char == '\'') { + possibly_quoted_string.remove_prefix(1); + } + } + if (possibly_quoted_string.size() > 0) { + auto check_char = possibly_quoted_string.back(); + if (check_char == '"' || check_char == '\'') { + possibly_quoted_string.remove_suffix(1); + } + } + return std::string{possibly_quoted_string}; +} + #ifdef REALM_MONGODB_ENDPOINT std::string get_base_url() { // allows configuration with or without quotes - std::string base_url = REALM_QUOTE(REALM_MONGODB_ENDPOINT); - if (base_url.size() > 0 && base_url[0] == '"') { - base_url.erase(0, 1); - } - if (base_url.size() > 0 && base_url[base_url.size() - 1] == '"') { - base_url.erase(base_url.size() - 1); - } - return base_url; + return unquote_string(REALM_QUOTE(REALM_MONGODB_ENDPOINT)); +} + +std::string get_admin_url() +{ +#ifdef REALM_ADMIN_ENDPOINT + // allows configuration with or without quotes + return unquote_string(REALM_QUOTE(REALM_ADMIN_ENDPOINT)); +#else + return get_base_url(); +#endif } #endif // REALM_MONGODB_ENDPOINT @@ -526,6 +550,13 @@ struct BaasClientReset : public TestClientReset { { } + TestClientReset* set_development_mode(bool enable) override + { + const AppSession& app_session = m_test_app_session.app_session(); + app_session.admin_api.set_development_mode_to(app_session.server_app_id, enable); + return this; + } + void run() override { m_did_run = true; @@ -651,6 +682,13 @@ struct BaasFLXClientReset : public TestClientReset { REALM_ASSERT(m_local_config.schema->find(c_object_schema_name) != m_local_config.schema->end()); } + TestClientReset* set_development_mode(bool enable) override + { + const AppSession& app_session = m_test_app_session.app_session(); + app_session.admin_api.set_development_mode_to(app_session.server_app_id, enable); + return this; + } + void run() override { m_did_run = true; @@ -826,6 +864,10 @@ TestClientReset* TestClientReset::on_post_reset(Callback&& post_reset) m_on_post_reset = std::move(post_reset); return this; } +TestClientReset* TestClientReset::set_development_mode(bool) +{ + return this; +} void TestClientReset::set_pk_of_object_driving_reset(const ObjectId& pk) { diff --git a/test/object-store/util/sync/sync_test_utils.hpp b/test/object-store/util/sync/sync_test_utils.hpp index b7f0b285cdf..195f8eb2f24 100644 --- a/test/object-store/util/sync/sync_test_utils.hpp +++ b/test/object-store/util/sync/sync_test_utils.hpp @@ -18,9 +18,9 @@ #pragma once -#include -#include -#include +#include "util/event_loop.hpp" +#include "util/test_file.hpp" +#include "util/test_utils.hpp" #include #include @@ -92,6 +92,8 @@ util::Future wait_for_future(util::Future&& input, std::chrono::millisecon { auto pf = util::make_promise_future(); auto shared_state = util::make_bind>(std::move(pf.promise)); + const auto delay = TEST_TIMEOUT_EXTRA > 0 ? max_ms + std::chrono::seconds(TEST_TIMEOUT_EXTRA) : max_ms; + std::move(input).get_async([shared_state](StatusOrStatusWith value) { std::unique_lock lk(shared_state->mutex); // If the state has already expired, then just return without doing anything. @@ -105,12 +107,12 @@ util::Future wait_for_future(util::Future&& input, std::chrono::millisecon }); std::unique_lock lk(shared_state->mutex); - if (!shared_state->cv.wait_for(lk, max_ms, [&] { + if (!shared_state->cv.wait_for(lk, delay, [&] { return shared_state->finished; })) { shared_state->finished = true; shared_state->promise.set_error( - {ErrorCodes::RuntimeError, util::format("timed_future wait exceeded %1 ms", max_ms.count())}); + {ErrorCodes::RuntimeError, util::format("wait_for_future exceeded %1 ms", delay.count())}); } return std::move(pf.future); @@ -147,6 +149,8 @@ void subscribe_to_all_and_bootstrap(Realm& realm); #ifdef REALM_MONGODB_ENDPOINT std::string get_base_url(); +std::string get_admin_url(); + #endif struct AutoVerifiedEmailCredentials : app::AppCredentials { @@ -193,6 +197,7 @@ struct TestClientReset { ObjectId get_pk_of_object_driving_reset() const; void disable_wait_for_reset_completion(); + virtual TestClientReset* set_development_mode(bool enable = true); virtual void run() = 0; protected: diff --git a/test/object-store/util/test_file.cpp b/test/object-store/util/test_file.cpp index 7c2a2f00db3..40ee67f286a 100644 --- a/test/object-store/util/test_file.cpp +++ b/test/object-store/util/test_file.cpp @@ -263,6 +263,7 @@ static Status wait_for_session(Realm& realm, void (SyncSession::*fn)(util::Uniqu { auto shared_state = std::make_shared(); auto& session = *realm.config().sync_config->user->session_for_on_disk_path(realm.config().path); + auto delay = TEST_TIMEOUT_EXTRA > 0 ? timeout + std::chrono::seconds(TEST_TIMEOUT_EXTRA) : timeout; (session.*fn)([weak_state = std::weak_ptr(shared_state)](Status s) { auto shared_state = weak_state.lock(); if (!shared_state) { @@ -274,11 +275,11 @@ static Status wait_for_session(Realm& realm, void (SyncSession::*fn)(util::Uniqu shared_state->cv.notify_one(); }); std::unique_lock lock(shared_state->mutex); - bool completed = shared_state->cv.wait_for(lock, timeout, [&]() { + bool completed = shared_state->cv.wait_for(lock, delay, [&]() { return shared_state->complete == true; }); if (!completed) { - throw std::runtime_error("wait_for_session() timed out"); + throw std::runtime_error(util::format("wait_for_session() exceeded %1 ms", delay.count())); } return shared_state->status; } @@ -323,7 +324,7 @@ void set_app_config_defaults(app::App::Config& app_config, #if REALM_ENABLE_AUTH_TESTS TestAppSession::TestAppSession() - : TestAppSession(get_runtime_app_session(get_base_url()), nullptr, DeleteApp{false}) + : TestAppSession(get_runtime_app_session(), nullptr, DeleteApp{false}) { } diff --git a/test/object-store/util/test_file.hpp b/test/object-store/util/test_file.hpp index 905cb95299a..4888199fd7f 100644 --- a/test/object-store/util/test_file.hpp +++ b/test/object-store/util/test_file.hpp @@ -38,6 +38,9 @@ #endif // REALM_ENABLE_SYNC +#ifndef TEST_TIMEOUT_EXTRA +#define TEST_TIMEOUT_EXTRA 0 +#endif namespace realm { struct AppSession; From 37c18a9f73b06beeee414771368195eee125b71d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 8 Nov 2023 18:53:09 -0800 Subject: [PATCH 07/16] Explicitly instantiate Set in set.cpp `Set` is the single type not implicitly instantiated in `Obj::get_setbase_ptr()`, so it needs to be explicitly instantiated. --- CHANGELOG.md | 1 + src/realm/set.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32aabb1c524..42bf5a5964b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) * A crash at a very specific time during a DiscardLocal client reset on a FLX Realm could leave subscriptions in an invalid state ([#7110](https://github.com/realm/realm-core/pull/7110), since v12.3.0). * Fixed an error "Invalid schema change (UPLOAD): cannot process AddColumn instruction for non-existent table" when using automatic client reset with recovery in dev mode to recover schema changes made locally while offline. ([#7042](https://github.com/realm/realm-core/pull/7042) since the server introduced the feature that allows client to redefine the server's schema if the server is in dev mode - fall 2023) +* Fix missing symbol linker error for `Set` when building with Clang and LTO enabled ([#7121](https://github.com/realm/realm-core/pull/7121), since v12.23.3). ### Breaking changes * None. diff --git a/src/realm/set.cpp b/src/realm/set.cpp index d208de64096..20689dfc1f4 100644 --- a/src/realm/set.cpp +++ b/src/realm/set.cpp @@ -404,6 +404,8 @@ void Set::do_clear() m_tree->set_context_flag(false); } +template class Set; + template <> void Set::do_insert(size_t ndx, ObjLink target_link) { From 97fb2b8018b29895ba35ed2ea92e6a60da0e9a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Tue, 14 Nov 2023 08:56:33 +0100 Subject: [PATCH 08/16] Prepeare release --- CHANGELOG.md | 4 +--- Package.swift | 2 +- dependencies.list | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42bf5a5964b..1eca46596c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,9 @@ -# NEXT RELEASE +# 13.23.4 Release notes ### Enhancements -* (PR [#????](https://github.com/realm/realm-core/pull/????)) * None. ### Fixed -* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) * A crash at a very specific time during a DiscardLocal client reset on a FLX Realm could leave subscriptions in an invalid state ([#7110](https://github.com/realm/realm-core/pull/7110), since v12.3.0). * Fixed an error "Invalid schema change (UPLOAD): cannot process AddColumn instruction for non-existent table" when using automatic client reset with recovery in dev mode to recover schema changes made locally while offline. ([#7042](https://github.com/realm/realm-core/pull/7042) since the server introduced the feature that allows client to redefine the server's schema if the server is in dev mode - fall 2023) * Fix missing symbol linker error for `Set` when building with Clang and LTO enabled ([#7121](https://github.com/realm/realm-core/pull/7121), since v12.23.3). diff --git a/Package.swift b/Package.swift index 127e3d53362..b5bde2c24f4 100644 --- a/Package.swift +++ b/Package.swift @@ -3,7 +3,7 @@ import PackageDescription import Foundation -let versionStr = "13.23.3" +let versionStr = "13.23.4" let versionPieces = versionStr.split(separator: "-") let versionCompontents = versionPieces[0].split(separator: ".") let versionExtra = versionPieces.count > 1 ? versionPieces[1] : "" diff --git a/dependencies.list b/dependencies.list index e01b8fc86da..aaf817e3d43 100644 --- a/dependencies.list +++ b/dependencies.list @@ -1,5 +1,5 @@ PACKAGE_NAME=realm-core -VERSION=13.23.3 +VERSION=13.23.4 OPENSSL_VERSION=3.0.8 ZLIB_VERSION=1.2.13 MDBREALM_TEST_SERVER_TAG=2023-10-20 From a2e743a388a3d70230ade165e0edc1d2f025bc95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Tue, 14 Nov 2023 10:37:16 +0100 Subject: [PATCH 09/16] Fix compilation --- src/realm/set.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/realm/set.cpp b/src/realm/set.cpp index 20689dfc1f4..dd9515d3eeb 100644 --- a/src/realm/set.cpp +++ b/src/realm/set.cpp @@ -404,6 +404,11 @@ void Set::do_clear() m_tree->set_context_flag(false); } +template <> +void Set::migrate() +{ +} + template class Set; template <> From 0801f9772d5ba0c35411740d4a32d4e0025c2d1d Mon Sep 17 00:00:00 2001 From: realm-ci Date: Tue, 14 Nov 2023 04:33:26 -0800 Subject: [PATCH 10/16] Updated release notes --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eca46596c8..a07c16532c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +# NEXT RELEASE + +### Enhancements +* (PR [#????](https://github.com/realm/realm-core/pull/????)) +* None. + +### Fixed +* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* None. + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v23. Reads and automatically upgrade from fileformat v5. + +----------- + +### Internals +* None. + +---------------------------------------------- + # 13.23.4 Release notes ### Enhancements From 870615dbc4b9c5f6a6412df1da34e9a76733d912 Mon Sep 17 00:00:00 2001 From: Michael Wilkerson-Barker Date: Thu, 16 Nov 2023 04:09:56 -0500 Subject: [PATCH 11/16] Disable the network tests (#7145) * Disable the network tests * Update to 'debug' log level for network tests * Added comment about the tag --- evergreen/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/evergreen/config.yml b/evergreen/config.yml index 940df4b78e1..7fbcfeef23a 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -7,6 +7,7 @@ # * Add the `allowed_requesters: [ "ad_hoc", "patch" ]` entry to the nightly task to only run for periodic tests # * Commit Builds: # * All other tasks will be included in the set run when a PR is merged to master +# * To prevent a task from running at all, add the `"disabled"` tag so it will be skipped by the any of the test runs. # Some tests take up to 4 hours to run, so give a very generous timeout project-wide, but generally try to complete tasks within 30 minutes. exec_timeout_secs: 14400 @@ -969,6 +970,7 @@ tasks: - name: baas-network-tests # Uncomment once tests are passing + tags: [ "disabled" ] # tags: [ "for_nightly_tests" ] # These tests can be manually requested for patches and pull requests allowed_requesters: [ "ad_hoc", "patch", "github_pr" ] @@ -1456,7 +1458,7 @@ buildvariants: cmake_build_type: RelWithDebInfo run_with_encryption: On baas_admin_port: 9098 - test_logging_level: trace + test_logging_level: debug test_timeout_extra: 60 proxy_toxics_file: evergreen/proxy-nonideal-transfer.toxics # RANDOM1: bandwidth-upstream limited to between 10-50 KB/s from the client to the server @@ -1478,7 +1480,7 @@ buildvariants: cmake_build_type: RelWithDebInfo run_with_encryption: On baas_admin_port: 9098 - test_logging_level: trace + test_logging_level: debug proxy_toxics_file: evergreen/proxy-network-faults.toxics # RANDOM1: limit-data-upstream to close connection after between 1000-3000 bytes have been sent # RANDOM2: limit-data-downstream to close connection after between 1000-3000 bytes have been received From 5faefc45b65c3712d219f37685ebfaa03115ab1f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 15 Nov 2023 09:20:18 -0800 Subject: [PATCH 12/16] Fix a use-after-free when intersecting a set with itself After calling `clear()`, the values to be inserted in the set would no longer be valid if the rhs was the same set and the values were StringData or BinaryData. --- CHANGELOG.md | 2 +- src/realm/collection.hpp | 22 +++--- src/realm/set.cpp | 14 ++++ src/realm/table.cpp | 5 ++ src/realm/table.hpp | 2 + test/test_set.cpp | 145 ++++++++++++++++++++++++++++++--------- 6 files changed, 144 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a07c16532c0..d6110e771a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Fixed * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) -* None. +* `Set::assign_intersection()` on `Set`, `Set`, and `Set` containing string or binary would cause a use-after-free if a set was intersected with itself ([PR #7144](https://github.com/realm/realm-core/pull/7144), since v10.0.0). ### Breaking changes * None. diff --git a/src/realm/collection.hpp b/src/realm/collection.hpp index ab2d99fcd72..35223ad06c9 100644 --- a/src/realm/collection.hpp +++ b/src/realm/collection.hpp @@ -124,6 +124,17 @@ class CollectionBase { return get_table()->get_column_name(get_col_key()); } + bool operator==(const CollectionBase& other) const noexcept + { + return get_table() == other.get_table() && get_owner_key() == other.get_owner_key() && + get_col_key() == other.get_col_key(); + } + + bool operator!=(const CollectionBase& other) const noexcept + { + return !(*this == other); + } + // These are shadowed by typed versions in subclasses using value_type = Mixed; CollectionIterator begin() const; @@ -373,17 +384,6 @@ class CollectionBaseImpl : public Interface, protected ArrayParent { using Interface::get_table; using Interface::get_target_table; - bool operator==(const CollectionBaseImpl& other) const noexcept - { - return get_table() == other.get_table() && get_owner_key() == other.get_owner_key() && - get_col_key() == other.get_col_key(); - } - - bool operator!=(const CollectionBaseImpl& other) const noexcept - { - return !(*this == other); - } - protected: Obj m_obj; ColKey m_col_key; diff --git a/src/realm/set.cpp b/src/realm/set.cpp index dd9515d3eeb..f9c4335375b 100644 --- a/src/realm/set.cpp +++ b/src/realm/set.cpp @@ -271,6 +271,9 @@ bool SetBase::set_equals(const CollectionBase& rhs) const void SetBase::assign_union(const CollectionBase& rhs) { + if (*this == rhs) { + return; + } if (auto other_set = dynamic_cast(&rhs)) { return assign_union(other_set->begin(), other_set->end()); } @@ -292,6 +295,9 @@ void SetBase::assign_union(It1 first, It2 last) void SetBase::assign_intersection(const CollectionBase& rhs) { + if (*this == rhs) { + return; + } if (auto other_set = dynamic_cast(&rhs)) { return assign_intersection(other_set->begin(), other_set->end()); } @@ -313,6 +319,10 @@ void SetBase::assign_intersection(It1 first, It2 last) void SetBase::assign_difference(const CollectionBase& rhs) { + if (*this == rhs) { + clear(); + return; + } if (auto other_set = dynamic_cast(&rhs)) { return assign_difference(other_set->begin(), other_set->end()); } @@ -334,6 +344,10 @@ void SetBase::assign_difference(It1 first, It2 last) void SetBase::assign_symmetric_difference(const CollectionBase& rhs) { + if (*this == rhs) { + clear(); + return; + } if (auto other_set = dynamic_cast(&rhs)) { return assign_symmetric_difference(other_set->begin(), other_set->end()); } diff --git a/src/realm/table.cpp b/src/realm/table.cpp index c22c1581f9c..f1a53e03c4a 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -2257,6 +2257,11 @@ void Table::set_collision_map(ref_type ref) m_top.set(top_position_for_collision_map, RefOrTagged::make_ref(ref)); } +void Table::set_col_key_sequence_number(uint64_t seq) +{ + m_top.set(top_position_for_column_key, RefOrTagged::make_tagged(seq)); +} + TableRef Table::get_link_target(ColKey col_key) noexcept { return get_opposite_table(col_key); diff --git a/src/realm/table.hpp b/src/realm/table.hpp index b1df78f058d..9e84b645abc 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -383,6 +383,8 @@ class Table { // Used by upgrade void set_sequence_number(uint64_t seq); void set_collision_map(ref_type ref); + // Used for testing purposes. + void set_col_key_sequence_number(uint64_t seq); // Get the key of this table directly, without needing a Table accessor. static TableKey get_key_direct(Allocator& alloc, ref_type top_ref); diff --git a/test/test_set.cpp b/test/test_set.cpp index b88f1dc4bea..c2d76a2ef75 100644 --- a/test/test_set.cpp +++ b/test/test_set.cpp @@ -369,44 +369,68 @@ TEST_TYPES(Set_Types, Prop, Prop, Prop, Prop, Propcreate_object(); - { - auto s = obj.get_set(col); - auto l = obj.get_list(col_list); - auto values = gen.values_from_int({0, 1, 2, 3}); - for (auto v : values) { + auto values = gen.values_from_int({0, 1, 2, 3}); + + auto l = obj.get_list(col_list); + for (auto&& v : values) { + l.add(v); + } + + auto s = obj.get_set(col); + auto populate_set = [&] { + s.clear(); + for (auto&& v : values) { s.insert(v); - l.add(v); - } - auto sz = values.size(); - CHECK_EQUAL(s.size(), sz); - auto s1 = s; - CHECK_EQUAL(s1.size(), sz); - CHECK(s.set_equals(l)); - for (auto v : values) { - auto ndx = s.find(v); - CHECK_NOT_EQUAL(ndx, realm::npos); } - auto [erased_ndx, erased] = s.erase(values[0]); - CHECK(erased); - CHECK_EQUAL(erased_ndx, 0); - CHECK_EQUAL(s.size(), values.size() - 1); - CHECK(s.is_subset_of(l)); + }; + + populate_set(); + auto sz = values.size(); + CHECK_EQUAL(s.size(), sz); + auto s1 = s; + CHECK_EQUAL(s1.size(), sz); + CHECK(s.set_equals(l)); + for (auto v : values) { + auto ndx = s.find(v); + CHECK_NOT_EQUAL(ndx, realm::npos); + } - s.clear(); + auto [erased_ndx, erased] = s.erase(values[0]); + CHECK(erased); + CHECK_EQUAL(erased_ndx, 0); + CHECK_EQUAL(s.size(), values.size() - 1); + CHECK(s.is_subset_of(l)); + + s.clear(); + CHECK_EQUAL(s.size(), 0); + + // Union and intersection with self is a no-op + populate_set(); + s.assign_union(s); + CHECK_EQUAL(s.size(), sz); + s.assign_intersection(s); + CHECK_EQUAL(s.size(), sz); + + // Difference with self is clear() + populate_set(); + s.assign_difference(s); + CHECK_EQUAL(s.size(), 0); + + populate_set(); + s.assign_symmetric_difference(s); + CHECK_EQUAL(s.size(), 0); + + if (TEST_TYPE::is_nullable) { + s.insert_null(); + CHECK_EQUAL(s.size(), 1); + auto null_value = TEST_TYPE::default_value(); + CHECK(value_is_null(null_value)); + auto ndx = s.find(null_value); + CHECK_NOT_EQUAL(ndx, realm::npos); + s.erase_null(); CHECK_EQUAL(s.size(), 0); - - if (TEST_TYPE::is_nullable) { - s.insert_null(); - CHECK_EQUAL(s.size(), 1); - auto null_value = TEST_TYPE::default_value(); - CHECK(value_is_null(null_value)); - auto ndx = s.find(null_value); - CHECK_NOT_EQUAL(ndx, realm::npos); - s.erase_null(); - CHECK_EQUAL(s.size(), 0); - ndx = s.find(null_value); - CHECK_EQUAL(ndx, realm::npos); - } + ndx = s.find(null_value); + CHECK_EQUAL(ndx, realm::npos); } } @@ -849,3 +873,56 @@ TEST(Set_SymmetricDifferenceString) CHECK_EQUAL(set1.get(2), "The fox jumps over the lazy dog"); CHECK_EQUAL(set1.get(3), "World"); } + +namespace realm { +template +static std::ostream& operator<<(std::ostream& os, const Set& set) +{ + os << "Set(" << set.get_table()->get_key() << ", " << set.get_obj().get_key() << ", " << set.get_col_key() << ")"; + return os; +} +} // namespace realm + +TEST(Set_Equality) +{ + // Table tries to avoid ColKey collisions, but we specifically want to trigger + // them to test that operator== handles them correctly + auto set_next_col_key = [](Table& table, int64_t value) { + table.set_col_key_sequence_number(value ^ table.get_key().value); + }; + + Group g_1; + auto table_1 = g_1.add_table("table 1"); + set_next_col_key(*table_1, 5); + table_1->add_column_set(type_Int, "set 1"); + set_next_col_key(*table_1, 10); + table_1->add_column_set(type_Int, "set 2"); + auto table_2 = g_1.add_table("table 2"); + set_next_col_key(*table_2, 5); + table_2->add_column_set(type_Int, "set 1"); + Group g_2; + auto table_3 = g_2.add_table("table 1"); + set_next_col_key(*table_3, 5); + table_3->add_column_set(type_Int, "set 1"); + + auto obj_1 = table_1->create_object(); + auto obj_2 = table_1->create_object(); + auto obj_3 = table_2->create_object(); + auto obj_4 = table_3->create_object(); + + // Validate our assumptions that we actually do have key overlaps + CHECK_EQUAL(table_1->get_key(), table_3->get_key()); + CHECK_EQUAL(table_1->get_column_key("set 1"), table_2->get_column_key("set 1")); + CHECK_EQUAL(obj_1.get_key(), obj_3.get_key()); + CHECK_EQUAL(obj_1.get_key(), obj_4.get_key()); + + CHECK_EQUAL(obj_1.get_set("set 1"), obj_1.get_set("set 1")); + // Same obj, different col + CHECK_NOT_EQUAL(obj_1.get_set("set 1"), obj_1.get_set("set 2")); + // Same col, different obj + CHECK_NOT_EQUAL(obj_1.get_set("set 1"), obj_2.get_set("set 1")); + // Same col, same obj, different table + CHECK_NOT_EQUAL(obj_1.get_set("set 1"), obj_3.get_set("set 1")); + // Same col, same obj, same table, different group + CHECK_NOT_EQUAL(obj_1.get_set("set 1"), obj_4.get_set("set 1")); +} From 7e7b52a3d3d3a5df3e9342eb16a43c8bf831e899 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 15 Nov 2023 11:30:50 -0800 Subject: [PATCH 13/16] Use typed comparisons for set algebra on strings and binary On platforms where `char` is signed, ordering StringData and Mixed containing the same strings gives different results. --- CHANGELOG.md | 1 + src/realm/set.cpp | 245 ++++++++++++++++++++++------------------------ src/realm/set.hpp | 22 ----- test/test_set.cpp | 33 +++++++ 4 files changed, 150 insertions(+), 151 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6110e771a9..870b89cc440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Fixed * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) * `Set::assign_intersection()` on `Set`, `Set`, and `Set` containing string or binary would cause a use-after-free if a set was intersected with itself ([PR #7144](https://github.com/realm/realm-core/pull/7144), since v10.0.0). +* Set algebra on `Set` and `Set` gave incorrect results when used on platforms where `char` is signed ([#7135](https://github.com/realm/realm-core/issues/7135), since v13.23.3). ### Breaking changes * None. diff --git a/src/realm/set.cpp b/src/realm/set.cpp index f9c4335375b..ca7de71c771 100644 --- a/src/realm/set.cpp +++ b/src/realm/set.cpp @@ -176,7 +176,22 @@ void SetBase::clear_repl(Replication* repl) const repl->set_clear(*this); } -static std::vector convert_to_set(const CollectionBase& rhs) +namespace { +template +std::vector convert_to_set(const CollectionBase& rhs) +{ + std::vector vec; + vec.reserve(rhs.size()); + for (Mixed value : rhs) { + vec.push_back(value.get()); + } + std::sort(vec.begin(), vec.end(), SetElementLessThan()); + vec.erase(std::unique(vec.begin(), vec.end()), vec.end()); + return vec; +} + +template <> +std::vector convert_to_set(const CollectionBase& rhs) { std::vector mixed(rhs.begin(), rhs.end()); std::sort(mixed.begin(), mixed.end(), SetElementLessThan()); @@ -184,191 +199,163 @@ static std::vector convert_to_set(const CollectionBase& rhs) return mixed; } -bool SetBase::is_subset_of(const CollectionBase& rhs) const +template +auto to_set_if_needed(const SetBase& set, const CollectionBase& rhs, Fn fn) { - if (auto other_set = dynamic_cast(&rhs)) { - return is_subset_of(other_set->begin(), other_set->end()); + SetElementLessThan cmp; + auto& typed_set = static_cast(set); + if (typeid(rhs) == typeid(Set)) { + return fn(typed_set, static_cast(rhs), cmp); } - auto other_set = convert_to_set(rhs); - return is_subset_of(other_set.begin(), other_set.end()); + return fn(typed_set, convert_to_set(rhs), cmp); } -template -bool SetBase::is_subset_of(It1 first, It2 last) const +template +auto as_typed_sets(const SetBase& set, const CollectionBase& rhs, Fn fn) { - return std::includes(first, last, begin(), end(), SetElementLessThan{}); + // Set and Set compare using `char` rather than + // `unsigned char`, resulting in a different order from Set on + // platforms where `char` is signed. This means that the elements will not + // be ordered properly when using SetElementLessThan, and we need + // to use the typed comparisons. + // This can go away in the next major version, as we've fixed the order + // of non-Mixed sets. + switch (set.get_col_key().get_type()) { + case col_type_String: + return to_set_if_needed, StringData>(set, rhs, fn); + case col_type_Binary: + return to_set_if_needed, BinaryData>(set, rhs, fn); + default: + return to_set_if_needed(set, rhs, fn); + } } +} // anonymous namespace -bool SetBase::is_strict_subset_of(const CollectionBase& rhs) const +bool SetBase::is_subset_of(const CollectionBase& rhs) const { - if (auto other_set = dynamic_cast(&rhs)) { - return size() != rhs.size() && is_subset_of(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return size() != other_set.size() && is_subset_of(other_set.begin(), other_set.end()); + return as_typed_sets(*this, rhs, [](auto&& lhs, auto&& rhs, auto cmp) { + return std::includes(rhs.begin(), rhs.end(), lhs.begin(), lhs.end(), cmp); + }); } -bool SetBase::is_superset_of(const CollectionBase& rhs) const +bool SetBase::is_strict_subset_of(const CollectionBase& rhs) const { - if (auto other_set = dynamic_cast(&rhs)) { - return is_superset_of(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return is_superset_of(other_set.begin(), other_set.end()); + return size() != rhs.size() && is_subset_of(rhs); } -template -bool SetBase::is_superset_of(It1 first, It2 last) const +bool SetBase::is_superset_of(const CollectionBase& rhs) const { - return std::includes(begin(), end(), first, last, SetElementLessThan{}); + return as_typed_sets(*this, rhs, [](auto&& lhs, auto&& rhs, auto cmp) { + return std::includes(lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), cmp); + }); } bool SetBase::is_strict_superset_of(const CollectionBase& rhs) const { - if (auto other_set = dynamic_cast(&rhs)) { - return size() != rhs.size() && is_superset_of(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return size() != other_set.size() && is_superset_of(other_set.begin(), other_set.end()); + return as_typed_sets(*this, rhs, [](auto&& lhs, auto&& rhs, auto cmp) { + return lhs.size() > rhs.size() && std::includes(lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), cmp); + }); } bool SetBase::intersects(const CollectionBase& rhs) const { - if (auto other_set = dynamic_cast(&rhs)) { - return intersects(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return intersects(other_set.begin(), other_set.end()); -} - -template -bool SetBase::intersects(It1 first, It2 last) const -{ - SetElementLessThan less; - auto it = begin(); - while (it != end() && first != last) { - if (less(*it, *first)) { - ++it; - } - else if (less(*first, *it)) { - ++first; - } - else { - return true; + return as_typed_sets(*this, rhs, [](auto&& lhs, auto&& rhs, auto cmp) { + auto first = rhs.begin(), last = rhs.end(); + auto it = lhs.begin(); + while (it != lhs.end() && first != last) { + if (cmp(*it, *first)) { + ++it; + } + else if (cmp(*first, *it)) { + ++first; + } + else { + return true; + } } - } - return false; + return false; + }); } bool SetBase::set_equals(const CollectionBase& rhs) const { - if (auto other_set = dynamic_cast(&rhs)) { - return size() == rhs.size() && is_subset_of(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return size() == other_set.size() && is_subset_of(other_set.begin(), other_set.end()); + return as_typed_sets(*this, rhs, [](auto&& lhs, auto&& rhs, auto cmp) { + return lhs.size() == rhs.size() && std::includes(lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), cmp); + }); } void SetBase::assign_union(const CollectionBase& rhs) { if (*this == rhs) { + // Union of a set with itself is itself return; } - if (auto other_set = dynamic_cast(&rhs)) { - return assign_union(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return assign_union(other_set.begin(), other_set.end()); -} - -template -void SetBase::assign_union(It1 first, It2 last) -{ - std::vector the_diff; - std::set_difference(first, last, begin(), end(), std::back_inserter(the_diff), SetElementLessThan{}); - // 'the_diff' now contains all the elements that are in foreign set, but not in 'this' - // Now insert those elements - for (auto&& value : the_diff) { - insert_any(value); - } + return as_typed_sets(*this, rhs, [&](auto&& lhs, auto&& rhs, auto cmp) { + std::vector the_diff; + std::set_difference(rhs.begin(), rhs.end(), lhs.begin(), lhs.end(), std::back_inserter(the_diff), cmp); + // 'the_diff' now contains all the elements that are in foreign set, but not in 'this' + // Now insert those elements + for (auto&& value : the_diff) { + insert_any(value); + } + }); } void SetBase::assign_intersection(const CollectionBase& rhs) { if (*this == rhs) { + // Intersection of a set with itself is itself return; } - if (auto other_set = dynamic_cast(&rhs)) { - return assign_intersection(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return assign_intersection(other_set.begin(), other_set.end()); -} - -template -void SetBase::assign_intersection(It1 first, It2 last) -{ - std::vector intersection; - std::set_intersection(first, last, begin(), end(), std::back_inserter(intersection), SetElementLessThan{}); - clear(); - // Elements in intersection comes from foreign set, so ok to use here - for (auto&& value : intersection) { - insert_any(value); - } + return as_typed_sets(*this, rhs, [&](auto&& lhs, auto&& rhs, auto cmp) { + std::vector intersection; + std::set_intersection(rhs.begin(), rhs.end(), lhs.begin(), lhs.end(), std::back_inserter(intersection), cmp); + clear(); + // Elements in intersection comes from foreign set, so ok to use here + for (auto&& value : intersection) { + insert_any(value); + } + }); } void SetBase::assign_difference(const CollectionBase& rhs) { if (*this == rhs) { + // Difference between a set and itself is empty clear(); return; } - if (auto other_set = dynamic_cast(&rhs)) { - return assign_difference(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return assign_difference(other_set.begin(), other_set.end()); -} - -template -void SetBase::assign_difference(It1 first, It2 last) -{ - std::vector intersection; - std::set_intersection(first, last, begin(), end(), std::back_inserter(intersection), SetElementLessThan{}); - // 'intersection' now contains all the elements that are in both foreign set and 'this'. - // Remove those elements. The elements comes from the foreign set, so ok to refer to. - for (auto&& value : intersection) { - erase_any(value); - } + return as_typed_sets(*this, rhs, [&](auto&& lhs, auto&& rhs, auto cmp) { + std::vector intersection; + std::set_intersection(rhs.begin(), rhs.end(), lhs.begin(), lhs.end(), std::back_inserter(intersection), cmp); + // 'intersection' now contains all the elements that are in both foreign set and 'this'. + // Remove those elements. The elements comes from the foreign set, so ok to refer to. + for (auto&& value : intersection) { + erase_any(value); + } + }); } void SetBase::assign_symmetric_difference(const CollectionBase& rhs) { if (*this == rhs) { + // Difference between a set and itself is empty clear(); return; } - if (auto other_set = dynamic_cast(&rhs)) { - return assign_symmetric_difference(other_set->begin(), other_set->end()); - } - auto other_set = convert_to_set(rhs); - return assign_symmetric_difference(other_set.begin(), other_set.end()); -} - -template -void SetBase::assign_symmetric_difference(It1 first, It2 last) -{ - std::vector difference; - std::set_difference(first, last, begin(), end(), std::back_inserter(difference), SetElementLessThan{}); - std::vector intersection; - std::set_intersection(first, last, begin(), end(), std::back_inserter(intersection), SetElementLessThan{}); - // Now remove the common elements and add the differences - for (auto&& value : intersection) { - erase_any(value); - } - for (auto&& value : difference) { - insert_any(value); - } + return as_typed_sets(*this, rhs, [&](auto&& lhs, auto&& rhs, auto cmp) { + std::vector difference; + std::set_difference(rhs.begin(), rhs.end(), lhs.begin(), lhs.end(), std::back_inserter(difference), cmp); + std::vector intersection; + std::set_intersection(rhs.begin(), rhs.end(), lhs.begin(), lhs.end(), std::back_inserter(intersection), cmp); + // Now remove the common elements and add the differences + for (auto&& value : intersection) { + erase_any(value); + } + for (auto&& value : difference) { + insert_any(value); + } + }); } template <> diff --git a/src/realm/set.hpp b/src/realm/set.hpp index 27fb605b813..093047c5eb9 100644 --- a/src/realm/set.hpp +++ b/src/realm/set.hpp @@ -61,28 +61,6 @@ class SetBase : public CollectionBase { throw InvalidArgument(ErrorCodes::PropertyNotNullable, util::format("Set: %1", CollectionBase::get_property_name())); } - -private: - template - bool is_subset_of(It1, It2) const; - - template - bool is_superset_of(It1, It2) const; - - template - bool intersects(It1, It2) const; - - template - void assign_union(It1, It2); - - template - void assign_intersection(It1, It2); - - template - void assign_difference(It1, It2); - - template - void assign_symmetric_difference(It1, It2); }; template diff --git a/test/test_set.cpp b/test/test_set.cpp index c2d76a2ef75..a2dc83d49b6 100644 --- a/test/test_set.cpp +++ b/test/test_set.cpp @@ -775,6 +775,8 @@ TEST(Set_Difference) Group g; auto foos = g.add_table("class_Foo"); ColKey col_set = foos->add_column_set(type_Int, "int set"); + ColKey col_set_bin = foos->add_column_set(type_Binary, "bin set"); + ColKey col_set_str = foos->add_column_set(type_String, "str set"); ColKey col_list = foos->add_column_list(type_Int, "int list"); auto obj1 = foos->create_object(); @@ -805,6 +807,37 @@ TEST(Set_Difference) CHECK_EQUAL(set1.size(), 2); CHECK_EQUAL(set1.get(0), 2); CHECK_EQUAL(set1.get(1), 3); + + auto set3 = obj1.get_set(col_set_bin); + auto set4 = obj2.get_set(col_set_bin); + + char buffer[5]; + auto gen_bin = [&](char c) { + for (auto& b : buffer) { + b = c; + } + return BinaryData(buffer, 5); + }; + for (char x : {1, 2}) { + set3.insert(gen_bin(x)); + } + for (char x : {255, 1, 2, 3, 2}) { + set4.insert(gen_bin(x)); + } + set3.assign_difference(set4); + CHECK_EQUAL(set3.size(), 0); + + auto set5 = obj1.get_set(col_set_str); + auto set6 = obj2.get_set(col_set_str); + + for (const char* x : {"Apple", "Banana"}) { + set5.insert(x); + } + for (const char* x : {"Æbler", "Apple", "Banana", "Citrus", "Dade"}) { + set6.insert(x); + } + set5.assign_difference(set6); + CHECK_EQUAL(set5.size(), 0); } TEST(Set_SymmetricDifference) From 2e6629522509993e9e040163643fbf8e35da92aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Sat, 18 Nov 2023 15:13:11 +0100 Subject: [PATCH 14/16] Index on list of strings (#7142) --- CHANGELOG.md | 2 +- src/realm/index_string.cpp | 117 ++++++++++++++++------- src/realm/index_string.hpp | 25 +++-- src/realm/list.cpp | 65 +++++++++++++ src/realm/list.hpp | 9 ++ src/realm/object-store/object_store.cpp | 7 +- src/realm/object-store/property.hpp | 2 +- src/realm/query_expression.hpp | 38 +++++++- src/realm/table.cpp | 17 +++- test/object-store/primitive_list.cpp | 59 ++++++++++++ test/test_index_string.cpp | 119 ++++++++++++++++++++++++ test/test_parser.cpp | 6 +- 12 files changed, 414 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 870b89cc440..ca28cf11449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Enhancements * (PR [#????](https://github.com/realm/realm-core/pull/????)) -* None. +* Index on list of strings property now supported (PR [#7142](https://github.com/realm/realm-core/pull/7142)) ### Fixed * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) diff --git a/src/realm/index_string.cpp b/src/realm/index_string.cpp index 216a9adcae6..8f4b49701ba 100644 --- a/src/realm/index_string.cpp +++ b/src/realm/index_string.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -82,6 +83,12 @@ Mixed ClusterColumn::get_value(ObjKey key) const return obj.get_any(m_column_key); } +Lst ClusterColumn::get_list(ObjKey key) const +{ + const Obj obj{m_cluster_tree->get(key)}; + return obj.get_list(m_column_key); +} + std::vector ClusterColumn::get_all_keys() const { std::vector ret; @@ -253,8 +260,8 @@ int64_t IndexArray::index_string(Mixed value, InternalFindResult& result_ref, co if (ref & 1) { int64_t key_value = int64_t(ref >> 1); - Mixed a = column.is_fulltext() ? reconstruct_string(stringoffset, key, index_data) - : column.get_value(ObjKey(key_value)); + Mixed a = column.full_word() ? reconstruct_string(stringoffset, key, index_data) + : column.get_value(ObjKey(key_value)); if (a == value) { result_ref.payload = key_value; return first ? key_value : get_count ? 1 : FindRes_single; @@ -268,7 +275,7 @@ int64_t IndexArray::index_string(Mixed value, InternalFindResult& result_ref, co // List of row indices with common prefix up to this point, in sorted order. if (!sub_isindex) { const IntegerColumn sub(m_alloc, ref_type(ref)); - if (column.is_fulltext()) { + if (column.full_word()) { result_ref.payload = ref; result_ref.start_ndx = 0; result_ref.end_ndx = sub.size(); @@ -335,6 +342,15 @@ void IndexArray::from_list_all_ins(StringData upper_value, std::vector& void IndexArray::from_list_all(Mixed value, std::vector& result, const IntegerColumn& rows, const ClusterColumn& column) const { + if (column.full_word()) { + result.reserve(rows.size()); + for (IntegerColumn::const_iterator it = rows.cbegin(); it != rows.cend(); ++it) { + result.push_back(ObjKey(*it)); + } + + return; + } + SortedListComparator slc(column); IntegerColumn::const_iterator it_end = rows.cend(); @@ -356,8 +372,6 @@ void IndexArray::from_list_all(Mixed value, std::vector& result, const I for (IntegerColumn::const_iterator it = lower; it != upper; ++it) { result.push_back(ObjKey(*it)); } - - return; } @@ -592,8 +606,7 @@ void IndexArray::index_string_all(Mixed value, std::vector& result, cons if (ref & 1) { ObjKey k(int64_t(ref >> 1)); - Mixed a = column.get_value(k); - if (a == value) { + if (column.full_word() || column.get_value(k) == value) { result.push_back(k); return; } @@ -802,11 +815,18 @@ void StringIndex::insert_with_offset(ObjKey obj_key, StringData index_data, cons void StringIndex::insert_to_existing_list_at_lower(ObjKey key, Mixed value, IntegerColumn& list, const IntegerColumnIterator& lower) { - SortedListComparator slc(m_target_column); // At this point there exists duplicates of this value, we need to // insert value beside it's duplicates so that rows are also sorted // in ascending order. - IntegerColumn::const_iterator upper = std::upper_bound(lower, list.cend(), value, slc); + IntegerColumn::const_iterator upper = [&]() { + if (m_target_column.full_word()) { + return list.cend(); + } + else { + SortedListComparator slc(m_target_column); + return std::upper_bound(lower, list.cend(), value, slc); + } + }(); // find insert position (the list has to be kept in sorted order) // In most cases the refs will be added to the end. So we test for that // first to see if we can avoid the binary search for insert position @@ -817,7 +837,9 @@ void StringIndex::insert_to_existing_list_at_lower(ObjKey key, Mixed value, Inte } else { IntegerColumn::const_iterator inner_lower = std::lower_bound(lower, upper, key.value); - list.insert(inner_lower.get_position(), key.value); + if (*inner_lower != key.value) { + list.insert(inner_lower.get_position(), key.value); + } } } @@ -1120,7 +1142,7 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin // When key is outside current range, we can just add it keys.add(key); - if (!m_target_column.is_fulltext() || is_at_string_end) { + if (!m_target_column.full_word() || is_at_string_end) { int64_t shifted = int64_t((uint64_t(obj_key.value) << 1) + 1); // shift to indicate literal m_array->add(shifted); } @@ -1141,7 +1163,7 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin return false; keys.insert(ins_pos, key); - if (!m_target_column.is_fulltext() || is_at_string_end) { + if (!m_target_column.full_word() || is_at_string_end) { int64_t shifted = int64_t((uint64_t(obj_key.value) << 1) + 1); // shift to indicate literal m_array->insert(ins_pos_refs, shifted); } @@ -1162,17 +1184,19 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin // Single match (lowest bit set indicates literal row_ndx) if ((slot_value & 1) != 0) { ObjKey obj_key2 = ObjKey(int64_t(slot_value >> 1)); - Mixed v2 = m_target_column.is_fulltext() ? reconstruct_string(offset, key, index_data) : get(obj_key2); + Mixed v2 = m_target_column.full_word() ? reconstruct_string(offset, key, index_data) : get(obj_key2); if (v2 == value) { - // Strings are equal but this is not a list. - // Create a list and add both rows. + if (obj_key.value != obj_key2.value) { + // Strings are equal but this is not a list. + // Create a list and add both rows. - // convert to list (in sorted order) - Array row_list(alloc); - row_list.create(Array::type_Normal); // Throws - row_list.add(obj_key < obj_key2 ? obj_key.value : obj_key2.value); - row_list.add(obj_key < obj_key2 ? obj_key2.value : obj_key.value); - m_array->set(ins_pos_refs, row_list.get_ref()); + // convert to list (in sorted order) + Array row_list(alloc); + row_list.create(Array::type_Normal); // Throws + row_list.add(obj_key < obj_key2 ? obj_key.value : obj_key2.value); + row_list.add(obj_key < obj_key2 ? obj_key2.value : obj_key.value); + m_array->set(ins_pos_refs, row_list.get_ref()); + } } else { StringConversionBuffer buffer; @@ -1213,7 +1237,8 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin IntegerColumn::const_iterator lower = it_end; auto value_exists_in_list = [&]() { - if (m_target_column.is_fulltext()) { + if (m_target_column.full_word()) { + lower = sub.cbegin(); return reconstruct_string(offset, key, index_data) == value.get_string(); } SortedListComparator slc(m_target_column); @@ -1240,15 +1265,15 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin // point and insert into the existing list. ObjKey key_of_any_dup = ObjKey(sub.get(0)); StringConversionBuffer buffer; - StringData index_data_2 = m_target_column.is_fulltext() ? reconstruct_string(offset, key, index_data) - : get(key_of_any_dup).get_index_data(buffer); + StringData index_data_2 = m_target_column.full_word() ? reconstruct_string(offset, key, index_data) + : get(key_of_any_dup).get_index_data(buffer); if (index_data == index_data_2 || suboffset > s_max_offset) { insert_to_existing_list(obj_key, value, sub); } else { #ifdef REALM_DEBUG bool contains_only_duplicates = true; - if (!m_target_column.is_fulltext() && sub.size() > 1) { + if (!m_target_column.full_word() && sub.size() > 1) { ObjKey first_key = ObjKey(sub.get(0)); ObjKey last_key = ObjKey(sub.back()); auto first = get(first_key); @@ -1287,15 +1312,37 @@ Mixed StringIndex::get(ObjKey key) const void StringIndex::erase(ObjKey key) { StringConversionBuffer buffer; - std::string_view value{(get(key).get_index_data(buffer))}; - if (m_target_column.is_fulltext()) { - auto words = Tokenizer::get_instance()->reset(value).get_all_tokens(); - for (auto& w : words) { - erase_string(key, w); + if (m_target_column.full_word()) { + if (m_target_column.tokenize()) { + // This is a full text index + auto index_data(get(key).get_index_data(buffer)); + auto words = Tokenizer::get_instance()->reset(std::string_view(index_data)).get_all_tokens(); + for (auto& w : words) { + erase_string(key, w); + } + } + else { + // This is a list (of strings) + erase_list(key, m_target_column.get_list(key)); } } else { - erase_string(key, value); + erase_string(key, get(key).get_index_data(buffer)); + } +} + +void StringIndex::erase_list(ObjKey key, const Lst& list) +{ + std::vector strings; + strings.reserve(list.size()); + for (auto& val : list) { + strings.push_back(val); + } + + std::sort(strings.begin(), strings.end()); + auto last = std::unique(strings.begin(), strings.end()); + for (auto it = strings.begin(); it != last; ++it) { + erase_string(key, *it); } } @@ -1659,7 +1706,7 @@ void StringIndex::insert(ObjKey key, StringData value) { StringConversionBuffer buffer; - if (this->m_target_column.is_fulltext()) { + if (this->m_target_column.tokenize()) { auto words = Tokenizer::get_instance()->reset(std::string_view(value)).get_all_tokens(); for (auto& word : words) { @@ -1680,7 +1727,7 @@ void StringIndex::set(ObjKey key, StringData new_value) Mixed old_value = get(key); Mixed new_value2 = Mixed(new_value); - if (this->m_target_column.is_fulltext()) { + if (this->m_target_column.tokenize()) { auto tokenizer = Tokenizer::get_instance(); StringData old_string = old_value.get_index_data(buffer); std::set old_words; @@ -1966,6 +2013,10 @@ void StringIndex::dump_node_structure(const Array& node, std::ostream& out, int } } +void StringIndex::dump_node_structure() const +{ + do_dump_node_structure(std::cout, 0); +} void StringIndex::do_dump_node_structure(std::ostream& out, int level) const { diff --git a/src/realm/index_string.hpp b/src/realm/index_string.hpp index 3e40828af4b..9d34fd1b497 100644 --- a/src/realm/index_string.hpp +++ b/src/realm/index_string.hpp @@ -125,7 +125,8 @@ class ClusterColumn { ClusterColumn(const ClusterTree* cluster_tree, ColKey column_key, IndexType type) : m_cluster_tree(cluster_tree) , m_column_key(column_key) - , m_type(type) + , m_tokenize(type == IndexType::Fulltext) + , m_full_word(m_tokenize | column_key.is_collection()) { } size_t size() const @@ -152,17 +153,23 @@ class ClusterColumn { { return m_column_key.is_nullable(); } - bool is_fulltext() const + bool tokenize() const { - return m_type == IndexType::Fulltext; + return m_tokenize; + } + bool full_word() const + { + return m_full_word; } Mixed get_value(ObjKey key) const; + Lst get_list(ObjKey key) const; std::vector get_all_keys() const; private: const ClusterTree* m_cluster_tree; ColKey m_column_key; - IndexType m_type; + bool m_tokenize; + bool m_full_word; }; class StringIndex { @@ -203,7 +210,7 @@ class StringIndex { bool is_empty() const; bool is_fulltext_index() const { - return this->m_target_column.is_fulltext(); + return this->m_target_column.tokenize(); } template @@ -217,6 +224,10 @@ class StringIndex { void set(ObjKey key, util::Optional new_value); void erase(ObjKey key); + void erase_list(ObjKey key, const Lst&); + // Erase without getting value from parent column (useful when string stored + // does not directly match string in parent, like with full-text indexing) + void erase_string(ObjKey key, StringData value); template ObjKey find_first(T value) const; @@ -237,6 +248,7 @@ class StringIndex { #ifdef REALM_DEBUG template void verify_entries(const ClusterColumn& column) const; + void dump_node_structure() const; void do_dump_node_structure(std::ostream&, int) const; #endif @@ -316,9 +328,6 @@ class StringIndex { bool noextend = false); void node_insert_split(size_t ndx, size_t new_ref); void node_insert(size_t ndx, size_t ref); - // Erase without getting value from parent column (useful when string stored - // does not directly match string in parent, like with full-text indexing) - void erase_string(ObjKey key, StringData value); void do_delete(ObjKey key, StringData, size_t offset); Mixed get(ObjKey key) const; diff --git a/src/realm/list.cpp b/src/realm/list.cpp index f1e2f93d546..ad08694a567 100644 --- a/src/realm/list.cpp +++ b/src/realm/list.cpp @@ -35,6 +35,7 @@ #include "realm/table_view.hpp" #include "realm/group.hpp" #include "realm/replication.hpp" +#include "realm/index_string.hpp" namespace realm { @@ -192,6 +193,70 @@ void Lst::distinct(std::vector& indices, util::Optional sort_or } } +/***************************** Lst ******************************/ + +template <> +void Lst::do_insert(size_t ndx, StringData value) +{ + if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { + // Inserting a value already present is idempotent + index->insert(m_obj.get_key(), value); + } + m_tree->insert(ndx, value); +} + +template <> +void Lst::do_set(size_t ndx, StringData value) +{ + if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { + auto old_value = m_tree->get(ndx); + size_t nb_old = 0; + m_tree->for_all([&](StringData val) { + if (val == old_value) { + nb_old++; + } + return !(nb_old > 1); + }); + + if (nb_old == 1) { + // Remove last one + index->erase_string(m_obj.get_key(), old_value); + } + // Inserting a value already present is idempotent + index->insert(m_obj.get_key(), value); + } + m_tree->set(ndx, value); +} + +template <> +inline void Lst::do_remove(size_t ndx) +{ + if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { + auto old_value = m_tree->get(ndx); + size_t nb_old = 0; + m_tree->for_all([&](StringData val) { + if (val == old_value) { + nb_old++; + } + return !(nb_old > 1); + }); + + if (nb_old == 1) { + index->erase_string(m_obj.get_key(), old_value); + } + } + m_tree->erase(ndx); +} + +template <> +inline void Lst::do_clear() +{ + if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { + index->erase_list(m_obj.get_key(), *this); + } + m_tree->clear(); +} + /********************************* Lst *********************************/ template <> diff --git a/src/realm/list.hpp b/src/realm/list.hpp index 30eb6996763..f9ad7e1bee1 100644 --- a/src/realm/list.hpp +++ b/src/realm/list.hpp @@ -290,6 +290,15 @@ class Lst final : public CollectionBaseImpl { T do_get(size_t ndx, const char* msg) const; }; +// Specialization of Lst: +template <> +void Lst::do_insert(size_t, StringData); +template <> +void Lst::do_set(size_t, StringData); +template <> +void Lst::do_remove(size_t); +template <> +void Lst::do_clear(); // Specialization of Lst: template <> void Lst::do_set(size_t, ObjKey); diff --git a/src/realm/object-store/object_store.cpp b/src/realm/object-store/object_store.cpp index 6c38f2d117c..db383cd6a32 100644 --- a/src/realm/object-store/object_store.cpp +++ b/src/realm/object-store/object_store.cpp @@ -131,8 +131,11 @@ ColKey add_column(Group& group, Table& table, Property const& property) } } else if (is_array(property.type)) { - return table.add_column_list(to_core_type(property.type & ~PropertyType::Flags), property.name, - is_nullable(property.type)); + auto key = table.add_column_list(to_core_type(property.type & ~PropertyType::Flags), property.name, + is_nullable(property.type)); + if (property.requires_index()) + table.add_search_index(key); + return key; } else if (is_set(property.type)) { return table.add_column_set(to_core_type(property.type & ~PropertyType::Flags), property.name, diff --git a/src/realm/object-store/property.hpp b/src/realm/object-store/property.hpp index 8ec66878f30..5c2823f14d2 100644 --- a/src/realm/object-store/property.hpp +++ b/src/realm/object-store/property.hpp @@ -338,7 +338,7 @@ inline Property::Property(std::string name, PropertyType type, std::string objec inline bool Property::type_is_indexable() const noexcept { - return !is_collection(type) && + return (!is_collection(type) || (is_array(type) && type == PropertyType::String)) && (type == PropertyType::Int || type == PropertyType::Bool || type == PropertyType::Date || type == PropertyType::String || type == PropertyType::ObjectId || type == PropertyType::UUID || type == PropertyType::Mixed); diff --git a/src/realm/query_expression.hpp b/src/realm/query_expression.hpp index b1e70c65123..19fe2a92942 100644 --- a/src/realm/query_expression.hpp +++ b/src/realm/query_expression.hpp @@ -3028,8 +3028,42 @@ class Columns> : public ColumnsCollection { { return make_subexpr>>(*this); } - friend class Table; - friend class LinkChain; +}; + +template <> +class Columns> : public ColumnsCollection { +public: + using ColumnsCollection::ColumnsCollection; + using ColumnListBase::m_column_key; + using ColumnListBase::m_link_map; + + std::unique_ptr clone() const override + { + return make_subexpr>>(*this); + } + + bool has_search_index() const final + { + auto target_table = m_link_map.get_target_table(); + return target_table->search_index_type(m_column_key) == IndexType::General; + } + + std::vector find_all(Mixed value) const final + { + std::vector ret; + std::vector result; + + StringIndex* index = m_link_map.get_target_table()->get_search_index(m_column_key); + REALM_ASSERT(index); + index->find_all(result, value); + + for (ObjKey k : result) { + auto ndxs = m_link_map.get_origin_ndxs(k); + ret.insert(ret.end(), ndxs.begin(), ndxs.end()); + } + + return ret; + } }; template diff --git a/src/realm/table.cpp b/src/realm/table.cpp index f1a53e03c4a..76fdf448fa0 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -770,8 +770,16 @@ void Table::populate_search_index(ColKey col_key) } } else if (type == type_String) { - StringData value = o.get(col_key); - index->insert(key, value); // Throws + if (col_key.is_list()) { + auto list = o.get_list(col_key); + for (auto& s : list) { + index->insert(key, s); // Throws + } + } + else { + StringData value = o.get(col_key); + index->insert(key, value); // Throws + } } else if (type == type_Timestamp) { Timestamp value = o.get(col_key); @@ -840,6 +848,8 @@ void Table::update_indexes(ObjKey key, const FieldValues& values) if (auto&& index = m_index_accessors[column_ndx]) { // There is an index for this column auto col_key = m_leaf_ndx2colkey[column_ndx]; + if (col_key.is_collection()) + continue; auto type = col_key.get_type(); auto attr = col_key.get_attrs(); bool nullable = attr.test(col_attr_Nullable); @@ -919,7 +929,8 @@ void Table::do_add_search_index(ColKey col_key, IndexType type) if (m_index_accessors[column_ndx] != nullptr) return; - if (!StringIndex::type_supported(DataType(col_key.get_type())) || col_key.is_collection() || + if (!StringIndex::type_supported(DataType(col_key.get_type())) || + (col_key.is_collection() && !(col_key.is_list() && col_key.get_type() == col_type_String)) || (type == IndexType::Fulltext && col_key.get_type() != col_type_String)) { // Not ideal, but this is what we used to throw, so keep throwing that for compatibility reasons, even though // it should probably be a type mismatch exception instead. diff --git a/test/object-store/primitive_list.cpp b/test/object-store/primitive_list.cpp index 05a0a6627aa..b45a37dd358 100644 --- a/test/object-store/primitive_list.cpp +++ b/test/object-store/primitive_list.cpp @@ -989,3 +989,62 @@ TEST_CASE("list of mixed links", "[primitives]") { } } } + +TEST_CASE("list of strings - with index", "[primitives]") { + InMemoryTestFile config; + config.cache = false; + config.automatic_change_notifications = false; + config.schema = Schema{ + {"object", + {{"strings", PropertyType::Array | PropertyType::String, Property::IsPrimary{false}, + Property::IsIndexed{true}}}}, + }; + + auto r = Realm::get_shared_realm(config); + + auto table = r->read_group().get_table("class_object"); + ColKey col = table->get_column_key("strings"); + Results has_banana(r, table->query("strings = 'Banana'")); + Results has_pear(r, table->query("strings = 'Pear'")); + + auto write = [&](auto&& fn) { + r->begin_transaction(); + fn(); + r->commit_transaction(); + }; + + r->begin_transaction(); + Obj obj = table->create_object(); + List list(r, obj, col); + r->commit_transaction(); + + write([&] { + list.add(StringData("Banana")); + list.add(StringData("Apple")); + list.add(StringData("Orange")); + }); + + CHECK(has_banana.size() == 1); + CHECK(has_pear.size() == 0); + + write([&] { + list.set(0, StringData("Pear")); // Add Pear - remove banana + list.add(StringData("Pear")); // Already there + list.add(StringData("Grape")); // Add + }); + CHECK(has_banana.size() == 0); + CHECK(has_pear.size() == 1); + + write([&] { + list.set(1, StringData("Orange")); // Already Orange - remove Apple + list.set(3, StringData("Banana")); // Add Banana - keep Pear + }); + CHECK(has_banana.size() == 1); + CHECK(has_pear.size() == 1); + + write([&] { + list.set(2, StringData("Banana")); // No change in index + }); + CHECK(has_banana.size() == 1); + CHECK(has_pear.size() == 1); +} diff --git a/test/test_index_string.cpp b/test/test_index_string.cpp index 631d68b42ff..1e07d9bbc24 100644 --- a/test/test_index_string.cpp +++ b/test/test_index_string.cpp @@ -1900,4 +1900,123 @@ TEST(Unicode_Casemap) CHECK_EQUAL(*out, inp); } } + +static std::string random_string(std::string::size_type length) +{ + static auto& chrs = "0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + thread_local static std::mt19937 rg{std::random_device{}()}; + thread_local static std::uniform_int_distribution pick(0, sizeof(chrs) - 2); + + std::string s; + + s.reserve(length); + + while (length--) + s += chrs[pick(rg)]; + + return s; +} + +TEST(StringIndex_ListOfRandomStrings) +{ + using namespace std::chrono; + + SHARED_GROUP_TEST_PATH(path); + auto db = DB::create(path); + auto wt = db->start_write(); + + auto t = wt->add_table_with_primary_key("foo", type_Int, "_id"); + ColKey col_codes = t->add_column_list(type_String, "codes"); + std::string some_string; + + for (size_t i = 0; i < 10000; i++) { + auto obj = t->create_object_with_primary_key(int64_t(i)); + auto list = obj.get_list(col_codes); + for (size_t j = 0; j < 3; j++) { + std::string str(random_string(14)); + if (i == 5000 && j == 0) { + some_string = str; + } + list.add(StringData(str)); + } + } + + std::vector arguments{Mixed(some_string)}; + auto q = wt->get_table("foo")->query("codes = $0", arguments); + // auto t1 = steady_clock::now(); + auto tv = q.find_all(); + // auto t2 = steady_clock::now(); + // std::cout << "time without index: " << duration_cast(t2 - t1).count() << " us" << std::endl; + CHECK_EQUAL(tv.size(), 1); + t->add_search_index(col_codes); + + // t1 = steady_clock::now(); + tv = q.find_all(); + // t2 = steady_clock::now(); + // std::cout << "time with index: " << duration_cast(t2 - t1).count() << " us" << std::endl; + CHECK_EQUAL(tv.size(), 1); + t->add_search_index(col_codes); + + // std::cout << tv.get_object(0).get("_id") << std::endl; +} + +TEST_TYPES(StringIndex_ListOfStrings, std::true_type, std::false_type) +{ + constexpr bool add_index = TEST_TYPE::value; + Group g; + + auto t = g.add_table("foo"); + ColKey col = t->add_column_list(type_String, "names", true); + if constexpr (add_index) { + t->add_search_index(col); + } + + auto obj1 = t->create_object(); + auto obj2 = t->create_object(); + auto obj3 = t->create_object(); + + for (Obj* obj : {&obj2, &obj3}) { + auto list = obj->get_list(col); + list.add("Johnny"); + list.add("John"); + } + + auto list = obj1.get_list(col); + list.add("Johnny"); + list.add("John"); + list.add("Ivan"); + list.add("Ivan"); + list.add(StringData()); + + CHECK_EQUAL(t->query(R"(names = "John")").count(), 3); + CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 3); + CHECK_EQUAL(t->query(R"(names = NULL)").count(), 1); + + list.set(0, "Paul"); + CHECK_EQUAL(t->query(R"(names = "John")").count(), 3); + CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 2); + CHECK_EQUAL(t->query(R"(names = "Paul")").count(), 1); + + list.remove(1); + CHECK_EQUAL(t->query(R"(names = "John")").count(), 2); + CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 2); + CHECK_EQUAL(t->query(R"(names = "Paul")").count(), 1); + CHECK_EQUAL(t->query(R"(names = "Ivan")").count(), 1); + + list.clear(); + CHECK_EQUAL(t->query(R"(names = "John")").count(), 2); + CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 2); + CHECK_EQUAL(t->query(R"(names = "Paul")").count(), 0); + + list = obj2.get_list(col); + list.insert(0, "Adam"); + list.insert(0, "Adam"); + obj2.remove(); + CHECK_EQUAL(t->query(R"(names = "John")").count(), 1); + CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 1); +} + #endif // TEST_INDEX_STRING diff --git a/test/test_parser.cpp b/test/test_parser.cpp index 843f965e60b..e53cb88844b 100644 --- a/test/test_parser.cpp +++ b/test/test_parser.cpp @@ -2222,14 +2222,16 @@ TEST(Parser_list_of_primitive_ints) CHECK_EQUAL(message, "Cannot convert 'string' to a number"); } -TEST(Parser_list_of_primitive_strings) +TEST_TYPES(Parser_list_of_primitive_strings, std::true_type, std::false_type) { Group g; TableRef t = g.add_table("table"); constexpr bool nullable = true; auto col_str_list = t->add_column_list(type_String, "strings", nullable); - CHECK_THROW_ANY(t->add_search_index(col_str_list)); + if constexpr (TEST_TYPE::value) { + t->add_search_index(col_str_list); + } auto get_string = [](size_t i) -> std::string { return util::format("string_%1", i); From c5174be4f4cf424c8421dda1c65671f5b34cd77d Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Mon, 20 Nov 2023 10:36:58 +0000 Subject: [PATCH 15/16] Stop failing when sync is disabled and MacOs debug tests are set to run (#7146) --- Jenkinsfile | 9 ++++++--- test/object-store/benchmarks/CMakeLists.txt | 3 +-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7f53f8280eb..59aaa0db17c 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -201,7 +201,7 @@ jobWrapper { ] if (releaseTesting) { extendedChecks = [ - checkMacOsDebug : doBuildMacOs(buildOptions + [buildType: "Release"]), + checkMacOsDebug : doBuildMacOs(buildOptions + [buildType: "Debug"]), checkAndroidarmeabiDebug : doAndroidBuildInDocker('armeabi-v7a', 'Debug', TestAction.Run), // FIXME: https://github.com/realm/realm-core/issues/4159 //checkAndroidx86Release : doAndroidBuildInDocker('x86', 'Release', TestAction.Run), @@ -606,8 +606,11 @@ def doBuildMacOs(Map options = [:]) { dir('build-macosx') { withEnv(['DEVELOPER_DIR=/Applications/Xcode-14.app/Contents/Developer/']) { - sh "cmake ${cmakeDefinitions} -G Xcode .." - + try { + sh "cmake ${cmakeDefinitions} -G Xcode .." + } catch(Exception e) { + archiveArtifacts '**/*' + } runAndCollectWarnings( parser: 'clang', script: "cmake --build . --config ${buildType} --target package -- ONLY_ACTIVE_ARCH=NO -destination generic/name=macOS -sdk macosx", diff --git a/test/object-store/benchmarks/CMakeLists.txt b/test/object-store/benchmarks/CMakeLists.txt index 75d330ebec4..d77f5f021b4 100644 --- a/test/object-store/benchmarks/CMakeLists.txt +++ b/test/object-store/benchmarks/CMakeLists.txt @@ -35,9 +35,8 @@ target_include_directories(object-store-benchmarks PRIVATE if(REALM_ENABLE_SYNC) target_link_libraries(object-store-benchmarks SyncServer) - enable_stdfilesystem(object-store-benchmarks) endif() - +enable_stdfilesystem(object-store-benchmarks) target_link_libraries(object-store-benchmarks ObjectStore TestUtil Catch2::Catch2) add_dependencies(benchmarks object-store-benchmarks) From 062049346198c172fc643fddef7f9c14725a0bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 20 Nov 2023 12:34:28 +0100 Subject: [PATCH 16/16] Revert "Index on list of strings (#7142)" This reverts commit 2e6629522509993e9e040163643fbf8e35da92aa. --- CHANGELOG.md | 2 +- src/realm/index_string.cpp | 117 +++++++---------------- src/realm/index_string.hpp | 25 ++--- src/realm/list.cpp | 65 ------------- src/realm/list.hpp | 9 -- src/realm/object-store/object_store.cpp | 7 +- src/realm/object-store/property.hpp | 2 +- src/realm/query_expression.hpp | 38 +------- src/realm/table.cpp | 17 +--- test/object-store/primitive_list.cpp | 59 ------------ test/test_index_string.cpp | 119 ------------------------ test/test_parser.cpp | 6 +- 12 files changed, 52 insertions(+), 414 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca28cf11449..870b89cc440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Enhancements * (PR [#????](https://github.com/realm/realm-core/pull/????)) -* Index on list of strings property now supported (PR [#7142](https://github.com/realm/realm-core/pull/7142)) +* None. ### Fixed * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) diff --git a/src/realm/index_string.cpp b/src/realm/index_string.cpp index 8f4b49701ba..216a9adcae6 100644 --- a/src/realm/index_string.cpp +++ b/src/realm/index_string.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -83,12 +82,6 @@ Mixed ClusterColumn::get_value(ObjKey key) const return obj.get_any(m_column_key); } -Lst ClusterColumn::get_list(ObjKey key) const -{ - const Obj obj{m_cluster_tree->get(key)}; - return obj.get_list(m_column_key); -} - std::vector ClusterColumn::get_all_keys() const { std::vector ret; @@ -260,8 +253,8 @@ int64_t IndexArray::index_string(Mixed value, InternalFindResult& result_ref, co if (ref & 1) { int64_t key_value = int64_t(ref >> 1); - Mixed a = column.full_word() ? reconstruct_string(stringoffset, key, index_data) - : column.get_value(ObjKey(key_value)); + Mixed a = column.is_fulltext() ? reconstruct_string(stringoffset, key, index_data) + : column.get_value(ObjKey(key_value)); if (a == value) { result_ref.payload = key_value; return first ? key_value : get_count ? 1 : FindRes_single; @@ -275,7 +268,7 @@ int64_t IndexArray::index_string(Mixed value, InternalFindResult& result_ref, co // List of row indices with common prefix up to this point, in sorted order. if (!sub_isindex) { const IntegerColumn sub(m_alloc, ref_type(ref)); - if (column.full_word()) { + if (column.is_fulltext()) { result_ref.payload = ref; result_ref.start_ndx = 0; result_ref.end_ndx = sub.size(); @@ -342,15 +335,6 @@ void IndexArray::from_list_all_ins(StringData upper_value, std::vector& void IndexArray::from_list_all(Mixed value, std::vector& result, const IntegerColumn& rows, const ClusterColumn& column) const { - if (column.full_word()) { - result.reserve(rows.size()); - for (IntegerColumn::const_iterator it = rows.cbegin(); it != rows.cend(); ++it) { - result.push_back(ObjKey(*it)); - } - - return; - } - SortedListComparator slc(column); IntegerColumn::const_iterator it_end = rows.cend(); @@ -372,6 +356,8 @@ void IndexArray::from_list_all(Mixed value, std::vector& result, const I for (IntegerColumn::const_iterator it = lower; it != upper; ++it) { result.push_back(ObjKey(*it)); } + + return; } @@ -606,7 +592,8 @@ void IndexArray::index_string_all(Mixed value, std::vector& result, cons if (ref & 1) { ObjKey k(int64_t(ref >> 1)); - if (column.full_word() || column.get_value(k) == value) { + Mixed a = column.get_value(k); + if (a == value) { result.push_back(k); return; } @@ -815,18 +802,11 @@ void StringIndex::insert_with_offset(ObjKey obj_key, StringData index_data, cons void StringIndex::insert_to_existing_list_at_lower(ObjKey key, Mixed value, IntegerColumn& list, const IntegerColumnIterator& lower) { + SortedListComparator slc(m_target_column); // At this point there exists duplicates of this value, we need to // insert value beside it's duplicates so that rows are also sorted // in ascending order. - IntegerColumn::const_iterator upper = [&]() { - if (m_target_column.full_word()) { - return list.cend(); - } - else { - SortedListComparator slc(m_target_column); - return std::upper_bound(lower, list.cend(), value, slc); - } - }(); + IntegerColumn::const_iterator upper = std::upper_bound(lower, list.cend(), value, slc); // find insert position (the list has to be kept in sorted order) // In most cases the refs will be added to the end. So we test for that // first to see if we can avoid the binary search for insert position @@ -837,9 +817,7 @@ void StringIndex::insert_to_existing_list_at_lower(ObjKey key, Mixed value, Inte } else { IntegerColumn::const_iterator inner_lower = std::lower_bound(lower, upper, key.value); - if (*inner_lower != key.value) { - list.insert(inner_lower.get_position(), key.value); - } + list.insert(inner_lower.get_position(), key.value); } } @@ -1142,7 +1120,7 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin // When key is outside current range, we can just add it keys.add(key); - if (!m_target_column.full_word() || is_at_string_end) { + if (!m_target_column.is_fulltext() || is_at_string_end) { int64_t shifted = int64_t((uint64_t(obj_key.value) << 1) + 1); // shift to indicate literal m_array->add(shifted); } @@ -1163,7 +1141,7 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin return false; keys.insert(ins_pos, key); - if (!m_target_column.full_word() || is_at_string_end) { + if (!m_target_column.is_fulltext() || is_at_string_end) { int64_t shifted = int64_t((uint64_t(obj_key.value) << 1) + 1); // shift to indicate literal m_array->insert(ins_pos_refs, shifted); } @@ -1184,19 +1162,17 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin // Single match (lowest bit set indicates literal row_ndx) if ((slot_value & 1) != 0) { ObjKey obj_key2 = ObjKey(int64_t(slot_value >> 1)); - Mixed v2 = m_target_column.full_word() ? reconstruct_string(offset, key, index_data) : get(obj_key2); + Mixed v2 = m_target_column.is_fulltext() ? reconstruct_string(offset, key, index_data) : get(obj_key2); if (v2 == value) { - if (obj_key.value != obj_key2.value) { - // Strings are equal but this is not a list. - // Create a list and add both rows. + // Strings are equal but this is not a list. + // Create a list and add both rows. - // convert to list (in sorted order) - Array row_list(alloc); - row_list.create(Array::type_Normal); // Throws - row_list.add(obj_key < obj_key2 ? obj_key.value : obj_key2.value); - row_list.add(obj_key < obj_key2 ? obj_key2.value : obj_key.value); - m_array->set(ins_pos_refs, row_list.get_ref()); - } + // convert to list (in sorted order) + Array row_list(alloc); + row_list.create(Array::type_Normal); // Throws + row_list.add(obj_key < obj_key2 ? obj_key.value : obj_key2.value); + row_list.add(obj_key < obj_key2 ? obj_key2.value : obj_key.value); + m_array->set(ins_pos_refs, row_list.get_ref()); } else { StringConversionBuffer buffer; @@ -1237,8 +1213,7 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin IntegerColumn::const_iterator lower = it_end; auto value_exists_in_list = [&]() { - if (m_target_column.full_word()) { - lower = sub.cbegin(); + if (m_target_column.is_fulltext()) { return reconstruct_string(offset, key, index_data) == value.get_string(); } SortedListComparator slc(m_target_column); @@ -1265,15 +1240,15 @@ bool StringIndex::leaf_insert(ObjKey obj_key, key_type key, size_t offset, Strin // point and insert into the existing list. ObjKey key_of_any_dup = ObjKey(sub.get(0)); StringConversionBuffer buffer; - StringData index_data_2 = m_target_column.full_word() ? reconstruct_string(offset, key, index_data) - : get(key_of_any_dup).get_index_data(buffer); + StringData index_data_2 = m_target_column.is_fulltext() ? reconstruct_string(offset, key, index_data) + : get(key_of_any_dup).get_index_data(buffer); if (index_data == index_data_2 || suboffset > s_max_offset) { insert_to_existing_list(obj_key, value, sub); } else { #ifdef REALM_DEBUG bool contains_only_duplicates = true; - if (!m_target_column.full_word() && sub.size() > 1) { + if (!m_target_column.is_fulltext() && sub.size() > 1) { ObjKey first_key = ObjKey(sub.get(0)); ObjKey last_key = ObjKey(sub.back()); auto first = get(first_key); @@ -1312,37 +1287,15 @@ Mixed StringIndex::get(ObjKey key) const void StringIndex::erase(ObjKey key) { StringConversionBuffer buffer; - if (m_target_column.full_word()) { - if (m_target_column.tokenize()) { - // This is a full text index - auto index_data(get(key).get_index_data(buffer)); - auto words = Tokenizer::get_instance()->reset(std::string_view(index_data)).get_all_tokens(); - for (auto& w : words) { - erase_string(key, w); - } - } - else { - // This is a list (of strings) - erase_list(key, m_target_column.get_list(key)); + std::string_view value{(get(key).get_index_data(buffer))}; + if (m_target_column.is_fulltext()) { + auto words = Tokenizer::get_instance()->reset(value).get_all_tokens(); + for (auto& w : words) { + erase_string(key, w); } } else { - erase_string(key, get(key).get_index_data(buffer)); - } -} - -void StringIndex::erase_list(ObjKey key, const Lst& list) -{ - std::vector strings; - strings.reserve(list.size()); - for (auto& val : list) { - strings.push_back(val); - } - - std::sort(strings.begin(), strings.end()); - auto last = std::unique(strings.begin(), strings.end()); - for (auto it = strings.begin(); it != last; ++it) { - erase_string(key, *it); + erase_string(key, value); } } @@ -1706,7 +1659,7 @@ void StringIndex::insert(ObjKey key, StringData value) { StringConversionBuffer buffer; - if (this->m_target_column.tokenize()) { + if (this->m_target_column.is_fulltext()) { auto words = Tokenizer::get_instance()->reset(std::string_view(value)).get_all_tokens(); for (auto& word : words) { @@ -1727,7 +1680,7 @@ void StringIndex::set(ObjKey key, StringData new_value) Mixed old_value = get(key); Mixed new_value2 = Mixed(new_value); - if (this->m_target_column.tokenize()) { + if (this->m_target_column.is_fulltext()) { auto tokenizer = Tokenizer::get_instance(); StringData old_string = old_value.get_index_data(buffer); std::set old_words; @@ -2013,10 +1966,6 @@ void StringIndex::dump_node_structure(const Array& node, std::ostream& out, int } } -void StringIndex::dump_node_structure() const -{ - do_dump_node_structure(std::cout, 0); -} void StringIndex::do_dump_node_structure(std::ostream& out, int level) const { diff --git a/src/realm/index_string.hpp b/src/realm/index_string.hpp index 9d34fd1b497..3e40828af4b 100644 --- a/src/realm/index_string.hpp +++ b/src/realm/index_string.hpp @@ -125,8 +125,7 @@ class ClusterColumn { ClusterColumn(const ClusterTree* cluster_tree, ColKey column_key, IndexType type) : m_cluster_tree(cluster_tree) , m_column_key(column_key) - , m_tokenize(type == IndexType::Fulltext) - , m_full_word(m_tokenize | column_key.is_collection()) + , m_type(type) { } size_t size() const @@ -153,23 +152,17 @@ class ClusterColumn { { return m_column_key.is_nullable(); } - bool tokenize() const + bool is_fulltext() const { - return m_tokenize; - } - bool full_word() const - { - return m_full_word; + return m_type == IndexType::Fulltext; } Mixed get_value(ObjKey key) const; - Lst get_list(ObjKey key) const; std::vector get_all_keys() const; private: const ClusterTree* m_cluster_tree; ColKey m_column_key; - bool m_tokenize; - bool m_full_word; + IndexType m_type; }; class StringIndex { @@ -210,7 +203,7 @@ class StringIndex { bool is_empty() const; bool is_fulltext_index() const { - return this->m_target_column.tokenize(); + return this->m_target_column.is_fulltext(); } template @@ -224,10 +217,6 @@ class StringIndex { void set(ObjKey key, util::Optional new_value); void erase(ObjKey key); - void erase_list(ObjKey key, const Lst&); - // Erase without getting value from parent column (useful when string stored - // does not directly match string in parent, like with full-text indexing) - void erase_string(ObjKey key, StringData value); template ObjKey find_first(T value) const; @@ -248,7 +237,6 @@ class StringIndex { #ifdef REALM_DEBUG template void verify_entries(const ClusterColumn& column) const; - void dump_node_structure() const; void do_dump_node_structure(std::ostream&, int) const; #endif @@ -328,6 +316,9 @@ class StringIndex { bool noextend = false); void node_insert_split(size_t ndx, size_t new_ref); void node_insert(size_t ndx, size_t ref); + // Erase without getting value from parent column (useful when string stored + // does not directly match string in parent, like with full-text indexing) + void erase_string(ObjKey key, StringData value); void do_delete(ObjKey key, StringData, size_t offset); Mixed get(ObjKey key) const; diff --git a/src/realm/list.cpp b/src/realm/list.cpp index ad08694a567..f1e2f93d546 100644 --- a/src/realm/list.cpp +++ b/src/realm/list.cpp @@ -35,7 +35,6 @@ #include "realm/table_view.hpp" #include "realm/group.hpp" #include "realm/replication.hpp" -#include "realm/index_string.hpp" namespace realm { @@ -193,70 +192,6 @@ void Lst::distinct(std::vector& indices, util::Optional sort_or } } -/***************************** Lst ******************************/ - -template <> -void Lst::do_insert(size_t ndx, StringData value) -{ - if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { - // Inserting a value already present is idempotent - index->insert(m_obj.get_key(), value); - } - m_tree->insert(ndx, value); -} - -template <> -void Lst::do_set(size_t ndx, StringData value) -{ - if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { - auto old_value = m_tree->get(ndx); - size_t nb_old = 0; - m_tree->for_all([&](StringData val) { - if (val == old_value) { - nb_old++; - } - return !(nb_old > 1); - }); - - if (nb_old == 1) { - // Remove last one - index->erase_string(m_obj.get_key(), old_value); - } - // Inserting a value already present is idempotent - index->insert(m_obj.get_key(), value); - } - m_tree->set(ndx, value); -} - -template <> -inline void Lst::do_remove(size_t ndx) -{ - if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { - auto old_value = m_tree->get(ndx); - size_t nb_old = 0; - m_tree->for_all([&](StringData val) { - if (val == old_value) { - nb_old++; - } - return !(nb_old > 1); - }); - - if (nb_old == 1) { - index->erase_string(m_obj.get_key(), old_value); - } - } - m_tree->erase(ndx); -} - -template <> -inline void Lst::do_clear() -{ - if (auto index = m_obj.get_table()->get_search_index(m_col_key)) { - index->erase_list(m_obj.get_key(), *this); - } - m_tree->clear(); -} - /********************************* Lst *********************************/ template <> diff --git a/src/realm/list.hpp b/src/realm/list.hpp index f9ad7e1bee1..30eb6996763 100644 --- a/src/realm/list.hpp +++ b/src/realm/list.hpp @@ -290,15 +290,6 @@ class Lst final : public CollectionBaseImpl { T do_get(size_t ndx, const char* msg) const; }; -// Specialization of Lst: -template <> -void Lst::do_insert(size_t, StringData); -template <> -void Lst::do_set(size_t, StringData); -template <> -void Lst::do_remove(size_t); -template <> -void Lst::do_clear(); // Specialization of Lst: template <> void Lst::do_set(size_t, ObjKey); diff --git a/src/realm/object-store/object_store.cpp b/src/realm/object-store/object_store.cpp index db383cd6a32..6c38f2d117c 100644 --- a/src/realm/object-store/object_store.cpp +++ b/src/realm/object-store/object_store.cpp @@ -131,11 +131,8 @@ ColKey add_column(Group& group, Table& table, Property const& property) } } else if (is_array(property.type)) { - auto key = table.add_column_list(to_core_type(property.type & ~PropertyType::Flags), property.name, - is_nullable(property.type)); - if (property.requires_index()) - table.add_search_index(key); - return key; + return table.add_column_list(to_core_type(property.type & ~PropertyType::Flags), property.name, + is_nullable(property.type)); } else if (is_set(property.type)) { return table.add_column_set(to_core_type(property.type & ~PropertyType::Flags), property.name, diff --git a/src/realm/object-store/property.hpp b/src/realm/object-store/property.hpp index 5c2823f14d2..8ec66878f30 100644 --- a/src/realm/object-store/property.hpp +++ b/src/realm/object-store/property.hpp @@ -338,7 +338,7 @@ inline Property::Property(std::string name, PropertyType type, std::string objec inline bool Property::type_is_indexable() const noexcept { - return (!is_collection(type) || (is_array(type) && type == PropertyType::String)) && + return !is_collection(type) && (type == PropertyType::Int || type == PropertyType::Bool || type == PropertyType::Date || type == PropertyType::String || type == PropertyType::ObjectId || type == PropertyType::UUID || type == PropertyType::Mixed); diff --git a/src/realm/query_expression.hpp b/src/realm/query_expression.hpp index 19fe2a92942..b1e70c65123 100644 --- a/src/realm/query_expression.hpp +++ b/src/realm/query_expression.hpp @@ -3028,42 +3028,8 @@ class Columns> : public ColumnsCollection { { return make_subexpr>>(*this); } -}; - -template <> -class Columns> : public ColumnsCollection { -public: - using ColumnsCollection::ColumnsCollection; - using ColumnListBase::m_column_key; - using ColumnListBase::m_link_map; - - std::unique_ptr clone() const override - { - return make_subexpr>>(*this); - } - - bool has_search_index() const final - { - auto target_table = m_link_map.get_target_table(); - return target_table->search_index_type(m_column_key) == IndexType::General; - } - - std::vector find_all(Mixed value) const final - { - std::vector ret; - std::vector result; - - StringIndex* index = m_link_map.get_target_table()->get_search_index(m_column_key); - REALM_ASSERT(index); - index->find_all(result, value); - - for (ObjKey k : result) { - auto ndxs = m_link_map.get_origin_ndxs(k); - ret.insert(ret.end(), ndxs.begin(), ndxs.end()); - } - - return ret; - } + friend class Table; + friend class LinkChain; }; template diff --git a/src/realm/table.cpp b/src/realm/table.cpp index 76fdf448fa0..f1a53e03c4a 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -770,16 +770,8 @@ void Table::populate_search_index(ColKey col_key) } } else if (type == type_String) { - if (col_key.is_list()) { - auto list = o.get_list(col_key); - for (auto& s : list) { - index->insert(key, s); // Throws - } - } - else { - StringData value = o.get(col_key); - index->insert(key, value); // Throws - } + StringData value = o.get(col_key); + index->insert(key, value); // Throws } else if (type == type_Timestamp) { Timestamp value = o.get(col_key); @@ -848,8 +840,6 @@ void Table::update_indexes(ObjKey key, const FieldValues& values) if (auto&& index = m_index_accessors[column_ndx]) { // There is an index for this column auto col_key = m_leaf_ndx2colkey[column_ndx]; - if (col_key.is_collection()) - continue; auto type = col_key.get_type(); auto attr = col_key.get_attrs(); bool nullable = attr.test(col_attr_Nullable); @@ -929,8 +919,7 @@ void Table::do_add_search_index(ColKey col_key, IndexType type) if (m_index_accessors[column_ndx] != nullptr) return; - if (!StringIndex::type_supported(DataType(col_key.get_type())) || - (col_key.is_collection() && !(col_key.is_list() && col_key.get_type() == col_type_String)) || + if (!StringIndex::type_supported(DataType(col_key.get_type())) || col_key.is_collection() || (type == IndexType::Fulltext && col_key.get_type() != col_type_String)) { // Not ideal, but this is what we used to throw, so keep throwing that for compatibility reasons, even though // it should probably be a type mismatch exception instead. diff --git a/test/object-store/primitive_list.cpp b/test/object-store/primitive_list.cpp index b45a37dd358..05a0a6627aa 100644 --- a/test/object-store/primitive_list.cpp +++ b/test/object-store/primitive_list.cpp @@ -989,62 +989,3 @@ TEST_CASE("list of mixed links", "[primitives]") { } } } - -TEST_CASE("list of strings - with index", "[primitives]") { - InMemoryTestFile config; - config.cache = false; - config.automatic_change_notifications = false; - config.schema = Schema{ - {"object", - {{"strings", PropertyType::Array | PropertyType::String, Property::IsPrimary{false}, - Property::IsIndexed{true}}}}, - }; - - auto r = Realm::get_shared_realm(config); - - auto table = r->read_group().get_table("class_object"); - ColKey col = table->get_column_key("strings"); - Results has_banana(r, table->query("strings = 'Banana'")); - Results has_pear(r, table->query("strings = 'Pear'")); - - auto write = [&](auto&& fn) { - r->begin_transaction(); - fn(); - r->commit_transaction(); - }; - - r->begin_transaction(); - Obj obj = table->create_object(); - List list(r, obj, col); - r->commit_transaction(); - - write([&] { - list.add(StringData("Banana")); - list.add(StringData("Apple")); - list.add(StringData("Orange")); - }); - - CHECK(has_banana.size() == 1); - CHECK(has_pear.size() == 0); - - write([&] { - list.set(0, StringData("Pear")); // Add Pear - remove banana - list.add(StringData("Pear")); // Already there - list.add(StringData("Grape")); // Add - }); - CHECK(has_banana.size() == 0); - CHECK(has_pear.size() == 1); - - write([&] { - list.set(1, StringData("Orange")); // Already Orange - remove Apple - list.set(3, StringData("Banana")); // Add Banana - keep Pear - }); - CHECK(has_banana.size() == 1); - CHECK(has_pear.size() == 1); - - write([&] { - list.set(2, StringData("Banana")); // No change in index - }); - CHECK(has_banana.size() == 1); - CHECK(has_pear.size() == 1); -} diff --git a/test/test_index_string.cpp b/test/test_index_string.cpp index 1e07d9bbc24..631d68b42ff 100644 --- a/test/test_index_string.cpp +++ b/test/test_index_string.cpp @@ -1900,123 +1900,4 @@ TEST(Unicode_Casemap) CHECK_EQUAL(*out, inp); } } - -static std::string random_string(std::string::size_type length) -{ - static auto& chrs = "0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - thread_local static std::mt19937 rg{std::random_device{}()}; - thread_local static std::uniform_int_distribution pick(0, sizeof(chrs) - 2); - - std::string s; - - s.reserve(length); - - while (length--) - s += chrs[pick(rg)]; - - return s; -} - -TEST(StringIndex_ListOfRandomStrings) -{ - using namespace std::chrono; - - SHARED_GROUP_TEST_PATH(path); - auto db = DB::create(path); - auto wt = db->start_write(); - - auto t = wt->add_table_with_primary_key("foo", type_Int, "_id"); - ColKey col_codes = t->add_column_list(type_String, "codes"); - std::string some_string; - - for (size_t i = 0; i < 10000; i++) { - auto obj = t->create_object_with_primary_key(int64_t(i)); - auto list = obj.get_list(col_codes); - for (size_t j = 0; j < 3; j++) { - std::string str(random_string(14)); - if (i == 5000 && j == 0) { - some_string = str; - } - list.add(StringData(str)); - } - } - - std::vector arguments{Mixed(some_string)}; - auto q = wt->get_table("foo")->query("codes = $0", arguments); - // auto t1 = steady_clock::now(); - auto tv = q.find_all(); - // auto t2 = steady_clock::now(); - // std::cout << "time without index: " << duration_cast(t2 - t1).count() << " us" << std::endl; - CHECK_EQUAL(tv.size(), 1); - t->add_search_index(col_codes); - - // t1 = steady_clock::now(); - tv = q.find_all(); - // t2 = steady_clock::now(); - // std::cout << "time with index: " << duration_cast(t2 - t1).count() << " us" << std::endl; - CHECK_EQUAL(tv.size(), 1); - t->add_search_index(col_codes); - - // std::cout << tv.get_object(0).get("_id") << std::endl; -} - -TEST_TYPES(StringIndex_ListOfStrings, std::true_type, std::false_type) -{ - constexpr bool add_index = TEST_TYPE::value; - Group g; - - auto t = g.add_table("foo"); - ColKey col = t->add_column_list(type_String, "names", true); - if constexpr (add_index) { - t->add_search_index(col); - } - - auto obj1 = t->create_object(); - auto obj2 = t->create_object(); - auto obj3 = t->create_object(); - - for (Obj* obj : {&obj2, &obj3}) { - auto list = obj->get_list(col); - list.add("Johnny"); - list.add("John"); - } - - auto list = obj1.get_list(col); - list.add("Johnny"); - list.add("John"); - list.add("Ivan"); - list.add("Ivan"); - list.add(StringData()); - - CHECK_EQUAL(t->query(R"(names = "John")").count(), 3); - CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 3); - CHECK_EQUAL(t->query(R"(names = NULL)").count(), 1); - - list.set(0, "Paul"); - CHECK_EQUAL(t->query(R"(names = "John")").count(), 3); - CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 2); - CHECK_EQUAL(t->query(R"(names = "Paul")").count(), 1); - - list.remove(1); - CHECK_EQUAL(t->query(R"(names = "John")").count(), 2); - CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 2); - CHECK_EQUAL(t->query(R"(names = "Paul")").count(), 1); - CHECK_EQUAL(t->query(R"(names = "Ivan")").count(), 1); - - list.clear(); - CHECK_EQUAL(t->query(R"(names = "John")").count(), 2); - CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 2); - CHECK_EQUAL(t->query(R"(names = "Paul")").count(), 0); - - list = obj2.get_list(col); - list.insert(0, "Adam"); - list.insert(0, "Adam"); - obj2.remove(); - CHECK_EQUAL(t->query(R"(names = "John")").count(), 1); - CHECK_EQUAL(t->query(R"(names = "Johnny")").count(), 1); -} - #endif // TEST_INDEX_STRING diff --git a/test/test_parser.cpp b/test/test_parser.cpp index e53cb88844b..843f965e60b 100644 --- a/test/test_parser.cpp +++ b/test/test_parser.cpp @@ -2222,16 +2222,14 @@ TEST(Parser_list_of_primitive_ints) CHECK_EQUAL(message, "Cannot convert 'string' to a number"); } -TEST_TYPES(Parser_list_of_primitive_strings, std::true_type, std::false_type) +TEST(Parser_list_of_primitive_strings) { Group g; TableRef t = g.add_table("table"); constexpr bool nullable = true; auto col_str_list = t->add_column_list(type_String, "strings", nullable); - if constexpr (TEST_TYPE::value) { - t->add_search_index(col_str_list); - } + CHECK_THROW_ANY(t->add_search_index(col_str_list)); auto get_string = [](size_t i) -> std::string { return util::format("string_%1", i);