Skip to content

Commit

Permalink
fix several hangs in tests
Browse files Browse the repository at this point in the history 
download/upload/advance may have already happened
by the time the test code gets to the wait_for_upload() line
which results in the test hanging forever because there are no further
changes. A fix is to do a timed wait for the expected state which will
succeed immediately if the same race occurs.
  • Loading branch information
@ironage
ironage committed Nov 1, 2023
1 parent f361c5c commit 3caa441
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 46 deletions.
8 changes: 6 additions & 2 deletions test/object-store/sync/flx_migration.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,8 +441,12 @@ TEST_CASE("Test client migration and rollback with recovery", "[sync][flx][flx m
// Migrate back to FLX - and keep the realm session open
trigger_server_migration(session.app_session(), MigrateToFLX, logger_ptr);

REQUIRE(!wait_for_upload(*outer_realm));
REQUIRE(!wait_for_download(*outer_realm));
// wait for the subscription store to initialize after downloading
timed_wait_for(
[&outer_realm]() {
return outer_realm->sync_session() && outer_realm->sync_session()->get_flx_subscription_store();
},
std::chrono::seconds(180));

// Verify data has been sync'ed and there is only 1 subscription for the Object table
{
Expand Down
98 changes: 54 additions & 44 deletions test/object-store/sync/flx_sync.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,6 @@ TEST_CASE("flx: connect to FLX-enabled app", "[sync][flx][baas]") {


harness.do_with_new_realm([&](SharedRealm realm) {
wait_for_download(*realm);
{
auto empty_subs = realm->get_latest_subscription_set();
CHECK(empty_subs.size() == 0);
Expand All @@ -174,9 +173,13 @@ TEST_CASE("flx: connect to FLX-enabled app", "[sync][flx][baas]") {
subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get();
}

wait_for_download(*realm);
{
wait_for_advance(*realm);
timed_wait_for(
[&]() {
Results results(realm, table);
return results.size() > 0;
},
std::chrono::seconds(60));
Results results(realm, table);
CHECK(results.size() == 1);
auto obj = results.get<Obj>(0);
Expand All @@ -193,11 +196,12 @@ TEST_CASE("flx: connect to FLX-enabled app", "[sync][flx][baas]") {
subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get();
}

{
wait_for_advance(*realm);
Results results(realm, Query(table));
CHECK(results.size() == 2);
}
timed_wait_for(
[&]() {
Results results(realm, Query(table));
return results.size() == 2;
},
std::chrono::seconds(60));

{
auto mut_subs = realm->get_latest_subscription_set().make_mutable_copy();
Expand All @@ -210,7 +214,12 @@ TEST_CASE("flx: connect to FLX-enabled app", "[sync][flx][baas]") {
}

{
wait_for_advance(*realm);
timed_wait_for(
[&]() {
Results results(realm, Query(table));
return results.size() == 1;
},
std::chrono::seconds(60));
Results results(realm, Query(table));
CHECK(results.size() == 1);
auto obj = results.get<Obj>(0);
Expand All @@ -225,11 +234,12 @@ TEST_CASE("flx: connect to FLX-enabled app", "[sync][flx][baas]") {
subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get();
}

{
wait_for_advance(*realm);
Results results(realm, table);
CHECK(results.size() == 0);
}
timed_wait_for(
[&]() {
Results results(realm, Query(table));
return results.size() == 0;
},
std::chrono::seconds(60));
});
}

Expand Down Expand Up @@ -2014,7 +2024,7 @@ TEST_CASE("flx: geospatial", "[sync][flx][geospatial][baas]") {
#endif // REALM_ENABLE_GEOSPATIAL

TEST_CASE("flx: interrupted bootstrap restarts/recovers on reconnect", "[sync][flx][bootstrap][baas]") {
FLXSyncTestHarness harness("flx_bootstrap_batching", {g_large_array_schema, {"queryable_int_field"}});
FLXSyncTestHarness harness("flx_bootstrap_reconnect", {g_large_array_schema, {"queryable_int_field"}});

std::vector<ObjectId> obj_ids_at_end = fill_large_array_schema(harness);
SyncTestFile interrupted_realm_config(harness.app()->current_user(), harness.schema(),
Expand Down Expand Up @@ -2084,17 +2094,18 @@ TEST_CASE("flx: interrupted bootstrap restarts/recovers on reconnect", "[sync][f
}

auto realm = Realm::get_shared_realm(interrupted_realm_config);
auto table = realm->read_group().get_table("class_TopLevel");
realm->get_latest_subscription_set().get_state_change_notification(sync::SubscriptionSet::State::Complete).get();
wait_for_upload(*realm);
wait_for_download(*realm);

wait_for_advance(*realm);
REQUIRE(table->size() == obj_ids_at_end.size());
for (auto& id : obj_ids_at_end) {
REQUIRE(table->find_primary_key(Mixed{id}));
}
timed_wait_for(
[&]() -> bool {
auto table = realm->read_group().get_table("class_TopLevel");
return table->size() == obj_ids_at_end.size() &&
std::all_of(obj_ids_at_end.begin(), obj_ids_at_end.end(), [&table](auto pk) {
return bool(table->find_primary_key(Mixed{pk}));
});
},
std::chrono::seconds(120));

realm->get_latest_subscription_set().get_state_change_notification(sync::SubscriptionSet::State::Complete).get();
auto active_subs = realm->get_active_subscription_set();
auto latest_subs = realm->get_latest_subscription_set();
REQUIRE(active_subs.version() == latest_subs.version());
Expand Down Expand Up @@ -2705,16 +2716,16 @@ TEST_CASE("flx: bootstrap batching prevents orphan documents", "[sync][flx][boot
realm->get_latest_subscription_set()
.get_state_change_notification(sync::SubscriptionSet::State::Complete)
.get();
wait_for_upload(*realm);
wait_for_download(*realm);

wait_for_advance(*realm);
auto expected_obj_ids = util::Span<ObjectId>(obj_ids_at_end).sub_span(0, 3);

REQUIRE(table->size() == expected_obj_ids.size());
for (auto& id : expected_obj_ids) {
REQUIRE(table->find_primary_key(Mixed{id}));
}
timed_wait_for(
[&]() {
return table->size() == expected_obj_ids.size() &&
std::all_of(expected_obj_ids.begin(), expected_obj_ids.end(), [&table](auto pk) {
return bool(table->find_primary_key(Mixed{pk}));
});
},
std::chrono::seconds(60));
}

SECTION("interrupted after final bootstrap message before processing") {
Expand Down Expand Up @@ -2828,17 +2839,16 @@ TEST_CASE("flx: bootstrap batching prevents orphan documents", "[sync][flx][boot
realm->get_latest_subscription_set()
.get_state_change_notification(sync::SubscriptionSet::State::Complete)
.get();
wait_for_upload(*realm);
wait_for_download(*realm);
wait_for_advance(*realm);

auto expected_obj_ids = util::Span<ObjectId>(obj_ids_at_end).sub_span(0, 3);

// After we've downloaded all the mutations there should only by 3 objects left.
REQUIRE(table->size() == expected_obj_ids.size());
for (auto& id : expected_obj_ids) {
REQUIRE(table->find_primary_key(Mixed{id}));
}
timed_wait_for(
[&]() {
return table->size() == expected_obj_ids.size() &&
std::all_of(expected_obj_ids.begin(), expected_obj_ids.end(), [&table](auto pk) {
return bool(table->find_primary_key(Mixed{pk}));
});
},
std::chrono::seconds(60));
}
}

Expand Down Expand Up @@ -3900,7 +3910,7 @@ TEST_CASE("flx: compensating write errors get re-sent across sessions", "[sync][
}

TEST_CASE("flx: bootstrap changesets are applied continuously", "[sync][flx][bootstrap][baas]") {
FLXSyncTestHarness harness("flx_bootstrap_batching", {g_large_array_schema, {"queryable_int_field"}});
FLXSyncTestHarness harness("flx_bootstrap_ordering", {g_large_array_schema, {"queryable_int_field"}});
fill_large_array_schema(harness);

std::unique_ptr<std::thread> th;
Expand Down Expand Up @@ -3983,7 +3993,7 @@ TEST_CASE("flx: bootstrap changesets are applied continuously", "[sync][flx][boo

TEST_CASE("flx: open realm + register subscription callback while bootstrapping",
"[sync][flx][bootstrap][async open][baas]") {
FLXSyncTestHarness harness("flx_bootstrap_batching");
FLXSyncTestHarness harness("flx_bootstrap_and_subscribe");
auto foo_obj_id = ObjectId::gen();
harness.load_initial_data([&](SharedRealm realm) {
CppContext c(realm);
Expand Down Expand Up @@ -4250,7 +4260,7 @@ TEST_CASE("flx: open realm + register subscription callback while bootstrapping"
}
}
TEST_CASE("flx sync: Client reset during async open", "[sync][flx][client reset][async open][baas]") {
FLXSyncTestHarness harness("flx_bootstrap_batching");
FLXSyncTestHarness harness("flx_bootstrap_reset");
auto foo_obj_id = ObjectId::gen();
std::atomic<bool> subscription_invoked = false;
harness.load_initial_data([&](SharedRealm realm) {
Expand Down

0 comments on commit 3caa441

Please sign in to comment.