From c9d2662835347c2860cab92bdd448763a462478b Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Fri, 19 Apr 2019 14:00:19 -0700 Subject: [PATCH 1/3] Added delete tests into correctness --- test/correctness/document-correctness.py | 98 ++++++++++++++++++++---- test/correctness/mongo_model.py | 25 ++++++ 2 files changed, 108 insertions(+), 15 deletions(-) diff --git a/test/correctness/document-correctness.py b/test/correctness/document-correctness.py index c53fcc3..0a285b9 100755 --- a/test/correctness/document-correctness.py +++ b/test/correctness/document-correctness.py @@ -277,6 +277,62 @@ def test_update(collection1, collection2, verbose=False): return (True, False) +def test_delete_many(collection1, collection2, verbose=False): + for i in range(1, 10): + exceptionOne = None + exceptionTwo = None + query = gen.random_query(gen.global_prng.random()) + + util.trace('debug', '\n========== Delete_many No.', i, '==========') + util.trace('debug', 'Query:', query) + util.trace('debug', 'Number results from collection: ', gen.count_query_results( + collection1, query)) + for item in collection1.find(query): + util.trace('debug', 'Find Result1:', item) + for item in collection2.find(query): + util.trace('debug', 'Find Result2:', item) + + try: + if verbose: + all = [x for x in collection1.find(dict())] + for item in collection1.find(query): + print '[{}] Before delete doc:{}'.format(type(collection1), item) + print 'Before delete collection1 size: ', len(all) + collection1.delete_many(query) + except pymongo.errors.OperationFailure as e: + exceptionOne = e + except MongoModelException as e: + exceptionOne = e + try: + if verbose: + all = [x for x in collection2.find(dict())] + for item in collection2.find(query): + print '[{}]Before delete doc:{}'.format(type(collection2), item) + print 'Before delete collection2 size: ', len(all) + collection2.delete_many(query) + except pymongo.errors.OperationFailure as e: + exceptionTwo = e + except MongoModelException as e: + exceptionTwo = e + + if (exceptionOne is None and exceptionTwo is None): + # happy case, proceed to consistency check + pass + elif exceptionOne is not None and exceptionTwo is not None and exceptionOne.code == exceptionTwo.code: + # TODO re-enable consistency check when failure happened + return (True, True) + else: + print 'Unmatched result: ' + print type(exceptionOne), ': ', str(exceptionOne) + print type(exceptionTwo), ': ', str(exceptionTwo) + ignored_exception_check(exceptionOne) + ignored_exception_check(exceptionTwo) + return (False, False) + + if not check_query(dict(), collection1, collection2): + return (False, False) + + return (True, False) class IgnoredException(Exception): def __init__(self, message): @@ -289,10 +345,11 @@ def ignored_exception_check(e): def one_iteration(collection1, collection2, ns, seed): - update_tests_enabled = ns['no_updates'] + update_tests_enabled = not ns['no_updates'] + delete_many_tests_enabled = not ns['no_delete_many'] sorting_tests_enabled = gen.generator_options.allow_sorts - indexes_enabled = ns['no_indexes'] - projections_enabled = ns['no_projections'] + indexes_enabled = not ns['no_indexes'] + projections_enabled = not ns['no_projections'] verbose = ns['verbose'] num_doc = ns['num_doc'] fname = "unknown" @@ -406,6 +463,15 @@ def _run_operation_(op1, op2): if not okay: return (okay, fname, None) + if delete_many_tests_enabled: + okay, skip_current_iteration = test_delete_many(collection1, collection2, verbose) + if skip_current_iteration: + if verbose: + print "Skipping current iteration due to the failure from update." + return (True, fname, None) + if not okay: + return (okay, fname, None) + if update_tests_enabled: okay, skip_current_iteration = test_update(collection1, collection2, verbose) if skip_current_iteration: @@ -507,10 +573,10 @@ def test_forever(ns): def start_forever_test(ns): - gen.generator_options.test_nulls = ns['no_nulls'] - gen.generator_options.upserts_enabled = ns['no_upserts'] - gen.generator_options.numeric_fieldnames = ns['no_numeric_fieldnames'] - gen.generator_options.allow_sorts = ns['no_sort'] + gen.generator_options.test_nulls = not ns['no_nulls'] + gen.generator_options.upserts_enabled = not ns['no_upserts'] + gen.generator_options.numeric_fieldnames = not ns['no_numeric_fieldnames'] + gen.generator_options.allow_sorts = not ns['no_sort'] util.weaken_tests(ns) @@ -540,6 +606,7 @@ def tester_thread(c1, c2): collection2=c2, seed=random.random(), update_tests_enabled=True, + delete_many_tests_enabled=True, sorting_tests_enabled=True, indexes_enabled=False, projections_enabled=True, @@ -598,22 +665,23 @@ def tester_thread(c1, c2): parser_forever.add_argument('2', choices=['mongo', 'mm', 'doclayer'], help='second tester') parser_forever.add_argument( '-s', '--seed', type=int, default=random.randint(0, sys.maxint), help='random seed to use') - parser_forever.add_argument('--no-updates', default=True, action='store_false', help='disable update tests') + parser_forever.add_argument('--no-updates', default=False, action='store_true', help='disable update tests') + parser_forever.add_argument('--no-delete-many', default=False, action='store_true', help='disable delete_many tests') parser_forever.add_argument( - '--no-sort', default=True, action='store_false', help='disable non-deterministic sort tests') + '--no-sort', default=False, action='store_true', help='disable non-deterministic sort tests') parser_forever.add_argument( '--no-numeric-fieldnames', - default=True, - action='store_false', + default=False, + action='store_true', help='disable use of numeric fieldnames in subobjects') parser_forever.add_argument( - '--no-nulls', default=True, action='store_false', help='disable generation of null values') + '--no-nulls', default=False, action='store_true', help='disable generation of null values') parser_forever.add_argument( - '--no-upserts', default=True, action='store_false', help='disable operator-operator upserts in update tests') + '--no-upserts', default=False, action='store_true', help='disable operator-operator upserts in update tests') parser_forever.add_argument( - '--no-indexes', default=True, action='store_false', help='disable generation of random indexes') + '--no-indexes', default=False, action='store_true', help='disable generation of random indexes') parser_forever.add_argument( - '--no-projections', default=True, action='store_false', help='disable generation of random query projections') + '--no-projections', default=False, action='store_true', help='disable generation of random query projections') parser_forever.add_argument('--num-doc', type=int, default=300, help='number of documents in the collection') parser_forever.add_argument('--buggify', default=False, action='store_true', help='enable buggification') parser_forever.add_argument('--num-iter', type=int, default=0, help='number of iterations of this type of test') diff --git a/test/correctness/mongo_model.py b/test/correctness/mongo_model.py index 3fabb79..b8e137d 100644 --- a/test/correctness/mongo_model.py +++ b/test/correctness/mongo_model.py @@ -1112,6 +1112,31 @@ def transform_operator_query_to_upsert(self, selector): raise MongoModelException("bad query!") return selector + def delete_one(self, query): + """Delete one document that matches the query. Notice when comparing the result of this with the result from + document layer, user may see difference because MongoModel does not use indexes when doing query, while + document layer does and thus document layer might delete a different doc here. + """ + if len(query) == 0: + for k in self.data.keys(): + del self.data[k] + return + queryKey = query.keys()[0] + for k, item in self.data.iteritems(): + if evaluate(queryKey, query[queryKey], item, self.options): + del self.data[k] + return + + def delete_many(self, query): + if len(query) == 0: + self.data = SortedDict() + return + queryKey = query.keys()[0] + for k, item in self.data.iteritems(): + if evaluate(queryKey, query[queryKey], item, self.options): + del self.data[k] + + def update(self, query, update, upsert, multi): isOperatorUpdate = self.has_operator_expressions(update) if not isOperatorUpdate and multi: From 51fa86261ee81deecfcd2f3ccc605d3d04426ed7 Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Mon, 22 Apr 2019 16:21:57 -0700 Subject: [PATCH 2/3] The command generated now handles the new semantics used in main --- test/correctness/util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/correctness/util.py b/test/correctness/util.py index 34c77cc..72e7172 100644 --- a/test/correctness/util.py +++ b/test/correctness/util.py @@ -1176,13 +1176,14 @@ def command_line_str(ns, seed): cmd_line = cmd_line + " --seed " + str(seed) cmd_line = cmd_line + " --num-doc " + str(ns['num_doc']) cmd_line = cmd_line + " --num-iter " + str(ns['num_iter']) - cmd_line = cmd_line + ("" if ns['no_updates'] else " --no-update") + cmd_line = cmd_line + (" --no-update" if ns['no_updates'] else "") cmd_line = cmd_line + ("" if gen.generator_options.allow_sorts else " --no-sort") cmd_line = cmd_line + ("" if gen.generator_options.numeric_fieldnames else " --no-numeric-fieldnames") cmd_line = cmd_line + ("" if gen.generator_options.test_nulls else " --no-nulls") cmd_line = cmd_line + ("" if gen.generator_options.upserts_enabled else " --no-upserts") - cmd_line = cmd_line + ("" if ns['no_indexes'] else " --no-indexes") - cmd_line = cmd_line + ("" if ns['no_projections'] else " --no-projections") + cmd_line = cmd_line + (" --no-indexes" if ns['no_indexes'] else "") + cmd_line = cmd_line + (" --no-projections" if ns['no_projections'] else "") + cmd_line = cmd_line + (" --no-delete-many" if ns['no_delete_many'] else "") cmd_line = cmd_line + ("" if instance_id == 0 else " --instance-id " + str(instance_id)) return cmd_line + '\n' From 0b6ff9c35f768a8262e379c8ae2c67af2489c6bf Mon Sep 17 00:00:00 2001 From: Xin Dong Date: Wed, 24 Apr 2019 10:55:15 -0700 Subject: [PATCH 3/3] Re-enabled the None value generating --- test/correctness/gen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/correctness/gen.py b/test/correctness/gen.py index 0bd8fe2..db640af 100644 --- a/test/correctness/gen.py +++ b/test/correctness/gen.py @@ -183,9 +183,9 @@ def random_id_value(): def random_value(): r = global_prng.random() while True: - # if (r < 0.1) and generator_options.test_nulls: - # val = None - if (r < 0.2): + if (r < 0.1) and generator_options.test_nulls: + val = None + elif (r < 0.2): val = random_float() elif (r < 0.4): val = random_string(global_prng.randint(1, 8))