From 68af99f5d592fcb1e11096bf1423d9e725188cd1 Mon Sep 17 00:00:00 2001 From: marshtompsxd Date: Sat, 11 Sep 2021 23:49:23 -0500 Subject: [PATCH] format with black --- analyze.py | 186 +++++++++++----- analyze_event.py | 36 ++- analyze_gen.py | 98 +++++--- analyze_sql.py | 54 ++++- analyze_util.py | 56 +++-- build.py | 255 +++++++++++++++------ check_env.py | 76 +++++-- common.py | 30 ++- controllers.py | 214 +++++++++++++----- eva-learn.py | 47 ++-- oracle.py | 345 +++++++++++++++++++++-------- sieve.py | 553 +++++++++++++++++++++++++++++++++++----------- sieve_config.py | 1 - sieve_tui.py | 154 +++++++------ test_framework.py | 113 +++++++--- workloads.py | 277 +++++++++++++++++------ 16 files changed, 1798 insertions(+), 697 deletions(-) diff --git a/analyze.py b/analyze.py index 4465e911cab..6d439f11bbd 100644 --- a/analyze.py +++ b/analyze.py @@ -47,7 +47,9 @@ def sanity_check_sieve_log(path): for key in event_status: assert event_status[key] == 1 or event_status[key] == 2 for key in reconcile_status: - assert reconcile_status[reconcile_id] == 0 or reconcile_status[reconcile_id] == 1 + assert ( + reconcile_status[reconcile_id] == 0 or reconcile_status[reconcile_id] == 1 + ) def parse_events(path): @@ -150,11 +152,16 @@ def parse_side_effects(path, compress_trivial_reconcile=True): if controller_name not in cur_reconcile_start_timestamp: cur_reconcile_start_timestamp[controller_name] = -1 cur_reconcile_is_trivial[controller_name] = False - if (not compress_trivial_reconcile) or (compress_trivial_reconcile and not cur_reconcile_is_trivial[controller_name]): + if (not compress_trivial_reconcile) or ( + compress_trivial_reconcile + and not cur_reconcile_is_trivial[controller_name] + ): # When compress_trivial_reconcile is disabled, we directly set prev_reconcile_start_timestamp # When compress_trivial_reconcile is enabled, we do not set prev_reconcile_start_timestamp # if no side effects happen during the last reconcile - prev_reconcile_start_timestamp[controller_name] = cur_reconcile_start_timestamp[controller_name] + prev_reconcile_start_timestamp[ + controller_name + ] = cur_reconcile_start_timestamp[controller_name] cur_reconcile_start_timestamp[controller_name] = i # Reset cur_reconcile_is_trivial[controller_name] to True as a new round of reconcile just starts cur_reconcile_is_trivial[controller_name] = True @@ -179,16 +186,19 @@ def parse_side_effects(path, compress_trivial_reconcile=True): def extract_events_and_effects(path, compress_trivial_reconcile): event_list, event_key_map, event_id_map = parse_events(path) - events_data_structure = EventsDataStructure( - event_list, event_key_map, event_id_map) + events_data_structure = EventsDataStructure(event_list, event_key_map, event_id_map) side_effect_list, side_effect_id_map = parse_side_effects( - path, compress_trivial_reconcile) + path, compress_trivial_reconcile + ) side_effects_data_structure = SideEffectsDataStructure( - side_effect_list, side_effect_id_map) + side_effect_list, side_effect_id_map + ) return events_data_structure, side_effects_data_structure -def base_pass(event_vertices: List[CausalityVertex], side_effect_vertices: List[CausalityVertex]): +def base_pass( + event_vertices: List[CausalityVertex], side_effect_vertices: List[CausalityVertex] +): print("Running base pass ...") vertex_pairs = [] for side_effect_vertex in side_effect_vertices: @@ -207,8 +217,7 @@ def write_read_overlap_filtering_pass(vertex_pairs: List[List[CausalityVertex]]) side_effect_vertex = pair[1] if side_effect_vertex.content.interest_overlap(event_vertex.content): pruned_vertex_pairs.append(pair) - print(" pairs: %d -> %d" % - (len(vertex_pairs), len(pruned_vertex_pairs))) + print(" pairs: %d -> %d" % (len(vertex_pairs), len(pruned_vertex_pairs))) return pruned_vertex_pairs @@ -219,43 +228,53 @@ def error_msg_filtering_pass(vertex_pairs: List[List[CausalityVertex]]): side_effect_vertex = pair[1] if side_effect_vertex.content.error in ALLOWED_ERROR_TYPE: pruned_vertex_pairs.append(pair) - print(" pairs: %d -> %d" % - (len(vertex_pairs), len(pruned_vertex_pairs))) + print(" pairs: %d -> %d" % (len(vertex_pairs), len(pruned_vertex_pairs))) return pruned_vertex_pairs -def generate_event_effect_edges(event_vertices: List[CausalityVertex], side_effect_vertices: List[CausalityVertex]): +def generate_event_effect_edges( + event_vertices: List[CausalityVertex], side_effect_vertices: List[CausalityVertex] +): edges = [] vertex_pairs = base_pass(event_vertices, side_effect_vertices) if ERROR_MSG_FILTER_FLAG: - vertex_pairs = error_msg_filtering_pass( - vertex_pairs) + vertex_pairs = error_msg_filtering_pass(vertex_pairs) if WRITE_READ_FILTER_FLAG: - vertex_pairs = write_read_overlap_filtering_pass( - vertex_pairs) + vertex_pairs = write_read_overlap_filtering_pass(vertex_pairs) for pair in vertex_pairs: - edges.append(CausalityEdge( - pair[0], pair[1], INTER_THREAD_EDGE)) + edges.append(CausalityEdge(pair[0], pair[1], INTER_THREAD_EDGE)) return edges -def generate_effect_event_edges(event_vertices: List[CausalityVertex], side_effect_vertices: List[CausalityVertex], event_key_map): +def generate_effect_event_edges( + event_vertices: List[CausalityVertex], + side_effect_vertices: List[CausalityVertex], + event_key_map, +): edges = [] vertex_pairs = [] for side_effect_vertex in side_effect_vertices: if side_effect_vertex.content.key in event_key_map: for event_vertex in event_vertices: - if event_vertex.content.obj_str == side_effect_vertex.content.obj_str and side_effect_vertex.content.start_timestamp < event_vertex.content.start_timestamp and consistent_type(event_vertex.content.etype, side_effect_vertex.content.etype): - vertex_pairs.append( - [side_effect_vertex, event_vertex]) + if ( + event_vertex.content.obj_str == side_effect_vertex.content.obj_str + and side_effect_vertex.content.start_timestamp + < event_vertex.content.start_timestamp + and consistent_type( + event_vertex.content.etype, side_effect_vertex.content.etype + ) + ): + vertex_pairs.append([side_effect_vertex, event_vertex]) break for pair in vertex_pairs: - edges.append(CausalityEdge( - pair[0], pair[1], INTER_THREAD_EDGE)) + edges.append(CausalityEdge(pair[0], pair[1], INTER_THREAD_EDGE)) return edges -def build_causality_graph(events_data_structure: EventsDataStructure, side_effects_data_structure: SideEffectsDataStructure): +def build_causality_graph( + events_data_structure: EventsDataStructure, + side_effects_data_structure: SideEffectsDataStructure, +): causality_graph = CausalityGraph() for event in events_data_structure.event_list: causality_graph.add_vertex(CausalityVertex(event)) @@ -266,9 +285,11 @@ def build_causality_graph(events_data_structure: EventsDataStructure, side_effec side_effect_vertices = causality_graph.get_side_effect_vertex_list() event_effect_edges = generate_event_effect_edges( - event_vertices, side_effect_vertices) + event_vertices, side_effect_vertices + ) effect_event_edges = generate_effect_event_edges( - event_vertices, side_effect_vertices, events_data_structure.event_key_map) + event_vertices, side_effect_vertices, events_data_structure.event_key_map + ) for edge in event_effect_edges: causality_graph.add_edge(edge) @@ -281,51 +302,80 @@ def build_causality_graph(events_data_structure: EventsDataStructure, side_effec return causality_graph -def generate_test_config(analysis_mode, project, log_dir, two_sided, causality_graph, events_data_structure): - generated_config_dir = os.path.join( - log_dir, analysis_mode) +def generate_test_config( + analysis_mode, project, log_dir, two_sided, causality_graph, events_data_structure +): + generated_config_dir = os.path.join(log_dir, analysis_mode) if os.path.isdir(generated_config_dir): shutil.rmtree(generated_config_dir) os.makedirs(generated_config_dir, exist_ok=True) if analysis_mode == sieve_modes.TIME_TRAVEL: analyze_gen.time_travel_analysis( - causality_graph, events_data_structure, generated_config_dir, project) + causality_graph, events_data_structure, generated_config_dir, project + ) if two_sided: analyze_gen.time_travel_analysis( - causality_graph, events_data_structure, generated_config_dir, project, "before") + causality_graph, + events_data_structure, + generated_config_dir, + project, + "before", + ) elif analysis_mode == sieve_modes.OBS_GAP: analyze_gen.obs_gap_analysis( - causality_graph, events_data_structure, generated_config_dir, project) + causality_graph, events_data_structure, generated_config_dir, project + ) elif analysis_mode == sieve_modes.ATOM_VIO: analyze_gen.atom_vio_analysis( - causality_graph, events_data_structure, generated_config_dir, project) - - -def analyze_trace(project, log_dir, generate_oracle=True, generate_config=True, two_sided=False, use_sql=False, compress_trivial_reconcile=True): - print("generate-oracle feature is %s" % - ("enabled" if generate_oracle else "disabled")) - print("generate-config feature is %s" % - ("enabled" if generate_config else "disabled")) + causality_graph, events_data_structure, generated_config_dir, project + ) + + +def analyze_trace( + project, + log_dir, + generate_oracle=True, + generate_config=True, + two_sided=False, + use_sql=False, + compress_trivial_reconcile=True, +): + print( + "generate-oracle feature is %s" % ("enabled" if generate_oracle else "disabled") + ) + print( + "generate-config feature is %s" % ("enabled" if generate_config else "disabled") + ) if not generate_config: two_sided = False use_sql = False - print("two-sided feature is %s" % - ("enabled" if two_sided else "disabled")) - print("use-sql feature is %s" % - ("enabled" if use_sql else "disabled")) + print("two-sided feature is %s" % ("enabled" if two_sided else "disabled")) + print("use-sql feature is %s" % ("enabled" if use_sql else "disabled")) log_path = os.path.join(log_dir, "sieve-server.log") print("Sanity checking the sieve log %s ..." % log_path) sanity_check_sieve_log(log_path) events_data_structure, side_effects_data_structure = extract_events_and_effects( - log_path, compress_trivial_reconcile) + log_path, compress_trivial_reconcile + ) causality_graph = build_causality_graph( - events_data_structure, side_effects_data_structure) + events_data_structure, side_effects_data_structure + ) if generate_config: - for analysis_mode in [sieve_modes.TIME_TRAVEL, sieve_modes.OBS_GAP, sieve_modes.ATOM_VIO]: - generate_test_config(analysis_mode, project, log_dir, - two_sided, causality_graph, events_data_structure) + for analysis_mode in [ + sieve_modes.TIME_TRAVEL, + sieve_modes.OBS_GAP, + sieve_modes.ATOM_VIO, + ]: + generate_test_config( + analysis_mode, + project, + log_dir, + two_sided, + causality_graph, + events_data_structure, + ) if generate_oracle: oracle.generate_test_oracle(log_dir) @@ -334,16 +384,34 @@ def analyze_trace(project, log_dir, generate_oracle=True, generate_config=True, if __name__ == "__main__": usage = "usage: python3 analyze.py [options]" parser = optparse.OptionParser(usage=usage) - parser.add_option("-p", "--project", dest="project", - help="specify PROJECT", metavar="PROJECT", default="cassandra-operator") - parser.add_option("-t", "--test", dest="test", - help="specify TEST to run", metavar="TEST", default="recreate") + parser.add_option( + "-p", + "--project", + dest="project", + help="specify PROJECT", + metavar="PROJECT", + default="cassandra-operator", + ) + parser.add_option( + "-t", + "--test", + dest="test", + help="specify TEST to run", + metavar="TEST", + default="recreate", + ) (options, args) = parser.parse_args() project = options.project test = options.test - print("Analyzing controller trace for %s's test workload %s ..." % - (project, test)) + print("Analyzing controller trace for %s's test workload %s ..." % (project, test)) # hardcoded to time travel config only for now dir = os.path.join("log", project, test, "learn", "learn") - analyze_trace(project, dir, generate_oracle=False, - generate_config=True, two_sided=False, use_sql=False, compress_trivial_reconcile=True) + analyze_trace( + project, + dir, + generate_oracle=False, + generate_config=True, + two_sided=False, + use_sql=False, + compress_trivial_reconcile=True, + ) diff --git a/analyze_event.py b/analyze_event.py index ac65beca4d8..4884037f6b1 100644 --- a/analyze_event.py +++ b/analyze_event.py @@ -12,17 +12,23 @@ def find_previous_event(event, event_map): if i == 0: return None, event_map[key][i] else: - return event_map[key][i-1], event_map[key][i] + return event_map[key][i - 1], event_map[key][i] -def compress_event_object_for_list(prev_object, cur_object, slim_prev_object, slim_cur_object): +def compress_event_object_for_list( + prev_object, cur_object, slim_prev_object, slim_cur_object +): for i in range(min(len(cur_object), len(prev_object))): if str(cur_object[i]) != str(prev_object[i]): if isinstance(cur_object[i], dict): if not isinstance(prev_object[i], dict): continue if compress_event_object( - prev_object[i], cur_object[i], slim_prev_object[i], slim_cur_object[i]): + prev_object[i], + cur_object[i], + slim_prev_object[i], + slim_cur_object[i], + ): # SIEVE_SKIP means we can skip the value in list when later comparing to the events in testing run slim_cur_object[i] = SIEVE_SKIP_MARKER slim_prev_object[i] = SIEVE_SKIP_MARKER @@ -30,7 +36,11 @@ def compress_event_object_for_list(prev_object, cur_object, slim_prev_object, sl if not isinstance(prev_object[i], list): continue if compress_event_object_for_list( - prev_object[i], cur_object[i], slim_prev_object[i], slim_cur_object[i]): + prev_object[i], + cur_object[i], + slim_prev_object[i], + slim_cur_object[i], + ): slim_cur_object[i] = SIEVE_SKIP_MARKER slim_prev_object[i] = SIEVE_SKIP_MARKER else: @@ -60,20 +70,27 @@ def compress_event_object(prev_object, cur_object, slim_prev_object, slim_cur_ob if not isinstance(prev_object[key], dict): continue if compress_event_object( - prev_object[key], cur_object[key], slim_prev_object[key], slim_cur_object[key]): + prev_object[key], + cur_object[key], + slim_prev_object[key], + slim_cur_object[key], + ): to_del.append(key) elif isinstance(cur_object[key], list): if not isinstance(prev_object[key], list): continue if compress_event_object_for_list( - prev_object[key], cur_object[key], slim_prev_object[key], slim_cur_object[key]): + prev_object[key], + cur_object[key], + slim_prev_object[key], + slim_cur_object[key], + ): to_del.append(key) else: continue else: to_del.append(key) - sym_different_keys = set( - cur_object.keys()).symmetric_difference(prev_object.keys()) + sym_different_keys = set(cur_object.keys()).symmetric_difference(prev_object.keys()) for key in sym_different_keys: if key in BORING_EVENT_OBJECT_FIELDS: if key in cur_object.keys(): @@ -105,8 +122,7 @@ def diff_events(prev_event: Event, cur_event: Event): cur_object = cur_event.obj_map slim_prev_object = copy.deepcopy(prev_object) slim_cur_object = copy.deepcopy(cur_object) - compress_event_object(prev_object, cur_object, - slim_prev_object, slim_cur_object) + compress_event_object(prev_object, cur_object, slim_prev_object, slim_cur_object) return slim_prev_object, slim_cur_object diff --git a/analyze_gen.py b/analyze_gen.py index 0fc35d86508..e58ebfd9015 100644 --- a/analyze_gen.py +++ b/analyze_gen.py @@ -21,7 +21,9 @@ def delete_only_filtering_pass(causality_edges: List[CausalityEdge]): return candidate_edges -def delete_then_recreate_filtering_pass(causality_edges: List[CausalityEdge], event_key_map): +def delete_then_recreate_filtering_pass( + causality_edges: List[CausalityEdge], event_key_map +): print("Running optional pass: delete-then-recreate-filtering ...") # this should only be applied to time travel mode candidate_edges = [] @@ -46,27 +48,47 @@ def delete_then_recreate_filtering_pass(causality_edges: List[CausalityEdge], ev return candidate_edges -def time_travel_analysis(causality_graph: CausalityGraph, events_data_structure: EventsDataStructure, path: str, project: str, timing="after"): +def time_travel_analysis( + causality_graph: CausalityGraph, + events_data_structure: EventsDataStructure, + path: str, + project: str, + timing="after", +): causality_edges = causality_graph.get_event_effect_edge_list() candidate_edges = delete_only_filtering_pass(causality_edges) candidate_edges = delete_then_recreate_filtering_pass( - candidate_edges, events_data_structure.event_key_map) + candidate_edges, events_data_structure.event_key_map + ) generate_time_travel_yaml( - candidate_edges, path, project, events_data_structure.event_key_map, timing) + candidate_edges, path, project, events_data_structure.event_key_map, timing + ) -def obs_gap_analysis(causality_graph: CausalityGraph, events_data_structure: EventsDataStructure, path: str, project: str): +def obs_gap_analysis( + causality_graph: CausalityGraph, + events_data_structure: EventsDataStructure, + path: str, + project: str, +): causality_edges = causality_graph.get_event_effect_edge_list() candidate_edges = causality_edges generate_obs_gap_yaml( - candidate_edges, path, project, events_data_structure.event_key_map) + candidate_edges, path, project, events_data_structure.event_key_map + ) -def atom_vio_analysis(causality_graph: CausalityGraph, events_data_structure: EventsDataStructure, path: str, project: str): +def atom_vio_analysis( + causality_graph: CausalityGraph, + events_data_structure: EventsDataStructure, + path: str, + project: str, +): causality_edges = causality_graph.get_event_effect_edge_list() candidate_edges = causality_edges generate_atom_vio_yaml( - candidate_edges, path, project, events_data_structure.event_key_map) + candidate_edges, path, project, events_data_structure.event_key_map + ) def generate_time_travel_yaml(causality_edges, path, project, event_key_map, timing): @@ -87,12 +109,10 @@ def generate_time_travel_yaml(causality_edges, path, project, event_key_map, tim assert isinstance(event, Event) assert isinstance(side_effect, SideEffect) - prev_event, cur_event = analyze_event.find_previous_event( - event, event_key_map) + prev_event, cur_event = analyze_event.find_previous_event(event, event_key_map) if prev_event is None: continue - slim_prev_obj, slim_cur_obj = analyze_event.diff_events( - prev_event, cur_event) + slim_prev_obj, slim_cur_obj = analyze_event.diff_events(prev_event, cur_event) if len(slim_prev_obj) == 0 and len(slim_cur_obj) == 0: continue @@ -101,9 +121,11 @@ def generate_time_travel_yaml(causality_edges, path, project, event_key_map, tim yaml_map["ce-rtype"] = event.rtype yaml_map["ce-diff-current"] = json.dumps( - analyze_event.canonicalize_event(copy.deepcopy(slim_cur_obj))) + analyze_event.canonicalize_event(copy.deepcopy(slim_cur_obj)) + ) yaml_map["ce-diff-previous"] = json.dumps( - analyze_event.canonicalize_event(copy.deepcopy(slim_prev_obj))) + analyze_event.canonicalize_event(copy.deepcopy(slim_prev_obj)) + ) yaml_map["ce-etype-current"] = cur_event.etype yaml_map["ce-etype-previous"] = prev_event.etype @@ -113,8 +135,14 @@ def generate_time_travel_yaml(causality_edges, path, project, event_key_map, tim yaml_map["se-etype"] = "ADDED" if side_effect.etype == "Delete" else "DELETED" i += 1 - yaml.dump(yaml_map, open( - os.path.join(path, "time-travel-config-%s%s.yaml" % (str(i), suffix)), "w"), sort_keys=False) + yaml.dump( + yaml_map, + open( + os.path.join(path, "time-travel-config-%s%s.yaml" % (str(i), suffix)), + "w", + ), + sort_keys=False, + ) print("Generated %d time-travel config(s) in %s" % (i, path)) @@ -137,12 +165,10 @@ def generate_obs_gap_yaml(causality_edges, path, project, event_key_map): else: continue - prev_event, cur_event = analyze_event.find_previous_event( - event, event_key_map) + prev_event, cur_event = analyze_event.find_previous_event(event, event_key_map) if prev_event is None: continue - slim_prev_obj, slim_cur_obj = analyze_event.diff_events( - prev_event, cur_event) + slim_prev_obj, slim_cur_obj = analyze_event.diff_events(prev_event, cur_event) if len(slim_prev_obj) == 0 and len(slim_cur_obj) == 0: continue @@ -151,15 +177,20 @@ def generate_obs_gap_yaml(causality_edges, path, project, event_key_map): yaml_map["ce-rtype"] = event.rtype yaml_map["ce-diff-current"] = json.dumps( - analyze_event.canonicalize_event(copy.deepcopy(slim_cur_obj))) + analyze_event.canonicalize_event(copy.deepcopy(slim_cur_obj)) + ) yaml_map["ce-diff-previous"] = json.dumps( - analyze_event.canonicalize_event(copy.deepcopy(slim_prev_obj))) + analyze_event.canonicalize_event(copy.deepcopy(slim_prev_obj)) + ) yaml_map["ce-etype-current"] = cur_event.etype yaml_map["ce-etype-previous"] = prev_event.etype i += 1 - yaml.dump(yaml_map, open( - os.path.join(path, "obs-gap-config-%s.yaml" % (str(i))), "w"), sort_keys=False) + yaml.dump( + yaml_map, + open(os.path.join(path, "obs-gap-config-%s.yaml" % (str(i))), "w"), + sort_keys=False, + ) print("Generated %d obs-gap config(s) in %s" % (i, path)) @@ -178,12 +209,10 @@ def generate_atom_vio_yaml(causality_edges, path, project, event_key_map): assert isinstance(event, Event) assert isinstance(side_effect, SideEffect) - prev_event, cur_event = analyze_event.find_previous_event( - event, event_key_map) + prev_event, cur_event = analyze_event.find_previous_event(event, event_key_map) if prev_event is None: continue - slim_prev_obj, slim_cur_obj = analyze_event.diff_events( - prev_event, cur_event) + slim_prev_obj, slim_cur_obj = analyze_event.diff_events(prev_event, cur_event) if len(slim_prev_obj) == 0 and len(slim_cur_obj) == 0: continue @@ -192,9 +221,11 @@ def generate_atom_vio_yaml(causality_edges, path, project, event_key_map): yaml_map["ce-rtype"] = event.rtype yaml_map["ce-diff-current"] = json.dumps( - analyze_event.canonicalize_event(copy.deepcopy(slim_cur_obj))) + analyze_event.canonicalize_event(copy.deepcopy(slim_cur_obj)) + ) yaml_map["ce-diff-previous"] = json.dumps( - analyze_event.canonicalize_event(copy.deepcopy(slim_prev_obj))) + analyze_event.canonicalize_event(copy.deepcopy(slim_prev_obj)) + ) yaml_map["ce-etype-current"] = cur_event.etype yaml_map["ce-etype-previous"] = prev_event.etype @@ -206,6 +237,9 @@ def generate_atom_vio_yaml(causality_edges, path, project, event_key_map): # TODO: should find a way to determine crash location yaml_map["crash-location"] = "before" i += 1 - yaml.dump(yaml_map, open( - os.path.join(path, "atomic-config-%s.yaml" % (str(i))), "w"), sort_keys=False) + yaml.dump( + yaml_map, + open(os.path.join(path, "atomic-config-%s.yaml" % (str(i))), "w"), + sort_keys=False, + ) print("Generated %d atomic config(s) in %s" % (i, path)) diff --git a/analyze_sql.py b/analyze_sql.py index e22317e7764..673067ddbba 100644 --- a/analyze_sql.py +++ b/analyze_sql.py @@ -32,7 +32,8 @@ def create_sqlite_db(): # TODO: SQlite3 does not type check by default, but # tighten the column types later - conn.execute(''' + conn.execute( + """ create table events ( id integer not null primary key, @@ -46,8 +47,10 @@ def create_sqlite_db(): event_cache_update_time integer not null, fully_qualified_name text not null ) - ''') - conn.execute(''' + """ + ) + conn.execute( + """ create table side_effects ( id integer not null primary key, @@ -64,7 +67,8 @@ def create_sqlite_db(): end_timestamp integer not null, owner_controllers text not null ) - ''') + """ + ) return conn @@ -72,8 +76,21 @@ def record_event_list_in_sqlite(event_list, conn): for e in event_list: json_form = json.dumps(e.obj) # Skip the first column: Sqlite will use an auto-incrementing ID - conn.execute("insert into events values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (None, e.id, e.etype, e.rtype, json_form, e.namespace, e.name, e.start_timestamp, e.end_timestamp, e.key)) + conn.execute( + "insert into events values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + None, + e.id, + e.etype, + e.rtype, + json_form, + e.namespace, + e.name, + e.start_timestamp, + e.end_timestamp, + e.key, + ), + ) conn.commit() @@ -83,15 +100,34 @@ def record_side_effect_list_in_sqlite(side_effect_list, conn): json_read_keys = json.dumps(list(e.read_keys)) json_owner_controllers = json.dumps(list(e.owner_controllers)) # Skip the first column: Sqlite will use an auto-incrementing ID - conn.execute("insert into side_effects values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (None, e.id, e.etype, e.rtype, e.namespace, e.name, e.error, json_read_types, json_read_keys, e.range_start_timestamp, e.range_end_timestamp, e.end_timestamp, json_owner_controllers)) + conn.execute( + "insert into side_effects values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + None, + e.id, + e.etype, + e.rtype, + e.namespace, + e.name, + e.error, + json_read_types, + json_read_keys, + e.range_start_timestamp, + e.range_end_timestamp, + e.end_timestamp, + json_owner_controllers, + ), + ) conn.commit() def passes_as_sql_query(analysis_mode): query = SQL_BASE_PASS_QUERY first_optional_pass = True - if analyze_util.DELETE_ONLY_FILTER_FLAG and analysis_mode == sieve_modes.TIME_TRAVEL: + if ( + analyze_util.DELETE_ONLY_FILTER_FLAG + and analysis_mode == sieve_modes.TIME_TRAVEL + ): query += " where " if first_optional_pass else " and " query += SQL_DELETE_ONLY_FILTER first_optional_pass = False diff --git a/analyze_util.py b/analyze_util.py index 142e7731722..cb7616626d5 100644 --- a/analyze_util.py +++ b/analyze_util.py @@ -17,8 +17,15 @@ SIEVE_BEFORE_RECONCILE_MARK = "[SIEVE-BEFORE-RECONCILE]" SIEVE_AFTER_RECONCILE_MARK = "[SIEVE-AFTER-RECONCILE]" -BORING_EVENT_OBJECT_FIELDS = ["resourceVersion", "time", - "managedFields", "lastTransitionTime", "generation", "annotations", "deletionGracePeriodSeconds"] +BORING_EVENT_OBJECT_FIELDS = [ + "resourceVersion", + "time", + "managedFields", + "lastTransitionTime", + "generation", + "annotations", + "deletionGracePeriodSeconds", +] SIEVE_SKIP_MARKER = "SIEVE-SKIP" SIEVE_CANONICALIZATION_MARKER = "SIEVE-NON-NIL" @@ -220,7 +227,10 @@ def range_overlap(self, event: Event): assert event.end_timestamp != -1 assert self.range_start_timestamp < self.range_end_timestamp assert event.start_timestamp < event.end_timestamp - return self.range_start_timestamp < event.end_timestamp and self.range_end_timestamp > event.start_timestamp + return ( + self.range_start_timestamp < event.end_timestamp + and self.range_end_timestamp > event.start_timestamp + ) def interest_overlap(self, event: Event): return event.key in self.read_keys or event.rtype in self.read_types @@ -293,33 +303,42 @@ def round_id(self): class EventsDataStructure: - def __init__(self, event_list: List[Event], event_key_map: Dict[str, Event], event_id_map: Dict[int, Event]): + def __init__( + self, + event_list: List[Event], + event_key_map: Dict[str, Event], + event_id_map: Dict[int, Event], + ): self.event_list = event_list self.event_key_map = event_key_map self.event_id_map = event_id_map class SideEffectsDataStructure: - def __init__(self, side_effect_list: List[SideEffect], side_effect_id_map: Dict[int, SideEffect]): + def __init__( + self, + side_effect_list: List[SideEffect], + side_effect_id_map: Dict[int, SideEffect], + ): self.side_effect_list = side_effect_list self.side_effect_id_map = side_effect_id_map def parse_event(line: str) -> Event: assert SIEVE_BEFORE_EVENT_MARK in line - tokens = line[line.find(SIEVE_BEFORE_EVENT_MARK):].strip("\n").split("\t") + tokens = line[line.find(SIEVE_BEFORE_EVENT_MARK) :].strip("\n").split("\t") return Event(tokens[1], tokens[2], tokens[3], tokens[4]) def parse_side_effect(line: str) -> SideEffect: assert SIEVE_AFTER_SIDE_EFFECT_MARK in line - tokens = line[line.find(SIEVE_AFTER_SIDE_EFFECT_MARK) :].strip("\n").split("\t") + tokens = line[line.find(SIEVE_AFTER_SIDE_EFFECT_MARK) :].strip("\n").split("\t") return SideEffect(tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]) def parse_cache_read(line: str) -> CacheRead: assert SIEVE_AFTER_READ_MARK in line - tokens = line[line.find(SIEVE_AFTER_READ_MARK):].strip("\n").split("\t") + tokens = line[line.find(SIEVE_AFTER_READ_MARK) :].strip("\n").split("\t") if tokens[1] == "Get": return CacheRead(tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]) else: @@ -331,34 +350,32 @@ def parse_cache_read(line: str) -> CacheRead: def parse_event_id_only(line: str) -> EventIDOnly: assert SIEVE_AFTER_EVENT_MARK in line or SIEVE_BEFORE_EVENT_MARK in line if SIEVE_AFTER_EVENT_MARK in line: - tokens = line[line.find(SIEVE_AFTER_EVENT_MARK) :].strip("\n").split("\t") + tokens = line[line.find(SIEVE_AFTER_EVENT_MARK) :].strip("\n").split("\t") return EventIDOnly(tokens[1]) else: - tokens = line[line.find(SIEVE_BEFORE_EVENT_MARK) :].strip("\n").split("\t") + tokens = line[line.find(SIEVE_BEFORE_EVENT_MARK) :].strip("\n").split("\t") return EventIDOnly(tokens[1]) def parse_side_effect_id_only(line: str) -> SideEffectIDOnly: assert SIEVE_AFTER_SIDE_EFFECT_MARK in line or SIEVE_BEFORE_SIDE_EFFECT_MARK in line if SIEVE_AFTER_SIDE_EFFECT_MARK in line: - tokens = line[line.find(SIEVE_AFTER_SIDE_EFFECT_MARK):].strip( - "\n").split("\t") + tokens = line[line.find(SIEVE_AFTER_SIDE_EFFECT_MARK) :].strip("\n").split("\t") return SideEffectIDOnly(tokens[1]) else: - tokens = line[line.find(SIEVE_BEFORE_SIDE_EFFECT_MARK):].strip( - "\n").split("\t") + tokens = ( + line[line.find(SIEVE_BEFORE_SIDE_EFFECT_MARK) :].strip("\n").split("\t") + ) return SideEffectIDOnly(tokens[1]) def parse_reconcile(line: str) -> Reconcile: assert SIEVE_BEFORE_RECONCILE_MARK in line or SIEVE_AFTER_RECONCILE_MARK in line if SIEVE_BEFORE_RECONCILE_MARK in line: - tokens = line[line.find(SIEVE_BEFORE_RECONCILE_MARK):].strip( - "\n").split("\t") + tokens = line[line.find(SIEVE_BEFORE_RECONCILE_MARK) :].strip("\n").split("\t") return Reconcile(tokens[1], tokens[2]) else: - tokens = line[line.find(SIEVE_AFTER_RECONCILE_MARK):].strip( - "\n").split("\t") + tokens = line[line.find(SIEVE_AFTER_RECONCILE_MARK) :].strip("\n").split("\t") return Reconcile(tokens[1], tokens[2]) @@ -433,7 +450,8 @@ def sanity_check(self): assert edge.source.id in self.vertices assert edge.sink.id in self.vertices assert (edge.source.is_event() and edge.sink.is_side_effect()) or ( - edge.sink.is_event() and edge.source.is_side_effect()) + edge.sink.is_event() and edge.source.is_side_effect() + ) def add_vertex(self, vertex: CausalityVertex): if vertex.id not in self.__vertices: diff --git a/build.py b/build.py index 12ed84dbd62..59787e0df7e 100644 --- a/build.py +++ b/build.py @@ -12,37 +12,45 @@ def download_kubernetes(): cmd_early_exit("rm -rf fakegopath") cmd_early_exit("mkdir -p fakegopath/src/k8s.io") cmd_early_exit( - "git clone --single-branch --branch v1.18.9 https://github.com/kubernetes/kubernetes.git fakegopath/src/k8s.io/kubernetes >> /dev/null") + "git clone --single-branch --branch v1.18.9 https://github.com/kubernetes/kubernetes.git fakegopath/src/k8s.io/kubernetes >> /dev/null" + ) os.chdir("fakegopath/src/k8s.io/kubernetes") cmd_early_exit("git checkout -b sieve >> /dev/null") os.chdir(ORIGINAL_DIR) def install_lib_for_kubernetes(): - with open("fakegopath/src/k8s.io/kubernetes/staging/src/k8s.io/apiserver/go.mod", "a") as go_mod_file: + with open( + "fakegopath/src/k8s.io/kubernetes/staging/src/k8s.io/apiserver/go.mod", "a" + ) as go_mod_file: go_mod_file.write("require sieve.client v0.0.0\n") go_mod_file.write("replace sieve.client => ../../sieve.client\n") cmd_early_exit( - "cp -r sieve-client fakegopath/src/k8s.io/kubernetes/staging/src/sieve.client") + "cp -r sieve-client fakegopath/src/k8s.io/kubernetes/staging/src/sieve.client" + ) cmd_early_exit( - "ln -s ../staging/src/sieve.client fakegopath/src/k8s.io/kubernetes/vendor/sieve.client") + "ln -s ../staging/src/sieve.client fakegopath/src/k8s.io/kubernetes/vendor/sieve.client" + ) def instrument_kubernetes(mode): os.chdir("instrumentation") cmd_early_exit("go build") cmd_early_exit( - "./instrumentation kubernetes %s %s/fakegopath/src/k8s.io/kubernetes" % (mode, ORIGINAL_DIR)) + "./instrumentation kubernetes %s %s/fakegopath/src/k8s.io/kubernetes" + % (mode, ORIGINAL_DIR) + ) os.chdir(ORIGINAL_DIR) def build_kubernetes(img_repo, img_tag): os.chdir("fakegopath/src/k8s.io/kubernetes") cmd_early_exit( - "GOPATH=%s/fakegopath KUBE_GIT_VERSION=v1.18.9-sieve-`git rev-parse HEAD` kind build node-image" % ORIGINAL_DIR) + "GOPATH=%s/fakegopath KUBE_GIT_VERSION=v1.18.9-sieve-`git rev-parse HEAD` kind build node-image" + % ORIGINAL_DIR + ) os.chdir(ORIGINAL_DIR) - cmd_early_exit("docker build --no-cache -t %s/node:%s ." % - (img_repo, img_tag)) + cmd_early_exit("docker build --no-cache -t %s/node:%s ." % (img_repo, img_tag)) cmd_early_exit("docker push %s/node:%s" % (img_repo, img_tag)) @@ -55,12 +63,15 @@ def setup_kubernetes(mode, img_repo, img_tag): def download_controller(project, link, sha): # If for some permission issue that we can't remove the operator, try sudo - if cmd_early_exit("rm -rf %s" % controllers.app_dir[project], early_exit=False) != 0: - print("We cannot remove %s, try sudo instead" % - controllers.app_dir[project]) + if ( + cmd_early_exit("rm -rf %s" % controllers.app_dir[project], early_exit=False) + != 0 + ): + print("We cannot remove %s, try sudo instead" % controllers.app_dir[project]) cmd_early_exit("sudo rm -rf %s" % controllers.app_dir[project]) - cmd_early_exit("git clone %s %s >> /dev/null" % - (link, controllers.app_dir[project])) + cmd_early_exit( + "git clone %s %s >> /dev/null" % (link, controllers.app_dir[project]) + ) os.chdir(controllers.app_dir[project]) cmd_early_exit("git checkout %s >> /dev/null" % sha) cmd_early_exit("git checkout -b sieve >> /dev/null") @@ -80,72 +91,114 @@ def remove_replacement_in_go_mod_file(file): go_mod_file.write(line) -def install_lib_for_controller(project, controller_runtime_version, client_go_version, docker_file_path): +def install_lib_for_controller( + project, controller_runtime_version, client_go_version, docker_file_path +): # download controller_runtime and client_go libs cmd_early_exit( - "go mod download sigs.k8s.io/controller-runtime@%s >> /dev/null" % controller_runtime_version) - cmd_early_exit("mkdir -p %s/dep-sieve/src/sigs.k8s.io" % - controllers.app_dir[project]) - cmd_early_exit("cp -r ${GOPATH}/pkg/mod/sigs.k8s.io/controller-runtime@%s %s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s" % - (controller_runtime_version, controllers.app_dir[project], controller_runtime_version)) - cmd_early_exit("chmod +w -R %s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s" % - (controllers.app_dir[project], controller_runtime_version)) - cmd_early_exit("go mod download k8s.io/client-go@%s >> /dev/null" % - client_go_version) - cmd_early_exit("mkdir -p %s/dep-sieve/src/k8s.io" % - controllers.app_dir[project]) + "go mod download sigs.k8s.io/controller-runtime@%s >> /dev/null" + % controller_runtime_version + ) + cmd_early_exit( + "mkdir -p %s/dep-sieve/src/sigs.k8s.io" % controllers.app_dir[project] + ) + cmd_early_exit( + "cp -r ${GOPATH}/pkg/mod/sigs.k8s.io/controller-runtime@%s %s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s" + % ( + controller_runtime_version, + controllers.app_dir[project], + controller_runtime_version, + ) + ) + cmd_early_exit( + "chmod +w -R %s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s" + % (controllers.app_dir[project], controller_runtime_version) + ) + cmd_early_exit( + "go mod download k8s.io/client-go@%s >> /dev/null" % client_go_version + ) + cmd_early_exit("mkdir -p %s/dep-sieve/src/k8s.io" % controllers.app_dir[project]) cmd_early_exit( - "cp -r ${GOPATH}/pkg/mod/k8s.io/client-go@%s %s/dep-sieve/src/k8s.io/client-go@%s" % (client_go_version, controllers.app_dir[project], client_go_version)) + "cp -r ${GOPATH}/pkg/mod/k8s.io/client-go@%s %s/dep-sieve/src/k8s.io/client-go@%s" + % (client_go_version, controllers.app_dir[project], client_go_version) + ) cmd_early_exit( - "chmod +w -R %s/dep-sieve/src/k8s.io/client-go@%s" % (controllers.app_dir[project], client_go_version)) - cmd_early_exit("cp -r sieve-client %s/dep-sieve/src/sieve.client" % - controllers.app_dir[project]) + "chmod +w -R %s/dep-sieve/src/k8s.io/client-go@%s" + % (controllers.app_dir[project], client_go_version) + ) + cmd_early_exit( + "cp -r sieve-client %s/dep-sieve/src/sieve.client" + % controllers.app_dir[project] + ) if project == "yugabyte-operator": # Ad-hoc fix for api incompatibility in golang # Special handling of yugabyte-operator as it depends on an older apimachinery which is # incompatible with the one sieve.client depends on - with fileinput.FileInput("%s/dep-sieve/src/sieve.client/go.mod" % controllers.app_dir[project], - inplace=True, backup='.bak') as sieve_client_go_mod: + with fileinput.FileInput( + "%s/dep-sieve/src/sieve.client/go.mod" % controllers.app_dir[project], + inplace=True, + backup=".bak", + ) as sieve_client_go_mod: for line in sieve_client_go_mod: if "k8s.io/apimachinery" in line: - print("\tk8s.io/apimachinery v0.17.4", end='\n') + print("\tk8s.io/apimachinery v0.17.4", end="\n") else: - print(line, end='') + print(line, end="") os.chdir(controllers.app_dir[project]) cmd_early_exit("git add -A >> /dev/null") - cmd_early_exit("git commit -m \"download the lib\" >> /dev/null") + cmd_early_exit('git commit -m "download the lib" >> /dev/null') os.chdir(ORIGINAL_DIR) # modify the go.mod to import the libs - remove_replacement_in_go_mod_file( - "%s/go.mod" % controllers.app_dir[project]) + remove_replacement_in_go_mod_file("%s/go.mod" % controllers.app_dir[project]) with open("%s/go.mod" % controllers.app_dir[project], "a") as go_mod_file: go_mod_file.write("require sieve.client v0.0.0\n") + go_mod_file.write("replace sieve.client => ./dep-sieve/src/sieve.client\n") go_mod_file.write( - "replace sieve.client => ./dep-sieve/src/sieve.client\n") - go_mod_file.write( - "replace sigs.k8s.io/controller-runtime => ./dep-sieve/src/sigs.k8s.io/controller-runtime@%s\n" % controller_runtime_version) + "replace sigs.k8s.io/controller-runtime => ./dep-sieve/src/sigs.k8s.io/controller-runtime@%s\n" + % controller_runtime_version + ) go_mod_file.write( - "replace k8s.io/client-go => ./dep-sieve/src/k8s.io/client-go@%s\n" % client_go_version) - with open("%s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s/go.mod" % (controllers.app_dir[project], controller_runtime_version), "a") as go_mod_file: + "replace k8s.io/client-go => ./dep-sieve/src/k8s.io/client-go@%s\n" + % client_go_version + ) + with open( + "%s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s/go.mod" + % (controllers.app_dir[project], controller_runtime_version), + "a", + ) as go_mod_file: go_mod_file.write("require sieve.client v0.0.0\n") go_mod_file.write("replace sieve.client => ../../sieve.client\n") go_mod_file.write( - "replace k8s.io/client-go => ../../k8s.io/client-go@%s\n" % client_go_version) - with open("%s/dep-sieve/src/k8s.io/client-go@%s/go.mod" % (controllers.app_dir[project], client_go_version), "a") as go_mod_file: + "replace k8s.io/client-go => ../../k8s.io/client-go@%s\n" + % client_go_version + ) + with open( + "%s/dep-sieve/src/k8s.io/client-go@%s/go.mod" + % (controllers.app_dir[project], client_go_version), + "a", + ) as go_mod_file: go_mod_file.write("require sieve.client v0.0.0\n") go_mod_file.write("replace sieve.client => ../../sieve.client\n") # copy the build.sh and Dockerfile - cmd_early_exit("cp %s/build/build.sh %s/build.sh" % - (controllers.test_dir[project], controllers.app_dir[project])) - cmd_early_exit("cp %s/build/Dockerfile %s/%s" % - (controllers.test_dir[project], controllers.app_dir[project], docker_file_path)) + cmd_early_exit( + "cp %s/build/build.sh %s/build.sh" + % (controllers.test_dir[project], controllers.app_dir[project]) + ) + cmd_early_exit( + "cp %s/build/Dockerfile %s/%s" + % ( + controllers.test_dir[project], + controllers.app_dir[project], + docker_file_path, + ) + ) os.chdir(controllers.app_dir[project]) cmd_early_exit("git add -A >> /dev/null") - cmd_early_exit("git commit -m \"import the lib\" >> /dev/null") + cmd_early_exit('git commit -m "import the lib" >> /dev/null') os.chdir(ORIGINAL_DIR) @@ -153,7 +206,18 @@ def instrument_controller(project, mode, controller_runtime_version, client_go_v os.chdir("instrumentation") cmd_early_exit("go build") cmd_early_exit( - "./instrumentation %s %s %s/%s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s %s/%s/dep-sieve/src/k8s.io/client-go@%s" % (project, mode, ORIGINAL_DIR, controllers.app_dir[project], controller_runtime_version, ORIGINAL_DIR, controllers.app_dir[project], client_go_version)) + "./instrumentation %s %s %s/%s/dep-sieve/src/sigs.k8s.io/controller-runtime@%s %s/%s/dep-sieve/src/k8s.io/client-go@%s" + % ( + project, + mode, + ORIGINAL_DIR, + controllers.app_dir[project], + controller_runtime_version, + ORIGINAL_DIR, + controllers.app_dir[project], + client_go_version, + ) + ) os.chdir(ORIGINAL_DIR) @@ -163,29 +227,72 @@ def build_controller(project, img_repo, img_tag): os.chdir(ORIGINAL_DIR) -def setup_controller(project, mode, img_repo, img_tag, link, sha, controller_runtime_version, client_go_version, docker_file_path, build_only): +def setup_controller( + project, + mode, + img_repo, + img_tag, + link, + sha, + controller_runtime_version, + client_go_version, + docker_file_path, + build_only, +): if not build_only: download_controller(project, link, sha) install_lib_for_controller( - project, controller_runtime_version, client_go_version, docker_file_path) + project, controller_runtime_version, client_go_version, docker_file_path + ) instrument_controller( - project, mode, controller_runtime_version, client_go_version) + project, mode, controller_runtime_version, client_go_version + ) build_controller(project, img_repo, img_tag) if __name__ == "__main__": usage = "usage: python3 build.py [options]" parser = optparse.OptionParser(usage=usage) - parser.add_option("-p", "--project", dest="project", - help="specify PROJECT to build: cassandra-operator or zookeeper-operator", metavar="PROJECT", default="cassandra-operator") - parser.add_option("-m", "--mode", dest="mode", - help="build MODE: learn, time-travel, obs-gap, atom-vio", metavar="MODE", default="learn") - parser.add_option("-s", "--sha", dest="sha", - help="SHA of the project", metavar="SHA", default="none") - parser.add_option("-d", "--docker", dest="docker", - help="DOCKER repo that you have access", metavar="DOCKER", default="none") - parser.add_option("-b", "--build", dest="build_only", action="store_true", - help="build only", default=False) + parser.add_option( + "-p", + "--project", + dest="project", + help="specify PROJECT to build: cassandra-operator or zookeeper-operator", + metavar="PROJECT", + default="cassandra-operator", + ) + parser.add_option( + "-m", + "--mode", + dest="mode", + help="build MODE: learn, time-travel, obs-gap, atom-vio", + metavar="MODE", + default="learn", + ) + parser.add_option( + "-s", + "--sha", + dest="sha", + help="SHA of the project", + metavar="SHA", + default="none", + ) + parser.add_option( + "-d", + "--docker", + dest="docker", + help="DOCKER repo that you have access", + metavar="DOCKER", + default="none", + ) + parser.add_option( + "-b", + "--build", + dest="build_only", + action="store_true", + help="build only", + default=False, + ) (options, args) = parser.parse_args() if options.mode == "obs-gap": @@ -193,15 +300,25 @@ def setup_controller(project, mode, img_repo, img_tag, link, sha, controller_run elif options.mode == "atom-vio": options.mode = sieve_modes.ATOM_VIO - img_repo = options.docker if options.docker != "none" else sieve_config.config[ - "docker_repo"] + img_repo = ( + options.docker + if options.docker != "none" + else sieve_config.config["docker_repo"] + ) img_tag = options.mode if options.project == "kubernetes": setup_kubernetes(options.mode, img_repo, img_tag) else: sha = options.sha if options.sha != "none" else controllers.sha[options.project] - setup_controller(options.project, options.mode, img_repo, - img_tag, controllers.github_link[options.project], sha, - controllers.controller_runtime_version[options.project], - controllers.client_go_version[options.project], - controllers.docker_file[options.project], options.build_only) + setup_controller( + options.project, + options.mode, + img_repo, + img_tag, + controllers.github_link[options.project], + sha, + controllers.controller_runtime_version[options.project], + controllers.client_go_version[options.project], + controllers.docker_file[options.project], + options.build_only, + ) diff --git a/check_env.py b/check_env.py index 11950efccc0..8bd3e634e32 100644 --- a/check_env.py +++ b/check_env.py @@ -8,32 +8,39 @@ def check_go_env(): if os.system("go version > /dev/null 2>&1") != 0: fail( - "golang environment not detected, please install it according to https://golang.org/doc/install") + "golang environment not detected, please install it according to https://golang.org/doc/install" + ) return else: ok("golang environment detected") - goenv = {x.split("=")[0]: x.split("=")[1].strip('"') for x in subprocess.check_output( - "go env", shell=True, encoding='UTF-8').strip().split("\n")} - if 'GOVERSION' in goenv: - version = goenv['GOVERSION'][2:] + goenv = { + x.split("=")[0]: x.split("=")[1].strip('"') + for x in subprocess.check_output("go env", shell=True, encoding="UTF-8") + .strip() + .split("\n") + } + if "GOVERSION" in goenv: + version = goenv["GOVERSION"][2:] else: - version = os.popen('go version').read().split()[2][2:] + version = os.popen("go version").read().split()[2][2:] version_breakdown = version.split(".") major = int(version_breakdown[0]) minor = int(version_breakdown[1]) if major > 1 or (major == 1 and minor >= 13): - ok("go version %s satisfies the requirement" % - (version)) + ok("go version %s satisfies the requirement" % (version)) else: - warn("go version %s not satisfies the requirement, the minimum go version should be above 1.13.0" % ( - version)) + warn( + "go version %s not satisfies the requirement, the minimum go version should be above 1.13.0" + % (version) + ) - if 'GOPATH' in os.environ: + if "GOPATH" in os.environ: ok("environment variable $GOPATH detected") else: fail( - "environment variable $GOPATH not detected, try to set it according to https://golang.org/doc/gopath_code#GOPATH") + "environment variable $GOPATH not detected, try to set it according to https://golang.org/doc/gopath_code#GOPATH" + ) return @@ -41,7 +48,8 @@ def check_go_env(): def check_kind_env(): if os.system("kind version > /dev/null 2>&1") != 0: fail( - "kind not detected, please install it according to https://kind.sigs.k8s.io/docs/user/quick-start/#installation") + "kind not detected, please install it according to https://kind.sigs.k8s.io/docs/user/quick-start/#installation" + ) return else: ok("kind detected") @@ -56,54 +64,74 @@ def check_kind_env(): ok("kind version %s satisfies the requirement" % (version)) else: warn( - "kind version %s not satisfies the requirement, the minimum kind version should be above 0.10.0" % (version)) + "kind version %s not satisfies the requirement, the minimum kind version should be above 0.10.0" + % (version) + ) - if 'KUBECONFIG' in os.environ: + if "KUBECONFIG" in os.environ: ok("environment variable $KUBECONFIG detected") else: - fail("environment variable $KUBECONFIG not detected, try to set it according to https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig") + fail( + "environment variable $KUBECONFIG not detected, try to set it according to https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig" + ) return def check_sqlite_env(): - warn("sqlite3 version 3.32 and pysqlite3 are only required for learning stage, please ignore any failures below if you only want to reproduce the bugs") + warn( + "sqlite3 version 3.32 and pysqlite3 are only required for learning stage, please ignore any failures below if you only want to reproduce the bugs" + ) if os.system("sqlite3 -version > /dev/null 2>&1") != 0: - fail("sqlite3 not detected, please install it with version above 3.32 according to https://help.dreamhost.com/hc/en-us/articles/360028047592-Installing-a-custom-version-of-SQLite3") + fail( + "sqlite3 not detected, please install it with version above 3.32 according to https://help.dreamhost.com/hc/en-us/articles/360028047592-Installing-a-custom-version-of-SQLite3" + ) return else: ok("sqlite3 detected") - version = subprocess.check_output( - "sqlite3 -version", shell=True, encoding='UTF-8').strip().split()[0] + version = ( + subprocess.check_output("sqlite3 -version", shell=True, encoding="UTF-8") + .strip() + .split()[0] + ) major = int(version.split(".")[0]) minor = int(version.split(".")[1]) if major > 3 or (major == 3 and minor >= 32): ok("sqlite3 version %s satisfies the requirement" % (version)) else: - fail("sqlite3 version %s not satisfies the requirement, the minimum sqlite3 version should be above 3.32, please update according to https://help.dreamhost.com/hc/en-us/articles/360028047592-Installing-a-custom-version-of-SQLite3" % (version)) + fail( + "sqlite3 version %s not satisfies the requirement, the minimum sqlite3 version should be above 3.32, please update according to https://help.dreamhost.com/hc/en-us/articles/360028047592-Installing-a-custom-version-of-SQLite3" + % (version) + ) try: import sqlite3 + ok("python module pysqlite3 detected") except Exception as err: fail( - "python module pysqlite3 not detected, try to install it by `pip3 install pysqlite3`") + "python module pysqlite3 not detected, try to install it by `pip3 install pysqlite3`" + ) def check_python_env(): try: import kubernetes + ok("python module kubernetes detected") except Exception as err: fail( - "python module pysqlite3 not detected, try to install it by `pip3 install kubernetes`") + "python module pysqlite3 not detected, try to install it by `pip3 install kubernetes`" + ) try: import yaml + ok("python module pyyaml detected") except Exception as err: fail( - "python module pysqlite3 not detected, try to install it by `pip3 install pyyaml`") + "python module pysqlite3 not detected, try to install it by `pip3 install pyyaml`" + ) if __name__ == "__main__": diff --git a/common.py b/common.py index 4bb2a75c767..06eab6a1558 100644 --- a/common.py +++ b/common.py @@ -36,15 +36,15 @@ class sieve_modes: class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" def warn(message): @@ -64,7 +64,17 @@ def cprint(message, color): class Suite: - def __init__(self, workload, config, mode, two_sided=False, num_apiservers=1, num_workers=2, pvc_resize=False, oracle_config={}): + def __init__( + self, + workload, + config, + mode, + two_sided=False, + num_apiservers=1, + num_workers=2, + pvc_resize=False, + oracle_config={}, + ): self.workload = workload self.config = config self.mode = mode diff --git a/controllers.py b/controllers.py index e38f823e009..e57a53c667a 100644 --- a/controllers.py +++ b/controllers.py @@ -54,73 +54,165 @@ test_suites = { "cassandra-operator": { "scaledown": Suite( - workloads.workloads["cassandra-operator"]["scaledown"], "test-cassandra-operator/test/obs-gap-1.yaml", sieve_modes.OBS_GAP), + workloads.workloads["cassandra-operator"]["scaledown"], + "test-cassandra-operator/test/obs-gap-1.yaml", + sieve_modes.OBS_GAP, + ), "recreate": Suite( - workloads.workloads["cassandra-operator"]["recreate"], "test-cassandra-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["cassandra-operator"]["recreate"], + "test-cassandra-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + ), "scaledown-scaleup": Suite( - workloads.workloads["cassandra-operator"]["scaledown-scaleup"], "test-cassandra-operator/test/time-travel-2.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["cassandra-operator"]["scaledown-scaleup"], + "test-cassandra-operator/test/time-travel-2.yaml", + sieve_modes.TIME_TRAVEL, + ), }, "zookeeper-operator": { "recreate": Suite( - workloads.workloads["zookeeper-operator"]["recreate"], "test-zookeeper-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["zookeeper-operator"]["recreate"], + "test-zookeeper-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + ), "scaledown-scaleup": Suite( - workloads.workloads["zookeeper-operator"]["scaledown-scaleup"], "test-zookeeper-operator/test/time-travel-2.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["zookeeper-operator"]["scaledown-scaleup"], + "test-zookeeper-operator/test/time-travel-2.yaml", + sieve_modes.TIME_TRAVEL, + ), "scaledown-scaleup-obs": Suite( - workloads.workloads["zookeeper-operator"]["scaledown-scaleup-obs"], "test-zookeeper-operator/test/obs-gap-1.yaml", sieve_modes.OBS_GAP), + workloads.workloads["zookeeper-operator"]["scaledown-scaleup-obs"], + "test-zookeeper-operator/test/obs-gap-1.yaml", + sieve_modes.OBS_GAP, + ), }, "rabbitmq-operator": { "recreate": Suite( - workloads.workloads["rabbitmq-operator"]["recreate"], "test-rabbitmq-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["rabbitmq-operator"]["recreate"], + "test-rabbitmq-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + ), "resize-pvc": Suite( - workloads.workloads["rabbitmq-operator"]["resize-pvc"], "test-rabbitmq-operator/test/time-travel-2.yaml", sieve_modes.TIME_TRAVEL, two_sided=True), + workloads.workloads["rabbitmq-operator"]["resize-pvc"], + "test-rabbitmq-operator/test/time-travel-2.yaml", + sieve_modes.TIME_TRAVEL, + two_sided=True, + ), "scaleup-scaledown": Suite( - workloads.workloads["rabbitmq-operator"]["scaleup-scaledown"], "test-rabbitmq-operator/test/obs-gap-1.yaml", sieve_modes.OBS_GAP), + workloads.workloads["rabbitmq-operator"]["scaleup-scaledown"], + "test-rabbitmq-operator/test/obs-gap-1.yaml", + sieve_modes.OBS_GAP, + ), "resize-pvc-atomic": Suite( - workloads.workloads["rabbitmq-operator"]["resize-pvc-atomic"], "test-rabbitmq-operator/test/atomic-1.yaml", sieve_modes.ATOM_VIO, pvc_resize=True) + workloads.workloads["rabbitmq-operator"]["resize-pvc-atomic"], + "test-rabbitmq-operator/test/atomic-1.yaml", + sieve_modes.ATOM_VIO, + pvc_resize=True, + ), }, "mongodb-operator": { "recreate": Suite( - workloads.workloads["mongodb-operator"]["recreate"], "test-mongodb-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL, num_workers=3), + workloads.workloads["mongodb-operator"]["recreate"], + "test-mongodb-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + num_workers=3, + ), "disable-enable-shard": Suite( - workloads.workloads["mongodb-operator"]["disable-enable-shard"], "test-mongodb-operator/test/time-travel-2.yaml", sieve_modes.TIME_TRAVEL, num_workers=3), + workloads.workloads["mongodb-operator"]["disable-enable-shard"], + "test-mongodb-operator/test/time-travel-2.yaml", + sieve_modes.TIME_TRAVEL, + num_workers=3, + ), "disable-enable-arbiter": Suite( - workloads.workloads["mongodb-operator"]["disable-enable-arbiter"], "test-mongodb-operator/test/time-travel-3.yaml", sieve_modes.TIME_TRAVEL, num_workers=5), + workloads.workloads["mongodb-operator"]["disable-enable-arbiter"], + "test-mongodb-operator/test/time-travel-3.yaml", + sieve_modes.TIME_TRAVEL, + num_workers=5, + ), }, "cass-operator": { "recreate": Suite( - workloads.workloads["cass-operator"]["recreate"], "test-cass-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["cass-operator"]["recreate"], + "test-cass-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + ), }, "casskop-operator": { "recreate": Suite( - workloads.workloads["casskop-operator"]["recreate"], "test-casskop-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["casskop-operator"]["recreate"], + "test-casskop-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + ), "reducepdb": Suite( - workloads.workloads["casskop-operator"]["reducepdb"], "test-casskop-operator/test/time-travel-2.yaml", sieve_modes.TIME_TRAVEL, two_sided=True), + workloads.workloads["casskop-operator"]["reducepdb"], + "test-casskop-operator/test/time-travel-2.yaml", + sieve_modes.TIME_TRAVEL, + two_sided=True, + ), "nodesperrack": Suite( - workloads.workloads["casskop-operator"]["nodesperrack"], "test-casskop-operator/test/obs-gap-1.yaml", sieve_modes.OBS_GAP), + workloads.workloads["casskop-operator"]["nodesperrack"], + "test-casskop-operator/test/obs-gap-1.yaml", + sieve_modes.OBS_GAP, + ), "scaledown-obs-gap": Suite( - workloads.workloads["casskop-operator"]["scaledown"], "test-casskop-operator/test/obs-gap-2.yaml", sieve_modes.OBS_GAP), + workloads.workloads["casskop-operator"]["scaledown"], + "test-casskop-operator/test/obs-gap-2.yaml", + sieve_modes.OBS_GAP, + ), }, "xtradb-operator": { "recreate": Suite( - workloads.workloads["xtradb-operator"]["recreate"], "test-xtradb-operator/test/time-travel-1.yaml", sieve_modes.TIME_TRAVEL, num_workers=4), + workloads.workloads["xtradb-operator"]["recreate"], + "test-xtradb-operator/test/time-travel-1.yaml", + sieve_modes.TIME_TRAVEL, + num_workers=4, + ), "disable-enable-haproxy": Suite( - workloads.workloads["xtradb-operator"]["disable-enable-haproxy"], "test-xtradb-operator/test/time-travel-2.yaml", sieve_modes.TIME_TRAVEL, num_workers=4), + workloads.workloads["xtradb-operator"]["disable-enable-haproxy"], + "test-xtradb-operator/test/time-travel-2.yaml", + sieve_modes.TIME_TRAVEL, + num_workers=4, + ), "disable-enable-proxysql": Suite( - workloads.workloads["xtradb-operator"]["disable-enable-proxysql"], "test-xtradb-operator/test/time-travel-3.yaml", sieve_modes.TIME_TRAVEL, num_workers=4), + workloads.workloads["xtradb-operator"]["disable-enable-proxysql"], + "test-xtradb-operator/test/time-travel-3.yaml", + sieve_modes.TIME_TRAVEL, + num_workers=4, + ), }, "yugabyte-operator": { "recreate": Suite( - workloads.workloads["yugabyte-operator"]["recreate"], "null", sieve_modes.TIME_TRAVEL), + workloads.workloads["yugabyte-operator"]["recreate"], + "null", + sieve_modes.TIME_TRAVEL, + ), "disable-enable-tls": Suite( - workloads.workloads["yugabyte-operator"]["disable-enable-tls"], "test-yugabyte-operator/test/time-travel-tls.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["yugabyte-operator"]["disable-enable-tls"], + "test-yugabyte-operator/test/time-travel-tls.yaml", + sieve_modes.TIME_TRAVEL, + ), "disable-enable-tserverUIPort": Suite( - workloads.workloads["yugabyte-operator"]["disable-enable-tserverUIPort"], "test-yugabyte-operator/test/time-travel-tserverUIPort.yaml", sieve_modes.TIME_TRAVEL), + workloads.workloads["yugabyte-operator"]["disable-enable-tserverUIPort"], + "test-yugabyte-operator/test/time-travel-tserverUIPort.yaml", + sieve_modes.TIME_TRAVEL, + ), "scaleup-scaledown-tserver": Suite( - workloads.workloads["yugabyte-operator"]["scaleup-scaledown-tserver"], "test-yugabyte-operator/test/obs-gap-config-1.yaml", sieve_modes.OBS_GAP) + workloads.workloads["yugabyte-operator"]["scaleup-scaledown-tserver"], + "test-yugabyte-operator/test/obs-gap-config-1.yaml", + sieve_modes.OBS_GAP, + ), }, "nifikop-operator": { "change-config": Suite( - workloads.workloads["nifikop-operator"]["change-config"], "test-nifikop-operator/test/atomic-1.yaml", sieve_modes.ATOM_VIO, oracle_config={"interest_objects": [{"rtype": "pod", "namespace": "default", "name": "simplenifi-1.*"}], "effect_to_check": ["Create"]} + workloads.workloads["nifikop-operator"]["change-config"], + "test-nifikop-operator/test/atomic-1.yaml", + sieve_modes.ATOM_VIO, + oracle_config={ + "interest_objects": [ + {"rtype": "pod", "namespace": "default", "name": "simplenifi-1.*"} + ], + "effect_to_check": ["Create"], + }, ), }, } @@ -128,13 +220,26 @@ # This should be all lower case # TODO: we should make the CRD checking in learn client case insensitive CRDs = { - "cassandra-operator": ["cassandradatacenter", "cassandracluster", "cassandrabackup"], + "cassandra-operator": [ + "cassandradatacenter", + "cassandracluster", + "cassandrabackup", + ], "zookeeper-operator": ["zookeepercluster"], "rabbitmq-operator": ["rabbitmqcluster"], - "mongodb-operator": ["perconaservermongodb", "perconaservermongodbbackup", "perconaservermongodbrestore"], + "mongodb-operator": [ + "perconaservermongodb", + "perconaservermongodbbackup", + "perconaservermongodbrestore", + ], "cass-operator": ["cassandradatacenter"], "casskop-operator": ["cassandracluster", "cassandrarestore", "cassandrabackup"], - "xtradb-operator": ["perconaxtradbcluster", "perconaxtradbclusterbackup", "perconaxtradbclusterrestore", "perconaxtradbbackup"], + "xtradb-operator": [ + "perconaxtradbcluster", + "perconaxtradbclusterbackup", + "perconaxtradbclusterrestore", + "perconaxtradbbackup", + ], "yugabyte-operator": ["ybcluster"], "nifikop-operator": ["nificluster"], } @@ -213,7 +318,7 @@ def make_safe_filename(filename): - return re.sub(r'[^\w\d-]', '_', filename) + return re.sub(r"[^\w\d-]", "_", filename) def replace_docker_repo(path, dr, dt): @@ -222,8 +327,8 @@ def replace_docker_repo(path, dr, dt): data = data.replace("${SIEVE-DR}", dr) data = data.replace("${SIEVE-DT}", dt) fin.close() - tokens = path.rsplit('.', 1) - new_path = tokens[0] + "-" + make_safe_filename(dr) + '.' + tokens[1] + tokens = path.rsplit(".", 1) + new_path = tokens[0] + "-" + make_safe_filename(dr) + "." + tokens[1] fin = open(new_path, "w") fin.write(data) fin.close() @@ -231,42 +336,40 @@ def replace_docker_repo(path, dr, dt): def cassandra_operator_deploy(dr, dt): - new_path = replace_docker_repo( - "test-cassandra-operator/deploy/bundle.yaml", dr, dt) + new_path = replace_docker_repo("test-cassandra-operator/deploy/bundle.yaml", dr, dt) os.system("kubectl apply -f test-cassandra-operator/deploy/crds.yaml") - os.system( - "kubectl apply -f %s" % new_path) + os.system("kubectl apply -f %s" % new_path) os.system("rm %s" % new_path) def zookeeper_operator_deploy(dr, dt): new_path = replace_docker_repo( - "test-zookeeper-operator/deploy/default_ns/operator.yaml", dr, dt) + "test-zookeeper-operator/deploy/default_ns/operator.yaml", dr, dt + ) os.system("kubectl create -f test-zookeeper-operator/deploy/crds") - os.system( - "kubectl create -f test-zookeeper-operator/deploy/default_ns/rbac.yaml") - os.system( - "kubectl create -f %s" % new_path) + os.system("kubectl create -f test-zookeeper-operator/deploy/default_ns/rbac.yaml") + os.system("kubectl create -f %s" % new_path) os.system("rm %s" % new_path) def rabbitmq_operator_deploy(dr, dt): new_path = replace_docker_repo( - "test-rabbitmq-operator/deploy/cluster-operator.yaml", dr, dt) + "test-rabbitmq-operator/deploy/cluster-operator.yaml", dr, dt + ) os.system("kubectl apply -f %s" % new_path) os.system("rm %s" % new_path) def mongodb_operator_deploy(dr, dt): - new_path = replace_docker_repo( - "test-mongodb-operator/deploy/bundle.yaml", dr, dt) + new_path = replace_docker_repo("test-mongodb-operator/deploy/bundle.yaml", dr, dt) os.system("kubectl apply -f %s" % new_path) os.system("rm %s" % new_path) def cass_operator_deploy(dr, dt): new_path = replace_docker_repo( - "test-cass-operator/deploy/controller-manifest.yaml", dr, dt) + "test-cass-operator/deploy/controller-manifest.yaml", dr, dt + ) os.system("kubectl apply -f %s" % new_path) os.system("kubectl apply -f test-cass-operator/deploy/storageClass.yaml") os.system("rm %s" % new_path) @@ -274,37 +377,38 @@ def cass_operator_deploy(dr, dt): def casskop_operator_deploy(dr, dt): # Using helm - new_path = replace_docker_repo( - "test-casskop-operator/deploy/values.yaml", dr, dt) + new_path = replace_docker_repo("test-casskop-operator/deploy/values.yaml", dr, dt) os.system( - "helm install -f %s casskop-operator test-casskop-operator/deploy" % (new_path)) + "helm install -f %s casskop-operator test-casskop-operator/deploy" % (new_path) + ) def xtradb_operator_deploy(dr, dt): - new_path = replace_docker_repo( - "test-xtradb-operator/deploy/bundle.yaml", dr, dt) + new_path = replace_docker_repo("test-xtradb-operator/deploy/bundle.yaml", dr, dt) os.system("kubectl apply -f %s" % new_path) os.system("rm %s" % new_path) def yugabyte_operator_deploy(dr, dt): new_path = replace_docker_repo( - "test-yugabyte-operator/deploy/operator.yaml", dr, dt) + "test-yugabyte-operator/deploy/operator.yaml", dr, dt + ) os.system("cp %s .." % new_path) os.system( - "kubectl create -f test-yugabyte-operator/deploy/crds/yugabyte.com_ybclusters_crd.yaml") + "kubectl create -f test-yugabyte-operator/deploy/crds/yugabyte.com_ybclusters_crd.yaml" + ) os.system("kubectl create -f %s" % new_path) os.system("rm %s" % new_path) def nifikop_operator_deploy(dr, dt): # Using helm - new_path = replace_docker_repo( - "test-nifikop-operator/deploy/values.yaml", dr, dt) + new_path = replace_docker_repo("test-nifikop-operator/deploy/values.yaml", dr, dt) os.system("kubectl apply -f test-nifikop-operator/deploy/role.yaml") os.system("test-nifikop-operator/deploy/zk.sh") os.system( - "helm install -f %s nifikop-operator test-nifikop-operator/deploy" % (new_path)) + "helm install -f %s nifikop-operator test-nifikop-operator/deploy" % (new_path) + ) os.system("rm %s" % (new_path)) diff --git a/eva-learn.py b/eva-learn.py index 268621198b5..9d553885128 100644 --- a/eva-learn.py +++ b/eva-learn.py @@ -24,9 +24,9 @@ def evaluate_single(cmd, project, test_suite, mode): os.system(cmd) total_time = time.time() - s generated_config_dir = os.path.join( - "log", project, test_suite, "learn", mode, "analysis", "gen-" + mode) - num_configs = len( - glob.glob(os.path.join(generated_config_dir, "*.yaml"))) + "log", project, test_suite, "learn", mode, "analysis", "gen-" + mode + ) + num_configs = len(glob.glob(os.path.join(generated_config_dir, "*.yaml"))) return total_time, num_configs @@ -35,14 +35,14 @@ def evaluate(build): os.system("mkdir -p %s" % eva_dir) docker_repo_name = sieve_config.config["docker_repo"] if build: - os.system("python3 build.py -p kubernetes -m learn -d %s" % - docker_repo_name) + os.system("python3 build.py -p kubernetes -m learn -d %s" % docker_repo_name) project_workload_map = controllers.test_suites f = open(os.path.join(eva_dir, "output.tsv"), "w") for project in project_workload_map.keys(): if build: - os.system("python3 build.py -p %s -m learn -d %s" % - (project, docker_repo_name)) + os.system( + "python3 build.py -p %s -m learn -d %s" % (project, docker_repo_name) + ) for test_suite in project_workload_map[project].keys(): if project_workload_map[project][test_suite].mode != "time-travel": continue @@ -50,17 +50,34 @@ def evaluate(build): continue cmd = "python3 sieve.py -p %s -t %s -d %s -s learn -r" % ( - project, test_suite, docker_repo_name) - total_time_with_rl, num_configs_w_rl = evaluate_single(cmd, project, test_suite, - project_workload_map[project][test_suite].mode) + project, + test_suite, + docker_repo_name, + ) + total_time_with_rl, num_configs_w_rl = evaluate_single( + cmd, project, test_suite, project_workload_map[project][test_suite].mode + ) cmd = "python3 sieve.py -p %s -t %s -d %s -s learn" % ( - project, test_suite, docker_repo_name) - total_time, num_configs = evaluate_single(cmd, project, test_suite, - project_workload_map[project][test_suite].mode) + project, + test_suite, + docker_repo_name, + ) + total_time, num_configs = evaluate_single( + cmd, project, test_suite, project_workload_map[project][test_suite].mode + ) - f.write("%s\t%s\t%s\t%s\t%s\t%s\n" % (project, test_suite, - total_time, num_configs, total_time_with_rl, num_configs_w_rl)) + f.write( + "%s\t%s\t%s\t%s\t%s\t%s\n" + % ( + project, + test_suite, + total_time, + num_configs, + total_time_with_rl, + num_configs_w_rl, + ) + ) f.flush() f.close() diff --git a/oracle.py b/oracle.py index ffcb9fd166a..bd035139b63 100644 --- a/oracle.py +++ b/oracle.py @@ -10,18 +10,51 @@ import sieve_config import re -not_care_keys = ['uid', 'resourceVersion', 'creationTimestamp', 'ownerReferences', 'managedFields', 'generateName', 'selfLink', 'annotations', - 'pod-template-hash', 'secretName', 'image', 'lastTransitionTime', 'nodeName', 'podIPs', 'podIP', 'hostIP', 'containerID', 'imageID', - 'startTime', 'startedAt', 'volumeMounts', 'finishedAt', 'volumeName', 'lastUpdateTime', 'env', 'message', 'currentRevision', 'updateRevision', - 'controller-revision-hash'] -not_care = [jd.delete, jd.insert] + not_care_keys + ['name'] -side_effect_empty_entry = {"Create": 0, "Update": 0, - "Delete": 0, "Patch": 0, "DeleteAllOf": 0} +not_care_keys = [ + "uid", + "resourceVersion", + "creationTimestamp", + "ownerReferences", + "managedFields", + "generateName", + "selfLink", + "annotations", + "pod-template-hash", + "secretName", + "image", + "lastTransitionTime", + "nodeName", + "podIPs", + "podIP", + "hostIP", + "containerID", + "imageID", + "startTime", + "startedAt", + "volumeMounts", + "finishedAt", + "volumeName", + "lastUpdateTime", + "env", + "message", + "currentRevision", + "updateRevision", + "controller-revision-hash", +] +not_care = [jd.delete, jd.insert] + not_care_keys + ["name"] +side_effect_empty_entry = { + "Create": 0, + "Update": 0, + "Delete": 0, + "Patch": 0, + "DeleteAllOf": 0, +} def dump_json_file(dir, data, json_file_name): - json.dump(data, open(os.path.join( - dir, json_file_name), "w"), indent=4, sort_keys=True) + json.dump( + data, open(os.path.join(dir, json_file_name), "w"), indent=4, sort_keys=True + ) def generate_test_oracle(log_dir): @@ -61,7 +94,8 @@ def generate_side_effect(path): side_effect_map[rtype][namespace] = {} if name not in side_effect_map[rtype][namespace]: side_effect_map[rtype][namespace][name] = copy.deepcopy( - side_effect_empty_entry) + side_effect_empty_entry + ) side_effect_map[rtype][namespace][name][etype] += 1 return side_effect_map @@ -81,7 +115,9 @@ def generate_status(): status[ktype] = copy.deepcopy(status_empty_entry) for pod in core_v1.list_namespaced_pod(k8s_namespace, watch=False).items: resources[POD].append(pod) - for pvc in core_v1.list_namespaced_persistent_volume_claim(k8s_namespace, watch=False).items: + for pvc in core_v1.list_namespaced_persistent_volume_claim( + k8s_namespace, watch=False + ).items: resources[PVC].append(pvc) for dp in apps_v1.list_namespaced_deployment(k8s_namespace, watch=False).items: resources[DEPLOYMENT].append(dp) @@ -108,7 +144,7 @@ def trim_resource(cur, key_trace=[]): for key in list(cur): if key in not_care_keys: cur[key] = "SIEVE-IGNORE" - elif key == 'name' and key_trace[1] != "metadata": + elif key == "name" and key_trace[1] != "metadata": cur[key] = "SIEVE-IGNORE" else: trim_resource(cur[key], key_trace + [key]) @@ -118,13 +154,13 @@ def get_resource_helper(func): k8s_namespace = sieve_config.config["namespace"] response = func(k8s_namespace, _preload_content=False, watch=False) data = json.loads(response.data) - return [resource for resource in data['items']] + return [resource for resource in data["items"]] def get_crd_list(): data = [] try: - for item in json.loads(os.popen('kubectl get crd -o json').read())["items"]: + for item in json.loads(os.popen("kubectl get crd -o json").read())["items"]: data.append(item["spec"]["names"]["singular"]) except Exception as e: print("get_crd_list fail", e) @@ -134,7 +170,9 @@ def get_crd_list(): def get_crd(crd): data = [] try: - for item in json.loads(os.popen('kubectl get %s -o json' % (crd)).read())["items"]: + for item in json.loads(os.popen("kubectl get %s -o json" % (crd)).read())[ + "items" + ]: data.append(item) except Exception as e: print("get_crd fail", e) @@ -153,14 +191,13 @@ def generate_resources(path=""): "persistentvolumeclaim": core_v1.list_namespaced_persistent_volume_claim, "pod": core_v1.list_namespaced_pod, "service": core_v1.list_namespaced_service, - "statefulset": apps_v1.list_namespaced_stateful_set + "statefulset": apps_v1.list_namespaced_stateful_set, } resources = {} crd_list = get_crd_list() - resource_set = set( - ["statefulset", "pod", "persistentvolumeclaim", "deployment"]) + resource_set = set(["statefulset", "pod", "persistentvolumeclaim", "deployment"]) if path != "": for line in open(path).readlines(): if analyze_util.SIEVE_AFTER_SIDE_EFFECT_MARK not in line: @@ -171,8 +208,7 @@ def generate_resources(path=""): for resource in resource_set: if resource in resource_handler: # Normal resource - resources[resource] = get_resource_helper( - resource_handler[resource]) + resources[resource] = get_resource_helper(resource_handler[resource]) # Fetch for crd for crd in crd_list: @@ -184,28 +220,33 @@ def generate_resources(path=""): def check_status(learning_status, testing_status): alarm = 0 - all_keys = set(learning_status.keys()).union( - testing_status.keys()) + all_keys = set(learning_status.keys()).union(testing_status.keys()) bug_report = "" for rtype in all_keys: if rtype not in learning_status: - bug_report += "[ERROR] %s not in learning status digest\n" % ( - rtype) + bug_report += "[ERROR] %s not in learning status digest\n" % (rtype) alarm += 1 continue elif rtype not in testing_status: - bug_report += "[ERROR] %s not in testing status digest\n" % ( - rtype) + bug_report += "[ERROR] %s not in testing status digest\n" % (rtype) alarm += 1 continue else: for attr in learning_status[rtype]: if learning_status[rtype][attr] != testing_status[rtype][attr]: alarm += 1 - bug_report += "[ERROR] %s %s inconsistency: %s seen after learning run, but %s seen after testing run\n" % ( - rtype, attr.upper(), str(learning_status[rtype][attr]), str(testing_status[rtype][attr])) - final_bug_report = "Liveness assertion failed:\n" + \ - bug_report if bug_report != "" else "" + bug_report += ( + "[ERROR] %s %s inconsistency: %s seen after learning run, but %s seen after testing run\n" + % ( + rtype, + attr.upper(), + str(learning_status[rtype][attr]), + str(testing_status[rtype][attr]), + ) + ) + final_bug_report = ( + "Liveness assertion failed:\n" + bug_report if bug_report != "" else "" + ) return alarm, final_bug_report @@ -234,28 +275,48 @@ def preprocess_side_effect(side_effect, interest_objects): return result -def check_side_effect(learning_side_effect, testing_side_effect, interest_objects, effect_to_check, selective=True): +def check_side_effect( + learning_side_effect, + testing_side_effect, + interest_objects, + effect_to_check, + selective=True, +): alarm = 0 bug_report = "" # Preporcess regex learning_side_effect = preprocess_side_effect( - learning_side_effect, interest_objects) - testing_side_effect = preprocess_side_effect( - testing_side_effect, interest_objects) + learning_side_effect, interest_objects + ) + testing_side_effect = preprocess_side_effect(testing_side_effect, interest_objects) for interest in interest_objects: rtype = interest["rtype"] namespace = interest["namespace"] name = interest["name"] exist = True - if rtype not in learning_side_effect or namespace not in learning_side_effect[rtype] or name not in learning_side_effect[rtype][namespace]: + if ( + rtype not in learning_side_effect + or namespace not in learning_side_effect[rtype] + or name not in learning_side_effect[rtype][namespace] + ): bug_report += "[ERROR] %s/%s/%s not in learning side effect digest\n" % ( - rtype, namespace, name) + rtype, + namespace, + name, + ) alarm += 1 exist = False - if rtype not in testing_side_effect or namespace not in testing_side_effect[rtype] or name not in testing_side_effect[rtype][namespace]: + if ( + rtype not in testing_side_effect + or namespace not in testing_side_effect[rtype] + or name not in testing_side_effect[rtype][namespace] + ): bug_report += "[ERROR] %s/%s/%s not in testing side effect digest\n" % ( - rtype, namespace, name) + rtype, + namespace, + name, + ) alarm += 1 exist = False if exist: @@ -267,44 +328,89 @@ def check_side_effect(learning_side_effect, testing_side_effect, interest_object continue if learning_entry[attr] != testing_entry[attr]: alarm += 1 - bug_report += "[ERROR] %s/%s/%s %s inconsistency: %s events seen during learning run, but %s seen during testing run\n" % ( - rtype, namespace, name, attr.upper(), str(learning_entry[attr]), str(testing_entry[attr])) - final_bug_report = "Safety assertion failed:\n" + \ - bug_report if bug_report != "" else "" + bug_report += ( + "[ERROR] %s/%s/%s %s inconsistency: %s events seen during learning run, but %s seen during testing run\n" + % ( + rtype, + namespace, + name, + attr.upper(), + str(learning_entry[attr]), + str(testing_entry[attr]), + ) + ) + final_bug_report = ( + "Safety assertion failed:\n" + bug_report if bug_report != "" else "" + ) return alarm, final_bug_report def generate_time_travel_debugging_hint(testing_config): - desc = "Sieve makes the controller time travel back to the history to see the status just %s %s: %s" % ( - testing_config["timing"], - testing_config["ce-rtype"] + "/" + - testing_config["ce-namespace"] + "/" + testing_config["ce-name"], - testing_config["ce-diff-current"]) - suggestion = "Please check how controller reacts when seeing %s: %s, the controller might issue %s to %s without proper checking" % ( - testing_config["ce-rtype"] + "/" + - testing_config["ce-namespace"] + "/" + testing_config["ce-name"], - testing_config["ce-diff-current"], - "deletion" if testing_config["se-etype"] == "ADDED" else "creation", - testing_config["se-rtype"] + "/" + testing_config["se-namespace"] + "/" + testing_config["se-name"]) + desc = ( + "Sieve makes the controller time travel back to the history to see the status just %s %s: %s" + % ( + testing_config["timing"], + testing_config["ce-rtype"] + + "/" + + testing_config["ce-namespace"] + + "/" + + testing_config["ce-name"], + testing_config["ce-diff-current"], + ) + ) + suggestion = ( + "Please check how controller reacts when seeing %s: %s, the controller might issue %s to %s without proper checking" + % ( + testing_config["ce-rtype"] + + "/" + + testing_config["ce-namespace"] + + "/" + + testing_config["ce-name"], + testing_config["ce-diff-current"], + "deletion" if testing_config["se-etype"] == "ADDED" else "creation", + testing_config["se-rtype"] + + "/" + + testing_config["se-namespace"] + + "/" + + testing_config["se-name"], + ) + ) return desc + "\n" + suggestion + "\n" def generate_obs_gap_debugging_hint(testing_config): desc = "Sieve makes the controller miss the event %s: %s" % ( - testing_config["ce-rtype"] + "/" + - testing_config["ce-namespace"] + "/" + testing_config["ce-name"], - testing_config["ce-diff-current"]) - suggestion = "Please check how controller reacts when seeing %s: %s, the event can trigger a controller side effect, and it might be cancelled by following events" % ( - testing_config["ce-rtype"] + "/" + - testing_config["ce-namespace"] + "/" + testing_config["ce-name"], - testing_config["ce-diff-current"]) + testing_config["ce-rtype"] + + "/" + + testing_config["ce-namespace"] + + "/" + + testing_config["ce-name"], + testing_config["ce-diff-current"], + ) + suggestion = ( + "Please check how controller reacts when seeing %s: %s, the event can trigger a controller side effect, and it might be cancelled by following events" + % ( + testing_config["ce-rtype"] + + "/" + + testing_config["ce-namespace"] + + "/" + + testing_config["ce-name"], + testing_config["ce-diff-current"], + ) + ) return desc + "\n" + suggestion + "\n" -def look_for_discrepancy_in_digest(learning_side_effect, learning_status, testing_side_effect, testing_status, config, oracle_config): +def look_for_discrepancy_in_digest( + learning_side_effect, + learning_status, + testing_side_effect, + testing_status, + config, + oracle_config, +): testing_config = yaml.safe_load(open(config)) - alarm_status, bug_report_status = check_status( - learning_status, testing_status) + alarm_status, bug_report_status = check_status(learning_status, testing_status) alarm = alarm_status bug_report = bug_report_status # TODO: implement side effect checking for obs gap @@ -312,7 +418,12 @@ def look_for_discrepancy_in_digest(learning_side_effect, learning_status, testin interest_objects = [] if testing_config["mode"] == sieve_modes.TIME_TRAVEL: interest_objects.append( - {"rtype": testing_config["se-rtype"], "namespace": testing_config["se-namespace"], "name": testing_config["se-name"]}) + { + "rtype": testing_config["se-rtype"], + "namespace": testing_config["se-namespace"], + "name": testing_config["se-name"], + } + ) if "interest_objects" in oracle_config: interest_objects.extend(oracle_config["interest_objects"]) @@ -321,7 +432,8 @@ def look_for_discrepancy_in_digest(learning_side_effect, learning_status, testin effect_to_check.extend(oracle_config["effect_to_check"]) alarm_side_effect, bug_report_side_effect = check_side_effect( - learning_side_effect, testing_side_effect, interest_objects, effect_to_check) + learning_side_effect, testing_side_effect, interest_objects, effect_to_check + ) alarm += alarm_side_effect bug_report += bug_report_side_effect return alarm, bug_report @@ -333,12 +445,15 @@ def look_for_panic_in_operator_log(operator_log): file = open(operator_log) for line in file.readlines(): if "Observed a panic" in line: - panic_in_file = line[line.find("Observed a panic"):] + panic_in_file = line[line.find("Observed a panic") :] panic_in_file = panic_in_file.strip() bug_report += "[ERROR] %s\n" % panic_in_file alarm += 1 - final_bug_report = "Checking for any panic in operator log...\n" + \ - bug_report if bug_report != "" else "" + final_bug_report = ( + "Checking for any panic in operator log...\n" + bug_report + if bug_report != "" + else "" + ) return alarm, final_bug_report @@ -359,28 +474,48 @@ def generate_debugging_hint(testing_config): assert False -def check(learned_side_effect, learned_status, learned_resources, testing_side_effect, testing_status, testing_resources, test_config, operator_log, server_log, oracle_config): +def check( + learned_side_effect, + learned_status, + learned_resources, + testing_side_effect, + testing_status, + testing_resources, + test_config, + operator_log, + server_log, + oracle_config, +): testing_config = yaml.safe_load(open(test_config)) # Skip case which target side effect event not appear in operator log under time-travel mode - if testing_config["mode"] == sieve_modes.TIME_TRAVEL and not look_for_sleep_over_in_server_log(server_log): - bug_report = "[WARN] target side effect event did't appear under time-travel workload" + if testing_config[ + "mode" + ] == sieve_modes.TIME_TRAVEL and not look_for_sleep_over_in_server_log(server_log): + bug_report = ( + "[WARN] target side effect event did't appear under time-travel workload" + ) print(bug_report) return 0, bug_report discrepancy_alarm, discrepancy_bug_report = look_for_discrepancy_in_digest( - learned_side_effect, learned_status, testing_side_effect, testing_status, test_config, oracle_config) - panic_alarm, panic_bug_report = look_for_panic_in_operator_log( - operator_log) + learned_side_effect, + learned_status, + testing_side_effect, + testing_status, + test_config, + oracle_config, + ) + panic_alarm, panic_bug_report = look_for_panic_in_operator_log(operator_log) alarm = discrepancy_alarm + panic_alarm bug_report = discrepancy_bug_report + panic_bug_report # TODO(urgent): we should use learned_resources to replace learned_status, instead of using both # and look_for_resouces_diff() should return alarm as well if learned_resources != None: - bug_report += "\n" + \ - look_for_resouces_diff(learned_resources, testing_resources) + bug_report += "\n" + look_for_resouces_diff( + learned_resources, testing_resources + ) if alarm != 0: bug_report = "[BUG FOUND]\n" + bug_report - hint = "[DEBUGGING SUGGESTION]\n" + \ - generate_debugging_hint(testing_config) + hint = "[DEBUGGING SUGGESTION]\n" + generate_debugging_hint(testing_config) print(bcolors.FAIL + bug_report + bcolors.WARNING + hint + bcolors.ENDC) bug_report += hint return alarm, bug_report @@ -392,7 +527,7 @@ def look_for_resouces_diff(learn, test): test_trim = copy.deepcopy(test) trim_resource(learn_trim) trim_resource(test_trim) - diff = jd.diff(learn_trim, test_trim, syntax='symmetric') + diff = jd.diff(learn_trim, test_trim, syntax="symmetric") print("[RESOURCE DIFF REPORT]", file=f) for rtype in diff: @@ -401,21 +536,31 @@ def look_for_resouces_diff(learn, test): if jd.delete in rdiff: # Any resource deleted for (idx, item) in rdiff[jd.delete]: - print("[resource deleted]", rtype, item['metadata'] - ['namespace'], item['metadata']['name'], file=f) + print( + "[resource deleted]", + rtype, + item["metadata"]["namespace"], + item["metadata"]["name"], + file=f, + ) if jd.insert in rdiff: # Any resource added for (idx, item) in rdiff[jd.insert]: - print("[resource added]", rtype, item['metadata'] - ['namespace'], item['metadata']['name'], file=f) + print( + "[resource added]", + rtype, + item["metadata"]["namespace"], + item["metadata"]["name"], + file=f, + ) # Check for resource internal fields for idx in rdiff: if not type(idx) is int: continue resource = test[rtype][idx] - name = resource['metadata']['name'] - namespace = resource['metadata']['namespace'] + name = resource["metadata"]["name"] + namespace = resource["metadata"]["namespace"] item = rdiff[idx] for majorfield in item: if majorfield == jd.delete: @@ -427,14 +572,32 @@ def look_for_resouces_diff(learn, test): data = item[majorfield] for subfield in data: if not subfield in not_care: - print("[%s field changed]" % (majorfield), rtype, name, - subfield, "changed", "delta: ", data[subfield], file=f) + print( + "[%s field changed]" % (majorfield), + rtype, + name, + subfield, + "changed", + "delta: ", + data[subfield], + file=f, + ) if jd.delete in data: - print("[%s field deleted]" % (majorfield), - rtype, name, data[jd.delete], file=f) + print( + "[%s field deleted]" % (majorfield), + rtype, + name, + data[jd.delete], + file=f, + ) if jd.insert in data: - print("[%s field added]" % (majorfield), - rtype, name, data[jd.insert], file=f) + print( + "[%s field added]" % (majorfield), + rtype, + name, + data[jd.insert], + file=f, + ) result = f.getvalue() f.close() diff --git a/sieve.py b/sieve.py index b5f65c6b013..1508f0d4e4c 100644 --- a/sieve.py +++ b/sieve.py @@ -19,8 +19,7 @@ def watch_crd(project, addrs): for addr in addrs: for crd in controllers.CRDs[project]: - cmd_early_exit("kubectl get %s -s %s --ignore-not-found=true" % - (crd, addr)) + cmd_early_exit("kubectl get %s -s %s --ignore-not-found=true" % (crd, addr)) def generate_configmap(test_config): @@ -42,11 +41,11 @@ def generate_configmap(test_config): def generate_kind_config(mode, num_apiservers, num_workers): - kind_config_filename = "kind-%sa-%sw.yaml" % ( - str(num_apiservers), str(num_workers)) + kind_config_filename = "kind-%sa-%sw.yaml" % (str(num_apiservers), str(num_workers)) kind_config_file = open(kind_config_filename, "w") kind_config_file.writelines( - ["kind: Cluster\n", "apiVersion: kind.x-k8s.io/v1alpha4\n", "nodes:\n"]) + ["kind: Cluster\n", "apiVersion: kind.x-k8s.io/v1alpha4\n", "nodes:\n"] + ) for i in range(num_apiservers): kind_config_file.write("- role: control-plane\n") for i in range(num_workers): @@ -58,19 +57,22 @@ def generate_kind_config(mode, num_apiservers, num_workers): def redirect_workers(num_workers): target_master = sieve_config.config["time_travel_front_runner"] for i in range(num_workers): - worker = "kind-worker" + (str(i+1) if i > 0 else "") - cmd_early_exit("docker exec %s bash -c \"sed -i 's/kind-external-load-balancer/%s/g' /etc/kubernetes/kubelet.conf\"" % - (worker, target_master)) + worker = "kind-worker" + (str(i + 1) if i > 0 else "") cmd_early_exit( - "docker exec %s bash -c \"systemctl restart kubelet\"" % worker) + "docker exec %s bash -c \"sed -i 's/kind-external-load-balancer/%s/g' /etc/kubernetes/kubelet.conf\"" + % (worker, target_master) + ) + cmd_early_exit('docker exec %s bash -c "systemctl restart kubelet"' % worker) def redirect_kubectl(): client = docker.from_env() - cp_port = client.containers.get( - "kind-control-plane").attrs["NetworkSettings"]["Ports"]["6443/tcp"][0]["HostPort"] - balancer_port = client.containers.get( - "kind-external-load-balancer").attrs["NetworkSettings"]["Ports"]["6443/tcp"][0]["HostPort"] + cp_port = client.containers.get("kind-control-plane").attrs["NetworkSettings"][ + "Ports" + ]["6443/tcp"][0]["HostPort"] + balancer_port = client.containers.get("kind-external-load-balancer").attrs[ + "NetworkSettings" + ]["Ports"]["6443/tcp"][0]["HostPort"] kube_config = os.getenv("KUBECONFIG") target_prefix = " server: https://127.0.0.1:" fin = open(kube_config) @@ -95,25 +97,39 @@ def prepare_sieve_server(test_config): def start_sieve_server(): cmd_early_exit( - "docker exec kind-control-plane bash -c 'cd /sieve-server && ./sieve-server &> sieve-server.log &'") + "docker exec kind-control-plane bash -c 'cd /sieve-server && ./sieve-server &> sieve-server.log &'" + ) def stop_sieve_server(): - cmd_early_exit( - "docker exec kind-control-plane bash -c 'pkill sieve-server'") + cmd_early_exit("docker exec kind-control-plane bash -c 'pkill sieve-server'") def setup_kind_cluster(kind_config, docker_repo, docker_tag): - cmd_early_exit("kind create cluster --image %s/node:%s --config %s" % - (docker_repo, docker_tag, kind_config)) cmd_early_exit( - "docker exec kind-control-plane bash -c 'mkdir -p /root/.kube/ && cp /etc/kubernetes/admin.conf /root/.kube/config'") - - -def setup_cluster(project, stage, mode, test_config, docker_repo, docker_tag, num_apiservers, num_workers, pvc_resize): + "kind create cluster --image %s/node:%s --config %s" + % (docker_repo, docker_tag, kind_config) + ) + cmd_early_exit( + "docker exec kind-control-plane bash -c 'mkdir -p /root/.kube/ && cp /etc/kubernetes/admin.conf /root/.kube/config'" + ) + + +def setup_cluster( + project, + stage, + mode, + test_config, + docker_repo, + docker_tag, + num_apiservers, + num_workers, + pvc_resize, +): cmd_early_exit("kind delete cluster") - setup_kind_cluster(generate_kind_config( - mode, num_apiservers, num_workers), docker_repo, docker_tag) + setup_kind_cluster( + generate_kind_config(mode, num_apiservers, num_workers), docker_repo, docker_tag + ) # cmd_early_exit("kubectl create namespace %s" % sieve_config["namespace"]) # cmd_early_exit("kubectl config set-context --current --namespace=%s" % @@ -136,20 +152,25 @@ def setup_cluster(project, stage, mode, test_config, docker_repo, docker_tag, nu # Then we wait apiservers to be ready apiserver_list = [] for i in range(num_apiservers): - apiserver_name = "kube-apiserver-kind-control-plane" + \ - ("" if i == 0 else str(i + 1)) + apiserver_name = "kube-apiserver-kind-control-plane" + ( + "" if i == 0 else str(i + 1) + ) apiserver_list.append(apiserver_name) for tick in range(600): created = core_v1.list_namespaced_pod( - "kube-system", watch=False, label_selector="component=kube-apiserver").items - if len(created) == len(apiserver_list) and len(created) == len([item for item in created if item.status.phase == "Running"]): + "kube-system", watch=False, label_selector="component=kube-apiserver" + ).items + if len(created) == len(apiserver_list) and len(created) == len( + [item for item in created if item.status.phase == "Running"] + ): break time.sleep(1) for apiserver in apiserver_list: - cmd_early_exit("kubectl cp %s %s:/sieve.yaml -n kube-system" % - (test_config, apiserver)) + cmd_early_exit( + "kubectl cp %s %s:/sieve.yaml -n kube-system" % (test_config, apiserver) + ) # Preload operator image to kind nodes image = "%s/%s:%s" % (docker_repo, project, docker_tag) @@ -175,7 +196,10 @@ def start_operator(project, docker_repo, docker_tag, num_apiservers): print("Wait for operator pod ready...") for tick in range(600): project_pod = core_v1.list_namespaced_pod( - sieve_config.config["namespace"], watch=False, label_selector="sievetag="+project).items + sieve_config.config["namespace"], + watch=False, + label_selector="sievetag=" + project, + ).items if len(project_pod) >= 1: if project_pod[0].status.phase == "Running": break @@ -183,15 +207,31 @@ def start_operator(project, docker_repo, docker_tag, num_apiservers): apiserver_addr_list = [] for i in range(num_apiservers): - label_selector = "kubernetes.io/hostname=kind-control-plane" + \ - ("" if i == 0 else str(i + 1)) - apiserver_addr = "https://" + core_v1.list_node( - watch=False, label_selector=label_selector).items[0].status.addresses[0].address + ":6443" + label_selector = "kubernetes.io/hostname=kind-control-plane" + ( + "" if i == 0 else str(i + 1) + ) + apiserver_addr = ( + "https://" + + core_v1.list_node(watch=False, label_selector=label_selector) + .items[0] + .status.addresses[0] + .address + + ":6443" + ) apiserver_addr_list.append(apiserver_addr) watch_crd(project, apiserver_addr_list) -def run_workload(project, mode, test_workload, test_config, log_dir, docker_repo, docker_tag, num_apiservers): +def run_workload( + project, + mode, + test_workload, + test_config, + log_dir, + docker_repo, + docker_tag, + num_apiservers, +): cprint("Setting up Sieve server ...", bcolors.OKGREEN) start_sieve_server() ok("Sieve server set up") @@ -201,85 +241,194 @@ def run_workload(project, mode, test_workload, test_config, log_dir, docker_repo ok("Operator deployed") kubernetes.config.load_kube_config() - pod_name = kubernetes.client.CoreV1Api().list_namespaced_pod( - sieve_config.config["namespace"], watch=False, label_selector="sievetag="+project).items[0].metadata.name + pod_name = ( + kubernetes.client.CoreV1Api() + .list_namespaced_pod( + sieve_config.config["namespace"], + watch=False, + label_selector="sievetag=" + project, + ) + .items[0] + .metadata.name + ) streamed_log_file = open("%s/streamed-operator.log" % (log_dir), "w+") - streaming = subprocess.Popen("kubectl logs %s -f" % - pod_name, stdout=streamed_log_file, stderr=streamed_log_file, shell=True, preexec_fn=os.setsid) + streaming = subprocess.Popen( + "kubectl logs %s -f" % pod_name, + stdout=streamed_log_file, + stderr=streamed_log_file, + shell=True, + preexec_fn=os.setsid, + ) cprint("Running test workload ...", bcolors.OKGREEN) test_workload.run(mode) ok("Test workload finished") - pod_name = kubernetes.client.CoreV1Api().list_namespaced_pod( - sieve_config.config["namespace"], watch=False, label_selector="sievetag="+project).items[0].metadata.name + pod_name = ( + kubernetes.client.CoreV1Api() + .list_namespaced_pod( + sieve_config.config["namespace"], + watch=False, + label_selector="sievetag=" + project, + ) + .items[0] + .metadata.name + ) for i in range(num_apiservers): - apiserver_name = "kube-apiserver-kind-control-plane" + \ - ("" if i == 0 else str(i + 1)) + apiserver_name = "kube-apiserver-kind-control-plane" + ( + "" if i == 0 else str(i + 1) + ) apiserver_log = "apiserver%s.log" % (str(i + 1)) cmd_early_exit( - "kubectl logs %s -n kube-system > %s/%s" % (apiserver_name, log_dir, apiserver_log)) + "kubectl logs %s -n kube-system > %s/%s" + % (apiserver_name, log_dir, apiserver_log) + ) cmd_early_exit( - "docker cp kind-control-plane:/sieve-server/sieve-server.log %s/sieve-server.log" % (log_dir)) + "docker cp kind-control-plane:/sieve-server/sieve-server.log %s/sieve-server.log" + % (log_dir) + ) - cmd_early_exit( - "kubectl logs %s > %s/operator.log" % (pod_name, log_dir)) + cmd_early_exit("kubectl logs %s > %s/operator.log" % (pod_name, log_dir)) os.killpg(streaming.pid, signal.SIGTERM) streamed_log_file.close() stop_sieve_server() -def check_result(project, mode, stage, test_config, log_dir, data_dir, two_sided, oracle_config): +def check_result( + project, mode, stage, test_config, log_dir, data_dir, two_sided, oracle_config +): if stage == "learn": analyze.analyze_trace(project, log_dir, two_sided=two_sided) cmd_early_exit("mkdir -p %s" % data_dir) - cmd_early_exit("cp %s %s" % (os.path.join(log_dir, "status.json"), os.path.join( - data_dir, "status.json"))) - cmd_early_exit("cp %s %s" % (os.path.join(log_dir, "side-effect.json"), os.path.join( - data_dir, "side-effect.json"))) - cmd_early_exit("cp %s %s" % (os.path.join(log_dir, "resources.json"), os.path.join( - data_dir, "resources.json"))) + cmd_early_exit( + "cp %s %s" + % ( + os.path.join(log_dir, "status.json"), + os.path.join(data_dir, "status.json"), + ) + ) + cmd_early_exit( + "cp %s %s" + % ( + os.path.join(log_dir, "side-effect.json"), + os.path.join(data_dir, "side-effect.json"), + ) + ) + cmd_early_exit( + "cp %s %s" + % ( + os.path.join(log_dir, "resources.json"), + os.path.join(data_dir, "resources.json"), + ) + ) else: if os.path.exists(test_config): - open(os.path.join(log_dir, "config.yaml"), - "w").write(open(test_config).read()) + open(os.path.join(log_dir, "config.yaml"), "w").write( + open(test_config).read() + ) if mode != sieve_modes.VANILLA: - learned_side_effect = json.load(open(os.path.join( - data_dir, "side-effect.json"))) - learned_status = json.load(open(os.path.join( - data_dir, "status.json"))) + learned_side_effect = json.load( + open(os.path.join(data_dir, "side-effect.json")) + ) + learned_status = json.load(open(os.path.join(data_dir, "status.json"))) resources_path = os.path.join(data_dir, "resources.json") - learned_resources = json.load( - open(resources_path)) if os.path.isfile(resources_path) else None + learned_resources = ( + json.load(open(resources_path)) + if os.path.isfile(resources_path) + else None + ) server_log = os.path.join(log_dir, "sieve-server.log") - testing_side_effect, testing_status, testing_resources = oracle.generate_digest( - server_log) + ( + testing_side_effect, + testing_status, + testing_resources, + ) = oracle.generate_digest(server_log) operator_log = os.path.join(log_dir, "streamed-operator.log") - alarm, bug_report = oracle.check(learned_side_effect, learned_status, learned_resources, - testing_side_effect, testing_status, testing_resources, test_config, operator_log, server_log, oracle_config) + alarm, bug_report = oracle.check( + learned_side_effect, + learned_status, + learned_resources, + testing_side_effect, + testing_status, + testing_resources, + test_config, + operator_log, + server_log, + oracle_config, + ) open(os.path.join(log_dir, "bug-report.txt"), "w").write(bug_report) - json.dump(testing_side_effect, open(os.path.join( - log_dir, "side-effect.json"), "w"), indent=4) - json.dump(testing_status, open(os.path.join( - log_dir, "status.json"), "w"), indent=4) - json.dump(testing_resources, open(os.path.join( - log_dir, "resources.json"), "w"), indent=4) + json.dump( + testing_side_effect, + open(os.path.join(log_dir, "side-effect.json"), "w"), + indent=4, + ) + json.dump( + testing_status, + open(os.path.join(log_dir, "status.json"), "w"), + indent=4, + ) + json.dump( + testing_resources, + open(os.path.join(log_dir, "resources.json"), "w"), + indent=4, + ) return alarm return 0 -def run_test(project, mode, stage, test_workload, test_config, log_dir, docker_repo, docker_tag, num_apiservers, num_workers, pvc_resize, oracle_config, data_dir, two_sided, phase): +def run_test( + project, + mode, + stage, + test_workload, + test_config, + log_dir, + docker_repo, + docker_tag, + num_apiservers, + num_workers, + pvc_resize, + oracle_config, + data_dir, + two_sided, + phase, +): if phase == "all" or phase == "setup_only": - setup_cluster(project, stage, mode, test_config, - docker_repo, docker_tag, num_apiservers, num_workers, pvc_resize) + setup_cluster( + project, + stage, + mode, + test_config, + docker_repo, + docker_tag, + num_apiservers, + num_workers, + pvc_resize, + ) if phase == "all" or phase == "workload_only" or phase == "workload_and_check": - run_workload(project, mode, test_workload, test_config, - log_dir, docker_repo, docker_tag, num_apiservers) + run_workload( + project, + mode, + test_workload, + test_config, + log_dir, + docker_repo, + docker_tag, + num_apiservers, + ) if phase == "all" or phase == "check_only" or phase == "workload_and_check": - return check_result(project, mode, stage, test_config, - log_dir, data_dir, two_sided, oracle_config) + return check_result( + project, + mode, + stage, + test_config, + log_dir, + data_dir, + two_sided, + oracle_config, + ) return 0 @@ -299,7 +448,18 @@ def generate_learn_config(learn_config, project, mode, rate_limiter_enabled): yaml.dump(learn_config_map, open(learn_config, "w"), sort_keys=False) -def run(test_suites, project, test, log_dir, mode, stage, config, docker, rate_limiter_enabled=False, phase="all"): +def run( + test_suites, + project, + test, + log_dir, + mode, + stage, + config, + docker, + rate_limiter_enabled=False, + phase="all", +): suite = test_suites[project][test] data_dir = os.path.join("data", project, test, "learn") print("Log dir: %s" % log_dir) @@ -310,25 +470,68 @@ def run(test_suites, project, test, log_dir, mode, stage, config, docker, rate_l if stage == "learn": learn_config = os.path.join(log_dir, "learn.yaml") print("Learning stage with config %s" % learn_config) - generate_learn_config(learn_config, project, - mode, rate_limiter_enabled) - return run_test(project, mode, stage, suite.workload, - learn_config, log_dir, docker, stage, suite.num_apiservers, suite.num_workers, suite.pvc_resize, suite.oracle_config, data_dir, suite.two_sided, phase) + generate_learn_config(learn_config, project, mode, rate_limiter_enabled) + return run_test( + project, + mode, + stage, + suite.workload, + learn_config, + log_dir, + docker, + stage, + suite.num_apiservers, + suite.num_workers, + suite.pvc_resize, + suite.oracle_config, + data_dir, + suite.two_sided, + phase, + ) else: if mode == sieve_modes.VANILLA: blank_config = "config/none.yaml" - return run_test(project, mode, stage, suite.workload, - blank_config, log_dir, docker, mode, suite.num_apiservers, suite.num_workers, suite.pvc_resize, suite.oracle_config, data_dir, suite.two_sided, phase) + return run_test( + project, + mode, + stage, + suite.workload, + blank_config, + log_dir, + docker, + mode, + suite.num_apiservers, + suite.num_workers, + suite.pvc_resize, + suite.oracle_config, + data_dir, + suite.two_sided, + phase, + ) else: test_config = config if config != "none" else suite.config print("Testing with config: %s" % test_config) - test_config_to_use = os.path.join( - log_dir, os.path.basename(test_config)) + test_config_to_use = os.path.join(log_dir, os.path.basename(test_config)) cmd_early_exit("cp %s %s" % (test_config, test_config_to_use)) if mode == sieve_modes.TIME_TRAVEL: suite.num_apiservers = 3 - return run_test(project, mode, stage, suite.workload, - test_config_to_use, log_dir, docker, mode, suite.num_apiservers, suite.num_workers, suite.pvc_resize, suite.oracle_config, data_dir, suite.two_sided, phase) + return run_test( + project, + mode, + stage, + suite.workload, + test_config_to_use, + log_dir, + docker, + mode, + suite.num_apiservers, + suite.num_workers, + suite.pvc_resize, + suite.oracle_config, + data_dir, + suite.two_sided, + phase, + ) def run_batch(project, test, dir, mode, stage, docker): @@ -338,15 +541,25 @@ def run_batch(project, test, dir, mode, stage, docker): configs.sort(key=lambda config: config.split("-")[-1].split(".")[0]) print("Configs to test:") print("\n".join(configs)) - batch_test_result = open("bt_%s_%s_%s_%s.tsv" % ( - project, test, mode, datetime.now().strftime("%Y%m%d%H%M%S")), "w") + batch_test_result = open( + "bt_%s_%s_%s_%s.tsv" + % (project, test, mode, datetime.now().strftime("%Y%m%d%H%M%S")), + "w", + ) for config in configs: num = os.path.basename(config).split(".")[0] - log_dir = os.path.join( - dir, project, test, stage, mode + "-batch", num) + log_dir = os.path.join(dir, project, test, stage, mode + "-batch", num) try: - alarm = run(controllers.test_suites, project, - test, log_dir, mode, stage, config, docker) + alarm = run( + controllers.test_suites, + project, + test, + log_dir, + mode, + stage, + config, + docker, + ) batch_test_result.write("%s\t%s\n" % (config, str(alarm))) if alarm != 0: cprint("Bug happens when running %s" % config, bcolors.FAIL) @@ -359,26 +572,75 @@ def run_batch(project, test, dir, mode, stage, docker): s = time.time() usage = "usage: python3 sieve.py [options]" parser = optparse.OptionParser(usage=usage) - parser.add_option("-p", "--project", dest="project", - help="specify PROJECT to test: cassandra-operator or zookeeper-operator", metavar="PROJECT", default="cassandra-operator") - parser.add_option("-t", "--test", dest="test", - help="specify TEST to run", metavar="TEST", default="recreate") - parser.add_option("-d", "--docker", dest="docker", - help="DOCKER repo that you have access", metavar="DOCKER", default=sieve_config.config["docker_repo"]) - parser.add_option("-l", "--log", dest="log", - help="save to LOG", metavar="LOG", default="log") - parser.add_option("-m", "--mode", dest="mode", - help="test MODE: vanilla, time-travel, obs-gap, atom-vio", metavar="MODE", default="none") - parser.add_option("-c", "--config", dest="config", - help="test CONFIG", metavar="CONFIG", default="none") - parser.add_option("-b", "--batch", dest="batch", action="store_true", - help="batch mode or not", default=False) - parser.add_option("--phase", dest="phase", - help="run the PHASE: setup_only, workload_only, check_only or all", metavar="PHASE", default="all") - parser.add_option("-s", "--stage", dest="stage", - help="STAGE: learn, test", default="test") - parser.add_option("-r", "--rate_limiter", dest="rate_limiter", action="store_true", - help="use RATE LIMITER in learning stage or not", default=False) + parser.add_option( + "-p", + "--project", + dest="project", + help="specify PROJECT to test: cassandra-operator or zookeeper-operator", + metavar="PROJECT", + default="cassandra-operator", + ) + parser.add_option( + "-t", + "--test", + dest="test", + help="specify TEST to run", + metavar="TEST", + default="recreate", + ) + parser.add_option( + "-d", + "--docker", + dest="docker", + help="DOCKER repo that you have access", + metavar="DOCKER", + default=sieve_config.config["docker_repo"], + ) + parser.add_option( + "-l", "--log", dest="log", help="save to LOG", metavar="LOG", default="log" + ) + parser.add_option( + "-m", + "--mode", + dest="mode", + help="test MODE: vanilla, time-travel, obs-gap, atom-vio", + metavar="MODE", + default="none", + ) + parser.add_option( + "-c", + "--config", + dest="config", + help="test CONFIG", + metavar="CONFIG", + default="none", + ) + parser.add_option( + "-b", + "--batch", + dest="batch", + action="store_true", + help="batch mode or not", + default=False, + ) + parser.add_option( + "--phase", + dest="phase", + help="run the PHASE: setup_only, workload_only, check_only or all", + metavar="PHASE", + default="all", + ) + parser.add_option( + "-s", "--stage", dest="stage", help="STAGE: learn, test", default="test" + ) + parser.add_option( + "-r", + "--rate_limiter", + dest="rate_limiter", + action="store_true", + help="use RATE LIMITER in learning stage or not", + default=False, + ) (options, args) = parser.parse_args() @@ -393,22 +655,49 @@ def run_batch(project, test, dir, mode, stage, docker): if options.mode == "none" and options.stage == "test": options.mode = controllers.test_suites[options.project][options.test].mode - assert options.stage in [ - "learn", "test"], "invalid stage option: %s" % options.stage - assert options.mode in [sieve_modes.VANILLA, sieve_modes.TIME_TRAVEL, - sieve_modes.OBS_GAP, sieve_modes.ATOM_VIO, "learn"], "invalid mode option: %s" % options.mode - assert options.phase in ["all", "setup_only", "workload_only", - "check_only", "workload_and_check"], "invalid phase option: %s" % options.phase - - print("Running Sieve with %s: %s..." % - (options.stage, options.mode)) + assert options.stage in ["learn", "test"], ( + "invalid stage option: %s" % options.stage + ) + assert options.mode in [ + sieve_modes.VANILLA, + sieve_modes.TIME_TRAVEL, + sieve_modes.OBS_GAP, + sieve_modes.ATOM_VIO, + "learn", + ], ("invalid mode option: %s" % options.mode) + assert options.phase in [ + "all", + "setup_only", + "workload_only", + "check_only", + "workload_and_check", + ], ("invalid phase option: %s" % options.phase) + + print("Running Sieve with %s: %s..." % (options.stage, options.mode)) if options.batch: - run_batch(options.project, options.test, - options.log, options.mode, options.stage, options.docker) + run_batch( + options.project, + options.test, + options.log, + options.mode, + options.stage, + options.docker, + ) else: - log_dir = os.path.join(options.log, options.project, - options.test, options.stage, options.mode) - run(controllers.test_suites, options.project, options.test, log_dir, - options.mode, options.stage, options.config, options.docker, options.rate_limiter, options.phase) + log_dir = os.path.join( + options.log, options.project, options.test, options.stage, options.mode + ) + run( + controllers.test_suites, + options.project, + options.test, + log_dir, + options.mode, + options.stage, + options.config, + options.docker, + options.rate_limiter, + options.phase, + ) print("Total time: {} seconds".format(time.time() - s)) diff --git a/sieve_config.py b/sieve_config.py index ccea78181ea..a0023713313 100644 --- a/sieve_config.py +++ b/sieve_config.py @@ -1,4 +1,3 @@ - config = { "docker_repo": "xudongs", "namespace": "default", diff --git a/sieve_tui.py b/sieve_tui.py index c565764fd64..407a56cd537 100755 --- a/sieve_tui.py +++ b/sieve_tui.py @@ -5,80 +5,100 @@ import json - def run(): - data = json.loads(open(".sieve_tui_session").read()) - operator = data["operator"] - workload = data["workload"] - action = data["action"] - mode = controllers.test_suites[operator][workload].mode - print(operator, workload, action, mode) - if action == "learn": - os.system("python3 sieve.py -p %s -t %s -s learn"%(operator, workload)) - elif action == "test": - os.system("python3 sieve.py -p %s -t %s"%(operator, workload)) - elif action == "build for learn": - os.system("python3 build.py -p %s -m learn"%(operator)) - elif action == "build for test": - os.system("python3 build.py -p %s -m %s"%(operator, mode)) - elif action == "learn check only": - os.system("python3 sieve.py -p %s -t %s -s learn --phase=check_only"%(operator, workload)) - elif action == "test check only": - os.system("python3 sieve.py -p %s -t %s --phase=check_only"%(operator, workload)) - -class SieveTUI: - - # We add type annotations to our master PyCUI objects for improved intellisense - def __init__(self, master: py_cui.PyCUI): - - self.master = master - - # The scrolled list cells that will contain our tasks in each of the three categories - self.operator_scroll_cell = self.master.add_scroll_menu('Operator', 0, 0, row_span=6, column_span=2) - self.workload_scroll_cell = self.master.add_scroll_menu('Workload', 0, 2, row_span=6, column_span=2) - self.action_scroll_cell = self.master.add_scroll_menu('Action', 0, 4, row_span=6, column_span=2) - - self.operator_scroll_cell.add_item_list(controllers.test_suites.keys()) + data = json.loads(open(".sieve_tui_session").read()) + operator = data["operator"] + workload = data["workload"] + action = data["action"] + mode = controllers.test_suites[operator][workload].mode + print(operator, workload, action, mode) + if action == "learn": + os.system("python3 sieve.py -p %s -t %s -s learn" % (operator, workload)) + elif action == "test": + os.system("python3 sieve.py -p %s -t %s" % (operator, workload)) + elif action == "build for learn": + os.system("python3 build.py -p %s -m learn" % (operator)) + elif action == "build for test": + os.system("python3 build.py -p %s -m %s" % (operator, mode)) + elif action == "learn check only": + os.system( + "python3 sieve.py -p %s -t %s -s learn --phase=check_only" + % (operator, workload) + ) + elif action == "test check only": + os.system( + "python3 sieve.py -p %s -t %s --phase=check_only" % (operator, workload) + ) - self.operator_scroll_cell.add_key_command(py_cui.keys.KEY_ENTER, self.goto_workload) - self.workload_scroll_cell.add_key_command(py_cui.keys.KEY_ENTER, self.goto_action) - self.action_scroll_cell.add_key_command(py_cui.keys.KEY_ENTER, self.take_action) - self.master.set_selected_widget(self.operator_scroll_cell.get_id()) - - - def goto_workload(self): - operator = self.operator_scroll_cell.get() - self.workload_scroll_cell.clear() - self.workload_scroll_cell.add_item_list(controllers.test_suites[operator].keys()) - self.master.set_selected_widget(self.workload_scroll_cell.get_id()) - - def goto_action(self): - operator = self.operator_scroll_cell.get() - workload = self.workload_scroll_cell.get() - actions = ["learn", "test", "build for learn", "build for test", "learn check only", "test check only"] - self.action_scroll_cell.clear() - self.action_scroll_cell.add_item_list(actions) - self.master.set_selected_widget(self.action_scroll_cell.get_id()) - - def take_action(self): - operator = self.operator_scroll_cell.get() - workload = self.workload_scroll_cell.get() - action = self.action_scroll_cell.get() - - data = { - "operator": operator, - "workload": workload, - "action": action - } +class SieveTUI: - open(".sieve_tui_session", "w").write(json.dumps(data)) + # We add type annotations to our master PyCUI objects for improved intellisense + def __init__(self, master: py_cui.PyCUI): + + self.master = master + + # The scrolled list cells that will contain our tasks in each of the three categories + self.operator_scroll_cell = self.master.add_scroll_menu( + "Operator", 0, 0, row_span=6, column_span=2 + ) + self.workload_scroll_cell = self.master.add_scroll_menu( + "Workload", 0, 2, row_span=6, column_span=2 + ) + self.action_scroll_cell = self.master.add_scroll_menu( + "Action", 0, 4, row_span=6, column_span=2 + ) + + self.operator_scroll_cell.add_item_list(controllers.test_suites.keys()) + + self.operator_scroll_cell.add_key_command( + py_cui.keys.KEY_ENTER, self.goto_workload + ) + self.workload_scroll_cell.add_key_command( + py_cui.keys.KEY_ENTER, self.goto_action + ) + self.action_scroll_cell.add_key_command(py_cui.keys.KEY_ENTER, self.take_action) + + self.master.set_selected_widget(self.operator_scroll_cell.get_id()) + + def goto_workload(self): + operator = self.operator_scroll_cell.get() + self.workload_scroll_cell.clear() + self.workload_scroll_cell.add_item_list( + controllers.test_suites[operator].keys() + ) + self.master.set_selected_widget(self.workload_scroll_cell.get_id()) + + def goto_action(self): + operator = self.operator_scroll_cell.get() + workload = self.workload_scroll_cell.get() + actions = [ + "learn", + "test", + "build for learn", + "build for test", + "learn check only", + "test check only", + ] + self.action_scroll_cell.clear() + self.action_scroll_cell.add_item_list(actions) + self.master.set_selected_widget(self.action_scroll_cell.get_id()) + + def take_action(self): + operator = self.operator_scroll_cell.get() + workload = self.workload_scroll_cell.get() + action = self.action_scroll_cell.get() + + data = {"operator": operator, "workload": workload, "action": action} + + open(".sieve_tui_session", "w").write(json.dumps(data)) + + self.master.run_on_exit(run) + self.master.stop() - self.master.run_on_exit(run) - self.master.stop() # Create the CUI with 7 rows 6 columns, pass it to the wrapper object, and start it root = py_cui.PyCUI(7, 6) -root.set_title('Sieve Terminal UI') +root.set_title("Sieve Terminal UI") s = SieveTUI(root) root.start() diff --git a/test_framework.py b/test_framework.py index 09ebc46dda3..18d3d0cb672 100644 --- a/test_framework.py +++ b/test_framework.py @@ -10,7 +10,8 @@ def get_pod(resource_name): kubernetes.config.load_kube_config() core_v1 = kubernetes.client.CoreV1Api() pods = core_v1.list_namespaced_pod( - namespace=sieve_config.config["namespace"], watch=False).items + namespace=sieve_config.config["namespace"], watch=False + ).items target_pod = None for pod in pods: if pod.metadata.name == resource_name: @@ -28,7 +29,8 @@ def get_sts(resource_name): kubernetes.config.load_kube_config() apps_v1 = kubernetes.client.AppsV1Api() statefulsets = apps_v1.list_namespaced_stateful_set( - namespace=sieve_config.config["namespace"], watch=False).items + namespace=sieve_config.config["namespace"], watch=False + ).items target_sts = None for sts in statefulsets: if sts.metadata.name == resource_name: @@ -40,7 +42,8 @@ def get_pvc(resource_name): kubernetes.config.load_kube_config() core_v1 = kubernetes.client.CoreV1Api() pvcs = core_v1.list_namespaced_persistent_volume_claim( - namespace=sieve_config.config["namespace"], watch=False).items + namespace=sieve_config.config["namespace"], watch=False + ).items target_pvc = None for pvc in pvcs: if pvc.metadata.name == resource_name: @@ -57,7 +60,8 @@ def get_secret(resource_name): kubernetes.config.load_kube_config() core_v1 = kubernetes.client.CoreV1Api() secrets = core_v1.list_namespaced_secret( - namespace=sieve_config.config["namespace"], watch=False).items + namespace=sieve_config.config["namespace"], watch=False + ).items for secret in secrets: if secret.metadata.name == resource_name: return secret @@ -73,7 +77,8 @@ def get_service(resource_name): kubernetes.config.load_kube_config() core_v1 = kubernetes.client.CoreV1Api() services = core_v1.list_namespaced_service( - namespace=sieve_config.config["namespace"], watch=False).items + namespace=sieve_config.config["namespace"], watch=False + ).items for service in services: if service.metadata.name == resource_name: return service @@ -102,7 +107,9 @@ def run(self, mode): class TestWaitForStatus: - def __init__(self, resource_type, resource_name, status, obs_gap_waiting_time, time_out=600): + def __init__( + self, resource_type, resource_name, status, obs_gap_waiting_time, time_out=600 + ): self.resource_type = resource_type self.resource_name = resource_name self.status = status @@ -151,16 +158,20 @@ def check_pvc(self): def run(self, mode): s = time.time() - print("wait until %s %s becomes %s..." % - (self.resource_type, self.resource_name, self.status)) + print( + "wait until %s %s becomes %s..." + % (self.resource_type, self.resource_name, self.status) + ) if mode == common.sieve_modes.OBS_GAP and self.obs_gap_waiting_time != -1: time.sleep(self.obs_gap_waiting_time) print("obs gap waiting time is reached") else: while True: if time.time() - s > float(self.time_out): - print("[ERROR] waiting timeout: %s does not become %s within %d seconds" % - (self.resource_name, self.status, self.time_out)) + print( + "[ERROR] waiting timeout: %s does not become %s within %d seconds" + % (self.resource_name, self.status, self.time_out) + ) os.system("kubectl describe pods") return 1 if self.resource_type == common.POD: @@ -177,7 +188,14 @@ def run(self, mode): class TestWaitForStorage: - def __init__(self, resource_type, resource_name, storage_size, obs_gap_waiting_time, time_out=600): + def __init__( + self, + resource_type, + resource_name, + storage_size, + obs_gap_waiting_time, + time_out=600, + ): self.resource_type = resource_type self.resource_name = resource_name self.storage_size = storage_size @@ -190,22 +208,29 @@ def check_sts(self): if sts is None: return False for volume_claim_template in sts.spec.volume_claim_templates: - if volume_claim_template.spec.resources.requests["storage"] == self.storage_size: + if ( + volume_claim_template.spec.resources.requests["storage"] + == self.storage_size + ): return True return True def run(self, mode): s = time.time() - print("wait until %s %s has storage size %s..." % - (self.resource_type, self.resource_name, self.storage_size)) + print( + "wait until %s %s has storage size %s..." + % (self.resource_type, self.resource_name, self.storage_size) + ) if mode == common.sieve_modes.OBS_GAP and self.obs_gap_waiting_time != -1: time.sleep(self.obs_gap_waiting_time) print("obs gap waiting time is reached") else: while True: if time.time() - s > float(self.time_out): - print("[ERROR] waiting timeout: %s does not have storage size %s within %d seconds" % - (self.resource_name, self.storage_size, self.time_out)) + print( + "[ERROR] waiting timeout: %s does not have storage size %s within %d seconds" + % (self.resource_name, self.storage_size, self.time_out) + ) return 1 if self.resource_type == common.STS: if self.check_sts(): @@ -218,14 +243,21 @@ def run(self, mode): class TestWaitForExistence: - def __init__(self, resource_type, resource_name, exist: bool, obs_gap_waiting_time, time_out=600): - '''Constructor + def __init__( + self, + resource_type, + resource_name, + exist: bool, + obs_gap_waiting_time, + time_out=600, + ): + """Constructor Parameters: resource_type -- type of resource, e.g. Secret, Service resource_name -- name of the resource exist -- True for waiting for the resource to exist, False to waiting for the resource to disappear - ''' + """ self.resource_type = resource_type self.resource_name = resource_name self.exist = exist @@ -268,16 +300,24 @@ def check_service(self): def run(self, mode): s = time.time() - print("wait until %s %s %s..." % - (self.resource_type, self.resource_name, "exist" if self.exist else "nonexist")) + print( + "wait until %s %s %s..." + % ( + self.resource_type, + self.resource_name, + "exist" if self.exist else "nonexist", + ) + ) if mode == common.sieve_modes.OBS_GAP and self.obs_gap_waiting_time != -1: time.sleep(self.obs_gap_waiting_time) print("obs gap waiting time is reached") else: while True: if time.time() - s > float(self.time_out): - print("[ERROR] waiting timeout: %s does not become %s within %d seconds" % - (self.resource_name, self.status, self.time_out)) + print( + "[ERROR] waiting timeout: %s does not become %s within %d seconds" + % (self.resource_name, self.status, self.time_out) + ) os.system("kubectl describe pods") return 1 if self.resource_type == common.SECRET: @@ -304,31 +344,42 @@ def cmd(self, cmd): def wait_for_pod_status(self, pod_name, status, obs_gap_waiting_time=-1): test_wait = TestWaitForStatus( - common.POD, pod_name, status, obs_gap_waiting_time) + common.POD, pod_name, status, obs_gap_waiting_time + ) self.work_list.append(test_wait) return self def wait_for_pvc_status(self, pvc_name, status, obs_gap_waiting_time=-1): test_wait = TestWaitForStatus( - common.PVC, pvc_name, status, obs_gap_waiting_time) + common.PVC, pvc_name, status, obs_gap_waiting_time + ) self.work_list.append(test_wait) return self - def wait_for_secret_existence(self, secret_name, exist: bool, obs_gap_waiting_time=-1): + def wait_for_secret_existence( + self, secret_name, exist: bool, obs_gap_waiting_time=-1 + ): test_wait = TestWaitForExistence( - common.SECRET, secret_name, exist, obs_gap_waiting_time) + common.SECRET, secret_name, exist, obs_gap_waiting_time + ) self.work_list.append(test_wait) return self - def wait_for_service_existence(self, service_name, exist: bool, obs_gap_waiting_time=-1): + def wait_for_service_existence( + self, service_name, exist: bool, obs_gap_waiting_time=-1 + ): test_wait = TestWaitForExistence( - common.SERVICE, service_name, exist, obs_gap_waiting_time) + common.SERVICE, service_name, exist, obs_gap_waiting_time + ) self.work_list.append(test_wait) return self - def wait_for_sts_storage_size(self, sts_name, storage_size, obs_gap_waiting_time=-1): + def wait_for_sts_storage_size( + self, sts_name, storage_size, obs_gap_waiting_time=-1 + ): test_wait = TestWaitForStorage( - common.STS, sts_name, storage_size, obs_gap_waiting_time) + common.STS, sts_name, storage_size, obs_gap_waiting_time + ) self.work_list.append(test_wait) return self diff --git a/workloads.py b/workloads.py index 270f437cae1..5f7ef767517 100644 --- a/workloads.py +++ b/workloads.py @@ -5,109 +5,208 @@ workloads = { "cassandra-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-cassandra-operator/test/cdc-1.yaml").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.RUNNING) - .cmd("kubectl delete CassandraDataCenter cassandra-datacenter").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.TERMINATED) - .cmd("kubectl apply -f test-cassandra-operator/test/cdc-1.yaml").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.RUNNING) + .cmd("kubectl apply -f test-cassandra-operator/test/cdc-1.yaml") + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.RUNNING) + .cmd("kubectl delete CassandraDataCenter cassandra-datacenter") + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.TERMINATED) + .cmd("kubectl apply -f test-cassandra-operator/test/cdc-1.yaml") + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.RUNNING) .wait(50), "scaledown-scaleup": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-cassandra-operator/test/cdc-2.yaml").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.RUNNING) - .cmd("kubectl patch CassandraDataCenter cassandra-datacenter --type merge -p='{\"spec\":{\"nodes\":1}}'").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.TERMINATED, 80).wait_for_pvc_status("data-volume-cassandra-test-cluster-dc1-rack1-1", common.TERMINATED, 0) - .cmd("kubectl patch CassandraDataCenter cassandra-datacenter --type merge -p='{\"spec\":{\"nodes\":2}}'").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.RUNNING) + .cmd("kubectl apply -f test-cassandra-operator/test/cdc-2.yaml") + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.RUNNING) + .cmd( + 'kubectl patch CassandraDataCenter cassandra-datacenter --type merge -p=\'{"spec":{"nodes":1}}\'' + ) + .wait_for_pod_status( + "cassandra-test-cluster-dc1-rack1-1", common.TERMINATED, 80 + ) + .wait_for_pvc_status( + "data-volume-cassandra-test-cluster-dc1-rack1-1", common.TERMINATED, 0 + ) + .cmd( + 'kubectl patch CassandraDataCenter cassandra-datacenter --type merge -p=\'{"spec":{"nodes":2}}\'' + ) + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.RUNNING) .wait(50), "scaledown": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-cassandra-operator/test/cdc-2.yaml").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.RUNNING).wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.RUNNING) - .cmd("kubectl apply -f test-cassandra-operator/test/cdc-1.yaml").wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.TERMINATED) + .cmd("kubectl apply -f test-cassandra-operator/test/cdc-2.yaml") + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-0", common.RUNNING) + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.RUNNING) + .cmd("kubectl apply -f test-cassandra-operator/test/cdc-1.yaml") + .wait_for_pod_status("cassandra-test-cluster-dc1-rack1-1", common.TERMINATED) .wait(50), }, "casskop-operator": { "recreate": test_framework.new_built_in_workload() .cmd("kubectl apply -f test-casskop-operator/test/cassandra-configmap-v1.yaml") - .cmd("kubectl apply -f test-casskop-operator/test/cc-1.yaml").wait_for_pod_status("cassandra-cluster-dc1-rack1-0", common.RUNNING) - .cmd("kubectl delete CassandraCluster cassandra-cluster").wait_for_pod_status("cassandra-cluster-dc1-rack1-0", common.TERMINATED) - .cmd("kubectl apply -f test-casskop-operator/test/cc-1.yaml").wait_for_pod_status("cassandra-cluster-dc1-rack1-0", common.RUNNING) + .cmd("kubectl apply -f test-casskop-operator/test/cc-1.yaml") + .wait_for_pod_status("cassandra-cluster-dc1-rack1-0", common.RUNNING) + .cmd("kubectl delete CassandraCluster cassandra-cluster") + .wait_for_pod_status("cassandra-cluster-dc1-rack1-0", common.TERMINATED) + .cmd("kubectl apply -f test-casskop-operator/test/cc-1.yaml") + .wait_for_pod_status("cassandra-cluster-dc1-rack1-0", common.RUNNING) .wait(50), "reducepdb": test_framework.new_built_in_workload() .cmd("kubectl apply -f test-casskop-operator/test/cassandra-configmap-v1.yaml") - .cmd("kubectl apply -f test-casskop-operator/test/cc-2.yaml").wait(60) - .cmd("kubectl apply -f test-casskop-operator/test/cc-1.yaml").wait(60) + .cmd("kubectl apply -f test-casskop-operator/test/cc-2.yaml") + .wait(60) + .cmd("kubectl apply -f test-casskop-operator/test/cc-1.yaml") + .wait(60) .wait(50), "nodesperrack": test_framework.new_built_in_workload() .cmd("kubectl apply -f test-casskop-operator/test/cassandra-configmap-v1.yaml") - .cmd("kubectl apply -f test-casskop-operator/test/nodes-2.yaml").wait_for_pod_status("cassandra-cluster-dc1-rack1-1", common.RUNNING) - .cmd("kubectl apply -f test-casskop-operator/test/nodes-1.yaml").wait(10) - .cmd("kubectl apply -f test-casskop-operator/test/nodes-0.yaml").wait(10) + .cmd("kubectl apply -f test-casskop-operator/test/nodes-2.yaml") + .wait_for_pod_status("cassandra-cluster-dc1-rack1-1", common.RUNNING) + .cmd("kubectl apply -f test-casskop-operator/test/nodes-1.yaml") + .wait(10) + .cmd("kubectl apply -f test-casskop-operator/test/nodes-0.yaml") + .wait(10) .wait(50), - "scaledown": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-casskop-operator/test/cassandra-configmap-v1.yaml") + "scaledown": test_framework.new_built_in_workload().cmd( + "kubectl apply -f test-casskop-operator/test/cassandra-configmap-v1.yaml" + ) # Init 3 .cmd("kubectl apply -f test-casskop-operator/test/dc-3.yaml").wait(100) # Old 3, now 2, crash defer update cc. Now dc is 2, but old is still 3, and we crash the operator # Inside 10s, the operator should handle for the change, and resatrted after 10s .cmd("kubectl apply -f test-casskop-operator/test/dc-2.yaml").wait(10) # Issue this, and start the operator, see old = 3 - .cmd("kubectl apply -f test-casskop-operator/test/dc-1.yaml").wait(60) - .wait(50), + .cmd("kubectl apply -f test-casskop-operator/test/dc-1.yaml").wait(60).wait(50), }, "cass-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-cass-operator/test/cdc-1.yaml").wait_for_pod_status("cluster1-cassandra-datacenter-default-sts-0", common.RUNNING) - .cmd("kubectl delete CassandraDatacenter cassandra-datacenter").wait_for_pod_status("cluster1-cassandra-datacenter-default-sts-0", common.TERMINATED).wait_for_pvc_status("server-data-cluster1-cassandra-datacenter-default-sts-0", common.TERMINATED) - .cmd("kubectl apply -f test-cass-operator/test/cdc-1.yaml").wait_for_pod_status("cluster1-cassandra-datacenter-default-sts-0", common.RUNNING) + .cmd("kubectl apply -f test-cass-operator/test/cdc-1.yaml") + .wait_for_pod_status( + "cluster1-cassandra-datacenter-default-sts-0", common.RUNNING + ) + .cmd("kubectl delete CassandraDatacenter cassandra-datacenter") + .wait_for_pod_status( + "cluster1-cassandra-datacenter-default-sts-0", common.TERMINATED + ) + .wait_for_pvc_status( + "server-data-cluster1-cassandra-datacenter-default-sts-0", common.TERMINATED + ) + .cmd("kubectl apply -f test-cass-operator/test/cdc-1.yaml") + .wait_for_pod_status( + "cluster1-cassandra-datacenter-default-sts-0", common.RUNNING + ) .wait(50), }, "zookeeper-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-1.yaml").wait_for_pod_status("zookeeper-cluster-0", common.RUNNING) - .cmd("kubectl delete ZookeeperCluster zookeeper-cluster").wait_for_pod_status("zookeeper-cluster-0", common.TERMINATED).wait_for_pvc_status("data-zookeeper-cluster-0", common.TERMINATED) - .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-1.yaml").wait_for_pod_status("zookeeper-cluster-0", common.RUNNING) + .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-1.yaml") + .wait_for_pod_status("zookeeper-cluster-0", common.RUNNING) + .cmd("kubectl delete ZookeeperCluster zookeeper-cluster") + .wait_for_pod_status("zookeeper-cluster-0", common.TERMINATED) + .wait_for_pvc_status("data-zookeeper-cluster-0", common.TERMINATED) + .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-1.yaml") + .wait_for_pod_status("zookeeper-cluster-0", common.RUNNING) .wait(50), "scaledown-scaleup": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-2.yaml").wait_for_pod_status("zookeeper-cluster-1", common.RUNNING).wait(30) - .cmd("kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p='{\"spec\":{\"replicas\":1}}'").wait_for_pod_status("zookeeper-cluster-1", common.TERMINATED).wait_for_pvc_status("data-zookeeper-cluster-1", common.TERMINATED) - .cmd("kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p='{\"spec\":{\"replicas\":2}}'").wait_for_pod_status("zookeeper-cluster-1", common.RUNNING) + .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-2.yaml") + .wait_for_pod_status("zookeeper-cluster-1", common.RUNNING) + .wait(30) + .cmd( + 'kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p=\'{"spec":{"replicas":1}}\'' + ) + .wait_for_pod_status("zookeeper-cluster-1", common.TERMINATED) + .wait_for_pvc_status("data-zookeeper-cluster-1", common.TERMINATED) + .cmd( + 'kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p=\'{"spec":{"replicas":2}}\'' + ) + .wait_for_pod_status("zookeeper-cluster-1", common.RUNNING) .wait(50), "scaledown-scaleup-obs": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-2.yaml").wait_for_pod_status("zookeeper-cluster-1", common.RUNNING).wait(30) - .cmd("kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p='{\"spec\":{\"replicas\":1}}'").wait(55) - .cmd("kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p='{\"spec\":{\"replicas\":2}}'").wait(60) + .cmd("kubectl apply -f test-zookeeper-operator/test/zkc-2.yaml") + .wait_for_pod_status("zookeeper-cluster-1", common.RUNNING) + .wait(30) + .cmd( + 'kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p=\'{"spec":{"replicas":1}}\'' + ) + .wait(55) + .cmd( + 'kubectl patch ZookeeperCluster zookeeper-cluster --type merge -p=\'{"spec":{"replicas":2}}\'' + ) + .wait(60) .wait(50), }, "rabbitmq-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml").wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) - .cmd("kubectl delete RabbitmqCluster rabbitmq-cluster").wait_for_pod_status("rabbitmq-cluster-server-0", common.TERMINATED) - .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml").wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) + .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml") + .wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) + .cmd("kubectl delete RabbitmqCluster rabbitmq-cluster") + .wait_for_pod_status("rabbitmq-cluster-server-0", common.TERMINATED) + .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml") + .wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) .wait(50), "resize-pvc": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml").wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) - .cmd("kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p='{\"spec\":{\"persistence\":{\"storage\":\"15Gi\"}}}'").wait_for_sts_storage_size("rabbitmq-cluster-server", "15Gi") + .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml") + .wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) + .cmd( + 'kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p=\'{"spec":{"persistence":{"storage":"15Gi"}}}\'' + ) + .wait_for_sts_storage_size("rabbitmq-cluster-server", "15Gi") .wait(50), "scaleup-scaledown": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml").wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) - .cmd("kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p='{\"spec\":{\"replicas\":3}}'").wait(10) - .cmd("kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p='{\"spec\":{\"replicas\":2}}'").wait(10) + .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml") + .wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) + .cmd( + 'kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p=\'{"spec":{"replicas":3}}\'' + ) + .wait(10) + .cmd( + 'kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p=\'{"spec":{"replicas":2}}\'' + ) + .wait(10) .wait(50), "resize-pvc-atomic": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml").wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) + .cmd("kubectl apply -f test-rabbitmq-operator/test/rmqc-1.yaml") + .wait_for_pod_status("rabbitmq-cluster-server-0", common.RUNNING) # 10Gi -> 15Gi - .cmd("kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p='{\"spec\":{\"persistence\":{\"storage\":\"15Gi\"}}}'").wait_for_sts_storage_size("rabbitmq-cluster-server", "15Gi") + .cmd( + 'kubectl patch RabbitmqCluster rabbitmq-cluster --type merge -p=\'{"spec":{"persistence":{"storage":"15Gi"}}}\'' + ) + .wait_for_sts_storage_size("rabbitmq-cluster-server", "15Gi") .wait(120), }, "mongodb-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-mongodb-operator/test/cr.yaml").wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING) - .cmd("kubectl delete PerconaServerMongoDB mongodb-cluster").wait_for_pod_status("mongodb-cluster-rs0-2", common.TERMINATED).wait_for_pvc_status("mongod-data-mongodb-cluster-rs0-2", common.TERMINATED) - .cmd("kubectl apply -f test-mongodb-operator/test/cr.yaml").wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING) + .cmd("kubectl apply -f test-mongodb-operator/test/cr.yaml") + .wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING) + .cmd("kubectl delete PerconaServerMongoDB mongodb-cluster") + .wait_for_pod_status("mongodb-cluster-rs0-2", common.TERMINATED) + .wait_for_pvc_status("mongod-data-mongodb-cluster-rs0-2", common.TERMINATED) + .cmd("kubectl apply -f test-mongodb-operator/test/cr.yaml") + .wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING) .wait(50), "disable-enable-shard": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-mongodb-operator/test/cr-shard.yaml").wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING).wait_for_pod_status("mongodb-cluster-cfg-2", common.RUNNING) - .cmd("kubectl patch PerconaServerMongoDB mongodb-cluster --type merge -p='{\"spec\":{\"sharding\":{\"enabled\":false}}}'").wait_for_pod_status("mongodb-cluster-cfg-2", common.TERMINATED) - .cmd("kubectl patch PerconaServerMongoDB mongodb-cluster --type merge -p='{\"spec\":{\"sharding\":{\"enabled\":true}}}'").wait_for_pod_status("mongodb-cluster-cfg-2", common.RUNNING) + .cmd("kubectl apply -f test-mongodb-operator/test/cr-shard.yaml") + .wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING) + .wait_for_pod_status("mongodb-cluster-cfg-2", common.RUNNING) + .cmd( + 'kubectl patch PerconaServerMongoDB mongodb-cluster --type merge -p=\'{"spec":{"sharding":{"enabled":false}}}\'' + ) + .wait_for_pod_status("mongodb-cluster-cfg-2", common.TERMINATED) + .cmd( + 'kubectl patch PerconaServerMongoDB mongodb-cluster --type merge -p=\'{"spec":{"sharding":{"enabled":true}}}\'' + ) + .wait_for_pod_status("mongodb-cluster-cfg-2", common.RUNNING) .wait(50), "disable-enable-arbiter": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-mongodb-operator/test/cr-arbiter.yaml").wait_for_pod_status("mongodb-cluster-rs0-3", common.RUNNING).wait_for_pod_status("mongodb-cluster-rs0-arbiter-0", common.RUNNING) - .cmd("kubectl patch PerconaServerMongoDB mongodb-cluster --type='json' -p='[{\"op\": \"replace\", \"path\": \"/spec/replsets/0/arbiter/enabled\", \"value\": false}]'").wait_for_pod_status("mongodb-cluster-rs0-arbiter-0", common.TERMINATED).wait_for_pod_status("mongodb-cluster-rs0-4", common.RUNNING) - .cmd("kubectl patch PerconaServerMongoDB mongodb-cluster --type='json' -p='[{\"op\": \"replace\", \"path\": \"/spec/replsets/0/arbiter/enabled\", \"value\": true}]'").wait_for_pod_status("mongodb-cluster-rs0-arbiter-0", common.RUNNING).wait_for_pod_status("mongodb-cluster-rs0-4", common.TERMINATED) + .cmd("kubectl apply -f test-mongodb-operator/test/cr-arbiter.yaml") + .wait_for_pod_status("mongodb-cluster-rs0-3", common.RUNNING) + .wait_for_pod_status("mongodb-cluster-rs0-arbiter-0", common.RUNNING) + .cmd( + 'kubectl patch PerconaServerMongoDB mongodb-cluster --type=\'json\' -p=\'[{"op": "replace", "path": "/spec/replsets/0/arbiter/enabled", "value": false}]\'' + ) + .wait_for_pod_status("mongodb-cluster-rs0-arbiter-0", common.TERMINATED) + .wait_for_pod_status("mongodb-cluster-rs0-4", common.RUNNING) + .cmd( + 'kubectl patch PerconaServerMongoDB mongodb-cluster --type=\'json\' -p=\'[{"op": "replace", "path": "/spec/replsets/0/arbiter/enabled", "value": true}]\'' + ) + .wait_for_pod_status("mongodb-cluster-rs0-arbiter-0", common.RUNNING) + .wait_for_pod_status("mongodb-cluster-rs0-4", common.TERMINATED) .wait(50), # "enable-shard": test_framework.new_built_in_workload() # .cmd("kubectl apply -f test-mongodb-operator/test/cr.yaml").wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING).wait_for_pod_status("mongodb-cluster-rs0-2", common.RUNNING) @@ -117,55 +216,86 @@ }, "xtradb-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-xtradb-operator/test/cr.yaml").wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING) - .cmd("kubectl delete perconaxtradbcluster xtradb-cluster").wait_for_pod_status("xtradb-cluster-pxc-0", common.TERMINATED).wait_for_pod_status("xtradb-cluster-pxc-1", common.TERMINATED).wait_for_pod_status("xtradb-cluster-pxc-2", common.TERMINATED) - .cmd("kubectl apply -f test-xtradb-operator/test/cr.yaml").wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING) + .cmd("kubectl apply -f test-xtradb-operator/test/cr.yaml") + .wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING) + .cmd("kubectl delete perconaxtradbcluster xtradb-cluster") + .wait_for_pod_status("xtradb-cluster-pxc-0", common.TERMINATED) + .wait_for_pod_status("xtradb-cluster-pxc-1", common.TERMINATED) + .wait_for_pod_status("xtradb-cluster-pxc-2", common.TERMINATED) + .cmd("kubectl apply -f test-xtradb-operator/test/cr.yaml") + .wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING) .wait(70), "disable-enable-haproxy": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-xtradb-operator/test/cr-haproxy-enabled.yaml").wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING).wait_for_pod_status("xtradb-cluster-haproxy-0", common.RUNNING) - .cmd("kubectl apply -f test-xtradb-operator/test/cr-haproxy-disabled.yaml").wait_for_pod_status("xtradb-cluster-haproxy-0", common.TERMINATED) - .cmd("kubectl apply -f test-xtradb-operator/test/cr-haproxy-enabled.yaml").wait_for_pod_status("xtradb-cluster-haproxy-0", common.RUNNING) + .cmd("kubectl apply -f test-xtradb-operator/test/cr-haproxy-enabled.yaml") + .wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING) + .wait_for_pod_status("xtradb-cluster-haproxy-0", common.RUNNING) + .cmd("kubectl apply -f test-xtradb-operator/test/cr-haproxy-disabled.yaml") + .wait_for_pod_status("xtradb-cluster-haproxy-0", common.TERMINATED) + .cmd("kubectl apply -f test-xtradb-operator/test/cr-haproxy-enabled.yaml") + .wait_for_pod_status("xtradb-cluster-haproxy-0", common.RUNNING) .wait(70), "disable-enable-proxysql": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-xtradb-operator/test/cr-proxysql-enabled.yaml").wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING).wait_for_pod_status("xtradb-cluster-proxysql-0", common.RUNNING) - .cmd("kubectl apply -f test-xtradb-operator/test/cr-proxysql-disabled.yaml").wait_for_pod_status("xtradb-cluster-proxysql-0", common.TERMINATED) - .cmd("kubectl apply -f test-xtradb-operator/test/cr-proxysql-enabled.yaml").wait_for_pod_status("xtradb-cluster-proxysql-0", common.RUNNING) + .cmd("kubectl apply -f test-xtradb-operator/test/cr-proxysql-enabled.yaml") + .wait_for_pod_status("xtradb-cluster-pxc-2", common.RUNNING) + .wait_for_pod_status("xtradb-cluster-proxysql-0", common.RUNNING) + .cmd("kubectl apply -f test-xtradb-operator/test/cr-proxysql-disabled.yaml") + .wait_for_pod_status("xtradb-cluster-proxysql-0", common.TERMINATED) + .cmd("kubectl apply -f test-xtradb-operator/test/cr-proxysql-enabled.yaml") + .wait_for_pod_status("xtradb-cluster-proxysql-0", common.RUNNING) .wait(70), }, "yugabyte-operator": { "recreate": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-yugabyte-operator/test/yb-1.yaml").wait_for_pod_status("yb-master-0", common.RUNNING) - .cmd("kubectl delete YBCluster example-ybcluster").wait_for_pod_status("yb-master-0", common.TERMINATED) - .wait_for_pod_status("yb-master-1", common.TERMINATED).wait_for_pod_status("yb-master-2", common.TERMINATED) - .cmd("kubectl apply -f test-yugabyte-operator/test/yb-1.yaml").wait_for_pod_status("yb-master-0", common.RUNNING) + .cmd("kubectl apply -f test-yugabyte-operator/test/yb-1.yaml") + .wait_for_pod_status("yb-master-0", common.RUNNING) + .cmd("kubectl delete YBCluster example-ybcluster") + .wait_for_pod_status("yb-master-0", common.TERMINATED) + .wait_for_pod_status("yb-master-1", common.TERMINATED) + .wait_for_pod_status("yb-master-2", common.TERMINATED) + .cmd("kubectl apply -f test-yugabyte-operator/test/yb-1.yaml") + .wait_for_pod_status("yb-master-0", common.RUNNING) .wait(70), "disable-enable-tls": test_framework.new_built_in_workload() .cmd("kubectl apply -f test-yugabyte-operator/test/yb-tls-enabled.yaml") .wait_for_pod_status("yb-master-2", common.RUNNING) .wait_for_pod_status("yb-tserver-2", common.RUNNING) - .cmd("kubectl patch YBCluster example-ybcluster --type merge -p='{\"spec\":{\"tls\":{\"enabled\":false}}}'") + .cmd( + 'kubectl patch YBCluster example-ybcluster --type merge -p=\'{"spec":{"tls":{"enabled":false}}}\'' + ) .wait_for_secret_existence("yb-master-yugabyte-tls-cert", common.NONEXIST) .wait_for_secret_existence("yb-tserver-yugabyte-tls-cert", common.NONEXIST) - .cmd("kubectl patch YBCluster example-ybcluster --type merge -p='{\"spec\":{\"tls\":{\"enabled\":true}}}'") + .cmd( + 'kubectl patch YBCluster example-ybcluster --type merge -p=\'{"spec":{"tls":{"enabled":true}}}\'' + ) .wait_for_secret_existence("yb-master-yugabyte-tls-cert", common.EXIST) .wait_for_secret_existence("yb-tserver-yugabyte-tls-cert", common.EXIST) .wait(70), "disable-enable-tserverUIPort": test_framework.new_built_in_workload() - .cmd("kubectl apply -f test-yugabyte-operator/test/yb-tserverUIPort-enabled.yaml") + .cmd( + "kubectl apply -f test-yugabyte-operator/test/yb-tserverUIPort-enabled.yaml" + ) .wait_for_pod_status("yb-master-2", common.RUNNING) .wait_for_pod_status("yb-tserver-2", common.RUNNING) - .cmd("kubectl patch YBCluster example-ybcluster --type merge -p='{\"spec\":{\"tserver\":{\"tserverUIPort\": 0}}}'") + .cmd( + 'kubectl patch YBCluster example-ybcluster --type merge -p=\'{"spec":{"tserver":{"tserverUIPort": 0}}}\'' + ) .wait_for_service_existence("yb-tserver-ui", common.NONEXIST) - .cmd("kubectl patch YBCluster example-ybcluster --type merge -p='{\"spec\":{\"tserver\":{\"tserverUIPort\": 7000}}}'") + .cmd( + 'kubectl patch YBCluster example-ybcluster --type merge -p=\'{"spec":{"tserver":{"tserverUIPort": 7000}}}\'' + ) .wait_for_service_existence("yb-tserver-ui", common.EXIST) .wait(70), "scaleup-scaledown-tserver": test_framework.new_built_in_workload() .cmd("kubectl apply -f test-yugabyte-operator/test/yb-1.yaml") .wait_for_pod_status("yb-master-2", common.RUNNING) .wait_for_pod_status("yb-tserver-2", common.RUNNING) - .cmd("kubectl patch YBCluster example-ybcluster --type merge -p='{\"spec\":{\"tserver\":{\"replicas\":4},\"replicationFactor\":4}}'") + .cmd( + 'kubectl patch YBCluster example-ybcluster --type merge -p=\'{"spec":{"tserver":{"replicas":4},"replicationFactor":4}}\'' + ) .wait_for_pod_status("yb-tserver-3", common.RUNNING, 20) - .cmd("kubectl patch YBCluster example-ybcluster --type merge -p='{\"spec\":{\"tserver\":{\"replicas\":3},\"replicationFactor\":4}}'") + .cmd( + 'kubectl patch YBCluster example-ybcluster --type merge -p=\'{"spec":{"tserver":{"replicas":3},"replicationFactor":4}}\'' + ) .wait(70), }, "nifikop-operator": { @@ -173,7 +303,8 @@ .cmd("kubectl apply -f test-nifikop-operator/test/nc.yaml") .wait_for_pod_status("simplenifi-1-*", common.RUNNING) .cmd("kubectl apply -f test-nifikop-operator/test/nc1.yaml") - .wait(30).wait_for_pod_status("simplenifi-1-*", common.RUNNING) + .wait(30) + .wait_for_pod_status("simplenifi-1-*", common.RUNNING) .wait(60), - } + }, }