diff --git a/ssg/yaml.py b/ssg/yaml.py index b955f427c7e..e2745023687 100644 --- a/ssg/yaml.py +++ b/ssg/yaml.py @@ -15,6 +15,15 @@ except ImportError: from yaml import SafeLoader as yaml_SafeLoader +try: + from yaml import CLoader as yaml_Loader +except ImportError: + from yaml import Loader as yaml_Loader + +try: + from yaml import CDumper as yaml_Dumper +except ImportError: + from yaml import Dumper as yaml_Dumper def _bool_constructor(self, node): return self.construct_scalar(node) @@ -121,7 +130,7 @@ def open_raw(yaml_file): return yaml_contents -def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): +def ordered_load(stream, Loader=yaml_Loader, object_pairs_hook=OrderedDict): """ Drop-in replacement for yaml.load(), but preserves order of dictionaries """ @@ -137,7 +146,7 @@ def construct_mapping(loader, node): return yaml.load(stream, OrderedLoader) -def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): +def ordered_dump(data, stream=None, Dumper=yaml_Dumper, **kwds): """ Drop-in replacement for yaml.dump(), but preserves order of dictionaries """ @@ -165,6 +174,10 @@ def _str_representer(dumper, data): unformatted_yaml = yaml.dump(data, None, OrderedDumper, **kwds) formatted_yaml = re.sub(r"[\n]+([\s]*)- name", r"\n\n\1- name", unformatted_yaml) + # Fix CDumper issue where it adds yaml document ending '...' + # in some templated ansible remediations + formatted_yaml = re.sub(r"\n\s*\.\.\.\s*", r"\n", formatted_yaml) + if stream is not None: return stream.write(formatted_yaml) else: diff --git a/utils/build_profiler_report.py b/utils/build_profiler_report.py index 782a506b35c..2025d191302 100755 --- a/utils/build_profiler_report.py +++ b/utils/build_profiler_report.py @@ -2,7 +2,6 @@ """Compare and present build times to user and generate an HTML interactive graph""" import sys import argparse -import re def load_log_file(file) -> dict: @@ -13,18 +12,34 @@ def load_log_file(file) -> dict: # {Target: duration} dict target_duration_dict = {} - for line in lines: - line = line.strip() - - # pattern to match target names that are an absolute path - these should be skipped - # --> an issue appeared with new versions of cmake where the targets are duplicated for some - # reason and therefore they must be filtered here - duplicate_pattern = re.compile("[0-9]+\s+[0-9]+\s+[0-9]+\s+/.*") - if not line.startswith('#') and not duplicate_pattern.match(line): - # calculate target compilation duration and add it to dict - line = line.split() - duration = int(line[1]) - int(line[0]) - target_duration_dict[line[3]] = duration + + # an issue appeared with new versions of cmake where the targets are duplicated with a relative + # and absolute path and therefore they must be filtered here; filters comments too + lines = [line for line in lines if not line.strip().split()[3].startswith('/') and not + line.startswith('#')] + splitLines = [line.strip().split() for line in lines] + + # calculate target compilation duration and add it to dict + hash_target_duration_dict = {} + for line in splitLines: + duration = int(line[1]) - int(line[0]) + hash = line[2] + target = line[3] + + # filter out lines with duplicate times and concatenate target names + if hash not in hash_target_duration_dict: + target_duration_dict[target] = duration + # add target,duration with new hash + hash_target_duration_dict[hash] = (target, duration) + else: + previous_target = hash_target_duration_dict[hash][0] + # concatenate previous target with same hash to this one + concated_target = previous_target + ";" + target + # remove old target entry and add concatenated target = duraiton + target_duration_dict[concated_target] = \ + target_duration_dict.pop(previous_target) + # update target name in hash dict + hash_target_duration_dict[hash] = (concated_target, duration) return target_duration_dict @@ -138,6 +153,13 @@ def print_report(current_dict: dict, baseline_dict: dict = None) -> None: # if target was not in baseline, append note if baseline_dict and target not in baseline_dict.keys(): line.append("Not in baseline") + + # if target has multiple output files, print them on separate lines + # (times only on the last line) + if(';' in target): + print("\n".join(target.rsplit(';', 1)[0].split(';'))) + split_target = target.rsplit(';', 1)[1] + line[0] = f'{split_target:60}' print(' | '.join(line)) # Print time and % delta of the whole build time