Buildsystem optimization

ssg/yaml.py

@@ -15,6 +15,15 @@
 except ImportError:
     from yaml import SafeLoader as yaml_SafeLoader
 
+try:
+    from yaml import CLoader as yaml_Loader
+except ImportError:
+    from yaml import Loader as yaml_Loader
+
+try:
+    from yaml import CDumper as yaml_Dumper
+except ImportError:
+    from yaml import Dumper as yaml_Dumper
 
 def _bool_constructor(self, node):
     return self.construct_scalar(node)
@@ -121,7 +130,7 @@ def open_raw(yaml_file):
     return yaml_contents
 
 
-def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
+def ordered_load(stream, Loader=yaml_Loader, object_pairs_hook=OrderedDict):
     """
     Drop-in replacement for yaml.load(), but preserves order of dictionaries
     """
@@ -137,7 +146,7 @@ def construct_mapping(loader, node):
     return yaml.load(stream, OrderedLoader)
 
 
-def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds):
+def ordered_dump(data, stream=None, Dumper=yaml_Dumper, **kwds):
     """
     Drop-in replacement for yaml.dump(), but preserves order of dictionaries
     """
@@ -165,6 +174,10 @@ def _str_representer(dumper, data):
     unformatted_yaml = yaml.dump(data, None, OrderedDumper, **kwds)
     formatted_yaml = re.sub(r"[\n]+([\s]*)- name", r"\n\n\1- name", unformatted_yaml)
 
+    # Fix CDumper issue where it adds yaml document ending '...'
+    # in some templated ansible remediations
+    formatted_yaml = re.sub(r"\n\s*\.\.\.\s*", r"\n", formatted_yaml)
+
     if stream is not None:
         return stream.write(formatted_yaml)
     else:

utils/build_profiler_report.py

@@ -2,7 +2,6 @@
 """Compare and present build times to user and generate an HTML interactive graph"""
 import sys
 import argparse
-import re
 
 
 def load_log_file(file) -> dict:
@@ -13,18 +12,34 @@ def load_log_file(file) -> dict:
 
     # {Target: duration} dict
     target_duration_dict = {}
-    for line in lines:
-        line = line.strip()
-
-        # pattern to match target names that are an absolute path - these should be skipped
-        # --> an issue appeared with new versions of cmake where the targets are duplicated for some
-        # reason and therefore they must be filtered here
-        duplicate_pattern = re.compile("[0-9]+\s+[0-9]+\s+[0-9]+\s+/.*")
-        if not line.startswith('#') and not duplicate_pattern.match(line):
-            # calculate target compilation duration and add it to dict
-            line = line.split()
-            duration = int(line[1]) - int(line[0])
-            target_duration_dict[line[3]] = duration
+
+    # an issue appeared with new versions of cmake where the targets are duplicated with a relative
+    # and absolute path and therefore they must be filtered here; filters comments too
+    lines = [line for line in lines if not line.strip().split()[3].startswith('/') and not
+             line.startswith('#')]
+    splitLines = [line.strip().split() for line in lines]
+
+    # calculate target compilation duration and add it to dict
+    hash_target_duration_dict = {}
+    for line in splitLines:
+        duration = int(line[1]) - int(line[0])
+        hash = line[2]
+        target = line[3]
+
+        # filter out lines with duplicate times and concatenate target names
+        if hash not in hash_target_duration_dict:
+            target_duration_dict[target] = duration
+            # add target,duration with new hash
+            hash_target_duration_dict[hash] = (target, duration)
+        else:
+            previous_target = hash_target_duration_dict[hash][0]
+            # concatenate previous target with same hash to this one
+            concated_target = previous_target + ";" + target
+            # remove old target entry and add concatenated target = duraiton
+            target_duration_dict[concated_target] = \
+                target_duration_dict.pop(previous_target)
+            # update target name in hash dict
+            hash_target_duration_dict[hash] = (concated_target, duration)
 
     return target_duration_dict
 
@@ -138,6 +153,13 @@ def print_report(current_dict: dict, baseline_dict: dict = None) -> None:
         # if target was not in baseline, append note
         if baseline_dict and target not in baseline_dict.keys():
             line.append("Not in baseline")
+
+        # if target has multiple output files, print them on separate lines
+        # (times only on the last line)
+        if(';' in target):
+            print("\n".join(target.rsplit(';', 1)[0].split(';')))
+            split_target = target.rsplit(';', 1)[1]
+            line[0] = f'{split_target:60}'
         print(' | '.join(line))
 
     # Print time and % delta of the whole build time