S. Riette 7 may 2024: ZTC metric for testprogs

tools/check_commit_testprogs.sh

@@ -503,6 +503,7 @@ if [ $run -ge 1 -a "$perffile" != "" ]; then
   echo "### Evaluate performance for commit $commit"
 
   ZTD_sum=0
+  ZTC_sum=0
   firstrun=1
   for t in $(echo $tests | sed 's/,/ /g'); do
     if echo $allowedTests | grep -w $t > /dev/null; then
@@ -549,14 +550,23 @@ if [ $run -ge 1 -a "$perffile" != "" ]; then
           ZTD=-999
           ZTD_sum=-999
         fi
+        ZTC=$(grep -m 1 "ZTC =" $file | awk '{print $4}')
+        if [ "$ZTC" != "" ]; then
+          ZTC_sum=$(python3 -c "print(${ZTC_sum} if ${ZTC_sum} < 0. else (${ZTC_sum} + ${ZTC}))")
+        else
+          ZTC=-999
+          ZTC_sum=-999
+        fi
       else
         ZTD=-999
         ZTD_sum=-999
+        ZTC=-999
+        ZTC_sum=-999
       fi
-      echo "$commit testprogs $t $ZTD" >> "$perffile"
+      echo "$commit testprogs $t $ZTD $ZTC" >> "$perffile"
     fi
   done
-  echo "$commit testprogs ALL $ZTD_sum" >> "$perffile"
+  echo "$commit testprogs ALL $ZTD_sum $ZTC_sum" >> "$perffile"
 fi
 
 ####################

tools/plot_perf.py

@@ -4,31 +4,36 @@
 This script plots the data contained in the performance files obtained with the --perf option
 """
 
+import re
 import matplotlib.pyplot as plt
 import numpy
 import pandas
-import re
 
 class Perf():
+    """
+    This class manipulates performance files
+    """
     def __init__(self, perffile):
         """
         :param perffile: text file with each line having the form "commit model case time"
         """
-        self._df = df = pandas.read_csv(perffile, sep=' ',
-                                        names=['commit', 'model', 'case', 'time'])
+        self._df = pandas.read_csv(perffile, sep=' ',
+                                   names=['commit', 'model', 'case', 'time', 'time2'])
 
-    def plotPerf(self, outfile, model=None, title=None, num=None):
+    def plotPerf(self, outfile, model=None, title=None, num=None, allTimes=False):
         """
         :param outfile: output file
         :param model: None to plot each model on a subplot or the model to plot
         :param title: custom title to use (%M will be replaced by the model name)
         :param num: plot only last num values (None to plot all values)
+        :param allTimes: True to plot alternative metrics
         """
         models = [model] if model is not None else sorted(set(self._df['model']))
-        fig, ax = plt.subplots(nrows=len(models), sharex=True, sharey=True, figsize=(8, 8 * len(models)))
+        fig, ax = plt.subplots(nrows=len(models), sharex=True, sharey=True,
+                               figsize=(8, 8 * len(models)))
         if len(models) == 1:
             ax = [ax]
-    
+
         #Ordered commit list common to all models
         commits = []
         for commit in self._df['commit']:
@@ -37,7 +42,7 @@ def plotPerf(self, outfile, model=None, title=None, num=None):
         if num is not None:
             commits = commits[-num:]
         shortCommits = [self.shortenCommit(c) for c in commits]
-    
+
         df = self._df.groupby('model')
         for igrpM, grpM in enumerate(models):
             if title is None:
@@ -49,17 +54,22 @@ def plotPerf(self, outfile, model=None, title=None, num=None):
                 ax[igrpM].set_title(title.replace('%M', grpM))
             ax[igrpM].set_ylabel('time')
             ax[igrpM].set_yscale('log')
-    
+
             dfp = df.get_group(grpM).groupby('case')
             for grp in dfp.groups:
                 #Build time serie with possible missing value and mean aggregation if needed
                 time = []
+                time2 = []
                 for commit in commits:
                     f = dfp.get_group(grp)['commit'] == commit
                     #discard negative values
                     l = [numpy.nan if t < 0. else t for t in dfp.get_group(grp)[f]['time']]
                     time.append(numpy.nan if len(l) == 0 else numpy.ma.array(l).mean())
-                ax[igrpM].plot(range(len(commits)), numpy.ma.array(time), 'o-', label=grp)
+                    l = [numpy.nan if t < 0. else t for t in dfp.get_group(grp)[f]['time2']]
+                    time2.append(numpy.nan if len(l) == 0 else numpy.ma.array(l).mean())
+                p = ax[igrpM].plot(range(len(commits)), numpy.ma.array(time), 'o-', label=grp)
+                if allTimes:
+                    ax[igrpM].plot(range(len(commits)), numpy.ma.array(time2), 'o:', color=p[0].get_color())
             if igrpM == len(models) - 1:
                 ax[igrpM].set_xlabel('PHYEX version')
                 ax[igrpM].set_xticks(range(len(commits)))
@@ -97,11 +107,11 @@ def listModels(self):
                         help="Plot only last N values")
     parser.add_argument('--listModels', default=False, action='store_true',
                         help="returns the list of models present in the performance file")
+    parser.add_argument('--allTimes', default=False, action='store_true',
+                          help="to also plot the alternative metrics")
     args = parser.parse_args()
     perf = Perf(args.PERF_FILE)
     if args.plot is not None:
-        perf.plotPerf(args.plot, args.model, args.title, args.num)
+        perf.plotPerf(args.plot, args.model, args.title, args.num, args.allTimes)
     if args.listModels:
         print(' '.join(perf.listModels()))
-
-

tools/testing.sh

@@ -106,7 +106,7 @@ done
 [ ! -d ${WORKDIR} ] && mkdir -p ${WORKDIR}
 
 #stdout and stderr redirection
-logfile="${WORKDIR}/logfile"
+logfile="${WORKDIR}/logfile_${contextHostname}"
 if [ -f "${logfile}" ]; then
   mv "${logfile}" "${logfile}.old"
 fi