Merge branch 'master' of https://github.com/octoml/ck

Ailurus1 · Mar 1, 2023 · 7bcac7f · 7bcac7f
2 parents afc6152 + 4c60b64
commit 7bcac7f
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 5 deletions.
diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py
@@ -3506,7 +3506,20 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
 
         rc = os.system(cmd)
 
-        if rc>0 and not i.get('ignore_script_error',False):
+        if rc>0 and not i.get('ignore_script_error', False):
+            # Check if print files when error
+            print_files = meta.get('print_files_if_script_error', [])
+            if len(print_files)>0:
+               for pr in print_files:
+                   if os.path.isfile(pr):
+                       r = utils.load_txt(file_name = pr)
+                       if r['return'] == 0:
+                           print ("========================================================")
+                           print ("Print file {}:".format(pr))
+                           print ("")
+                           print (r['string'])
+                           print ("")
+
             note = '''Note that it is often a portability problem of the third-party tool or native script that is wrapped and unified by this CM script.
 The CM concept is to collaboratively fix such issues inside portable CM scripts to make existing tools and native script more portable, interoperable, deterministic and reproducible.
 

diff --git a/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py b/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py
@@ -90,20 +90,24 @@ def preprocess(i):
     value = None
     if scenario in [ 'Offline', 'Server' ]:
         metric = "target_qps"
+        tolerance = 1.01
         #value = env.get('CM_MLPERF_LOADGEN_SERVER_TARGET_QPS') if scenario == "Server" else env.get('CM_MLPERF_LOADGEN_OFFLINE_TARGET_QPS')
         if not value:
             value = env.get('CM_MLPERF_LOADGEN_TARGET_QPS')
     elif scenario in [ 'SingleStream', 'MultiStream' ]:
         metric = "target_latency"
+        tolerance = 0.95
         if not value:
             value = env.get('CM_MLPERF_LOADGEN_TARGET_LATENCY')
+    else:
+        return {'return': 1, 'error': 'Invalid scenario: {}'.format(scenario)}
 
     if value:
         metric_value = value
         conf[metric] = value
     else:
         if metric in conf:
-            metric_value = str(float(conf[metric]) / 1.3) #some tolerance
+            metric_value = str(float(conf[metric]) * tolerance) #some tolerance
         else:
             if env.get("CM_MLPERF_FIND_PERFORMANCE_MODE", '') == "yes":
                 if metric == "target_qps":

diff --git a/cm-mlops/script/get-cuda-devices/_cm.json b/cm-mlops/script/get-cuda-devices/_cm.json
@@ -17,6 +17,9 @@
       "tags": "get,cuda,_toolkit"
     }
   ],
+  "print_files_if_script_error": [
+    "tmp-run.out"
+  ],
   "tags": [
     "get",
     "cuda-devices"

diff --git a/cm-mlops/script/get-cuda/_cm.json b/cm-mlops/script/get-cuda/_cm.json
@@ -36,6 +36,9 @@
       "tags": "install,cuda,prebuilt"
     }
   ],
+  "print_files_if_script_error": [
+    "tmp-ver.out"
+  ],
   "tags": [
     "get",
     "cuda",

diff --git a/cm-mlops/script/get-cudnn/customize.py b/cm-mlops/script/get-cudnn/customize.py
@@ -70,9 +70,11 @@ def preprocess(i):
         # (unless forced by a user)
 
         if env.get('CM_INPUT','').strip()=='':
-            if env.get('CM_TMP_PATH','').strip()!='':
-               env['CM_TMP_PATH']+=':'
-            env['CM_TMP_PATH'] = '/usr/local/cuda/lib64:/usr/cuda/lib64:/usr/local/cuda/lib:/usr/cuda/lib:/usr/local/cuda-11/lib64:/usr/cuda-11/lib:/usr/local/cuda-12/lib:/usr/cuda-12/lib:/usr/local/packages/cuda/lib'
+            cm_tmp_path = env.get('CM_TMP_PATH','').strip()
+            if cm_tmp_path!='':
+               cm_tmp_path+=':'
+            cm_tmp_path+='/usr/local/cuda/lib64:/usr/cuda/lib64:/usr/local/cuda/lib:/usr/cuda/lib:/usr/local/cuda-11/lib64:/usr/cuda-11/lib:/usr/local/cuda-12/lib:/usr/cuda-12/lib:/usr/local/packages/cuda/lib'
+            env['CM_TMP_PATH'] = cm_tmp_path
             env['CM_TMP_PATH_IGNORE_NON_EXISTANT'] = 'yes'
 
             for lib_path in env.get('+CM_HOST_OS_DEFAULT_LIBRARY_PATH', []):

diff --git a/cm-mlops/script/get-cudnn/run.sh b/cm-mlops/script/get-cudnn/run.sh
@@ -4,6 +4,7 @@ if [ ${CM_TMP_RUN_COPY_SCRIPT} == "yes" ]; then
     echo $cmd
     eval $cmd
     test $? -eq 0 || exit 1
+
     cmd="${CM_SUDO} cp -P ${CM_TMP_LIB_PATH}/libcudnn* ${CM_CUDA_PATH_LIB}/"
     echo $cmd
     eval $cmd