Merge branch 'master' of https://github.com/octoml/ck

cm-mlops/script/app-mlperf-inference-reference/_cm.yaml

@@ -336,6 +336,16 @@ deps:
       CM_MLPERF_QUANTIZATION:
       - on
 
+  ## BERT (Deep-sparse format int8)
+  - tags: get,ml-model,language-processing,bert,_deepsparse,_int8
+    enable_if_env:
+      CM_MLPERF_BACKEND:
+      - deepsparse
+      CM_MODEL:
+      - bert-99
+      CM_MLPERF_QUANTIZATION:
+      - on
+
 
   ## BERT (TF format)
   - tags: get,ml-model,language-processing,bert,_tf
@@ -469,6 +479,18 @@ variations:
   tensorflow:
     alias: tf
 
+  deepsparse:
+    group: framework
+    env:
+      CM_MLPERF_BACKEND: deepsparse
+      CM_MLPERF_BATCH_SIZE: "1"
+      CM_MLPERF_BACKEND_VERSION: <<<CM_DEEPSPARSE_VERSION>>>
+    deps:
+    - tags: get,generic-python-lib,_deepsparse
+    add_deps_recursive:
+      inference-src:
+        version: deepsparse
+
   tvm-onnx:
     group: framework
     env:
@@ -531,6 +553,8 @@ variations:
       CM_MODEL: 3d-unet-99
 
   3d-unet:
+    env:
+      CM_TMP_IGNORE_MLPERF_QUERY_COUNT: yes
     deps:
     - tags: get,generic-python-lib,_torch
       names:

cm-mlops/script/app-mlperf-inference-reference/customize.py

@@ -60,7 +60,7 @@ def preprocess(i):
     if 'CM_MLPERF_LOADGEN_MAX_BATCHSIZE' in env:
         env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --max-batchsize " + env['CM_MLPERF_LOADGEN_MAX_BATCHSIZE']
 
-    if 'CM_MLPERF_LOADGEN_QUERY_COUNT' in env:
+    if 'CM_MLPERF_LOADGEN_QUERY_COUNT' in env and env.get('CM_TMP_IGNORE_MLPERF_QUERY_COUNT', 'no') != "yes":
         env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + env['CM_MLPERF_LOADGEN_QUERY_COUNT']
 
     print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'")
@@ -175,6 +175,8 @@ def get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, datas
             quantization_options = ""
         cmd = "cd '" + env['RUN_DIR'] + "' && "+env['CM_PYTHON_BIN_WITH_PATH']+ " run.py --backend=" + env['CM_MLPERF_BACKEND'] + " --scenario="+env['CM_MLPERF_LOADGEN_SCENARIO'] + \
             env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + scenario_extra_options + mode_extra_options + dataset_options + quantization_options
+        if env['CM_MLPERF_BACKEND'] == "deepsparse":
+            cmd += " --batch_size="+env['CM_MLPERF_BATCH_SIZE']+" --model_path=" + env['CM_ML_MODEL_FILE_WITH_PATH']
         cmd = cmd.replace("--count", "--max_examples")
         env['MODEL_FILE'] = env['CM_ML_MODEL_FILE_WITH_PATH']
         env['VOCAB_FILE'] = env['CM_ML_MODEL_BERT_VOCAB_FILE_WITH_PATH']
@@ -208,7 +210,7 @@ def get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, datas
         env['LOG_PATH'] = env['CM_MLPERF_OUTPUT_DIR']
         env['SKIP_VERIFY_ACCURACY'] = True
 
-    elif "dlrm" in env['CM_MODEL']:
+    elif "dlrm" in env['CM_MODEL']: # DLRM is in draft stage
 
         env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_DLRM_PATH'], "pytorch")
         if 'terabyte' in env['CM_ML_MODEL_DATASET']:

cm-mlops/script/app-mlperf-inference/_cm.yaml

@@ -52,6 +52,8 @@ input_mapping:
   test_query_count: CM_TEST_QUERY_COUNT
   new_tvm_model: CM_MLPERF_DELETE_COMPILED_MODEL
   clean: CM_MLPERF_CLEAN_SUBMISSION_DIR
+  target_qps: CM_MLPERF_LOADGEN_TARGET_QPS
+  target_latency: CM_MLPERF_LOADGEN_TARGET_LATENCY
 
 # Duplicate CM environment variables to the ones used in native apps
 env_key_mappings:
@@ -235,15 +237,30 @@ variations:
       - openimages-accuracy-script
       tags: run,accuracy,mlperf,_openimages
 
-  3dunet:
+  3d-unet-99:
     group:
       models
+    base:
+    - 3d-unet_
+    env:
+      CM_MODEL:
+        3d-unet-99
+    add_deps_recursive:
+      mlperf-inference-implementation:
+        tags: _3d-unet-99
+  3d-unet-99.9:
+    group:
+      models
+    base:
+    - 3d-unet_
     env:
       CM_MODEL:
-        3dunet
+        3d-unet-99.9
     add_deps_recursive:
       mlperf-inference-implementation:
-        tags: _3dunet
+        tags: _3d-unet-99.9
+
+  3d-unet_:
     post_deps:
     - enable_if_env:
         CM_MLPERF_LOADGEN_MODE:
@@ -342,6 +359,14 @@ variations:
     add_deps_recursive:
       mlperf-inference-implementation:
         tags: _pytorch
+  deepsparse:
+    group: backend
+    env:
+      CM_MLPERF_BACKEND:
+        deepsparse
+    add_deps_recursive:
+      mlperf-inference-implementation:
+        tags: _deepsparse
   tflite:
     group: backend
     env:
@@ -454,7 +479,9 @@ valid_variation_combinations:
   -
     - rnnt
   -
-    - 3d-unet
+    - 3d-unet-99
+  -
+    - 3d-unet-99.9
 
 invalid_variation_combinations:
   -

cm-mlops/script/app-mlperf-inference/customize.py

@@ -86,6 +86,12 @@ def preprocess(i):
 
     if "bert" not in env['CM_MODEL']:
         scenario_extra_options +=  " --threads " + NUM_THREADS
+
+    if env['CM_MODEL'] not in i['state']['CM_SUT_CONFIG'][env['CM_SUT_NAME']]:
+        i['state']['CM_SUT_CONFIG'][env['CM_SUT_NAME']][env['CM_MODEL']] = {}
+        i['state']['CM_SUT_CONFIG'][env['CM_SUT_NAME']][env['CM_MODEL']][scenario] = {}
+
+
     conf = i['state']['CM_SUT_CONFIG'][env['CM_SUT_NAME']][env['CM_MODEL']][scenario]
     user_conf = ''
     if ['CM_MLPERF_RUN_STYLE'] == "fast":
@@ -100,20 +106,39 @@ def preprocess(i):
 
     query_count = None
 
-    for metric in conf:
-        metric_value = conf[metric]
-        if env['CM_MLPERF_RUN_STYLE'] == "fast":
-            if metric == "target_qps" and scenario == "Offline":
-                metric_value /= fast_factor
-            if metric == "target_latency" and scenario in [ "SingleStream", "MultiStream" ]:
-                metric_value *= fast_factor
-            conf[metric] = metric_value
-        elif env['CM_MLPERF_RUN_STYLE'] == "test":
-            if metric == "target_qps" and scenario == "Offline":
-                metric_value = 1
-            if metric == "target_latency" and scenario in [ "SingleStream" ]:
-                metric_value = 1000
-        user_conf += ml_model_name + "." + scenario + "." + metric + " = " + str(metric_value) + "\n"
+    value = None
+    if scenario in [ 'Offline', 'Server' ]:
+        metric = "target_qps"
+        #value = env.get('CM_MLPERF_LOADGEN_SERVER_TARGET_QPS') if scenario == "Server" else env.get('CM_MLPERF_LOADGEN_OFFLINE_TARGET_QPS')
+        if not value:
+            value = env.get('CM_MLPERF_LOADGEN_TARGET_QPS')
+    elif scenario in [ 'SingleStream', 'MultiStream' ]:
+        metric = "target_latency"
+        if not value:
+            value = env.get('CM_MLPERF_LOADGEN_TARGET_LATENCY')
+
+    if value:
+        metric_value = value
+        conf[metric] = value
+    else:
+        if metric in conf:
+            metric_value = conf[metric]
+        else:
+            return {'return': 1, 'error': f"Config details missing for SUT:{env['CM_SUT_NAME']}, Model:{env['CM_MODEL']}, Scenario: {scenario}. Please input {metric} value"}
+
+    if env['CM_MLPERF_RUN_STYLE'] == "fast":
+        if scenario == "Offline":
+            metric_value /= fast_factor
+        if scenario in [ "SingleStream", "MultiStream" ]:
+            metric_value *= fast_factor
+    elif env['CM_MLPERF_RUN_STYLE'] == "test":
+        if scenario == "Offline":
+            metric_value = 1
+        if scenario in [ "SingleStream" ]:
+            metric_value = 1000
+    conf[metric] = metric_value
+    user_conf += ml_model_name + "." + scenario + "." + metric + " = " + str(metric_value) + "\n"
+
     if env['CM_MLPERF_RUN_STYLE'] == "test":
         query_count = env.get('CM_TEST_QUERY_COUNT', "5")
         user_conf += ml_model_name + "." + scenario + ".max_query_count = " + query_count + "\n"

cm-mlops/script/detect-cpu/customize.py

@@ -40,6 +40,7 @@ def postprocess(i):
             'CM_CPUINFO_L3_cache': 'CM_HOST_CPU_L3_CACHE_SIZE',
             'CM_CPUINFO_Sockets': 'CM_HOST_CPU_SOCKETS',
             'CM_CPUINFO_NUMA_nodes': 'CM_HOST_CPU_NUMA_NODES',
+            'CM_CPUINFO_Cores_per_socket': 'CM_HOST_CPU_PHYSICAL_CORES_PER_SOCKET',
             'CM_CPUINFO_Threads_per_core': 'CM_HOST_CPU_THREADS_PER_CORE',
             'CM_CPUINFO_Architecture': 'CM_HOST_CPU_ARCHITECTURE',
             'CM_CPUINFO_CPU_family': 'CM_HOST_CPU_FAMILY',
@@ -58,7 +59,7 @@ def postprocess(i):
 
     if env['CM_HOST_OS_TYPE'] == 'linux':
         vkeys = [ 'Architecture', 'Model name', 'Vendor ID', 'CPU family', 'NUMA node(s)', 'CPU(s)', \
-                'On-line CPU(s) list', 'Socket(s)', 'Thread(s) per core', 'L1d cache', 'L1i cache', 'L2 cache', \
+                'On-line CPU(s) list', 'Socket(s)', 'Core(s) per socket', 'Thread(s) per core', 'L1d cache', 'L1i cache', 'L2 cache', \
                 'L3 cache', 'CPU max MHz' ]
     elif env['CM_HOST_OS_FLAVOR'] == 'macos':
         vkeys = [ 'hw.physicalcpu', 'hw.logicalcpu', 'hw.packages', 'hw.ncpu', 'hw.memsize', 'hw.l1icachesize', \
@@ -77,7 +78,10 @@ def postprocess(i):
         env['CM_HOST_CPU_TOTAL_LOGICAL_CORES'] = env['CM_HOST_CPU_TOTAL_CORES']
 
     if 'CM_HOST_CPU_TOTAL_LOGICAL_CORES' in env and 'CM_HOST_CPU_TOTAL_PHYSICAL_CORES' in env and 'CM_HOST_CPU_THREADS_PER_CORE' not in env:
-        env['CM_HOST_CPU_THREADS_PER_CORE'] = str(int(int(env['CM_HOST_CPU_TOTAL_LOGICAL_CORES']) /
+        env['CM_HOST_CPU_THREADS_PER_CORE'] = str(int(int(env['CM_HOST_CPU_TOTAL_LOGICAL_CORES']) //
             int(env['CM_HOST_CPU_TOTAL_PHYSICAL_CORES'])))
 
+    if 'CM_HOST_CPU_SOCKETS' in env and 'CM_HOST_CPU_TOTAL_PHYSICAL_CORES' in env and 'CM_HOST_CPU_PHYSICAL_CORES_PER_SOCKET' not in env:
+        env['CM_HOST_CPU_PHYSICAL_CORES_PER_SOCKET'] = env['CM_HOST_CPU_TOTAL_PHYSICAL_CORES'] // env['CM_HOST_CPU_SOCKETS']
+
     return {'return':0}

cm-mlops/script/get-generic-python-lib/_cm.json

@@ -295,6 +295,14 @@
         "CM_SOX_VERSION"
       ]
     },
+    "deepsparse": {
+      "env": {
+        "CM_GENERIC_PYTHON_PACKAGE_NAME": "deepsparse"
+      },
+      "new_env_keys": [
+        "CM_DEEPSPARSE_VERSION"
+      ]
+    },
     "toml": {
       "env": {
         "CM_GENERIC_PYTHON_PACKAGE_NAME": "toml"

cm-mlops/script/get-git-repo/README-extra.md

@@ -0,0 +1,20 @@
+# Get GIT Repository
+This [CM script](https://github.com/mlcommons/ck/blob/master/cm/docs/specs/script.md) git clones any specified GIT repository.
+
+## Commands
+To install
+```
+cm run script --tags=get,git,repo,_repo.<repo_name>,[VARIATION] 
+```
+where [VARIATION] is one of
+* `patch:` Applies the `git.patch` to the cloned git repository
+* `short-history:` Uses a git depth of last 10 commits (significantly reduces the download size)
+* `full-history:` Uses the full git history
+* `no-recurse-submodules:` Only download the main repository
+
+## Exported Variables
+* `CM_GIT_CHECKOUT_PATH`: Directory path of the cloned git repository
+
+## Supported and Tested OS
+1. Ubuntu 18.04, 20.04, 22.04
+2. RHEL 9

cm-mlops/script/get-git-repo/_cm.json

@@ -0,0 +1,65 @@
+{
+  "alias": "get-git-repo",
+  "automation_alias": "script",
+  "automation_uid": "5b4e0237da074764",
+  "cache": true,
+  "category": "Modular MLPerf benchmarks",
+  "default_env": {
+    "CM_GIT_CHECKOUT": "main",
+    "CM_GIT_DEPTH": "--depth 4",
+    "CM_GIT_CHECKOUT_FOLDER": "repo",
+    "CM_GIT_PATCH": "no",
+    "CM_GIT_RECURSE_SUBMODULES": " --recurse-submodules",
+    "CM_GIT_URL": "https://github.com/mlcommons/ck.git"
+  },
+  "default_variation": "default",
+  "deps": [
+    {
+      "tags": "detect,os"
+    }
+  ],
+  "new_env_keys": [
+    "CM_GIT_CHECKOUT_PATH"
+  ],
+  "tags": [
+    "get",
+    "git",
+    "repo",
+    "repository",
+    "clone"
+  ],
+  "uid": "ed603e7292974f10",
+  "variations": {
+    "default": {
+      "base": [
+        "short-history"
+      ],
+      "env": {
+        "CM_GIT_PATCH": "no"
+      }
+    },
+    "full-history": {
+      "env": {
+        "CM_GIT_DEPTH": ""
+      }
+    },
+    "no-recurse-submodules": {
+      "env": {
+        "CM_GIT_RECURSE_SUBMODULES": ""
+      }
+    },
+    "patch": {
+      "env": {
+        "CM_GIT_PATCH": "yes"
+      }
+    },
+    "short-history": {
+      "env": {
+        "CM_GIT_DEPTH": "--depth 5"
+      }
+    },
+    "repo.#": {
+      "group": "repo"
+    }
+  }
+}

cm-mlops/script/get-git-repo/customize.py

@@ -0,0 +1,34 @@
+from cmind import utils
+import os
+import shutil
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    if os_info['platform'] == 'windows':
+        return {'return':1, 'error': 'Windows is not supported in this script yet'}
+
+    env = i['env']
+    meta = i['meta']
+
+    if 'CM_GIT_REPO_NAME' not in env:
+        env['CM_GIT_REPO_NAME'] = os.path.basename(env['CM_GIT_URL'])
+
+    if 'CM_GIT_DEPTH' not in env:
+        env['CM_GIT_DEPTH'] = ''
+
+    if 'CM_GIT_RECURSE_SUBMODULES' not in env:
+        env['CM_GIT_RECURSE_SUBMODULES'] = ''
+
+    env['CM_GIT_CHECKOUT_PATH'] = os.path.join(os.getcwd(), env['CM_GIT_CHECKOUT_FOLDER'])
+
+    return {'return':0}
+
+
+def postprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    return {'return':0}