Merge from OctoML (#480)

cm-mlops/automation/script/module.py

@@ -1801,7 +1801,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
                             'run_state':run_state
                         }
 
-                    ii.update(d)
+                    utils.merge_dicts({'dict1':ii, 'dict2':d, 'append_lists':True, 'append_unique':True})
 
                     r = self.cmind.access(ii)
                     if r['return']>0: return r
@@ -2902,7 +2902,7 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
 
         if verbose:
             print ('')
-            print (recursion_spaces + '  - Running script ...')
+            print (recursion_spaces + '  - Running script in {} ...'.format(os.getcwd()))
 
         # Prepare env variables
         import copy

cm-mlops/script/detect-os/_cm.json

@@ -10,7 +10,8 @@
     "CM_HOST_OS_*",
     "+CM_HOST_OS_*",
     "CM_HOST_PLATFORM_*",
-    "CM_HOST_PYTHON_*"
+    "CM_HOST_PYTHON_*",
+    "CM_HOST_SYSTEM_NAME"
   ],
   "new_state_keys": [
     "os_uname_*"

cm-mlops/script/detect-os/customize.py

@@ -53,5 +53,7 @@ def postprocess(i):
         state['os_uname_all'] = s[1]
 
         env['CM_HOST_OS_MACHINE'] = state['os_uname_machine']
+    import platform
+    env['CM_HOST_SYSTEM_NAME'] = platform.node()
 
     return {'return':0}

cm-mlops/script/generate-mlperf-inference-submission/_cm.json

@@ -45,7 +45,11 @@
           "yes"
         ]
       },
-      "tags": "submission,checker,mlc"
+      "tags": "submission,checker,mlc",
+      "names": [
+	"mlperf-inference-submission-checker",
+	"submission-checker"
+      ]
     },
     {
       "enable_if_env": {

cm-mlops/script/get-dataset-openimages/_cm.json

@@ -10,6 +10,13 @@
     {
       "tags": "get,sys-utils-cm"
     },
+    {
+      "tags": "get,python3",
+      "names": [
+	"python",
+	"python3"
+      ]
+    },
     {
       "names": [
         "inference-src"

cm-mlops/script/get-generic-python-lib/customize.py

@@ -6,6 +6,7 @@ def preprocess(i):
     os_info = i['os_info']
 
     env = i['env']
+    env['CM_PYTHON_PACKAGE_NAME_ENV'] = env['CM_PYTHON_PACKAGE_NAME'].replace("-", "_")
 
     recursion_spaces = i['recursion_spaces']
 
@@ -26,7 +27,7 @@ def detect_version(i):
     env = i['env']
     r = i['automation'].parse_version({'match_text': r'\s*([\d.a-z\-]+)',
                                        'group_number': 1,
-                                       'env_key':'CM_'+env['CM_PYTHON_PACKAGE_NAME'].upper()+'_VERSION',
+                                       'env_key':'CM_'+env['CM_PYTHON_PACKAGE_NAME_ENV'].upper()+'_VERSION',
                                        'which_env':i['env']})
     if r['return'] >0: return r
 
@@ -44,6 +45,6 @@ def postprocess(i):
 
     version = r['version']
 
-    env['CM_PYTHONLIB_'+env['CM_PYTHON_PACKAGE_NAME'].upper()+'_CACHE_TAGS'] = 'version-'+version
+    env['CM_PYTHONLIB_'+env['CM_PYTHON_PACKAGE_NAME_ENV'].upper()+'_CACHE_TAGS'] = 'version-'+version
 
     return {'return':0, 'version': version}

cm-mlops/script/get-mlperf-inference-sut-description/customize.py

@@ -49,6 +49,15 @@ def preprocess(i):
             state['CM_SUT_META']['submitter'] = submitter
             state['CM_SUT_META']['operating_system'] = os_name_string
             state['CM_SUT_META']['other_software_stack'] = "Python: " + python_version + ", " + compiler + "-" + compiler_version
+            if 'system_name' not in state['CM_SUT_META']:
+                system_name = env.get('CM_MLPERF_SYSTEM_NAME')
+                if not system_name:
+                    system_name = env.get('CM_HOST_SYSTEM_NAME')
+                    if system_name:
+                        system_name+=" (auto detected)"
+                    else:
+                        system_name = " (generic)"
+            state['CM_SUT_META']['system_name'] = system_name
             if 'host_processor_core_count' not in state['CM_SUT_META']:
                 state['CM_SUT_META']['host_processor_core_count'] = env.get('CM_HOST_CPU_TOTAL_CORES', '')
             if 'host_processor_model_name' not in state['CM_SUT_META']:
@@ -64,8 +73,8 @@ def preprocess(i):
                 state['CM_SUT_META']['host_processor_frequency'] = env.get('CM_HOST_CPU_MAX_MHZ','')
             if 'host_memory_capacity' not in state['CM_SUT_META']:
                 state['CM_SUT_META']['host_memory_capacity'] = env.get('CM_HOST_MEMORY_CAPACITY', '')
-            if 'host_disk_capacity' not in state['CM_SUT_META']:
-                state['CM_SUT_META']['host_disk_capacity'] = env.get('CM_HOST_DISK_CAPACITY', '')
+            if 'host_storage_capacity' not in state['CM_SUT_META']:
+                state['CM_SUT_META']['host_storage_capacity'] = env.get('CM_HOST_DISK_CAPACITY', '')
             if 'CM_SUT_SW_NOTES' in env:
                 sw_notes = env['CM_SUT_SW_NOTES']
             else:

cm-mlops/script/get-mlperf-inference-sut-description/hardware/default.json

@@ -20,6 +20,5 @@
   "status": "available",
   "submitter": "OctoML",
   "sw_notes": "Powered by Collective Mind (CK2)",
-  "system_name": "AWS g4dn.4xlarge",
-  "system_type": "datacenter"
+  "system_type": "edge"
 }

cm-mlops/script/run-mlperf-inference-app/_cm.json

@@ -104,7 +104,10 @@
       },
       "post_deps": [
         {
-          "tags": "generate,mlperf,inference,submission"
+          "tags": "generate,mlperf,inference,submission",
+	  "names": [
+	    "submission-generator"
+	  ]
         }
       ]
     },

cm-mlops/script/run-mlperf-inference-app/customize.py

@@ -71,6 +71,10 @@ def preprocess(i):
             r = cm.access({'action':'run', 'automation':'script', 'tags': tags, 'quiet': 'true',
                 'env': env, 'input': inp, 'state': state, 'add_deps': inp.get('add_deps',{}), 'add_deps_recursive':
                 inp.get('add_deps_recursive', {}), 'adr': inp.get('adr', {}), 'silent': silent})
+            if r['return'] > 0:
+                return r
+            if 'CM_MLPERF_RESULTS_DIR' in r['new_env']:
+                env['CM_MLPERF_RESULTS_DIR'] = r['new_env']['CM_MLPERF_RESULTS_DIR']
         if system_meta.get('division', '') == "open":
             env["CM_LOADGEN_COMPLIANCE"] = "no" #no compliance runs needed for open division
         if env.get("CM_LOADGEN_COMPLIANCE", "") == "yes":

cm-mlops/script/run-mlperf-submission-checker/_cm.json → cm-mlops/script/run-mlperf-inference-submission-checker/_cm.json

@@ -1,9 +1,9 @@
 {
-  "alias": "run-mlperf-submission-checker",
+  "alias": "run-mlperf-inference-submission-checker",
   "automation_alias": "script",
   "automation_uid": "5b4e0237da074764",
-  "category": "Modular MLPerf benchmarks",
   "cache": false,
+  "category": "Modular MLPerf benchmarks",
   "clean_files": [],
   "deps": [
     {
@@ -21,6 +21,9 @@
     "submission_dir": "CM_MLPERF_SUBMISSION_DIR",
     "submitter": "CM_MLPERF_SUBMITTER"
   },
+  "default_env": {
+    "CM_MLPERF_SHORT_RUN": "no"
+  },
   "tags": [
     "run",
     "mlc",
@@ -33,5 +36,12 @@
     "submission-checker",
     "mlc-submission-checker"
   ],
-  "uid": "15d03ec2c1af4297"
+  "uid": "15d03ec2c1af4297",
+  "variations": {
+    "short-run": {
+      "env": {
+	"CM_MLPERF_SHORT_RUN": "yes"
+      }
+    }
+  }
 }

cm-mlops/script/run-mlperf-inference-submission-checker/customize.py

@@ -0,0 +1,43 @@
+from cmind import utils
+import cmind as cm
+import os
+import subprocess
+from os.path import exists
+
+def preprocess(i):
+
+    os_info = i['os_info']
+    env = i['env']
+    submission_dir = env.get("CM_MLPERF_SUBMISSION_DIR", "")
+    if submission_dir == "":
+        return {'return': 1, 'error': 'Please set CM_MLPERF_SUBMISSION_DIR'}
+    submitter = env.get("CM_MLPERF_SUBMITTER", "default")
+    if 'CM_MLPERF_SKIP_COMPLIANCE' in env: 
+        skip_compliance = " --skip_compliance"
+    else:
+        skip_compliance = ""
+    submission_checker_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission",
+            "submission_checker.py")
+    if env['CM_MLPERF_SHORT_RUN'] == "yes":
+        import shutil
+        new_submission_checker_file = os.path.join(os.path.dirname(submission_checker_file), "submission_checker1.py")
+        with open(submission_checker_file, 'r') as file:
+            data = file.read()
+        data = data.replace("OFFLINE_MIN_SPQ = 24576", "OFFLINE_MIN_SPQ = 100")
+        data = data.replace("return is_valid, res, inferred", "return True, res, inferred")
+        with open(new_submission_checker_file, 'w') as file:
+            file.write(data)
+        submission_checker_file = new_submission_checker_file
+    CMD = env['CM_PYTHON_BIN'] + ' ' + submission_checker_file + " --input " + submission_dir + " --submitter " + submitter + \
+            skip_compliance
+    ret = os.system(CMD)
+
+    return {'return':0}
+
+def postprocess(i):
+    env = i['env']
+    import pandas
+    if exists('summary.csv'):
+        df = pandas.read_csv('summary.csv')
+        print(df)
+    return {'return':0}

docs/README.md

@@ -1,7 +1,7 @@
 # CM documentation
 
 This is the documentation for the [Collective Knowledge concept (CK)](https://arxiv.org/pdf/2011.01149.pdf) 
-and the Collective Mind automation meta-framework (CM)
+and the [Collective Mind automation meta-framework (CM)](https://github.com/mlcommons/ck)
 being developed by the [open MLCommons taskforce on education and reproducibility](mlperf-education-workgroup.md).
 
 
@@ -26,7 +26,7 @@ being developed by the [open MLCommons taskforce on education and reproducibilit
 ## Developer guide
 
 * [Contributing guidelines](../CONTRIBUTING.md)
-* [CM architecture and developer conventions](dev/README.md)
+* [CM architecture and developer conventions](development.md)
 * [CM core API](https://cknowledge.org/docs/cm/api/cmind.html)
 * [Adding new artifacts, scripts and workflows to CM](../cm/docs/tutorial-scripts.md#adding-new-artifacts-scripts-and-workflows-to-cm) 
 
@@ -40,16 +40,6 @@ being developed by the [open MLCommons taskforce on education and reproducibilit
 
 
 
-## Legacy MLCommons CK framework (CK1)
-
-The MLCommons CK (the 1st generation) was discontinued in summer 2022 after the stable release of the 2nd generation of this framework (Collective Mind aka CM or CK2).
-You can access this project [here](../ck1).
-
-
-## Maintainers
-
-* [Grigori Fursin](https://cknowledge.io/@gfursin) (OctoML, MLCommons, cTuning foundation)
-* [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) (OctoML, MLCommons)
 
 ## Acknowledgments
 

docs/overview.md

@@ -51,6 +51,11 @@ to provide your feedback and participate in further community developments!
 
 &copy; 2021-2022 [MLCommons](https://mlcommons.org)<br>
 
+## Legacy MLCommons CK framework (CK1)
+
+The MLCommons CK (the 1st generation) was discontinued in summer 2022 after the stable release of the 2nd generation of this framework (Collective Mind aka CM or CK2).
+You can access this project [here](../ck1).
+
 ## Resources
 
 * [MLOps resources](misc/MLOps.md)
@@ -60,8 +65,3 @@ to provide your feedback and participate in further community developments!
 
 * [Grigori Fursin](https://cknowledge.io/@gfursin) (OctoML, MLCommons, cTuning foundation)
 * [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) (OctoML, MLCommons)
-
-# Acknowledgments
-
-This project is supported by [MLCommons](https://mlcommons.org), [OctoML](https://octoml.ai) 
-and [individual contributors](https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md).