Merge from OctoML (#481)

mlcommons · Nov 8, 2022 · d5d1773 · d5d1773
2 parents 8f135bc + e001637
commit d5d1773
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 16 deletions.
diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py
@@ -1284,6 +1284,19 @@ def run(self, i):
                     remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state)
             if r['return']>0: return r
 
+            # Add extra tags from env updated by deps (such as python version and compiler version, etc)
+            extra_cache_tags_from_env = meta.get('extra_cache_tags_from_env',[])
+            for extra_cache_tags in extra_cache_tags_from_env:
+                key = extra_cache_tags['env']
+                prefix = extra_cache_tags.get('prefix','')
+
+                v = env.get(key,'').strip()
+                if v!='':
+                    for t in v.split(','):
+                        x = 'deps-' + prefix + t
+                        if x not in cached_tags: 
+                            cached_tags.append(x)
+
 
         ############################################################################################################
         ##################################### Finalize script
@@ -1439,11 +1452,6 @@ def run(self, i):
 
 
 
-
-
-
-
-
     ############################################################
     def version(self, i):
         """
@@ -2385,7 +2393,7 @@ def doc(self, i):
         if repos == '': repos='internal,a4705959af8e447a'
 
         parsed_artifact = i.get('parsed_artifact',[])
-        
+
         if len(parsed_artifact)<1:
             parsed_artifact = [('',''), ('','')]
         elif len(parsed_artifact)<2:
@@ -2394,7 +2402,7 @@ def doc(self, i):
             repos = parsed_artifact[1][0]
 
         list_of_repos = repos.split(',') if ',' in repos else [repos]
-        
+
         ii = utils.sub_input(i, self.cmind.cfg['artifact_keys'])
 
         ii['out'] = None
@@ -2449,7 +2457,7 @@ def doc(self, i):
 
                 if category not in toc_category_sort or category_sort>0:
                     toc_category_sort[category]=category_sort
-                
+
                 if alias not in toc_category[category]:
                     toc_category[category].append(alias)
 
@@ -2481,7 +2489,7 @@ def doc(self, i):
 
             md.append('## '+alias)
             md.append('\n')
-            
+
             if name!='':
                 md.append('*'+name+'.*')
                 md.append('\n')
@@ -2498,7 +2506,7 @@ def doc(self, i):
             meta_path = os.path.join(path, meta_file)
 
             meta_file += '.yaml' if os.path.isfile(meta_path+'.yaml') else '.json'
-            
+
             meta_url = url+'/'+meta_file
 
             md.append('* CM script meta description: *[GitHub]({})*'.format(meta_url))
@@ -2523,7 +2531,7 @@ def doc(self, i):
 
 
 
-            
+
             md.append('\n')
 
 
@@ -2653,7 +2661,7 @@ def find_cached_script(i):
             if t not in cached_tags: 
                 cached_tags.append(t)
 
-    # Add tags from deps
+    # Add tags from deps (will be also duplicated when creating new cache entry)
     extra_cache_tags_from_env = meta.get('extra_cache_tags_from_env',[])
     for extra_cache_tags in extra_cache_tags_from_env:
         key = extra_cache_tags['env']

diff --git a/cm-mlops/script/get-dataset-openimages/_cm.json b/cm-mlops/script/get-dataset-openimages/_cm.json
@@ -65,6 +65,26 @@
       "env": {
         "CM_DATASET_CALIBRATION": "no"
       }
+    },
+    "500": {
+      "env": {
+        "CM_DATASET_SIZE": "500"
+      }
+    },
+    "50": {
+      "env": {
+        "CM_DATASET_SIZE": "50"
+      }
+    },
+    "5": {
+      "env": {
+        "CM_DATASET_SIZE": "5"
+      }
+    },
+    "1": {
+      "env": {
+        "CM_DATASET_SIZE": "1"
+      }
     }
   }
 }
diff --git a/cm-mlops/script/get-dataset-openimages/run.sh b/cm-mlops/script/get-dataset-openimages/run.sh
@@ -4,13 +4,16 @@ CUR=${PWD}
 mkdir -p install
 INSTALL_DIR=${CUR}/install
 
-${CM_PYTHON_BIN} -m pip install fiftyone
-
 cd ${CM_MLPERF_INFERENCE_VISION_PATH}
 cd tools
 
 if [[ ${CM_DATASET_CALIBRATION} == "no" ]]; then
-  ./openimages_mlperf.sh -d ${INSTALL_DIR}
+  if [ ! -z {CM_DATASET_SIZE} ]; then
+    max_images=" -m ${CM_DATASET_SIZE}"
+  else
+    max_images=""
+  fi
+  ./openimages_mlperf.sh -d ${INSTALL_DIR} ${max_images}
   test $? -eq 0 || exit 1
   cd $CUR
   echo "CM_DATASET_PATH=${INSTALL_DIR}" > tmp-run-env.out

diff --git a/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json b/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json
@@ -54,16 +54,41 @@
     "1": {
       "env": {
         "CM_DATASET_SIZE": "1"
+      },
+      "adr": {
+	"original-dataset": {
+	  "tags": "_1"
+	}
+      }
+    },
+    "5": {
+      "env": {
+        "CM_DATASET_SIZE": "5"
+      },
+      "adr": {
+	"original-dataset": {
+	  "tags": "_5"
+	}
       }
     },
     "50": {
       "env": {
         "CM_DATASET_SIZE": "50"
+      },
+      "adr": {
+	"original-dataset": {
+	  "tags": "_50"
+	}
       }
     },
     "500": {
       "env": {
         "CM_DATASET_SIZE": "500"
+      },
+      "adr": {
+	"original-dataset": {
+	  "tags": "_500"
+	}
       }
     },
     "NCHW": {

diff --git a/cm-mlops/script/get-sys-utils-cm/run-macos.sh b/cm-mlops/script/get-sys-utils-cm/run-macos.sh
@@ -35,4 +35,4 @@ brew update && \
            zlib \
            python3
 
-python3 -m pip install ${CM_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${CM_PYTHON_PIP_USER}
+python3 -m pip install -r ${CM_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${CM_PYTHON_PIP_USER}
-Original file line number
+Diff line change
@@ Expand Up / @@ -35,4 +35,4 @@ brew update && \ @@
                zlib \
                python3
-    python3 -m pip install ${CM_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${CM_PYTHON_PIP_USER}
+    python3 -m pip install -r ${CM_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${CM_PYTHON_PIP_USER}