diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py index d6622160b6..71854d0a9b 100644 --- a/cm-mlops/automation/script/module.py +++ b/cm-mlops/automation/script/module.py @@ -1284,6 +1284,19 @@ def run(self, i): remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r + # Add extra tags from env updated by deps (such as python version and compiler version, etc) + extra_cache_tags_from_env = meta.get('extra_cache_tags_from_env',[]) + for extra_cache_tags in extra_cache_tags_from_env: + key = extra_cache_tags['env'] + prefix = extra_cache_tags.get('prefix','') + + v = env.get(key,'').strip() + if v!='': + for t in v.split(','): + x = 'deps-' + prefix + t + if x not in cached_tags: + cached_tags.append(x) + ############################################################################################################ ##################################### Finalize script @@ -1439,11 +1452,6 @@ def run(self, i): - - - - - ############################################################ def version(self, i): """ @@ -2385,7 +2393,7 @@ def doc(self, i): if repos == '': repos='internal,a4705959af8e447a' parsed_artifact = i.get('parsed_artifact',[]) - + if len(parsed_artifact)<1: parsed_artifact = [('',''), ('','')] elif len(parsed_artifact)<2: @@ -2394,7 +2402,7 @@ def doc(self, i): repos = parsed_artifact[1][0] list_of_repos = repos.split(',') if ',' in repos else [repos] - + ii = utils.sub_input(i, self.cmind.cfg['artifact_keys']) ii['out'] = None @@ -2449,7 +2457,7 @@ def doc(self, i): if category not in toc_category_sort or category_sort>0: toc_category_sort[category]=category_sort - + if alias not in toc_category[category]: toc_category[category].append(alias) @@ -2481,7 +2489,7 @@ def doc(self, i): md.append('## '+alias) md.append('\n') - + if name!='': md.append('*'+name+'.*') md.append('\n') @@ -2498,7 +2506,7 @@ def doc(self, i): meta_path = os.path.join(path, meta_file) meta_file += '.yaml' if os.path.isfile(meta_path+'.yaml') else '.json' - + meta_url = url+'/'+meta_file md.append('* CM script meta description: *[GitHub]({})*'.format(meta_url)) @@ -2523,7 +2531,7 @@ def doc(self, i): - + md.append('\n') @@ -2653,7 +2661,7 @@ def find_cached_script(i): if t not in cached_tags: cached_tags.append(t) - # Add tags from deps + # Add tags from deps (will be also duplicated when creating new cache entry) extra_cache_tags_from_env = meta.get('extra_cache_tags_from_env',[]) for extra_cache_tags in extra_cache_tags_from_env: key = extra_cache_tags['env'] diff --git a/cm-mlops/script/get-dataset-openimages/_cm.json b/cm-mlops/script/get-dataset-openimages/_cm.json index bb7eab4e23..0ab4f06ad5 100644 --- a/cm-mlops/script/get-dataset-openimages/_cm.json +++ b/cm-mlops/script/get-dataset-openimages/_cm.json @@ -65,6 +65,26 @@ "env": { "CM_DATASET_CALIBRATION": "no" } + }, + "500": { + "env": { + "CM_DATASET_SIZE": "500" + } + }, + "50": { + "env": { + "CM_DATASET_SIZE": "50" + } + }, + "5": { + "env": { + "CM_DATASET_SIZE": "5" + } + }, + "1": { + "env": { + "CM_DATASET_SIZE": "1" + } } } } diff --git a/cm-mlops/script/get-dataset-openimages/run.sh b/cm-mlops/script/get-dataset-openimages/run.sh index bcc5695953..8f767320d0 100644 --- a/cm-mlops/script/get-dataset-openimages/run.sh +++ b/cm-mlops/script/get-dataset-openimages/run.sh @@ -4,13 +4,16 @@ CUR=${PWD} mkdir -p install INSTALL_DIR=${CUR}/install -${CM_PYTHON_BIN} -m pip install fiftyone - cd ${CM_MLPERF_INFERENCE_VISION_PATH} cd tools if [[ ${CM_DATASET_CALIBRATION} == "no" ]]; then - ./openimages_mlperf.sh -d ${INSTALL_DIR} + if [ ! -z {CM_DATASET_SIZE} ]; then + max_images=" -m ${CM_DATASET_SIZE}" + else + max_images="" + fi + ./openimages_mlperf.sh -d ${INSTALL_DIR} ${max_images} test $? -eq 0 || exit 1 cd $CUR echo "CM_DATASET_PATH=${INSTALL_DIR}" > tmp-run-env.out diff --git a/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json b/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json index 2ae0e7a05a..4773ef887a 100644 --- a/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json +++ b/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json @@ -54,16 +54,41 @@ "1": { "env": { "CM_DATASET_SIZE": "1" + }, + "adr": { + "original-dataset": { + "tags": "_1" + } + } + }, + "5": { + "env": { + "CM_DATASET_SIZE": "5" + }, + "adr": { + "original-dataset": { + "tags": "_5" + } } }, "50": { "env": { "CM_DATASET_SIZE": "50" + }, + "adr": { + "original-dataset": { + "tags": "_50" + } } }, "500": { "env": { "CM_DATASET_SIZE": "500" + }, + "adr": { + "original-dataset": { + "tags": "_500" + } } }, "NCHW": { diff --git a/cm-mlops/script/get-sys-utils-cm/run-macos.sh b/cm-mlops/script/get-sys-utils-cm/run-macos.sh index 52697b5ab0..39e256cecf 100644 --- a/cm-mlops/script/get-sys-utils-cm/run-macos.sh +++ b/cm-mlops/script/get-sys-utils-cm/run-macos.sh @@ -35,4 +35,4 @@ brew update && \ zlib \ python3 -python3 -m pip install ${CM_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${CM_PYTHON_PIP_USER} +python3 -m pip install -r ${CM_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${CM_PYTHON_PIP_USER}