improving docs and minor fixes (#760)

cm-mlops/automation/script/module_misc.py

@@ -1111,9 +1111,19 @@ def dockerfile(i):
             "rhel": ["9"]
         }
     else:
-        docker_os = {
+        if i.get('docker_os'):
+            docker_os = {}
+            docker_os[i['docker_os']] = []
+        if i.get('docker_os_version'):
+            docker_os[i['docker_os']] = [i.get('docker_os_version')]
+        else:
+            if docker_os == "ubuntu":
+                docker_os[i['docker_os']] = ["22.04"]
+
+        if not docker_os:
+          docker_os = {
             "ubuntu": ["22.04"],
-        }
+          }
 
     for artifact in sorted(lst, key = lambda x: x.meta.get('alias','')):
 
@@ -1125,21 +1135,24 @@ def dockerfile(i):
         for _os in docker_os:
             for version in docker_os[_os]:
                 dockerfile_path = os.path.join(script_path,'dockerfiles', _os +'_'+version +'.Dockerfile')
-                cm_input = {'action': 'run',
+                if i.get('print_deps'):
+                  cm_input = {'action': 'run',
                             'automation': 'script',
                             'tags': f'{tag_string}',
                             'print_deps': True,
                             'quiet': True,
                             'silent': True,
                             'fake_run': True
                             }
-                r = self_module.cmind.access(cm_input)
-                if r['return'] > 0:
+                  r = self_module.cmind.access(cm_input)
+                  if r['return'] > 0:
                     return r
-                print_deps = r['new_state']['print_deps']
-                comments = [ "#RUN " + dep for dep in print_deps ]
-                comments.append("")
-                comments.append("# Run CM workflow")
+                  print_deps = r['new_state']['print_deps']
+                  comments = [ "#RUN " + dep for dep in print_deps ]
+                  comments.append("")
+                  comments.append("# Run CM workflow")
+                else:
+                  comments = []
 
                 cm_docker_input = {'action': 'run',
                             'automation': 'script',
@@ -1241,8 +1254,8 @@ def docker(i):
         script_alias = meta.get('alias')
         tag_string=",".join(tags)
 
-        _os="ubuntu"
-        version="22.04"
+        _os=i.get('docker_os', 'ubuntu')
+        version=i.get('docker_os_version', '22.04')
 
         dockerfile_path = os.path.join(script_path,'dockerfiles', _os +'_'+version +'.Dockerfile')
 

cm-mlops/automation/script/template/run.sh

@@ -1,3 +1,23 @@
 #!/bin/bash
 
-test $? -eq 0 || exit $?
+#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH}
+
+#To export any variable
+#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out
+
+#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
+
+function exit_if_error() {
+  test $? -eq 0 || exit $?
+}
+
+function run() {
+  echo "Running: "
+  echo "$1"
+  echo ""
+  if [[ ${CM_FAKE_RUN} != 'yes' ]]; then
+    eval $1
+  fi
+}
+
+#Add your run commands here...

cm-mlops/challenge/reproduce-and-automate-ipol-paper/_cm.json

@@ -2,6 +2,7 @@
   "alias": "reproduce-and-automate-ipol-paper",
   "automation_alias": "challenge",
   "automation_uid": "3d84abd768f34e08",
+  "date_close": "20230515",
   "date_open": "20230301",
   "experiments": [],
   "tags": [

cm-mlops/script/README.md

@@ -23,7 +23,7 @@ See the catalog [here](https://github.com/mlcommons/ck/blob/master/docs/list_of_
 ### Input flags
 When we run a CM script we can also pass inputs to it and any input added in `input_mapping` dictionary inside `_cm.json` gets converted to the corresponding `ENV` variable.
 
-### Consitional execution of any `deps`, `post_deps`
+### Conditional execution of any `deps`, `post_deps`
 We can use `skip_if_env` dictionary inside any `deps`, `prehook_deps`, `posthook_deps` or `post_deps` to make its executional conditional
 
 ### Versions
@@ -54,6 +54,10 @@ Sometimes it is difficult to add all variations needed for a script like say `ba
 * If `env['CM_GH_TOKEN']=TOKEN_VALUE` is set then git URLs (specified by `CM_GIT_URL`) are changed to add this token.
 * If `env['CM_GIT_SSH']=yes`, then git URLs are chnged to SSH from HTTPS.
 
+### Script Meta
+#### Special keys in script meta
+* TBD: `reuse_version`, `inherit_variation_tags`, `update_env_tags_from_env`
+
 ### How cache works?
 * If `cache=true` is set in a script meta, the result of the script execution is cached for further use. 
 * For a cached script, `env` and `state` updates are done using `new_env` and `new_state` dictionaries which are stored in the `cm-cached.json` file inside the cached folder.

cm-mlops/script/download-torrent/README.md

@@ -0,0 +1,155 @@
+<details>
+<summary>Click here to see the table of contents.</summary>
+
+* [Description](#description)
+* [Information](#information)
+* [Usage](#usage)
+  * [ CM installation](#cm-installation)
+  * [ CM script automation help](#cm-script-automation-help)
+  * [ CM CLI](#cm-cli)
+  * [ CM Python API](#cm-python-api)
+  * [ CM GUI](#cm-gui)
+  * [ CM modular Docker container](#cm-modular-docker-container)
+* [Customization](#customization)
+  * [ Variations](#variations)
+  * [ Default environment](#default-environment)
+* [Script workflow, dependencies and native scripts](#script-workflow-dependencies-and-native-scripts)
+* [Script output](#script-output)
+* [New environment keys (filter)](#new-environment-keys-(filter))
+* [New environment keys auto-detected from customize](#new-environment-keys-auto-detected-from-customize)
+* [Maintainers](#maintainers)
+
+</details>
+
+*Note that this README is automatically generated - don't edit! See [more info](README-extra.md).*
+
+### Description
+
+
+See [more info](README-extra.md).
+
+#### Information
+
+* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/master/cm-mlops)*
+* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent)*
+* CM meta description for this script: *[_cm.json](_cm.json)*
+* CM "database" tags to find this script: *download,torrent,download-torrent*
+* Output cached?: *True*
+___
+### Usage
+
+#### CM installation
+
+[Guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md)
+
+##### CM pull repository
+
+```cm pull repo mlcommons@ck```
+
+##### CM script automation help
+
+```cm run script --help```
+
+#### CM CLI
+
+1. `cm run script --tags=download,torrent,download-torrent[,variations] `
+
+2. `cm run script "download torrent download-torrent[,variations]" `
+
+3. `cm run script 69b752c5618e45bb `
+
+* `variations` can be seen [here](#variations)
+
+* `input_flags` can be seen [here](#script-flags-mapped-to-environment)
+
+#### CM Python API
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+```python
+
+import cmind
+
+r = cmind.access({'action':'run'
+                  'automation':'script',
+                  'tags':'download,torrent,download-torrent'
+                  'out':'con',
+                  ...
+                  (other input keys for this script)
+                  ...
+                 })
+
+if r['return']>0:
+    print (r['error'])
+
+```
+
+</details>
+
+
+#### CM GUI
+
+```cm run script --tags=gui --script="download,torrent,download-torrent"```
+
+Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=download,torrent,download-torrent) to generate CM CMD.
+
+#### CM modular Docker container
+
+*TBD*
+
+___
+### Customization
+
+
+#### Variations
+
+  * *No group (any variation can be selected)*
+    <details>
+    <summary>Click here to expand this section.</summary>
+
+    * `_torrent.#`
+      - Environment variables:
+        - *CM_TORRENT_FILE*: `#`
+      - Workflow:
+
+    </details>
+
+#### Default environment
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags.
+
+
+</details>
+
+___
+### Script workflow, dependencies and native scripts
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+  1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/_cm.json)***
+     * get,generic-sys-util,_transmission
+       - CM script: [get-generic-sys-util](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-sys-util)
+  1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/customize.py)***
+  1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/_cm.json)
+  1. ***Run native script if exists***
+     * [run.sh](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/run.sh)
+  1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/_cm.json)
+  1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/customize.py)***
+  1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-torrent/_cm.json)
+</details>
+
+___
+### Script output
+#### New environment keys (filter)
+
+#### New environment keys auto-detected from customize
+
+___
+### Maintainers
+
+* [Open MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)

cm-mlops/script/download-torrent/_cm.json

@@ -0,0 +1,32 @@
+{
+  "alias": "download-torrent",
+  "automation_alias": "script",
+  "automation_uid": "5b4e0237da074764",
+  "cache": true,
+  "deps": [
+    {
+      "tags": "get,generic-sys-util,_transmission"
+    }
+  ],
+  "input_description": {},
+  "input_mapping": {},
+  "new_env_keys": [],
+  "new_state_keys": [],
+  "post_deps": [],
+  "posthook_deps": [],
+  "prehook_deps": [],
+  "tags": [
+    "download",
+    "torrent",
+    "download-torrent"
+  ],
+  "uid": "69b752c5618e45bb",
+  "variations": {
+    "torrent.#": {
+      "env": {
+        "CM_TORRENT_FILE": "#"
+      }
+    }
+  },
+  "versions": {}
+}

cm-mlops/script/download-torrent/customize.py

@@ -0,0 +1,22 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    env = i['env']
+
+    meta = i['meta']
+
+    automation = i['automation']
+
+    quiet = (env.get('CM_QUIET', False) == 'yes')
+
+    return {'return':0}
+
+def postprocess(i):
+
+    env = i['env']
+
+    return {'return':0}

cm-mlops/script/download-torrent/run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+transmission-daemon --download-dir ${PWD}
+transmission-remote --no-auth -a ${CM_TORRENT_FILE}
+test $? -eq 0 || exit $?
+transmission-remote -l
+test $? -eq 0 || exit $?

cm-mlops/script/get-generic-python-lib/_cm.json

@@ -34,6 +34,11 @@
   ],
   "uid": "94b62a682bc44791",
   "variations": {
+    "package.#": {
+      "env": {
+        "CM_GENERIC_PYTHON_PACKAGE_NAME": "#"
+      }
+    },
     "apache-tvm": {
       "deps": [
         {
@@ -245,6 +250,12 @@
         "CM_MLPERF_LOGGING_VERSION"
       ]
     },
+    "dllogger": {
+      "env": {
+        "CM_GENERIC_PYTHON_PACKAGE_NAME": "dllogger",
+        "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/NVIDIA/dllogger#egg=dllogger"
+      }
+    },
     "mpld3": {
       "env": {
         "CM_GENERIC_PYTHON_PACKAGE_NAME": "mpld3"

cm-mlops/script/get-generic-sys-util/_cm.json

@@ -140,6 +140,19 @@
         }
       }
     },
+    "transmission": {
+      "env": {
+        "CM_SYS_UTIL_NAME": "transmission"
+      },
+      "state": {
+        "transmission": {
+          "apt": "transmission-daemon",
+          "dnf": "transmission-daemon",
+          "yum": "transmission-daemon",
+          "brew": "transmission"
+        }
+      }
+    },
     "libpci-dev": {
       "env": {
         "CM_SYS_UTIL_NAME": "libpci-dev"

cm-mlops/script/get-qaic-compute-sdk/run.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 wget https://codelinaro.jfrog.io/artifactory/codelinaro-toolchain-for-hexagon/v15.0.5/clang+llvm-15.0.5-cross-hexagon-unknown-linux-musl.tar.xz
+tar -xvf clang+llvm-15.0.5-cross-hexagon-unknown-linux-musl.tar.xz
 export HEXAGON_TOOLS_DIR=`pwd`/clang+llvm-15.0.5-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu
 cd ${CM_QAIC_COMPUTE_SDK_PATH}
 ./scripts/build.sh --${CM_QAIC_COMPUTE_SDK_INSTALL_MODE} --install