Merge pull request #19 from mlcommons/dev

Major update to support CM v2.2.0

README.md

@@ -1,7 +1,9 @@
-# Cross-platform CM interface for DevOps, MLOps, AIOps and MLPerf
+## Unified and cross-platform CM interface for DevOps, MLOps and MLPerf
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE.md)
+[![Python Version](https://img.shields.io/badge/python-3+-blue.svg)](https://github.com/mlcommons/ck/tree/master/cm/cmind)
 [![Powered by CM](https://img.shields.io/badge/Powered_by-MLCommons%20CM-blue)](https://github.com/mlcommons/ck).
+[![Downloads](https://static.pepy.tech/badge/cmind)](https://pepy.tech/project/cmind)
 
 This repository contains reusable and cross-platform automation recipes to run DevOps, MLOps, AIOps and MLPerf 
 via a simple and human-readable [Collective Mind interface (CM)](https://github.com/mlcommons/ck) 
@@ -16,14 +18,23 @@ These automation recipes are being developed and maintained
 by the [MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)
 with [great contributions](CONTRIBUTING.md) from the community.
 
-### Catalog
+## Tests
+
+[![CM script automation test](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-scripts.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-scripts.yml)
+[![CM script automation features test](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-script-features.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-script-features.yml)
+[![MLPerf loadgen with HuggingFace bert onnx fp32 squad model](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml)
+[![MLPerf inference MLCommons C++ ResNet50](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml)
+[![image classification with ONNX](https://github.com/mlcommons/cm4mlops/actions/workflows/test-image-classification-onnx.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-image-classification-onnx.yml)
+
+
+## Catalog
 
 See the automatically generated catalog [online](https://access.cknowledge.org/playground/?action=scripts).
 
-### License
+## License
 
 [Apache 2.0](LICENSE.md)
 
-### Copyright
+## Copyright
 
 2022-2024 [MLCommons](https://mlcommons.org)

automation/script/_cm.json

@@ -2,11 +2,12 @@
   "alias": "script",
   "automation_alias": "automation",
   "automation_uid": "bbeb15d8f0a944a4",
+  "min_cm_version": "2.2.0",
   "deps": {
     "cache": "cache,541d6f712a6b464e"
   },
   "desc": "Making native scripts more portable, interoperable and deterministic",
-  "developers": "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)",
+  "developers": "Arjun Suresh and Grigori Fursin",
   "actions_with_help":["run", "docker"],
   "sort": 1000,
   "tags": [

automation/script/module.py

@@ -14,6 +14,7 @@
 
 from cmind.automation import Automation
 from cmind import utils
+from cmind import __version__ as current_cm_version
 
 class CAutomation(Automation):
     """
@@ -150,6 +151,8 @@ def run(self, i):
                                       inside a script specified by these tags
 
           (debug_script) (bool): if True, debug current script (set debug_script_tags to the tags of a current script)
+          (debug_uid) (str): if True, set CM_TMP_DEBUG_UID to this number to enable
+                             remote python debugging of scripts and wrapped apps/tools
           (detected_versions) (dict): All the used scripts and their detected_versions
 
           (verbose) (bool): if True, prints all tech. info about script execution (False by default)
@@ -180,6 +183,9 @@ def run(self, i):
           (skip_sudo) (bool): if True, set env['CM_TMP_SKIP_SUDO']='yes'
                               to let scripts deal with that
 
+          (silent) (bool): if True, attempt to suppress all info if supported
+                           (sets CM_TMP_SILENT=yes)
+          (s) (bool): the same as 'silent'
           ...
 
         Returns:
@@ -309,14 +315,6 @@ def _run(self, i):
 
         print_env = i.get('print_env', False)
 
-        verbose = False
-
-        if 'verbose' in i: verbose=i['verbose']
-        elif 'v' in i: verbose=i['v']
-
-        if verbose:
-           env['CM_VERBOSE']='yes'
-
         show_time = i.get('time', False)
         show_space = i.get('space', False)
 
@@ -331,6 +329,10 @@ def _run(self, i):
         fake_run = i.get('fake_run', False)
         fake_run = i.get('fake_run', False) if 'fake_run' in i else i.get('prepare', False)
         if fake_run: env['CM_TMP_FAKE_RUN']='yes'
+
+        debug_uid = i.get('debug_uid', '')
+        if debug_uid!='':
+            env['CM_TMP_DEBUG_UID'] = debug_uid
 
         fake_deps = i.get('fake_deps', False)
         if fake_deps: env['CM_TMP_FAKE_DEPS']='yes'
@@ -348,6 +350,28 @@ def _run(self, i):
         if fake_deps:
             run_state['fake_deps'] = True
 
+        # Check verbose and silent
+        verbose = False
+
+        silent = True if str(i.get('silent', '')).lower() in ['true', 'yes', 'on'] else False
+
+        if not silent:
+            silent = True if str(i.get('s', '')).lower() in ['true', 'yes', 'on'] else False
+
+        if silent:
+            if 'verbose' in i: del(i['verbose'])
+            if 'v' in i: del(i['v'])
+            env['CM_TMP_SILENT']='yes'
+            run_state['tmp_silent']=True
+
+        if 'verbose' in i: verbose=i['verbose']
+        elif 'v' in i: verbose=i['v']
+
+        if verbose:
+           env['CM_VERBOSE']='yes'
+           run_state['tmp_verbose']=True
+
+
         print_deps = i.get('print_deps', False)
         print_readme = i.get('print_readme', False)
 
@@ -517,8 +541,9 @@ def _run(self, i):
 #        if verbose:
 #            print ('')
 
-        print ('')
-        print (recursion_spaces + '* ' + cm_script_info)
+        if not run_state.get('tmp_silent', False):
+            print ('')
+            print (recursion_spaces + '* ' + cm_script_info)
 
 
         #############################################################################
@@ -703,6 +728,15 @@ def _run(self, i):
         meta = script_artifact.meta
         path = script_artifact.path
 
+        # Check min CM version requirement
+        min_cm_version = meta.get('min_cm_version','').strip()
+        if min_cm_version != '':
+            # Check compare version while avoiding craches for older version
+            if 'compare_versions' in dir(utils):
+                comparison = utils.compare_versions(current_cm_version, min_cm_version)
+                if comparison < 0:
+                    return {'return':1, 'error':'CM script requires CM version >= {} while current CM version is {} - please update using "pip install cmind -U"'.format(min_cm_version, current_cm_version)}
+
         # Check path to repo
         script_repo_path = script_artifact.repo_path
 
@@ -1081,7 +1115,8 @@ def _run(self, i):
                     if r['return']>0: return r
                     version = r['meta'].get('version')
 
-                    print (recursion_spaces + '     ! load {}'.format(path_to_cached_state_file))
+                    if not run_state.get('tmp_silent', False):
+                        print (recursion_spaces + '     ! load {}'.format(path_to_cached_state_file))
 
 
                     ################################################################################################
@@ -1763,19 +1798,20 @@ def _run(self, i):
             input ('Press Enter to continue ...')
 
         # Check if need to print some final info such as path to model, etc
-        print_env_at_the_end = meta.get('print_env_at_the_end',{})
-        if len(print_env_at_the_end)>0:
-            print ('')
+        if not run_state.get('tmp_silent', False):
+            print_env_at_the_end = meta.get('print_env_at_the_end',{})
+            if len(print_env_at_the_end)>0:
+                print ('')
 
-            for p in sorted(print_env_at_the_end):
-                t = print_env_at_the_end[p]
-                if t == '': t = 'ENV[{}]'.format(p)
+                for p in sorted(print_env_at_the_end):
+                    t = print_env_at_the_end[p]
+                    if t == '': t = 'ENV[{}]'.format(p)
 
-                v = new_env.get(p, None)
+                    v = new_env.get(p, None)
 
-                print ('{}: {}'.format(t, str(v)))
+                    print ('{}: {}'.format(t, str(v)))
 
-            print ('')
+                print ('')
 
         return rr
 
@@ -2887,6 +2923,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
 
                     # Run collective script via CM API:
                     # Not very efficient but allows logging - can be optimized later
+
                     ii = {
                             'action':'run',
                             'automation':utils.assemble_cm_object(self.meta['alias'], self.meta['uid']),
@@ -2900,6 +2937,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
                             'add_deps_recursive':add_deps_recursive,
                             'debug_script_tags':debug_script_tags,
                             'verbose':verbose,
+                            'silent':run_state.get('tmp_silent', False),
                             'time':show_time,
                             'run_state':run_state
 
@@ -4295,8 +4333,9 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
             print (recursion_spaces + '  - Running native script "{}" from temporal script "{}" in "{}" ...'.format(path_to_run_script, run_script, cur_dir))
             print ('')
 
-        print (recursion_spaces + '       ! cd {}'.format(cur_dir))
-        print (recursion_spaces + '       ! call {} from {}'.format(path_to_run_script, run_script))
+        if not run_state.get('tmp_silent', False):
+            print (recursion_spaces + '       ! cd {}'.format(cur_dir))
+            print (recursion_spaces + '       ! call {} from {}'.format(path_to_run_script, run_script))
 
 
         # Prepare env variables
@@ -4388,7 +4427,8 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
 
 
     if postprocess != '' and customize_code is not None and postprocess in dir(customize_code):
-        print (recursion_spaces+'       ! call "{}" from {}'.format(postprocess, customize_code.__file__))
+        if not run_state.get('tmp_silent', False):
+            print (recursion_spaces+'       ! call "{}" from {}'.format(postprocess, customize_code.__file__))
 
     if len(posthook_deps)>0 and (postprocess == "postprocess"):
         r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state,

automation/script/template/run.sh

@@ -7,21 +7,11 @@
 
 #${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
 
-
-
-function exit_if_error() {
-  test $? -eq 0 || exit $?
-}
-
-function run() {
-  echo "Running: "
-  echo "$1"
-  echo ""
-  if [[ ${CM_FAKE_RUN} != 'yes' ]]; then
-    eval "$1"
-    exit_if_error
-  fi
-}
-
-#Add your run commands here...
-# run "$CM_RUN_CMD"
+echo "Running: "
+echo "${CM_RUN_CMD}"
+echo ""
+
+if [[ ${CM_FAKE_RUN} != "yes" ]]; then
+  eval "${CM_RUN_CMD}"
+  test $? -eq 0 || exit 1
+fi

cmr.yaml

@@ -3,6 +3,8 @@ uid: 9e97bb72b0474657
 
 git: true
 
+version: 2.2.0
+
 deps:
   - alias: mlcommons@ck
     uid: a4705959af8e447a

script/app-image-classification-onnx-py/customize.py

@@ -7,8 +7,8 @@ def preprocess(i):
     os_info = i['os_info']
     env = i['env']
 
-    print ('')
-    print ('Running preprocess function in customize.py ...')
+#    print ('')
+#    print ('Running preprocess function in customize.py ...')
 
     return {'return':0}
 
@@ -55,10 +55,11 @@ def postprocess(i):
 
     top_classification = data.get('top_classification','')
 
-    if top_classification!='':
-        print ('')
-        x = 'Top classification: {}'.format(top_classification)
-        print ('='*len(x))
-        print (x)
+    if env.get('CM_TMP_SILENT','')!='yes':
+        if top_classification!='':
+            print ('')
+            x = 'Top classification: {}'.format(top_classification)
+            print ('='*len(x))
+            print (x)
 
     return {'return':0}

script/create-custom-cache-entry/_cm.yaml

@@ -0,0 +1,27 @@
+alias: create-custom-cache-entry
+uid: 485741440fbe4236
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- create
+- custom
+- cache
+- entry
+
+category: CM automation
+
+cache: true
+
+input_mapping:
+  env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY
+  env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2
+  path: CM_CUSTOM_CACHE_ENTRY_PATH
+  to: CM_CUSTOM_CACHE_ENTRY_PATH
+
+new_env_keys:
+- CM_CUSTOM_CACHE_ENTRY*
+
+print_env_at_the_end:
+  CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry"

script/create-custom-cache-entry/customize.py

@@ -0,0 +1,44 @@
+from cmind import utils
+import os
+import shutil
+
+def preprocess(i):
+
+    # CM script internal variables
+    env = i['env']
+
+    extra_cache_tags = []
+    if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '':
+        print ('')
+        extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ')
+
+        extra_cache_tags = extra_cache_tags_str.strip().split(',')
+
+    return {'return':0, 'add_extra_cache_tags':extra_cache_tags}
+
+def postprocess(i):
+
+    env = i['env']
+
+    path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip()
+
+    if path!='':
+        if not os.path.isdir(path):
+            os.makedirs(path)
+    else:
+       path = os.getcwd()
+
+    x = ''
+    env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '')
+    if env_key != '': x = env_key+'_'
+
+    env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path
+    env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path
+
+    env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '')
+    v = env.get(env_key2, '')
+    real_path = v if v != '' else path
+
+    env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path
+
+    return {'return': 0}

script/download-file/_cm.json

@@ -16,11 +16,13 @@
   "input_description": {},
   "input_mapping": {
     "download_path": "CM_DOWNLOAD_PATH",
+    "output_file": "CM_DOWNLOAD_FILENAME",
     "from": "CM_DOWNLOAD_LOCAL_FILE_PATH",
     "local_path": "CM_DOWNLOAD_LOCAL_FILE_PATH",
     "store": "CM_DOWNLOAD_PATH",
     "url": "CM_DOWNLOAD_URL",
     "verify": "CM_VERIFY_SSL",
+    "verify_ssl": "CM_VERIFY_SSL",
     "md5sum": "CM_DOWNLOAD_CHECKSUM"
   },
   "new_env_keys": [