many CM fixes for MLOps and MLPerf inference v4.0 (#1204)

cm-mlops/automation/docs/_cm.json

@@ -0,0 +1,9 @@
+{
+  "alias": "docs",
+  "automation_alias": "automation",
+  "automation_uid": "bbeb15d8f0a944a4",
+  "tags": [
+    "automation"
+  ],
+  "uid": "9558c9e6ca124065"
+}

cm-mlops/automation/docs/module.py

@@ -0,0 +1,52 @@
+import os
+
+from cmind.automation import Automation
+from cmind import utils
+
+class CAutomation(Automation):
+    """
+    Automation actions
+    """
+
+    ############################################################
+    def __init__(self, cmind, automation_file):
+        super().__init__(cmind, __file__)
+
+    ############################################################
+    def test(self, i):
+        """
+        Test automation
+
+        Args:
+          (CM input dict): 
+
+          (out) (str): if 'con', output to console
+
+          automation (str): automation as CM string object
+
+          parsed_automation (list): prepared in CM CLI or CM access function
+                                    [ (automation alias, automation UID) ] or
+                                    [ (automation alias, automation UID), (automation repo alias, automation repo UID) ]
+
+          (artifact) (str): artifact as CM string object
+
+          (parsed_artifact) (list): prepared in CM CLI or CM access function
+                                    [ (artifact alias, artifact UID) ] or
+                                    [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ]
+
+          ...
+
+        Returns:
+          (CM return dict):
+
+          * return (int): return code == 0 if no error and >0 if error
+          * (error) (str): error string if return>0
+
+          * Output from this automation action
+
+        """
+
+        import json
+        print (json.dumps(i, indent=2))
+
+        return {'return':0}

cm-mlops/automation/experiment/README-extra.md

@@ -37,7 +37,7 @@ The goal is to provide a common interface to run, record, share, visualize and r
 on any platform with any software, hardware and data.
 
 The community helped us test a prototype of our "experiment" automation to record results in a unified CM format 
-from [several MLPerf benchmarks](https://github.com/mlcommons/ck_mlperf_results) 
+from [several MLPerf benchmarks](https://github.com/mlcommons/cm4mlperf-results) 
 including [MLPerf inference](https://github.com/mlcommons/inference) and [MLPerf Tiny](https://github.com/mlcommons/tiny),
 visualize them at the [MLCommons CM platform](https://access.cknowledge.org/playground/?action=experiments&tags=all),
 and improve them by the community via [public benchmarking, optimization and reproducibility challenges](https://access.cknowledge.org/playground/?action=challenges).

cm-mlops/automation/script/README-specs.md

@@ -55,7 +55,7 @@ Sometimes it is difficult to add all variations needed for a script like say `ba
 * `--input` is automatically converted to `CM_INPUT` env key
 * `version` is converted to `CM_VERSION`, ``version_min` to `CM_VERSION_MIN` and `version_max` to `CM_VERSION_MAX`
 * If `env['CM_GH_TOKEN']=TOKEN_VALUE` is set then git URLs (specified by `CM_GIT_URL`) are changed to add this token.
-* If `env['CM_GIT_SSH']=yes`, then git URLs are chnged to SSH from HTTPS.
+* If `env['CM_GIT_SSH']=yes`, then git URLs are changed to SSH from HTTPS.
 
 ### Script Meta
 #### Special keys in script meta

cm-mlops/automation/script/module.py

@@ -43,7 +43,7 @@ def __init__(self, cmind, automation_file):
         self.tmp_file_run_env = 'tmp-run-env.out'
         self.tmp_file_ver = 'tmp-ver.out'
 
-        self.__version__ = "1.1.6"
+        self.__version__ = "1.2.1"
 
         self.local_env_keys = ['CM_VERSION',
                                'CM_VERSION_MIN',
@@ -155,7 +155,8 @@ def run(self, i):
           (verbose) (bool): if True, prints all tech. info about script execution (False by default)
           (v) (bool): the same as verbose
 
-          (time) (bool): if True, print script execution time (on if verbose == True)
+          (time) (bool): if True, print script execution time (or if verbose == True)
+          (space) (bool): if True, print used disk space for this script (or if verbose == True)
 
           (ignore_script_error) (bool): if True, ignore error code in native tools and scripts
                                         and finish a given CM script. Useful to test/debug partial installations
@@ -172,6 +173,7 @@ def run(self, i):
           (repro_prefix) (str): if !='', use it to record above files {repro-prefix)-input.json ...                
           (repro_dir) (str): if !='', use this directory to dump info
 
+          (script_call_prefix) (str): how to call script in logs and READMEs (cm run script)
           ...
 
         Returns:
@@ -201,6 +203,7 @@ def _run(self, i):
         from cmind import utils
         import copy
         import time
+        import shutil
 
         # Check if save input/output to file
         repro = i.get('repro', False)
@@ -222,8 +225,9 @@ def _run(self, i):
         recursion = i.get('recursion', False)
 
         # If first script run, check if can write to current directory
-        if not recursion and not can_write_to_current_directory():
-            return {'return':1, 'error':'Current directory "{}" is not writable - please change it'.format(os.getcwd())}
+        if not recursion and not i.get('skip_write_test', False):
+            if not can_write_to_current_directory():
+                return {'return':1, 'error':'Current directory "{}" is not writable - please change it'.format(os.getcwd())}
 
         recursion_int = int(i.get('recursion_int',0))+1
 
@@ -308,6 +312,10 @@ def _run(self, i):
            env['CM_VERBOSE']='yes'
 
         show_time = i.get('time', False)
+        show_space = i.get('space', False)
+
+        if not recursion and show_space:
+            start_disk_stats = shutil.disk_usage("/")
 
         extra_recursion_spaces = '  '# if verbose else ''
 
@@ -470,7 +478,9 @@ def _run(self, i):
 #            print (recursion_spaces + '* Running ' + cm_script_info)
 
 
-        cm_script_info = 'cm run script '
+        cm_script_info = i.get('script_call_prefix', '').strip()
+        if cm_script_info == '': cm_script_info = 'cm run script'
+        if not cm_script_info.endswith(' '): cm_script_info+=' '
 
         x = '"'
         y = ' '
@@ -682,6 +692,16 @@ def _run(self, i):
         meta = script_artifact.meta
         path = script_artifact.path
 
+        # Check path to repo
+        script_repo_path = script_artifact.repo_path
+
+        script_repo_path_with_prefix = script_artifact.repo_path
+        if script_artifact.repo_meta.get('prefix', '') != '':
+            script_repo_path_with_prefix = os.path.join(script_repo_path, script_artifact.repo_meta['prefix'])
+
+        env['CM_TMP_CURRENT_SCRIPT_REPO_PATH'] = script_repo_path
+        env['CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX'] = script_repo_path_with_prefix
+
         # Check if has --help
         if i.get('help',False):
             return utils.call_internal_module(self, __file__, 'module_help', 'print_help', {'meta':meta, 'path':path})
@@ -1716,6 +1736,16 @@ def _run(self, i):
         if verbose or show_time:
             print (recursion_spaces+'  - running time of script "{}": {:.2f} sec.'.format(','.join(found_script_tags), elapsed_time))
 
+
+        if not recursion and show_space:
+            stop_disk_stats = shutil.disk_usage("/")
+
+            used_disk_space_in_mb = int((start_disk_stats.free - stop_disk_stats.free) / (1024*1024))
+
+            if used_disk_space_in_mb > 0:
+                print (recursion_spaces+'  - used disk space: {} MB'.format(used_disk_space_in_mb))
+
+
         # Check if pause (useful if running a given script in a new terminal that may close automatically)
         if i.get('pause', False):
             print ('')
@@ -1725,13 +1755,17 @@ def _run(self, i):
         print_env_at_the_end = meta.get('print_env_at_the_end',{})
         if len(print_env_at_the_end)>0:
             print ('')
-            for p in print_env_at_the_end:
+
+            for p in sorted(print_env_at_the_end):
                 t = print_env_at_the_end[p]
+                if t == '': t = 'ENV[{}]'.format(p)
 
                 v = new_env.get(p, None)
 
                 print ('{}: {}'.format(t, str(v)))
 
+            print ('')
+
         return rr
 
     ######################################################################################
@@ -2372,7 +2406,8 @@ def add(self, i):
 
         parsed_artifact = i.get('parsed_artifact',[])
 
-        artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','')
+        artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else None
+        artifact_repo = parsed_artifact[1] if len(parsed_artifact)>1 else None
 
         script_name = ''
         if 'script_name' in i:
@@ -2473,6 +2508,13 @@ def add(self, i):
 
         ii['automation']='script,5b4e0237da074764'
 
+        for k in ['parsed_automation', 'parsed_artifact']:
+            if k in ii: del ii[k]
+
+        if artifact_repo != None:
+            artifact = ii.get('artifact','')
+            ii['artifact'] = utils.assemble_cm_object2(artifact_repo) + ':' + artifact
+
         r_obj=self.cmind.access(ii)
         if r_obj['return']>0: return r_obj