Skip to content

Commit

Permalink
Merge pull request #19 from mlcommons/dev
Browse files Browse the repository at this point in the history
Major update to support CM v2.2.0
  • Loading branch information
ctuning-admin authored Apr 23, 2024
2 parents 8fa290e + b838091 commit a32c233
Show file tree
Hide file tree
Showing 42 changed files with 626 additions and 71 deletions.
19 changes: 15 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
# Cross-platform CM interface for DevOps, MLOps, AIOps and MLPerf
## Unified and cross-platform CM interface for DevOps, MLOps and MLPerf

[![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE.md)
[![Python Version](https://img.shields.io/badge/python-3+-blue.svg)](https://github.com/mlcommons/ck/tree/master/cm/cmind)
[![Powered by CM](https://img.shields.io/badge/Powered_by-MLCommons%20CM-blue)](https://github.com/mlcommons/ck).
[![Downloads](https://static.pepy.tech/badge/cmind)](https://pepy.tech/project/cmind)

This repository contains reusable and cross-platform automation recipes to run DevOps, MLOps, AIOps and MLPerf
via a simple and human-readable [Collective Mind interface (CM)](https://github.com/mlcommons/ck)
Expand All @@ -16,14 +18,23 @@ These automation recipes are being developed and maintained
by the [MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)
with [great contributions](CONTRIBUTING.md) from the community.

### Catalog
## Tests

[![CM script automation test](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-scripts.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-scripts.yml)
[![CM script automation features test](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-script-features.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-script-features.yml)
[![MLPerf loadgen with HuggingFace bert onnx fp32 squad model](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml)
[![MLPerf inference MLCommons C++ ResNet50](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml)
[![image classification with ONNX](https://github.com/mlcommons/cm4mlops/actions/workflows/test-image-classification-onnx.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-image-classification-onnx.yml)


## Catalog

See the automatically generated catalog [online](https://access.cknowledge.org/playground/?action=scripts).

### License
## License

[Apache 2.0](LICENSE.md)

### Copyright
## Copyright

2022-2024 [MLCommons](https://mlcommons.org)
3 changes: 2 additions & 1 deletion automation/script/_cm.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@
"alias": "script",
"automation_alias": "automation",
"automation_uid": "bbeb15d8f0a944a4",
"min_cm_version": "2.2.0",
"deps": {
"cache": "cache,541d6f712a6b464e"
},
"desc": "Making native scripts more portable, interoperable and deterministic",
"developers": "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)",
"developers": "Arjun Suresh and Grigori Fursin",
"actions_with_help":["run", "docker"],
"sort": 1000,
"tags": [
Expand Down
86 changes: 63 additions & 23 deletions automation/script/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

from cmind.automation import Automation
from cmind import utils
from cmind import __version__ as current_cm_version

class CAutomation(Automation):
"""
Expand Down Expand Up @@ -150,6 +151,8 @@ def run(self, i):
inside a script specified by these tags
(debug_script) (bool): if True, debug current script (set debug_script_tags to the tags of a current script)
(debug_uid) (str): if True, set CM_TMP_DEBUG_UID to this number to enable
remote python debugging of scripts and wrapped apps/tools
(detected_versions) (dict): All the used scripts and their detected_versions
(verbose) (bool): if True, prints all tech. info about script execution (False by default)
Expand Down Expand Up @@ -180,6 +183,9 @@ def run(self, i):
(skip_sudo) (bool): if True, set env['CM_TMP_SKIP_SUDO']='yes'
to let scripts deal with that
(silent) (bool): if True, attempt to suppress all info if supported
(sets CM_TMP_SILENT=yes)
(s) (bool): the same as 'silent'
...
Returns:
Expand Down Expand Up @@ -309,14 +315,6 @@ def _run(self, i):

print_env = i.get('print_env', False)

verbose = False

if 'verbose' in i: verbose=i['verbose']
elif 'v' in i: verbose=i['v']

if verbose:
env['CM_VERBOSE']='yes'

show_time = i.get('time', False)
show_space = i.get('space', False)

Expand All @@ -331,6 +329,10 @@ def _run(self, i):
fake_run = i.get('fake_run', False)
fake_run = i.get('fake_run', False) if 'fake_run' in i else i.get('prepare', False)
if fake_run: env['CM_TMP_FAKE_RUN']='yes'

debug_uid = i.get('debug_uid', '')
if debug_uid!='':
env['CM_TMP_DEBUG_UID'] = debug_uid

fake_deps = i.get('fake_deps', False)
if fake_deps: env['CM_TMP_FAKE_DEPS']='yes'
Expand All @@ -348,6 +350,28 @@ def _run(self, i):
if fake_deps:
run_state['fake_deps'] = True

# Check verbose and silent
verbose = False

silent = True if str(i.get('silent', '')).lower() in ['true', 'yes', 'on'] else False

if not silent:
silent = True if str(i.get('s', '')).lower() in ['true', 'yes', 'on'] else False

if silent:
if 'verbose' in i: del(i['verbose'])
if 'v' in i: del(i['v'])
env['CM_TMP_SILENT']='yes'
run_state['tmp_silent']=True

if 'verbose' in i: verbose=i['verbose']
elif 'v' in i: verbose=i['v']

if verbose:
env['CM_VERBOSE']='yes'
run_state['tmp_verbose']=True


print_deps = i.get('print_deps', False)
print_readme = i.get('print_readme', False)

Expand Down Expand Up @@ -517,8 +541,9 @@ def _run(self, i):
# if verbose:
# print ('')

print ('')
print (recursion_spaces + '* ' + cm_script_info)
if not run_state.get('tmp_silent', False):
print ('')
print (recursion_spaces + '* ' + cm_script_info)


#############################################################################
Expand Down Expand Up @@ -703,6 +728,15 @@ def _run(self, i):
meta = script_artifact.meta
path = script_artifact.path

# Check min CM version requirement
min_cm_version = meta.get('min_cm_version','').strip()
if min_cm_version != '':
# Check compare version while avoiding craches for older version
if 'compare_versions' in dir(utils):
comparison = utils.compare_versions(current_cm_version, min_cm_version)
if comparison < 0:
return {'return':1, 'error':'CM script requires CM version >= {} while current CM version is {} - please update using "pip install cmind -U"'.format(min_cm_version, current_cm_version)}

# Check path to repo
script_repo_path = script_artifact.repo_path

Expand Down Expand Up @@ -1081,7 +1115,8 @@ def _run(self, i):
if r['return']>0: return r
version = r['meta'].get('version')

print (recursion_spaces + ' ! load {}'.format(path_to_cached_state_file))
if not run_state.get('tmp_silent', False):
print (recursion_spaces + ' ! load {}'.format(path_to_cached_state_file))


################################################################################################
Expand Down Expand Up @@ -1763,19 +1798,20 @@ def _run(self, i):
input ('Press Enter to continue ...')

# Check if need to print some final info such as path to model, etc
print_env_at_the_end = meta.get('print_env_at_the_end',{})
if len(print_env_at_the_end)>0:
print ('')
if not run_state.get('tmp_silent', False):
print_env_at_the_end = meta.get('print_env_at_the_end',{})
if len(print_env_at_the_end)>0:
print ('')

for p in sorted(print_env_at_the_end):
t = print_env_at_the_end[p]
if t == '': t = 'ENV[{}]'.format(p)
for p in sorted(print_env_at_the_end):
t = print_env_at_the_end[p]
if t == '': t = 'ENV[{}]'.format(p)

v = new_env.get(p, None)
v = new_env.get(p, None)

print ('{}: {}'.format(t, str(v)))
print ('{}: {}'.format(t, str(v)))

print ('')
print ('')

return rr

Expand Down Expand Up @@ -2887,6 +2923,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a

# Run collective script via CM API:
# Not very efficient but allows logging - can be optimized later

ii = {
'action':'run',
'automation':utils.assemble_cm_object(self.meta['alias'], self.meta['uid']),
Expand All @@ -2900,6 +2937,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
'add_deps_recursive':add_deps_recursive,
'debug_script_tags':debug_script_tags,
'verbose':verbose,
'silent':run_state.get('tmp_silent', False),
'time':show_time,
'run_state':run_state

Expand Down Expand Up @@ -4295,8 +4333,9 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
print (recursion_spaces + ' - Running native script "{}" from temporal script "{}" in "{}" ...'.format(path_to_run_script, run_script, cur_dir))
print ('')

print (recursion_spaces + ' ! cd {}'.format(cur_dir))
print (recursion_spaces + ' ! call {} from {}'.format(path_to_run_script, run_script))
if not run_state.get('tmp_silent', False):
print (recursion_spaces + ' ! cd {}'.format(cur_dir))
print (recursion_spaces + ' ! call {} from {}'.format(path_to_run_script, run_script))


# Prepare env variables
Expand Down Expand Up @@ -4388,7 +4427,8 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):


if postprocess != '' and customize_code is not None and postprocess in dir(customize_code):
print (recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__))
if not run_state.get('tmp_silent', False):
print (recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__))

if len(posthook_deps)>0 and (postprocess == "postprocess"):
r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state,
Expand Down
26 changes: 8 additions & 18 deletions automation/script/template/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,11 @@

#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency



function exit_if_error() {
test $? -eq 0 || exit $?
}

function run() {
echo "Running: "
echo "$1"
echo ""
if [[ ${CM_FAKE_RUN} != 'yes' ]]; then
eval "$1"
exit_if_error
fi
}

#Add your run commands here...
# run "$CM_RUN_CMD"
echo "Running: "
echo "${CM_RUN_CMD}"
echo ""

if [[ ${CM_FAKE_RUN} != "yes" ]]; then
eval "${CM_RUN_CMD}"
test $? -eq 0 || exit 1
fi
2 changes: 2 additions & 0 deletions cmr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ uid: 9e97bb72b0474657

git: true

version: 2.2.0

deps:
- alias: mlcommons@ck
uid: a4705959af8e447a
Expand Down
15 changes: 8 additions & 7 deletions script/app-image-classification-onnx-py/customize.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ def preprocess(i):
os_info = i['os_info']
env = i['env']

print ('')
print ('Running preprocess function in customize.py ...')
# print ('')
# print ('Running preprocess function in customize.py ...')

return {'return':0}

Expand Down Expand Up @@ -55,10 +55,11 @@ def postprocess(i):

top_classification = data.get('top_classification','')

if top_classification!='':
print ('')
x = 'Top classification: {}'.format(top_classification)
print ('='*len(x))
print (x)
if env.get('CM_TMP_SILENT','')!='yes':
if top_classification!='':
print ('')
x = 'Top classification: {}'.format(top_classification)
print ('='*len(x))
print (x)

return {'return':0}
27 changes: 27 additions & 0 deletions script/create-custom-cache-entry/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
alias: create-custom-cache-entry
uid: 485741440fbe4236

automation_alias: script
automation_uid: 5b4e0237da074764

tags:
- create
- custom
- cache
- entry

category: CM automation

cache: true

input_mapping:
env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY
env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2
path: CM_CUSTOM_CACHE_ENTRY_PATH
to: CM_CUSTOM_CACHE_ENTRY_PATH

new_env_keys:
- CM_CUSTOM_CACHE_ENTRY*

print_env_at_the_end:
CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry"
44 changes: 44 additions & 0 deletions script/create-custom-cache-entry/customize.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
from cmind import utils
import os
import shutil

def preprocess(i):

# CM script internal variables
env = i['env']

extra_cache_tags = []
if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '':
print ('')
extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ')

extra_cache_tags = extra_cache_tags_str.strip().split(',')

return {'return':0, 'add_extra_cache_tags':extra_cache_tags}

def postprocess(i):

env = i['env']

path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip()

if path!='':
if not os.path.isdir(path):
os.makedirs(path)
else:
path = os.getcwd()

x = ''
env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '')
if env_key != '': x = env_key+'_'

env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path
env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path

env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '')
v = env.get(env_key2, '')
real_path = v if v != '' else path

env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path

return {'return': 0}
2 changes: 2 additions & 0 deletions script/download-file/_cm.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@
"input_description": {},
"input_mapping": {
"download_path": "CM_DOWNLOAD_PATH",
"output_file": "CM_DOWNLOAD_FILENAME",
"from": "CM_DOWNLOAD_LOCAL_FILE_PATH",
"local_path": "CM_DOWNLOAD_LOCAL_FILE_PATH",
"store": "CM_DOWNLOAD_PATH",
"url": "CM_DOWNLOAD_URL",
"verify": "CM_VERIFY_SSL",
"verify_ssl": "CM_VERIFY_SSL",
"md5sum": "CM_DOWNLOAD_CHECKSUM"
},
"new_env_keys": [
Expand Down
Loading

0 comments on commit a32c233

Please sign in to comment.