diff --git a/README.md b/README.md index 0c6183d42..3c45ff8be 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ -# Cross-platform CM interface for DevOps, MLOps, AIOps and MLPerf +## Unified and cross-platform CM interface for DevOps, MLOps and MLPerf [![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE.md) +[![Python Version](https://img.shields.io/badge/python-3+-blue.svg)](https://github.com/mlcommons/ck/tree/master/cm/cmind) [![Powered by CM](https://img.shields.io/badge/Powered_by-MLCommons%20CM-blue)](https://github.com/mlcommons/ck). +[![Downloads](https://static.pepy.tech/badge/cmind)](https://pepy.tech/project/cmind) This repository contains reusable and cross-platform automation recipes to run DevOps, MLOps, AIOps and MLPerf via a simple and human-readable [Collective Mind interface (CM)](https://github.com/mlcommons/ck) @@ -16,14 +18,23 @@ These automation recipes are being developed and maintained by the [MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) with [great contributions](CONTRIBUTING.md) from the community. -### Catalog +## Tests + +[![CM script automation test](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-scripts.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-scripts.yml) +[![CM script automation features test](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-script-features.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-cm-script-features.yml) +[![MLPerf loadgen with HuggingFace bert onnx fp32 squad model](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml) +[![MLPerf inference MLCommons C++ ResNet50](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml) +[![image classification with ONNX](https://github.com/mlcommons/cm4mlops/actions/workflows/test-image-classification-onnx.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-image-classification-onnx.yml) + + +## Catalog See the automatically generated catalog [online](https://access.cknowledge.org/playground/?action=scripts). -### License +## License [Apache 2.0](LICENSE.md) -### Copyright +## Copyright 2022-2024 [MLCommons](https://mlcommons.org) diff --git a/automation/script/_cm.json b/automation/script/_cm.json index 140662bfa..d1b827526 100644 --- a/automation/script/_cm.json +++ b/automation/script/_cm.json @@ -2,11 +2,12 @@ "alias": "script", "automation_alias": "automation", "automation_uid": "bbeb15d8f0a944a4", + "min_cm_version": "2.2.0", "deps": { "cache": "cache,541d6f712a6b464e" }, "desc": "Making native scripts more portable, interoperable and deterministic", - "developers": "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)", + "developers": "Arjun Suresh and Grigori Fursin", "actions_with_help":["run", "docker"], "sort": 1000, "tags": [ diff --git a/automation/script/module.py b/automation/script/module.py index 139ce5dc5..44286bbe4 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -14,6 +14,7 @@ from cmind.automation import Automation from cmind import utils +from cmind import __version__ as current_cm_version class CAutomation(Automation): """ @@ -150,6 +151,8 @@ def run(self, i): inside a script specified by these tags (debug_script) (bool): if True, debug current script (set debug_script_tags to the tags of a current script) + (debug_uid) (str): if True, set CM_TMP_DEBUG_UID to this number to enable + remote python debugging of scripts and wrapped apps/tools (detected_versions) (dict): All the used scripts and their detected_versions (verbose) (bool): if True, prints all tech. info about script execution (False by default) @@ -180,6 +183,9 @@ def run(self, i): (skip_sudo) (bool): if True, set env['CM_TMP_SKIP_SUDO']='yes' to let scripts deal with that + (silent) (bool): if True, attempt to suppress all info if supported + (sets CM_TMP_SILENT=yes) + (s) (bool): the same as 'silent' ... Returns: @@ -309,14 +315,6 @@ def _run(self, i): print_env = i.get('print_env', False) - verbose = False - - if 'verbose' in i: verbose=i['verbose'] - elif 'v' in i: verbose=i['v'] - - if verbose: - env['CM_VERBOSE']='yes' - show_time = i.get('time', False) show_space = i.get('space', False) @@ -331,6 +329,10 @@ def _run(self, i): fake_run = i.get('fake_run', False) fake_run = i.get('fake_run', False) if 'fake_run' in i else i.get('prepare', False) if fake_run: env['CM_TMP_FAKE_RUN']='yes' + + debug_uid = i.get('debug_uid', '') + if debug_uid!='': + env['CM_TMP_DEBUG_UID'] = debug_uid fake_deps = i.get('fake_deps', False) if fake_deps: env['CM_TMP_FAKE_DEPS']='yes' @@ -348,6 +350,28 @@ def _run(self, i): if fake_deps: run_state['fake_deps'] = True + # Check verbose and silent + verbose = False + + silent = True if str(i.get('silent', '')).lower() in ['true', 'yes', 'on'] else False + + if not silent: + silent = True if str(i.get('s', '')).lower() in ['true', 'yes', 'on'] else False + + if silent: + if 'verbose' in i: del(i['verbose']) + if 'v' in i: del(i['v']) + env['CM_TMP_SILENT']='yes' + run_state['tmp_silent']=True + + if 'verbose' in i: verbose=i['verbose'] + elif 'v' in i: verbose=i['v'] + + if verbose: + env['CM_VERBOSE']='yes' + run_state['tmp_verbose']=True + + print_deps = i.get('print_deps', False) print_readme = i.get('print_readme', False) @@ -517,8 +541,9 @@ def _run(self, i): # if verbose: # print ('') - print ('') - print (recursion_spaces + '* ' + cm_script_info) + if not run_state.get('tmp_silent', False): + print ('') + print (recursion_spaces + '* ' + cm_script_info) ############################################################################# @@ -703,6 +728,15 @@ def _run(self, i): meta = script_artifact.meta path = script_artifact.path + # Check min CM version requirement + min_cm_version = meta.get('min_cm_version','').strip() + if min_cm_version != '': + # Check compare version while avoiding craches for older version + if 'compare_versions' in dir(utils): + comparison = utils.compare_versions(current_cm_version, min_cm_version) + if comparison < 0: + return {'return':1, 'error':'CM script requires CM version >= {} while current CM version is {} - please update using "pip install cmind -U"'.format(min_cm_version, current_cm_version)} + # Check path to repo script_repo_path = script_artifact.repo_path @@ -1081,7 +1115,8 @@ def _run(self, i): if r['return']>0: return r version = r['meta'].get('version') - print (recursion_spaces + ' ! load {}'.format(path_to_cached_state_file)) + if not run_state.get('tmp_silent', False): + print (recursion_spaces + ' ! load {}'.format(path_to_cached_state_file)) ################################################################################################ @@ -1763,19 +1798,20 @@ def _run(self, i): input ('Press Enter to continue ...') # Check if need to print some final info such as path to model, etc - print_env_at_the_end = meta.get('print_env_at_the_end',{}) - if len(print_env_at_the_end)>0: - print ('') + if not run_state.get('tmp_silent', False): + print_env_at_the_end = meta.get('print_env_at_the_end',{}) + if len(print_env_at_the_end)>0: + print ('') - for p in sorted(print_env_at_the_end): - t = print_env_at_the_end[p] - if t == '': t = 'ENV[{}]'.format(p) + for p in sorted(print_env_at_the_end): + t = print_env_at_the_end[p] + if t == '': t = 'ENV[{}]'.format(p) - v = new_env.get(p, None) + v = new_env.get(p, None) - print ('{}: {}'.format(t, str(v))) + print ('{}: {}'.format(t, str(v))) - print ('') + print ('') return rr @@ -2887,6 +2923,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a # Run collective script via CM API: # Not very efficient but allows logging - can be optimized later + ii = { 'action':'run', 'automation':utils.assemble_cm_object(self.meta['alias'], self.meta['uid']), @@ -2900,6 +2937,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a 'add_deps_recursive':add_deps_recursive, 'debug_script_tags':debug_script_tags, 'verbose':verbose, + 'silent':run_state.get('tmp_silent', False), 'time':show_time, 'run_state':run_state @@ -4295,8 +4333,9 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): print (recursion_spaces + ' - Running native script "{}" from temporal script "{}" in "{}" ...'.format(path_to_run_script, run_script, cur_dir)) print ('') - print (recursion_spaces + ' ! cd {}'.format(cur_dir)) - print (recursion_spaces + ' ! call {} from {}'.format(path_to_run_script, run_script)) + if not run_state.get('tmp_silent', False): + print (recursion_spaces + ' ! cd {}'.format(cur_dir)) + print (recursion_spaces + ' ! call {} from {}'.format(path_to_run_script, run_script)) # Prepare env variables @@ -4388,7 +4427,8 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): if postprocess != '' and customize_code is not None and postprocess in dir(customize_code): - print (recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__)) + if not run_state.get('tmp_silent', False): + print (recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__)) if len(posthook_deps)>0 and (postprocess == "postprocess"): r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state, diff --git a/automation/script/template/run.sh b/automation/script/template/run.sh index 3a584c10c..4c23c380e 100644 --- a/automation/script/template/run.sh +++ b/automation/script/template/run.sh @@ -7,21 +7,11 @@ #${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency - - -function exit_if_error() { - test $? -eq 0 || exit $? -} - -function run() { - echo "Running: " - echo "$1" - echo "" - if [[ ${CM_FAKE_RUN} != 'yes' ]]; then - eval "$1" - exit_if_error - fi -} - -#Add your run commands here... -# run "$CM_RUN_CMD" +echo "Running: " +echo "${CM_RUN_CMD}" +echo "" + +if [[ ${CM_FAKE_RUN} != "yes" ]]; then + eval "${CM_RUN_CMD}" + test $? -eq 0 || exit 1 +fi diff --git a/cmr.yaml b/cmr.yaml index c22c97047..9765b5f42 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -3,6 +3,8 @@ uid: 9e97bb72b0474657 git: true +version: 2.2.0 + deps: - alias: mlcommons@ck uid: a4705959af8e447a diff --git a/script/app-image-classification-onnx-py/customize.py b/script/app-image-classification-onnx-py/customize.py index 43098c71f..0b2c7c0a4 100644 --- a/script/app-image-classification-onnx-py/customize.py +++ b/script/app-image-classification-onnx-py/customize.py @@ -7,8 +7,8 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - print ('') - print ('Running preprocess function in customize.py ...') +# print ('') +# print ('Running preprocess function in customize.py ...') return {'return':0} @@ -55,10 +55,11 @@ def postprocess(i): top_classification = data.get('top_classification','') - if top_classification!='': - print ('') - x = 'Top classification: {}'.format(top_classification) - print ('='*len(x)) - print (x) + if env.get('CM_TMP_SILENT','')!='yes': + if top_classification!='': + print ('') + x = 'Top classification: {}'.format(top_classification) + print ('='*len(x)) + print (x) return {'return':0} diff --git a/script/create-custom-cache-entry/_cm.yaml b/script/create-custom-cache-entry/_cm.yaml new file mode 100644 index 000000000..7272bb99a --- /dev/null +++ b/script/create-custom-cache-entry/_cm.yaml @@ -0,0 +1,27 @@ +alias: create-custom-cache-entry +uid: 485741440fbe4236 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +tags: +- create +- custom +- cache +- entry + +category: CM automation + +cache: true + +input_mapping: + env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY + env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2 + path: CM_CUSTOM_CACHE_ENTRY_PATH + to: CM_CUSTOM_CACHE_ENTRY_PATH + +new_env_keys: +- CM_CUSTOM_CACHE_ENTRY* + +print_env_at_the_end: + CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry" diff --git a/script/create-custom-cache-entry/customize.py b/script/create-custom-cache-entry/customize.py new file mode 100644 index 000000000..8d2d31db3 --- /dev/null +++ b/script/create-custom-cache-entry/customize.py @@ -0,0 +1,44 @@ +from cmind import utils +import os +import shutil + +def preprocess(i): + + # CM script internal variables + env = i['env'] + + extra_cache_tags = [] + if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '': + print ('') + extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ') + + extra_cache_tags = extra_cache_tags_str.strip().split(',') + + return {'return':0, 'add_extra_cache_tags':extra_cache_tags} + +def postprocess(i): + + env = i['env'] + + path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip() + + if path!='': + if not os.path.isdir(path): + os.makedirs(path) + else: + path = os.getcwd() + + x = '' + env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '') + if env_key != '': x = env_key+'_' + + env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path + env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path + + env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '') + v = env.get(env_key2, '') + real_path = v if v != '' else path + + env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path + + return {'return': 0} diff --git a/script/download-file/_cm.json b/script/download-file/_cm.json index 9b960e7f6..6d89ed2d4 100644 --- a/script/download-file/_cm.json +++ b/script/download-file/_cm.json @@ -16,11 +16,13 @@ "input_description": {}, "input_mapping": { "download_path": "CM_DOWNLOAD_PATH", + "output_file": "CM_DOWNLOAD_FILENAME", "from": "CM_DOWNLOAD_LOCAL_FILE_PATH", "local_path": "CM_DOWNLOAD_LOCAL_FILE_PATH", "store": "CM_DOWNLOAD_PATH", "url": "CM_DOWNLOAD_URL", "verify": "CM_VERIFY_SSL", + "verify_ssl": "CM_VERIFY_SSL", "md5sum": "CM_DOWNLOAD_CHECKSUM" }, "new_env_keys": [ diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index b15e2d9e0..0beed5d94 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -114,7 +114,8 @@ def detect_version(i): version = r['version'] current_detected_version = version - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) + if env.get('CM_TMP_SILENT','')!='yes': + print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) return {'return':0, 'version':version} diff --git a/script/get-python3/_cm.json b/script/get-python3/_cm.json index 470fff100..a65cbd71b 100644 --- a/script/get-python3/_cm.json +++ b/script/get-python3/_cm.json @@ -71,5 +71,9 @@ "shared": {}, "with-ssl": {}, "with-custom-ssl": {} + }, + "print_env_at_the_end" : { + "CM_PYTHON_BIN_WITH_PATH": "Path to Python", + "CM_PYTHON_VERSION": "Python version" } } diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 549a30430..555fe7c6a 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -31,3 +31,7 @@ variations: user: env: CM_PYTHON_PIP_USER: --user + + skip_python_deps: + env: + CM_SKIP_PYTHON_DEPS: "yes" diff --git a/script/get-sys-utils-cm/run-arch.sh b/script/get-sys-utils-cm/run-arch.sh index 1a5338c58..eb71848ad 100644 --- a/script/get-sys-utils-cm/run-arch.sh +++ b/script/get-sys-utils-cm/run-arch.sh @@ -1,7 +1,7 @@ #!/bin/bash -echo "************************************************" -echo "Installing some system dependencies via package manager" +echo "***************************************************" +echo "Installing some system dependencies via sudo pacman" if [[ "$CM_QUIET" != "yes" ]]; then @@ -31,5 +31,9 @@ ${CM_SUDO} ${CM_PACKAGE_TOOL} -Syu && \ xz \ zip -. ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh -test $? -eq 0 || exit $? +# Install Python deps though preference is to install them +# via cmr "get generic-python-lib _package.{Python PIP package name}" +if [[ "${CM_SKIP_PYTHON_DEPS}" != "yes" ]]; then + . ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh + test $? -eq 0 || exit $? +fi diff --git a/script/get-sys-utils-cm/run-debian.sh b/script/get-sys-utils-cm/run-debian.sh index c7de24464..488da76fb 100644 --- a/script/get-sys-utils-cm/run-debian.sh +++ b/script/get-sys-utils-cm/run-debian.sh @@ -52,5 +52,9 @@ ${CM_SUDO} ${CM_APT_TOOL} update && \ libgl1 \ libncurses5 -. ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh -test $? -eq 0 || exit $? +# Install Python deps though preference is to install them +# via cmr "get generic-python-lib _package.{Python PIP package name}" +if [[ "${CM_SKIP_PYTHON_DEPS}" != "yes" ]]; then + . ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh + test $? -eq 0 || exit $? +fi diff --git a/script/get-sys-utils-cm/run-macos.sh b/script/get-sys-utils-cm/run-macos.sh index 1e5eab4ad..afb2ef8e6 100644 --- a/script/get-sys-utils-cm/run-macos.sh +++ b/script/get-sys-utils-cm/run-macos.sh @@ -35,5 +35,9 @@ brew update && \ zlib \ python3 -. ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh -test $? -eq 0 || exit $? +# Install Python deps though preference is to install them +# via cmr "get generic-python-lib _package.{Python PIP package name}" +if [[ "${CM_SKIP_PYTHON_DEPS}" != "yes" ]]; then + . ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh + test $? -eq 0 || exit $? +fi diff --git a/script/get-sys-utils-cm/run-rhel.sh b/script/get-sys-utils-cm/run-rhel.sh index f247f807e..87b03b777 100644 --- a/script/get-sys-utils-cm/run-rhel.sh +++ b/script/get-sys-utils-cm/run-rhel.sh @@ -1,7 +1,7 @@ #!/bin/bash echo "************************************************" -echo "Installing some system dependencies via package manager" +echo "Installing some system dependencies via sudo dnf" if [[ "$CM_QUIET" != "yes" ]]; then @@ -38,5 +38,9 @@ ${CM_SUDO} ${CM_PACKAGE_TOOL} update && \ xz \ zip -. ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh -test $? -eq 0 || exit $? +# Install Python deps though preference is to install them +# via cmr "get generic-python-lib _package.{Python PIP package name}" +if [[ "${CM_SKIP_PYTHON_DEPS}" != "yes" ]]; then + . ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh + test $? -eq 0 || exit $? +fi diff --git a/script/get-sys-utils-cm/run-sles.sh b/script/get-sys-utils-cm/run-sles.sh index 32cfdbabc..845c0b069 100644 --- a/script/get-sys-utils-cm/run-sles.sh +++ b/script/get-sys-utils-cm/run-sles.sh @@ -1,7 +1,7 @@ #!/bin/bash -echo "************************************************" -echo "Installing some system dependencies via package manager" +echo "***************************************************" +echo "Installing some system dependencies via sudo zypper" if [[ "$CM_QUIET" != "yes" ]]; then @@ -34,5 +34,9 @@ ${CM_SUDO} ${CM_PACKAGE_TOOL} update && \ xz \ zip -. ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh -test $? -eq 0 || exit $? +# Install Python deps though preference is to install them +# via cmr "get generic-python-lib _package.{Python PIP package name}" +if [[ "${CM_SKIP_PYTHON_DEPS}" != "yes" ]]; then + . ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh + test $? -eq 0 || exit $? +fi diff --git a/script/get-sys-utils-cm/run-ubuntu.sh b/script/get-sys-utils-cm/run-ubuntu.sh index 1c4561e89..6fa7f5814 100644 --- a/script/get-sys-utils-cm/run-ubuntu.sh +++ b/script/get-sys-utils-cm/run-ubuntu.sh @@ -56,5 +56,9 @@ ${CM_SUDO} ${CM_APT_TOOL} update && \ libgl1-mesa-glx \ zlib1g-dev -. ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh -test $? -eq 0 || exit $? +# Install Python deps though preference is to install them +# via cmr "get generic-python-lib _package.{Python PIP package name}" +if [[ "${CM_SKIP_PYTHON_DEPS}" != "yes" ]]; then + . ${CM_TMP_CURRENT_SCRIPT_PATH}/do_pip_installs.sh + test $? -eq 0 || exit $? +fi diff --git a/script/print-any-text/_cm.yaml b/script/print-any-text/_cm.yaml new file mode 100644 index 000000000..2fd9bba2c --- /dev/null +++ b/script/print-any-text/_cm.yaml @@ -0,0 +1,34 @@ +alias: print-any-text +uid: f4bf2d1d33c24e31 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +default_env: + CM_PRINT_ANY_TEXT: '' + +input_mapping: + text: CM_PRINT_ANY_TEXT + cm_env_keys: CM_PRINT_ANY_CM_ENV_KEYS + os_env_keys: CM_PRINT_ANY_OS_ENV_KEYS + +tags: +- print +- any-text + +variations: + text.#: + env: + CM_PRINT_ANY_TEXT: "#" + + cm_env.#: + env: + CM_PRINT_ANY_CM_ENV_KEYS: "#" + + os_env.#: + env: + CM_PRINT_ANY_OS_ENV_KEYS: "#" diff --git a/script/print-any-text/customize.py b/script/print-any-text/customize.py new file mode 100644 index 000000000..093cafdcf --- /dev/null +++ b/script/print-any-text/customize.py @@ -0,0 +1,30 @@ +# Developer(s): Grigori Fursin + +from cmind import utils +import os + +def postprocess(i): + + env = i['env'] + + cm_env_keys = env.get('CM_PRINT_ANY_CM_ENV_KEYS', '').strip() + os_env_keys = env.get('CM_PRINT_ANY_OS_ENV_KEYS', '').strip() + + printed = False + for k,e,t in [(cm_env_keys, env, 'CM_ENV'), + (os_env_keys, os.environ, 'OS_ENV')]: + + if k!='': + for kk in k.split(','): + kk = kk.strip() + if kk!='': + vv = e.get(kk) + + print ('{}[{}]: {}'.format(t, kk, vv)) + printed = True + + if printed: + print ('') + + return {'return':0} + diff --git a/script/print-any-text/run.bat b/script/print-any-text/run.bat new file mode 100644 index 000000000..be97ff0a2 --- /dev/null +++ b/script/print-any-text/run.bat @@ -0,0 +1,5 @@ +if "%CM_PRINT_ANY_TEXT%" == "" ( + echo. +) else ( + echo %CM_PRINT_ANY_TEXT% +) diff --git a/script/print-any-text/run.sh b/script/print-any-text/run.sh new file mode 100644 index 000000000..7f04767d6 --- /dev/null +++ b/script/print-any-text/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "${CM_PRINT_ANY_TEXT}" diff --git a/script/test-cm-core/_cm.yaml b/script/test-cm-core/_cm.yaml index ca8ee7d17..420d119f8 100644 --- a/script/test-cm-core/_cm.yaml +++ b/script/test-cm-core/_cm.yaml @@ -4,6 +4,8 @@ uid: 2c2fb9d20dc64cf3 automation_alias: script automation_uid: 5b4e0237da074764 +category: Tests + tags: - test - cm diff --git a/script/test-cm-script-pipeline/README-extra.md b/script/test-cm-script-pipeline/README-extra.md new file mode 100644 index 000000000..480bea4f4 --- /dev/null +++ b/script/test-cm-script-pipeline/README-extra.md @@ -0,0 +1,8 @@ +# CM script + +This script prints internal pipeline run: + +```bash +cmr "test cm-script pipeline" +``` + diff --git a/script/test-cm-script-pipeline/_cm.yaml b/script/test-cm-script-pipeline/_cm.yaml new file mode 100644 index 000000000..9a2327ae6 --- /dev/null +++ b/script/test-cm-script-pipeline/_cm.yaml @@ -0,0 +1,37 @@ +alias: test-cm-script-pipeline +uid: ebe50aa281be4458 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +tags: +- test +- cm-script +- pipeline + +cache: false + +deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: deps" + +prehook_deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: prehook_deps" + +posthook_deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: posthook_deps" + +post_deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: post_deps" + CM_PRINT_ANY_CM_ENV_KEYS: 'CM_TMP_CURRENT_SCRIPT_PATH,CM_TMP_CURRENT_PATH,CM_QUIET' diff --git a/script/test-cm-script-pipeline/customize.py b/script/test-cm-script-pipeline/customize.py new file mode 100644 index 000000000..89311a22f --- /dev/null +++ b/script/test-cm-script-pipeline/customize.py @@ -0,0 +1,38 @@ +# Developers: Grigori Fursin + +from cmind import utils +import os + +def preprocess(i): + + print ('') + print ('customize.py: preprocess') + print ('') + + return {'return':0} + +def postprocess(i): + + automation = i['automation'] + run_script_input = i['run_script_input'] + env = i['env'] + + print ('') + print ('customize.py: postprocess') + print ('') + + r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'run2'}) + if r['return']>0: + return r + + return {'return':0} + +def detect_version(i): + + + print ('') + print ('customize.py: detect_version') + print ('') + + + return {'return':0} diff --git a/script/test-cm-script-pipeline/run.bat b/script/test-cm-script-pipeline/run.bat new file mode 100644 index 000000000..3103960b7 --- /dev/null +++ b/script/test-cm-script-pipeline/run.bat @@ -0,0 +1,5 @@ +rem native script + +echo. +echo run.bat +echo. diff --git a/script/test-cm-script-pipeline/run.sh b/script/test-cm-script-pipeline/run.sh new file mode 100644 index 000000000..51936abac --- /dev/null +++ b/script/test-cm-script-pipeline/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "" +echo "run.sh" +echo "" diff --git a/script/test-cm-script-pipeline/run2.bat b/script/test-cm-script-pipeline/run2.bat new file mode 100644 index 000000000..120bd24a8 --- /dev/null +++ b/script/test-cm-script-pipeline/run2.bat @@ -0,0 +1,5 @@ +rem native script + +echo. +echo run2.bat +echo. diff --git a/script/test-cm-script-pipeline/run2.sh b/script/test-cm-script-pipeline/run2.sh new file mode 100644 index 000000000..21664817b --- /dev/null +++ b/script/test-cm-script-pipeline/run2.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "" +echo "run2.sh" +echo "" diff --git a/script/test-debug/.vscode/launch.json b/script/test-debug/.vscode/launch.json new file mode 100644 index 000000000..49c4e19f0 --- /dev/null +++ b/script/test-debug/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "${workspaceFolder}" + } + ] + } +} \ No newline at end of file diff --git a/script/test-debug/README-extra.md b/script/test-debug/README-extra.md new file mode 100644 index 000000000..06a41746c --- /dev/null +++ b/script/test-debug/README-extra.md @@ -0,0 +1,20 @@ +Demo of debugging CM scripts and wrapped apps with Visual Studio Code. + + +Debug customize.py using remote debugging and follow instructions on command line: + +```bash +cmr "test cm-debug" --debug_uid=8d96cd9fa4734204 +``` + +Debug Python application or tool wrapped by the CM script (see [python/main.py](python/main.py)): + +```bash +cmr "test cm-debug" --debug_uid=45a7c3a500d24a63 +``` + + + +Debug CM internals using standard Python debugging: +* Open _demo.py and start debugging using "Python File" default configuration. + diff --git a/script/test-debug/_cm.yaml b/script/test-debug/_cm.yaml new file mode 100644 index 000000000..8a4ab37aa --- /dev/null +++ b/script/test-debug/_cm.yaml @@ -0,0 +1,29 @@ +alias: test-debug +uid: 5ccd3d701a9144f9 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: CM interface prototyping + +deps: + # Detect host OS features + - tags: detect,os + + # Detect host CPU features + - tags: detect,cpu + + # Get Python + - tags: get,python3 + names: + - python + - python3 + + # May need to use CM in automation recipes + - tags: get,generic-python-lib,_package.cmind + names: + - python-cmind + +tags: +- test +- cm-debug diff --git a/script/test-debug/_demo.py b/script/test-debug/_demo.py new file mode 100644 index 000000000..781bed321 --- /dev/null +++ b/script/test-debug/_demo.py @@ -0,0 +1,9 @@ +# Developer: Grigori Fursin + +import cmind +import sys + +print(sys.executable) + +r = cmind.access('run "cm-debug"') +print(r) diff --git a/script/test-debug/customize.py b/script/test-debug/customize.py new file mode 100644 index 000000000..a3af87f64 --- /dev/null +++ b/script/test-debug/customize.py @@ -0,0 +1,35 @@ +# Developer(s): Grigori Fursin + +import os + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + + print ("********************************************************") + print ('- Importing CM library ...') + import cmind + print (' SUCCESS!') + + cmind.utils.debug_here(__file__, port=5678, text='Debugging customize.py!', env=env, env_debug_uid='8d96cd9fa4734204').breakpoint() + + print ('') + print ('- List CM repos ...') + print ('') + r = cmind.access({'action':'show', 'automation':'repo', 'out':'con'}) + print ('') + print (' SUCCESS!') + print ("********************************************************") + + + return {'return':0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + return {'return':0} diff --git a/script/test-debug/python/.vscode/launch.json b/script/test-debug/python/.vscode/launch.json new file mode 100644 index 000000000..c2dc39156 --- /dev/null +++ b/script/test-debug/python/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "${workspaceFolder}" + } + ] + } + ] +} \ No newline at end of file diff --git a/script/test-debug/python/main.py b/script/test-debug/python/main.py new file mode 100644 index 000000000..0dfdf30f5 --- /dev/null +++ b/script/test-debug/python/main.py @@ -0,0 +1,25 @@ +""" +Testing CM debugging + +# Developer(s): Grigori Fursin +""" + +import os +import json + +print ("Hello World 1") + +env = os.environ + +import json +print ('') +print (json.dumps(dict(env), indent=2)) + +# Import cmind to test break points +import cmind.utils +if os.environ.get('CM_TMP_DEBUG_UID', '') == '45a7c3a500d24a63': + cmind.utils.debug_here(__file__, port=5678, text='Debugging main.py!').breakpoint() + +print ('') +print ("Hello World 2") + diff --git a/script/test-debug/run.bat b/script/test-debug/run.bat new file mode 100644 index 000000000..d45522c1d --- /dev/null +++ b/script/test-debug/run.bat @@ -0,0 +1,6 @@ +echo ======================================================== + +%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\python\main.py +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% + +echo ======================================================== diff --git a/script/test-debug/run.sh b/script/test-debug/run.sh new file mode 100644 index 000000000..ee43341da --- /dev/null +++ b/script/test-debug/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo "========================================================" + +${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/python/main.py +test $? -eq 0 || exit 1 + +echo ======================================================== diff --git a/script/test-deps-conditions/_cm.yaml b/script/test-deps-conditions/_cm.yaml index 1a342d4bf..bc74a5871 100644 --- a/script/test-deps-conditions/_cm.yaml +++ b/script/test-deps-conditions/_cm.yaml @@ -6,6 +6,8 @@ automation_uid: 5b4e0237da074764 category: Tests +developers: "Grigori Fursin" + deps: - tags: print,native,hello-world,_skip_print_env - tags: print,native,hello-world,_skip_print_env,_text.SKIP_IF_ALL_ENV diff --git a/script/test-deps-conditions2/README-extra.md b/script/test-deps-conditions2/README-extra.md new file mode 100644 index 000000000..d2b1f3033 --- /dev/null +++ b/script/test-deps-conditions2/README-extra.md @@ -0,0 +1,20 @@ +Checking some conditions to turn on or off deps: + +```bash +cmr "test deps conditions2" -s +cmr "test deps conditions2" -s --test +cmr "test deps conditions2" -s --test=xyz +``` + +Note that the last two will run with the following deps, +i.e. `True` tests not only for turning flag on, +but also for any non-empty value : +```yaml + - tags: print,any-text,_text.RUN_IF_ENV_IS_SET_TO_TRUE + enable_if_env: + TEST: + - True + +``` + +It is useful to check if flag turns on output or output to specific file for example ... diff --git a/script/test-deps-conditions2/_cm.yaml b/script/test-deps-conditions2/_cm.yaml new file mode 100644 index 000000000..08eb5ffb8 --- /dev/null +++ b/script/test-deps-conditions2/_cm.yaml @@ -0,0 +1,28 @@ +alias: test-deps-conditions2 +uid: 7a81ef941b3c4c6c + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +deps: + - tags: print,any-text,_cm_env.TEST + - tags: print,any-text,_text.RUN_IF_ENV_IS_SET_TO_TRUE + enable_if_env: + TEST: + - True + - tags: print,any-text,_text.RUN_IF_ENV_IS_NOT_SET_TO_TRUE + skip_if_env: + TEST: + - True + +input_mapping: + test: TEST + +tags: +- test +- deps +- conditions2