From f2fa8f5e08236b7794d3da412c4f2c6005108454 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 30 Apr 2024 21:47:01 +0100 Subject: [PATCH] Sync with dev (new) --- LICENSE.third-party.md | 1 + automation/cfg/README-extra.md | 8 + .../ubuntu-23.04.Dockerfile | 34 ++++ .../ubuntu-23.04.Dockerfile.build.bat | 1 + .../ubuntu-23.04.Dockerfile.build.sh | 3 + .../ubuntu-23.04.Dockerfile.run.bat | 1 + .../ubuntu-23.04.Dockerfile.run.sh | 3 + cfg/docker-basic-configurations/_cm.yaml | 10 ++ .../basic-archlinux.yaml | 9 ++ .../basic-rhel-9.yaml | 9 ++ .../basic-ubuntu-20.04.yaml | 9 ++ .../basic-ubuntu-22.04.yaml | 9 ++ .../basic-ubuntu-23.04.yaml | 9 ++ script/app-image-corner-detection/_cm.yaml | 32 ++++ script/benchmark-program/README-extra.md | 3 + script/build-docker-image/_cm.yaml | 46 ++++++ script/build-dockerfile/_cm.yaml | 67 ++++++++ script/create-custom-cache-entry/_cm.yaml | 27 ++++ script/create-custom-cache-entry/customize.py | 44 +++++ script/generate-docs-for-all-scripts.cmd | 1 + script/get-cuda-devices/_cm.yaml | 41 +++++ script/get-sys-utils-cm/_cm.yaml | 37 +++++ script/print-any-text/_cm.yaml | 34 ++++ script/print-any-text/customize.py | 30 ++++ script/print-any-text/run.bat | 5 + script/print-any-text/run.sh | 3 + script/print-hello-world/_cm.yaml | 48 ++++++ script/run-docker-container/_cm.yaml | 59 +++++++ script/test-cm-core/README-extra.md | 1 + script/test-cm-core/_cm.yaml | 14 ++ script/test-cm-core/customize.py | 16 ++ script/test-cm-core/run.bat | 3 + script/test-cm-core/run.sh | 13 ++ script/test-cm-core/src/script/check.py | 14 ++ .../src/script/process_dockerfile.py | 30 ++++ .../test-cm-core/src/script/process_readme.py | 26 +++ script/test-cm-core/src/script/test_deps.py | 23 +++ script/test-cm-core/src/script/test_docker.py | 21 +++ .../test-cm-core/src/script/test_features.py | 21 +++ .../test-cm-core/src/script/test_install.py | 10 ++ script/test-cm-core/src/test_cm.py | 14 ++ script/test-cm-core/src/test_search_speed.py | 25 +++ .../src/tutorials/test_tutorial_retinanet.py | 28 ++++ .../src/tutorials/test_tutorial_tvm.py | 24 +++ .../src/tutorials/test_tutorial_tvm_pip_ge.py | 21 +++ .../src/tutorials/test_tutorial_tvm_pip_vm.py | 23 +++ .../test-cm-script-pipeline/README-extra.md | 8 + script/test-cm-script-pipeline/_cm.yaml | 37 +++++ script/test-cm-script-pipeline/customize.py | 38 +++++ script/test-cm-script-pipeline/run.bat | 5 + script/test-cm-script-pipeline/run.sh | 5 + script/test-cm-script-pipeline/run2.bat | 5 + script/test-cm-script-pipeline/run2.sh | 5 + script/test-debug/.vscode/launch.json | 22 +++ script/test-debug/README-extra.md | 20 +++ script/test-debug/_cm.yaml | 29 ++++ script/test-debug/_demo.py | 9 ++ script/test-debug/customize.py | 35 ++++ script/test-debug/python/.vscode/launch.json | 23 +++ script/test-debug/python/main.py | 25 +++ script/test-debug/run.bat | 6 + script/test-debug/run.sh | 8 + script/test-deps-conditions/README-extra.md | 6 + script/test-deps-conditions/README.md | 151 ++++++++++++++++++ script/test-deps-conditions/_cm.yaml | 54 +++++++ script/test-deps-conditions2/README-extra.md | 20 +++ script/test-deps-conditions2/_cm.yaml | 28 ++++ 67 files changed, 1449 insertions(+) create mode 100644 LICENSE.third-party.md create mode 100644 automation/cfg/README-extra.md create mode 100644 automation/script/docker_repro_example/ubuntu-23.04.Dockerfile create mode 100644 automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat create mode 100644 automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh create mode 100644 automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat create mode 100644 automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh create mode 100644 cfg/docker-basic-configurations/_cm.yaml create mode 100644 cfg/docker-basic-configurations/basic-archlinux.yaml create mode 100644 cfg/docker-basic-configurations/basic-rhel-9.yaml create mode 100644 cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml create mode 100644 cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml create mode 100644 cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml create mode 100644 script/app-image-corner-detection/_cm.yaml create mode 100644 script/benchmark-program/README-extra.md create mode 100644 script/build-docker-image/_cm.yaml create mode 100644 script/build-dockerfile/_cm.yaml create mode 100644 script/create-custom-cache-entry/_cm.yaml create mode 100644 script/create-custom-cache-entry/customize.py create mode 100644 script/generate-docs-for-all-scripts.cmd create mode 100644 script/get-cuda-devices/_cm.yaml create mode 100644 script/get-sys-utils-cm/_cm.yaml create mode 100644 script/print-any-text/_cm.yaml create mode 100644 script/print-any-text/customize.py create mode 100644 script/print-any-text/run.bat create mode 100644 script/print-any-text/run.sh create mode 100644 script/print-hello-world/_cm.yaml create mode 100644 script/run-docker-container/_cm.yaml create mode 100644 script/test-cm-core/README-extra.md create mode 100644 script/test-cm-core/_cm.yaml create mode 100644 script/test-cm-core/customize.py create mode 100644 script/test-cm-core/run.bat create mode 100644 script/test-cm-core/run.sh create mode 100644 script/test-cm-core/src/script/check.py create mode 100644 script/test-cm-core/src/script/process_dockerfile.py create mode 100644 script/test-cm-core/src/script/process_readme.py create mode 100644 script/test-cm-core/src/script/test_deps.py create mode 100644 script/test-cm-core/src/script/test_docker.py create mode 100644 script/test-cm-core/src/script/test_features.py create mode 100644 script/test-cm-core/src/script/test_install.py create mode 100644 script/test-cm-core/src/test_cm.py create mode 100644 script/test-cm-core/src/test_search_speed.py create mode 100644 script/test-cm-core/src/tutorials/test_tutorial_retinanet.py create mode 100644 script/test-cm-core/src/tutorials/test_tutorial_tvm.py create mode 100644 script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py create mode 100644 script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py create mode 100644 script/test-cm-script-pipeline/README-extra.md create mode 100644 script/test-cm-script-pipeline/_cm.yaml create mode 100644 script/test-cm-script-pipeline/customize.py create mode 100644 script/test-cm-script-pipeline/run.bat create mode 100644 script/test-cm-script-pipeline/run.sh create mode 100644 script/test-cm-script-pipeline/run2.bat create mode 100644 script/test-cm-script-pipeline/run2.sh create mode 100644 script/test-debug/.vscode/launch.json create mode 100644 script/test-debug/README-extra.md create mode 100644 script/test-debug/_cm.yaml create mode 100644 script/test-debug/_demo.py create mode 100644 script/test-debug/customize.py create mode 100644 script/test-debug/python/.vscode/launch.json create mode 100644 script/test-debug/python/main.py create mode 100644 script/test-debug/run.bat create mode 100644 script/test-debug/run.sh create mode 100644 script/test-deps-conditions/README-extra.md create mode 100644 script/test-deps-conditions/README.md create mode 100644 script/test-deps-conditions/_cm.yaml create mode 100644 script/test-deps-conditions2/README-extra.md create mode 100644 script/test-deps-conditions2/_cm.yaml diff --git a/LICENSE.third-party.md b/LICENSE.third-party.md new file mode 100644 index 000000000..faa008458 --- /dev/null +++ b/LICENSE.third-party.md @@ -0,0 +1 @@ +This CM repository may contain CM scripts with third-party files licensed under Apache2, BSD or MIT license. diff --git a/automation/cfg/README-extra.md b/automation/cfg/README-extra.md new file mode 100644 index 000000000..cc94030ab --- /dev/null +++ b/automation/cfg/README-extra.md @@ -0,0 +1,8 @@ +Examples: + +```bash +cm set cfg default +cm set cfg default --key.script.silent +cm set cfg default --key.script.silent- + +``` diff --git a/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile new file mode 100644 index 000000000..a93507dc2 --- /dev/null +++ b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile @@ -0,0 +1,34 @@ +FROM ubuntu:23.04 + +# Automatically generated by the CM workflow automation meta-framework +# https://github.com/mlcommons/ck + +LABEL github="" +LABEL maintainer="" +LABEL license="" + +SHELL ["/bin/bash", "-c"] + +ARG CM_GH_TOKEN +ARG CM_ADD_DOCKER_GROUP_ID="" + +# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes +# Install system dependencies +RUN apt-get update -y +RUN apt-get install -y python3 python3-pip git sudo wget + +# Setup docker environment +ENTRYPOINT ["/bin/bash", "-c"] +ENV TZ="US/Pacific" +ENV PATH="${PATH}:/home/cmuser/.local/bin" +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone + +# Setup docker user +RUN groupadd ${CM_ADD_DOCKER_GROUP_ID} cm +RUN useradd -g cm --create-home --shell /bin/bash cmuser +RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +USER cmuser:cm +WORKDIR /home/cmuser + +# Install python packages +RUN python3 -m pip install --user cmind requests giturlparse tabulate --break-system-packages diff --git a/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat new file mode 100644 index 000000000..bd4ea665d --- /dev/null +++ b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat @@ -0,0 +1 @@ +docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" . diff --git a/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh new file mode 100644 index 000000000..92ec69ba1 --- /dev/null +++ b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" . diff --git a/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat new file mode 100644 index 000000000..c7c3fd198 --- /dev/null +++ b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat @@ -0,0 +1 @@ +docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash diff --git a/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh new file mode 100644 index 000000000..69425443a --- /dev/null +++ b/automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash diff --git a/cfg/docker-basic-configurations/_cm.yaml b/cfg/docker-basic-configurations/_cm.yaml new file mode 100644 index 000000000..d5d60a385 --- /dev/null +++ b/cfg/docker-basic-configurations/_cm.yaml @@ -0,0 +1,10 @@ +alias: docker-basic-configurations +uid: d2a0c5bb17664c93 + +automation_alias: cfg +automation_uid: 88dce9c160324c5d + +tags: +- docker +- basic +- configurations diff --git a/cfg/docker-basic-configurations/basic-archlinux.yaml b/cfg/docker-basic-configurations/basic-archlinux.yaml new file mode 100644 index 000000000..238a1e57c --- /dev/null +++ b/cfg/docker-basic-configurations/basic-archlinux.yaml @@ -0,0 +1,9 @@ +uid: 9960e9fb3cb24cb3 + +name: "Basic ArchLinux" + +input: + docker_base_image: 'archlinux' + docker_os: arch + docker_os_version: 'latest' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/basic-rhel-9.yaml b/cfg/docker-basic-configurations/basic-rhel-9.yaml new file mode 100644 index 000000000..1fba915fa --- /dev/null +++ b/cfg/docker-basic-configurations/basic-rhel-9.yaml @@ -0,0 +1,9 @@ +uid: 27b4afcdd8e042e8 + +name: "Basic RHEL 9" + +input: + docker_base_image: 'registry.access.redhat.com/ubi9' + docker_os: 'rhel' + docker_os_version: '9' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml b/cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml new file mode 100644 index 000000000..35c0b48f9 --- /dev/null +++ b/cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml @@ -0,0 +1,9 @@ +uid: 59311e6098c14b21 + +name: "Basic Ubuntu 20.04" + +input: + docker_base_image: 'ubuntu:20.04' + docker_os: ubuntu + docker_os_version: '20.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml b/cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml new file mode 100644 index 000000000..371ba2b6a --- /dev/null +++ b/cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml @@ -0,0 +1,9 @@ +uid: 614aa48d90724835 + +name: "Basic Ubuntu 22.04" + +input: + docker_base_image: 'ubuntu:22.04' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml b/cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml new file mode 100644 index 000000000..990d56cf5 --- /dev/null +++ b/cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml @@ -0,0 +1,9 @@ +uid: 276bd8ab39324f5f + +name: "Basic Ubuntu 23.04" + +input: + docker_base_image: 'ubuntu:23.04' + docker_os: ubuntu + docker_os_version: '23.04' + \ No newline at end of file diff --git a/script/app-image-corner-detection/_cm.yaml b/script/app-image-corner-detection/_cm.yaml new file mode 100644 index 000000000..1fd27d9b6 --- /dev/null +++ b/script/app-image-corner-detection/_cm.yaml @@ -0,0 +1,32 @@ +alias: app-image-corner-detection +uid: 998ffee0bc534d0a + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Modular application pipeline + +deps: +- tags: detect,os + +- tags: detect,cpu + +- tags: download,file,_url.https://cKnowledge.org/ai/data/data.pgm + md5sum: 0af279e557a8de252d7ff0751a999379 + force_cache: false + + +posthook_deps: +- skip_if_env: + CM_SKIP_COMPILE: + - 'on' + tags: compile,cpp-program +- skip_if_env: + CM_SKIP_RUN: + - 'on' + tags: benchmark-program + +tags: +- app +- image +- corner-detection diff --git a/script/benchmark-program/README-extra.md b/script/benchmark-program/README-extra.md new file mode 100644 index 000000000..d0cdc3143 --- /dev/null +++ b/script/benchmark-program/README-extra.md @@ -0,0 +1,3 @@ +This is a universal script to run and profile programs. + +It is a part of our universal benchmarking and optimization roadmap: https://github.com/mlcommons/cm4mlops/issues/23 diff --git a/script/build-docker-image/_cm.yaml b/script/build-docker-image/_cm.yaml new file mode 100644 index 000000000..9c958874d --- /dev/null +++ b/script/build-docker-image/_cm.yaml @@ -0,0 +1,46 @@ +alias: build-docker-image +uid: 2c3c4ba2413442e7 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: false + +category: Docker automation + +tags: +- build +- docker +- image +- docker-image +- dockerimage + +default_env: + CM_DOCKER_IMAGE_REPO: local + CM_DOCKER_IMAGE_TAG: latest + +input_mapping: + cache: CM_DOCKER_CACHE + cm_repo: CM_MLOPS_REPO + docker_os: CM_DOCKER_OS + docker_os_version: CM_DOCKER_OS_VERSION + dockerfile: CM_DOCKERFILE_WITH_PATH + gh_token: CM_GH_TOKEN + image_name: CM_DOCKER_IMAGE_NAME + image_repo: CM_DOCKER_IMAGE_REPO + image_tag: CM_DOCKER_IMAGE_TAG + post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS + pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS + real_run: CM_REAL_RUN + script_tags: CM_DOCKER_RUN_SCRIPT_TAGS + push_image: CM_DOCKER_PUSH_IMAGE + +new_env_keys: +- CM_DOCKER_* + +prehook_deps: +- enable_if_env: + CM_BUILD_DOCKERFILE: + - 'yes' + - '1' + tags: build,dockerfile diff --git a/script/build-dockerfile/_cm.yaml b/script/build-dockerfile/_cm.yaml new file mode 100644 index 000000000..74a6a528a --- /dev/null +++ b/script/build-dockerfile/_cm.yaml @@ -0,0 +1,67 @@ +alias: build-dockerfile +uid: e66a7483230d4641 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +tags: +- build +- dockerfile + +cache: false + +category: Docker automation + +default_env: + CM_DOCKER_BUILD_SLIM: 'no' + CM_DOCKER_IMAGE_EOL: ' + + ' + CM_DOCKER_OS: ubuntu + +input_mapping: + build: CM_BUILD_DOCKER_IMAGE + cache: CM_DOCKER_CACHE + cm_repo: CM_MLOPS_REPO + cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO + cm_repos: CM_DOCKER_EXTRA_CM_REPOS + comments: CM_DOCKER_RUN_COMMENTS + copy_files: CM_DOCKER_COPY_FILES + docker_base_image: CM_DOCKER_IMAGE_BASE + docker_os: CM_DOCKER_OS + docker_os_version: CM_DOCKER_OS_VERSION + extra_sys_deps: CM_DOCKER_EXTRA_SYS_DEPS + fake_docker_deps: CM_DOCKER_FAKE_DEPS + fake_run_option: CM_DOCKER_FAKE_RUN_OPTION + file_path: CM_DOCKERFILE_WITH_PATH + gh_token: CM_GH_TOKEN + image_repo: CM_DOCKER_IMAGE_REPO + image_tag: CM_DOCKER_IMAGE_TAG + package_manager_update_cmd: CM_PACKAGE_MANAGER_UPDATE_CMD + pip_extra_flags: CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS + post_file: DOCKER_IMAGE_POST_FILE + post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS + pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS + real_run: CM_REAL_RUN + run_cmd: CM_DOCKER_RUN_CMD + run_cmd_extra: CM_DOCKER_RUN_CMD_EXTRA + script_tags: CM_DOCKER_RUN_SCRIPT_TAGS + skip_cm_sys_upgrade: CM_DOCKER_SKIP_CM_SYS_UPGRADE + push_image: CM_DOCKER_PUSH_IMAGE + +new_env_keys: +- CM_DOCKERFILE_* + +post_deps: +- enable_if_env: + CM_BUILD_DOCKER_IMAGE: + - 'yes' + - '1' + names: + - build-docker-image + tags: build,docker,image + +variations: + slim: + env: + CM_DOCKER_BUILD_SLIM: 'yes' diff --git a/script/create-custom-cache-entry/_cm.yaml b/script/create-custom-cache-entry/_cm.yaml new file mode 100644 index 000000000..7272bb99a --- /dev/null +++ b/script/create-custom-cache-entry/_cm.yaml @@ -0,0 +1,27 @@ +alias: create-custom-cache-entry +uid: 485741440fbe4236 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +tags: +- create +- custom +- cache +- entry + +category: CM automation + +cache: true + +input_mapping: + env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY + env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2 + path: CM_CUSTOM_CACHE_ENTRY_PATH + to: CM_CUSTOM_CACHE_ENTRY_PATH + +new_env_keys: +- CM_CUSTOM_CACHE_ENTRY* + +print_env_at_the_end: + CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry" diff --git a/script/create-custom-cache-entry/customize.py b/script/create-custom-cache-entry/customize.py new file mode 100644 index 000000000..8d2d31db3 --- /dev/null +++ b/script/create-custom-cache-entry/customize.py @@ -0,0 +1,44 @@ +from cmind import utils +import os +import shutil + +def preprocess(i): + + # CM script internal variables + env = i['env'] + + extra_cache_tags = [] + if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '': + print ('') + extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ') + + extra_cache_tags = extra_cache_tags_str.strip().split(',') + + return {'return':0, 'add_extra_cache_tags':extra_cache_tags} + +def postprocess(i): + + env = i['env'] + + path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip() + + if path!='': + if not os.path.isdir(path): + os.makedirs(path) + else: + path = os.getcwd() + + x = '' + env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '') + if env_key != '': x = env_key+'_' + + env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path + env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path + + env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '') + v = env.get(env_key2, '') + real_path = v if v != '' else path + + env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path + + return {'return': 0} diff --git a/script/generate-docs-for-all-scripts.cmd b/script/generate-docs-for-all-scripts.cmd new file mode 100644 index 000000000..c21a77b71 --- /dev/null +++ b/script/generate-docs-for-all-scripts.cmd @@ -0,0 +1 @@ +cm doc script --repos=mlcommons@cm4mlops --output_dir=.. \ No newline at end of file diff --git a/script/get-cuda-devices/_cm.yaml b/script/get-cuda-devices/_cm.yaml new file mode 100644 index 000000000..b91e791af --- /dev/null +++ b/script/get-cuda-devices/_cm.yaml @@ -0,0 +1,41 @@ +alias: get-cuda-devices +uid: 7a3ede4d3558427a + +automation_alias: script +automation_uid: 5b4e0237da074764 + +tags: +- get +- cuda-devices + +cache: false + +can_force_cache: true + +category: CUDA automation + +clean_files: +- tmp-run.out + +deps: +- names: + - cuda + tags: get,cuda,_toolkit + +docker: + run: false + all_gpus: 'yes' + skip_run_cmd: 'no' + skip_cm_sys_upgrade: 'yes' + cm_repo_flags: '--checkout=dev' + use_host_group_id: 'yes' + image_tag_extra: '-cm-dev' + +new_env_keys: +- CM_CUDA_DEVICE_* + +new_state_keys: +- cm_cuda_device_prop + +print_files_if_script_error: +- tmp-run.out diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml new file mode 100644 index 000000000..555fe7c6a --- /dev/null +++ b/script/get-sys-utils-cm/_cm.yaml @@ -0,0 +1,37 @@ +alias: get-sys-utils-cm +uid: bc90993277e84b8e + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: true + +category: Detection or installation of tools and artifacts + +deps: +- tags: detect,os + +env: + CM_CLEAN_DIRS: bin + CM_PACKAGE_WIN_URL: https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip + ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 + CM_SUDO: sudo + +input_mapping: + skip: CM_SKIP_SYS_UTILS + +new_env_keys: +- +PATH + +tags: +- get +- sys-utils-cm + +variations: + user: + env: + CM_PYTHON_PIP_USER: --user + + skip_python_deps: + env: + CM_SKIP_PYTHON_DEPS: "yes" diff --git a/script/print-any-text/_cm.yaml b/script/print-any-text/_cm.yaml new file mode 100644 index 000000000..2fd9bba2c --- /dev/null +++ b/script/print-any-text/_cm.yaml @@ -0,0 +1,34 @@ +alias: print-any-text +uid: f4bf2d1d33c24e31 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +default_env: + CM_PRINT_ANY_TEXT: '' + +input_mapping: + text: CM_PRINT_ANY_TEXT + cm_env_keys: CM_PRINT_ANY_CM_ENV_KEYS + os_env_keys: CM_PRINT_ANY_OS_ENV_KEYS + +tags: +- print +- any-text + +variations: + text.#: + env: + CM_PRINT_ANY_TEXT: "#" + + cm_env.#: + env: + CM_PRINT_ANY_CM_ENV_KEYS: "#" + + os_env.#: + env: + CM_PRINT_ANY_OS_ENV_KEYS: "#" diff --git a/script/print-any-text/customize.py b/script/print-any-text/customize.py new file mode 100644 index 000000000..093cafdcf --- /dev/null +++ b/script/print-any-text/customize.py @@ -0,0 +1,30 @@ +# Developer(s): Grigori Fursin + +from cmind import utils +import os + +def postprocess(i): + + env = i['env'] + + cm_env_keys = env.get('CM_PRINT_ANY_CM_ENV_KEYS', '').strip() + os_env_keys = env.get('CM_PRINT_ANY_OS_ENV_KEYS', '').strip() + + printed = False + for k,e,t in [(cm_env_keys, env, 'CM_ENV'), + (os_env_keys, os.environ, 'OS_ENV')]: + + if k!='': + for kk in k.split(','): + kk = kk.strip() + if kk!='': + vv = e.get(kk) + + print ('{}[{}]: {}'.format(t, kk, vv)) + printed = True + + if printed: + print ('') + + return {'return':0} + diff --git a/script/print-any-text/run.bat b/script/print-any-text/run.bat new file mode 100644 index 000000000..be97ff0a2 --- /dev/null +++ b/script/print-any-text/run.bat @@ -0,0 +1,5 @@ +if "%CM_PRINT_ANY_TEXT%" == "" ( + echo. +) else ( + echo %CM_PRINT_ANY_TEXT% +) diff --git a/script/print-any-text/run.sh b/script/print-any-text/run.sh new file mode 100644 index 000000000..7f04767d6 --- /dev/null +++ b/script/print-any-text/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "${CM_PRINT_ANY_TEXT}" diff --git a/script/print-hello-world/_cm.yaml b/script/print-hello-world/_cm.yaml new file mode 100644 index 000000000..12b26efd4 --- /dev/null +++ b/script/print-hello-world/_cm.yaml @@ -0,0 +1,48 @@ +alias: print-hello-world +uid: b9f0acba4aca4baa + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +default_env: + CM_ENV_TEST1: TEST1 + +env: + CM_ENV_TEST2: TEST2 + +input_mapping: + test1: CM_ENV_TEST1 + +new_env_keys: +- CM_ENV_TEST* + +new_state_keys: +- hello_world* + +tags: +- print +- hello-world +- hello world +- hello +- world +- native-script +- native +- script + +variations: + text.#: + env: + CM_PRINT_HELLO_WORLD_TEXT: "#" + + skip_print_env: + env: + CM_PRINT_HELLO_WORLD_SKIP_PRINT_ENV: 'yes' + +docker: + skip_run_cmd: 'yes' + skip_cm_sys_upgrade: 'yes' + cm_repo_flags: '--checkout=dev' + use_host_group_id: 'yes' + image_tag_extra: '-cm-dev' diff --git a/script/run-docker-container/_cm.yaml b/script/run-docker-container/_cm.yaml new file mode 100644 index 000000000..864b5b202 --- /dev/null +++ b/script/run-docker-container/_cm.yaml @@ -0,0 +1,59 @@ +alias: run-docker-container +uid: 1e0c884107514b46 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +tags: +- run +- docker +- container + +cache: false + +category: Docker automation + +default_env: + CM_DOCKER_DETACHED_MODE: 'yes' + +input_mapping: + all_gpus: CM_DOCKER_ADD_ALL_GPUS + base: CM_DOCKER_IMAGE_BASE + cache: CM_DOCKER_CACHE + cm_repo: CM_MLOPS_REPO + detached: CM_DOCKER_DETACHED_MODE + device: CM_DOCKER_ADD_DEVICE + docker_image_base: CM_DOCKER_IMAGE_BASE + docker_os: CM_DOCKER_OS + docker_os_version: CM_DOCKER_OS_VERSION + extra_run_args: CM_DOCKER_EXTRA_RUN_ARGS + fake_run_option: CM_DOCKER_FAKE_RUN_OPTION + gh_token: CM_GH_TOKEN + image_name: CM_DOCKER_IMAGE_NAME + image_repo: CM_DOCKER_IMAGE_REPO + image_tag: CM_DOCKER_IMAGE_TAG + image_tag_extra: CM_DOCKER_IMAGE_TAG_EXTRA + interactive: CM_DOCKER_INTERACTIVE_MODE + it: CM_DOCKER_INTERACTIVE + mounts: CM_DOCKER_VOLUME_MOUNTS + pass_user_group: CM_DOCKER_PASS_USER_GROUP + port_maps: CM_DOCKER_PORT_MAPS + post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS + pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS + real_run: CM_REAL_RUN + recreate: CM_DOCKER_IMAGE_RECREATE + run_cmd: CM_DOCKER_RUN_CMD + run_cmd_extra: CM_DOCKER_RUN_CMD_EXTRA + save_script: CM_DOCKER_SAVE_SCRIPT + script_tags: CM_DOCKER_RUN_SCRIPT_TAGS + shm_size: CM_DOCKER_SHM_SIZE + +prehook_deps: +- names: + - build-docker-image + skip_if_any_env: + CM_DOCKER_IMAGE_EXISTS: + - 'yes' + CM_DOCKER_SKIP_BUILD: + - 'yes' + tags: build,docker,image diff --git a/script/test-cm-core/README-extra.md b/script/test-cm-core/README-extra.md new file mode 100644 index 000000000..582991f6d --- /dev/null +++ b/script/test-cm-core/README-extra.md @@ -0,0 +1 @@ +# CM script diff --git a/script/test-cm-core/_cm.yaml b/script/test-cm-core/_cm.yaml new file mode 100644 index 000000000..420d119f8 --- /dev/null +++ b/script/test-cm-core/_cm.yaml @@ -0,0 +1,14 @@ +alias: test-cm-core +uid: 2c2fb9d20dc64cf3 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +tags: +- test +- cm +- core + +cache: false diff --git a/script/test-cm-core/customize.py b/script/test-cm-core/customize.py new file mode 100644 index 000000000..831f0bdff --- /dev/null +++ b/script/test-cm-core/customize.py @@ -0,0 +1,16 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} diff --git a/script/test-cm-core/run.bat b/script/test-cm-core/run.bat new file mode 100644 index 000000000..271224ef9 --- /dev/null +++ b/script/test-cm-core/run.bat @@ -0,0 +1,3 @@ +rem native script + +echo "TBD" diff --git a/script/test-cm-core/run.sh b/script/test-cm-core/run.sh new file mode 100644 index 000000000..997c0c33e --- /dev/null +++ b/script/test-cm-core/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency + +echo "TBD" + +#Add your run commands here... +# run "$CM_RUN_CMD" diff --git a/script/test-cm-core/src/script/check.py b/script/test-cm-core/src/script/check.py new file mode 100644 index 000000000..7394406d8 --- /dev/null +++ b/script/test-cm-core/src/script/check.py @@ -0,0 +1,14 @@ +def check_return(r): + if 'return' not in r: + raise Exception('CM access function should always return key \'return\'!') + if 'error' in r: + raise Exception(r['error']) + +def check_list(r, string, found=True): + check_return(r) + if 'list' not in r: + raise Exception('CM search should return a list!') + if len(r['list']) < 1 and found: + raise Exception('CM search returned an empty list for ' + string) + if len(r['list']) > 0 and not found: + raise Exception('CM search returned at lease one entry for ' + string) diff --git a/script/test-cm-core/src/script/process_dockerfile.py b/script/test-cm-core/src/script/process_dockerfile.py new file mode 100644 index 000000000..62bf4d5f8 --- /dev/null +++ b/script/test-cm-core/src/script/process_dockerfile.py @@ -0,0 +1,30 @@ +import sys +import os +import cmind as cm +import check as checks +import json +import yaml + +files=sys.argv[1:] + +for file in files: + if not os.path.isfile(file): + continue + if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): + continue + if not file.startswith(os.path.join("script")): + continue + + script_path = os.path.dirname(file) + + f = open(file) + + if file.endswith(".json"): + data = json.load(f) + elif file.endswith(".yaml"): + data = yaml.safe_load(f) + + uid = data['uid'] + + r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + checks.check_return(r) diff --git a/script/test-cm-core/src/script/process_readme.py b/script/test-cm-core/src/script/process_readme.py new file mode 100644 index 000000000..e33f26528 --- /dev/null +++ b/script/test-cm-core/src/script/process_readme.py @@ -0,0 +1,26 @@ +import sys +import os +import cmind as cm +import check as checks +import json +import yaml + +files=sys.argv[1:] + +for file in files: + if not os.path.isfile(file): + continue + if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): + continue + if not file.startswith(os.path.join("script")): + continue + script_path = os.path.dirname(file) + f = open(file) + if file.endswith(".json"): + data = json.load(f) + elif file.endswith(".yaml"): + data = yaml.safe_load(f) + uid = data['uid'] + + r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + checks.check_return(r) diff --git a/script/test-cm-core/src/script/test_deps.py b/script/test-cm-core/src/script/test_deps.py new file mode 100644 index 000000000..aaf19bc81 --- /dev/null +++ b/script/test-cm-core/src/script/test_deps.py @@ -0,0 +1,23 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm +import check as checks + +# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests + +#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': +# {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50', +# 'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'}) +#checks.check_return(r) +# +#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) +#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") +# +#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) +#checks.check_list(r, "inference,src,version-r3.0") +# +#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +#checks.check_return(r) +# +#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +#checks.check_return(r) diff --git a/script/test-cm-core/src/script/test_docker.py b/script/test-cm-core/src/script/test_docker.py new file mode 100644 index 000000000..5a02c932e --- /dev/null +++ b/script/test-cm-core/src/script/test_docker.py @@ -0,0 +1,21 @@ +# This test covers version, variation, compilation from src, add_deps_recursive, post_deps + +import cmind as cm +import check as checks + +r = cm.access({'action':'run', + 'automation':'script', + 'tags': 'run,docker,container', + 'add_deps_recursive': { + 'compiler': {'tags': "gcc"} + }, + 'image_name':'cm-script-app-image-classification-onnx-py', + 'env': { + 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', + 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', + 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04' + }, + 'quiet': 'yes' + }) + +checks.check_return(r) diff --git a/script/test-cm-core/src/script/test_features.py b/script/test-cm-core/src/script/test_features.py new file mode 100644 index 000000000..d116cbd5b --- /dev/null +++ b/script/test-cm-core/src/script/test_features.py @@ -0,0 +1,21 @@ +# This test covers +# 1. python-virtual-env and update_deps inside customize.py +# 2. cache search using "-" prefix + +import cmind as cm +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'}) +checks.check_list(r, "get,python-venv") + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) +checks.check_list(r, "_NHWC") + +r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) +checks.check_list(r, "_NHWC", False) diff --git a/script/test-cm-core/src/script/test_install.py b/script/test-cm-core/src/script/test_install.py new file mode 100644 index 000000000..66fa164d5 --- /dev/null +++ b/script/test-cm-core/src/script/test_install.py @@ -0,0 +1,10 @@ +# This test covers script installation, version, shared library install + +import cmind as cm +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'}) +checks.check_return(r) + +r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'}) +checks.check_list(r, "python,src,install,_shared,version-3.9.10") diff --git a/script/test-cm-core/src/test_cm.py b/script/test-cm-core/src/test_cm.py new file mode 100644 index 000000000..41fb402c2 --- /dev/null +++ b/script/test-cm-core/src/test_cm.py @@ -0,0 +1,14 @@ +try: + import cmind as cm + + r = cm.access(['test', 'script']) + if 'return' not in r: + raise Exception('CM access function should always return key \'return\'!') + exit(0) + +except ImportError as e: + from sys import stderr + from subprocess import call + print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr) + retcode = call(['cm', 'test', 'script']) + exit(retcode) diff --git a/script/test-cm-core/src/test_search_speed.py b/script/test-cm-core/src/test_search_speed.py new file mode 100644 index 000000000..3086a8340 --- /dev/null +++ b/script/test-cm-core/src/test_search_speed.py @@ -0,0 +1,25 @@ +import cmind as cm +import time + +times = [] + +steps = 10 + +print ('Running search with tags {} times ...'.format(steps)) + +for step in range(steps): + + start = time.time() + r = cm.access({'action':'search', + 'automation':'script', + 'tags':'detect,os'}) + timer = time.time() - start + + if r['return']>0: cm.error(r) + + times.append(timer) + +step = 0 +for t in times: + step += 1 + print ("{}) {:0.3f} sec.".format(step, t)) diff --git a/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py b/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py new file mode 100644 index 000000000..404b69fe9 --- /dev/null +++ b/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py @@ -0,0 +1,28 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ + 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ + 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' }) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \ + 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \ + 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) diff --git a/script/test-cm-core/src/tutorials/test_tutorial_tvm.py b/script/test-cm-core/src/tutorials/test_tutorial_tvm.py new file mode 100644 index 000000000..930e3622d --- /dev/null +++ b/script/test-cm-core/src/tutorials/test_tutorial_tvm.py @@ -0,0 +1,24 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm + +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + diff --git a/script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py b/script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py new file mode 100644 index 000000000..0c9a4b9c3 --- /dev/null +++ b/script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py @@ -0,0 +1,21 @@ +import cmind as cm + +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ + 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ + 'device': 'cpu', 'scenario': 'Offline', 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ + 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ + 'device': 'cpu', 'scenario': 'Offline', 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + diff --git a/script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py b/script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py new file mode 100644 index 000000000..81069194d --- /dev/null +++ b/script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py @@ -0,0 +1,23 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm + +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + diff --git a/script/test-cm-script-pipeline/README-extra.md b/script/test-cm-script-pipeline/README-extra.md new file mode 100644 index 000000000..480bea4f4 --- /dev/null +++ b/script/test-cm-script-pipeline/README-extra.md @@ -0,0 +1,8 @@ +# CM script + +This script prints internal pipeline run: + +```bash +cmr "test cm-script pipeline" +``` + diff --git a/script/test-cm-script-pipeline/_cm.yaml b/script/test-cm-script-pipeline/_cm.yaml new file mode 100644 index 000000000..9a2327ae6 --- /dev/null +++ b/script/test-cm-script-pipeline/_cm.yaml @@ -0,0 +1,37 @@ +alias: test-cm-script-pipeline +uid: ebe50aa281be4458 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +tags: +- test +- cm-script +- pipeline + +cache: false + +deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: deps" + +prehook_deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: prehook_deps" + +posthook_deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: posthook_deps" + +post_deps: +- tags: print,any-text + env: + CM_PRINT_ANY_TEXT: "_cm.yaml: post_deps" + CM_PRINT_ANY_CM_ENV_KEYS: 'CM_TMP_CURRENT_SCRIPT_PATH,CM_TMP_CURRENT_PATH,CM_QUIET' diff --git a/script/test-cm-script-pipeline/customize.py b/script/test-cm-script-pipeline/customize.py new file mode 100644 index 000000000..89311a22f --- /dev/null +++ b/script/test-cm-script-pipeline/customize.py @@ -0,0 +1,38 @@ +# Developers: Grigori Fursin + +from cmind import utils +import os + +def preprocess(i): + + print ('') + print ('customize.py: preprocess') + print ('') + + return {'return':0} + +def postprocess(i): + + automation = i['automation'] + run_script_input = i['run_script_input'] + env = i['env'] + + print ('') + print ('customize.py: postprocess') + print ('') + + r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'run2'}) + if r['return']>0: + return r + + return {'return':0} + +def detect_version(i): + + + print ('') + print ('customize.py: detect_version') + print ('') + + + return {'return':0} diff --git a/script/test-cm-script-pipeline/run.bat b/script/test-cm-script-pipeline/run.bat new file mode 100644 index 000000000..3103960b7 --- /dev/null +++ b/script/test-cm-script-pipeline/run.bat @@ -0,0 +1,5 @@ +rem native script + +echo. +echo run.bat +echo. diff --git a/script/test-cm-script-pipeline/run.sh b/script/test-cm-script-pipeline/run.sh new file mode 100644 index 000000000..51936abac --- /dev/null +++ b/script/test-cm-script-pipeline/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "" +echo "run.sh" +echo "" diff --git a/script/test-cm-script-pipeline/run2.bat b/script/test-cm-script-pipeline/run2.bat new file mode 100644 index 000000000..120bd24a8 --- /dev/null +++ b/script/test-cm-script-pipeline/run2.bat @@ -0,0 +1,5 @@ +rem native script + +echo. +echo run2.bat +echo. diff --git a/script/test-cm-script-pipeline/run2.sh b/script/test-cm-script-pipeline/run2.sh new file mode 100644 index 000000000..21664817b --- /dev/null +++ b/script/test-cm-script-pipeline/run2.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "" +echo "run2.sh" +echo "" diff --git a/script/test-debug/.vscode/launch.json b/script/test-debug/.vscode/launch.json new file mode 100644 index 000000000..49c4e19f0 --- /dev/null +++ b/script/test-debug/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "${workspaceFolder}" + } + ] + } +} \ No newline at end of file diff --git a/script/test-debug/README-extra.md b/script/test-debug/README-extra.md new file mode 100644 index 000000000..06a41746c --- /dev/null +++ b/script/test-debug/README-extra.md @@ -0,0 +1,20 @@ +Demo of debugging CM scripts and wrapped apps with Visual Studio Code. + + +Debug customize.py using remote debugging and follow instructions on command line: + +```bash +cmr "test cm-debug" --debug_uid=8d96cd9fa4734204 +``` + +Debug Python application or tool wrapped by the CM script (see [python/main.py](python/main.py)): + +```bash +cmr "test cm-debug" --debug_uid=45a7c3a500d24a63 +``` + + + +Debug CM internals using standard Python debugging: +* Open _demo.py and start debugging using "Python File" default configuration. + diff --git a/script/test-debug/_cm.yaml b/script/test-debug/_cm.yaml new file mode 100644 index 000000000..8a4ab37aa --- /dev/null +++ b/script/test-debug/_cm.yaml @@ -0,0 +1,29 @@ +alias: test-debug +uid: 5ccd3d701a9144f9 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: CM interface prototyping + +deps: + # Detect host OS features + - tags: detect,os + + # Detect host CPU features + - tags: detect,cpu + + # Get Python + - tags: get,python3 + names: + - python + - python3 + + # May need to use CM in automation recipes + - tags: get,generic-python-lib,_package.cmind + names: + - python-cmind + +tags: +- test +- cm-debug diff --git a/script/test-debug/_demo.py b/script/test-debug/_demo.py new file mode 100644 index 000000000..781bed321 --- /dev/null +++ b/script/test-debug/_demo.py @@ -0,0 +1,9 @@ +# Developer: Grigori Fursin + +import cmind +import sys + +print(sys.executable) + +r = cmind.access('run "cm-debug"') +print(r) diff --git a/script/test-debug/customize.py b/script/test-debug/customize.py new file mode 100644 index 000000000..a3af87f64 --- /dev/null +++ b/script/test-debug/customize.py @@ -0,0 +1,35 @@ +# Developer(s): Grigori Fursin + +import os + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + + print ("********************************************************") + print ('- Importing CM library ...') + import cmind + print (' SUCCESS!') + + cmind.utils.debug_here(__file__, port=5678, text='Debugging customize.py!', env=env, env_debug_uid='8d96cd9fa4734204').breakpoint() + + print ('') + print ('- List CM repos ...') + print ('') + r = cmind.access({'action':'show', 'automation':'repo', 'out':'con'}) + print ('') + print (' SUCCESS!') + print ("********************************************************") + + + return {'return':0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + return {'return':0} diff --git a/script/test-debug/python/.vscode/launch.json b/script/test-debug/python/.vscode/launch.json new file mode 100644 index 000000000..c2dc39156 --- /dev/null +++ b/script/test-debug/python/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "${workspaceFolder}" + } + ] + } + ] +} \ No newline at end of file diff --git a/script/test-debug/python/main.py b/script/test-debug/python/main.py new file mode 100644 index 000000000..0dfdf30f5 --- /dev/null +++ b/script/test-debug/python/main.py @@ -0,0 +1,25 @@ +""" +Testing CM debugging + +# Developer(s): Grigori Fursin +""" + +import os +import json + +print ("Hello World 1") + +env = os.environ + +import json +print ('') +print (json.dumps(dict(env), indent=2)) + +# Import cmind to test break points +import cmind.utils +if os.environ.get('CM_TMP_DEBUG_UID', '') == '45a7c3a500d24a63': + cmind.utils.debug_here(__file__, port=5678, text='Debugging main.py!').breakpoint() + +print ('') +print ("Hello World 2") + diff --git a/script/test-debug/run.bat b/script/test-debug/run.bat new file mode 100644 index 000000000..d45522c1d --- /dev/null +++ b/script/test-debug/run.bat @@ -0,0 +1,6 @@ +echo ======================================================== + +%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\python\main.py +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% + +echo ======================================================== diff --git a/script/test-debug/run.sh b/script/test-debug/run.sh new file mode 100644 index 000000000..ee43341da --- /dev/null +++ b/script/test-debug/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo "========================================================" + +${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/python/main.py +test $? -eq 0 || exit 1 + +echo ======================================================== diff --git a/script/test-deps-conditions/README-extra.md b/script/test-deps-conditions/README-extra.md new file mode 100644 index 000000000..bde1360a8 --- /dev/null +++ b/script/test-deps-conditions/README-extra.md @@ -0,0 +1,6 @@ +```bash +cmr "test deps conditions" +cmr "test deps conditions" --test1 +cmr "test deps conditions" --test1 --test2 +cmr "test deps conditions" --test1 --test2 --test3 +``` diff --git a/script/test-deps-conditions/README.md b/script/test-deps-conditions/README.md new file mode 100644 index 000000000..4c0ee33ed --- /dev/null +++ b/script/test-deps-conditions/README.md @@ -0,0 +1,151 @@ +Automatically generated README for this automation recipe: **test-deps-conditions** + +Category: **Tests** + +License: **Apache 2.0** + +Maintainers: [Public MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) + +--- +*[ [Online info and GUI to run this CM script](https://access.cknowledge.org/playground/?action=scripts&name=test-deps-conditions,5cb82aee472640df) ] [ [Notes from the authors, contributors and users](README-extra.md) ]* + +--- +#### Summary + +* CM GitHub repository: *[mlcommons@cm4mlops](https://github.com/mlcommons/cm4mlops/tree/dev)* +* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/cm4mlops/tree/dev/script/test-deps-conditions)* +* CM meta description for this script: *[_cm.yaml](_cm.yaml)* +* All CM tags to find and reuse this script (see in above meta description): *test,deps,conditions* +* Output cached? *False* +* See [pipeline of dependencies](#dependencies-on-other-cm-scripts) on other CM scripts + + +--- +### Reuse this script in your project + +#### Install MLCommons CM automation meta-framework + +* [Install CM](https://access.cknowledge.org/playground/?action=install) +* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md) + +#### Pull CM repository with this automation recipe (CM script) + +```cm pull repo mlcommons@cm4mlops``` + +#### Print CM help from the command line + +````cmr "test deps conditions" --help```` + +#### Customize and run this script from the command line with different variations and flags + +`cm run script --tags=test,deps,conditions` + +`cm run script --tags=test,deps,conditions [--input_flags]` + +*or* + +`cmr "test deps conditions"` + +`cmr "test deps conditions " [--input_flags]` + + +#### Run this script from Python + +
+Click here to expand this section. + +```python + +import cmind + +r = cmind.access({'action':'run' + 'automation':'script', + 'tags':'test,deps,conditions' + 'out':'con', + ... + (other input keys for this script) + ... + }) + +if r['return']>0: + print (r['error']) + +``` + +
+ + +#### Run this script via GUI + +```cmr "cm gui" --script="test,deps,conditions"``` + +#### Run this script via Docker (beta) + +`cm docker script "test deps conditions" [--input_flags]` + +___ +### Customization + + +#### Script flags mapped to environment +
+Click here to expand this section. + +* `--test1=value` → `CM_ENV1=value` +* `--test2=value` → `CM_ENV2=value` +* `--test3=value` → `CM_ENV3=value` + +**Above CLI flags can be used in the Python CM API as follows:** + +```python +r=cm.access({... , "test1":...} +``` + +
+ +#### Default environment + +
+Click here to expand this section. + +These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags. + + +
+ +___ +### Dependencies on other CM scripts + + + 1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/cm4mlops/tree/dev/script/test-deps-conditions/_cm.yaml)*** + * print,native,hello-world,_skip_print_env + - CM script: [print-hello-world](https://github.com/mlcommons/cm4mlops/tree/master/script/print-hello-world) + * print,native,hello-world,_skip_print_env,_text.SKIP_IF_ALL_ENV + * Skip this dependenecy only if all ENV vars are set:
+`{'CM_ENV1': [True], 'CM_ENV2': [True], 'CM_ENV3': [True]}` + - CM script: [print-hello-world](https://github.com/mlcommons/cm4mlops/tree/master/script/print-hello-world) + * print,native,hello-world,_skip_print_env,_text.SKIP_IF_ANY_ENV + * Skip this dependenecy only if any of ENV vars are set:
+`{'CM_ENV1': [True], 'CM_ENV2': [True], 'CM_ENV3': [True]}` + - CM script: [print-hello-world](https://github.com/mlcommons/cm4mlops/tree/master/script/print-hello-world) + * print,native,hello-world,_skip_print_env,_text.ENABLE_IF_ALL_ENV + * Enable this dependency only if all ENV vars are set:
+`{'CM_ENV1': [True], 'CM_ENV2': [True], 'CM_ENV3': [True]}` + - CM script: [print-hello-world](https://github.com/mlcommons/cm4mlops/tree/master/script/print-hello-world) + * print,native,hello-world,_skip_print_env,_text.ENABLE_IF_ANY_ENV + * Enable this dependency only if any of ENV vars are set:
+`{'CM_ENV1': [True], 'CM_ENV2': [True], 'CM_ENV3': [True]}` + - CM script: [print-hello-world](https://github.com/mlcommons/cm4mlops/tree/master/script/print-hello-world) + 1. Run "preprocess" function from customize.py + 1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/cm4mlops/tree/dev/script/test-deps-conditions/_cm.yaml) + 1. ***Run native script if exists*** + 1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/cm4mlops/tree/dev/script/test-deps-conditions/_cm.yaml) + 1. Run "postrocess" function from customize.py + 1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/cm4mlops/tree/dev/script/test-deps-conditions/_cm.yaml) + +___ +### Script output +`cmr "test deps conditions " [--input_flags] -j` +#### New environment keys (filter) + +#### New environment keys auto-detected from customize diff --git a/script/test-deps-conditions/_cm.yaml b/script/test-deps-conditions/_cm.yaml new file mode 100644 index 000000000..bc74a5871 --- /dev/null +++ b/script/test-deps-conditions/_cm.yaml @@ -0,0 +1,54 @@ +alias: test-deps-conditions +uid: 5cb82aee472640df + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +deps: + - tags: print,native,hello-world,_skip_print_env + - tags: print,native,hello-world,_skip_print_env,_text.SKIP_IF_ALL_ENV + skip_if_env: + CM_ENV1: + - True + CM_ENV2: + - True + CM_ENV3: + - True + - tags: print,native,hello-world,_skip_print_env,_text.SKIP_IF_ANY_ENV + skip_if_any_env: + CM_ENV1: + - True + CM_ENV2: + - True + CM_ENV3: + - True + - tags: print,native,hello-world,_skip_print_env,_text.ENABLE_IF_ALL_ENV + enable_if_env: + CM_ENV1: + - True + CM_ENV2: + - True + CM_ENV3: + - True + - tags: print,native,hello-world,_skip_print_env,_text.ENABLE_IF_ANY_ENV + enable_if_any_env: + CM_ENV1: + - True + CM_ENV2: + - True + CM_ENV3: + - True + +input_mapping: + test1: CM_ENV1 + test2: CM_ENV2 + test3: CM_ENV3 + +tags: +- test +- deps +- conditions diff --git a/script/test-deps-conditions2/README-extra.md b/script/test-deps-conditions2/README-extra.md new file mode 100644 index 000000000..d2b1f3033 --- /dev/null +++ b/script/test-deps-conditions2/README-extra.md @@ -0,0 +1,20 @@ +Checking some conditions to turn on or off deps: + +```bash +cmr "test deps conditions2" -s +cmr "test deps conditions2" -s --test +cmr "test deps conditions2" -s --test=xyz +``` + +Note that the last two will run with the following deps, +i.e. `True` tests not only for turning flag on, +but also for any non-empty value : +```yaml + - tags: print,any-text,_text.RUN_IF_ENV_IS_SET_TO_TRUE + enable_if_env: + TEST: + - True + +``` + +It is useful to check if flag turns on output or output to specific file for example ... diff --git a/script/test-deps-conditions2/_cm.yaml b/script/test-deps-conditions2/_cm.yaml new file mode 100644 index 000000000..08eb5ffb8 --- /dev/null +++ b/script/test-deps-conditions2/_cm.yaml @@ -0,0 +1,28 @@ +alias: test-deps-conditions2 +uid: 7a81ef941b3c4c6c + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Tests + +developers: "Grigori Fursin" + +deps: + - tags: print,any-text,_cm_env.TEST + - tags: print,any-text,_text.RUN_IF_ENV_IS_SET_TO_TRUE + enable_if_env: + TEST: + - True + - tags: print,any-text,_text.RUN_IF_ENV_IS_NOT_SET_TO_TRUE + skip_if_env: + TEST: + - True + +input_mapping: + test: TEST + +tags: +- test +- deps +- conditions2