Sync with dev (new)

LICENSE.third-party.md

@@ -0,0 +1 @@
+This CM repository may contain CM scripts with third-party files licensed under Apache2, BSD or MIT license.

automation/cfg/README-extra.md

@@ -0,0 +1,8 @@
+Examples:
+
+```bash
+cm set cfg default 
+cm set cfg default --key.script.silent
+cm set cfg default --key.script.silent-
+
+```

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:23.04
+
+# Automatically generated by the CM workflow automation meta-framework
+# https://github.com/mlcommons/ck
+
+LABEL github=""
+LABEL maintainer=""
+LABEL license=""
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CM_GH_TOKEN
+ARG CM_ADD_DOCKER_GROUP_ID=""
+
+# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes
+# Install system dependencies
+RUN apt-get update -y
+RUN apt-get install -y python3 python3-pip git sudo wget
+
+# Setup docker environment
+ENTRYPOINT ["/bin/bash", "-c"]
+ENV TZ="US/Pacific"
+ENV PATH="${PATH}:/home/cmuser/.local/bin"
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone
+
+# Setup docker user
+RUN groupadd ${CM_ADD_DOCKER_GROUP_ID} cm
+RUN useradd  -g cm --create-home --shell /bin/bash cmuser
+RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER cmuser:cm
+WORKDIR /home/cmuser
+
+# Install python packages
+RUN python3 -m pip install --user cmind requests giturlparse tabulate  --break-system-packages 

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat

@@ -0,0 +1 @@
+docker build  -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build  -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat

@@ -0,0 +1 @@
+docker run  -it --entrypoint ""  cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run  -it --entrypoint ""  cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

cfg/docker-basic-configurations/_cm.yaml

@@ -0,0 +1,10 @@
+alias: docker-basic-configurations
+uid: d2a0c5bb17664c93
+
+automation_alias: cfg
+automation_uid: 88dce9c160324c5d
+
+tags:
+- docker
+- basic
+- configurations

cfg/docker-basic-configurations/basic-archlinux.yaml

@@ -0,0 +1,9 @@
+uid: 9960e9fb3cb24cb3
+
+name: "Basic ArchLinux"
+
+input:
+  docker_base_image: 'archlinux'
+  docker_os: arch
+  docker_os_version: 'latest'
+

cfg/docker-basic-configurations/basic-rhel-9.yaml

@@ -0,0 +1,9 @@
+uid: 27b4afcdd8e042e8
+
+name: "Basic RHEL 9"
+
+input:
+  docker_base_image: 'registry.access.redhat.com/ubi9'
+  docker_os: 'rhel'
+  docker_os_version: '9'
+

cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml

@@ -0,0 +1,9 @@
+uid: 59311e6098c14b21
+
+name: "Basic Ubuntu 20.04"
+
+input:
+  docker_base_image: 'ubuntu:20.04'
+  docker_os: ubuntu
+  docker_os_version: '20.04'
+

cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml

@@ -0,0 +1,9 @@
+uid: 614aa48d90724835
+
+name: "Basic Ubuntu 22.04"
+
+input:
+  docker_base_image: 'ubuntu:22.04'
+  docker_os: ubuntu
+  docker_os_version: '22.04'
+

cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml

@@ -0,0 +1,9 @@
+uid: 276bd8ab39324f5f
+
+name: "Basic Ubuntu 23.04"
+
+input:
+  docker_base_image: 'ubuntu:23.04'
+  docker_os: ubuntu
+  docker_os_version: '23.04'
+

script/app-image-corner-detection/_cm.yaml

@@ -0,0 +1,32 @@
+alias: app-image-corner-detection
+uid: 998ffee0bc534d0a
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+category: Modular application pipeline
+
+deps:
+- tags: detect,os
+
+- tags: detect,cpu
+
+- tags: download,file,_url.https://cKnowledge.org/ai/data/data.pgm
+  md5sum: 0af279e557a8de252d7ff0751a999379
+  force_cache: false
+
+
+posthook_deps:
+- skip_if_env:
+    CM_SKIP_COMPILE:
+    - 'on'
+  tags: compile,cpp-program
+- skip_if_env:
+    CM_SKIP_RUN:
+    - 'on'
+  tags: benchmark-program
+
+tags:
+- app
+- image
+- corner-detection

script/benchmark-program/README-extra.md

@@ -0,0 +1,3 @@
+This is a universal script to run and profile programs.
+
+It is a part of our universal benchmarking and optimization roadmap: https://github.com/mlcommons/cm4mlops/issues/23

script/build-docker-image/_cm.yaml

@@ -0,0 +1,46 @@
+alias: build-docker-image
+uid: 2c3c4ba2413442e7
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+cache: false
+
+category: Docker automation
+
+tags:
+- build
+- docker
+- image
+- docker-image
+- dockerimage
+
+default_env:
+  CM_DOCKER_IMAGE_REPO: local
+  CM_DOCKER_IMAGE_TAG: latest
+
+input_mapping:
+  cache: CM_DOCKER_CACHE
+  cm_repo: CM_MLOPS_REPO
+  docker_os: CM_DOCKER_OS
+  docker_os_version: CM_DOCKER_OS_VERSION
+  dockerfile: CM_DOCKERFILE_WITH_PATH
+  gh_token: CM_GH_TOKEN
+  image_name: CM_DOCKER_IMAGE_NAME
+  image_repo: CM_DOCKER_IMAGE_REPO
+  image_tag: CM_DOCKER_IMAGE_TAG
+  post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
+  pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
+  real_run: CM_REAL_RUN
+  script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
+  push_image: CM_DOCKER_PUSH_IMAGE
+
+new_env_keys:
+- CM_DOCKER_*
+
+prehook_deps:
+- enable_if_env:
+    CM_BUILD_DOCKERFILE:
+    - 'yes'
+    - '1'
+  tags: build,dockerfile

script/build-dockerfile/_cm.yaml

@@ -0,0 +1,67 @@
+alias: build-dockerfile
+uid: e66a7483230d4641
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- build
+- dockerfile
+
+cache: false
+
+category: Docker automation
+
+default_env:
+  CM_DOCKER_BUILD_SLIM: 'no'
+  CM_DOCKER_IMAGE_EOL: '
+
+    '
+  CM_DOCKER_OS: ubuntu
+
+input_mapping:
+  build: CM_BUILD_DOCKER_IMAGE
+  cache: CM_DOCKER_CACHE
+  cm_repo: CM_MLOPS_REPO
+  cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO
+  cm_repos: CM_DOCKER_EXTRA_CM_REPOS
+  comments: CM_DOCKER_RUN_COMMENTS
+  copy_files: CM_DOCKER_COPY_FILES
+  docker_base_image: CM_DOCKER_IMAGE_BASE
+  docker_os: CM_DOCKER_OS
+  docker_os_version: CM_DOCKER_OS_VERSION
+  extra_sys_deps: CM_DOCKER_EXTRA_SYS_DEPS
+  fake_docker_deps: CM_DOCKER_FAKE_DEPS
+  fake_run_option: CM_DOCKER_FAKE_RUN_OPTION
+  file_path: CM_DOCKERFILE_WITH_PATH
+  gh_token: CM_GH_TOKEN
+  image_repo: CM_DOCKER_IMAGE_REPO
+  image_tag: CM_DOCKER_IMAGE_TAG
+  package_manager_update_cmd: CM_PACKAGE_MANAGER_UPDATE_CMD
+  pip_extra_flags: CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS
+  post_file: DOCKER_IMAGE_POST_FILE
+  post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
+  pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
+  real_run: CM_REAL_RUN
+  run_cmd: CM_DOCKER_RUN_CMD
+  run_cmd_extra: CM_DOCKER_RUN_CMD_EXTRA
+  script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
+  skip_cm_sys_upgrade: CM_DOCKER_SKIP_CM_SYS_UPGRADE
+  push_image: CM_DOCKER_PUSH_IMAGE
+
+new_env_keys:
+- CM_DOCKERFILE_*
+
+post_deps:
+- enable_if_env:
+    CM_BUILD_DOCKER_IMAGE:
+    - 'yes'
+    - '1'
+  names:
+  - build-docker-image
+  tags: build,docker,image
+
+variations:
+  slim:
+    env:
+      CM_DOCKER_BUILD_SLIM: 'yes'

script/create-custom-cache-entry/_cm.yaml

@@ -0,0 +1,27 @@
+alias: create-custom-cache-entry
+uid: 485741440fbe4236
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- create
+- custom
+- cache
+- entry
+
+category: CM automation
+
+cache: true
+
+input_mapping:
+  env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY
+  env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2
+  path: CM_CUSTOM_CACHE_ENTRY_PATH
+  to: CM_CUSTOM_CACHE_ENTRY_PATH
+
+new_env_keys:
+- CM_CUSTOM_CACHE_ENTRY*
+
+print_env_at_the_end:
+  CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry"

script/create-custom-cache-entry/customize.py

@@ -0,0 +1,44 @@
+from cmind import utils
+import os
+import shutil
+
+def preprocess(i):
+
+    # CM script internal variables
+    env = i['env']
+
+    extra_cache_tags = []
+    if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '':
+        print ('')
+        extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ')
+
+        extra_cache_tags = extra_cache_tags_str.strip().split(',')
+
+    return {'return':0, 'add_extra_cache_tags':extra_cache_tags}
+
+def postprocess(i):
+
+    env = i['env']
+
+    path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip()
+
+    if path!='':
+        if not os.path.isdir(path):
+            os.makedirs(path)
+    else:
+       path = os.getcwd()
+
+    x = ''
+    env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '')
+    if env_key != '': x = env_key+'_'
+
+    env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path
+    env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path
+
+    env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '')
+    v = env.get(env_key2, '')
+    real_path = v if v != '' else path
+
+    env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path
+
+    return {'return': 0}

script/generate-docs-for-all-scripts.cmd

@@ -0,0 +1 @@
+cm doc script --repos=mlcommons@cm4mlops --output_dir=..

script/get-cuda-devices/_cm.yaml

@@ -0,0 +1,41 @@
+alias: get-cuda-devices
+uid: 7a3ede4d3558427a
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- get
+- cuda-devices
+
+cache: false
+
+can_force_cache: true
+
+category: CUDA automation
+
+clean_files:
+- tmp-run.out
+
+deps:
+- names:
+  - cuda
+  tags: get,cuda,_toolkit
+
+docker:
+  run: false
+  all_gpus: 'yes'
+  skip_run_cmd: 'no'
+  skip_cm_sys_upgrade: 'yes'
+  cm_repo_flags: '--checkout=dev'
+  use_host_group_id: 'yes'
+  image_tag_extra: '-cm-dev'
+
+new_env_keys:
+- CM_CUDA_DEVICE_*
+
+new_state_keys:
+- cm_cuda_device_prop
+
+print_files_if_script_error:
+- tmp-run.out