Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync with dev (new) #32

Merged
merged 1 commit into from
May 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions LICENSE.third-party.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This CM repository may contain CM scripts with third-party files licensed under Apache2, BSD or MIT license.
8 changes: 8 additions & 0 deletions automation/cfg/README-extra.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Examples:

```bash
cm set cfg default
cm set cfg default --key.script.silent
cm set cfg default --key.script.silent-

```
34 changes: 34 additions & 0 deletions automation/script/docker_repro_example/ubuntu-23.04.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
FROM ubuntu:23.04

# Automatically generated by the CM workflow automation meta-framework
# https://github.com/mlcommons/ck

LABEL github=""
LABEL maintainer=""
LABEL license=""

SHELL ["/bin/bash", "-c"]

ARG CM_GH_TOKEN
ARG CM_ADD_DOCKER_GROUP_ID=""

# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes
# Install system dependencies
RUN apt-get update -y
RUN apt-get install -y python3 python3-pip git sudo wget

# Setup docker environment
ENTRYPOINT ["/bin/bash", "-c"]
ENV TZ="US/Pacific"
ENV PATH="${PATH}:/home/cmuser/.local/bin"
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone

# Setup docker user
RUN groupadd ${CM_ADD_DOCKER_GROUP_ID} cm
RUN useradd -g cm --create-home --shell /bin/bash cmuser
RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
USER cmuser:cm
WORKDIR /home/cmuser

# Install python packages
RUN python3 -m pip install --user cmind requests giturlparse tabulate --break-system-packages
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash
10 changes: 10 additions & 0 deletions cfg/docker-basic-configurations/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
alias: docker-basic-configurations
uid: d2a0c5bb17664c93

automation_alias: cfg
automation_uid: 88dce9c160324c5d

tags:
- docker
- basic
- configurations
9 changes: 9 additions & 0 deletions cfg/docker-basic-configurations/basic-archlinux.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
uid: 9960e9fb3cb24cb3

name: "Basic ArchLinux"

input:
docker_base_image: 'archlinux'
docker_os: arch
docker_os_version: 'latest'

9 changes: 9 additions & 0 deletions cfg/docker-basic-configurations/basic-rhel-9.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
uid: 27b4afcdd8e042e8

name: "Basic RHEL 9"

input:
docker_base_image: 'registry.access.redhat.com/ubi9'
docker_os: 'rhel'
docker_os_version: '9'

9 changes: 9 additions & 0 deletions cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
uid: 59311e6098c14b21

name: "Basic Ubuntu 20.04"

input:
docker_base_image: 'ubuntu:20.04'
docker_os: ubuntu
docker_os_version: '20.04'

9 changes: 9 additions & 0 deletions cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
uid: 614aa48d90724835

name: "Basic Ubuntu 22.04"

input:
docker_base_image: 'ubuntu:22.04'
docker_os: ubuntu
docker_os_version: '22.04'

9 changes: 9 additions & 0 deletions cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
uid: 276bd8ab39324f5f

name: "Basic Ubuntu 23.04"

input:
docker_base_image: 'ubuntu:23.04'
docker_os: ubuntu
docker_os_version: '23.04'

32 changes: 32 additions & 0 deletions script/app-image-corner-detection/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
alias: app-image-corner-detection
uid: 998ffee0bc534d0a

automation_alias: script
automation_uid: 5b4e0237da074764

category: Modular application pipeline

deps:
- tags: detect,os

- tags: detect,cpu

- tags: download,file,_url.https://cKnowledge.org/ai/data/data.pgm
md5sum: 0af279e557a8de252d7ff0751a999379
force_cache: false


posthook_deps:
- skip_if_env:
CM_SKIP_COMPILE:
- 'on'
tags: compile,cpp-program
- skip_if_env:
CM_SKIP_RUN:
- 'on'
tags: benchmark-program

tags:
- app
- image
- corner-detection
3 changes: 3 additions & 0 deletions script/benchmark-program/README-extra.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
This is a universal script to run and profile programs.

It is a part of our universal benchmarking and optimization roadmap: https://github.com/mlcommons/cm4mlops/issues/23
46 changes: 46 additions & 0 deletions script/build-docker-image/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
alias: build-docker-image
uid: 2c3c4ba2413442e7

automation_alias: script
automation_uid: 5b4e0237da074764

cache: false

category: Docker automation

tags:
- build
- docker
- image
- docker-image
- dockerimage

default_env:
CM_DOCKER_IMAGE_REPO: local
CM_DOCKER_IMAGE_TAG: latest

input_mapping:
cache: CM_DOCKER_CACHE
cm_repo: CM_MLOPS_REPO
docker_os: CM_DOCKER_OS
docker_os_version: CM_DOCKER_OS_VERSION
dockerfile: CM_DOCKERFILE_WITH_PATH
gh_token: CM_GH_TOKEN
image_name: CM_DOCKER_IMAGE_NAME
image_repo: CM_DOCKER_IMAGE_REPO
image_tag: CM_DOCKER_IMAGE_TAG
post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
real_run: CM_REAL_RUN
script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
push_image: CM_DOCKER_PUSH_IMAGE

new_env_keys:
- CM_DOCKER_*

prehook_deps:
- enable_if_env:
CM_BUILD_DOCKERFILE:
- 'yes'
- '1'
tags: build,dockerfile
67 changes: 67 additions & 0 deletions script/build-dockerfile/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
alias: build-dockerfile
uid: e66a7483230d4641

automation_alias: script
automation_uid: 5b4e0237da074764

tags:
- build
- dockerfile

cache: false

category: Docker automation

default_env:
CM_DOCKER_BUILD_SLIM: 'no'
CM_DOCKER_IMAGE_EOL: '

'
CM_DOCKER_OS: ubuntu

input_mapping:
build: CM_BUILD_DOCKER_IMAGE
cache: CM_DOCKER_CACHE
cm_repo: CM_MLOPS_REPO
cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO
cm_repos: CM_DOCKER_EXTRA_CM_REPOS
comments: CM_DOCKER_RUN_COMMENTS
copy_files: CM_DOCKER_COPY_FILES
docker_base_image: CM_DOCKER_IMAGE_BASE
docker_os: CM_DOCKER_OS
docker_os_version: CM_DOCKER_OS_VERSION
extra_sys_deps: CM_DOCKER_EXTRA_SYS_DEPS
fake_docker_deps: CM_DOCKER_FAKE_DEPS
fake_run_option: CM_DOCKER_FAKE_RUN_OPTION
file_path: CM_DOCKERFILE_WITH_PATH
gh_token: CM_GH_TOKEN
image_repo: CM_DOCKER_IMAGE_REPO
image_tag: CM_DOCKER_IMAGE_TAG
package_manager_update_cmd: CM_PACKAGE_MANAGER_UPDATE_CMD
pip_extra_flags: CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS
post_file: DOCKER_IMAGE_POST_FILE
post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
real_run: CM_REAL_RUN
run_cmd: CM_DOCKER_RUN_CMD
run_cmd_extra: CM_DOCKER_RUN_CMD_EXTRA
script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
skip_cm_sys_upgrade: CM_DOCKER_SKIP_CM_SYS_UPGRADE
push_image: CM_DOCKER_PUSH_IMAGE

new_env_keys:
- CM_DOCKERFILE_*

post_deps:
- enable_if_env:
CM_BUILD_DOCKER_IMAGE:
- 'yes'
- '1'
names:
- build-docker-image
tags: build,docker,image

variations:
slim:
env:
CM_DOCKER_BUILD_SLIM: 'yes'
27 changes: 27 additions & 0 deletions script/create-custom-cache-entry/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
alias: create-custom-cache-entry
uid: 485741440fbe4236

automation_alias: script
automation_uid: 5b4e0237da074764

tags:
- create
- custom
- cache
- entry

category: CM automation

cache: true

input_mapping:
env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY
env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2
path: CM_CUSTOM_CACHE_ENTRY_PATH
to: CM_CUSTOM_CACHE_ENTRY_PATH

new_env_keys:
- CM_CUSTOM_CACHE_ENTRY*

print_env_at_the_end:
CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry"
44 changes: 44 additions & 0 deletions script/create-custom-cache-entry/customize.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
from cmind import utils
import os
import shutil

def preprocess(i):

# CM script internal variables
env = i['env']

extra_cache_tags = []
if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '':
print ('')
extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ')

extra_cache_tags = extra_cache_tags_str.strip().split(',')

return {'return':0, 'add_extra_cache_tags':extra_cache_tags}

def postprocess(i):

env = i['env']

path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip()

if path!='':
if not os.path.isdir(path):
os.makedirs(path)
else:
path = os.getcwd()

x = ''
env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '')
if env_key != '': x = env_key+'_'

env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path
env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path

env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '')
v = env.get(env_key2, '')
real_path = v if v != '' else path

env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path

return {'return': 0}
1 change: 1 addition & 0 deletions script/generate-docs-for-all-scripts.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cm doc script --repos=mlcommons@cm4mlops --output_dir=..
41 changes: 41 additions & 0 deletions script/get-cuda-devices/_cm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
alias: get-cuda-devices
uid: 7a3ede4d3558427a

automation_alias: script
automation_uid: 5b4e0237da074764

tags:
- get
- cuda-devices

cache: false

can_force_cache: true

category: CUDA automation

clean_files:
- tmp-run.out

deps:
- names:
- cuda
tags: get,cuda,_toolkit

docker:
run: false
all_gpus: 'yes'
skip_run_cmd: 'no'
skip_cm_sys_upgrade: 'yes'
cm_repo_flags: '--checkout=dev'
use_host_group_id: 'yes'
image_tag_extra: '-cm-dev'

new_env_keys:
- CM_CUDA_DEVICE_*

new_state_keys:
- cm_cuda_device_prop

print_files_if_script_error:
- tmp-run.out
Loading
Loading