Sync master (#1354)

.github/workflows/test-cm-image-classification-onnx.yml

@@ -0,0 +1,36 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: 'CM: image classification with ONNX'
+
+on:
+  pull_request:
+    branches: [ "master", "main", "dev" ]
+    paths:
+      - '.github/workflows/test-image-classification-onnx.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.10", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cm pull repo --url=https://github.com/mlcommons/cm4mlops
+        cm run script --quiet --tags=get,sys-utils-cm
+    - name: Test image classification with ONNX
+      run: |
+        cmr "python app image-classification onnx" --quiet

.github/workflows/test-mlperf-inference-resnet50.yml → .github/workflows/test-cm-mlperf-inference-resnet50.yml

@@ -1,10 +1,10 @@
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: MLPerf inference resnet50
+name: 'CM: MLPerf inference resnet50'
 
 on:
   pull_request:
-    branches: [ "master", "dev" ]
+    branches: [ "master", "main", "dev" ]
     paths:
       - '.github/workflows/test-mlperf-inference-resnet50.yml'
       - '**'

.github/workflows/test-cm-script-features.yml

@@ -4,7 +4,7 @@ name: CM script automation features test
 
 on:
   pull_request:
-    branches: [ "master", "dev" ]
+    branches: [ "master", "main", "dev" ]
     paths:
       - '.github/workflows/test-cm-script-features.yml'
       - '**'

.github/workflows/test-cm.yml

@@ -5,7 +5,7 @@ name: CM test
 
 on:
   pull_request:
-    branches: [ "master" ]
+    branches: [ "master", "main", "dev" ]
     paths:
       - '.github/workflows/test-cm.yml'
       - 'cm/**'

.github/workflows/test-cmx-image-classification-onnx.yml

@@ -0,0 +1,36 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: 'CMX: image classification with ONNX'
+
+on:
+  pull_request:
+    branches: [ "master", "main", "dev" ]
+    paths:
+      - '.github/workflows/test-image-classification-onnx.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.10", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cmx pull repo --url=https://github.com/mlcommons/cm4mlops
+        cmx test core
+    - name: Test image classification with ONNX
+      run: |
+        cmx run script "python app image-classification onnx" --quiet

CONTRIBUTING.md

@@ -14,11 +14,13 @@ Modify the project in your own fork and issue a pull request once you want other
 to take a look at what you have done and discuss the proposed changes. 
 Ensure that cla-bot and other checks pass for your Pull requests.
 
-Collective Knowledge (CK) and Collective Mind (CM) were created by [Grigori Fursin](https://cKnowledge.org/gfursin),
-sponsored by cKnowledge.org and cTuning.org, and donated to MLCommons to benefit everyone. 
-Since then, this open-source technology (CM, CM4MLOps, CM4ABTF, CM4Research, etc)
-is being developed as a community effort thanks to all our volunteers, collaborators and contributors
-listed here in alphabetical order:
+Collective Knowledge (CK) and Collective Mind (CM)
+were created by [Grigori Fursin](https://arxiv.org/abs/2406.16791),
+sponsored by cKnowledge.org and cTuning.org, and donated to MLCommons 
+to benefit everyone. Since then, this open-source automation technology
+(CM/CMX, CM4MLOps, CM4ABTF, CM4Research, etc) is being extended 
+as a community effort thanks to all our volunteers, collaborators 
+and contributors listed here in alphabetical order:
 
 * Resmi Arjun
 * Omar Benjelloun (Google)
@@ -62,7 +64,7 @@ listed here in alphabetical order:
 * Arun Tejusve Raghunath Rajan (Cruise)
 * Vijay Janapa Reddi (Harvard University)
 * Andrew Reusch (OctoML)
-* Anandhu S (Kerala Technical University)
+* Anandhu Sooraj (Kerala Technical University)
 * Sergey Serebryakov (HPE)
 * Warren Schultz (Principled Technologies)
 * Amrutha Sheleenderan (Kerala Technical University)

README.md

@@ -6,58 +6,46 @@
 [![arXiv](https://img.shields.io/badge/arXiv-2406.16791-b31b1b.svg)](https://arxiv.org/abs/2406.16791)
 [![CM test](https://github.com/mlcommons/ck/actions/workflows/test-cm.yml/badge.svg)](https://github.com/mlcommons/ck/actions/workflows/test-cm.yml)
 [![CM script automation features test](https://github.com/mlcommons/ck/actions/workflows/test-cm-script-features.yml/badge.svg)](https://github.com/mlcommons/ck/actions/workflows/test-cm-script-features.yml)
-[![MLPerf inference resnet50](https://github.com/mlcommons/ck/actions/workflows/test-mlperf-inference-resnet50.yml/badge.svg)](https://github.com/mlcommons/ck/actions/workflows/test-mlperf-inference-resnet50.yml)
-[![Publish documentation site](https://github.com/mlcommons/ck/actions/workflows/publish.yaml/badge.svg)](https://github.com/mlcommons/ck/actions/workflows/publish.yaml)
+[![MLPerf inference resnet50](https://github.com/mlcommons/ck/actions/workflows/test-cm-mlperf-inference-resnet50.yml/badge.svg)](https://github.com/mlcommons/ck/actions/workflows/test-cm-mlperf-inference-resnet50.yml)
+[![CMX: image classification with ONNX](https://github.com/mlcommons/ck/actions/workflows/test-cmx-image-classification-onnx.yml/badge.svg)](https://github.com/mlcommons/ck/actions/workflows/test-cmx-image-classification-onnx.yml)
 
 ### About
 
-Collective Knowledge (CK) in a community project to develop open-source tools, platforms and automation recipes 
-that can help researchers and engineers automate their repetitive, tedious and time-consuming tasks
+[Collective Knowledge (CK)](https://cKnowledge.org) in an educational project
+to help researchers and engineers automate their repetitive, tedious and time-consuming tasks
 to build, run, benchmark and optimize AI, ML and other applications and systems 
 across diverse and continuously changing models, data, software and hardware.
 
-CK consists of several ongoing sub-projects:
+CK consists of several sub-projects:
 
-* [Collective Mind framework (CM)](cm) - a very light-weight Python-based framework with minimal dependencies
+* [Collective Mind framework (CM)](cm) - a very lightweight Python-based framework with minimal dependencies
   to help users implement, share and reuse cross-platform automation recipes to 
   build, benchmark and optimize applications on any platform
-  with any software and hardware. CM attempts to extends the `cmake` concept 
-  with reusable automation recipes and workflows written in plain Python or native OS scripts,
-  accessible via a human readable interface with simple tags,
-  and shareable in public and private repositories in a decentralized way.
-  Furthermore, in comparison with cmake, these automation recipes can not only detect missing code 
-  but also download artifacts (models, data sets), preprocess them, build missing 
-  dependencies, install them and run the final code on diverse platforms in a unified and automated way.
-  You can learn more about the CM concept from this [white paper](https://arxiv.org/abs/2406.16791) 
-  and the [ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339).
+  with any software and hardware. 
 
+  * [CM interface to run MLPerf inference benchmarks](https://docs.mlcommons.org/inference)
 
   * [CM4MLOPS](https://github.com/mlcommons/cm4mlops) - 
     a collection of portable, extensible and technology-agnostic automation recipes
     with a human-friendly interface (aka CM scripts) to unify and automate all the manual steps required to compose, run, benchmark and optimize complex ML/AI applications 
-    on diverse platforms with any software and hardware: see [online cKnowledge catalog](https://access.cknowledge.org/playground/?action=scripts),
+    on diverse platforms with any software and hardware: see [online catalog at CK playground](https://access.cknowledge.org/playground/?action=scripts),
     [online MLCommons catalog](https://docs.mlcommons.org/cm4mlops/scripts) 
-    and [source code](https://github.com/mlcommons/cm4mlops/blob/master/script).
 
   * [CM4ABTF](https://github.com/mlcommons/cm4abtf) - a unified CM interface and automation recipes
     to run automotive benchmark across different models, data sets, software and hardware from different vendors.
 
-* [Collective Knowledge Playground](https://access.cKnowledge.org) - an external platform being developed by [cKnowledge](https://cKnowledge.org)
-  to list CM scripts similar to PYPI, aggregate AI/ML Systems benchmarking results in a reproducible format with CM workflows, 
-  and organize [public optimization challenges and reproducibility initiatives](https://access.cknowledge.org/playground/?action=challenges) 
-  to find the most performance and cost-effective AI/ML Systems.
+* [CMX (the next generation of CM and CM4MLOps)](cm/docs/cmx) - we are developing the next generation of CM 
+    to make it simpler and more flexible based on user feedback. Please follow 
+    this project [here]( https://github.com/orgs/mlcommons/projects/46 ).
 
-  * [GUI to run modular benchmarks](https://access.cknowledge.org/playground/?action=howtorun) - such benchmarks 
-    are composed from [CM scripts](https://access.cknowledge.org/playground/?action=scripts)
-    and can run via a unified CM interface.
 
-  * [MLCommons docs to run MLPerf inference benchmarks from command line via CM](https://docs.mlcommons.org/inference)
+* [Collective Knowledge Playground](https://access.cKnowledge.org) - a unified platform
+  to list CM scripts similar to PYPI, aggregate AI/ML Systems benchmarking results in a reproducible format with CM workflows, 
+  and organize [public optimization challenges and reproducibility initiatives](https://access.cknowledge.org/playground/?action=challenges) 
+  to co-design more efficient and cost-effiective software and hardware for emerging workloads.
 
-### Incubator
+* [Artifact Evaluation](https://cTuning.org/ae) - automating artifact evaluation and reproducibility initiatives at ML and systems conferences.
 
-We are preparing new projects based on user feedback - please contact [Grigori Fursin](https://cKnowledge.org/gfursin) for more details:
-* [The next generation of CM](_incubator/cm-next-gen) *(prototyping stage)*
-* [Collaborative testing of MLPerf benchmarks](_incubator/cm4mlops-testing) *(brainstorming stage)*
 
 
 ### License
@@ -69,8 +57,25 @@ We are preparing new projects based on user feedback - please contact [Grigori F
 * Copyright (c) 2021-2024 MLCommons
 * Copyright (c) 2014-2021 cTuning foundation
 
+### Maintainers
+
+* CM/CM4Research: [Grigori Fursin](https://cKnowledge.org/gfursin)
+* CM4MLOps: [Arjun Suresh](https://github.com/arjunsuresh) and [Anandhu Sooraj](https://github.com/anandhu-eng)
+* CMX (the next generation of CM) [Grigori Fursin](https://cKnowledge.org/gfursin)
+
+### Citing our project
+
+If you found the CM automation framework helpful, kindly reference this article:
+[ [ArXiv](https://arxiv.org/abs/2406.16791) ], [ [BibTex](https://github.com/mlcommons/ck/blob/master/citation.bib) ].
+
+To learn more about the motivation behind CK and CM technology, please explore the following presentations:
 
-### Documentation
+* "Enabling more efficient and cost-effective AI/ML systems with Collective Mind, virtualized MLOps, MLPerf, Collective Knowledge Playground and reproducible optimization tournaments": [ [ArXiv](https://arxiv.org/abs/2406.16791) ]
+* ACM REP'23 keynote about the MLCommons CM automation framework: [ [slides](https://doi.org/10.5281/zenodo.8105339) ] 
+* ACM TechTalk'21 about Collective Knowledge project: [ [YouTube](https://www.youtube.com/watch?v=7zpeIVwICa4) ] [ [slides](https://learning.acm.org/binaries/content/assets/leaning-center/webinar-slides/2021/grigorifursin_techtalk_slides.pdf) ]
+
+
+### CM Documentation
 
 * [CM installation GUI](https://access.cknowledge.org/playground/?action=install)
 * [CM Getting Started Guide and FAQ](docs/getting-started.md)
@@ -83,21 +88,13 @@ We are preparing new projects based on user feedback - please contact [Grigori F
 * [CM and CK history](docs/history.md)
 
 
-### Citing CM
-
-If you found CM useful, please cite this article: 
-[ [ArXiv](https://arxiv.org/abs/2406.16791) ], [ [BibTex](https://github.com/mlcommons/ck/blob/master/citation.bib) ].
-
-You can learn more about the motivation behind these projects from the following articles and presentations:
-
-* "Enabling more efficient and cost-effective AI/ML systems with Collective Mind, virtualized MLOps, MLPerf, Collective Knowledge Playground and reproducible optimization tournaments": [ [ArXiv](https://arxiv.org/abs/2406.16791) ] 
-* ACM REP'23 keynote about the MLCommons CM automation framework: [ [slides](https://doi.org/10.5281/zenodo.8105339) ] 
-* ACM TechTalk'21 about automating research projects: [ [YouTube](https://www.youtube.com/watch?v=7zpeIVwICa4) ] [ [slides](https://learning.acm.org/binaries/content/assets/leaning-center/webinar-slides/2021/grigorifursin_techtalk_slides.pdf) ]
-
 ### Acknowledgments
 
-Collective Knowledge (CK) and Collective Mind (CM) were created by [Grigori Fursin](https://cKnowledge.org/gfursin),
-sponsored by cKnowledge.org and cTuning.org, and donated to MLCommons to benefit everyone. 
-Since then, this open-source technology (CM, CM4MLOps, CM4MLPerf, CM4ABTF, CM4Research, etc)
-is being developed as a community effort thanks to all our
-[volunteers, collaborators and contributors](https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md)! 
+The open-source Collective Knowledge project (CK, CM, CM4MLOps/CM4MLPerf, 
+CM4Research and CMX) was created by [Grigori Fursin](https://cKnowledge.org/gfursin)
+and sponsored by cTuning.org, OctoAI and HiPEAC.
+Grigori donated CK to MLCommons to benefit the community
+and to advance its development as a collaborative, community-driven effort.
+We thank MLCommons and FlexAI for supporting this project,
+as well as our dedicated [volunteers and collaborators](https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md)
+for their feedback and contributions!

cm/CHANGES.md

@@ -1,3 +1,120 @@
+## V3.4.2
+   - added utils.flatten_dict
+   - added utils.safe_int
+   - added utils.safe_float
+   - added utils.get_set
+   - added utils.digits
+
+## V3.4.1
+   - reduced Python min version in pyproject.toml to 3.7 for backwards compatibility
+
+## V3.4.0
+   - added pyproject.toml to install CM and deprecated setup.py
+
+## V3.3.4
+   - added utils.path2: 
+     add quotes if spaces in path
+   - added utils.update_dict_with_flat_key: 
+     update dictionary via flat key (x.y.z)
+   - added utils.get_value_from_dict_with_flat_key
+     get value from dict via flat key (x.y.z)
+   - added utils.load_module
+     universal python module loader
+
+## V3.3.3
+   - fixed CM logger issue
+
+## V3.3.2
+   - moved "CMX stack error" to -log
+   - improved error reporting
+
+## V3.3.1
+   - allow "cm/cmx pull repo {URL}" along with "cm/cmx pull repo --url={URL}" 
+   - CMX: added "automation_full_path" to -log
+
+## V3.2.9
+   - fixed minor bug with JSON console output
+   - fixed minor bug with console in cmind.x function
+
+## V3.2.8
+   - fixed Docker test for mlperf-inference with cm4mlops
+   - minor improvements
+
+## V3.2.7
+   - minor improvement in error handling and messages
+
+## V3.2.6
+   - Fix type of "unknown_keys" to list to be serializable
+     https://github.com/mlcommons/ck/issues/1335
+   - Added support to print warning instead of error
+     (useful for help)
+
+## V3.2.5
+   - CMX: improved logging
+   - CMX: improved error handling (show module path and line number)
+   - CMX: fixed bug when detecting unknown control flag
+   - CMX: do not change output to json if -j or --json
+          just print json in the end ...
+
+## V3.2.3
+   - added --new_branch to `cm pull repo` and `cm checkout repo`
+   - fixed a bug in `cm show repo` (removed dependency on cm4mlops 
+     and used subprocess.check_output)
+
+## V3.2.2
+   - fixed action `help` and flag `-h` to work with all automations:
+     https://github.com/mlcommons/ck/issues/1325
+
+## V3.2.1
+   - added `-f` to the CMX core to be backwards compatible 
+     with CM commands like `cm rm cache -f`
+
+## V3.2.0
+   - added `utils.test_input` to test if input has keys
+     and report them as error
+   - added `prefix_cmx` key to cmr.yaml to customize `cmx pull repo`
+   - improved CMX logging (-log and -logfile):
+     https://github.com/mlcommons/ck/issues/1317
+   - print control flags in help (cmx -h | cmx -help):
+     https://github.com/mlcommons/ck/issues/1318
+   - fail if control flag is not recognized:
+     https://github.com/mlcommons/ck/issues/1315
+   - added -repro flag to record various info to cmx-repro directory
+     https://github.com/mlcommons/ck/issues/1319
+   - print call stack when error > 32 to be able to trace error cause:
+     https://github.com/mlcommons/ck/issues/1320
+     can be combined with -log=debug and -logfile
+
+## V3.1.0
+   - simplified and changed process_input function API
+
+## V3.0.4
+   - splitted automation, artifact and artifacts into name and repo 
+     in the access function to simplify handling of artifacts:
+     https://github.com/mlcommons/ck/issues/1311
+
+## V3.0.3
+   - added -raise for https://github.com/mlcommons/ck/issues/1309
+   - added --extra_cmd_git and --extra_cmd_pip to cm/cmx pull repo
+     https://github.com/mlcommons/ck/issues/1308
+
+## V3.0.2
+   - fixed cmx init
+
+## V3.0.1
+   - fixed minor bug in CM core
+
+## V3.0.0
+   - added `min_cm_version` to the CM repo description (cmr.yaml)
+     to check compatibility of repositories with CM. 
+     It is needed to implement new features in CM core and repositories
+     requested by our users that may not work with previous CM versions.
+     Hence, starting a new version.
+
+   - started prototyping a simpler and cleaner version of `cmind.access()`.
+     it should not influence existing automations and workflows 
+     and will co-exist in the future.
+
 ## V2.4.0
    - added `install_python_requirements` to the CM repo description (cmr.yaml)
      to install requirements to a current python with CM installation if needed