diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 0902c119a9..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,27 +0,0 @@ -# TBD: need to cover new code and CK modules - -[run] -source = ck.kernel - -[report] - -# Regexes for lines to exclude from consideration -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't complain about missing debug-only code: - def __repr__ - if self\.debug - - # Don't complain if tests don't hit defensive assertion code: - raise AssertionError - raise NotImplementedError - - # Don't complain if non-runnable code isn't run: - if 0: - if __name__ == .__main__.: - - if r\['return'\]>0: - - except Exception as e: diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..06ce1b0991 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# These owners will be the default owners for everything in the repo. +# Unless a later match takes precedence,they will be requested for review when someone opens a pull request. +* @mlcommons/wg-ck + +/CODEOWNERS @mlcommons/staff diff --git a/.github/workflows/check-all-broken-links.md b/.github/workflows/check-all-broken-links.md new file mode 100644 index 0000000000..6ade4bfd25 --- /dev/null +++ b/.github/workflows/check-all-broken-links.md @@ -0,0 +1,17 @@ +name: Check .md README files for broken links + +on: + push: [main] + +jobs: + markdown-link-check: + runs-on: ubuntu-latest + # check out the latest version of the code + steps: + - uses: actions/checkout@v3 + + # Checks the status of hyperlinks in .md files in verbose mode + - name: Check links + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' diff --git a/.github/workflows/check-broken-links.md b/.github/workflows/check-broken-links.md new file mode 100644 index 0000000000..a753ec75ba --- /dev/null +++ b/.github/workflows/check-broken-links.md @@ -0,0 +1,17 @@ +name: Check .md README files for broken links + +on: [pull_request] + +jobs: + markdown-link-check: + runs-on: ubuntu-latest + # check out the latest version of the code + steps: + - uses: actions/checkout@v3 + + # Checks the status of hyperlinks in .md files in verbose mode + - name: Check links + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + check-modified-files-only: 'yes' diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml new file mode 100644 index 0000000000..c0e1544d43 --- /dev/null +++ b/.github/workflows/cla.yml @@ -0,0 +1,36 @@ + +name: "cla-bot" +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened,closed,synchronize] + +jobs: + cla-check: + runs-on: ubuntu-latest + steps: + - name: "MLCommons CLA bot check" + if: (github.event.comment.body == 'recheck') || github.event_name == 'pull_request_target' + # Alpha Release + uses: mlcommons/cla-bot@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # the below token should have repo scope and must be manually added by you in the repository's secret + PERSONAL_ACCESS_TOKEN : ${{ secrets.MLCOMMONS_BOT_CLA_TOKEN }} + with: + path-to-signatures: 'cla-bot/v1/cla.json' + # branch should not be protected + branch: 'main' + allowlist: user1,bot* + remote-organization-name: mlcommons + remote-repository-name: systems + + #below are the optional inputs - If the optional inputs are not given, then default values will be taken + #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository) + #remote-repository-name: enter the remote repository name where the signatures should be stored (Default is storing the signatures in the same repository) + #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' + #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo' + #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' + #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' + #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' diff --git a/.github/workflows/test-cm-script-features.yml b/.github/workflows/test-cm-script-features.yml new file mode 100644 index 0000000000..674d8b03c1 --- /dev/null +++ b/.github/workflows/test-cm-script-features.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CM script automation features test + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-cm-script-features.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.11", "3.10", "3.9", "3.8"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test CM Script Features + run: | + python script/test-cm-core/src/script/test_install.py + python script/test-cm-core/src/script/test_docker.py + python script/test-cm-core/src/script/test_features.py diff --git a/.github/workflows/test-cm-scripts.yml b/.github/workflows/test-cm-scripts.yml new file mode 100644 index 0000000000..8fec3213c2 --- /dev/null +++ b/.github/workflows/test-cm-scripts.yml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CM script automation test + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-cm-scripts.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test CM Script Automation + run: | + python script/test-cm-core/src/script/test_deps.py diff --git a/.github/workflows/test-cm-tutorial-retinanet.yml b/.github/workflows/test-cm-tutorial-retinanet.yml new file mode 100644 index 0000000000..b08378a1b3 --- /dev/null +++ b/.github/workflows/test-cm-tutorial-retinanet.yml @@ -0,0 +1,35 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CM tutorial retinanet + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-cm-tutorial-retinanet.yml' + - '**' + - '!**.md' +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test CM Tutorial Retinanet + run: | + python script/test-cm-core/src/tutorials/test_tutorial_retinanet.py diff --git a/.github/workflows/test-cm-tutorial-tvm-pip.yml b/.github/workflows/test-cm-tutorial-tvm-pip.yml new file mode 100644 index 0000000000..c12f7308ec --- /dev/null +++ b/.github/workflows/test-cm-tutorial-tvm-pip.yml @@ -0,0 +1,57 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CM tutorial tvm pip install + +on: + pull_request: + branches: [ "main", "test" ] + paths: + - '.github/workflows/test-cm-tutorial-tvm-pip.yml' + - '**' + - '!**.md' + +jobs: + test_vm_runtime: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test CM Tutorial TVM pip install with VirtualMachine Runtime + run: | + python script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py + + test_ge_runtime: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test CM Tutorial TVM pip install with GraphExecutor Runtime + run: | + python script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py diff --git a/.github/workflows/test-cm-tutorial-tvm.yml b/.github/workflows/test-cm-tutorial-tvm.yml new file mode 100644 index 0000000000..0f0e73c3b8 --- /dev/null +++ b/.github/workflows/test-cm-tutorial-tvm.yml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CM tutorial tvm + +on: + pull_request: + branches: [ "test" ] + paths: + - '.github/workflows/test-cm-tutorial-tvm.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test CM Tutorial TVM + run: | + python script/test-cm-core/src/tutorials/test_tutorial_tvm.py diff --git a/.github/workflows/test-cm.yml b/.github/workflows/test-cm.yml new file mode 100644 index 0000000000..30b2162e2c --- /dev/null +++ b/.github/workflows/test-cm.yml @@ -0,0 +1,71 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CM test + +on: + pull_request: + branches: [ "master" ] + paths: + - '.github/workflows/test-cm.yml' + - 'cm/**' + - '!cm/**.md' + +jobs: + build: + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + on: [ubuntu-latest] + runs-on: "${{ matrix.on }}" + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install --ignore-installed --verbose pip setuptools + cd cm + python setup.py install + python -m cmind + # cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 cm/cmind --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 cm/cmind --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test + run: | + python tests/test_cm.py + + test_cm: + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + on: [ubuntu-latest, windows-latest] + runs-on: "${{ matrix.on }}" + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install cmind + # cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} + - name: Test CM + run: | + python tests/test_cm.py + diff --git a/.github/workflows/test-image-classification-onnx.yml b/.github/workflows/test-image-classification-onnx.yml new file mode 100644 index 0000000000..3c732cb391 --- /dev/null +++ b/.github/workflows/test-image-classification-onnx.yml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: image classification with ONNX + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-image-classification-onnx.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test image classification with ONNX + run: | + cmr "python app image-classification onnx" --quiet diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml new file mode 100644 index 0000000000..db4ea40d28 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch) + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # 3.12 didn't work on 20240305 - need to check + python-version: [ "3.11", "3.9" ] + backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ] + precision: [ "int8", "fp32" ] + exclude: + - backend: tf + - backend: pytorch + - backend: onnxruntime + - precision: fp32 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test MLPerf Inference Bert (DeepSparse, TF, ONNX, PyTorch) + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml new file mode 100644 index 0000000000..1207f3394a --- /dev/null +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference GPT-J + +on: + pull_request: + branches: [ "main1", "dev1" ] + paths: + - '.github/workflows/test-mlperf-inference-gptj.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9" ] + backend: [ "pytorch" ] + precision: [ "bfloat16" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test MLPerf Inference GPTJ + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=gptj --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml new file mode 100644 index 0000000000..294356e590 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference MLCommons C++ ResNet50 + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9" ] + llvm-version: [ "15.0.6", "16.0.4", "17.0.6" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + cm run script --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }} + - name: Test MLPerf Inference MLCommons C++ ResNet50 + run: | + cmr "app mlperf inference mlcommons cpp" -v --quiet diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml new file mode 100644 index 0000000000..cf09c53241 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -0,0 +1,43 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference resnet50 + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-resnet50.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + env: + CM_INDEX: "on" + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9" ] + backend: [ "onnxruntime", "tf" ] + implementation: [ "python", "cpp" ] + exclude: + - backend: tf + implementation: cpp + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test MLPerf Inference ResNet50 + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --hw_name=default --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml new file mode 100644 index 0000000000..513bb1abe6 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -0,0 +1,41 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference retinanet + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-retinanet.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9" ] + backend: [ "onnxruntime", "pytorch" ] + implementation: [ "python", "cpp" ] + exclude: + - backend: pytorch + implementation: cpp + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --hw_name=default --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet -v --target_qps=1 diff --git a/.github/workflows/test-mlperf-inference-rnnt.yml b/.github/workflows/test-mlperf-inference-rnnt.yml new file mode 100644 index 0000000000..8e97fc23c3 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-rnnt.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference rnnt + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-rnnt.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9" ] + backend: [ "pytorch" ] + precision: [ "fp32" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test MLPerf Inference RNNT + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_performance-only --submitter="cTuning" --model=rnnt --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=5 -v --quiet diff --git a/.github/workflows/test-mlperf-inference-tvm.yml b/.github/workflows/test-mlperf-inference-tvm.yml new file mode 100644 index 0000000000..7b11963929 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-tvm.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +# We are doing similar test on our tvm tutorial test. So, this test is not necessary +name: MLPerf inference resnet50 using TVM. + +on: + pull_request: + branches: [ "tvm-more" ] + paths: + - '.github/workflows/test-mlperf-inference-tvm.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.10" ] + backend: [ "tvm-onnx" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: MLPerf Inference ResNet50 using TVM + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds --hw_name=default --model=resnet50 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --target_qps=1 -v --quiet diff --git a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml new file mode 100644 index 0000000000..6ed90e67e6 --- /dev/null +++ b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf loadgen with HuggingFace bert onnx fp32 squad model + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml' + - '**' + - '!**.md' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12", "3.9" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm run script --quiet --tags=get,sys-utils-cm + - name: Test MLPerf loadgen with HuggingFace bert onnx fp32 squad model + run: | + cmr "python app loadgen-generic _onnxruntime _custom _huggingface _model-stub.ctuning/mlperf-inference-bert-onnx-fp32-squad-v1.1" --adr.hf-downloader.model_filename=model.onnx --quiet diff --git a/.github/workflows/test-qaic-compute-sdk-build.yml b/.github/workflows/test-qaic-compute-sdk-build.yml new file mode 100644 index 0000000000..b2e774a440 --- /dev/null +++ b/.github/workflows/test-qaic-compute-sdk-build.yml @@ -0,0 +1,35 @@ +name: Test Compilation of QAIC Compute SDK (build LLVM from src) + +on: + schedule: + - cron: "1 1 * * 2" + +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + llvm-version: [ "12.0.1", "13.0.1", "14.0.0", "15.0.6", "16.0.4", "17.0.6" ] + exclude: + - llvm-version: "13.0.1" + - llvm-version: "14.0.0" + - llvm-version: "15.0.6" + - llvm-version: "16.0.4" + - llvm-version: "17.0.6" + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo mlcommons@ck + cm run script --tags=get,sys-utils-cm --quiet + + - name: Test QAIC Compute SDK for compilation + run: | + cm run script --tags=get,qaic,compute,sdk --adr.llvm.version=${{ matrix.llvm-version }} --quiet diff --git a/.github/workflows/test-qaic-software-kit.yml b/.github/workflows/test-qaic-software-kit.yml new file mode 100644 index 0000000000..e3a186daae --- /dev/null +++ b/.github/workflows/test-qaic-software-kit.yml @@ -0,0 +1,41 @@ +name: Test QAIC Software kit Compilation + +on: + schedule: + - cron: "1 1 * * 1" + +jobs: + build_ubuntu_20_04: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + compiler: [ "gcc", "llvm" ] + llvm-version: [ "12.0.1", "13.0.1", "14.0.0", "15.0.6" ] + exclude: + - llvm-version: "12.0.1" + - llvm-version: "13.0.1" + - llvm-version: "14.0.0" + compiler: "gcc" + - llvm-version: "15.0.6" + compiler: "gcc" + include: + - llvm-version: " " + compiler: "gcc" + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo mlcommons@ck + cm run script --tags=get,sys-utils-cm --quiet + + - name: Test Software Kit for compilation on Ubuntu 20.04 + run: | + cm run script --tags=get,qaic,software,kit --adr.compiler.tags=${{ matrix.compiler }} --adr.compiler.version=${{ matrix.llvm-version }} --quiet + cm run script --tags=get,qaic,software,kit --adr.compiler.tags=${{ matrix.compiler }} --adr.compiler.version=${{ matrix.llvm-version }} --quiet diff --git a/.gitignore b/.gitignore index c87fc6d308..96523fae44 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,18 @@ build/* MANIFEST *.pyc __pycache__ +develop-eggs/ +dist/ +eggs/ +.eggs/ +lib/ +lib64/ +sdist/ +wheels/ .cache/ .coverage htmlcov *tmp/ *tmp-ck-*/ +local/cache/ + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6c7a13491d..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -os: linux -dist: xenial -language: python - -python: -# - "2.7" - - "3.6" - - "3.7" - -addons: - apt: - packages: - - python-pip - sources: - - ubuntu-toolchain-r-test - -before_install: - - | - if [ $TRAVIS_OS_NAME == "osx" ]; then - brew update # this swaps python versions and makes 3 the default one - - if [ "$WANTED_PYTHON_VERSION" == "2.7" ]; then - brew reinstall python\@2 || brew link --overwrite python\@2 # install and link python2 and pip2 to /usr/local/bin - export PATH=/usr/local/opt/python\@2/bin:$PATH - export PYTHON_EXE=python - export PIP_EXE=pip - else - brew reinstall python # install and link python3 and pip3 to /usr/local/bin - export PATH=/usr/local/opt/python/bin:$PATH - export PYTHON_EXE=python3 - export PIP_EXE=pip3 - fi - - export CK_PLATFORM_NAME="generic-macos " # used later by CK - else - sudo apt-get install python-pip - export WANTED_PYTHON_VERSION=$TRAVIS_PYTHON_VERSION # since Python is supported in Linux, get it from Travis - export CK_PLATFORM_NAME="generic-linux " # used later by CK (note the trailing space to make the choice unique) - export PYTHON_EXE=python - export PIP_EXE=pip - fi - -install: - - CWD=`pwd` - - $PIP_EXE install coveralls - - $PIP_EXE install pyyaml - - echo "TRAVIS_OS_NAME=${TRAVIS_OS_NAME}, WANTED_PYTHON_VERSION=${WANTED_PYTHON_VERSION}" - - which "${PYTHON_EXE}" - - ${PYTHON_EXE} --version - -script: - - ${PYTHON_EXE} -m pip install --ignore-installed --verbose pip setuptools - - ${PYTHON_EXE} setup.py install - - ck version - - coverage run -m tests.test - - ck pull repo:octoml@mlops - - echo "$CK_PLATFORM_NAME" | ck detect platform.os --update_platform_init - - ck detect soft:compiler.python --full_path=`which ${PYTHON_EXE}` - - ck install package --tags=lib,python-package,numpy - - ck install package --tags=lib,python-package,scipy - - ck install package --tags=lib,python-package,pillow - - ck install package:lib-tensorflow-1.13.1-cpu - - ck install package:imagenet-2012-val-min - - ck install package --tags=imagenet,2012,aux,from.dividiti # (from.berkeley doesn't work) - - ck install package --tags=dataset,imagenet,preprocessed,using-pillow,first.20 - - ck install package --quiet --tags=lib,python-package,onnx - - ck install package --quiet --tags=model,image-classification,onnx,mobilenet,non-quantized,nchw,downloaded - - ck install package --quiet --tags=model,onnx,mobilenet,nhwc,converted - - ck install package --quiet --tags=model,onnx,resnet,nchw,converted - - ck install package --quiet --tags=model,onnx,resnet,nhwc,converted - - ck install package --quiet --tags=lib,python-package,onnxruntime,cpu - - - ck detect soft:compiler.gcc --full_path=`which gcc` - - ck install package --tags=tool,cmake,prebuilt --quiet - - - ck install package --quiet --tags=mlperf,inference,src,r1.0 - - - ck install package --tags=lib,python-package,absl - - - ck install package --tags=lib,python-package,mlperf,loadgen - - ck install package --tags=lib,mlperf,loadgen,static - - - ck show env - - - ck compile program:cbench-automotive-susan --speed - - ck run program:cbench-automotive-susan --cmd_key=corners --dataset_uoa=image-pgm-0001 --repeat=1 --env.MY_ENV=123 --env.TEST=xyz --return_error_if_external_fail - - - ck run program:image-classification-onnx-py --quiet --dep_add_tags.weights=mobilenet,converted --cmd_key=barebones --env.CK_BATCH_SIZE=3 --return_error_if_external_fail - - ck run program:image-classification-onnx-py --dep_add_tags.weights=mobilenet,downloaded --cmd_key=preprocessed --env.CK_BATCH_COUNT=2 --return_error_if_external_fail - - ck run program:image-classification-onnx-py --dep_add_tags.weights=resnet,nchw,converted --cmd_key=preprocessed --return_error_if_external_fail - - ck run program:image-classification-onnx-py --dep_add_tags.weights=resnet,nhwc,converted --cmd_key=preprocessed --return_error_if_external_fail - - ck run program:image-classification-onnx-py --cmd_key=preprocessed --dep_add_tags.weights=resnet,nchw,converted --env.CK_IMAGE_FILE=ILSVRC2012_val_00000003.JPEG --return_error_if_external_fail - -after_success: coveralls diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000000..bcbe2d35b4 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,236 @@ +### 20240429 + * Added `cm set cfg` automation. For example, we can set default CM script to silent + using `cm set cfg default --key.script.silent` + * Added key `script_entry_repo_to_report_errors` to CM script meta + to customize where to report errors instead of the default repo. + For example, issues with the CM script `run-mlperf-inference-app` + should be reported at https://github.com/mlcommons/inference . + * Added saving running script from different deps without CM to tmp-run-without-cm.bat. + Example: `cmr "app image corner-detection" --debug-script-tags=benchmark,program` + * Generate Docker container sample during --repro (prototype) + + +### 20240427 + * Added cm run script --print_versions to print resolved versions of dependencies at the end of the run + +### 20240426 + * Improved cm run script --repro function to dump dependencies, versions and generate README + +### 20240425 + * CM script automation: fixed dumping versions (variations didn't have _ prefix) + +--- + +Since March 2023, all updates to CM automations are submitted via PRs. +You can follow our PRs at +* https://github.com/ctuning/mlcommons-ck/commits/master +* https://github.com/mlcommons/ck/pulls?q=is%3Apr+is%3Aclosed . + +--- + +### 20230214 + * experiment and graph gui are working now + +### 20230206: + * started prototyping cm run experiment + +### 20230123: + * added simple GUI to CM scripts + +### 20221206: + * added "script_name" to the CM "script" meta to specify any native script name + * added "--script_name" to "cm add script {alias} --script_name=my-native-script.sh" + +### 20221206: + * added CM_SCRIPT_EXTRA_CMD to force some flags to all scripts + +### 20221202: + * major updates for Windows (CL, CUDA, etc) + +### 20221111: + * various fixes for Student Cluster Competition at SuperComputing'22 + +### 20221110: + * added support to push MLPerf results to W&B dashboard + +### 20221103: + * added "cm json2yaml utils" and "cm yaml2json utils" + +### 20221024: + * added --verbose and --time to "cm run script" + +### 20221017: + * removed the need for echo-off script + +### 20221010: + * added cm run script --debug-script-tags to run cmd/bash before native script + * added cm run script --shell to set env and run shell after script execution + +* 20221007: + * added script template (used when adding new scripts) + * major clean up of all scripts + +### 20220916: + * treat alias as tags if spaces: + cm run script "get compiler" is converted to cm run script --tags=get,compiler + * improved gcc detection + * refactored "cm run script" to skip deps in cache if needed + +### 20220906 + * added --print_env flag to "cm run script" to print aggregated env + before running native scripts + * various fixes to support MLPerf automation + +### 20220823 + * various fixes for universal MLPerf inference submission automation + +### 20220803 + * various fixes for TVM and image classification + +### 20220802 + * added "run_script_after_post_deps" to script meta to run script after post deps + (useful to activate python virtual env) + * added "activate-python-venv" script to make it easier to debug Python deps installation + +### 20220722 + * added --accept-license and --skip-system-deps + (converted to env CM_ACCEPT_LICENSE ("True") and CM_SKIP_SYSTEM_DEPS ("True")) + +### 20220719 + * moved relatively stable MLOps automation scripts here + +### 20220718 + * fixed local_env_keys in get-python3 + * added new_env_only_keys to meta to specify which env to keep + * fixed problem with adding tags from the selected script during caching + * added --skip-compile and --skip-run to script (converted to env CM_SKIP_COMPILE and CM_SKIP_RUN) + * fixed local_env_keys in get-python3 + * added new_env_only_keys to get-python3 + +### 20220713 + * added local_env_keys to meta + * added "env" dict to os_info + +### 20220712 + * major script refactoring to support cache tags update from deps + * fixed version min/max propagations in deps + * improvements to support tags from deps + * added tags from deps (python, llvm) + +### 20220708 + * various fixes to handle versions (min/max/default) + * various fixes to avoid contamination of ENV from other scripts + * various fixes to handle versions (min/max/default) + +### 20220705 + * fixes for remembered selections + * added --skip-remembered-selections to "cm run script" + +### 20220704 + * fixed a bug with searching for scripts with variations + * added the possibility to update deps from pre/post processing + * added --extra-cache-tags and --name for "cm run script" + * added prototype of selection caching + * fixed get-python-venv + +### 20220701 + * added dummy "cm test script" + * added "--env" to "cm show cache" to show env and state + * added "cm show cache" + +### 20220629 + * added "detect_version_using_script" in script used to detect python packages + * major fix to properly support multiple scripts with the same tags, caching, selection, etc + * fixed a bug in version comparison (converting string to int) + * added recording of "version" to cache meta + +### 20220628 + * fixed local_env with deps + +### 20220623 + * important update of versions logic + +### 20220621 + * added support for --quiet + * changed CM_NEED_VERSION to CM_VERSION + * added CM_VERSION_MIN, CM_VERSION_MAX + * added cm compare_versions utils --version1=... --version2=... + * added support to detect min/max/correct versions + +### 20220617 + * fixed logic to handle variations (-_): https://github.com/mlcommons/ck/issues/243 + +### 20220616 + * changed "cached" to "cache" automation + +### 20220615 + * major update of script (remove parallel env/new_env and state/new_state). + keep global env & state and detect changes automatically + * major simplification of "script" + * removed "installed" to be more understandable + * added "cached" to be more understandable + +### 20220609 + * added "versions" key to the CM script meta + it works similar to "variations" and is forced by --version + * changed "ic" to "script" in "experiment" automation + +### 20220608 + * updated "variations" logic in "script"! + meta['default_variation'] (str): only one of many + meta['default_variations'] (list): multiple choices + * deprecated "ic" automation. Use "script" instead! + +### 20220607 + * added strip_folders to utils/unzip_file + * fixed minor bugs in CM script + +### 20220606 + * added "name" key to deps (list of names and UIDs) + * added "add_deps_tags" in variations and in CMD ({"name":"tag(s)"}) + * added "deps" to variations to be merged with the list of current deps + * added --input and --output for cm run script converted to env CM_INPUT and CM_OUTPUT + useful to create interactive CM scripts to process files + * Added prototype-test-deps-variations-tags to play with deps, variations, tags + +### 20220605 + * clean tmp files in "script" automation by default and keep them using --dirty flag + +### 20220603 + * added "skip" and "deps" to postprocess to call other scripts. + For example call install LLVM if detect LLVM fails... + * added "script" automation to substitute less intuitive "ic" + * Improved LLVM detection and installation + * Added example of image corner detection + * Added updated script entries + +### 20220601 + * added version, path, skip_install and post_deps to IC + * added --new to IC to detect new components + * Updating mechanisms to install and/or detect LLVM + * added support to install prebuilt LLVM for Linux, MacOs, Windows + +### 20220530 + * updated ic automation to read tmp-run-state.json + and merge it with the "new_state" dict + +### 20220524 + * changed directory ck2-repo-mlops to cm-devops + +### 20220517 + * Changed CM_PATH_LIST to +PATH + * Added general support for +ENV that is expanded to ENV=val1;val2;...:${ENV} + +### 20220511 + * Better handle exceptions in utils.download_file + * Added support for variations in intelligent components (ic) + * Fixed bugs in IC + * Added "_" prefix in tags to specify variation of IC + * Record env.sh in "installed artifacts even if bat file is not executed + * Fixed experiment directory naming on Windows + * Added "cm version ic" (#233) + * Added prototype of ic::prototype-get-ml-model-resnet50-onnx with variations + * Added prototype of ic::prototype-get-imagenet-val with variations + * Added prototype of ic::prototype-get-imagenet-aux with variations + * Added prototype of ic::prototype-get-llvm + * Added prototype of ic::prototype-get-tvm diff --git a/CHANGES.txt b/CHANGES.txt deleted file mode 100644 index f326ca4742..0000000000 --- a/CHANGES.txt +++ /dev/null @@ -1,747 +0,0 @@ -* v2.5.8.1 - * minor enhancement of "ck display dashboard" automation module - to support the latest format of the MLPerf benchmark - -* v2.5.8 - * fixed copyright note in the License file - * improved problem reporting in module:program - * important fix in "module:program" detected while preparing - MLPerf inference out-of-the-box benchmarking: - clean tmp directory when running CK workflow - that doesn't have compilation! - * added --remove_deps flag to module:program and module:env - as suggested by CK users to be able to remove some dependencies - from CK program workflows and thus use natively installed compilers, - tools, libraries and other components - useful to debugging and testing. - This flag takes a list of keys from dependencies from a given - program workflows separated by comma. - * added the latest contributors to the CK project. - -* v2.5.7 - * added 'ck_html_end_note' key to customize CK result dashboard - * fixed Pareto frontier filter - * added "ck filter_2d math.frontier" for MLPerf inference - -* v2.5.6 - * added --j flag to "ck install package" to update CK_HOST_CPU_NUMBER_OF_PROCESSORS env - and force number of processes in "make -j" if used by a given package - * added ck.cfg key "pip_user". If yes, add --user when to pip - when installing CK repositories. - Turn it on as follows: - $ ck set kernel var.pip_user=yes - See ticket #157 - * print path to the CK kernel when invoking "ck" - * extended "result" module to use meta.json as a base for all sub configurations - (to simplify configuration for multiple dashboards - useful for MLPerf) - -* v2.5.5 - * added result.cfg to configure CK dashboards - * improved module:result to push results to CK dashboards - * improved module:dashboard to work with new CK dashboards - * improved module:wfe to work with new CK dashboards - * added module:ck-platform to work with cKnowledge.io platform - (moved cbench functionality from the CK incubator to the CK module - to work with cKnowledge.io dashboards directly) - * provided better support to work with CK modules - as standard Python packages (see module:ck-platform as example) - -* V2.5.4 - * added support to automatically add simple packages from Git: - ck add package:my-git-repo --git={URL} --tags=my-repo --env_ext=MY_GIT_REPO - * added module:mlperf.result to abstract official MLPerf results - * added key "skip_global_deps" to program meta to skip global dependencies - for a given command line (for example, only to install python deps - for a given program) - * improved handling of a situation when CK environment has changed - and make it possible to continue running a workflow at user risk - (useful for debugging) - -* V2.5.3 - * added "pull" function to module:docker - * if only 1 tag exists in Docker, select it automatically - * updated docs - -* V2.5.2 - * fixed ck run docker --command="something" on Linux - -* V2.5.1 - * added --min_run to module:program to avoid pre/post processing and just run a command. - Useful to install Python requirements for a given program CMD. - We use it in MLPerf automation. - -* V2.5.0 - * auto-install Python requirements from pulled repositories - * added --bash to docker:module to run bash in a container - * removed outdated and unnecessary files - * added --command flag to "run docker" to run a specific command such as - ck run docker:xyz --command="ck show env" - -* V2.4.0 - * added module "docker.template" to support adaptive CK containers - * added rebuild function to module:docker to rebuild images (#145) - * added support for multiple Docker tags in module:docker - See example: https://github.com/octoml/mlops/tree/main/docker/ck-template-mlperf - * added "default_org" key in Docker image to specify default organization - * added "outdated" and "aging" keys in Docker images to warn user - about a status of a given Docker image - * added main CK and MLPerf repositories to Zenodo - to ensure reproducibilty - * added common function "tested" to mark CK entries as tested, - i.e. "ck tested docker:mlperf-inference-v0.7.openvino" - -* V2.3.0 - * improved "ck search" to search tags in entries with inheritance - * fixed function "ck status" - -* V2.2.0 - * enhancement of the inheritance mechanism and a few bug fixes: - we keep original meta of a given entry when updating, copying or moving it - rather than expanding it with all the base entries. - -* V2.1.0 - * added support for basic inheritance in CK entries via "_base_entry" key. - Useful to simplify program, package and soft modules. - See https://github.com/ctuning/ck/issues/143 . - * added program.template, soft.template and package.template modules - to describe base entries for inheritance. - -* V2.0.10 - * fixed a bug in the new installation function - -* V2.0.9 - * added missing requirements.txt to the distro - -* V2.0.8 - * added requirements.txt file with pyyaml - * simplified setup.py script - * added incubating projects to simplify CK - -* V2.0.7 - * fixed problem in module:program to return error when pre/post processing fails - * added more tests to catch bugs in program workflows - * fixed a new bug in the program workflow - * updated docs - -* V2.0.6 - * improved module:program meta to be able to use global CMD run_time parameters - for all command lines (useful to run different MLPerf scenarios - with similar run time parameters) - -* V2.0.5 - * added possibility to pull CK repositories from different GitHub organizations - as follows: "ck pull repo:{organization}@{repo}" - * removed old and not used meta "backup_data_uid" and "data_name" from program entries - -* V2.0.4 - * fixed modules mlperf.inference and mlperf.mobilenets - to cache results in local:tmp:{some cache} - -* V2.0.3 - * fixed a minor bug in module:program - -* V2.0.2 - * Added --print_skip_stats to "ck benchmark program" to reduce output for MLPerf benchmarks - * Added --print_files to "ck benchmark program" to print multiple files after MLPerf benchmarking - * Removed several data zips that are not required for the releases - -* V2.0.1 - * Updated deprecated function in web server to support Python 3.8+ - * We now use Python to generate ck and ck.bat scripts! - If you install CK for different python versions, - you can use a specific one as follows: - python3.7 -m ck version - -* V2.0.0 - * We now use Apache 2.0 license based on the feedback from the majority of users - * The code did not change from the last version 1.55.16. - * We plan to focus on incubating projects (see "incubator" directory) - to redesign CK and make it simpler, faster and more Pythonic - - we plan to use version 3.x for the new design. - -======================================================================================== - -* V1.55.17 - * fixed a bug in module:package when reinstalling existing package - -* V1.55.16 - * fixed a bug in module:env. - * added more MLPerf tests from https://github.com/ctuning/ck-ml to find above bug. - * detect packages with problems and print notes how to solve them - (useful for ImageNet 2012 packages where direct download is no longer available) - -* V1.55.15 - * added module:docs to be more compatible with standard repos - * added LICENSE.third-party.txt - -* V1.55.14 - * improved error reporting in packages and programs - -* V1.55.13 - * print "Enter -1 to use CK packages" when detecting soft - to install CK packages instead of detecting - already installed software - -* V1.55.12 - * added module:doc to the distribution - -* V1.55.11 - * Test environment variable "CK_SKIP_SAVING_PYTHON_BIN" during CK installation - to skip detecting and recording python binary to ck-python.cfg - to avoid mixing up virtual environments. - * When CK_PYTHON is defined during CK installation, - record it to ck-python.cfg to preserve the default version. - -* V1.55.10 - * do not ask to select variations in "ck install package" - when there are 2 or more variations! - -* V1.55.9 - * added --force_version to "ck detect soft" to force version. - Useful to automate autodetection of imagenet, for example. - -* V1.55.8 - * added --quiet flag to automatically enter default anser to most questions - -* V1.55.7 - * added --default flag to ck install package to select 0 - when multiple packages and variations available. - -* V1.55.6 - * extended module:package to ask a user to select a variation - for a given package when there are multiple versions available. - -* V1.55.5 - * added --tags to "ck find" action to be able to search for entries - by tags and return their paths - * added module:doc to abstract documentation - -* V1.55.4 - * [sztaylor] fix cpu & gpu set_freq commands - -* V1.55.3 - * fixed minor API typo - * improvements from Arm to 2 CK modules: platform.cpu and platform.gpu: - https://github.com/ctuning/ck-env/pull/113 - -* V1.55.2 - * removed two outdated modules - * updated module:program - * added incubator projects - * prepared for release - -* V1.55.0 - * Fixed missing CK modules - * Removed module/crowdnode/tests (binary files) - * Prepared new release - -======================================================================================== - -* V1.50.1 - * Fixed missing CK modules - -* V1.50.0 - * Added stable modules from different CK repositories (Issue #120) - - * Added ck list_tags {CID} to list all tags from found entries (uses search function) - * Converted python files to PEP8 style using autopep8 - * Fixed bug with "ck unzip repo" if ckr.zip if not present - * Added command "get_default_repo" to print the path to the default repo inside CK - -======================================================================================== - -* V1.17.0 - * Updated description of all CK APIs in Google format! - * Added support to load YAML files via CK CLI: ck action @file.yaml - * Fixed Yaml loader - * Added support to download components with depenendecies from cKnowledge.io - * Fixed issue when adding the "init" action to modules (this function always exists in CK modules). - We create "new_init" function in such case. - * Checking all function names in the CK module when adding a new action - * Added simplified "ck init repo" to create a CK repository in the current directory - * Added docs about "ck init repo" - -* V1.15.0 - * Improved the integration with the open CK portal: https://cKnowledge.io - * Fixed installation in virtual env / pip for Linux/Windows - * Started working on a new Sphinx-based documentation - * Check missing modules at cKnowledge.io - (can be turned off by "ck set kernel --var.check_missing_modules=no") - * Fixed ck.bat for Windows when CK is used from GitHub - * Fixed ElasticSearch indexing error when index is empty - * Fixed remote repo access - * Added experimental mode to support multiple aliases for the same UID - to be able to push renamed components to the cKnowledge.io portal - * Started breaking CK kernel into sub modules: - https://github.com/ctuning/ck/issues/107 - * Started improving ck scripts to automatically detect installation (Linux and Windows): - https://github.com/ctuning/ck/issues/106 - * Added support to download missing components from cKnowledge.io - -* V1.12.3 - * improved tests on Linux, MacOS and Windows - * prototyped testing of the status of repositories (stable and warning) - -* V1.12.2 - * fixed UTF-8 problem with README.md in setup.py - -* V1.12.1 - * added first draft of the docs in the Sphinx format - * fixed repo update - * added backward compatibility for the search via ElasticSearch - * do not ask for "web support" by default when adding actions - * ck webhelp now shows published components at the CodeReef portal - * ck webapi now shows module API at the CodeReef portal - * Added "ck codereef " to show information about published components - at the CodeReef portal - * Allowed ' character in names of CK entries - * Added 2020 - * Simplified Readme - -* V1.11.4 - * fixed problem when ck-python.cfg contains non-existent python - * fixed problem with allowed action_names - (should not start with number but can have numbers in the middle) - * fixed problem with ElasticSearch when renaming entries - * Added key "use_git_instead_of_https" to kernel cfg - to substitute https:// with git@ in repositories - * fixed problem with pip installation using Python 2.7 - (thanks to Alex Redshaw from Arm) - * added possibility to log all used CK entries to find out all dependencies - - turn on: ck set kernel --var.log_ck_entries={file} - turn off: ck set kernel --var.log_ck_entries="" - -* V1.11.3 - * minor clean up of the API doc - * now copying default repo to $HOME/.ck/{version} to be able to always find it - * added possibility to split dirs for the whole repo - (to be able to exchange repos with split directories) - * added commands "private", "recache", "split_all_dirs" to ck add/update repo - * added try/catch around encode/decode routines to better support non-UTF locale - * can use wildcards when searching for UIDs - * added repo_module_uoa to be able to execute module from a specific repository - -* V1.11.2 - * fixed typo - * added possibility to force index data for specific repos via local kernel meta: - "index_repos":[list of repo UIDs and aliases] - * bug fix for split directories if UID is used for an entry with alias - * fixed link to CK slack channel - * added zip_safe=False - -* V1.11.1 - * fixed ck.bat on Windows (if CK_ROOT has spaces) - * added possibility to add sub-directory for all entries of a given module based on first N characters of an entry name; - this is needed to handle millions of entries; - it can be configured via local kernel meta: "split_dirs":{"module UOA":N} - * improved "ck help" function - * added support for the latest ElasticSearch via Python client - * improved CK search via ElasticSearch - * added possibility to auto convert all entry names to lower case; - it can be configured via local kernel meta: "force_lower":"yes" - * added check for allowed entry names; - it can be configured as RE via local kernel meta: - "allowed_entry_names": "^[A-Za-z0-9_@. -]*$" - * added check for allowed action names in modules; - it can be configured as RE via local kernel meta: - "allowed_action_names": "^[A-Za-z_-]*$" - * added possibility to index data for only specific modules via local kernel meta: - "index_modules":[list of module UIDs and aliases] - * removed AUTHORS.txt as non-standard (leaving CONTRIBUTIONS.txt) - * added possibility to skip SSL certificate check for remote repositories - * added CK kernel functions save_state(), restore_state, reinit - to let multiple CK instances co-exist - -* V1.10.3 - * fixed major bug with kernel version checking - -* V1.10.2 - * fixed mistake in PyPi release - -* V1.10.1 - * added support for setuptools - * added report on Python executable used by CK - * recording python executable to ck-python.cfg during installation - - it can now be used by default when calling CK CLI - * added experimental functionality to automatically download - missing components from cKnowledge.org. - It is turned off by default. - To turn it on use "ck set kernel var.download_missing_components=yes" - -* V1.9.9 - * fixed installation bug (non UTF-8 character) - -* V1.9.8 - * improved OpenME to call CK in dynamic websites - * added passing of $_SESSION, $_POST, $_GET to CK via OpenME - * added "Access-Control-Allow-Origin=*" to module:web - to be able to access CK from local web pages - * fixed CK execution with Anaconda python (Linux and MacOS) - when installed via pip - * added possibility to change default repo - using "CK_DEFAULT_REPO" env var - * added flag --module_version to use different Python module: - (module{.{module_version}}.py) - -* V1.9.7 - * allowed local widget to fetch data from cKnowledge.org (or other) website: - https://github.com/ctuning/ck/pull/93 - * automatically print API of redirected functions (ck --help) - * improved module API printing - * added pypi download stats - * fixed minor problem with "ck renew repo" - now takes already existing UID instead of generating a new one - -* V1.9.6 - * fixed minor bug in "ck rm" when deleting entries only from allowed repos (for crowdsourcing) - * fixed problem with wrong repo UID when repo is installed from zip archive - * added flag --all to "ck zip" to archive all files including .git - * fixed "ck add repo --zip" if zip contains Git repository (making it shared) - -* V1.9.5 - * added "debug":"yes" to kernel for GitHub versions - * fixed minor bug in detection of minimal CK kernel version - (important since current python3-ck in official Ubunutu 17.04 is quite outdated: v1.7.2) - * fixed tests to report the error detected by running ck.access() or equivalent - * testfix: stop testing 3.5dev and nightly Python, but add 3.6 which everyone uses - * improved auto-generation of a list of all modules and actions: - http://cKnowledge.org/shared-modules.html - * made removal of \r in text files safer on Windows (across different Python versions) - * added TARGET remark to all CK kernel functions to show target audience (end users vs CK kernel developers). - Used in "ck list_kernel_functions misc". - * improved recaching of repos (taking latest updates from .ckr.json into account) - * blocked copying of repositories to avoid duplication of CK entries - * added "ck python_version" to print Python version used by CK - * now when calling ck without arguments, it will print just a short help - -* V1.9.4 - * fixed minor bug when searching repo by UID when repo doesn't exist - * added CK internal var "debug". If set to "yes", print error when CK entries are broken or ignore them (now default) - See https://github.com/ctuning/ck-tensorflow/commit/c4ddfcde1baacf6bebb23f345735b38006be08d1#commitcomment-25230616 - -* V1.9.3 - * fixed issue #80 (adding zip repo if starts with ~) - * fixed issue #81 (when renaming entry with data_name==data_alias, change it to the new alias) - * added flag --name (duplicate of --show_name) to "ck ls" - * fixed issue #82 (detecting name of repo from the zip filename) - * fixed repo renaming (now updates .ckr.json and recache repo properly) - -* V1.9.2 - * added "ck cd {CK entry}" to open a new shell in a given CK entry - * added ck.compare_dicts examples - * fixed some self tests - * added safe recording of JSON (even if some keys are not serializable) - * printing user-friendly name (from info.json "data_name") when removing entries. - Useful when deleting env entries (ck rm env:*) - * added more user-friendly configuration of internal CK variables via - $ ck set kernel - $ ck set kernel var.install_to_env=yes - * added safe deletion of CK entries with Git files on Windows - * automated adding CK .zip repos without questions - -* V1.9.1 - * added ck.debug_out(i) to print dict to console - * fixed strange warnings for Python 3.5+ on Windows and Linux (when installed via pip) - * fixed repo renewing on latest Windows (when access error to Git files) - * added ck show repo to show git status of all repos - * started highly experimental support for stable repos or different branches/checkouts - (particularly useful for Artifact Evaluation to check out only stable repos) - -* V1.8.7 - * fixed major bug with default license/author (to avoid crashing when adding new module) - * fixed minor bug in API when starting CK web service and viewing entry (wcid option) - * allow ~ in an archive name when archiving CK entries using 'ck zip ...' - -* V1.8.6 - * added direct update of kernel vars via CMD similar to Git. For example: - - $ ck setup kernel --var.default_license="See CK LICENSE.txt for licensing details" - $ ck setup kernel --var.default_copyright="See CK COPYRIGHT.txt for copyright details" - $ ck setup kernel --var.default_developer="cTuning foundation" - $ ck setup kernel --var.default_developer_email="admin@cTuning.org" - $ ck setup kernel --var.default_developer_webpage="http://cTuning.org" - - * added -f flag to force removal of entries (i.e. ck rm x:y -f) to be compatible - with bash rm -f ... - - * added JSON sorting when viewing entries from CMD (via ck load x:y) - -* V1.8.5 - * do not crash if renewing repo while staying inside it - * fixed bug in "system_with_timeout" (was always returning error) - required by ck-clsmith - * added "skip_sort" to "select" and "select_uoa" functions - * added cfg['internal_keys'] to be able to check them in input - * fixed minor bug when @file.json from CMD not found - -* V1.8.4 - * fixed 'ck status' and version comparison - * minor fix when listing all modules in mediawiki format - -* V1.8.3dev - * fixed minor issue in ck.list_all_files() with 'limit' parameter - -* V1.8.2 - * fixed bug in 'remove_action' in CK kernel - * fixed issue in "run_and_get_stdout" on Windows - * printing module UOA in square brackets when error (useful for debugging) - * added 'shell' parameter to kernel 'run_and_get_stdout' function - (needed for ck-crowdnode) - * added 'eout' function to kernel to print to stderr - (useful for ck-crowdnode-server) - -* V1.8.1 - * added continuous integration via Travis (Linux) and AppVeyor (Windows) - * added CoverAlls to measure test coverage - * added possibility to skip listing of all modules from private repositories - * fixed error in 'ck edit' - * unitests added (we now support Python 2.7 and 3.3+) - * added 'dump_json' instead of 'dumps_json' - -* V1.7.4 - * extended run_and_get_stdout kernel function - * fixed problem in run_and_get_stdout (ValueError: I/O operation on closed file) - * fixed problem with --help or --get_api from kernel - -* V1.7.3 - * fixed rare bug when 'return' is non-integer in remote CK access (such as during crowd-tuning and crowd-testing) - * fixed copying to clipboard on Windows - * fixed __init__.py when using CK from scripts/ipython/jupyter (calling init()) - * added ck.jerr(r) - print error with raise KeyboardInterrupt for Jupyter Notebooks - * fixed major problem with using CK without CK_ROOT env, i.e. from python scripts or Jupyter Notebooks - * fixed minor problem with CK_ROOT env (when is not set but can be detected automatically) - * added run_and_get_stdout function and fixed various problems with crowdtuning on MacOS - -* V1.7.2 - * added the --wfe_host and --wfe_host flags to "ck start web". - * improved API when starting CK server and a browser at the same time ... - * added info about Docker support in CK - * simplified adding remote repo using --hostname --port --hostext - * added possibility to describe repository for Artifact Evaluation - using flag --describe (see http://cTuning.org/ae), i.e. - ck add repo:my-artifact-pack --describe - * return 16 when adding new entry (including repository) and it already exists - * added often used 'load_yaml_file' and 'save_yaml_to_file' functions to kernel - * changed yaml.dump to yaml.safe_dump - * added --install option to "ck pull all (--kernel)" to install CK kernel as python module - -* V1.7.1 - * fixing detection of CK installation even when ck is soft linked - * changing remote-ck repo URL pointing to cknowledge.org - (after moving personal CK server to Azure cloud) - * adding support for basic http authentication for remote repositories - * adding automatic recaching of repositories if has remote access ... - * adding support to automatically open web service and browser via - "ck browser" (or "ck browser --template=