diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index a6e6b86ef..9fed7358c 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -4,7 +4,7 @@ name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch) on: - pull_request: + pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml' @@ -13,8 +13,7 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: @@ -22,11 +21,13 @@ jobs: python-version: [ "3.11" ] backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ] precision: [ "int8", "fp32" ] + os: [ubuntu-latest, windows-latest, macos-latest] exclude: - backend: tf - backend: pytorch - backend: onnxruntime - precision: fp32 + - os: windows-latest steps: - uses: actions/checkout@v3 @@ -38,6 +39,26 @@ jobs: run: | python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - - name: Test MLPerf Inference Bert (DeepSparse, TF, ONNX, PyTorch) + - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os == 'windows-latest' + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --precision=${{ matrix.precision }} --target_qps=1 -v --quiet + - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os != 'windows-latest' run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet + - name: Push Results + if: github.repository_owner == 'gateoverflow' + env: + USER: "GitHub Action" + EMAIL: "admin@gateoverflow.com" + run: | + git config --global user.name "${{ env.USER }}" + git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + + cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index 61f9d30cd..54217d71a 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -4,7 +4,7 @@ name: MLPerf inference retinanet on: - pull_request: + pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-retinanet.yml' @@ -13,17 +13,19 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, windows-latest, macos-latest] python-version: [ "3.12" ] backend: [ "onnxruntime", "pytorch" ] implementation: [ "python", "cpp" ] exclude: - backend: pytorch implementation: cpp + - os: windows + implementation: cpp steps: - uses: actions/checkout@v3 @@ -35,6 +37,27 @@ jobs: run: | python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} + - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=retinanet --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet -v --target_qps=1 + - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os != 'windows-latest' run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --hw_name=default --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet -v --target_qps=1 + - name: Push Results + if: github.repository_owner == 'gateoverflow' + env: + USER: "GitHub Action" + EMAIL: "admin@gateoverflow.com" + run: | + git config --global user.name "${{ env.USER }}" + git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + + cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index b68d90e27..54ab2dd48 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1470,6 +1470,8 @@ def dockerfile(i): run_cmd = r['run_cmd_string'] cm_repo = i.get('docker_cm_repo', docker_settings.get('cm_repo', 'mlcommons@cm4mlops')) + cm_repo_branch = i.get('docker_cm_repo_branch', docker_settings.get('cm_repo_branch', 'mlperf-inference')) + cm_repo_flags = i.get('docker_cm_repo_flags', docker_settings.get('cm_repo_flags', '')) docker_base_image = i.get('docker_base_image', docker_settings.get('base_image')) @@ -1542,6 +1544,7 @@ def dockerfile(i): 'automation': 'script', 'tags': 'build,dockerfile', 'cm_repo': cm_repo, + 'cm_repo_branch': cm_repo_branch, 'cm_repo_flags': cm_repo_flags, 'docker_base_image': docker_base_image, 'docker_os': docker_os, diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index b648dc54e..0576f6eca 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -434,6 +434,9 @@ deps: skip_if_env: CM_MLPERF_CUSTOM_MODEL_PATH: - "on" + skip_if_env: + CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: + - 'yes' ## 3d-unet - tags: get,ml-model,medical-imaging,3d-unet @@ -555,6 +558,9 @@ deps: enable_if_env: CM_MODEL: - mixtral-8x7b + skip_if_env: + CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: + - 'yes' ## Kits19 for 3d-unet - tags: get,dataset,kits19,preprocessed diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index ca97bbf0d..037629348 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -78,7 +78,7 @@ def preprocess(i): if env.get('CM_NETWORK_LOADGEN', '') != "lon" and env.get('CM_MLPERF_INFERENCE_API_SERVER','')=='' and "llama2-70b" not in env['CM_MODEL']: env['MODEL_DIR'] = env.get('CM_ML_MODEL_PATH') if not env['MODEL_DIR']: - env['MODEL_DIR'] = os.path.dirname(env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH'))) + env['MODEL_DIR'] = os.path.dirname(env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH', ''))) RUN_CMD = "" state['RUN'] = {} @@ -332,14 +332,15 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio device = env['CM_MLPERF_DEVICE'] if env['CM_MLPERF_DEVICE'] != "gpu" else "cuda" cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " main.py " \ " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + \ - " --dataset-path " + env['CM_DATASET_PREPROCESSED_PATH'] + \ + " --dataset-path " + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] + \ " --device " + device.replace("cuda", "cuda:0") + \ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ scenario_extra_options + mode_extra_options + \ " --output-log-dir " + env['CM_MLPERF_OUTPUT_DIR'] + \ ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \ - " --model-path " + env['MODEL_DIR'] + " --model-path " + env['MIXTRAL_CHECKPOINT_PATH'] cmd = cmd.replace("--count", "--total-sample-count") + cmd = cmd.replace("--max-batchsize", "--batch-size") elif "3d-unet" in env['CM_MODEL']: diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 2b8186c88..0eb89752b 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -799,6 +799,26 @@ variations: - openorca-gsm8k-mbxp-combined-accuracy-script tags: run,accuracy,mlperf,_openorca-gsm8k-mbxp,_int32 + mixtral-8x7b,reference: + docker: + deps: + - tags: get,ml-model,mixtral + names: + - ml-model + - mixtral-model + enable_if_env: + CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: + - 'yes' + - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined + names: + - openorca-mbxp-gsm8k-combined-preprocessed + enable_if_env: + CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: + - 'yes' + mounts: + - "${{ MIXTRAL_CHECKPOINT_PATH }}:${{ MIXTRAL_CHECKPOINT_PATH }}" + - "${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}:${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}" + rnnt: group: model @@ -1611,7 +1631,7 @@ docker: extra_run_args: ' --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined' os: ubuntu cm_repo: mlcommons@cm4mlops - cm_repo_flags: ' --branch=mlperf-inference ' + cm_repo_branch: mlperf-inference real_run: False os_version: '22.04' docker_input_mapping: diff --git a/script/build-dockerfile/_cm.yaml b/script/build-dockerfile/_cm.yaml index da00f24de..7706597ee 100644 --- a/script/build-dockerfile/_cm.yaml +++ b/script/build-dockerfile/_cm.yaml @@ -19,6 +19,7 @@ default_env: ' CM_DOCKER_OS: ubuntu CM_DOCKER_NOT_PULL_UPDATE: False + CM_MLOPS_REPO_BRANCH: mlperf-inference input_mapping: build: CM_BUILD_DOCKER_IMAGE @@ -26,6 +27,7 @@ input_mapping: cm_repo: CM_MLOPS_REPO cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO cm_repos: CM_DOCKER_EXTRA_CM_REPOS + cm_repo_branch: CM_MLOPS_REPO_BRANCH comments: CM_DOCKER_RUN_COMMENTS copy_files: CM_DOCKER_COPY_FILES docker_base_image: CM_DOCKER_IMAGE_BASE diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 896454fb9..2a8c96b63 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -69,11 +69,8 @@ def preprocess(i): else: cm_mlops_repo = "mlcommons@cm4mlops" - if env.get("CM_MLOPS_REPO_BRANCH", '') != '': - cm_mlops_repo_branch_string = f" --branch {env['CM_MLOPS_REPO_BRANCH']}" - else: - cm_mlops_repo_branch_string = "" - + cm_mlops_repo_branch_string = f" --branch={env['CM_MLOPS_REPO_BRANCH']}" + if env.get('CM_DOCKERFILE_WITH_PATH', '') == '': env['CM_DOCKERFILE_WITH_PATH'] = os.path.join(os.getcwd(), "Dockerfile") @@ -206,7 +203,7 @@ def preprocess(i): x = env.get('CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO','') if x!='': x=' '+x - f.write('RUN cm pull repo ' + cm_mlops_repo + x + EOL) + f.write('RUN cm pull repo ' + cm_mlops_repo + cm_mlops_repo_branch_string + x + EOL) # Check extra repositories x = env.get('CM_DOCKER_EXTRA_CM_REPOS','') diff --git a/script/get-dataset-mixtral/customize.py b/script/get-dataset-mixtral/customize.py new file mode 100644 index 000000000..31d72127e --- /dev/null +++ b/script/get-dataset-mixtral/customize.py @@ -0,0 +1,18 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + return {'return':0} + + +def postprocess(i): + env = i['env'] + + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['CM_DATASET_PREPROCESSED_PATH'] + + return {'return':0} diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 30939d556..4d3e755ed 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -14,8 +14,6 @@ deps: env: CM_CLEAN_DIRS: bin CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 - ; https://www.dropbox.com/scl/fi/cj7xmgmk4wu1r0iuoedcq/zlib123dllx64-bin.zip?rlkey=sqmldotiste2zhk6nhmijg09e&st=i5w7esjx&dl=1 - ; https://www.dropbox.com/scl/fi/x94qvrmac55ht03257139/xz-5.2.9-win64.zip?rlkey=adzixlsqrapncrusy9ndd2qji&st=tqobzlfo&dl=1 CM_SUDO: sudo input_mapping: diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index cd4028a53..1addc3cbf 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -340,16 +340,34 @@ }, "openorca-gsm8k-mbxp": { "deps": [ + { + "tags": "get,generic-python-lib,_package.rouge_score", + "names": + [ + "pip-package", + "rouge-score" + ] + }, { "names": [ "openorca-gsm8k-mbxp-combined" ], + "skip_if_env": { + "CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [ + "yes" + ] + }, "tags": "get,dataset-mixtral,openorca-mbxp-gsm8k-combined" }, { "names": [ "mixtral-8x7b-model" ], + "skip_if_env": { + "CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [ + "yes" + ] + }, "tags": "get,ml-model,mixtral" } ], diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index d6536e13f..765f96a35 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -87,8 +87,8 @@ def preprocess(i): elif dataset == "openorca-gsm8k-mbxp-combined": accuracy_checker_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b", "evaluate-accuracy.py") - CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['CM_ML_MODEL_MIXTRAL_FILE_WITH_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --dataset-file '" + env['CM_DATASET_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32") +" > '" + out_file + "'" + CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['MIXTRAL_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --dataset-file '" + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32") +" > '" + out_file + "'" elif dataset == "coco2014": diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index ebf497a8b..d482a1ecf 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -8,8 +8,9 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH ( echo %cd% echo "Doing cd" +echo %CM_GIT_REPO_CHECKOUT_PATH% REM Change directory -cd "%CM_GIT_REPO_CHECKOUT_PATH%" || ( +chdir %CM_GIT_REPO_CHECKOUT_PATH% || ( echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" exit /b 1 )