Merge branch 'mlperf-inference' into mixtral+gha+selfhosted

.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml

@@ -4,7 +4,7 @@
 name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch)
 
 on:
-  pull_request:
+  pull_request_target:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml'
@@ -13,20 +13,21 @@ on:
 
 jobs:
   build:
-
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         # 3.12 didn't work on 20240305 - need to check
         python-version: [ "3.11" ]
         backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ]
         precision: [ "int8", "fp32" ]
+        os: [ubuntu-latest, windows-latest, macos-latest]
         exclude:
           - backend: tf
           - backend: pytorch
           - backend: onnxruntime
           - precision: fp32
+          - os: windows-latest
 
     steps:
     - uses: actions/checkout@v3
@@ -38,6 +39,26 @@ jobs:
       run: |
         python3 -m pip install cmind
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
-    - name: Test MLPerf Inference Bert (DeepSparse, TF, ONNX, PyTorch)
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os == 'windows-latest'
+      run: |
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --precision=${{ matrix.precision }} --target_qps=1 -v --quiet
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os != 'windows-latest'
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet
+    - name: Push Results
+      if: github.repository_owner == 'gateoverflow'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@gateoverflow.com"
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://git.luolix.top.helper ""
+        git config --global credential.https://git.luolix.top.helper "!gh auth git-credential"
+        git config --global credential.https://gist.git.luolix.top.helper ""
+        git config --global credential.https://gist.git.luolix.top.helper "!gh auth git-credential"
+
+        cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}"
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-mlperf-inference-retinanet.yml

@@ -4,7 +4,7 @@
 name: MLPerf inference retinanet
 
 on:
-  pull_request:
+  pull_request_target:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-mlperf-inference-retinanet.yml'
@@ -13,17 +13,19 @@ on:
 
 jobs:
   build:
-
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: [ "3.12" ]
         backend: [ "onnxruntime", "pytorch" ]
         implementation: [ "python", "cpp" ]
         exclude:
           - backend: pytorch
             implementation: cpp
+          - os: windows
+            implementation: cpp
 
     steps:
     - uses: actions/checkout@v3
@@ -35,6 +37,27 @@ jobs:
       run: |
         python3 -m pip install cmind
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
-    - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }}
+    - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=retinanet --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet  -v --target_qps=1
+    - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os != 'windows-latest'
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --hw_name=default --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet  -v --target_qps=1
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet  -v --target_qps=1
+    - name: Push Results
+      if: github.repository_owner == 'gateoverflow'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@gateoverflow.com"
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://git.luolix.top.helper ""
+        git config --global credential.https://git.luolix.top.helper "!gh auth git-credential"
+        git config --global credential.https://gist.git.luolix.top.helper ""
+        git config --global credential.https://gist.git.luolix.top.helper "!gh auth git-credential"
+
+        cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}"
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet

automation/script/module_misc.py

@@ -1470,6 +1470,8 @@ def dockerfile(i):
         run_cmd  = r['run_cmd_string']
 
         cm_repo = i.get('docker_cm_repo', docker_settings.get('cm_repo', 'mlcommons@cm4mlops'))
+        cm_repo_branch = i.get('docker_cm_repo_branch', docker_settings.get('cm_repo_branch', 'mlperf-inference'))
+
         cm_repo_flags = i.get('docker_cm_repo_flags', docker_settings.get('cm_repo_flags', ''))
 
         docker_base_image = i.get('docker_base_image', docker_settings.get('base_image'))
@@ -1542,6 +1544,7 @@ def dockerfile(i):
                            'automation': 'script',
                            'tags': 'build,dockerfile',
                            'cm_repo': cm_repo,
+                           'cm_repo_branch': cm_repo_branch,
                            'cm_repo_flags': cm_repo_flags,
                            'docker_base_image': docker_base_image,
                            'docker_os': docker_os,

script/app-mlperf-inference-mlcommons-python/_cm.yaml

@@ -434,6 +434,9 @@ deps:
     skip_if_env:
       CM_MLPERF_CUSTOM_MODEL_PATH:
       - "on"
+    skip_if_env:
+      CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST:
+      - 'yes'
 
   ## 3d-unet
   - tags: get,ml-model,medical-imaging,3d-unet
@@ -555,6 +558,9 @@ deps:
     enable_if_env:
       CM_MODEL:
       - mixtral-8x7b
+    skip_if_env:
+      CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST:
+      - 'yes'
 
   ## Kits19 for 3d-unet
   - tags: get,dataset,kits19,preprocessed

script/app-mlperf-inference-mlcommons-python/customize.py

@@ -78,7 +78,7 @@ def preprocess(i):
     if env.get('CM_NETWORK_LOADGEN', '') != "lon" and env.get('CM_MLPERF_INFERENCE_API_SERVER','')=='' and "llama2-70b" not in env['CM_MODEL']:
         env['MODEL_DIR'] = env.get('CM_ML_MODEL_PATH')
         if not env['MODEL_DIR']:
-            env['MODEL_DIR'] = os.path.dirname(env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH')))
+            env['MODEL_DIR'] = os.path.dirname(env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH', '')))
 
     RUN_CMD = ""
     state['RUN'] = {}
@@ -332,14 +332,15 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio
         device = env['CM_MLPERF_DEVICE'] if env['CM_MLPERF_DEVICE'] != "gpu" else "cuda"
         cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " main.py " \
                 " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + \
-                " --dataset-path " + env['CM_DATASET_PREPROCESSED_PATH'] + \
+                " --dataset-path " + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] + \
                 " --device " + device.replace("cuda", "cuda:0") + \
                  env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \
                  scenario_extra_options + mode_extra_options + \
                 " --output-log-dir " + env['CM_MLPERF_OUTPUT_DIR'] + \
                 ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \
-                " --model-path " + env['MODEL_DIR']
+                " --model-path " + env['MIXTRAL_CHECKPOINT_PATH']
         cmd = cmd.replace("--count", "--total-sample-count")
+        cmd = cmd.replace("--max-batchsize", "--batch-size")
 
     elif "3d-unet" in env['CM_MODEL']:
 

script/app-mlperf-inference/_cm.yaml

@@ -799,6 +799,26 @@ variations:
       - openorca-gsm8k-mbxp-combined-accuracy-script
       tags: run,accuracy,mlperf,_openorca-gsm8k-mbxp,_int32
 
+  mixtral-8x7b,reference:
+    docker:
+      deps:
+        - tags: get,ml-model,mixtral
+          names:
+          - ml-model
+          - mixtral-model
+          enable_if_env:
+            CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST:
+            - 'yes'
+        - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined
+          names:
+          - openorca-mbxp-gsm8k-combined-preprocessed
+          enable_if_env:
+            CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST:
+            - 'yes'
+      mounts:
+        - "${{ MIXTRAL_CHECKPOINT_PATH }}:${{ MIXTRAL_CHECKPOINT_PATH }}"
+        - "${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}:${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}"
+
   rnnt:
     group:
       model
@@ -1611,7 +1631,7 @@ docker:
   extra_run_args: ' --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined'
   os: ubuntu
   cm_repo: mlcommons@cm4mlops
-  cm_repo_flags: ' --branch=mlperf-inference '
+  cm_repo_branch: mlperf-inference
   real_run: False
   os_version: '22.04'
   docker_input_mapping:

script/build-dockerfile/_cm.yaml

@@ -19,13 +19,15 @@ default_env:
     '
   CM_DOCKER_OS: ubuntu
   CM_DOCKER_NOT_PULL_UPDATE: False
+  CM_MLOPS_REPO_BRANCH: mlperf-inference
 
 input_mapping:
   build: CM_BUILD_DOCKER_IMAGE
   cache: CM_DOCKER_CACHE
   cm_repo: CM_MLOPS_REPO
   cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO
   cm_repos: CM_DOCKER_EXTRA_CM_REPOS
+  cm_repo_branch: CM_MLOPS_REPO_BRANCH
   comments: CM_DOCKER_RUN_COMMENTS
   copy_files: CM_DOCKER_COPY_FILES
   docker_base_image: CM_DOCKER_IMAGE_BASE

script/build-dockerfile/customize.py

@@ -69,11 +69,8 @@ def preprocess(i):
     else:
         cm_mlops_repo = "mlcommons@cm4mlops"
 
-    if env.get("CM_MLOPS_REPO_BRANCH", '') != '':
-        cm_mlops_repo_branch_string = f" --branch {env['CM_MLOPS_REPO_BRANCH']}"
-    else:
-        cm_mlops_repo_branch_string = ""
-
+    cm_mlops_repo_branch_string = f" --branch={env['CM_MLOPS_REPO_BRANCH']}"
+
     if env.get('CM_DOCKERFILE_WITH_PATH', '') == '':
         env['CM_DOCKERFILE_WITH_PATH'] = os.path.join(os.getcwd(), "Dockerfile")
 
@@ -206,7 +203,7 @@ def preprocess(i):
     x = env.get('CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO','')
     if x!='': x=' '+x
 
-    f.write('RUN cm pull repo ' + cm_mlops_repo + x + EOL)
+    f.write('RUN cm pull repo ' + cm_mlops_repo + cm_mlops_repo_branch_string + x + EOL)
 
     # Check extra repositories
     x = env.get('CM_DOCKER_EXTRA_CM_REPOS','')

script/get-dataset-mixtral/customize.py

@@ -0,0 +1,18 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    env = i['env']
+
+    return {'return':0}
+
+
+def postprocess(i):
+    env = i['env']
+
+    env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['CM_DATASET_PREPROCESSED_PATH']
+
+    return {'return':0}

script/get-sys-utils-cm/_cm.yaml

@@ -14,8 +14,6 @@ deps:
 env:
   CM_CLEAN_DIRS: bin
   CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1
-    ; https://www.dropbox.com/scl/fi/cj7xmgmk4wu1r0iuoedcq/zlib123dllx64-bin.zip?rlkey=sqmldotiste2zhk6nhmijg09e&st=i5w7esjx&dl=1
-    ; https://www.dropbox.com/scl/fi/x94qvrmac55ht03257139/xz-5.2.9-win64.zip?rlkey=adzixlsqrapncrusy9ndd2qji&st=tqobzlfo&dl=1
   CM_SUDO: sudo
 
 input_mapping:

script/process-mlperf-accuracy/_cm.json

@@ -340,16 +340,34 @@
     },
     "openorca-gsm8k-mbxp": {
       "deps": [
+	{
+          "tags": "get,generic-python-lib,_package.rouge_score",
+          "names":
+          [
+            "pip-package",
+            "rouge-score"
+          ]
+        },
         {
           "names": [
             "openorca-gsm8k-mbxp-combined"
           ],
+	  "skip_if_env": {
+            "CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [
+              "yes"
+            ]
+          },
           "tags": "get,dataset-mixtral,openorca-mbxp-gsm8k-combined"
         },
         {
           "names": [
             "mixtral-8x7b-model"
           ],
+	  "skip_if_env": {
+            "CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [
+              "yes"
+            ]
+          },
           "tags": "get,ml-model,mixtral"
         }
       ],

script/process-mlperf-accuracy/customize.py

@@ -87,8 +87,8 @@ def preprocess(i):
         elif dataset == "openorca-gsm8k-mbxp-combined":
             accuracy_checker_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b",
                 "evaluate-accuracy.py")
-            CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['CM_ML_MODEL_MIXTRAL_FILE_WITH_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \
-                "' --dataset-file '" + env['CM_DATASET_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32")  +" > '" + out_file + "'"
+            CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['MIXTRAL_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \
+                "' --dataset-file '" + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32")  +" > '" + out_file + "'"
 
 
         elif dataset == "coco2014":

script/push-mlperf-inference-results-to-github/run.bat

@@ -8,8 +8,9 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH (
 
 echo %cd%
 echo "Doing cd"
+echo %CM_GIT_REPO_CHECKOUT_PATH%
 REM Change directory
-cd "%CM_GIT_REPO_CHECKOUT_PATH%" || (
+chdir %CM_GIT_REPO_CHECKOUT_PATH% || (
     echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%"
     exit /b 1
 )