Enable update mlperf inference measurements.json file from existing file

.github/workflows/test-mlperf-inference-mixtral.yml

@@ -5,19 +5,19 @@ name: MLPerf inference MIXTRAL-8x7B
 
 on:
   schedule:
-    - cron: "45 4 * * 5"   # 30th minute and 20th hour => 20:30 UTC => 2 AM IST 
+    - cron: "32 22 * * *"   # 30th minute and 20th hour => 20:30 UTC => 2 AM IST 
 
 jobs:
   build_reference:
     if: github.repository_owner == 'gateoverflow'
-    runs-on: [ self-hosted, GO-spr, linux, x64 ]
+    runs-on: [ self-hosted, phoenix, linux, x64 ]
     strategy:
       fail-fast: false
       matrix:
         python-version: [ "3.12" ]
         backend: [ "pytorch" ]
         device: [ "cpu" ]
-        precision: [ "bfloat16" ]
+        precision: [ "float16" ]
 
     steps:
     - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation
@@ -31,4 +31,4 @@ jobs:
         huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
         cm pull repo
         cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=0.001 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes
-        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-mlperf-inference-sdxl.yaml

@@ -1,7 +1,7 @@
 name: MLPerf inference SDXL
 on:
   schedule:
-    - cron: "30 2 * * *"
+    - cron: "11 15 * * *"
 
 jobs:
   build_reference:
@@ -21,5 +21,5 @@ jobs:
         export CM_REPOS=$HOME/GH_CM
         python3 -m pip install cm4mlops
         cm pull repo
-        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions  --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
+        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --adr.mlperf-implementation.tags=_branch.dev --adr.mlperf-implementation.version=custom  --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions  --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
         cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions

automation/script/module.py

@@ -836,6 +836,10 @@ def _run(self, i):
         script_artifact_state = meta.get('state',{})
         utils.merge_dicts({'dict1':state, 'dict2':script_artifact_state, 'append_lists':True, 'append_unique':True})
 
+        # Store the default_version in run_state -> may be overridden by variations
+        default_version = meta.get('default_version', '') #not used if version is given
+        run_state['default_version'] = default_version
+
 
 
 
@@ -957,9 +961,6 @@ def _run(self, i):
                 #Processing them again using updated deps for add_deps_recursive
                 r = update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive, env)
 
-        default_version = meta.get('default_version', '') #not used if version is given
-        run_state['default_version'] = default_version
-
         # STEP 1100: Update deps from input
         r = update_deps_from_input(deps, post_deps, prehook_deps, posthook_deps, i)
         if r['return']>0: return r

script/app-mlperf-inference/_cm.yaml

@@ -873,6 +873,7 @@ variations:
 
   mixtral-8x7b,reference:
     docker:
+      image_name: mlperf-inference-mlcommons-python-implementation-mixtral-8x7b
       deps:
         - tags: get,ml-model,mixtral
           names:

script/app-mlperf-inference/customize.py

@@ -181,12 +181,17 @@ def postprocess(i):
 
 
     if mode in [ "performance", "accuracy" ]:
-        measurements = {}
-        measurements['starting_weights_filename'] = env.get('CM_ML_MODEL_STARTING_WEIGHTS_FILENAME', env.get('CM_ML_MODEL_FILE', ''))
-        measurements['retraining'] = env.get('CM_ML_MODEL_RETRAINING','no')
-        measurements['input_data_types'] = env.get('CM_ML_MODEL_INPUTS_DATA_TYPE', 'fp32')
-        measurements['weight_data_types'] = env.get('CM_ML_MODEL_WEIGHTS_DATA_TYPE', 'fp32')
-        measurements['weight_transformations'] = env.get('CM_ML_MODEL_WEIGHT_TRANSFORMATIONS', 'none')
+        #if measurements file exist read it
+        if os.path.exists("measurements.json"):
+            with open("measurements.json", "r") as file:
+                measurements = json.load(file)  # Load JSON data from the file
+        else:
+            measurements = {}
+        measurements['starting_weights_filename'] = env.get('CM_ML_MODEL_STARTING_WEIGHTS_FILENAME', env.get('CM_ML_MODEL_FILE', measurements.get('starting_weights_filename', '')))
+        measurements['retraining'] = env.get('CM_ML_MODEL_RETRAINING', measurements.get('retraining', 'no'))
+        measurements['input_data_types'] = env.get('CM_ML_MODEL_INPUTS_DATA_TYPE', measurements.get('input_data_types', 'fp32'))
+        measurements['weight_data_types'] = env.get('CM_ML_MODEL_WEIGHTS_DATA_TYPE', measurements.get('weight_data_types', 'fp32'))
+        measurements['weight_transformations'] = env.get('CM_ML_MODEL_WEIGHT_TRANSFORMATIONS', measurements.get('weight_transformations', 'none'))
 
         os.chdir(output_dir)
 

script/get-ml-model-mixtral/_cm.json

@@ -69,6 +69,7 @@
       "default": true,
       "env": {
         "CM_GIT_CHECKOUT_FOLDER": "Mixtral-8x7B-Instruct-v0.1",
+        "CM_ML_MODEL_STARTING_WEIGHTS_FILENAME": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
         "CM_MODEL_ZOO_ENV_KEY": "MIXTRAL"
       },
       "group": "huggingface-stub"

script/get-mlperf-inference-src/_cm.json

@@ -4,6 +4,7 @@
   "automation_uid": "5b4e0237da074764",
   "cache": true,
   "category": "MLPerf benchmark support",
+  "default_version": "master",
   "default_env": {
     "CM_GIT_CHECKOUT_FOLDER": "inference",
     "CM_GIT_DEPTH": "--depth 4",
@@ -92,7 +93,8 @@
       "env": {
         "CM_GIT_CHECKOUT": "#"
       },
-      "group": "checkout"
+      "group": "checkout",
+      "default_version": "custom"
     },
     "deeplearningexamples": {
       "env": {

script/get-mlperf-inference-src/customize.py

@@ -45,7 +45,7 @@ def preprocess(i):
             env["CM_GIT_URL"] = "https://github.com/mlcommons/inference"
 
     if env.get("CM_MLPERF_LAST_RELEASE", '') == '':
-        env["CM_MLPERF_LAST_RELEASE"] = "v4.0"
+        env["CM_MLPERF_LAST_RELEASE"] = "v4.1"
 
     if 'CM_GIT_DEPTH' not in env:
         env['CM_GIT_DEPTH'] = ''