MLPerf Inference Updates

.github/workflows/test-mlperf-inference-abtf-poc.yml

@@ -3,7 +3,7 @@
 name: MLPerf inference ABTF POC Test
 
 on:
-  pull_request_target:
+  pull_request:
     branches: [ "main", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-mlperf-inference-abtf-poc.yml'
@@ -37,7 +37,7 @@ jobs:
         cm pull repo mlcommons@cm4abtf --branch=poc
     - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker
       run: |
-        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet --docker --docker_it=no  -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
+        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet  -v
 
   build2:
     runs-on: ${{ matrix.os }}
@@ -65,5 +65,31 @@ jobs:
         cm pull repo mlcommons@cm4abtf --branch=poc
     - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
       run: |
-        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet  -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
+        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet  -v
 
+  build3:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest]
+        python-version: [ "3.8", "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+        exclude:
+        - python-version: "3.8"
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+        cm pull repo mlcommons@cm4abtf --branch=poc
+    - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
+      run: |
+        cm run script --tags=run-abtf,inference,_poc-demo --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off  -v

automation/script/module.py

@@ -3,7 +3,7 @@
 # as portable and reusable automation recipes with simple tags, native scripts 
 # and a unified CLI, Python API and JSON/YAML meta descriptions.
 #
-# This is a stable prototype of the CM script automation
+# This is a stable prototype of the CM script automation being developed by Grigori Fursin and Arjun Suresh
 #
 # TBD: when we have bandwidth and resources, we should refactor it
 # and make it cleaner and simpler while keeping full backwards compatibility.
@@ -4443,7 +4443,11 @@ def update_env_with_values(env, fail_on_not_found=False):
             if tmp_value not in env and fail_on_not_found:
                 return {'return':1, 'error':'variable {} is not in env'.format(tmp_value)}
             if tmp_value in env:
-                value = value.replace("<<<"+tmp_value+">>>", str(env[tmp_value]))
+                if type(value) == str:
+                    value = value.replace("<<<"+tmp_value+">>>", str(env[tmp_value]))
+                elif type(value) == list:
+                    for i,val in enumerate(value):
+                        value[i] = value[i].replace("<<<"+tmp_value+">>>", str(env[tmp_value]))
 
         env[key] = value
 

cm-repro/cm-run-script-input.json

@@ -0,0 +1,41 @@
+{
+  "action": "run",
+  "automation": "script",
+  "tags": "run-mlperf,inference,_find-performance,_full",
+  "model": "mixtral-8x7b-99",
+  "implementation": "reference",
+  "framework": "pytorch",
+  "category": "edge",
+  "scenario": "Offline",
+  "execution_mode": "test",
+  "device": "cuda",
+  "test_query_count": "100",
+  "adr": {
+    "cuda": {
+      "version": "12.4.1"
+    }
+  },
+  "quiet": true,
+  "repro": true,
+  "cmd": [
+    "--tags=run-mlperf,inference,_find-performance,_full",
+    "--model=mixtral-8x7b-99",
+    "--implementation=reference",
+    "--framework=pytorch",
+    "--category=edge",
+    "--scenario=Offline",
+    "--execution_mode=test",
+    "--device=cuda",
+    "--test_query_count=100",
+    "--adr.cuda.version=12.4.1",
+    "--quiet",
+    "--repro"
+  ],
+  "out": "con",
+  "parsed_automation": [
+    [
+      "script",
+      "5b4e0237da074764"
+    ]
+  ]
+}