Skip to content

Commit

Permalink
Merge pull request #94 from anandhu-eng/mlperf-inference
Browse files Browse the repository at this point in the history
MLPerf Inference Updates
  • Loading branch information
arjunsuresh authored Jul 2, 2024
2 parents a07fbee + 61440ca commit b18ff89
Show file tree
Hide file tree
Showing 66 changed files with 1,308 additions and 375 deletions.
32 changes: 29 additions & 3 deletions .github/workflows/test-mlperf-inference-abtf-poc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
name: MLPerf inference ABTF POC Test

on:
pull_request_target:
pull_request:
branches: [ "main", "mlperf-inference" ]
paths:
- '.github/workflows/test-mlperf-inference-abtf-poc.yml'
Expand Down Expand Up @@ -37,7 +37,7 @@ jobs:
cm pull repo mlcommons@cm4abtf --branch=poc
- name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker
run: |
cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet --docker --docker_it=no -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet -v
build2:
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -65,5 +65,31 @@ jobs:
cm pull repo mlcommons@cm4abtf --branch=poc
- name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
run: |
cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet -v
build3:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [windows-latest]
python-version: [ "3.8", "3.12" ]
backend: [ "pytorch" ]
implementation: [ "python" ]
exclude:
- python-version: "3.8"

steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python3 -m pip install cmind
cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
cm pull repo mlcommons@cm4abtf --branch=poc
- name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
run: |
cm run script --tags=run-abtf,inference,_poc-demo --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off -v
8 changes: 6 additions & 2 deletions automation/script/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# as portable and reusable automation recipes with simple tags, native scripts
# and a unified CLI, Python API and JSON/YAML meta descriptions.
#
# This is a stable prototype of the CM script automation
# This is a stable prototype of the CM script automation being developed by Grigori Fursin and Arjun Suresh
#
# TBD: when we have bandwidth and resources, we should refactor it
# and make it cleaner and simpler while keeping full backwards compatibility.
Expand Down Expand Up @@ -4443,7 +4443,11 @@ def update_env_with_values(env, fail_on_not_found=False):
if tmp_value not in env and fail_on_not_found:
return {'return':1, 'error':'variable {} is not in env'.format(tmp_value)}
if tmp_value in env:
value = value.replace("<<<"+tmp_value+">>>", str(env[tmp_value]))
if type(value) == str:
value = value.replace("<<<"+tmp_value+">>>", str(env[tmp_value]))
elif type(value) == list:
for i,val in enumerate(value):
value[i] = value[i].replace("<<<"+tmp_value+">>>", str(env[tmp_value]))

env[key] = value

Expand Down
41 changes: 41 additions & 0 deletions cm-repro/cm-run-script-input.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{
"action": "run",
"automation": "script",
"tags": "run-mlperf,inference,_find-performance,_full",
"model": "mixtral-8x7b-99",
"implementation": "reference",
"framework": "pytorch",
"category": "edge",
"scenario": "Offline",
"execution_mode": "test",
"device": "cuda",
"test_query_count": "100",
"adr": {
"cuda": {
"version": "12.4.1"
}
},
"quiet": true,
"repro": true,
"cmd": [
"--tags=run-mlperf,inference,_find-performance,_full",
"--model=mixtral-8x7b-99",
"--implementation=reference",
"--framework=pytorch",
"--category=edge",
"--scenario=Offline",
"--execution_mode=test",
"--device=cuda",
"--test_query_count=100",
"--adr.cuda.version=12.4.1",
"--quiet",
"--repro"
],
"out": "con",
"parsed_automation": [
[
"script",
"5b4e0237da074764"
]
]
}
Loading

0 comments on commit b18ff89

Please sign in to comment.