From ee55ade430d3c6c8bc6cfea301d659eab53bc908 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 26 Sep 2024 12:30:02 +0530 Subject: [PATCH 1/5] initial commit for self hosted mixtral gha --- .../test-mlperf-inference-mixtral.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-mixtral.yml diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml new file mode 100644 index 000000000..a5c3f452e --- /dev/null +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference MIXTRAL-8x7B + +on: + schedule: + - cron: "1 2 * * *" + +jobs: + build_reference: + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64 ] + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + backend: [ "pytorch" ] + device: [ "cpu", "cuda" ] + + steps: + - name: Install dependencies + run: | + source gh_action/bin/deactivate || python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation + run: | + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean From 51346cf0ae9969e5cc928e67b2775c190282d9df Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:17:05 +0530 Subject: [PATCH 2/5] Updated device tag and cron job schedule --- .github/workflows/test-mlperf-inference-mixtral.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index a5c3f452e..9de8f2f70 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -5,12 +5,12 @@ name: MLPerf inference MIXTRAL-8x7B on: schedule: - - cron: "1 2 * * *" + - cron: "30 20 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST jobs: build_reference: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64 ] + runs-on: [ self-hosted, GO-i9, linux, x64 ] strategy: fail-fast: false matrix: @@ -24,7 +24,8 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + python3 -m pip install cm4mlops + cm pull repo - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean From bcec9ec54eb3b0408cc2033e6f86e05a995c5b23 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 8 Oct 2024 00:46:41 +0530 Subject: [PATCH 3/5] Updated run commands --- .github/workflows/test-mlperf-inference-mixtral.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index 9de8f2f70..e09988f29 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -28,4 +28,5 @@ jobs: cm pull repo - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions From 7b8cacf9a167e8656b6cef9a47f4f7860cf68ba9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 18:55:49 +0100 Subject: [PATCH 4/5] Update test-mlperf-inference-mixtral.yml --- .github/workflows/test-mlperf-inference-mixtral.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index e09988f29..d7a358ae1 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -19,14 +19,12 @@ jobs: device: [ "cpu", "cuda" ] steps: - - name: Install dependencies + - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation run: | source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation - run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions From 17114558ec5c24995158d14d8ec6780fd95eab96 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 8 Oct 2024 23:41:28 +0530 Subject: [PATCH 5/5] removed cuda device --- .github/workflows/test-mlperf-inference-mixtral.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index d7a358ae1..b5716e78f 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -16,7 +16,7 @@ jobs: matrix: python-version: [ "3.12" ] backend: [ "pytorch" ] - device: [ "cpu", "cuda" ] + device: [ "cpu" ] steps: - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation