From de84fde617c2679a1b48b8a8971a866e08a5d2b6 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 21:25:08 +0000
Subject: [PATCH 01/18] Create nightly job for torch.compile benchmarks

---
 .../benchmark_torch_compile_nightly.yml       | 78 +++++++++++++++++++
 .../benchmark_config_torch_compile_gpu.yaml   | 48 ++++++++++++
 .../models_config/bert_torch_compile_gpu.yaml | 42 ++++++++++
 .../resnet50_torch_compile_gpu.yaml           | 42 ++++++++++
 .../vgg16_torch_compile_gpu.yaml              | 42 ++++++++++
 5 files changed, 252 insertions(+)
 create mode 100644 .github/workflows/benchmark_torch_compile_nightly.yml
 create mode 100644 benchmarks/benchmark_config_torch_compile_gpu.yaml
 create mode 100644 benchmarks/models_config/bert_torch_compile_gpu.yaml
 create mode 100644 benchmarks/models_config/resnet50_torch_compile_gpu.yaml
 create mode 100644 benchmarks/models_config/vgg16_torch_compile_gpu.yaml

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
new file mode 100644
index 0000000000..679e923058
--- /dev/null
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -0,0 +1,78 @@
+name: Benchmark torch.compile models nightly
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  merge_group:
+  # run every day at 9:15pm
+  schedule:
+    - cron:  '15 21 * * *'
+
+jobs:
+  nightly:
+    strategy:
+      fail-fast: false
+      matrix:
+        hardware: [gpu]
+    runs-on:
+      - self-hosted
+    timeout-minutes: 1320
+    steps:
+      - name: Clean up previous run
+        run: |
+          echo "Cleaning up previous run"
+          cd $RUNNER_WORKSPACE
+          pwd
+          cd ..
+          pwd
+          rm -rf _tool
+      - name: Setup Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          architecture: x64
+      - name: Setup Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: Checkout TorchServe
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y apache2-utils
+          pip install -r benchmarks/requirements-ab.txt
+          pip uninstall torchtext torchdata torch torchvision torchaudio -y
+          pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121 --ignore-installed
+      - name: Benchmark gpu nightly
+        if: ${{ matrix.hardware == 'gpu' }}
+        run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false
+      - name: Save benchmark artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: nightly ${{ matrix.hardware }} torch compile artifact
+          path: /tmp/ts_benchmark
+      - name: Download benchmark artifacts for auto validation
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: ${{ github.event.workflow_run.workflow_id }}
+          workflow_conclusion: success
+          if_no_artifact_found: ignore
+          path: /tmp/ts_artifacts
+          name: ${{ matrix.hardware }}_torch_compile_benchmark_validation
+      - name: Validate Benchmark result
+        run: python benchmarks/validate_report.py --input-artifacts-dir /tmp/ts_artifacts/${{ matrix.hardware }}_torch_compile_benchmark_validation
+      - name: Update benchmark artifacts for auto validation
+        run: python benchmarks/utils/update_artifacts.py --output /tmp/ts_artifacts/${{ matrix.hardware }}_torch_compile_benchmark_validation
+      - name: Upload the updated benchmark artifacts for auto validation
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.hardware }}_torch_compile_benchmark_validation
+          path: /tmp/ts_artifacts
\ No newline at end of file
diff --git a/benchmarks/benchmark_config_torch_compile_gpu.yaml b/benchmarks/benchmark_config_torch_compile_gpu.yaml
new file mode 100644
index 0000000000..a2c222a792
--- /dev/null
+++ b/benchmarks/benchmark_config_torch_compile_gpu.yaml
@@ -0,0 +1,48 @@
+# Torchserve version is to be installed. It can be one of the options
+#  - branch : "master"
+#  - nightly: "2022.3.16"
+#  - release: "0.5.3"
+# Nightly build will be installed if "ts_version" is not specifiged
+#ts_version:
+#    branch: &ts_version "master"
+
+# a list of model configure yaml files defined in benchmarks/models_config
+# or a list of model configure yaml files with full path
+models:
+    - "bert_torch_compile_gpu.yaml"
+    - "resnet50_torch_compile_gpu.yaml"
+    - "vgg16_torch_compile_gpu.yaml"
+
+# benchmark on "cpu" or "gpu".
+# "cpu" is set if "hardware" is not specified
+hardware: &hardware "gpu"
+
+# load prometheus metrics report to remote storage or local different path if "metrics_cmd" is set.
+# the command line to load prometheus metrics report to remote system.
+# Here is an example of AWS cloudwatch command:
+# Note:
+#    - keep the values order as the same as the command definition.
+#    - set up the command before enabling `metrics_cmd`.
+#      For example, aws client and AWS credentials need to be setup before trying this example.
+metrics_cmd:
+    - "cmd": "aws cloudwatch put-metric-data"
+    - "--namespace": ["torchserve_benchmark_nightly_torch_compile_", *hardware]
+    - "--region": "us-east-2"
+    - "--metric-data": 'file:///tmp/benchmark/logs/stats_metrics.json'
+
+# load report to remote storage or local different path if "report_cmd" is set.
+# the command line to load report to remote storage.
+# Here is an example of AWS cloudwatch command:
+# Note:
+#    - keep the values order as the same as the command.
+#    - set up the command before enabling `report_cmd`.
+#      For example, aws client, AWS credentials and S3 bucket
+#      need to be setup before trying this example.
+#    - "today()" is a keyword to apply current date in the path
+#      For example, the dest path in the following example is
+#      s3://torchserve-model-serving/benchmark/2022-03-18/gpu
+report_cmd:
+    - "cmd": "aws s3 cp --recursive"
+    - "source": '/tmp/ts_benchmark/'
+    - "dest": ['s3://torchserve-benchmark/torch-compile-nightly', "today()", *hardware]
+
diff --git a/benchmarks/models_config/bert_torch_compile_gpu.yaml b/benchmarks/models_config/bert_torch_compile_gpu.yaml
new file mode 100644
index 0000000000..2895d09b5d
--- /dev/null
+++ b/benchmarks/models_config/bert_torch_compile_gpu.yaml
@@ -0,0 +1,42 @@
+---
+bert:
+    scripted_mode:
+        benchmark_engine: "ab"
+        url: https://torchserve.pytorch.org/mar_files/bert-scripted.mar
+        workers:
+            - 4
+        batch_delay: 100
+        batch_size:
+            - 1
+            - 2
+            - 4
+            - 8
+            - 16
+        input: "./examples/Huggingface_Transformers/Seq_classification_artifacts/sample_text_captum_input.txt"
+        requests: 50000
+        concurrency: 100
+        backend_profiling: False
+        exec_env: "local"
+        processors:
+            - "cpu"
+            - "gpus": "all"
+    torch_compile_default_mode:
+        benchmark_engine: "ab"
+        url: https://torchserve.pytorch.org/mar_files/bert-default.mar
+        workers:
+            - 4
+        batch_delay: 100
+        batch_size:
+            - 1
+            - 2
+            - 4
+            - 8
+            - 16
+        input: "./examples/Huggingface_Transformers/Seq_classification_artifacts/sample_text_captum_input.txt"
+        requests: 50000
+        concurrency: 100
+        backend_profiling: False
+        exec_env: "local"
+        processors:
+            - "cpu"
+            - "gpus": "all"
diff --git a/benchmarks/models_config/resnet50_torch_compile_gpu.yaml b/benchmarks/models_config/resnet50_torch_compile_gpu.yaml
new file mode 100644
index 0000000000..70a50988c9
--- /dev/null
+++ b/benchmarks/models_config/resnet50_torch_compile_gpu.yaml
@@ -0,0 +1,42 @@
+---
+resnet50:
+    scripted_mode:
+        benchmark_engine: "ab"
+        url: https://torchserve.pytorch.org/mar_files/resnet-50-scripted.mar
+        workers:
+            - 4
+        batch_delay: 100
+        batch_size:
+            - 1
+            - 2
+            - 4
+            - 8
+            - 16
+        input: "./examples/image_classifier/kitten.jpg"
+        requests: 10000
+        concurrency: 100
+        backend_profiling: False
+        exec_env: "local"
+        processors:
+            - "cpu"
+            - "gpus": "all"
+    torch_compile_default_mode:
+        benchmark_engine: "ab"
+        url: https://torchserve.pytorch.org/mar_files/resnet-50-default.mar
+        workers:
+            - 4
+        batch_delay: 100
+        batch_size:
+            - 1
+            - 2
+            - 4
+            - 8
+            - 16
+        input: "./examples/image_classifier/kitten.jpg"
+        requests: 10000
+        concurrency: 100
+        backend_profiling: False
+        exec_env: "local"
+        processors:
+            - "cpu"
+            - "gpus": "all"
diff --git a/benchmarks/models_config/vgg16_torch_compile_gpu.yaml b/benchmarks/models_config/vgg16_torch_compile_gpu.yaml
new file mode 100644
index 0000000000..3af84a2a72
--- /dev/null
+++ b/benchmarks/models_config/vgg16_torch_compile_gpu.yaml
@@ -0,0 +1,42 @@
+---
+vgg16:
+    scripted_mode:
+        benchmark_engine: "ab"
+        url: https://torchserve.pytorch.org/mar_files/vgg-16-scripted.mar
+        workers:
+            - 4
+        batch_delay: 100
+        batch_size:
+            - 1
+            - 2
+            - 4
+            - 8
+            - 16
+        input: "./examples/image_classifier/kitten.jpg"
+        requests: 10000
+        concurrency: 100
+        backend_profiling: False
+        exec_env: "local"
+        processors:
+            - "cpu"
+            - "gpus": "all"
+    torch_compile_default_mode:
+        benchmark_engine: "ab"
+        url: https://torchserve.pytorch.org/mar_files/vgg-16-default.mar
+        workers:
+            - 4
+        batch_delay: 100
+        batch_size:
+            - 1
+            - 2
+            - 4
+            - 8
+            - 16
+        input: "./examples/image_classifier/kitten.jpg"
+        requests: 10000
+        concurrency: 100
+        backend_profiling: False
+        exec_env: "local"
+        processors:
+            - "cpu"
+            - "gpus": "all"

From 8cdc6d7531175ca6aece707fe062562315db96fc Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 21:39:05 +0000
Subject: [PATCH 02/18] Run with torch nightly

---
 .github/workflows/benchmark_torch_compile_nightly.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index 679e923058..63d07f95e4 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -49,8 +49,6 @@ jobs:
           sudo apt-get update -y
           sudo apt-get install -y apache2-utils
           pip install -r benchmarks/requirements-ab.txt
-          pip uninstall torchtext torchdata torch torchvision torchaudio -y
-          pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121 --ignore-installed
       - name: Benchmark gpu nightly
         if: ${{ matrix.hardware == 'gpu' }}
         run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false

From 89cfd2bc70aa6aa3aab8bcd6998f9d8b252d425e Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 21:45:46 +0000
Subject: [PATCH 03/18] Add 300 second timeout in ab command

---
 benchmarks/benchmark-ab.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmarks/benchmark-ab.py b/benchmarks/benchmark-ab.py
index 6dd7966117..0204c9ae97 100644
--- a/benchmarks/benchmark-ab.py
+++ b/benchmarks/benchmark-ab.py
@@ -229,7 +229,7 @@ def warm_up():
     click.secho("\n\nExecuting warm-up ...", fg="green")
 
     ab_cmd = (
-        f"ab -c {execution_params['concurrency']}  -n {execution_params['requests']/10} -k -p "
+        f"ab -c {execution_params['concurrency']} -s 300 -n {execution_params['requests']/10} -k -p "
         f"{execution_params['tmp_dir']}/benchmark/input -T  {execution_params['content_type']} "
         f"{execution_params['inference_url']}/{execution_params['inference_model_url']} > "
         f"{execution_params['result_file']}"
@@ -247,7 +247,7 @@ def run_benchmark():
 
     click.secho("\n\nExecuting inference performance tests ...", fg="green")
     ab_cmd = (
-        f"ab -c {execution_params['concurrency']}  -n {execution_params['requests']} -k -p "
+        f"ab -c {execution_params['concurrency']} -s 300 -n {execution_params['requests']} -k -p "
         f"{execution_params['tmp_dir']}/benchmark/input -T  {execution_params['content_type']} "
         f"{execution_params['inference_url']}/{execution_params['inference_model_url']} > "
         f"{execution_params['result_file']}"

From 19633e235b59e185c78be4ee1d0a38efadb07de7 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 21:54:24 +0000
Subject: [PATCH 04/18] Retrigger tests


From 11a553819b16ae8f92631483f3b69830a4aa1909 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 23:03:22 +0000
Subject: [PATCH 05/18] Add support for nightly PyTorch in auto_benchmark.py

---
 benchmarks/auto_benchmark.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/benchmarks/auto_benchmark.py b/benchmarks/auto_benchmark.py
index 3849fdc502..a5e298b3bd 100644
--- a/benchmarks/auto_benchmark.py
+++ b/benchmarks/auto_benchmark.py
@@ -122,14 +122,14 @@ def load_benchmark_config(bm_config_path, skip_ts_install, skip_upload):
     return benchmark_config.bm_config
 
 
-def benchmark_env_setup(bm_config, skip_ts_install):
-    install_torchserve(skip_ts_install, bm_config["hardware"], bm_config["version"])
+def benchmark_env_setup(bm_config, skip_ts_install, nightly):
+    install_torchserve(skip_ts_install, bm_config["hardware"], bm_config["version"], nightly)
     setup_benchmark_path(bm_config["model_config_path"])
     build_model_json_config(bm_config["models"])
     enable_launcher_with_logical_core(bm_config["hardware"])
 
 
-def install_torchserve(skip_ts_install, hw, ts_version):
+def install_torchserve(skip_ts_install, hw, ts_version, nightly):
     if skip_ts_install:
         return
 
@@ -154,6 +154,8 @@ def install_torchserve(skip_ts_install, hw, ts_version):
         cmd = "python ts_scripts/install_dependencies.py --environment dev --neuronx"
     else:
         cmd = "python ts_scripts/install_dependencies.py --environment dev"
+    if nightly:
+        cmd += " --nightly_torch"
     execute(cmd, wait=True)
     print("successfully install install_dependencies.py")
 
@@ -290,9 +292,12 @@ def main():
     )
     parser.add_argument(
         "--skip_upload",
-        help="true: skip uploading commands . default: false",
+        help="true: skip uploading commands. default: false",
+    )
+    parser.add_argument(
+        "--nightly",
+        help="true: install nightly version of torch package. default: false"
     )
-
     arguments = parser.parse_args()
     skip_ts_config = (
         False
@@ -304,8 +309,13 @@ def main():
         if arguments.skip_upload is not None and arguments.skip_upload.lower() == "true"
         else False
     )
+    nightly = (
+        True
+        if arguments.nightly is not None and arguments.nightly.lower() == "true"
+        else False
+    )
     bm_config = load_benchmark_config(arguments.input, skip_ts_config, skip_upload)
-    benchmark_env_setup(bm_config, skip_ts_config)
+    benchmark_env_setup(bm_config, skip_ts_config, nightly)
     run_benchmark(bm_config)
     clean_up_benchmark_env(bm_config)
     print("benchmark_serving.sh finished successfully.")

From 1963abca4243fd304d3bc769f0496e481b0f7269 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 23:07:57 +0000
Subject: [PATCH 06/18] Retrigger tests


From a89e1b5fcaf4a637b2c128a85a279cd7542c37e2 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 23:08:28 +0000
Subject: [PATCH 07/18] Retrigger tests


From 80259d105a42b9af003666ab3de0f4da7cd0ca0b Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 23:09:04 +0000
Subject: [PATCH 08/18] Retrigger tests

---
 .github/workflows/benchmark_torch_compile_nightly.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index 63d07f95e4..0034a227e7 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -51,7 +51,7 @@ jobs:
           pip install -r benchmarks/requirements-ab.txt
       - name: Benchmark gpu nightly
         if: ${{ matrix.hardware == 'gpu' }}
-        run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false
+        run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false --nightly
       - name: Save benchmark artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -73,4 +73,4 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.hardware }}_torch_compile_benchmark_validation
-          path: /tmp/ts_artifacts
\ No newline at end of file
+          path: /tmp/ts_artifacts

From 17d1b35ae5605ec14580291e72fc945e39e58ac5 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 23:10:28 +0000
Subject: [PATCH 09/18] Retrigger tests

---
 .github/workflows/benchmark_torch_compile_nightly.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index 0034a227e7..c540c3c220 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -51,7 +51,7 @@ jobs:
           pip install -r benchmarks/requirements-ab.txt
       - name: Benchmark gpu nightly
         if: ${{ matrix.hardware == 'gpu' }}
-        run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false --nightly
+        run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false --nightly True
       - name: Save benchmark artifacts
         uses: actions/upload-artifact@v2
         with:

From e6000d563c395e208d5f47a75c4e79e081f0db5f Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Thu, 7 Dec 2023 23:40:51 +0000
Subject: [PATCH 10/18] Retrigger tests


From ed931b5aa307e47875596947f7a782bd1440164d Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 00:31:36 +0000
Subject: [PATCH 11/18] Retrigger tests

---
 .github/workflows/benchmark_torch_compile_nightly.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index c540c3c220..992f3879e3 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -20,6 +20,7 @@ jobs:
         hardware: [gpu]
     runs-on:
       - self-hosted
+      - ${{ matrix.hardware }}
     timeout-minutes: 1320
     steps:
       - name: Clean up previous run

From 3cb21307ab63bf7bdfeed13990f803a963133bce Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 00:39:37 +0000
Subject: [PATCH 12/18] Retrigger tests


From 322aeda811474cfdcf41bad618829ad82bf20d96 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 01:12:20 +0000
Subject: [PATCH 13/18] Retrigger tests

---
 .github/workflows/benchmark_torch_compile_nightly.yml |  6 +-----
 benchmarks/auto_benchmark.py                          | 10 ++++++----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index 992f3879e3..0afb1ada89 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -16,11 +16,7 @@ jobs:
   nightly:
     strategy:
       fail-fast: false
-      matrix:
-        hardware: [gpu]
-    runs-on:
-      - self-hosted
-      - ${{ matrix.hardware }}
+    runs-on: [self-hosted, gpu]
     timeout-minutes: 1320
     steps:
       - name: Clean up previous run
diff --git a/benchmarks/auto_benchmark.py b/benchmarks/auto_benchmark.py
index a5e298b3bd..177fb70e3f 100644
--- a/benchmarks/auto_benchmark.py
+++ b/benchmarks/auto_benchmark.py
@@ -123,7 +123,9 @@ def load_benchmark_config(bm_config_path, skip_ts_install, skip_upload):
 
 
 def benchmark_env_setup(bm_config, skip_ts_install, nightly):
-    install_torchserve(skip_ts_install, bm_config["hardware"], bm_config["version"], nightly)
+    install_torchserve(
+        skip_ts_install, bm_config["hardware"], bm_config["version"], nightly
+    )
     setup_benchmark_path(bm_config["model_config_path"])
     build_model_json_config(bm_config["models"])
     enable_launcher_with_logical_core(bm_config["hardware"])
@@ -134,8 +136,8 @@ def install_torchserve(skip_ts_install, hw, ts_version, nightly):
         return
 
     # git checkout branch if it is needed
-    cmd = "git checkout master && git reset --hard && git clean -dffx . && git pull --rebase"
-    execute(cmd, wait=True)
+    # cmd = "git checkout master && git reset --hard && git clean -dffx . && git pull --rebase"
+    # execute(cmd, wait=True)
     print("successfully reset git")
 
     ts_install_cmd = None
@@ -296,7 +298,7 @@ def main():
     )
     parser.add_argument(
         "--nightly",
-        help="true: install nightly version of torch package. default: false"
+        help="true: install nightly version of torch package. default: false",
     )
     arguments = parser.parse_args()
     skip_ts_config = (

From 806099ffeb961d5eaee6a9875db5fbbad0e75e83 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 01:15:53 +0000
Subject: [PATCH 14/18] Retrigger tests

---
 .github/workflows/benchmark_torch_compile_nightly.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index 0afb1ada89..65859b80d5 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -47,12 +47,11 @@ jobs:
           sudo apt-get install -y apache2-utils
           pip install -r benchmarks/requirements-ab.txt
       - name: Benchmark gpu nightly
-        if: ${{ matrix.hardware == 'gpu' }}
         run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false --nightly True
       - name: Save benchmark artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: nightly ${{ matrix.hardware }} torch compile artifact
+          name: nightly gpu torch compile artifact
           path: /tmp/ts_benchmark
       - name: Download benchmark artifacts for auto validation
         uses: dawidd6/action-download-artifact@v2
@@ -61,13 +60,13 @@ jobs:
           workflow_conclusion: success
           if_no_artifact_found: ignore
           path: /tmp/ts_artifacts
-          name: ${{ matrix.hardware }}_torch_compile_benchmark_validation
+          name: gpu_torch_compile_benchmark_validation
       - name: Validate Benchmark result
-        run: python benchmarks/validate_report.py --input-artifacts-dir /tmp/ts_artifacts/${{ matrix.hardware }}_torch_compile_benchmark_validation
+        run: python benchmarks/validate_report.py --input-artifacts-dir /tmp/ts_artifacts/gpu_torch_compile_benchmark_validation
       - name: Update benchmark artifacts for auto validation
-        run: python benchmarks/utils/update_artifacts.py --output /tmp/ts_artifacts/${{ matrix.hardware }}_torch_compile_benchmark_validation
+        run: python benchmarks/utils/update_artifacts.py --output /tmp/ts_artifacts/gpu_torch_compile_benchmark_validation
       - name: Upload the updated benchmark artifacts for auto validation
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ matrix.hardware }}_torch_compile_benchmark_validation
+          name: gpu_torch_compile_benchmark_validation
           path: /tmp/ts_artifacts

From fd3021228b4acdc8f718b5a7eab9765ab1161689 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 16:28:27 +0000
Subject: [PATCH 15/18] Retrigger tests


From 6ba753de764fd8fad94ed99a36eccd02a518fe11 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 20:09:20 +0000
Subject: [PATCH 16/18] Remove auto-validation of benchmark results from
 workflow

---
 .../benchmark_torch_compile_nightly.yml       | 22 -------------------
 1 file changed, 22 deletions(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index 65859b80d5..d118b16b38 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -48,25 +48,3 @@ jobs:
           pip install -r benchmarks/requirements-ab.txt
       - name: Benchmark gpu nightly
         run: python benchmarks/auto_benchmark.py --input benchmarks/benchmark_config_torch_compile_gpu.yaml --skip false --nightly True
-      - name: Save benchmark artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: nightly gpu torch compile artifact
-          path: /tmp/ts_benchmark
-      - name: Download benchmark artifacts for auto validation
-        uses: dawidd6/action-download-artifact@v2
-        with:
-          workflow: ${{ github.event.workflow_run.workflow_id }}
-          workflow_conclusion: success
-          if_no_artifact_found: ignore
-          path: /tmp/ts_artifacts
-          name: gpu_torch_compile_benchmark_validation
-      - name: Validate Benchmark result
-        run: python benchmarks/validate_report.py --input-artifacts-dir /tmp/ts_artifacts/gpu_torch_compile_benchmark_validation
-      - name: Update benchmark artifacts for auto validation
-        run: python benchmarks/utils/update_artifacts.py --output /tmp/ts_artifacts/gpu_torch_compile_benchmark_validation
-      - name: Upload the updated benchmark artifacts for auto validation
-        uses: actions/upload-artifact@v2
-        with:
-          name: gpu_torch_compile_benchmark_validation
-          path: /tmp/ts_artifacts

From 416bf4323a5313da871540bd12dc73ed8b97e948 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 20:43:00 +0000
Subject: [PATCH 17/18] Retrigger tests


From f8946011dff2109bc84cc072d3dfc40e7abefba0 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-15-190.us-west-2.compute.internal>
Date: Fri, 8 Dec 2023 22:57:45 +0000
Subject: [PATCH 18/18] Revert changes in auto_benchmark.py and remove push
 trigger in workflow

---
 .github/workflows/benchmark_torch_compile_nightly.yml | 7 -------
 benchmarks/auto_benchmark.py                          | 4 ++--
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/benchmark_torch_compile_nightly.yml b/.github/workflows/benchmark_torch_compile_nightly.yml
index d118b16b38..827977ef05 100644
--- a/.github/workflows/benchmark_torch_compile_nightly.yml
+++ b/.github/workflows/benchmark_torch_compile_nightly.yml
@@ -1,13 +1,6 @@
 name: Benchmark torch.compile models nightly
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-  merge_group:
   # run every day at 9:15pm
   schedule:
     - cron:  '15 21 * * *'
diff --git a/benchmarks/auto_benchmark.py b/benchmarks/auto_benchmark.py
index 177fb70e3f..c0d132dd2a 100644
--- a/benchmarks/auto_benchmark.py
+++ b/benchmarks/auto_benchmark.py
@@ -136,8 +136,8 @@ def install_torchserve(skip_ts_install, hw, ts_version, nightly):
         return
 
     # git checkout branch if it is needed
-    # cmd = "git checkout master && git reset --hard && git clean -dffx . && git pull --rebase"
-    # execute(cmd, wait=True)
+    cmd = "git checkout master && git reset --hard && git clean -dffx . && git pull --rebase"
+    execute(cmd, wait=True)
     print("successfully reset git")
 
     ts_install_cmd = None