From 59af71cf4e324e94bb019ece50e70f4b2b44a926 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Sun, 29 Sep 2024 05:13:39 +0000
Subject: [PATCH 01/15] update plotting util and readme

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 docs/user/InstructionsForAutoTuner.md       |  12 +++
 tools/AutoTuner/src/autotuner/utils/plot.py | 102 ++++++++++++++++++++
 2 files changed, 114 insertions(+)
 create mode 100644 tools/AutoTuner/src/autotuner/utils/plot.py

diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md
index 7b43950e05..c7012d3780 100644
--- a/docs/user/InstructionsForAutoTuner.md
+++ b/docs/user/InstructionsForAutoTuner.md
@@ -147,6 +147,18 @@ python3 distributed.py --design gcd --platform sky130hd \
                        sweep
 ```
 
+#### Plot images
+
+After running the autotuner experiments, you can visualize the results as follows.
+Currently, we support the following metrics:
+- QoR
+- Runtime per trial
+- Clock Period
+- Worst slack
+
+```shell
+python3 utils/plot.py --results_dir <your-autotuner-result-path>
+```
 
 ### Google Cloud Platform (GCP) distribution with Ray
 
diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
new file mode 100644
index 0000000000..fafaa7fa8f
--- /dev/null
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -0,0 +1,102 @@
+import glob
+import json
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import re
+import argparse
+
+AT_REGEX = r"variant-AutoTunerBase-([\w-]+)-\w+"
+
+
+def load_dir(dir: str) -> pd.DataFrame:
+    # Concatenate progress DFs
+    df = pd.concat([pd.read_csv(fname) for fname in glob.glob(f"{dir}/*/progress.csv")])
+
+    # Concatenate params.json & metrics.json file
+    params = []
+    for fname in glob.glob(f"{dir}/*/params.json"):
+        try:
+            with open(fname, "r") as f:
+                _dict = json.load(f)
+                _dict["trial_id"] = re.search(AT_REGEX, fname).group(1)
+            with open(fname.replace("params.json", "metrics.json"), "r") as f:
+                metrics = json.load(f)
+                ws = metrics["finish"]["timing__setup__ws"]
+                metrics["worst_slack"] = ws
+                _dict.update(metrics)
+            params.append(_dict)
+        except Exception as e:
+            print(f"Error in {fname}: {e}")
+            continue
+    tmp_df = pd.DataFrame(params)
+
+    # Merge all dataframe
+    df = df.merge(tmp_df, on="trial_id")
+    return df
+
+
+def preprocess(df: pd.DataFrame) -> pd.DataFrame:
+    cols_to_remove = [
+        "done",
+        "training_iteration",
+        "date",
+        "pid",
+        "hostname",
+        "node_ip",
+        "time_since_restore",
+        "time_total_s",
+        "iterations_since_restore",
+    ]
+    rename_dict = {
+        "time_this_iter_s": "runtime",
+        "_SDC_CLK_PERIOD": "clk_period",
+        "minimum": "qor",
+    }
+    df = df.rename(columns=rename_dict)
+    df = df.drop(columns=cols_to_remove)
+    df = df[df["qor"] != 9e99]
+    df["timestamp"] -= df["timestamp"].min()
+    return df
+
+
+def plot(df: pd.DataFrame, key: str):
+    # Plot box plot and time series plot for key
+    fig, ax = plt.subplots(1, figsize=(15, 10))
+    ax.scatter(df["timestamp"], df[key])
+    ax.set_xlabel("Time (s)")
+    ax.set_ylabel(key)
+    ax.set_title(f"{key} vs Time")
+    z = np.polyfit(df["timestamp"], df[key], 1)
+    p = np.poly1d(z)
+    ax.plot(
+        df["timestamp"], p(df["timestamp"]), "r--", label=f"y={z[0]:.2f}x+{z[1]:.2f}"
+    )
+    ax.legend()
+    fig.savefig(f"images/{key}.png")
+
+    plt.figure(figsize=(15, 10))
+    plt.boxplot(df[key])
+    plt.ylabel(key)
+    plt.title(f"{key} Boxplot")
+    plt.savefig(f"images/{key}-boxplot.png")
+
+
+def main(results_dir: str):
+    df = load_dir(results_dir)
+    df = preprocess(df)
+    keys = ["qor", "runtime", "clk_period", "worst_slack"]
+    for key in keys:
+        plot(df, key)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Plot AutoTuner results.")
+    parser.add_argument(
+        "results_dir",
+        type=str,
+        help="Directory containing the results.",
+        default="../../../../../flow/logs/asap7/gcd/test-tune-2024-09-17-12-00-44",
+    )
+    args = parser.parse_args()
+    main(args.results_dir)

From c4b78314d6f1bf1f664e7c692c6d4be79f72940d Mon Sep 17 00:00:00 2001
From: luarss <jluar@precisioninno.com>
Date: Thu, 10 Oct 2024 22:05:15 +0800
Subject: [PATCH 02/15] make arg required and update docs

Signed-off-by: luarss <jluar@precisioninno.com>
---
 docs/user/InstructionsForAutoTuner.md       | 5 +++--
 tools/AutoTuner/src/autotuner/utils/plot.py | 5 +----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md
index c7012d3780..0e9ac9e43a 100644
--- a/docs/user/InstructionsForAutoTuner.md
+++ b/docs/user/InstructionsForAutoTuner.md
@@ -149,8 +149,9 @@ python3 distributed.py --design gcd --platform sky130hd \
 
 #### Plot images
 
-After running the autotuner experiments, you can visualize the results as follows.
-Currently, we support the following metrics:
+After running an AutoTuner experiment, you can generate a graph to understand the results better.
+The graph will show one of the metrics listed below progression over the execution of the experiment.
+
 - QoR
 - Runtime per trial
 - Clock Period
diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index fafaa7fa8f..bf45b7cdeb 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -93,10 +93,7 @@ def main(results_dir: str):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Plot AutoTuner results.")
     parser.add_argument(
-        "results_dir",
-        type=str,
-        help="Directory containing the results.",
-        default="../../../../../flow/logs/asap7/gcd/test-tune-2024-09-17-12-00-44",
+        "results_dir", type=str, help="Directory containing the results.", required=True
     )
     args = parser.parse_args()
     main(args.results_dir)

From 989f88c1e514e4d4dd27b2d6fc65ce22901659ef Mon Sep 17 00:00:00 2001
From: Vitor Bandeira <vvbandeira@users.noreply.github.com>
Date: Thu, 10 Oct 2024 13:58:04 -0300
Subject: [PATCH 03/15] Update docs/user/InstructionsForAutoTuner.md

Signed-off-by: Vitor Bandeira <vvbandeira@users.noreply.github.com>
---
 docs/user/InstructionsForAutoTuner.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md
index 0e9ac9e43a..51db4d5d01 100644
--- a/docs/user/InstructionsForAutoTuner.md
+++ b/docs/user/InstructionsForAutoTuner.md
@@ -150,7 +150,7 @@ python3 distributed.py --design gcd --platform sky130hd \
 #### Plot images
 
 After running an AutoTuner experiment, you can generate a graph to understand the results better.
-The graph will show one of the metrics listed below progression over the execution of the experiment.
+The graph will show the progression of one metric (see list below) over the execution of the experiment.
 
 - QoR
 - Runtime per trial

From a7eeaaf487fb43d2b79114ed897fb9a8b2c7d572 Mon Sep 17 00:00:00 2001
From: luarss <jluar@precisioninno.com>
Date: Fri, 11 Oct 2024 22:58:15 +0800
Subject: [PATCH 04/15] add one call to test plot.py

Signed-off-by: luarss <jluar@precisioninno.com>
---
 tools/AutoTuner/src/autotuner/utils/plot.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index bf45b7cdeb..3ba4dc5aa3 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -4,9 +4,11 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 import re
+import os
 import argparse
 
 AT_REGEX = r"variant-AutoTunerBase-([\w-]+)-\w+"
+IMG_DIR = "images"
 
 
 def load_dir(dir: str) -> pd.DataFrame:
@@ -83,6 +85,8 @@ def plot(df: pd.DataFrame, key: str):
 
 
 def main(results_dir: str):
+    # Default: saves to <REPO_ROOT>/images. Change the IMG_DIR above.
+    os.makedirs(IMG_DIR, exist_ok=True)
     df = load_dir(results_dir)
     df = preprocess(df)
     keys = ["qor", "runtime", "clk_period", "worst_slack"]
@@ -93,7 +97,10 @@ def main(results_dir: str):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Plot AutoTuner results.")
     parser.add_argument(
-        "results_dir", type=str, help="Directory containing the results.", required=True
+        "--results_dir",
+        type=str,
+        help="Directory containing the results.",
+        required=True,
     )
     args = parser.parse_args()
     main(args.results_dir)

From 9eb4e16739cccb778f09b1e9dedbab01b54fc0a0 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Sat, 11 Jan 2025 07:02:28 +0000
Subject: [PATCH 05/15] add test_autotuner call, matplotlib reqs

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh      | 4 ++++
 tools/AutoTuner/requirements.txt | 1 +
 2 files changed, 5 insertions(+)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index 7f8c258753..d1b794297e 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -26,6 +26,10 @@ if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then
   echo "Running Autotuner ref file test (only once)"
   python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files
 
+  echo "Running Autotuner plotting smoke test (only once)"`
+  latest_folder=$(ls -dt ./flow/logs/asap7/gcd/test-tune*/ | tail -n 1)
+  python3 tools/AutoTuner/src/autotuner/utils/plot.py --results_dir $latest_folder
+
   echo "Running AutoTuner resume test (only once)"
   python3 -m unittest tools.AutoTuner.test.resume_check.ResumeCheck.test_tune_resume
 fi
diff --git a/tools/AutoTuner/requirements.txt b/tools/AutoTuner/requirements.txt
index 5bf65305cc..1870eb73e4 100644
--- a/tools/AutoTuner/requirements.txt
+++ b/tools/AutoTuner/requirements.txt
@@ -9,3 +9,4 @@ tensorboard>=2.14.0,<=2.16.2
 protobuf==3.20.3
 SQLAlchemy==1.4.17
 urllib3<=1.26.15
+matplotlib==3.10.0

From 6c550c326d180433b958f4a36c8b0ef4adcc4edb Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Sat, 11 Jan 2025 08:26:33 +0000
Subject: [PATCH 06/15] temporarily disable unstable tests

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index d1b794297e..279fbf423d 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -26,7 +26,7 @@ if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then
   echo "Running Autotuner ref file test (only once)"
   python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files
 
-  echo "Running Autotuner plotting smoke test (only once)"`
+  echo "Running Autotuner plotting smoke test (only once)"
   latest_folder=$(ls -dt ./flow/logs/asap7/gcd/test-tune*/ | tail -n 1)
   python3 tools/AutoTuner/src/autotuner/utils/plot.py --results_dir $latest_folder
 

From e98a5b36dcbb0b0dfb6d43cec7e438b573e70191 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Sat, 11 Jan 2025 16:48:50 +0000
Subject: [PATCH 07/15] print all files at the end of load_dir

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 tools/AutoTuner/src/autotuner/utils/plot.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index 3ba4dc5aa3..f21d5d605e 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -17,24 +17,27 @@ def load_dir(dir: str) -> pd.DataFrame:
 
     # Concatenate params.json & metrics.json file
     params = []
-    for fname in glob.glob(f"{dir}/*/params.json"):
+    failed = []
+    for params_fname in glob.glob(f"{dir}/*/params.json"):
         try:
-            with open(fname, "r") as f:
+            with open(params_fname, "r") as f:
                 _dict = json.load(f)
-                _dict["trial_id"] = re.search(AT_REGEX, fname).group(1)
-            with open(fname.replace("params.json", "metrics.json"), "r") as f:
+                _dict["trial_id"] = re.search(AT_REGEX, params_fname).group(1)
+            metrics_fname = params_fname.replace("params.json", "metrics.json")
+            with open(metrics_fname, "r") as f:
                 metrics = json.load(f)
                 ws = metrics["finish"]["timing__setup__ws"]
                 metrics["worst_slack"] = ws
                 _dict.update(metrics)
             params.append(_dict)
         except Exception as e:
-            print(f"Error in {fname}: {e}")
+            failed.append(metrics_fname)
             continue
     tmp_df = pd.DataFrame(params)
 
     # Merge all dataframe
     df = df.merge(tmp_df, on="trial_id")
+    print(f"Failed to load {len(failed)} files:\n{'\n'.join(failed)}")
     return df
 
 

From c476a5d18d3ab76740b5614aadd3f249d49bbd39 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Sun, 12 Jan 2025 04:46:27 +0000
Subject: [PATCH 08/15] refactors: - run plot.py for all log files in the
 autotuner supported design/platforms - parameterised img_dir, so we can store
 all AT image runs in build artifacts - argparse interface now takes in
 platform, design, experiment for easier usage - chdir before running plot.py
 - add docstrings - more error handling

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh                 |  18 ++-
 tools/AutoTuner/src/autotuner/utils/plot.py | 143 ++++++++++++++++----
 2 files changed, 127 insertions(+), 34 deletions(-)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index 279fbf423d..3e027a8f83 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -26,12 +26,22 @@ if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then
   echo "Running Autotuner ref file test (only once)"
   python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files
 
-  echo "Running Autotuner plotting smoke test (only once)"
-  latest_folder=$(ls -dt ./flow/logs/asap7/gcd/test-tune*/ | tail -n 1)
-  python3 tools/AutoTuner/src/autotuner/utils/plot.py --results_dir $latest_folder
-
   echo "Running AutoTuner resume test (only once)"
   python3 -m unittest tools.AutoTuner.test.resume_check.ResumeCheck.test_tune_resume
 fi
 
+echo "Running Autotuner plotting smoke test"
+all_experiments=$(ls -d ./flow/logs/${PLATFORM}/${DESIGN_NAME}/*/)
+if [ -z "$all_experiments" ]; then
+  echo "No experiments found for plotting"
+  exit 0
+fi
+all_experiments=$(basename -a $all_experiments)
+for expt in $all_experiments; do
+  python3 tools/AutoTuner/src/autotuner/utils/plot.py \
+    --platform ${PLATFORM} \
+    --design ${DESIGN_NAME} \
+    --experiment $expt
+done
+
 exit $ret
diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index f21d5d605e..00beac5275 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -6,24 +6,43 @@
 import re
 import os
 import argparse
+import sys
 
+# Only does plotting for AutoTunerBase variants
 AT_REGEX = r"variant-AutoTunerBase-([\w-]+)-\w+"
-IMG_DIR = "images"
+
+cur_dir = os.path.dirname(os.path.abspath(__file__))
+root_dir = os.path.join(cur_dir, "../../../../../")
+os.chdir(root_dir)
 
 
 def load_dir(dir: str) -> pd.DataFrame:
+    """
+    Load and merge progress, parameters, and metrics data from a specified directory.
+    This function searches for `progress.csv`, `params.json`, and `metrics.json` files within the given directory,
+    concatenates the data, and merges them into a single pandas DataFrame.
+    Args:
+        dir (str): The directory path containing the subdirectories with `progress.csv`, `params.json`, and `metrics.json` files.
+    Returns:
+        pd.DataFrame: A DataFrame containing the merged data from the progress, parameters, and metrics files.
+    """
+
     # Concatenate progress DFs
-    df = pd.concat([pd.read_csv(fname) for fname in glob.glob(f"{dir}/*/progress.csv")])
+    progress_csvs = glob.glob(f"{dir}/*/progress.csv")
+    if len(progress_csvs) == 0:
+        print("No progress.csv files found.")
+        sys.exit(0)
+    progress_df = pd.concat([pd.read_csv(f) for f in progress_csvs])
 
     # Concatenate params.json & metrics.json file
     params = []
     failed = []
     for params_fname in glob.glob(f"{dir}/*/params.json"):
+        metrics_fname = params_fname.replace("params.json", "metrics.json")
         try:
             with open(params_fname, "r") as f:
                 _dict = json.load(f)
                 _dict["trial_id"] = re.search(AT_REGEX, params_fname).group(1)
-            metrics_fname = params_fname.replace("params.json", "metrics.json")
             with open(metrics_fname, "r") as f:
                 metrics = json.load(f)
                 ws = metrics["finish"]["timing__setup__ws"]
@@ -33,15 +52,34 @@ def load_dir(dir: str) -> pd.DataFrame:
         except Exception as e:
             failed.append(metrics_fname)
             continue
-    tmp_df = pd.DataFrame(params)
 
     # Merge all dataframe
-    df = df.merge(tmp_df, on="trial_id")
-    print(f"Failed to load {len(failed)} files:\n{'\n'.join(failed)}")
-    return df
+    params_df = pd.DataFrame(params)
+    try:
+        progress_df = progress_df.merge(params_df, on="trial_id")
+    except KeyError:
+        print(
+            "Unable to merge DFs due to missing trial_id in params.json (possibly due to failed trials.)"
+        )
+        sys.exit(0)
+
+    # Print failed, if any
+    if failed:
+        print(f"Failed to load {len(failed)} files:\n{'\n'.join(failed)}")
+    return progress_df
 
 
 def preprocess(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Preprocess the input DataFrame by renaming columns, removing unnecessary columns,
+    filtering out invalid rows, and normalizing the timestamp.
+    Args:
+        df (pd.DataFrame): The input DataFrame to preprocess.
+    Returns:
+        pd.DataFrame: The preprocessed DataFrame with renamed columns, removed columns,
+                      filtered rows, and normalized timestamp.
+    """
+
     cols_to_remove = [
         "done",
         "training_iteration",
@@ -58,52 +96,97 @@ def preprocess(df: pd.DataFrame) -> pd.DataFrame:
         "_SDC_CLK_PERIOD": "clk_period",
         "minimum": "qor",
     }
-    df = df.rename(columns=rename_dict)
-    df = df.drop(columns=cols_to_remove)
-    df = df[df["qor"] != 9e99]
-    df["timestamp"] -= df["timestamp"].min()
-    return df
+    try:
+        df = df.rename(columns=rename_dict)
+        df = df.drop(columns=cols_to_remove)
+        df = df[df["qor"] != 9e99]
+        df["timestamp"] -= df["timestamp"].min()
+        return df
+    except KeyError as e:
+        print(
+            f"KeyError: {e} in the DataFrame. Dataframe does not contain necessary columns."
+        )
+        sys.exit(0)
+
 
+def plot(df: pd.DataFrame, key: str, dir: str):
+    """
+    Plots a scatter plot with a linear fit and a box plot for a specified key from a DataFrame.
+    Args:
+        df (pd.DataFrame): The DataFrame containing the data to plot.
+        key (str): The column name in the DataFrame to plot.
+        dir (str): The directory where the plots will be saved. The directory must exist.
+    Returns:
+        None
+    """
 
-def plot(df: pd.DataFrame, key: str):
+    assert os.path.exists(dir), f"Directory {dir} does not exist."
     # Plot box plot and time series plot for key
     fig, ax = plt.subplots(1, figsize=(15, 10))
     ax.scatter(df["timestamp"], df[key])
     ax.set_xlabel("Time (s)")
     ax.set_ylabel(key)
     ax.set_title(f"{key} vs Time")
-    z = np.polyfit(df["timestamp"], df[key], 1)
-    p = np.poly1d(z)
-    ax.plot(
-        df["timestamp"], p(df["timestamp"]), "r--", label=f"y={z[0]:.2f}x+{z[1]:.2f}"
-    )
-    ax.legend()
-    fig.savefig(f"images/{key}.png")
+
+    try:
+        coeff = np.polyfit(df["timestamp"], df[key], 1)
+        poly_func = np.poly1d(coeff)
+        ax.plot(
+            df["timestamp"],
+            poly_func(df["timestamp"]),
+            "r--",
+            label=f"y={coeff[0]:.2f}x+{coeff[1]:.2f}",
+        )
+        ax.legend()
+    except np.linalg.LinAlgError:
+        print("Cannot fit a line to the data, plotting only scatter plot.")
+
+    fig.savefig(f"{dir}/{key}.png")
 
     plt.figure(figsize=(15, 10))
     plt.boxplot(df[key])
     plt.ylabel(key)
     plt.title(f"{key} Boxplot")
-    plt.savefig(f"images/{key}-boxplot.png")
+    plt.savefig(f"{dir}/{key}-boxplot.png")
+
 
+def main(platform: str, design: str, experiment: str):
+    """
+    Main function to process results from a specified directory and plot the results.
+    Args:
+        platform (str): The platform name.
+        design (str): The design name.
+        experiment (str): The experiment name.
+    Returns:
+        None
+    """
 
-def main(results_dir: str):
-    # Default: saves to <REPO_ROOT>/images. Change the IMG_DIR above.
-    os.makedirs(IMG_DIR, exist_ok=True)
+    results_dir = os.path.join(
+        root_dir, f"./flow/logs/{platform}/{design}/{experiment}"
+    )
+    img_dir = os.path.join(
+        root_dir, f"./flow/reports/images/{platform}/{design}/{experiment}"
+    )
+    print("Processing results from", results_dir)
+    os.makedirs(img_dir, exist_ok=True)
     df = load_dir(results_dir)
     df = preprocess(df)
     keys = ["qor", "runtime", "clk_period", "worst_slack"]
+
+    # Plot only if more than one entry
+    if len(df) < 2:
+        print("Less than 2 entries, skipping plotting.")
+        sys.exit(0)
     for key in keys:
-        plot(df, key)
+        plot(df, key, img_dir)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Plot AutoTuner results.")
+    parser.add_argument("--platform", type=str, help="Platform name.", required=True)
+    parser.add_argument("--design", type=str, help="Design name.", required=True)
     parser.add_argument(
-        "--results_dir",
-        type=str,
-        help="Directory containing the results.",
-        required=True,
+        "--experiment", type=str, help="Experiment name.", required=True
     )
     args = parser.parse_args()
-    main(args.results_dir)
+    main(platform=args.platform, design=args.design, experiment=args.experiment)

From e320d883781b7c31fa94df06705d51ef2552cc3a Mon Sep 17 00:00:00 2001
From: luarss <jluar@precisioninno.com>
Date: Mon, 13 Jan 2025 21:49:55 +0800
Subject: [PATCH 09/15] fix lowercase platform

Signed-off-by: luarss <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index 3e027a8f83..94bba719c1 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -9,9 +9,9 @@ cd ../
 . ./tools/AutoTuner/setup.sh
 
 # remove dashes and capitalize platform name
-PLATFORM=${PLATFORM//-/}
+LOWERCASE_PLATFORM=${PLATFORM//-/}
 # convert to uppercase
-PLATFORM=${PLATFORM^^}
+PLATFORM=${LOWERCASE_PLATFORM^^}
 
 echo "Running Autotuner smoke tune test"
 python3 -m unittest tools.AutoTuner.test.smoke_test_tune.${PLATFORM}TuneSmokeTest.test_tune
@@ -31,7 +31,7 @@ if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then
 fi
 
 echo "Running Autotuner plotting smoke test"
-all_experiments=$(ls -d ./flow/logs/${PLATFORM}/${DESIGN_NAME}/*/)
+all_experiments=$(ls -d ./flow/logs/${LOWERCASE_PLATFORM}/${DESIGN_NAME}/*/)
 if [ -z "$all_experiments" ]; then
   echo "No experiments found for plotting"
   exit 0

From f3b90e67b8e1fd9178936404f5ac7c44813e1528 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Mon, 13 Jan 2025 17:39:47 +0000
Subject: [PATCH 10/15] fix syntax for string join

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 tools/AutoTuner/src/autotuner/utils/plot.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index 00beac5275..f1dfc8606b 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -65,7 +65,8 @@ def load_dir(dir: str) -> pd.DataFrame:
 
     # Print failed, if any
     if failed:
-        print(f"Failed to load {len(failed)} files:\n{'\n'.join(failed)}")
+        failed_files = "\n".join(failed)
+        print(f"Failed to load {len(failed)} files:\n{failed_files}")
     return progress_df
 
 

From d2ee915f0de94f3240000ca7cbaf6fceecbb26a8 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Tue, 14 Jan 2025 12:46:39 +0000
Subject: [PATCH 11/15] fix LOWERCASE_PLATFORM

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index 94bba719c1..907ace830e 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -39,7 +39,7 @@ fi
 all_experiments=$(basename -a $all_experiments)
 for expt in $all_experiments; do
   python3 tools/AutoTuner/src/autotuner/utils/plot.py \
-    --platform ${PLATFORM} \
+    --platform ${LOWERCASE_PLATFORM} \
     --design ${DESIGN_NAME} \
     --experiment $expt
 done

From 91756069f49dc636efd9425b44d753f57a1be334 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Tue, 14 Jan 2025 16:55:00 +0000
Subject: [PATCH 12/15] Change qor key to metric

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 tools/AutoTuner/src/autotuner/utils/plot.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index f1dfc8606b..2ef03cb643 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -11,6 +11,9 @@
 # Only does plotting for AutoTunerBase variants
 AT_REGEX = r"variant-AutoTunerBase-([\w-]+)-\w+"
 
+# TODO: Make sure the distributed.py METRIC variable is consistent with this, single source of truth.
+METRIC = "metric"
+
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 root_dir = os.path.join(cur_dir, "../../../../../")
 os.chdir(root_dir)
@@ -94,13 +97,12 @@ def preprocess(df: pd.DataFrame) -> pd.DataFrame:
     ]
     rename_dict = {
         "time_this_iter_s": "runtime",
-        "_SDC_CLK_PERIOD": "clk_period",
-        "minimum": "qor",
+        "_SDC_CLK_PERIOD": "clk_period",  # param
     }
     try:
         df = df.rename(columns=rename_dict)
         df = df.drop(columns=cols_to_remove)
-        df = df[df["qor"] != 9e99]
+        df = df[df[METRIC] != 9e99]
         df["timestamp"] -= df["timestamp"].min()
         return df
     except KeyError as e:
@@ -172,7 +174,7 @@ def main(platform: str, design: str, experiment: str):
     os.makedirs(img_dir, exist_ok=True)
     df = load_dir(results_dir)
     df = preprocess(df)
-    keys = ["qor", "runtime", "clk_period", "worst_slack"]
+    keys = [METRIC] + ["runtime", "clk_period", "worst_slack"]
 
     # Plot only if more than one entry
     if len(df) < 2:

From 869e5759bf43de4157454861fabdec011cdb7bc3 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Thu, 16 Jan 2025 14:38:55 +0000
Subject: [PATCH 13/15] PLATFORM now strictly lowercase in scripts

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh                         | 9 ++-------
 tools/AutoTuner/test/smoke_test_algo_eval.py        | 6 +++---
 tools/AutoTuner/test/smoke_test_sample_iteration.py | 6 +++---
 tools/AutoTuner/test/smoke_test_sweep.py            | 6 +++---
 tools/AutoTuner/test/smoke_test_tune.py             | 6 +++---
 5 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index 907ace830e..f620b9799c 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -8,11 +8,6 @@ cd ../
 ./tools/AutoTuner/installer.sh
 . ./tools/AutoTuner/setup.sh
 
-# remove dashes and capitalize platform name
-LOWERCASE_PLATFORM=${PLATFORM//-/}
-# convert to uppercase
-PLATFORM=${LOWERCASE_PLATFORM^^}
-
 echo "Running Autotuner smoke tune test"
 python3 -m unittest tools.AutoTuner.test.smoke_test_tune.${PLATFORM}TuneSmokeTest.test_tune
 
@@ -31,7 +26,7 @@ if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then
 fi
 
 echo "Running Autotuner plotting smoke test"
-all_experiments=$(ls -d ./flow/logs/${LOWERCASE_PLATFORM}/${DESIGN_NAME}/*/)
+all_experiments=$(ls -d ./flow/logs/${PLATFORM}/${DESIGN_NAME}/*/)
 if [ -z "$all_experiments" ]; then
   echo "No experiments found for plotting"
   exit 0
@@ -39,7 +34,7 @@ fi
 all_experiments=$(basename -a $all_experiments)
 for expt in $all_experiments; do
   python3 tools/AutoTuner/src/autotuner/utils/plot.py \
-    --platform ${LOWERCASE_PLATFORM} \
+    --platform ${PLATFORM} \
     --design ${DESIGN_NAME} \
     --experiment $expt
 done
diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py
index a695489b48..7e364e9b8b 100644
--- a/tools/AutoTuner/test/smoke_test_algo_eval.py
+++ b/tools/AutoTuner/test/smoke_test_algo_eval.py
@@ -46,7 +46,7 @@ def make_base(self):
         os.chdir(src_dir)
 
 
-class ASAP7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
+class asap7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
     platform = "asap7"
     design = "gcd"
 
@@ -60,7 +60,7 @@ def test_algo_eval(self):
             self.assertTrue(successful)
 
 
-class IHPSG13G2AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
+class ihpsg13g2AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
     platform = "ihp-sg13g2"
     design = "gcd"
 
@@ -74,7 +74,7 @@ def test_algo_eval(self):
             self.assertTrue(successful)
 
 
-class SKY130HDAlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
+class sky130hdAlgoEvalSmokeTest(BaseAlgoEvalSmokeTest):
     platform = "sky130hd"
     design = "gcd"
 
diff --git a/tools/AutoTuner/test/smoke_test_sample_iteration.py b/tools/AutoTuner/test/smoke_test_sample_iteration.py
index f49c22a088..af8788cf99 100644
--- a/tools/AutoTuner/test/smoke_test_sample_iteration.py
+++ b/tools/AutoTuner/test/smoke_test_sample_iteration.py
@@ -29,7 +29,7 @@ def setUp(self):
         ]
 
 
-class ASAP7SampleIterationSmokeTest(BaseSampleIterationSmokeTest):
+class asap7SampleIterationSmokeTest(BaseSampleIterationSmokeTest):
     platform = "asap7"
     design = "gcd"
 
@@ -41,7 +41,7 @@ def test_sample_iteration(self):
             self.assertTrue(successful)
 
 
-class SKY130HDSampleIterationSmokeTest(BaseSampleIterationSmokeTest):
+class sky130hdSampleIterationSmokeTest(BaseSampleIterationSmokeTest):
     platform = "sky130hd"
     design = "gcd"
 
@@ -53,7 +53,7 @@ def test_sample_iteration(self):
             self.assertTrue(successful)
 
 
-class IHPSG13G2SampleIterationSmokeTest(BaseSampleIterationSmokeTest):
+class ihpsg13g2SampleIterationSmokeTest(BaseSampleIterationSmokeTest):
     platform = "ihp-sg13g2"
     design = "gcd"
 
diff --git a/tools/AutoTuner/test/smoke_test_sweep.py b/tools/AutoTuner/test/smoke_test_sweep.py
index 7a1b013911..389cd5e730 100644
--- a/tools/AutoTuner/test/smoke_test_sweep.py
+++ b/tools/AutoTuner/test/smoke_test_sweep.py
@@ -46,7 +46,7 @@ def test_sweep(self):
         )
 
 
-class ASAP7SweepSmokeTest(BaseSweepSmokeTest):
+class asap7SweepSmokeTest(BaseSweepSmokeTest):
     platform = "asap7"
     design = "gcd"
 
@@ -56,7 +56,7 @@ def test_sweep(self):
         self.assertTrue(successful)
 
 
-class SKY130HDSweepSmokeTest(BaseSweepSmokeTest):
+class sky130hdSweepSmokeTest(BaseSweepSmokeTest):
     platform = "sky130hd"
     design = "gcd"
 
@@ -66,7 +66,7 @@ def test_sweep(self):
         self.assertTrue(successful)
 
 
-class IHPSG13G2SweepSmokeTest(BaseSweepSmokeTest):
+class ihpsg13g2SweepSmokeTest(BaseSweepSmokeTest):
     platform = "ihp-sg13g2"
     design = "gcd"
 
diff --git a/tools/AutoTuner/test/smoke_test_tune.py b/tools/AutoTuner/test/smoke_test_tune.py
index 9710416745..129d2c1ffe 100644
--- a/tools/AutoTuner/test/smoke_test_tune.py
+++ b/tools/AutoTuner/test/smoke_test_tune.py
@@ -32,7 +32,7 @@ def test_tune(self):
         )
 
 
-class ASAP7TuneSmokeTest(BaseTuneSmokeTest):
+class asap7TuneSmokeTest(BaseTuneSmokeTest):
     platform = "asap7"
     design = "gcd"
 
@@ -42,7 +42,7 @@ def test_tune(self):
         self.assertTrue(successful)
 
 
-class SKY130HDTuneSmokeTest(BaseTuneSmokeTest):
+class sky130hdTuneSmokeTest(BaseTuneSmokeTest):
     platform = "sky130hd"
     design = "gcd"
 
@@ -52,7 +52,7 @@ def test_tune(self):
         self.assertTrue(successful)
 
 
-class IHPSG13G2TuneSmokeTest(BaseTuneSmokeTest):
+class ihpsg13g2TuneSmokeTest(BaseTuneSmokeTest):
     platform = "ihp-sg13g2"
     design = "gcd"
 

From ad154f55a5b10fc6fec7d9ceccbe1cfc1a79e924 Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Thu, 16 Jan 2025 14:42:07 +0000
Subject: [PATCH 14/15] change plot.py sys exits

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/test/test_autotuner.sh                 | 6 +-----
 tools/AutoTuner/src/autotuner/utils/plot.py | 6 +++---
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh
index f620b9799c..8157689a5a 100755
--- a/flow/test/test_autotuner.sh
+++ b/flow/test/test_autotuner.sh
@@ -26,11 +26,7 @@ if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then
 fi
 
 echo "Running Autotuner plotting smoke test"
-all_experiments=$(ls -d ./flow/logs/${PLATFORM}/${DESIGN_NAME}/*/)
-if [ -z "$all_experiments" ]; then
-  echo "No experiments found for plotting"
-  exit 0
-fi
+all_experiments=$(ls -d ./flow/logs/${PLATFORM}/${DESIGN_NAME}/smoke-test-tune*)
 all_experiments=$(basename -a $all_experiments)
 for expt in $all_experiments; do
   python3 tools/AutoTuner/src/autotuner/utils/plot.py \
diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
index 2ef03cb643..6e2af4d827 100644
--- a/tools/AutoTuner/src/autotuner/utils/plot.py
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -34,7 +34,7 @@ def load_dir(dir: str) -> pd.DataFrame:
     progress_csvs = glob.glob(f"{dir}/*/progress.csv")
     if len(progress_csvs) == 0:
         print("No progress.csv files found.")
-        sys.exit(0)
+        sys.exit(1)
     progress_df = pd.concat([pd.read_csv(f) for f in progress_csvs])
 
     # Concatenate params.json & metrics.json file
@@ -64,7 +64,7 @@ def load_dir(dir: str) -> pd.DataFrame:
         print(
             "Unable to merge DFs due to missing trial_id in params.json (possibly due to failed trials.)"
         )
-        sys.exit(0)
+        sys.exit(1)
 
     # Print failed, if any
     if failed:
@@ -109,7 +109,7 @@ def preprocess(df: pd.DataFrame) -> pd.DataFrame:
         print(
             f"KeyError: {e} in the DataFrame. Dataframe does not contain necessary columns."
         )
-        sys.exit(0)
+        sys.exit(1)
 
 
 def plot(df: pd.DataFrame, key: str, dir: str):

From 9a6566c54b45bb040210b9b60fd9e742c15b1cef Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Thu, 16 Jan 2025 18:54:40 +0000
Subject: [PATCH 15/15] Fix genReport

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 flow/util/genReport.py               | 5 ++++-
 tools/AutoTuner/test/resume_check.py | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/flow/util/genReport.py b/flow/util/genReport.py
index c6bf63b1ba..ebc9a7f625 100755
--- a/flow/util/genReport.py
+++ b/flow/util/genReport.py
@@ -19,6 +19,7 @@
 METRICS_CHECK_FMT = "{}/metadata-{}-check.log"
 REGEX_ERROR = re.compile(r"^\[error ?(\w+-\d+)?\]", re.IGNORECASE)
 REGEX_WARNING = re.compile(r"^\[warning ?(\w+-\d+)?\]", re.IGNORECASE)
+SKIPPED_FLOW_VARIANT_KEYWORDS = ["test", "tune"]
 STATUS_GREEN = "Passing"
 STATUS_RED = "Failing"
 
@@ -248,7 +249,9 @@ def write_summary():
     dir_list = log_dir.split(os.sep)
     # Handles autotuner folders, which do not have `report.log` natively.
     # TODO: Can we log something for autotuner?
-    if len(dir_list) != 4 or "test-" in dir_list[-1]:
+    if len(dir_list) != 4 or any(
+        word in dir_list[-1] for word in SKIPPED_FLOW_VARIANT_KEYWORDS
+    ):
         continue
     report_dir = log_dir.replace(LOGS_FOLDER, REPORTS_FOLDER)
 
diff --git a/tools/AutoTuner/test/resume_check.py b/tools/AutoTuner/test/resume_check.py
index 92219eed22..0fa6423e87 100644
--- a/tools/AutoTuner/test/resume_check.py
+++ b/tools/AutoTuner/test/resume_check.py
@@ -51,7 +51,7 @@ def setUp(self):
             f" --platform {self.platform}"
             f" --config {self.config}"
             f" --jobs {self.jobs}"
-            f" --experiment test_resume"
+            f" --experiment test-resume"
             f" tune --iterations {self.iterations} --samples {self.samples}"
             f" --resources_per_trial {res_per_trial}"
             f" {c}"