From 8c62a06dba31389e7613ef4622b6a4f8aa0e189e Mon Sep 17 00:00:00 2001
From: Jack Luar <jluar@precisioninno.com>
Date: Sun, 29 Sep 2024 05:13:39 +0000
Subject: [PATCH] update plotting util and readme

Signed-off-by: Jack Luar <jluar@precisioninno.com>
---
 docs/user/InstructionsForAutoTuner.md       |  12 +++
 tools/AutoTuner/src/autotuner/utils/plot.py | 102 ++++++++++++++++++++
 2 files changed, 114 insertions(+)
 create mode 100644 tools/AutoTuner/src/autotuner/utils/plot.py

diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md
index c1e2fc1c41..5fc38d2acb 100644
--- a/docs/user/InstructionsForAutoTuner.md
+++ b/docs/user/InstructionsForAutoTuner.md
@@ -142,6 +142,18 @@ python3 distributed.py --design gcd --platform sky130hd \
                        sweep
 ```
 
+#### Plot images
+
+After running the autotuner experiments, you can visualize the results as follows.
+Currently, we support the following metrics:
+- QoR
+- Runtime per trial
+- Clock Period
+- Worst slack
+
+```shell
+python3 utils/plot.py --results_dir <your-autotuner-result-path>
+```
 
 ### Google Cloud Platform (GCP) distribution with Ray
 
diff --git a/tools/AutoTuner/src/autotuner/utils/plot.py b/tools/AutoTuner/src/autotuner/utils/plot.py
new file mode 100644
index 0000000000..fafaa7fa8f
--- /dev/null
+++ b/tools/AutoTuner/src/autotuner/utils/plot.py
@@ -0,0 +1,102 @@
+import glob
+import json
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import re
+import argparse
+
+AT_REGEX = r"variant-AutoTunerBase-([\w-]+)-\w+"
+
+
+def load_dir(dir: str) -> pd.DataFrame:
+    # Concatenate progress DFs
+    df = pd.concat([pd.read_csv(fname) for fname in glob.glob(f"{dir}/*/progress.csv")])
+
+    # Concatenate params.json & metrics.json file
+    params = []
+    for fname in glob.glob(f"{dir}/*/params.json"):
+        try:
+            with open(fname, "r") as f:
+                _dict = json.load(f)
+                _dict["trial_id"] = re.search(AT_REGEX, fname).group(1)
+            with open(fname.replace("params.json", "metrics.json"), "r") as f:
+                metrics = json.load(f)
+                ws = metrics["finish"]["timing__setup__ws"]
+                metrics["worst_slack"] = ws
+                _dict.update(metrics)
+            params.append(_dict)
+        except Exception as e:
+            print(f"Error in {fname}: {e}")
+            continue
+    tmp_df = pd.DataFrame(params)
+
+    # Merge all dataframe
+    df = df.merge(tmp_df, on="trial_id")
+    return df
+
+
+def preprocess(df: pd.DataFrame) -> pd.DataFrame:
+    cols_to_remove = [
+        "done",
+        "training_iteration",
+        "date",
+        "pid",
+        "hostname",
+        "node_ip",
+        "time_since_restore",
+        "time_total_s",
+        "iterations_since_restore",
+    ]
+    rename_dict = {
+        "time_this_iter_s": "runtime",
+        "_SDC_CLK_PERIOD": "clk_period",
+        "minimum": "qor",
+    }
+    df = df.rename(columns=rename_dict)
+    df = df.drop(columns=cols_to_remove)
+    df = df[df["qor"] != 9e99]
+    df["timestamp"] -= df["timestamp"].min()
+    return df
+
+
+def plot(df: pd.DataFrame, key: str):
+    # Plot box plot and time series plot for key
+    fig, ax = plt.subplots(1, figsize=(15, 10))
+    ax.scatter(df["timestamp"], df[key])
+    ax.set_xlabel("Time (s)")
+    ax.set_ylabel(key)
+    ax.set_title(f"{key} vs Time")
+    z = np.polyfit(df["timestamp"], df[key], 1)
+    p = np.poly1d(z)
+    ax.plot(
+        df["timestamp"], p(df["timestamp"]), "r--", label=f"y={z[0]:.2f}x+{z[1]:.2f}"
+    )
+    ax.legend()
+    fig.savefig(f"images/{key}.png")
+
+    plt.figure(figsize=(15, 10))
+    plt.boxplot(df[key])
+    plt.ylabel(key)
+    plt.title(f"{key} Boxplot")
+    plt.savefig(f"images/{key}-boxplot.png")
+
+
+def main(results_dir: str):
+    df = load_dir(results_dir)
+    df = preprocess(df)
+    keys = ["qor", "runtime", "clk_period", "worst_slack"]
+    for key in keys:
+        plot(df, key)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Plot AutoTuner results.")
+    parser.add_argument(
+        "results_dir",
+        type=str,
+        help="Directory containing the results.",
+        default="../../../../../flow/logs/asap7/gcd/test-tune-2024-09-17-12-00-44",
+    )
+    args = parser.parse_args()
+    main(args.results_dir)