aggregate.py: fix relative error computation (pytorch#6522)

Ouch.
amithrm · Mar 1, 2024 · d471269 · d471269
1 parent 016748d
commit d471269
Show file tree

Hide file tree

Showing 10 changed files with 16 additions and 16 deletions.
diff --git a/benchmarks/aggregate.py b/benchmarks/aggregate.py
@@ -168,8 +168,8 @@ def process_file(args, results_map: Dict[str, Any], filename: str):
 
 # Speedup of a over baseline ("b"), with errors.
 def compute_speedup(a: NamedTuple, b: NamedTuple) -> NamedTuple:
-  rel_err_a = a.avg * a.std
-  rel_err_b = b.avg * b.std
+  rel_err_a = a.std / a.avg
+  rel_err_b = b.std / b.avg
   rel_err = math.sqrt(rel_err_a**2 + rel_err_b**2)
   speedup = b.avg / a.avg
   err = rel_err * speedup

diff --git a/test/benchmarks/v100.inference.latest.tab.test b/test/benchmarks/v100.inference.latest.tab.test
@@ -8,5 +8,5 @@
 ╞════════════╪════════════╪══════════╪════════════════════╪══════════════╪══════════╪════════════════════╪═══════════════════╪══════════╪════════════════════╪══════════════════╪══════════╪════════════════════╡
 │          0 │       1.30 │     0.00 │ Background_Matting │         0.77 │     0.00 │ Background_Matting │              0.73 │     0.00 │ Background_Matting │             0.41 │     0.00 │ Background_Matting │
 ├────────────┼────────────┼──────────┼────────────────────┼──────────────┼──────────┼────────────────────┼───────────────────┼──────────┼────────────────────┼──────────────────┼──────────┼────────────────────┤
-│          1 │       1.52 │     0.00 │ BERT_pytorch       │         1.57 │     0.00 │ BERT_pytorch       │              1.37 │     0.00 │ BERT_pytorch       │                  │          │                    │
+│          1 │       1.52 │     0.07 │ BERT_pytorch       │         1.57 │     0.07 │ BERT_pytorch       │              1.37 │     0.06 │ BERT_pytorch       │                  │          │                    │
 ╘════════════╧════════════╧══════════╧════════════════════╧══════════════╧══════════╧════════════════════╧═══════════════════╧══════════╧════════════════════╧══════════════════╧══════════╧════════════════════╛
diff --git a/test/benchmarks/v100.inference.latest.test b/test/benchmarks/v100.inference.latest.test
@@ -1,4 +1,4 @@
 # ARGS: --backends inductor openxla+dynamo openxla_eval+dynamo openxla+lazytensor --
 # Workload,Speedup(Inductor/Oldest Inductor),StdDev,ModelName(Inductor),Speedup(XLA+Dynamo/Oldest Inductor),StdDev,ModelName(XLA+Dynamo),Speedup(XLA_Eval+Dynamo/Oldest Inductor),StdDev,ModelName(XLA_Eval+Dynamo),Speedup(XLA+LazyTensor/Oldest Inductor),StdDev,ModelName(XLA+LazyTensor)
 0,1.2957024,0.0,Background_Matting,0.77297688,0.0,Background_Matting,0.7341254,0.0,Background_Matting,0.41071322,0.0,Background_Matting
-1,1.51952596,6.914e-05,BERT_pytorch,1.56880282,7.138e-05,BERT_pytorch,1.36859903,6.227e-05,BERT_pytorch,,,
+1,1.51952596,0.06679279,BERT_pytorch,1.56880282,0.06895882,BERT_pytorch,1.36859903,0.06015859,BERT_pytorch,,,
diff --git a/test/benchmarks/v100.inference.latest.tier1.test b/test/benchmarks/v100.inference.latest.tier1.test
@@ -1,3 +1,3 @@
 # ARGS: --filter-by-tier=1
 # Workload,Speedup(Inductor/Oldest Inductor),StdDev,ModelName(Inductor),Speedup(XLA+Dynamo/Oldest Inductor),StdDev,ModelName(XLA+Dynamo),Speedup(XLA_Eval+Dynamo/Oldest Inductor),StdDev,ModelName(XLA_Eval+Dynamo)
-0,1.51952596,6.914e-05,BERT_pytorch,1.56880282,7.138e-05,BERT_pytorch,1.36859903,6.227e-05,BERT_pytorch
+0,1.51952596,0.06679279,BERT_pytorch,1.56880282,0.06895882,BERT_pytorch,1.36859903,0.06015859,BERT_pytorch
diff --git a/test/benchmarks/v100.inference.latest_grouped.test b/test/benchmarks/v100.inference.latest_grouped.test
@@ -1,5 +1,5 @@
 # ARGS: --backends inductor openxla+dynamo openxla_eval+dynamo openxla+lazytensor --
 # ModelName,Speedup(Inductor/Oldest Inductor),StdDev,Speedup(XLA+Dynamo/Oldest Inductor),StdDev,Speedup(XLA_Eval+Dynamo/Oldest Inductor),StdDev,Speedup(XLA+LazyTensor/Oldest Inductor),StdDev
 Background_Matting,1.2957024,0.0,0.77297688,0.0,0.7341254,0.0,0.41071322,0.0
-BERT_pytorch,1.51952596,6.914e-05,1.56880282,7.138e-05,1.36859903,6.227e-05,,
-GEOMEAN,1.40315838,3.192e-05,1.10120312,2.505e-05,1.00235887,2.28e-05,0.41071322,0.0
+BERT_pytorch,1.51952596,0.06679279,1.56880282,0.06895882,1.36859903,0.06015859,,
+GEOMEAN,1.40315838,0.03083885,1.10120312,0.02420242,1.00235887,0.02203001,0.41071322,0.0
diff --git a/test/benchmarks/v100.inference.speedup.baseline_latest.test b/test/benchmarks/v100.inference.speedup.baseline_latest.test
@@ -1,4 +1,4 @@
 # ARGS: --baseline=latest
 # Datetime(UTC),Speedup(Inductor/Latest Inductor),StdDev,Speedup(XLA+Dynamo/Latest Inductor),StdDev,Speedup(XLA_Eval+Dynamo/Latest Inductor),StdDev
-2023-11-11 05:32:18.723407,0.71267792,1.621e-05,0.60245072,0.0,0.55375084,0.0
+2023-11-11 05:32:18.723407,0.71267792,0.01566335,0.60245072,0.0,0.55375084,0.0
 2023-11-12 05:32:18,1.0,0.0,0.78480315,0.0,0.71435904,0.0
diff --git a/test/benchmarks/v100.inference.speedup.lazytensor.test b/test/benchmarks/v100.inference.speedup.lazytensor.test
@@ -1,4 +1,4 @@
 # ARGS: --backends inductor openxla+lazytensor --
 # Datetime(UTC),Speedup(Inductor/Oldest Inductor),StdDev,Speedup(XLA+LazyTensor/Oldest Inductor),StdDev
-2023-11-11 05:32:18.723407,1.0,3.217e-05,,
-2023-11-12 05:32:18,1.40315838,3.192e-05,0.41071322,0.0
+2023-11-11 05:32:18.723407,1.0,0.03108182,,
+2023-11-12 05:32:18,1.40315838,0.03083885,0.41071322,0.0
diff --git a/test/benchmarks/v100.inference.speedup.lazytensor_tab.test b/test/benchmarks/v100.inference.speedup.lazytensor_tab.test
@@ -6,7 +6,7 @@
 │                            │     Oldest │          │           Oldest │          │
 │                            │   Inductor │          │         Inductor │          │
 ╞════════════════════════════╪════════════╪══════════╪══════════════════╪══════════╡
-│ 2023-11-11 05:32:18.723407 │       1.00 │     0.00 │                  │          │
+│ 2023-11-11 05:32:18.723407 │       1.00 │     0.03 │                  │          │
 ├────────────────────────────┼────────────┼──────────┼──────────────────┼──────────┤
-│ 2023-11-12 05:32:18        │       1.40 │     0.00 │             0.41 │     0.00 │
+│ 2023-11-12 05:32:18        │       1.40 │     0.03 │             0.41 │     0.00 │
 ╘════════════════════════════╧════════════╧══════════╧══════════════════╧══════════╛
diff --git a/test/benchmarks/v100.inference.speedup.tab.test b/test/benchmarks/v100.inference.speedup.tab.test
@@ -6,7 +6,7 @@
 │                            │     Oldest │          │       Oldest │          │            Oldest │          │
 │                            │   Inductor │          │     Inductor │          │          Inductor │          │
 ╞════════════════════════════╪════════════╪══════════╪══════════════╪══════════╪═══════════════════╪══════════╡
-│ 2023-11-11 05:32:18.723407 │       1.00 │     0.00 │         0.85 │     0.00 │              0.78 │     0.00 │
+│ 2023-11-11 05:32:18.723407 │       1.00 │     0.03 │         0.85 │     0.02 │              0.78 │     0.02 │
 ├────────────────────────────┼────────────┼──────────┼──────────────┼──────────┼───────────────────┼──────────┤
-│ 2023-11-12 05:32:18        │       1.40 │     0.00 │         1.10 │     0.00 │              1.00 │     0.00 │
+│ 2023-11-12 05:32:18        │       1.40 │     0.03 │         1.10 │     0.02 │              1.00 │     0.02 │
 ╘════════════════════════════╧════════════╧══════════╧══════════════╧══════════╧═══════════════════╧══════════╛
diff --git a/test/benchmarks/v100.inference.speedup.test b/test/benchmarks/v100.inference.speedup.test
@@ -1,3 +1,3 @@
 # Datetime(UTC),Speedup(Inductor/Oldest Inductor),StdDev,Speedup(XLA+Dynamo/Oldest Inductor),StdDev,Speedup(XLA_Eval+Dynamo/Oldest Inductor),StdDev
-2023-11-11 05:32:18.723407,1.0,3.217e-05,0.84533378,1.923e-05,0.77700013,1.768e-05
-2023-11-12 05:32:18,1.40315838,3.192e-05,1.10120312,2.505e-05,1.00235887,2.28e-05
+2023-11-11 05:32:18.723407,1.0,0.03108182,0.84533378,0.01857889,0.77700013,0.01707704
+2023-11-12 05:32:18,1.40315838,0.03083885,1.10120312,0.02420242,1.00235887,0.02203001