Re-enable detectron2 model accuracy checks. (#2317)

Summary: Detectron2 models can pass the accuracy check when the deterministic mode is disabled. Revert abed1e7. Fixes #2290 Pull Request resolved: #2317 Reviewed By: aaronenyeshi Differential Revision: D58794036 Pulled By: xuzhao9 fbshipit-source-id: b0c38bd63c63b71e33f9692ad2169f8267e93ab1
pytorch · Jun 20, 2024 · 3bd542d · 3bd542d
1 parent b2b4158
commit 3bd542d
Show file tree

Hide file tree

Showing 14 changed files with 2 additions and 21 deletions.
diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_101_c4/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_101_c4/metadata.yaml
@@ -6,5 +6,3 @@ eval_deterministic: false
 eval_nograd: true
 train_benchmark: false
 train_deterministic: false
-not_implemented:
-  - test: example
diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_50_c4/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_50_c4/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_maskrcnn/__init__.py b/torchbenchmark/models/detectron2_maskrcnn/__init__.py
@@ -58,9 +58,7 @@ class Model(BenchmarkModel):
     model_file = os.path.join(MODEL_DIR, ".data", f"{MODEL_NAME}.pkl")
     DEFAULT_TRAIN_BSIZE = 1
     DEFAULT_EVAL_BSIZE = 1
-    # Skip correctness check, because the output tensor can't be verified using
-    # cosine similarity or torch.close()
-    SKIP_CORRECTNESS_CHECK = True
+    DISABLE_DETERMINISM = True
 
     def __init__(self, test, device, batch_size=None, extra_args=[]):
         super().__init__(

diff --git a/torchbenchmark/models/detectron2_maskrcnn/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_101_c4/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_101_c4/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_101_fpn/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_101_fpn/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_50_c4/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_50_c4/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_50_fpn/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_50_fpn/metadata.yaml
@@ -6,6 +6,5 @@ eval_deterministic: false
 eval_nograd: true
 not_implemented:
 - device: cpu
-- test: example
 train_benchmark: false
 train_deterministic: false
diff --git a/torchbenchmark/models/maml/__init__.py b/torchbenchmark/models/maml/__init__.py
@@ -16,9 +16,6 @@ class Model(BenchmarkModel):
     DEFAULT_EVAL_BSIZE = 1
     ALLOW_CUSTOMIZE_BSIZE = False
     CANNOT_SET_CUSTOM_OPTIMIZER = True
-    # Skip correctness check, because maml runs backward and optimizer in eval()
-    # Which will return non-deterministic results
-    SKIP_CORRECTNESS_CHECK = True
 
     def __init__(self, test, device, batch_size=None, extra_args=[]):
         super().__init__(

diff --git a/torchbenchmark/util/framework/detectron2/model_factory.py b/torchbenchmark/util/framework/detectron2/model_factory.py
@@ -90,9 +90,7 @@ class Detectron2Model(BenchmarkModel):
     # Default batch sizes
     DEFAULT_TRAIN_BSIZE = 1
     DEFAULT_EVAL_BSIZE = 1
-    # Skip correctness check, because the output tensor can't be verified using
-    # cosine similarity or torch.close()
-    SKIP_CORRECTNESS_CHECK = True
+    DISABLE_DETERMINISM = True
 
     def __init__(self, variant, test, device, batch_size=None, extra_args=[]):
         super().__init__(