diff --git a/benchmarks/experiment_runner.py b/benchmarks/experiment_runner.py index f0afbda0cadd..336e36c4287e 100644 --- a/benchmarks/experiment_runner.py +++ b/benchmarks/experiment_runner.py @@ -120,8 +120,7 @@ def generate_and_run_all_configs(self): # # Otherwise, we should go ahead and execute it. if (not self._args.no_skip and not self.model_loader.is_compatible( - benchmark_model, benchmark_experiment, - self._args.strict_compatible)): + benchmark_model, benchmark_experiment)): logger.warning("SKIP incompatible model and experiment configs.") self._save_results(benchmark_experiment.to_dict(), benchmark_model.to_dict(), {"error": "SKIP"}) @@ -883,11 +882,6 @@ def __str__(self): action="store_true", help="""If set, verifies the model output with PT Eager mode, and saves relative error to the output file.""" ) - parser.add_argument( - "--strict-compatible", - action="store_true", - help="Strictly skips some models including models without installation file or causing stackdump.", - ) parser.add_argument( "--no-skip", action="store_true", diff --git a/benchmarks/torchbench_model.py b/benchmarks/torchbench_model.py index eb7ed0e24c13..fd30ee359871 100644 --- a/benchmarks/torchbench_model.py +++ b/benchmarks/torchbench_model.py @@ -135,25 +135,6 @@ ], } -# This strict deny list denies tests that hold for too long and timeoout. -STRICT_DENY_LIST = { - **{ - "opacus_cifar10": [{ - "accelerator": "tpu", - },], # stackdump issue in TPU - "pytorch_stargan": [{ - "accelerator": "tpu", - },], # stackdump issue in TPU - "soft_actor_critic": [{ - "accelerator": "tpu", - },], # stackdump issue in TPU - "speech_transformer": [{ - "accelerator": "tpu", - },], # stackdump issue in TPU - }, - **DENY_LIST -} - class TorchBenchModelLoader(ModelLoader): @@ -234,12 +215,8 @@ def list_model_configs(self): return model_configs - def is_compatible(self, - dummy_benchmark_model, - benchmark_experiment, - use_strict_deny=False): + def is_compatible(self, dummy_benchmark_model, benchmark_experiment): name = dummy_benchmark_model.model_name - deny_list = STRICT_DENY_LIST if use_strict_deny else DENY_LIST if name in self.skip["skip"]: return False @@ -258,7 +235,7 @@ def is_compatible(self, def is_attr_eq(k, v): return getattr(benchmark_experiment, k) == v - for deny_experiment_config in deny_list.get(name, []): + for deny_experiment_config in DENY_LIST.get(name, []): if all(is_attr_eq(k, v) for k, v in deny_experiment_config.items()): return False