Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[benchmarks] Remove --strict-compatible argument. #6491

Merged
merged 1 commit into from
Feb 8, 2024
Merged

[benchmarks] Remove --strict-compatible argument. #6491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions benchmarks/experiment_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,7 @@ def generate_and_run_all_configs(self):
#
# Otherwise, we should go ahead and execute it.
if (not self._args.no_skip and not self.model_loader.is_compatible(
benchmark_model, benchmark_experiment,
self._args.strict_compatible)):
benchmark_model, benchmark_experiment)):
logger.warning("SKIP incompatible model and experiment configs.")
self._save_results(benchmark_experiment.to_dict(),
benchmark_model.to_dict(), {"error": "SKIP"})
Expand Down Expand Up @@ -883,11 +882,6 @@ def __str__(self):
action="store_true",
help="""If set, verifies the model output with PT Eager mode, and saves relative error to the output file."""
)
parser.add_argument(
"--strict-compatible",
action="store_true",
help="Strictly skips some models including models without installation file or causing stackdump.",
)
parser.add_argument(
"--no-skip",
action="store_true",
Expand Down
27 changes: 2 additions & 25 deletions benchmarks/torchbench_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,25 +135,6 @@
],
}

# This strict deny list denies tests that hold for too long and timeoout.
STRICT_DENY_LIST = {
**{
"opacus_cifar10": [{
"accelerator": "tpu",
},], # stackdump issue in TPU
"pytorch_stargan": [{
"accelerator": "tpu",
},], # stackdump issue in TPU
"soft_actor_critic": [{
"accelerator": "tpu",
},], # stackdump issue in TPU
"speech_transformer": [{
"accelerator": "tpu",
},], # stackdump issue in TPU
},
**DENY_LIST
}


class TorchBenchModelLoader(ModelLoader):

Expand Down Expand Up @@ -234,12 +215,8 @@ def list_model_configs(self):

return model_configs

def is_compatible(self,
dummy_benchmark_model,
benchmark_experiment,
use_strict_deny=False):
def is_compatible(self, dummy_benchmark_model, benchmark_experiment):
name = dummy_benchmark_model.model_name
deny_list = STRICT_DENY_LIST if use_strict_deny else DENY_LIST

if name in self.skip["skip"]:
return False
Expand All @@ -258,7 +235,7 @@ def is_compatible(self,
def is_attr_eq(k, v):
return getattr(benchmark_experiment, k) == v

for deny_experiment_config in deny_list.get(name, []):
for deny_experiment_config in DENY_LIST.get(name, []):
if all(is_attr_eq(k, v) for k, v in deny_experiment_config.items()):
return False

Expand Down
Loading