diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc index 5a18c32f90..08d1d03dee 100644 --- a/loadgen/test_settings_internal.cc +++ b/loadgen/test_settings_internal.cc @@ -334,6 +334,8 @@ void LogRequestedTestSettings(const TestSettings &s) { s.performance_issue_same_index); MLPERF_LOG(detail, "requested_performance_sample_count_override", s.performance_sample_count_override); + MLPERF_LOG(detail, "requested_sample_concatenate_permutation", + s.sample_concatenate_permutation); // Token latencies specific values if (s.use_token_latencies){ MLPERF_LOG(detail, "requested_use_token_latencies", s.use_token_latencies); @@ -443,6 +445,8 @@ void TestSettingsInternal::LogEffectiveSettings() const { s.performance_issue_same_index); MLPERF_LOG(detail, "effective_performance_sample_count", s.performance_sample_count); + MLPERF_LOG(detail, "effective_sample_concatenate_permutation", + s.sample_concatenate_permutation); #else detail(""); detail("Effective Settings:"); diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 3f0b0da5e7..3e3dc76c17 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -776,6 +776,34 @@ def uses_early_stopping(self, scenario): return ( scenario in ["Server", "SingleStream", "MultiStream"] ) + + def requires_equal_issue(self, model, division): + return ( + division in ["closed", "network"] and + model in [ + "3d-unet-99", + "3d-unet-99.9" + "llama2-70b-99", + "llama2-70b-99.9" + "llama2-70b-99", + "llama2-70b-99.9", + "mixtral-8x7b" + ] + and self.version in ["v4.1"] + ) + + def guess_equal_issue_mode(self, model): + return ( + self.version in ["v2.1", "v3.0", "v3.1", "v4.0", "v4.1"] and + model in [ + "3d-unet-99", + "3d-unet-99.9" + "llama2-70b-99", + "llama2-70b-99.9" + "llama2-70b-99", + "llama2-70b-99.9", + ] + ) def get_args(): @@ -1126,6 +1154,7 @@ def check_performance_dir( if scenario == "SingleStream" or scenario == "MultiStream": res /= MS_TO_NS + # Check if the current scenario uses early stopping uses_early_stopping = config.uses_early_stopping(scenario) @@ -1922,12 +1951,14 @@ def log_result( continue else: if not check_measurement_dir( + config, measurement_dir, name, system_desc, os.path.join(division, submitter), model_name, scenario, + division, has_power, skip_meaningful_fields_emptiness_check, skip_empty_files_check, @@ -2285,12 +2316,14 @@ def check_system_desc_id_power( def check_measurement_dir( + config, measurement_dir, fname, system_desc, root, model, scenario, + division, has_power, skip_meaningful_fields_emptiness_check, skip_empty_files_check, @@ -2381,6 +2414,40 @@ def check_measurement_dir( if not os.path.exists(os.path.dirname(code_dir)): log.error("%s is missing code_dir %s", fname, code_dir) is_valid = False + + # Check equal issue mode + if "mlperf.conf" in files and config.requires_equal_issue(model, division): + with open(f"{measurement_dir}/mlperf.conf") as f: + lines = f.readlines() + equal_issue_used = False + conf_ref_model = model.replace("-99.9", "").replace("-99", "") + for line in lines: + line = line.replace(" ", "").replace("\n", "") + if line.startswith("#"): + continue + elif line == "": + continue + else: + key, val = line.split("=") + key.replace(" ", "") + val.replace(" ", "") + conf_model, conf_scenario, conf_key = key.split(".") + if ( + (conf_key == "sample_concatenate_permutation") and + ((conf_model == conf_ref_model) or conf_model == "*") and + ((conf_scenario == scenario) or conf_scenario == "*") + ): + if val.isnumeric(): + val = int(val) + equal_issue_used = (val == 1) + break + if not equal_issue_used: + is_valid = False + log.error( + "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=%s", + fname, + equal_issue_used, + ) else: log.error("%s is missing %s*.json", fname, system_desc) is_valid = False