Check for equal issue mode in submission checker

mlcommons · Jul 23, 2024 · 5c306ac · 5c306ac
1 parent a248595
commit 5c306ac
Show file tree

Hide file tree

Showing 2 changed files with 71 additions and 0 deletions.
diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc
@@ -334,6 +334,8 @@ void LogRequestedTestSettings(const TestSettings &s) {
                s.performance_issue_same_index);
     MLPERF_LOG(detail, "requested_performance_sample_count_override",
                s.performance_sample_count_override);
+    MLPERF_LOG(detail, "requested_sample_concatenate_permutation",
+            s.sample_concatenate_permutation);
     // Token latencies specific values
     if (s.use_token_latencies){
       MLPERF_LOG(detail, "requested_use_token_latencies", s.use_token_latencies);
@@ -443,6 +445,8 @@ void TestSettingsInternal::LogEffectiveSettings() const {
                s.performance_issue_same_index);
     MLPERF_LOG(detail, "effective_performance_sample_count",
                s.performance_sample_count);
+    MLPERF_LOG(detail, "effective_sample_concatenate_permutation",
+               s.sample_concatenate_permutation);
 #else
     detail("");
     detail("Effective Settings:");

diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py
@@ -776,6 +776,34 @@ def uses_early_stopping(self, scenario):
         return (
             scenario in ["Server", "SingleStream", "MultiStream"]
         )
+
+    def requires_equal_issue(self, model, division):
+        return (
+            division in ["closed", "network"] and
+            model in [
+                "3d-unet-99",
+                "3d-unet-99.9"
+                "llama2-70b-99",
+                "llama2-70b-99.9"
+                "llama2-70b-99",
+                "llama2-70b-99.9", 
+                "mixtral-8x7b"
+            ]
+            and self.version in ["v4.1"]
+        )
+
+    def guess_equal_issue_mode(self, model):
+        return (
+            self.version in ["v2.1", "v3.0", "v3.1", "v4.0", "v4.1"] and
+            model in [
+                "3d-unet-99",
+                "3d-unet-99.9"
+                "llama2-70b-99",
+                "llama2-70b-99.9"
+                "llama2-70b-99",
+                "llama2-70b-99.9", 
+            ]
+        )
 
 
 def get_args():
@@ -1126,6 +1154,7 @@ def check_performance_dir(
 
     if scenario == "SingleStream" or scenario == "MultiStream":
         res /= MS_TO_NS
+
 
     # Check if the current scenario uses early stopping
     uses_early_stopping = config.uses_early_stopping(scenario)
@@ -1922,12 +1951,14 @@ def log_result(
                             continue
                         else:
                             if not check_measurement_dir(
+                                config,
                                 measurement_dir,
                                 name,
                                 system_desc,
                                 os.path.join(division, submitter),
                                 model_name,
                                 scenario,
+                                division,
                                 has_power,
                                 skip_meaningful_fields_emptiness_check,
                                 skip_empty_files_check,
@@ -2285,12 +2316,14 @@ def check_system_desc_id_power(
 
 
 def check_measurement_dir(
+    config,
     measurement_dir,
     fname,
     system_desc,
     root,
     model,
     scenario,
+    division,
     has_power,
     skip_meaningful_fields_emptiness_check,
     skip_empty_files_check,
@@ -2381,6 +2414,40 @@ def check_measurement_dir(
             if not os.path.exists(os.path.dirname(code_dir)):
                 log.error("%s is missing code_dir %s", fname, code_dir)
                 is_valid = False
+
+        # Check equal issue mode        
+        if "mlperf.conf" in files and config.requires_equal_issue(model, division):
+            with open(f"{measurement_dir}/mlperf.conf") as f:
+                lines = f.readlines()
+                equal_issue_used = False
+                conf_ref_model = model.replace("-99.9", "").replace("-99", "")
+                for line in lines:
+                    line = line.replace(" ", "").replace("\n", "")
+                    if line.startswith("#"):
+                        continue
+                    elif line == "":
+                        continue
+                    else:
+                        key, val = line.split("=")
+                        key.replace(" ", "")
+                        val.replace(" ", "")
+                        conf_model, conf_scenario, conf_key = key.split(".")
+                        if (
+                            (conf_key == "sample_concatenate_permutation") and 
+                            ((conf_model == conf_ref_model) or conf_model == "*") and
+                            ((conf_scenario == scenario) or conf_scenario == "*")
+                        ):
+                            if val.isnumeric():
+                                val = int(val)
+                                equal_issue_used = (val == 1)
+                                break
+                if not equal_issue_used:
+                    is_valid = False
+                    log.error(
+                        "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=%s",
+                        fname,
+                        equal_issue_used,
+                    )
     else:
         log.error("%s is missing %s*.json", fname, system_desc)
         is_valid = False