Fixes to SDXL accuracy output, submission preprocessor, fixes #1938

.github/workflows/build_wheels.yml

@@ -82,7 +82,12 @@ jobs:
       - uses: actions/setup-python@v3
 
       - name: Install requirements
-        run: python -m pip install cibuildwheel twine
+        run: python -m pip install cibuildwheel twine build
+
+      - name: Build src dist
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          python -m build --sdist --outdir wheels loadgen
 
       - name: Build wheels
         run: git pull && python -m cibuildwheel loadgen/ --output-dir wheels

loadgen/VERSION.txt

@@ -1 +1 @@
-4.1.23
+4.1.25

text_to_image/tools/accuracy_coco.py

@@ -195,8 +195,8 @@ def compute_accuracy(
                 generated_img).item())
     fid_score = compute_fid(result_list, statistics_path, device)
 
-    result_dict["FID_SCORE"] = fid_score
-    result_dict["CLIP_SCORE"] = np.mean(clip_scores)
+    result_dict["FID_SCORE"] = f"{fid_score}"
+    result_dict["CLIP_SCORE"] = f"{np.mean(clip_scores)}"
     print(f"Accuracy Results: {result_dict}")
 
     with open(output_file, "w") as fp:

text_to_image/tools/fid/fid_score.py

@@ -355,6 +355,10 @@ def compute_fid(
         imgs, model, batch_size, dims, device, num_workers
     )
 
+    # Ensure dimensions match before calculating FID
+    assert s1.shape == s2.shape, f"Covariance shapes mismatch: {s1.shape} vs {s2.shape}"
+    assert m1.shape == m2.shape, f"Mean shapes mismatch: {m1.shape} vs {m2.shape}"
+
     fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 
     return fid_value

tools/submission/preprocess_submission.py

@@ -144,18 +144,25 @@ def clean_model_dir(model_results_dir):
         model_results_dir, "results", "compliance")
 
     print(f"rmtree {model_results_dir}")
-    shutil.rmtree(model_results_dir)
-    shutil.rmtree(model_measurements_dir)
-    shutil.rmtree(model_compliance_dir)
+    if os.path.exists(model_results_dir):
+        shutil.rmtree(model_results_dir)
+    if os.path.exists(model_measurements_dir):
+        shutil.rmtree(model_measurements_dir)
+    if os.path.exists(model_compliance_dir):
+        shutil.rmtree(model_compliance_dir)
+
     sut_results_dir = os.path.dirname(model_results_dir)
-    if not os.listdir(sut_results_dir):
+    if os.path.exists(sut_results_dir) and not os.listdir(sut_results_dir):
         # clean sut dir
         sut = os.path.basename(sut_results_dir)
         log.info(
             f"No benchmark results remaining for {sut}. rmtree {sut_results_dir}")
-        shutil.rmtree(sut_results_dir)
-        shutil.rmtree(os.path.dirname(model_measurements_dir))
-        shutil.rmtree(os.path.dirname(model_compliance_dir))
+        if os.path.exists(sut_results_dir):
+            shutil.rmtree(sut_results_dir)
+        if os.path.exists(os.path.dirname(model_measurements_dir)):
+            shutil.rmtree(os.path.dirname(model_measurements_dir))
+        if os.path.exists(os.path.dirname(model_compliance_dir)):
+            shutil.rmtree(os.path.dirname(model_compliance_dir))
 
 
 def clean_invalid_results(args, log_path, config, system_desc, system_json,
@@ -200,7 +207,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                 ranging_path = os.path.join(
                     scenario_path, "performance", "ranging")
                 try:
-                    ranging_r = get_performance_metric(
+                    ranging_r = checker.get_performance_metric(
                         config,
                         mlperf_model,
                         ranging_path,
@@ -224,9 +231,13 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                 if not power_is_valid:
                     log.warning(
                         f"Power result is invalid for {system_desc}: {model} {scenario} scenario in {division} division. Removing...")
-                    shutil.rmtree(power_path)
-                    shutil.rmtree(ranging_path)
-                    shutil.rmtree(os.path.join(perf_path, "spl.txt"))
+                    if os.path.exists(power_path):
+                        shutil.rmtree(power_path)
+                    if os.path.exists(ranging_path):
+                        shutil.rmtree(ranging_path)
+                    spl_path = os.path.join(perf_path, "spl.txt")
+                    if os.path.exists(spl_path):
+                        os.remove(spl_path)
 
             compliance_is_valid = True
             if is_closed_or_network:
@@ -253,9 +264,12 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                     scenario_path, "results", "compliance")
                 log.warning(
                     f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...")
-                shutil.rmtree(scenario_path)
-                shutil.rmtree(scenario_measurements_path)
-                shutil.rmtree(scenario_compliance_path)
+                if os.path.exists(scenario_path):
+                    shutil.rmtree(scenario_path)
+                if os.path.exists(scenario_measurements_path):
+                    shutil.rmtree(scenario_measurements_path)
+                if os.path.exists(scenario_compliance_path):
+                    shutil.rmtree(scenario_compliance_path)
             elif division in ["closed", "network"]:
                 model_results_path = os.path.dirname(scenario_path)
                 model_measurements_path = change_folder_name_in_path(
@@ -279,10 +293,12 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                     if not perf_is_valid:
                         log.warning(
                             f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...")
-                        shutil.rmtree(scenario_path)
+                        if os.path.exists(scenario_path):
+                            shutil.rmtree(scenario_path)
                         scenario_measurements_path = change_folder_name_in_path(
                             scenario_path, "results", "measurements")
-                        shutil.rmtree(scenario_measurements_path)
+                        if os.path.exists(scenario_measurements_path):
+                            shutil.rmtree(scenario_measurements_path)
                     if not os.path.exists(target_results_path):
                         shutil.copytree(
                             model_results_path, target_results_path)
@@ -311,9 +327,11 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json,
                 clean_model_dir(model_results_path)
             else:  # delete this result
                 # delete other scenario results too
-                shutil.rmtree(scenario_path)
+                if os.path.exists(scenario_path):
+                    shutil.rmtree(scenario_path)
                 # delete other scenario results too
-                shutil.rmtree(scenario_measurements_path)
+                if os.path.exists(scenario_measurements_path):
+                    shutil.rmtree(scenario_measurements_path)
                 log.warning(
                     f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...")