diff --git a/.github/scripts/format-cpp.sh b/.github/scripts/format-cpp.sh deleted file mode 100755 index c0a237489..000000000 --- a/.github/scripts/format-cpp.sh +++ /dev/null @@ -1,26 +0,0 @@ -#================================================================ -# HEADER -#================================================================ -# DESCRIPTION -# This is a script containing some commands to automatically -# format the c/c++ code contained in one folder. -# This will help to maintain good quality code in the github -# repository. -# SET UP -# You need to allow the correct permissions for this file. -# This can be done by running: -# chmod u+x -# REQUIREMENTS -# clang-format -#================================================================ -# END_OF_HEADER -#================================================================ - - -# Checks all the modified c/c++ files, format them and adds them -# to the commit. -for FILE in $(git diff $1/$2 --name-only | grep -E '.*\.(cc|cpp|h|hpp)$') -do - clang-format -i -style=file $FILE - git add $FILE -done diff --git a/.github/scripts/format-py.sh b/.github/scripts/format-py.sh deleted file mode 100755 index ca3ca5671..000000000 --- a/.github/scripts/format-py.sh +++ /dev/null @@ -1,26 +0,0 @@ -#================================================================ -# HEADER -#================================================================ -# DESCRIPTION -# This is a script containing some commands to automatically -# format the c/c++ code contained in one folder. -# This will help to maintain good quality code in the github -# repository. -# SET UP -# You need to allow the correct permissions for this file. -# This can be done by running: -# chmod u+x -# REQUIREMENTS -# clang-format -#================================================================ -# END_OF_HEADER -#================================================================ - - -# Checks all the modified c/c++ files, format them and adds them -# to the commit. -for FILE in $(git diff $1/$2 --name-only | grep -E '.*\.py$') -do - autopep8 --in-place -a $FILE - git add $FILE -done diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e62f010df..6f67f56de 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest, macos-13] steps: - uses: actions/checkout@v3 @@ -82,7 +82,12 @@ jobs: - uses: actions/setup-python@v3 - name: Install requirements - run: python -m pip install cibuildwheel twine + run: python -m pip install cibuildwheel twine build + + - name: Build src dist + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + python -m build --sdist --outdir wheels loadgen - name: Build wheels run: git pull && python -m cibuildwheel loadgen/ --output-dir wheels @@ -110,12 +115,18 @@ jobs: with: name: wheels-ubuntu-latest path: wheels - # Download the built wheels from macOS - - name: Download macOS wheels + # Download the built wheels from macOS-latest + - name: Download macOS-latest wheels uses: actions/download-artifact@v4 with: name: wheels-macos-latest path: wheels + # Download the built wheels from macOS-13 (x86) + - name: Download macOS-13 (x86) wheels + uses: actions/download-artifact@v4 + with: + name: wheels-macos-13 + path: wheels # Download the built wheels from Windows - name: Download Windows wheels uses: actions/download-artifact@v4 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml deleted file mode 100644 index 829a3fcb7..000000000 --- a/.github/workflows/format.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Automatic code formatting -name: "format" -on: - pull_request: - branches: [ master ] - types: [opened, closed, synchronize] - - -env: - python_version: "3.9" - HEAD_REF: ${{ github.head_ref }} - -jobs: - format-code: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up Python ${{ env.python_version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ env.python_version }} - - - name: Install dependencies - run: | - python3 -m pip install autopep8 - - - name: Grant permissions - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/format-cpp.sh" - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/format-py.sh" - - - name: Format Codebase - run: | - git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} - git fetch upstream ${{ github.event.pull_request.base.ref }} - ".github/scripts/format-cpp.sh" "upstream" "${{ github.event.pull_request.base.ref }}" - ".github/scripts/format-py.sh" "upstream" "${{ github.event.pull_request.base.ref }}" - - - name: Commit - run: | - HAS_CHANGES=$(git diff --staged --name-only) - if [ ${#HAS_CHANGES} -gt 0 ]; then - git checkout -B "$HEAD_REF" - git config --global user.email "${{ github.actor }}@users.noreply.github.com" - git config --global user.name "${{ github.actor }}" - git commit -m '[Automated Commit] Format Codebase' - git push origin "$HEAD_REF" - fi diff --git a/loadgen/MANIFEST.in b/loadgen/MANIFEST.in index 74282fcee..152b53111 100644 --- a/loadgen/MANIFEST.in +++ b/loadgen/MANIFEST.in @@ -1 +1,2 @@ include VERSION.txt +include mlperf.conf diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 59fe035d9..e7c468635 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.23 +4.1.29 diff --git a/text_to_image/tools/accuracy_coco.py b/text_to_image/tools/accuracy_coco.py index b5f1be378..dcf7ed889 100644 --- a/text_to_image/tools/accuracy_coco.py +++ b/text_to_image/tools/accuracy_coco.py @@ -195,8 +195,8 @@ def compute_accuracy( generated_img).item()) fid_score = compute_fid(result_list, statistics_path, device) - result_dict["FID_SCORE"] = fid_score - result_dict["CLIP_SCORE"] = np.mean(clip_scores) + result_dict["FID_SCORE"] = f"{fid_score}" + result_dict["CLIP_SCORE"] = f"{np.mean(clip_scores)}" print(f"Accuracy Results: {result_dict}") with open(output_file, "w") as fp: diff --git a/text_to_image/tools/fid/fid_score.py b/text_to_image/tools/fid/fid_score.py index b2de332ce..1df208dd8 100644 --- a/text_to_image/tools/fid/fid_score.py +++ b/text_to_image/tools/fid/fid_score.py @@ -355,6 +355,10 @@ def compute_fid( imgs, model, batch_size, dims, device, num_workers ) + # Ensure dimensions match before calculating FID + assert s1.shape == s2.shape, f"Covariance shapes mismatch: {s1.shape} vs {s2.shape}" + assert m1.shape == m2.shape, f"Mean shapes mismatch: {m1.shape} vs {m2.shape}" + fid_value = calculate_frechet_distance(m1, s1, m2, s2) return fid_value diff --git a/tools/submission/preprocess_submission.py b/tools/submission/preprocess_submission.py index a737c1f16..977af4d47 100644 --- a/tools/submission/preprocess_submission.py +++ b/tools/submission/preprocess_submission.py @@ -144,18 +144,25 @@ def clean_model_dir(model_results_dir): model_results_dir, "results", "compliance") print(f"rmtree {model_results_dir}") - shutil.rmtree(model_results_dir) - shutil.rmtree(model_measurements_dir) - shutil.rmtree(model_compliance_dir) + if os.path.exists(model_results_dir): + shutil.rmtree(model_results_dir) + if os.path.exists(model_measurements_dir): + shutil.rmtree(model_measurements_dir) + if os.path.exists(model_compliance_dir): + shutil.rmtree(model_compliance_dir) + sut_results_dir = os.path.dirname(model_results_dir) - if not os.listdir(sut_results_dir): + if os.path.exists(sut_results_dir) and not os.listdir(sut_results_dir): # clean sut dir sut = os.path.basename(sut_results_dir) log.info( f"No benchmark results remaining for {sut}. rmtree {sut_results_dir}") - shutil.rmtree(sut_results_dir) - shutil.rmtree(os.path.dirname(model_measurements_dir)) - shutil.rmtree(os.path.dirname(model_compliance_dir)) + if os.path.exists(sut_results_dir): + shutil.rmtree(sut_results_dir) + if os.path.exists(os.path.dirname(model_measurements_dir)): + shutil.rmtree(os.path.dirname(model_measurements_dir)) + if os.path.exists(os.path.dirname(model_compliance_dir)): + shutil.rmtree(os.path.dirname(model_compliance_dir)) def clean_invalid_results(args, log_path, config, system_desc, system_json, @@ -200,7 +207,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, ranging_path = os.path.join( scenario_path, "performance", "ranging") try: - ranging_r = get_performance_metric( + ranging_r = checker.get_performance_metric( config, mlperf_model, ranging_path, @@ -224,9 +231,13 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, if not power_is_valid: log.warning( f"Power result is invalid for {system_desc}: {model} {scenario} scenario in {division} division. Removing...") - shutil.rmtree(power_path) - shutil.rmtree(ranging_path) - shutil.rmtree(os.path.join(perf_path, "spl.txt")) + if os.path.exists(power_path): + shutil.rmtree(power_path) + if os.path.exists(ranging_path): + shutil.rmtree(ranging_path) + spl_path = os.path.join(perf_path, "spl.txt") + if os.path.exists(spl_path): + os.remove(spl_path) compliance_is_valid = True if is_closed_or_network: @@ -253,9 +264,12 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, scenario_path, "results", "compliance") log.warning( f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...") - shutil.rmtree(scenario_path) - shutil.rmtree(scenario_measurements_path) - shutil.rmtree(scenario_compliance_path) + if os.path.exists(scenario_path): + shutil.rmtree(scenario_path) + if os.path.exists(scenario_measurements_path): + shutil.rmtree(scenario_measurements_path) + if os.path.exists(scenario_compliance_path): + shutil.rmtree(scenario_compliance_path) elif division in ["closed", "network"]: model_results_path = os.path.dirname(scenario_path) model_measurements_path = change_folder_name_in_path( @@ -279,10 +293,12 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, if not perf_is_valid: log.warning( f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") - shutil.rmtree(scenario_path) + if os.path.exists(scenario_path): + shutil.rmtree(scenario_path) scenario_measurements_path = change_folder_name_in_path( scenario_path, "results", "measurements") - shutil.rmtree(scenario_measurements_path) + if os.path.exists(scenario_measurements_path): + shutil.rmtree(scenario_measurements_path) if not os.path.exists(target_results_path): shutil.copytree( model_results_path, target_results_path) @@ -311,9 +327,11 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, clean_model_dir(model_results_path) else: # delete this result # delete other scenario results too - shutil.rmtree(scenario_path) + if os.path.exists(scenario_path): + shutil.rmtree(scenario_path) # delete other scenario results too - shutil.rmtree(scenario_measurements_path) + if os.path.exists(scenario_measurements_path): + shutil.rmtree(scenario_measurements_path) log.warning( f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...")