From 4e1ba93b0d0e45184527335bf209967f16744416 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 18:25:20 +0530 Subject: [PATCH 01/61] Removed mlperf.conf check in submission checker, removed equal issue mode check in conf files --- tools/submission/submission_checker.py | 65 +++----------------------- 1 file changed, 6 insertions(+), 59 deletions(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 52c85be7e..445ce97f3 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -412,7 +412,7 @@ } } } -REQUIRED_MEASURE_FILES = ["mlperf.conf", "user.conf", "README.md"] +REQUIRED_MEASURE_FILES = ["user.conf", "README.md"] REQUIRED_POWER_MEASURE_FILES = ["analyzer_table.*", "power_settings.*"] MS_TO_NS = 1000 * 1000 S_TO_MS = 1000 @@ -2021,7 +2021,7 @@ def log_result( errors += 1 continue else: - measurement_check, conf_equal_issue_check, weight_data_types = check_measurement_dir( + measurement_check, weight_data_types = check_measurement_dir( config, measurement_dir, name, @@ -2134,12 +2134,10 @@ def log_result( ) # Check equal issue mode - if not (conf_equal_issue_check or performance_equal_issue_check): + if not performance_equal_issue_check: log.error( - "%s %s requires equal issue mode (sample_concatenate_permutation), expected=true, found=%s", - perf_path, - measurement_dir, - not (conf_equal_issue_check or performance_equal_issue_check), + "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=false", + perf_path ) is_valid, r = False, None except Exception as e: @@ -2501,62 +2499,11 @@ def check_measurement_dir( log.error("%s is missing code_dir %s", fname, code_dir) is_valid = False - # Check equal issue mode - equal_issue_used = False - if "mlperf.conf" in files and config.requires_equal_issue(model, division): - with open(f"{measurement_dir}/mlperf.conf") as f: - lines = f.readlines() - conf_ref_model = model.replace("-99.9", "").replace("-99", "") - for line in lines: - line = line.replace(" ", "").replace("\n", "") - if line.startswith("#"): - continue - elif line == "": - continue - else: - key, val = line.split("=") - key.replace(" ", "") - val.replace(" ", "") - conf_model, conf_scenario, conf_key = key.split(".") - if ( - (conf_key == "sample_concatenate_permutation") and - ((conf_model == conf_ref_model) or conf_model == "*") and - ((conf_scenario == scenario) or conf_scenario == "*") - ): - if val.isnumeric(): - val = int(val) - equal_issue_used = (val == 1) - break - - if "user.conf" in files and config.requires_equal_issue(model, division): - with open(f"{measurement_dir}/user.conf") as f: - lines = f.readlines() - conf_ref_model = model.replace("-99.9", "").replace("-99", "") - for line in lines: - line = line.replace(" ", "").replace("\n", "") - if line.startswith("#"): - continue - elif line == "": - continue - else: - key, val = line.split("=") - key.replace(" ", "") - val.replace(" ", "") - conf_model, conf_scenario, conf_key = key.split(".") - if ( - (conf_key == "sample_concatenate_permutation") and - ((conf_model == conf_ref_model) or conf_model == "*") and - ((conf_scenario == scenario) or conf_scenario == "*") - ): - if val.isnumeric(): - val = int(val) - equal_issue_used = (val == 1) - break else: log.error("%s is missing %s*.json", fname, system_desc) is_valid = False - return is_valid, equal_issue_used, weight_data_types + return is_valid, weight_data_types def check_compliance_perf_dir(test_dir): From d85fa9e8555bb90965743250b6474b0154385aa6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 19:13:35 +0530 Subject: [PATCH 02/61] Fix loadgen wheel package --- .github/workflows/build_wheels.yml | 1 + loadgen/setup.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d7f98764e..a3fd86a4e 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -7,6 +7,7 @@ on: branches: - master - loadgen-release + - dev paths: - loadgen/** diff --git a/loadgen/setup.py b/loadgen/setup.py index 1e8cb0a8d..fd76d9a60 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -110,8 +110,6 @@ url="https://mlcommons.org/", cmdclass={"build_ext": build_ext}, ext_modules=[mlperf_loadgen_module], - packages=[''], - package_data={'': ['mlperf.conf']}, include_package_data=True, long_description=mlperf_long_description, long_description_content_type='text/markdown') From de458a16280fd7d1a49a2d3711f0a1cd521923c3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 22:57:20 +0530 Subject: [PATCH 03/61] Embed mlperf.conf as a binary string in the distribution --- .github/workflows/build_wheels.yml | 2 +- loadgen/CMakeLists.txt | 7 +++--- loadgen/VERSION.txt | 2 +- loadgen/setup.py | 14 ++++++++--- loadgen/test_settings_internal.cc | 37 +++++++++++++++++++----------- 5 files changed, 41 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a3fd86a4e..45233cef3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -80,7 +80,7 @@ jobs: run: python -m pip install cibuildwheel twine - name: Build wheels - run: python -m cibuildwheel loadgen/ --output-dir wheels + run: git pull && python -m cibuildwheel loadgen/ --output-dir wheels # Save wheels as artifacts - name: Upload built wheels diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index 66a68f6ab..a3b1eb374 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -38,6 +38,9 @@ message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}") find_package(PythonInterp) message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") +# Generate source file with version info. +execute_process(COMMAND xdd -i ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf > ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h) + # Generate source file with version info. execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR}) @@ -58,6 +61,7 @@ set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/results.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.h + ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h ${CMAKE_BINARY_DIR}/version_generated.cc ) @@ -82,6 +86,3 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h") install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/include) - -# Define preprocessor macro with the path to mlperf.conf -add_definitions(-DMLPERF_CONF_PATH=\"${CMAKE_INSTALL_PREFIX}/include/mlperf.conf\") diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 9edf2a44f..18837e703 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.7 +4.1.9 diff --git a/loadgen/setup.py b/loadgen/setup.py index fd76d9a60..b23c56cad 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -29,6 +29,7 @@ from pybind11 import get_include from pybind11.setup_helpers import Pybind11Extension, build_ext from version_generator import generate_loadgen_version_definitions +import subprocess generated_version_source_filename = "generated/version_generated.cc" generate_loadgen_version_definitions(generated_version_source_filename, ".") @@ -41,7 +42,7 @@ "test_settings.h", "issue_query_controller.h", "early_stopping.h", - "query_dispatch_library.h", + "query_dispatch_library.h" ] lib_headers = [ @@ -52,7 +53,8 @@ "version.h", "results.h", "bindings/c_api.h", - "version_generator.py" + "version_generator.py", + "mlperf_conf.h" ] lib_sources = [ @@ -88,8 +90,14 @@ version_split = version.split(".") if len(version_split) < 2: - print("Version is incomplete. Needs a format like 4.1 in VERSION file") + print("Version is incomplete. Needs a format like 4.1.1 in VERSION file") +command = ["xxd", "-i", "mlperf.conf", "mlperf_conf.h" ] +try: + subprocess.check_call(command) +except subprocess.CalledProcessError as e: + print(f"Failed to generate the mlperf_conf.h file: {e}") + raise mlperf_loadgen_module = Pybind11Extension( "mlperf_loadgen", diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc index 3dd49c25d..a75e892f4 100644 --- a/loadgen/test_settings_internal.cc +++ b/loadgen/test_settings_internal.cc @@ -11,6 +11,7 @@ limitations under the License. ==============================================================================*/ #include "test_settings_internal.h" +#include "mlperf_conf.h" #include #include @@ -527,7 +528,7 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, if (configCount == 0) { // Only allow userConf as the single configFile and loadgen loads the // mlperfConf automatically - FromConfig(MLPERF_CONF_PATH, model, scenario, true); + FromConfig(NULL, model, scenario, true); } else { @@ -579,24 +580,34 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, return true; }; - // dirt simple config parser - std::ifstream fss(path); - std::string line; int line_nr = 0; int errors = 0; - if (!fss.is_open()) { - LogDetail([p = path](AsyncDetail &detail) { + // Declare the input stream before the if-else block + std::unique_ptr fss; + std::string line; + + if (!is_mlperf_conf) { + // dirt simple config parser + fss = std::make_unique(path); + if (!static_cast(fss.get())->is_open()) { + LogDetail([p = path](AsyncDetail &detail) { #if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "can't open file " << p; - MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); + std::stringstream ss; + ss << "can't open file " << p; + MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); #else - detail.Error("can't open file ", p); + detail.Error("can't open file ", p); #endif - }); - return -ENOENT; + }); + return -ENOENT; + } + } else { + // Convert unsigned char array to std::string + std::string config_str(reinterpret_cast(mlperf_conf), mlperf_conf_len); + fss = std::make_unique(config_str); + } - while (std::getline(fss, line)) { + while (std::getline(*fss, line)) { line_nr++; std::istringstream iss(line); std::string s, k; From 11405495cd78c920d794b6b4b92f8c433da39bbd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 23:02:59 +0530 Subject: [PATCH 04/61] Fix xxd command for loadgen --- loadgen/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index a3b1eb374..4e38e706a 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -39,7 +39,7 @@ find_package(PythonInterp) message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") # Generate source file with version info. -execute_process(COMMAND xdd -i ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf > ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h) +execute_process(COMMAND xxd -i ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h) # Generate source file with version info. execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR}) From 13c47f69b9923a27fb283c1dfa7b072c3081f4f0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 23:19:51 +0530 Subject: [PATCH 05/61] Fix mlperf_conf.h include --- loadgen/CMakeLists.txt | 4 +--- loadgen/setup.py | 7 ++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index 4e38e706a..288a91139 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -39,7 +39,7 @@ find_package(PythonInterp) message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") # Generate source file with version info. -execute_process(COMMAND xxd -i ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h) +execute_process(COMMAND xxd -i mlperf.conf mlperf_conf.h) # Generate source file with version info. execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR}) @@ -84,5 +84,3 @@ install(TARGETS mlperf_loadgen DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h") -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf - DESTINATION ${CMAKE_INSTALL_PREFIX}/include) diff --git a/loadgen/setup.py b/loadgen/setup.py index b23c56cad..fcd6517cc 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -83,8 +83,6 @@ "README.md").read_text( encoding="utf-8") -config_file_path = Path(__file__).parent / "mlperf.conf" - with open("VERSION.txt", "r") as f: version = f.read() version_split = version.split(".") @@ -105,9 +103,8 @@ ("MAJOR_VERSION", version_split[0]), ("MINOR_VERSION", - version_split[1]), - ("MLPERF_CONF_PATH", - f'"{config_file_path}"')], + version_split[1]) + ], include_dirs=[".", get_include()], sources=mlperf_loadgen_sources, depends=mlperf_loadgen_headers) From e50006071787628b2f23932050a09019a32d11f2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 00:15:53 +0530 Subject: [PATCH 06/61] Fix mlperf_conf.h include --- loadgen/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index 288a91139..caeac55bd 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -38,8 +38,11 @@ message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}") find_package(PythonInterp) message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") + # Generate source file with version info. -execute_process(COMMAND xxd -i mlperf.conf mlperf_conf.h) +execute_process(COMMAND xxd -i mlperf.conf ${CMAKE_BINARY_DIR}/mlperf_conf.h + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +message(STATUS "Output config: ${CMAKE_BINARY_DIR}/mlperf_conf.h") # Generate source file with version info. execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR}) @@ -84,3 +87,5 @@ install(TARGETS mlperf_loadgen DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h") +install(FILES ${CMAKE_BINARY_DIR}/mlperf_conf.h + DESTINATION ${CMAKE_INSTALL_PREFIX}/include) From 95883558c4cc498983328a632574d298b2328d4c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 00:28:14 +0530 Subject: [PATCH 07/61] Fix mlperf_conf.h include --- loadgen/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index caeac55bd..9d3c40f9b 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -64,7 +64,7 @@ set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/results.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.h - ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h + ${CMAKE_BINARY_DIR}/mlperf_conf.h ${CMAKE_BINARY_DIR}/version_generated.cc ) From f26302e7c13e16adbad39edd4dd14f6a34f5b274 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 01:14:52 +0530 Subject: [PATCH 08/61] Fix mlperf_conf.h include --- loadgen/CMakeLists.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index 9d3c40f9b..cdf3ead6e 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -40,7 +40,7 @@ message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") # Generate source file with version info. -execute_process(COMMAND xxd -i mlperf.conf ${CMAKE_BINARY_DIR}/mlperf_conf.h +execute_process(COMMAND xxd -i mlperf.conf mlperf_conf.h WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) message(STATUS "Output config: ${CMAKE_BINARY_DIR}/mlperf_conf.h") @@ -64,7 +64,7 @@ set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/results.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.h - ${CMAKE_BINARY_DIR}/mlperf_conf.h + ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h ${CMAKE_BINARY_DIR}/version_generated.cc ) @@ -87,5 +87,3 @@ install(TARGETS mlperf_loadgen DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h") -install(FILES ${CMAKE_BINARY_DIR}/mlperf_conf.h - DESTINATION ${CMAKE_INSTALL_PREFIX}/include) From f506644a296d2cf3cf59cfc3576c0fe78ae3aadd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 01:24:30 +0530 Subject: [PATCH 09/61] Fix mlperf_conf.h include --- loadgen/test_settings_internal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc index a75e892f4..1b51adec1 100644 --- a/loadgen/test_settings_internal.cc +++ b/loadgen/test_settings_internal.cc @@ -528,7 +528,7 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, if (configCount == 0) { // Only allow userConf as the single configFile and loadgen loads the // mlperfConf automatically - FromConfig(NULL, model, scenario, true); + FromConfig("", model, scenario, true); } else { From 95c554160dcdcdc1822de1c2b4988ec56b425d54 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 01:33:12 +0530 Subject: [PATCH 10/61] Fix loadgen VERSION to include patch number, remove hardcoded versions --- loadgen/CMakeLists.txt | 15 +++++++++++---- loadgen/version_generator.py | 7 ++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index cdf3ead6e..55bae8446 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -5,14 +5,21 @@ project(mlperf_loadgen) # Read the version file file(READ "${CMAKE_SOURCE_DIR}/VERSION.txt" VERSION_CONTENTS) -# Extract the major and minor versions from the VERSION file (assuming "MAJOR.MINOR.PATCH" format) -string(REGEX MATCH "^([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION_CONTENTS}) +# Extract the major, minor, and patch versions from the VERSION file (assuming "MAJOR.MINOR.PATCH" format) +string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION_CONTENTS}) -# Set the variables for the major and minor versions +# Set the variables for the major, minor, and patch versions set(mlperf_loadgen_VERSION_MAJOR "${CMAKE_MATCH_1}") set(mlperf_loadgen_VERSION_MINOR "${CMAKE_MATCH_2}") +set(mlperf_loadgen_VERSION_PATCH "${CMAKE_MATCH_3}") -message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}") +# Check if the version format was parsed correctly +if(NOT mlperf_loadgen_VERSION_MAJOR OR NOT mlperf_loadgen_VERSION_MINOR OR NOT mlperf_loadgen_VERSION_PATCH) + message(FATAL_ERROR "Version format in VERSION.txt is incorrect. Expected format: MAJOR.MINOR.PATCH") +endif() + +# Print out the version +message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}.${mlperf_loadgen_VERSION_PATCH}") # Set build options. NB: CXX_STANDARD is supported since CMake 3.1. if (NOT MSVC) diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index a013af0c6..8c978b2f1 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -93,7 +93,12 @@ def generate_loadgen_version_definitions(cc_filename, loadgen_root): ofile.write("// DO NOT EDIT: Autogenerated by version_generator.py.\n\n") ofile.write("#include \n\n") ofile.write("namespace mlperf {\n\n") - ofile.write(func_def("Version", "\"4.1\"")) + # Open and read the VERSION.txt file + with open("VERSION.txt", "r") as version_file: + version_contents = version_file.read().strip() # Read and strip any extra whitespace/newlines + + # Write the version into the function definition + ofile.write(func_def("Version", f"\"{version_contents}\"")) date_time_now_local = datetime.datetime.now().isoformat() date_time_now_utc = datetime.datetime.utcnow().isoformat() From 5d8838ff79e80a8b9c91dfbf65fbb84033813f7b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 01:41:49 +0530 Subject: [PATCH 11/61] Added vim-common deps for loadgen build on Linux gh action --- .github/workflows/build_wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 45233cef3..5eed37d9c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -76,6 +76,10 @@ jobs: - uses: actions/setup-python@v3 + - name: Install requirements (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y vim-common # for xxd + - name: Install requirements run: python -m pip install cibuildwheel twine From e32a3a7ce0b5723836f686b6ea4beb6f17368fa1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 01:59:12 +0530 Subject: [PATCH 12/61] Support mlperf_loadgen name --- loadgen/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/loadgen/__init__.py b/loadgen/__init__.py index e69de29bb..4551a307d 100644 --- a/loadgen/__init__.py +++ b/loadgen/__init__.py @@ -0,0 +1,4 @@ +import sys + +# Aliasing mlcommons_loadgen as mlperf_loadgen +sys.modules['mlperf_loadgen'] = sys.modules[__name__] From fba22221e474c3194cdab016e6f4ce4a7188b5b6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 02:05:48 +0530 Subject: [PATCH 13/61] Added VERSION.txt src dependency in cmake --- loadgen/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index 55bae8446..715983481 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -72,6 +72,7 @@ set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/version.cc ${CMAKE_CURRENT_SOURCE_DIR}/version.h ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h + ${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt ${CMAKE_BINARY_DIR}/version_generated.cc ) From 2c6641cdc3500227fb8cde62409b235904693080 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 02:12:38 +0530 Subject: [PATCH 14/61] Added VERSION.txt src dependency in cmake --- loadgen/version_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index 8c978b2f1..229016455 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -94,7 +94,7 @@ def generate_loadgen_version_definitions(cc_filename, loadgen_root): ofile.write("#include \n\n") ofile.write("namespace mlperf {\n\n") # Open and read the VERSION.txt file - with open("VERSION.txt", "r") as version_file: + with open(os.path.join(loadgen_root, "VERSION.txt"), "r") as version_file: version_contents = version_file.read().strip() # Read and strip any extra whitespace/newlines # Write the version into the function definition From 5b2ebfa5a7875d8aab40952bdb6edfb1af21f6c2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 02:27:46 +0530 Subject: [PATCH 15/61] Fix xxd deps for loadgen --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 5eed37d9c..5653d45e1 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -78,7 +78,7 @@ jobs: - name: Install requirements (Linux) if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y vim-common # for xxd + run: sudo apt-get update && sudo apt-get install --reinstall -y vim-common # for xxd - name: Install requirements run: python -m pip install cibuildwheel twine From e3df0dbea8b356c0e0e39a655bb880066aa15bc1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 02:31:22 +0530 Subject: [PATCH 16/61] Increment loadgen patch version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 18837e703..5d30083e9 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.9 +4.1.10 From 7921ed275e19519056928be987a696d3900d0ee0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 02:34:35 +0530 Subject: [PATCH 17/61] Increment loadgen patch version --- .github/workflows/build_wheels.yml | 2 +- loadgen/VERSION.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 5653d45e1..2f5c2fbef 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -78,7 +78,7 @@ jobs: - name: Install requirements (Linux) if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install --reinstall -y vim-common # for xxd + run: sudo apt-get update && sudo apt-get install --reinstall -y vim # for xxd - name: Install requirements run: python -m pip install cibuildwheel twine diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 5d30083e9..152e4522c 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.10 +4.1.11 From c5bdacad78ed050d42fe1e09ae113e6e51a044fa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 02:57:57 +0530 Subject: [PATCH 18/61] Store mlperf.conf as a text string --- .github/workflows/build_wheels.yml | 4 ---- loadgen/CMakeLists.txt | 22 +++++++++++++++++++--- loadgen/test_settings_internal.cc | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 2f5c2fbef..45233cef3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -76,10 +76,6 @@ jobs: - uses: actions/setup-python@v3 - - name: Install requirements (Linux) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install --reinstall -y vim # for xxd - - name: Install requirements run: python -m pip install cibuildwheel twine diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index 715983481..e68600480 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -45,10 +45,26 @@ message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}") find_package(PythonInterp) message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") +# Specify the source and destination files +set(CONF_FILE "mlperf.conf") +set(HEADER_FILE "mlperf_conf.h") + +# Read the content of the configuration file +file(READ ${CONF_FILE} CONF_CONTENTS) + +# Escape all double quotes and backslashes +string(REPLACE "\\" "\\\\" CONF_CONTENTS "${CONF_CONTENTS}") +string(REPLACE "\"" "\\\"" CONF_CONTENTS "${CONF_CONTENTS}") + +# Handle new lines +string(REPLACE "\n" "\\n\"\n\"" CONF_CONTENTS "${CONF_CONTENTS}") + +# Wrap the content in a C++ string declaration +set(FORMATTED_CONTENT "const char* mlperf_conf =\n\"${CONF_CONTENTS}\";\n") + +# Write the formatted content to the header file +file(WRITE ${HEADER_FILE} "${FORMATTED_CONTENT}") -# Generate source file with version info. -execute_process(COMMAND xxd -i mlperf.conf mlperf_conf.h - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) message(STATUS "Output config: ${CMAKE_BINARY_DIR}/mlperf_conf.h") # Generate source file with version info. diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc index 1b51adec1..a22bda025 100644 --- a/loadgen/test_settings_internal.cc +++ b/loadgen/test_settings_internal.cc @@ -603,7 +603,7 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, } } else { // Convert unsigned char array to std::string - std::string config_str(reinterpret_cast(mlperf_conf), mlperf_conf_len); + std::string config_str(mlperf_conf); fss = std::make_unique(config_str); } From 98313d95e14422904d36a9152b412ade515e713e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 03:03:29 +0530 Subject: [PATCH 19/61] Fix setup.py and removed xxd --- loadgen/setup.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/loadgen/setup.py b/loadgen/setup.py index fcd6517cc..c4e4392fe 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -90,12 +90,24 @@ if len(version_split) < 2: print("Version is incomplete. Needs a format like 4.1.1 in VERSION file") -command = ["xxd", "-i", "mlperf.conf", "mlperf_conf.h" ] + try: - subprocess.check_call(command) -except subprocess.CalledProcessError as e: - print(f"Failed to generate the mlperf_conf.h file: {e}") - raise + with open("mlperf.conf", 'r') as file: + conf_contents = file.read() + + # Escape backslashes and double quotes + conf_contents = conf_contents.replace('\\', '\\\\').replace('"', '\\"') + + # Convert newlines + conf_contents = conf_contents.replace('\n', '\\n"\n"') + + formatted_content = f'const char* mlperf_conf =\n"{conf_contents}";\n' + + with open("mlperf_conf.h", 'w') as header_file: + header_file.write(formatted_content) + +except IOError as e: + raise RuntimeError(f"Failed to generate header file: {e}") mlperf_loadgen_module = Pybind11Extension( "mlperf_loadgen", From 8ca64f9fe0ee5508fa579113127df83d2a2b836e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 03:30:57 +0530 Subject: [PATCH 20/61] Added package_dir for loadgen --- loadgen/MANIFEST.in | 1 - loadgen/setup.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/MANIFEST.in b/loadgen/MANIFEST.in index 1cdd95c6d..74282fcee 100644 --- a/loadgen/MANIFEST.in +++ b/loadgen/MANIFEST.in @@ -1,2 +1 @@ -include mlperf.conf include VERSION.txt diff --git a/loadgen/setup.py b/loadgen/setup.py index c4e4392fe..e03514b16 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -127,6 +127,7 @@ url="https://mlcommons.org/", cmdclass={"build_ext": build_ext}, ext_modules=[mlperf_loadgen_module], + package_dir={'': 'mlcommons_loadgen'}, include_package_data=True, long_description=mlperf_long_description, long_description_content_type='text/markdown') From decce8c3e97f545c846758365c7a6d9e7110eefc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 03:44:47 +0530 Subject: [PATCH 21/61] Added package_dir for loadgen, added pypi loadgen test in github action --- .github/workflows/test-resnet50.yml | 3 ++- loadgen/CMakeLists.txt | 2 +- loadgen/setup.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-resnet50.yml b/.github/workflows/test-resnet50.yml index f1e3e6c39..64636a343 100755 --- a/.github/workflows/test-resnet50.yml +++ b/.github/workflows/test-resnet50.yml @@ -22,6 +22,7 @@ jobs: matrix: python-version: [ "3.9" ] backend: [ "onnxruntime", "tf" ] + loadgen-flag: [ "", "--adr.loadgen.tags=_from-pip --pip_loadgen=yes" ] steps: - uses: actions/checkout@v3 @@ -34,4 +35,4 @@ jobs: python3 -m pip install cm4mlops - name: Test Resnet50 and end to end submission generation run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.loadgen.version=custom + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }} diff --git a/loadgen/CMakeLists.txt b/loadgen/CMakeLists.txt index e68600480..37270ac24 100644 --- a/loadgen/CMakeLists.txt +++ b/loadgen/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.12) project(mlperf_loadgen) diff --git a/loadgen/setup.py b/loadgen/setup.py index e03514b16..c9675f0fc 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -127,7 +127,8 @@ url="https://mlcommons.org/", cmdclass={"build_ext": build_ext}, ext_modules=[mlperf_loadgen_module], - package_dir={'': 'mlcommons_loadgen'}, + packages=['mlperf_loadgen'], + package_dir={'mlperf_loadgen': '.'}, include_package_data=True, long_description=mlperf_long_description, long_description_content_type='text/markdown') From 06e6afdb846e876337b1fd0fa10a3457b05b2ca8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 03:53:32 +0530 Subject: [PATCH 22/61] Added package_dir for loadgen, added pypi loadgen test in github action --- loadgen/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loadgen/setup.py b/loadgen/setup.py index c9675f0fc..5ee556e51 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -127,8 +127,8 @@ url="https://mlcommons.org/", cmdclass={"build_ext": build_ext}, ext_modules=[mlperf_loadgen_module], - packages=['mlperf_loadgen'], - package_dir={'mlperf_loadgen': '.'}, + packages=['mlcommons_loadgen'], + package_dir={'mlcommons_loadgen': '.'}, include_package_data=True, long_description=mlperf_long_description, long_description_content_type='text/markdown') From 184786ae217aeb8e237bef0129009156ba0a5acc Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Wed, 23 Oct 2024 22:23:47 +0000 Subject: [PATCH 23/61] Increment version to 4.1.12 --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 152e4522c..b05079e54 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.11 +4.1.12 From b6046a5e2a08e167dc73150575575677746aa111 Mon Sep 17 00:00:00 2001 From: arjunsuresh <4791823+arjunsuresh@users.noreply.github.com> Date: Wed, 23 Oct 2024 22:24:09 +0000 Subject: [PATCH 24/61] [Automated Commit] Format Codebase --- loadgen/setup.py | 2 +- loadgen/test_settings_internal.cc | 5 +- loadgen/version_generator.py | 14 +- tools/submission/submission_checker.py | 364 ++++++++++++++++--------- 4 files changed, 253 insertions(+), 132 deletions(-) diff --git a/loadgen/setup.py b/loadgen/setup.py index 5ee556e51..c0dc59aee 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -116,7 +116,7 @@ version_split[0]), ("MINOR_VERSION", version_split[1]) - ], + ], include_dirs=[".", get_include()], sources=mlperf_loadgen_sources, depends=mlperf_loadgen_headers) diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc index a22bda025..1a7387f59 100644 --- a/loadgen/test_settings_internal.cc +++ b/loadgen/test_settings_internal.cc @@ -11,7 +11,6 @@ limitations under the License. ==============================================================================*/ #include "test_settings_internal.h" -#include "mlperf_conf.h" #include #include @@ -19,6 +18,7 @@ limitations under the License. #include #include "logging.h" +#include "mlperf_conf.h" #include "utils.h" namespace mlperf { @@ -589,7 +589,7 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, if (!is_mlperf_conf) { // dirt simple config parser fss = std::make_unique(path); - if (!static_cast(fss.get())->is_open()) { + if (!static_cast(fss.get())->is_open()) { LogDetail([p = path](AsyncDetail &detail) { #if USE_NEW_LOGGING_FORMAT std::stringstream ss; @@ -605,7 +605,6 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, // Convert unsigned char array to std::string std::string config_str(mlperf_conf); fss = std::make_unique(config_str); - } while (std::getline(*fss, line)) { line_nr++; diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index 229016455..1d8fd028a 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================= -## \file +# \file # \brief A script run by the build to generate the version definitions # expected at link time. @@ -27,10 +27,11 @@ # Creates a C++ raw string literal using a delimiter that is very # unlikely to show up in a git stats. -def make_raw_string(str) : +def make_raw_string(str): delimeter = "LGVG_RSLD" return "R\"" + delimeter + "(" + str + ")" + delimeter + "\"" + def func_def(name, string): return ("const std::string& Loadgen" + name + "() {\n" + " static const std::string str = " + string + ";\n" + @@ -42,10 +43,12 @@ def func_def(name, string): # any modifications. def generate_loadgen_version_definitions_git(ofile, git_command): git_rev = os.popen(git_command + "rev-parse --short=10 HEAD").read() - git_commit_date = os.popen(git_command + "log --format=\"%cI\" -n 1").read() + git_commit_date = os.popen( + git_command + + "log --format=\"%cI\" -n 1").read() git_status = os.popen(git_command + "status -s -uno .").read() git_log = subprocess.Popen( - git_command + "log --pretty=oneline -n 16 --no-decorate", stdout=subprocess.PIPE, shell=True, encoding='ascii', errors="ignore" ).stdout.read() + git_command + "log --pretty=oneline -n 16 --no-decorate", stdout=subprocess.PIPE, shell=True, encoding='ascii', errors="ignore").stdout.read() ofile.write(func_def("GitRevision", "\"" + git_rev[0:-1] + "\"")) ofile.write(func_def("GitCommitDate", "\"" + git_commit_date[0:-1] + "\"")) ofile.write(func_def("GitStatus", make_raw_string(git_status[0:-1]))) @@ -95,7 +98,8 @@ def generate_loadgen_version_definitions(cc_filename, loadgen_root): ofile.write("namespace mlperf {\n\n") # Open and read the VERSION.txt file with open(os.path.join(loadgen_root, "VERSION.txt"), "r") as version_file: - version_contents = version_file.read().strip() # Read and strip any extra whitespace/newlines + # Read and strip any extra whitespace/newlines + version_contents = version_file.read().strip() # Write the version into the function definition ofile.write(func_def("Version", f"\"{version_contents}\"")) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 445ce97f3..2acfdb838 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -98,16 +98,16 @@ "dlrm-v2-99.9": ("AUC", 80.31 * 0.999), "3d-unet-99": ("DICE", 0.86170 * 0.99), "3d-unet-99.9": ("DICE", 0.86170 * 0.999), - "gptj-99" : ("ROUGE1", 42.9865 * 0.99, "ROUGE2", 20.1235 * 0.99, "ROUGEL", 29.9881 * 0.99, "GEN_LEN", 4016878*0.9), - "gptj-99.9" : ("ROUGE1", 42.9865 * 0.999, "ROUGE2", 20.1235 * 0.999, "ROUGEL", 29.9881 * 0.999, "GEN_LEN", 4016878*0.9), - "llama2-70b-99" : ("ROUGE1", 44.4312 * 0.99, "ROUGE2", 22.0352 * 0.99, "ROUGEL", 28.6162 * 0.99, "TOKENS_PER_SAMPLE", 294.45*0.9), - "llama2-70b-99.9" : ("ROUGE1", 44.4312 * 0.999, "ROUGE2", 22.0352 * 0.999, "ROUGEL", 28.6162 * 0.999, "TOKENS_PER_SAMPLE", 294.45*0.9), + "gptj-99": ("ROUGE1", 42.9865 * 0.99, "ROUGE2", 20.1235 * 0.99, "ROUGEL", 29.9881 * 0.99, "GEN_LEN", 4016878 * 0.9), + "gptj-99.9": ("ROUGE1", 42.9865 * 0.999, "ROUGE2", 20.1235 * 0.999, "ROUGEL", 29.9881 * 0.999, "GEN_LEN", 4016878 * 0.9), + "llama2-70b-99": ("ROUGE1", 44.4312 * 0.99, "ROUGE2", 22.0352 * 0.99, "ROUGEL", 28.6162 * 0.99, "TOKENS_PER_SAMPLE", 294.45 * 0.9), + "llama2-70b-99.9": ("ROUGE1", 44.4312 * 0.999, "ROUGE2", 22.0352 * 0.999, "ROUGEL", 28.6162 * 0.999, "TOKENS_PER_SAMPLE", 294.45 * 0.9), "stable-diffusion-xl": ("CLIP_SCORE", 31.68631873, "FID_SCORE", 23.01085758) }, "accuracy-upper-limit": { "stable-diffusion-xl": ("CLIP_SCORE", 31.81331801, "FID_SCORE", 23.95007626), - "llama2-70b-99" : ("TOKENS_PER_SAMPLE", 294.45*1.1), - "llama2-70b-99.9" : ("TOKENS_PER_SAMPLE", 294.45*1.1) + "llama2-70b-99": ("TOKENS_PER_SAMPLE", 294.45 * 1.1), + "llama2-70b-99.9": ("TOKENS_PER_SAMPLE", 294.45 * 1.1) }, "accuracy-delta-perc": { "stable-diffusion-xl": { @@ -163,7 +163,7 @@ "gptj-99.9": {"Server": 20000000000}, "llama2-70b-99": {"Server": 20000000000}, "llama2-70b-99.9": {"Server": 20000000000}, - "stable-diffusion-xl" : {"Server": 20000000000} + "stable-diffusion-xl": {"Server": 20000000000} }, "min-queries": { "resnet": { @@ -263,18 +263,18 @@ "dlrm-v2-99.9": ("AUC", 80.31 * 0.999), "3d-unet-99": ("DICE", 0.86170 * 0.99), "3d-unet-99.9": ("DICE", 0.86170 * 0.999), - "gptj-99" : ("ROUGE1", 42.9865 * 0.99, "ROUGE2", 20.1235 * 0.99, "ROUGEL", 29.9881 * 0.99, "GEN_LEN", 4016878*0.9), - "gptj-99.9" : ("ROUGE1", 42.9865 * 0.999, "ROUGE2", 20.1235 * 0.999, "ROUGEL", 29.9881 * 0.999, "GEN_LEN", 4016878*0.9), - "llama2-70b-99" : ("ROUGE1", 44.4312 * 0.99, "ROUGE2", 22.0352 * 0.99, "ROUGEL", 28.6162 * 0.99, "TOKENS_PER_SAMPLE", 294.45*0.9), - "llama2-70b-99.9" : ("ROUGE1", 44.4312 * 0.999, "ROUGE2", 22.0352 * 0.999, "ROUGEL", 28.6162 * 0.999, "TOKENS_PER_SAMPLE", 294.45*0.9), + "gptj-99": ("ROUGE1", 42.9865 * 0.99, "ROUGE2", 20.1235 * 0.99, "ROUGEL", 29.9881 * 0.99, "GEN_LEN", 4016878 * 0.9), + "gptj-99.9": ("ROUGE1", 42.9865 * 0.999, "ROUGE2", 20.1235 * 0.999, "ROUGEL", 29.9881 * 0.999, "GEN_LEN", 4016878 * 0.9), + "llama2-70b-99": ("ROUGE1", 44.4312 * 0.99, "ROUGE2", 22.0352 * 0.99, "ROUGEL", 28.6162 * 0.99, "TOKENS_PER_SAMPLE", 294.45 * 0.9), + "llama2-70b-99.9": ("ROUGE1", 44.4312 * 0.999, "ROUGE2", 22.0352 * 0.999, "ROUGEL", 28.6162 * 0.999, "TOKENS_PER_SAMPLE", 294.45 * 0.9), "stable-diffusion-xl": ("CLIP_SCORE", 31.68631873, "FID_SCORE", 23.01085758), - "mixtral-8x7b" : ("ROUGE1", 45.4911 * 0.99, "ROUGE2", 23.2829 * 0.99, "ROUGEL", 30.3615 * 0.99, "TOKENS_PER_SAMPLE", 145.9 * 0.9, "gsm8k_accuracy", 73.78*0.99, "mbxp_accuracy", 60.12 * 0.99), + "mixtral-8x7b": ("ROUGE1", 45.4911 * 0.99, "ROUGE2", 23.2829 * 0.99, "ROUGEL", 30.3615 * 0.99, "TOKENS_PER_SAMPLE", 145.9 * 0.9, "gsm8k_accuracy", 73.78 * 0.99, "mbxp_accuracy", 60.12 * 0.99), }, "accuracy-upper-limit": { "stable-diffusion-xl": ("CLIP_SCORE", 31.81331801, "FID_SCORE", 23.95007626), - "llama2-70b-99" : ("TOKENS_PER_SAMPLE", 294.45*1.1), - "llama2-70b-99.9" : ("TOKENS_PER_SAMPLE", 294.45*1.1), - "mixtral-8x7b" : ("TOKENS_PER_SAMPLE", 145.9 * 1.1) + "llama2-70b-99": ("TOKENS_PER_SAMPLE", 294.45 * 1.1), + "llama2-70b-99.9": ("TOKENS_PER_SAMPLE", 294.45 * 1.1), + "mixtral-8x7b": ("TOKENS_PER_SAMPLE", 145.9 * 1.1) }, "accuracy-delta-perc": { "stable-diffusion-xl": { @@ -329,7 +329,7 @@ "gptj-99.9": {"Server": 20000000000}, "llama2-70b-99": {"Server": 20000000000}, "llama2-70b-99.9": {"Server": 20000000000}, - "stable-diffusion-xl" : {"Server": 20000000000} + "stable-diffusion-xl": {"Server": 20000000000} # TODO: Mixtral metrics # "mixtral-8x7b" : {"Server": 20000000000} }, @@ -514,19 +514,19 @@ } LLM_LATENCY_LIMITS = { - "llama2-70b-99":{ + "llama2-70b-99": { "conversational": { "ttft": 2000 * 1000000, "tpot": 200 * 1000000 } }, - "llama2-70b-99.9":{ + "llama2-70b-99.9": { "conversational": { "ttft": 2000 * 1000000, "tpot": 200 * 1000000 } }, - "mixtral-8x7b":{ + "mixtral-8x7b": { "conversational": { "ttft": 2000 * 1000000, "tpot": 200 * 1000000 @@ -755,7 +755,7 @@ def get_accuracy_target(self, model): if model not in self.accuracy_target: raise ValueError("model not known: " + model) return self.accuracy_target[model] - + def get_accuracy_upper_limit(self, model): return self.accuracy_upper_limit.get(model, None) @@ -781,12 +781,12 @@ def get_min_query_count(self, model, scenario): if model not in self.min_queries: raise ValueError("model not known: " + model) return self.min_queries[model].get(scenario) - + def get_delta_perc(self, model, metric): if model in self.accuracy_delta_perc: if metric in self.accuracy_delta_perc[model]: return self.accuracy_delta_perc[model][metric] - + more_accurate = model.find("99.9") if more_accurate == -1: required_delta_perc = 1 @@ -797,12 +797,11 @@ def get_delta_perc(self, model, metric): def has_new_logging_format(self): return True - def uses_early_stopping(self, scenario): return ( scenario in ["Server", "SingleStream", "MultiStream"] ) - + def requires_equal_issue(self, model, division): return ( division in ["closed", "network"] and @@ -812,7 +811,7 @@ def requires_equal_issue(self, model, division): "gptj-99", "gptj-99.9", "llama2-70b-99", - "llama2-70b-99.9", + "llama2-70b-99.9", "mixtral-8x7b" ] and self.version in ["v4.1"] @@ -830,7 +829,10 @@ def get_args(): help="mlperf version", ) parser.add_argument("--submitter", help="filter to submitter") - parser.add_argument("--csv", default="summary.csv", help="csv file with results") + parser.add_argument( + "--csv", + default="summary.csv", + help="csv file with results") parser.add_argument( "--skip_compliance", action="store_true", @@ -841,7 +843,10 @@ def get_args(): help="File containing extra custom model mapping. It is assumed to be inside the folder open/", default="model_mapping.json", ) - parser.add_argument("--debug", action="store_true", help="extra debug output") + parser.add_argument( + "--debug", + action="store_true", + help="extra debug output") parser.add_argument( "--submission-exceptions", action="store_true", @@ -883,17 +888,20 @@ def get_args(): def list_dir(*path): path = os.path.join(*path) - return [f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))] + return [f for f in os.listdir( + path) if os.path.isdir(os.path.join(path, f))] def list_files(*path): path = os.path.join(*path) - return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + return [f for f in os.listdir( + path) if os.path.isfile(os.path.join(path, f))] def list_empty_dirs_recursively(*path): path = os.path.join(*path) - return [dirpath for dirpath, dirs, files in os.walk(path) if not dirs and not files] + return [dirpath for dirpath, dirs, files in os.walk( + path) if not dirs and not files] def list_dirs_recursively(*path): @@ -919,19 +927,23 @@ def check_extra_files(path, target_files): check_pass = False missing_files.append(os.path.join(path, dir)) else: - files = [f.split(".")[0] for f in list_files(os.path.join(path, dir))] + files = [f.split(".")[0] + for f in list_files(os.path.join(path, dir))] for target_file in target_files[dir]: if target_file not in files: check_pass = False - missing_files.append(f"{os.path.join(path, dir, target_file)}.png") + missing_files.append( + f"{os.path.join(path, dir, target_file)}.png") if "captions" not in files: - missing_files.append(f"{os.path.join(path, dir, 'captions.txt')}") + missing_files.append( + f"{os.path.join(path, dir, 'captions.txt')}") return check_pass, missing_files def split_path(m): return m.replace("\\", "/").split("/") + def get_boolean(s): if s is None: return False @@ -942,7 +954,8 @@ def get_boolean(s): elif isinstance(s, int): return bool(s) else: - raise TypeError(f"Variable should be bool, string or int, got {type(s)} instead") + raise TypeError( + f"Variable should be bool, string or int, got {type(s)} instead") def find_error_in_detail_log(config, fname): @@ -987,19 +1000,20 @@ def check_accuracy_dir(config, model, path, verbose): up_patterns = [] acc_limit_check = True for i in range(0, len(acc_upper_limit), 2): - acc_type, acc_target = acc_upper_limit[i:i+2] + acc_type, acc_target = acc_upper_limit[i:i + 2] acc_limits.append(acc_target) up_patterns.append(ACC_PATTERN[acc_type]) for i in range(0, len(target), 2): - acc_type, acc_target = target[i:i+2] + acc_type, acc_target = target[i:i + 2] patterns.append(ACC_PATTERN[acc_type]) acc_targets.append(acc_target) acc_types.append(acc_type) acc_seen = [False for _ in acc_targets] with open(os.path.join(path, "accuracy.txt"), "r", encoding="utf-8") as f: for line in f: - for i, (pattern, acc_target, acc_type) in enumerate(zip(patterns, acc_targets, acc_types)): + for i, (pattern, acc_target, acc_type) in enumerate( + zip(patterns, acc_targets, acc_types)): m = re.match(pattern, line) if m: acc = m.group(1) @@ -1011,24 +1025,34 @@ def check_accuracy_dir(config, model, path, verbose): acc_seen[i] = True elif acc is not None: all_accuracy_valid = False - log.warning("%s accuracy not met: expected=%f, found=%s", path, acc_target, acc) + log.warning( + "%s accuracy not met: expected=%f, found=%s", + path, + acc_target, + acc) if acc: result_acc[acc_type] = acc acc = None if acc_upper_limit is not None: - for i, (pattern, acc_limit) in enumerate(zip(up_patterns, acc_limits)): + for i, (pattern, acc_limit) in enumerate( + zip(up_patterns, acc_limits)): m = re.match(pattern, line) if m: acc = m.group(1) m = re.match(r"^hash=([\w\d]+)$", line) if m: hash_val = m.group(1) - if acc is not None and acc_upper_limit is not None and float(acc) > acc_limit: + if acc is not None and acc_upper_limit is not None and float( + acc) > acc_limit: acc_limit_check = False - log.warning("%s accuracy not met: upper limit=%f, found=%s", path, acc_limit, acc) + log.warning( + "%s accuracy not met: upper limit=%f, found=%s", + path, + acc_limit, + acc) acc = None if all(acc_seen) and hash_val: - break; + break is_valid = all_accuracy_valid & all(acc_seen) if acc_upper_limit is not None: is_valid &= acc_limit_check @@ -1062,20 +1086,23 @@ def extra_check_llm(mlperf_log, scenario, model): return None, True else: for constraint, limits in LLM_LATENCY_LIMITS[model].items(): - if mlperf_log["result_first_token_99.00_percentile_latency_ns"] < limits["ttft"] and mlperf_log["result_time_per_output_token_99.00_percentile_ns"] < limits["tpot"]: + if mlperf_log["result_first_token_99.00_percentile_latency_ns"] < limits[ + "ttft"] and mlperf_log["result_time_per_output_token_99.00_percentile_ns"] < limits["tpot"]: return constraint, True else: - log.error(f'use_token_latencies flag needs to be enabled for Llama2 benchmark') + log.error( + f'use_token_latencies flag needs to be enabled for Llama2 benchmark') return None, False - log.error(f'Failed Llama2 extra check for TTFT and TPOT. TTFT 99-tile: {mlperf_log["result_first_token_99.00_percentile_latency_ns"]}, TPOT 99-tile: {mlperf_log["result_time_per_output_token_99.00_percentile_ns"]}') + log.error( + f'Failed Llama2 extra check for TTFT and TPOT. TTFT 99-tile: {mlperf_log["result_first_token_99.00_percentile_latency_ns"]}, TPOT 99-tile: {mlperf_log["result_time_per_output_token_99.00_percentile_ns"]}') return None, False - + def get_performance_metric( config, model, path, scenario_fixed, division, system_json, has_power=False ): - #Assumes new logging format + # Assumes new logging format version = config.version fname = os.path.join(path, "mlperf_log_detail.txt") @@ -1088,15 +1115,19 @@ def get_performance_metric( scenario = mlperf_log["effective_scenario"] res = float(mlperf_log[RESULT_FIELD_NEW[version][scenario]]) - if version in RESULT_FIELD_BENCHMARK_OVERWRITE and model in RESULT_FIELD_BENCHMARK_OVERWRITE[version] and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model]: - res = float(mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version][model][scenario]]) + if version in RESULT_FIELD_BENCHMARK_OVERWRITE and model in RESULT_FIELD_BENCHMARK_OVERWRITE[ + version] and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model]: + res = float( + mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version][model][scenario]]) inferred = False if scenario_fixed != scenario: - inferred, res = get_inferred_result(scenario_fixed, scenario, res, mlperf_log, config, False) + inferred, res = get_inferred_result( + scenario_fixed, scenario, res, mlperf_log, config, False) return res + def check_performance_dir( config, model, path, scenario_fixed, division, system_json, has_power=False ): @@ -1119,12 +1150,14 @@ def check_performance_dir( scenario = mlperf_log["effective_scenario"] res = float(mlperf_log[RESULT_FIELD_NEW[version][scenario]]) - if version in RESULT_FIELD_BENCHMARK_OVERWRITE and model in RESULT_FIELD_BENCHMARK_OVERWRITE[version] and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model]: - res = float(mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version][model][scenario]]) + if version in RESULT_FIELD_BENCHMARK_OVERWRITE and model in RESULT_FIELD_BENCHMARK_OVERWRITE[ + version] and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][model]: + res = float( + mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version][model][scenario]]) - if model in ["llama2-70b-99", "llama2-70b-99.9", "mixtral-8x7b"]: - llama_constraint, is_valid = extra_check_llm(mlperf_log, scenario_fixed, model) + llama_constraint, is_valid = extra_check_llm( + mlperf_log, scenario_fixed, model) latency_99_percentile = mlperf_log["result_99.00_percentile_latency_ns"] latency_mean = mlperf_log["result_mean_latency_ns"] @@ -1136,7 +1169,8 @@ def check_performance_dir( min_query_count = mlperf_log["effective_min_query_count"] samples_per_query = mlperf_log["effective_samples_per_query"] min_duration = mlperf_log["effective_min_duration_ms"] - equal_issue_used_check = (mlperf_log["effective_sample_concatenate_permutation"] == "true") + equal_issue_used_check = ( + mlperf_log["effective_sample_concatenate_permutation"] == "true") if not config.requires_equal_issue(model, division): equal_issue_used_check = True sut_name = mlperf_log["sut_name"] @@ -1146,7 +1180,8 @@ def check_performance_dir( if not find_error_in_detail_log(config, fname): is_valid = False - required_performance_sample_count = config.get_performance_sample_count(model) + required_performance_sample_count = config.get_performance_sample_count( + model) if performance_sample_count < required_performance_sample_count: log.error( "%s performance_sample_count, found %d, needs to be >= %d", @@ -1184,7 +1219,6 @@ def check_performance_dir( if scenario == "SingleStream" or scenario == "MultiStream": res /= MS_TO_NS - # Check if the current scenario uses early stopping uses_early_stopping = config.uses_early_stopping(scenario) @@ -1201,7 +1235,8 @@ def check_performance_dir( # If the scenario has a target latency (Server scenario), check # that the target latency that was passed to the early stopping # is less than the target latency. - target_latency = config.latency_constraint.get(model, dict()).get(scenario) + target_latency = config.latency_constraint.get( + model, dict()).get(scenario) if target_latency: early_stopping_latency_ns = mlperf_log["effective_target_latency_ns"] log.info( @@ -1221,7 +1256,8 @@ def check_performance_dir( else: # check if the benchmark meets latency constraint - target_latency = config.latency_constraint.get(model, dict()).get(scenario) + target_latency = config.latency_constraint.get( + model, dict()).get(scenario) log.info( "Target latency: %s, Latency: %s, Scenario: %s", target_latency, @@ -1252,7 +1288,8 @@ def check_performance_dir( ) is_valid = False - if scenario == "Offline" and (samples_per_query < OFFLINE_MIN_SPQ_SINCE_V4[model]): + if scenario == "Offline" and ( + samples_per_query < OFFLINE_MIN_SPQ_SINCE_V4[model]): log.error( "%s Required minimum samples per query not met by user config, Expected=%s, Found=%s", fname, @@ -1275,14 +1312,16 @@ def check_performance_dir( inferred = False if scenario_fixed != scenario: - inferred, res = get_inferred_result(scenario_fixed, scenario, res, mlperf_log, config, True) + inferred, res = get_inferred_result( + scenario_fixed, scenario, res, mlperf_log, config, True) is_network_system, is_network_mode_valid = is_system_over_network( division, system_json, path ) is_valid &= is_network_mode_valid if is_network_system: - # for network mode verify the SUT name is valid, according to the rules (must include "Network SUT" in name) + # for network mode verify the SUT name is valid, according to the rules + # (must include "Network SUT" in name) if NETWORK_MODE_REQUIRED_SUBSTRING_IN_SUT_NAME not in sut_name: log.error( f"{fname} invalid sut name for network mode. expecting the substring '{NETWORK_MODE_REQUIRED_SUBSTRING_IN_SUT_NAME}' got '{sut_name}'" @@ -1291,7 +1330,9 @@ def check_performance_dir( return is_valid, res, inferred, equal_issue_used_check -def get_inferred_result(scenario_fixed, scenario, res, mlperf_log, config, log_error=False): + +def get_inferred_result(scenario_fixed, scenario, res, + mlperf_log, config, log_error=False): inferred = False # Check if current scenario (and version) uses early stopping @@ -1305,7 +1346,8 @@ def get_inferred_result(scenario_fixed, scenario, res, mlperf_log, config, log_e latency_mean = mlperf_log["result_mean_query_latency_ns"] samples_per_query = mlperf_log["effective_samples_per_query"] if scenario == "SingleStream": - # qps_wo_loadgen_overhead is only used for inferring Offline from SingleStream; only for old submissions + # qps_wo_loadgen_overhead is only used for inferring Offline from + # SingleStream; only for old submissions qps_wo_loadgen_overhead = mlperf_log["result_qps_without_loadgen_overhead"] # special case for results inferred from different scenario @@ -1338,11 +1380,12 @@ def get_inferred_result(scenario_fixed, scenario, res, mlperf_log, config, log_e res = (latency_99_percentile * samples_per_query) / MS_TO_NS return inferred, res + def get_power_metric(config, scenario_fixed, log_path, is_valid, res): # parse the power logs server_timezone = datetime.timedelta(0) client_timezone = datetime.timedelta(0) - + detail_log_fname = os.path.join(log_path, "mlperf_log_detail.txt") mlperf_log = MLPerfLog(detail_log_fname) datetime_format = "%m-%d-%Y %H:%M:%S.%f" @@ -1356,7 +1399,7 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res): ) # Obtain the scenario also from logs to check if power is inferred scenario = mlperf_log["effective_scenario"] - + spl_fname = os.path.join(log_path, "spl.txt") power_list = [] with open(spl_fname) as f: @@ -1389,7 +1432,8 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res): avg_power_efficiency = res / avg_power else: - # In SingleStream and MultiStream scenarios, the power metric is in mJ/query. + # In SingleStream and MultiStream scenarios, the power metric is in + # mJ/query. assert scenario_fixed in [ "MultiStream", "SingleStream", @@ -1432,11 +1476,17 @@ def check_power_dir( # check if all the required files are present required_files = REQUIRED_PERF_FILES + REQUIRED_PERF_POWER_FILES - diff = files_diff(list_files(testing_path), required_files, OPTIONAL_PERF_FILES) + diff = files_diff( + list_files(testing_path), + required_files, + OPTIONAL_PERF_FILES) if diff: log.error("%s has file list mismatch (%s)", testing_path, diff) is_valid = False - diff = files_diff(list_files(ranging_path), required_files, OPTIONAL_PERF_FILES) + diff = files_diff( + list_files(ranging_path), + required_files, + OPTIONAL_PERF_FILES) if diff: log.error("%s has file list mismatch (%s)", ranging_path, diff) is_valid = False @@ -1474,7 +1524,9 @@ def check_power_dir( sys.stdout.flush() sys.stderr.flush() if check_power_result != 0: - log.error("Power WG power_checker.py did not pass for: %s", perf_path) + log.error( + "Power WG power_checker.py did not pass for: %s", + perf_path) is_valid = False return is_valid, power_metric, power_efficiency_testing @@ -1620,19 +1672,19 @@ def log_result( "Offline": "Tokens/s", "Server": "Tokens/s", }, - "llama2-70b-99" : { + "llama2-70b-99": { "SingleStream": "Latency (ms)", "MultiStream": "Latency (ms)", "Offline": "Tokens/s", "Server": "Tokens/s", }, - "llama2-70b-99.9" : { + "llama2-70b-99.9": { "SingleStream": "Latency (ms)", "MultiStream": "Latency (ms)", "Offline": "Tokens/s", "Server": "Tokens/s", }, - "mixtral-8x7b" : { + "mixtral-8x7b": { "SingleStream": "Latency (ms)", "MultiStream": "Latency (ms)", "Offline": "Tokens/s", @@ -1657,8 +1709,8 @@ def log_result( unit = special_unit_dict.get(model_name, unit_dict)[scenario_fixed] power_unit = power_unit_dict[scenario_fixed] - - if (power_metric <= 0) or (not get_boolean(system_json.get("system_power_only"))): + if (power_metric <= 0) or (not get_boolean( + system_json.get("system_power_only"))): csv.write( fmt.format( submitter, @@ -1690,7 +1742,7 @@ def log_result( unit, '"' + weight_data_types + '"', ) - ) + ) if power_metric > 0: csv.write( @@ -1770,7 +1822,7 @@ def log_result( if not os.path.exists(results_path): continue - ## Apply folder checks + # Apply folder checks dirs = list_dirs_recursively(division, submitter) files = list_files_recursively(division, submitter) @@ -1871,7 +1923,9 @@ def log_result( extra_model_mapping = json.load(fp) for system_desc in list_dir(results_path): - # we are looking at ./$division/$submitter/results/$system_desc, ie ./closed/mlperf_org/results/t4-ort + # we are looking at + # ./$division/$submitter/results/$system_desc, ie + # ./closed/mlperf_org/results/t4-ort # # check if system_id is good. @@ -1900,7 +1954,8 @@ def log_result( if config.version not in ["v0.5"]: valid_system_types = ["datacenter", "edge"] if config.version not in ["v0.7"]: - valid_system_types += ["datacenter,edge", "edge,datacenter"] + valid_system_types += ["datacenter,edge", + "edge,datacenter"] if system_type not in valid_system_types: log.error( "%s has invalid system type (%s)", @@ -1934,7 +1989,8 @@ def log_result( if is_closed_or_network and mlperf_model not in config.models: # for closed/network divisions we want the model name to match. - # for open division the model_name might be different than the task + # for open division the model_name might be different + # than the task log.error( "%s has an invalid model %s for closed/network division", name, @@ -1962,9 +2018,12 @@ def log_result( list(required_scenarios) + list(config.get_optional(mlperf_model)) ) - for scenario in list_dir(results_path, system_desc, model_name): - # some submissions in v0.5 use lower case scenarios - map them for now - scenario_fixed = SCENARIO_MAPPING.get(scenario, scenario) + for scenario in list_dir( + results_path, system_desc, model_name): + # some submissions in v0.5 use lower case scenarios - + # map them for now + scenario_fixed = SCENARIO_MAPPING.get( + scenario, scenario) # Skip scenario for debug purposes if scenario in scenarios_to_skip: @@ -2016,7 +2075,8 @@ def log_result( scenario, ) if not os.path.exists(measurement_dir): - log.error("no measurement_dir for %s", measurement_dir) + log.error( + "no measurement_dir for %s", measurement_dir) results[measurement_dir] = None errors += 1 continue @@ -2046,7 +2106,8 @@ def log_result( # check accuracy accuracy_is_valid = False acc_path = os.path.join(name, "accuracy") - if not os.path.exists(os.path.join(acc_path, "accuracy.txt")): + if not os.path.exists( + os.path.join(acc_path, "accuracy.txt")): log.error( "%s has no accuracy.txt. Generate it with accuracy-imagenet.py or accuracy-coco.py or " "process_accuracy.py", @@ -2055,7 +2116,8 @@ def log_result( errors += 1 continue elif scenario not in scenarios_to_skip: - diff = files_diff(list_files(acc_path), REQUIRED_ACC_FILES) + diff = files_diff( + list_files(acc_path), REQUIRED_ACC_FILES) if diff: log.error( "%s has file list mismatch (%s)", acc_path, diff @@ -2068,10 +2130,19 @@ def log_result( acc_path, debug or is_closed_or_network, ) - acc = json.dumps(acc).replace(",", " ").replace('"', "").replace("{", "").replace("}", "") + acc = json.dumps(acc).replace( + ",", + " ").replace( + '"', + "").replace( + "{", + "").replace( + "}", + "") if mlperf_model in REQUIRED_ACC_BENCHMARK: if config.version in REQUIRED_ACC_BENCHMARK[mlperf_model]: - extra_files_pass, missing_files = check_extra_files(acc_path, REQUIRED_ACC_BENCHMARK[mlperf_model][config.version]) + extra_files_pass, missing_files = check_extra_files( + acc_path, REQUIRED_ACC_BENCHMARK[mlperf_model][config.version]) if not extra_files_pass: log.error( "%s expected to have the following extra files (%s)", acc_path, missing_files @@ -2085,7 +2156,8 @@ def log_result( ) accuracy_is_valid = True if not accuracy_is_valid: - # a little below we'll not copy this into the results csv + # a little below we'll not copy this into the + # results csv errors += 1 log.error("%s, accuracy not valid", acc_path) @@ -2278,14 +2350,16 @@ def log_result( ) else: results[name] = None - log.error("%s is OK but accuracy has issues", name) + log.error( + "%s is OK but accuracy has issues", name) # Discard scenarios that we want to skip for scenario in scenarios_to_skip: required_scenarios.discard(scenario) if required_scenarios: - name = os.path.join(results_path, system_desc, model_name) + name = os.path.join( + results_path, system_desc, model_name) if is_closed_or_network: results[name] = None log.error( @@ -2313,9 +2387,8 @@ def check_system_desc_id( ): is_valid = True # check all required fields - - required_fields = SYSTEM_DESC_REQUIRED_FIELDS.copy() + required_fields = SYSTEM_DESC_REQUIRED_FIELDS.copy() is_network_system, is_network_mode_valid = is_system_over_network( division, systems_json, fname @@ -2340,7 +2413,6 @@ def check_system_desc_id( "%s, field %s requires a meaningful response but is empty", fname, k ) - # SYSTEM_DESC_REQUIRED_FIELDS_POWER should be mandatory when a submission has power logs, but since we # check power submission in check_results_dir, the information is not available yet at this stage and we do # this check later @@ -2428,21 +2500,45 @@ def check_measurement_dir( if has_power and not skip_check_power_measure_files: path = measurement_dir - all_files_1 = [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + all_files_1 = [ + os.path.join( + path, + f) for f in os.listdir(path) if os.path.isfile( + os.path.join( + path, + f))] path = os.path.join(path, "..") - all_files_2 = [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + all_files_2 = [ + os.path.join( + path, + f) for f in os.listdir(path) if os.path.isfile( + os.path.join( + path, + f))] path = os.path.join(path, "..") - all_files_3 = [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + all_files_3 = [ + os.path.join( + path, + f) for f in os.listdir(path) if os.path.isfile( + os.path.join( + path, + f))] path = os.path.join(path, "..") - all_files_4 = [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + all_files_4 = [ + os.path.join( + path, + f) for f in os.listdir(path) if os.path.isfile( + os.path.join( + path, + f))] all_files = all_files_1 + all_files_2 + all_files_3 + all_files_4 for i in REQUIRED_POWER_MEASURE_FILES: found = False for file in all_files: if re.match(i, os.path.basename(file)): - found = True - file_path = file + found = True + file_path = file if not found: log.error("%s is missing %s", measurement_dir, i) is_valid = False @@ -2482,9 +2578,10 @@ def check_measurement_dir( log.error("%s, field %s is missing", fname, k) elif check_empty_fields and not j[k]: is_valid = False - log.error("%s, field %s is missing meaningful value", fname, k) + log.error( + "%s, field %s is missing meaningful value", fname, k) - impl = system_file[len(system_desc) + 1 : -end] + impl = system_file[len(system_desc) + 1: -end] code_dir = os.path.join(root, "code", model) if os.path.isfile(code_dir): with open(code_dir, "r") as f: @@ -2498,7 +2595,7 @@ def check_measurement_dir( if not os.path.exists(os.path.dirname(code_dir)): log.error("%s is missing code_dir %s", fname, code_dir) is_valid = False - + else: log.error("%s is missing %s*.json", fname, system_desc) is_valid = False @@ -2521,7 +2618,9 @@ def check_compliance_perf_dir(test_dir): is_valid = True break if is_valid == False: - log.error("Compliance test performance check in %s failed", test_dir) + log.error( + "Compliance test performance check in %s failed", + test_dir) # Check performance dir test_perf_path = os.path.join(test_dir, "performance", "run_1") @@ -2535,7 +2634,10 @@ def check_compliance_perf_dir(test_dir): ["mlperf_log_accuracy.json"], ) if diff: - log.error("%s has file list mismatch (%s)", test_perf_path, diff) + log.error( + "%s has file list mismatch (%s)", + test_perf_path, + diff) is_valid = False return is_valid @@ -2577,14 +2679,17 @@ def check_compliance_acc_dir(test_dir, model, config): else REQUIRED_TEST01_ACC_FILES, ) if diff: - log.error("%s has file list mismatch (%s)", test_acc_path, diff) + log.error( + "%s has file list mismatch (%s)", + test_acc_path, + diff) is_valid = False elif not acc_passed: target = config.get_accuracy_target(model) patterns = [] acc_types = [] for i in range(0, len(target), 2): - acc_type = target[i:i+2] + acc_type = target[i:i + 2] acc_types.append(acc_type) patterns.append(ACC_PATTERN[acc_type[0]]) acc_seen = [False for _ in acc_type] @@ -2613,14 +2718,17 @@ def check_compliance_acc_dir(test_dir, model, config): for acc_type, pattern in zip(acc_types, patterns): m = re.match(pattern, line) if m: - acc_compliance[acc_type] = float(m.group(1)) + acc_compliance[acc_type] = float( + m.group(1)) for acc_type in acc_types: if acc_baseline[acc_type] == 0 or acc_compliance[acc_type] == 0: is_valid = False break else: - required_delta_perc = config.get_delta_perc(model, acc_type[0]) - delta_perc = abs(1 - acc_baseline[acc_type] / acc_compliance[acc_type]) * 100 + required_delta_perc = config.get_delta_perc( + model, acc_type[0]) + delta_perc = abs( + 1 - acc_baseline[acc_type] / acc_compliance[acc_type]) * 100 if delta_perc <= required_delta_perc: is_valid = True else: @@ -2641,9 +2749,11 @@ def check_compliance_acc_dir(test_dir, model, config): length_check_pass = "Sample length check pass: True" in lines is_valid = first_token_pass and eos_pass and length_check_pass if not is_valid: - log.error(f"TEST06 accuracy check failed. first_token_check: {first_token_pass} eos_check: {eos_pass} length_check: {length_check_pass}.") + log.error( + f"TEST06 accuracy check failed. first_token_check: {first_token_pass} eos_check: {eos_pass} length_check: {length_check_pass}.") else: - raise NotImplemented(f"{test_dir} is neither TEST01 and TEST06, which doesn't require accuracy check") + raise NotImplemented( + f"{test_dir} is neither TEST01 and TEST06, which doesn't require accuracy check") return is_valid @@ -2694,17 +2804,16 @@ def check_compliance_dir( if model in [ "stable-diffusion-xl" - ] and config.version in [ "v4.0" ]: + ] and config.version in ["v4.0"]: test_list.remove("TEST01") test_list.remove("TEST04") - if model in [ "llama2-70b-99", "llama2-70b-99.9", "mixtral-8x7b" ]: - test_list.append("TEST06") + test_list.append("TEST06") if test_list and not os.path.exists(compliance_dir): log.error("no compliance dir for %s: %s", name, compliance_dir) @@ -2728,7 +2837,10 @@ def check_compliance_dir( config, model, compliance_perf_dir, scenario, division, system_json ) if is_inferred: - log.info("%s has inferred results, qps=%s", compliance_perf_dir, r) + log.info( + "%s has inferred results, qps=%s", + compliance_perf_dir, + r) except Exception as e: log.error( "%s caused exception in check_performance_dir: %s", @@ -2742,7 +2854,7 @@ def check_compliance_dir( and compliance_perf_valid ) - compliance_acc_pass= True + compliance_acc_pass = True for test in ["TEST01", "TEST06"]: if test in test_list: # Check accuracy for TEST01 @@ -2750,7 +2862,6 @@ def check_compliance_dir( os.path.join(compliance_dir, test), model, config ) - return compliance_perf_pass and compliance_acc_pass and compliance_perf_dir_pass @@ -2836,7 +2947,8 @@ def merge_two_dict(x, y): unique_closed_systems = merge_two_dict( closed_power_systems, closed_non_power_systems ) - unique_open_systems = merge_two_dict(open_power_systems, open_non_power_systems) + unique_open_systems = merge_two_dict( + open_power_systems, open_non_power_systems) unique_network_systems = merge_two_dict( network_power_systems, network_non_power_systems ) @@ -2845,8 +2957,10 @@ def merge_two_dict(x, y): unique_systems = merge_two_dict(unique_systems, unique_network_systems) # power systems can be repeating in open, closed and network - unique_power_systems = merge_two_dict(closed_power_systems, open_power_systems) - unique_power_systems = merge_two_dict(unique_power_systems, network_power_systems) + unique_power_systems = merge_two_dict( + closed_power_systems, open_power_systems) + unique_power_systems = merge_two_dict( + unique_power_systems, network_power_systems) number_systems = len(unique_systems) number_power_systems = len(unique_power_systems) @@ -2867,7 +2981,8 @@ def sum_dict_values(x): count_open_results = count_open_power_results + count_open_non_power_results count_network_power_results = sum_dict_values(network_power_systems) - count_network_non_power_results = sum_dict_values(network_non_power_systems) + count_network_non_power_results = sum_dict_values( + network_non_power_systems) count_network_results = ( count_network_power_results + count_network_non_power_results ) @@ -2905,7 +3020,10 @@ def sum_dict_values(x): ) log.info("---") - log.info("Systems=%d, Power Systems=%d", number_systems, number_power_systems) + log.info( + "Systems=%d, Power Systems=%d", + number_systems, + number_power_systems) log.info( "Closed Systems=%d, Closed Power Systems=%d", number_closed_systems, From 0e224cd21fccf2c87834de44b3f6593cbfbd2fbf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 04:19:47 +0530 Subject: [PATCH 25/61] Increment version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index b05079e54..467d09372 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.12 +4.1.13 From fe973c88fc5cbbfb3c0d78755b113c606842dec2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 04:21:30 +0530 Subject: [PATCH 26/61] Increment version --- .github/workflows/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 45233cef3..8417cc06c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -27,12 +27,12 @@ jobs: echo "::set-output name=version_changed::true" new_version=$(cat VERSION.txt) else - echo "VERSION file has NOT been modified" + echo "VERSION.txt file has NOT been modified" echo "::set-output name=version_changed::false" fi echo "::set-output name=new_version::$new_version" - # Step 4: Increment version if VERSION was not changed + # Step 4: Increment version if VERSION.txt was not changed - name: Increment version if necessary id: do_version_increment if: steps.version_changed.outputs.version_changed == 'false' From 8a77640f66b93e628d577a985cd061f2be63909d Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Wed, 23 Oct 2024 22:51:52 +0000 Subject: [PATCH 27/61] Increment version to 4.1.14 --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 467d09372..5888f1a46 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.13 +4.1.14 From 21b1bfa00d4a2f11048d75335fd740aca0195ce2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:06:45 +0530 Subject: [PATCH 28/61] Cleanup for build_wheels gh action --- .github/workflows/build_wheels.yml | 44 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8417cc06c..548316c6a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -13,29 +13,31 @@ on: jobs: update_version: - name: Update version only on ubuntu but used by windows and macos - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + name: Update version only on ubuntu but used by windows and macos + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + # Check if VERSION.txt file has changed in this push + - name: Check if VERSION.txt file has changed + id: version_changed + run: | + echo "version_changed=false" >> $GITHUB_ENV + echo "new_version=" >> $GITHUB_ENV # Initialize with empty value + if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then + echo "VERSION.txt file has been modified" + echo "version_changed=true" >> $GITHUB_ENV + new_version=$(cat VERSION.txt) + echo "new_version=$new_version" >> $GITHUB_ENV + else + echo "VERSION.txt file has NOT been modified" + fi - # Step 3: Check if VERSION.txt file has changed in this push - - name: Check if VERSION.txt file has changed - id: version_changed - run: | - if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then - echo "VERSION.txt file has been modified" - echo "::set-output name=version_changed::true" - new_version=$(cat VERSION.txt) - else - echo "VERSION.txt file has NOT been modified" - echo "::set-output name=version_changed::false" - fi - echo "::set-output name=new_version::$new_version" # Step 4: Increment version if VERSION.txt was not changed - name: Increment version if necessary id: do_version_increment - if: steps.version_changed.outputs.version_changed == 'false' + if: env.version_changed == 'false' run: | cd loadgen # Check if VERSION file exists, else initialize it @@ -49,17 +51,17 @@ jobs: new_version="$major.$minor.$patch" echo $new_version > VERSION.txt echo "New version: $new_version" - echo "::set-output name=new_version::$new_version" + echo "new_version=$new_version" >> $GITHUB_ENV # Step 5: Commit the updated version to the repository - name: Commit updated version - if: steps.version_changed.outputs.version_changed == 'false' + if: env.version_changed == 'false' run: | cd loadgen git config --global user.name "${{ github.actor }}" git config --global user.email "${{ github.actor }}@users.noreply.github.com" git add VERSION.txt - git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}" + git commit -m "Increment version to $new_version" git push build_wheels: From fd9a6d8bf1c75b8160edef5ca9bf7a7d3322cb42 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:12:23 +0530 Subject: [PATCH 29/61] Cleanup for build_wheels gh action --- .github/workflows/build_wheels.yml | 37 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 548316c6a..841e4ca2a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -13,26 +13,25 @@ on: jobs: update_version: - name: Update version only on ubuntu but used by windows and macos - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - # Check if VERSION.txt file has changed in this push - - name: Check if VERSION.txt file has changed - id: version_changed - run: | - echo "version_changed=false" >> $GITHUB_ENV - echo "new_version=" >> $GITHUB_ENV # Initialize with empty value - if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then - echo "VERSION.txt file has been modified" - echo "version_changed=true" >> $GITHUB_ENV - new_version=$(cat VERSION.txt) - echo "new_version=$new_version" >> $GITHUB_ENV - else - echo "VERSION.txt file has NOT been modified" - fi + name: Update version only on ubuntu but used by windows and macos + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + # Check if VERSION.txt file has changed in this push + - name: Check if VERSION.txt file has changed + id: version_changed + run: | + echo "version_changed=false" >> $GITHUB_ENV + echo "new_version=" >> $GITHUB_ENV # Initialize with empty value + if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then + echo "VERSION.txt file has been modified" + echo "version_changed=true" >> $GITHUB_ENV + new_version=$(cat VERSION.txt) + echo "new_version=$new_version" >> $GITHUB_ENV + else + echo "VERSION.txt file has NOT been modified" + fi # Step 4: Increment version if VERSION.txt was not changed - name: Increment version if necessary From ee7e5e1f8f86383dda8932cbe8aeaabcb1ac94c5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:16:02 +0530 Subject: [PATCH 30/61] Increment version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 5888f1a46..796262d5e 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.14 +4.1.15 From e6b5e7c93f5dffbf2f2f37d62ee45922746f8f66 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 24 Oct 2024 13:46:18 +0000 Subject: [PATCH 31/61] Increment version to 4.1.16 --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 796262d5e..1b94a070e 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.15 +4.1.16 From d65eabe1b9431ee94e0d49bbc98109936116303f Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Wed, 23 Oct 2024 22:51:52 +0000 Subject: [PATCH 32/61] Increment version to 4.1.14 --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 467d09372..5888f1a46 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.13 +4.1.14 From cc935e0d74814ec68adfe212a2875159f8201319 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:12:23 +0530 Subject: [PATCH 33/61] Cleanup for build_wheels gh action --- .github/workflows/build_wheels.yml | 37 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 548316c6a..841e4ca2a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -13,26 +13,25 @@ on: jobs: update_version: - name: Update version only on ubuntu but used by windows and macos - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - # Check if VERSION.txt file has changed in this push - - name: Check if VERSION.txt file has changed - id: version_changed - run: | - echo "version_changed=false" >> $GITHUB_ENV - echo "new_version=" >> $GITHUB_ENV # Initialize with empty value - if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then - echo "VERSION.txt file has been modified" - echo "version_changed=true" >> $GITHUB_ENV - new_version=$(cat VERSION.txt) - echo "new_version=$new_version" >> $GITHUB_ENV - else - echo "VERSION.txt file has NOT been modified" - fi + name: Update version only on ubuntu but used by windows and macos + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + # Check if VERSION.txt file has changed in this push + - name: Check if VERSION.txt file has changed + id: version_changed + run: | + echo "version_changed=false" >> $GITHUB_ENV + echo "new_version=" >> $GITHUB_ENV # Initialize with empty value + if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then + echo "VERSION.txt file has been modified" + echo "version_changed=true" >> $GITHUB_ENV + new_version=$(cat VERSION.txt) + echo "new_version=$new_version" >> $GITHUB_ENV + else + echo "VERSION.txt file has NOT been modified" + fi # Step 4: Increment version if VERSION.txt was not changed - name: Increment version if necessary From d5e9431f52c5f503535000e8923c6c1464bb8443 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:16:02 +0530 Subject: [PATCH 34/61] Increment version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 5888f1a46..796262d5e 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.14 +4.1.15 From 746a74a567afe29d37bb2f0134606d3fc58f17e2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:22:49 +0530 Subject: [PATCH 35/61] Cleanup for build_wheels gh action --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 841e4ca2a..90ebf87b7 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -24,7 +24,7 @@ jobs: run: | echo "version_changed=false" >> $GITHUB_ENV echo "new_version=" >> $GITHUB_ENV # Initialize with empty value - if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then + if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then echo "VERSION.txt file has been modified" echo "version_changed=true" >> $GITHUB_ENV new_version=$(cat VERSION.txt) From d0b567c237090d547d54cf6d22029f6cf91c5a85 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 19:35:27 +0530 Subject: [PATCH 36/61] Increment version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 73274fd69..60623b5c5 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.17 +4.1.18 From 3b698d025e8488fa80e375008e3152a7b2443d7a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 20:11:24 +0530 Subject: [PATCH 37/61] Increment version --- .github/workflows/build_wheels.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 51a068e34..90ebf87b7 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -24,11 +24,7 @@ jobs: run: | echo "version_changed=false" >> $GITHUB_ENV echo "new_version=" >> $GITHUB_ENV # Initialize with empty value -<<<<<<< HEAD if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then -======= - if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q "VERSION.txt"; then ->>>>>>> origin/dev echo "VERSION.txt file has been modified" echo "version_changed=true" >> $GITHUB_ENV new_version=$(cat VERSION.txt) From cdaa4dda2d872858ecbbbf5124022fb8ec838bcf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 20:13:38 +0530 Subject: [PATCH 38/61] Increment version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 60623b5c5..3cee4a07a 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.18 +4.1.19 From a408b9f9aa0ad5ff35dc81345a672d7fad4b02a0 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 24 Oct 2024 14:43:53 +0000 Subject: [PATCH 39/61] Increment version to 4.1.20 --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 3cee4a07a..b76ffaade 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.19 +4.1.20 From aa07e7f1061b964cce70e25cc42a828805b7298b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 20:19:27 +0530 Subject: [PATCH 40/61] Increment version --- .github/workflows/build_wheels.yml | 2 ++ loadgen/VERSION.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 90ebf87b7..8141beb72 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + fetch-depth: 2 # Check if VERSION.txt file has changed in this push - name: Check if VERSION.txt file has changed diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 3cee4a07a..b76ffaade 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.19 +4.1.20 From d6e2d69064038214bf9d76f698eb6ed228fe19aa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 20:22:16 +0530 Subject: [PATCH 41/61] Increment version --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index b76ffaade..7fbb6275f 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.20 +4.1.21 From 6489ef4aa725e4445657309c3bb632d31087c94d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 20:24:05 +0530 Subject: [PATCH 42/61] Increment version --- .github/workflows/build_wheels.yml | 2 +- loadgen/VERSION.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8141beb72..dc029bf3a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -29,7 +29,7 @@ jobs: if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then echo "VERSION.txt file has been modified" echo "version_changed=true" >> $GITHUB_ENV - new_version=$(cat VERSION.txt) + new_version=$(cat loadgen/VERSION.txt) echo "new_version=$new_version" >> $GITHUB_ENV else echo "VERSION.txt file has NOT been modified" diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 7fbb6275f..566261e13 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.21 +4.1.22 From 1cd31f937a42eccc156f97479e9f1e1ec614b6c4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 14:29:10 +0100 Subject: [PATCH 43/61] Update submission_checker.py | replace .json by model-info.json --- tools/submission/submission_checker.py | 48 +++++++------------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 2acfdb838..861936c2e 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -412,7 +412,7 @@ } } } -REQUIRED_MEASURE_FILES = ["user.conf", "README.md"] +REQUIRED_MEASURE_FILES = ["user.conf", "README.md", "model-info.json"] REQUIRED_POWER_MEASURE_FILES = ["analyzer_table.*", "power_settings.*"] MS_TO_NS = 1000 * 1000 S_TO_MS = 1000 @@ -1951,19 +1951,17 @@ def log_result( results[name] = None continue system_type = system_json.get("system_type") - if config.version not in ["v0.5"]: - valid_system_types = ["datacenter", "edge"] - if config.version not in ["v0.7"]: - valid_system_types += ["datacenter,edge", - "edge,datacenter"] - if system_type not in valid_system_types: - log.error( - "%s has invalid system type (%s)", - system_id_json, - system_type, - ) - results[name] = None - continue + valid_system_types = ["datacenter", "edge", "datacenter,edge", "edge,datacenter"] + + if system_type not in valid_system_types: + log.error( + "%s has invalid system type (%s)", + system_id_json, + system_type, + ) + results[name] = None + continue + config.set_type(system_type) if not check_system_desc_id( name, @@ -2546,26 +2544,7 @@ def check_measurement_dir( log.error("%s is having empty %s", measurement_dir, i) is_valid = False - for i in files: - if i.startswith(system_desc) and i.endswith("_" + scenario + ".json"): - system_file = i - end = len("_" + scenario + ".json") - break - elif i.startswith(system_desc) and i.endswith(".json"): - system_file = i - end = len(".json") - break - - if not system_file and os.environ.get("INFER_SYSTEM_FILE", "") == "yes": - for i in files: - if i.endswith(".json"): - system_file = system_desc + ".json" - os.rename( - os.path.join(measurement_dir, i), - os.path.join(measurement_dir, system_file), - ) - end = len(".json") - break + system_file = "model-info.json" weight_data_types = None if system_file: @@ -2767,7 +2746,6 @@ def check_compliance_dir( test_list = ["TEST01", "TEST04", "TEST05"] if model in [ - "rnnt", "bert-99", "bert-99.9", "dlrm-v2-99", From f63f3041f4db7b6ea11f70179b8e7153caf720f9 Mon Sep 17 00:00:00 2001 From: arjunsuresh <4791823+arjunsuresh@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:29:56 +0000 Subject: [PATCH 44/61] [Automated Commit] Format Codebase --- tools/submission/submission_checker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 861936c2e..937ad22c1 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -1951,8 +1951,9 @@ def log_result( results[name] = None continue system_type = system_json.get("system_type") - valid_system_types = ["datacenter", "edge", "datacenter,edge", "edge,datacenter"] - + valid_system_types = [ + "datacenter", "edge", "datacenter,edge", "edge,datacenter"] + if system_type not in valid_system_types: log.error( "%s has invalid system type (%s)", From 99f9c7f39fb8cddec2b606eafce7fe22dc14a19b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 16:35:24 +0100 Subject: [PATCH 45/61] Update submission_checker.py --- tools/submission/submission_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 937ad22c1..bad2fa15e 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -1,4 +1,4 @@ -"""A checker for MLPerf Inference submissions +"""A checker for MLPerf Inference submissions from v4.0 onwards (for checking older submissions please use the submission checker from the respective release) """ from __future__ import division From 6ec901310a78841a3e051068197b4059cbee0079 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 16:42:30 +0100 Subject: [PATCH 46/61] Retain backward compatibility of the submission checker since v4.0 --- tools/submission/submission_checker.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index bad2fa15e..02d247b0c 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2545,7 +2545,28 @@ def check_measurement_dir( log.error("%s is having empty %s", measurement_dir, i) is_valid = False - system_file = "model-info.json" + if config.version in [ "v4.0", "v4.1" ]: + for i in files: + if i.startswith(system_desc) and i.endswith("_" + scenario + ".json"): + system_file = i + end = len("_" + scenario + ".json") + break + elif i.startswith(system_desc) and i.endswith(".json"): + system_file = i + end = len(".json") + break + if not system_file and os.environ.get("INFER_SYSTEM_FILE", "") == "yes": + for i in files: + if i.endswith(".json"): + system_file = system_desc + ".json" + os.rename( + os.path.join(measurement_dir, i), + os.path.join(measurement_dir, system_file), + ) + end = len(".json") + break + else: + system_file = "model-info.json" weight_data_types = None if system_file: From 54072ed5a9e8b27a8714bd2f6027556183d270ea Mon Sep 17 00:00:00 2001 From: arjunsuresh <4791823+arjunsuresh@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:43:02 +0000 Subject: [PATCH 47/61] [Automated Commit] Format Codebase --- tools/submission/submission_checker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 02d247b0c..02d681351 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2545,9 +2545,10 @@ def check_measurement_dir( log.error("%s is having empty %s", measurement_dir, i) is_valid = False - if config.version in [ "v4.0", "v4.1" ]: + if config.version in ["v4.0", "v4.1"]: for i in files: - if i.startswith(system_desc) and i.endswith("_" + scenario + ".json"): + if i.startswith(system_desc) and i.endswith( + "_" + scenario + ".json"): system_file = i end = len("_" + scenario + ".json") break @@ -2555,7 +2556,8 @@ def check_measurement_dir( system_file = i end = len(".json") break - if not system_file and os.environ.get("INFER_SYSTEM_FILE", "") == "yes": + if not system_file and os.environ.get( + "INFER_SYSTEM_FILE", "") == "yes": for i in files: if i.endswith(".json"): system_file = system_desc + ".json" From 621b6dd94f7a77a49704bd9b45f95bbc3de3fa21 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 16:55:46 +0100 Subject: [PATCH 48/61] Support impl in model-info.json --- tools/submission/submission_checker.py | 38 ++++++++++---------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 02d681351..fe504b8f9 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -412,7 +412,7 @@ } } } -REQUIRED_MEASURE_FILES = ["user.conf", "README.md", "model-info.json"] +REQUIRED_MEASURE_FILES = ["user.conf", "README.md"] REQUIRED_POWER_MEASURE_FILES = ["analyzer_table.*", "power_settings.*"] MS_TO_NS = 1000 * 1000 S_TO_MS = 1000 @@ -2546,29 +2546,19 @@ def check_measurement_dir( is_valid = False if config.version in ["v4.0", "v4.1"]: - for i in files: - if i.startswith(system_desc) and i.endswith( - "_" + scenario + ".json"): - system_file = i - end = len("_" + scenario + ".json") - break - elif i.startswith(system_desc) and i.endswith(".json"): - system_file = i - end = len(".json") - break - if not system_file and os.environ.get( - "INFER_SYSTEM_FILE", "") == "yes": - for i in files: - if i.endswith(".json"): - system_file = system_desc + ".json" - os.rename( - os.path.join(measurement_dir, i), - os.path.join(measurement_dir, system_file), - ) - end = len(".json") - break - else: - system_file = "model-info.json" + system_file_prefix = system_desc + else: + system_file_prefix = "model-info" + for i in files: + if i.startswith(system_desc) and i.endswith( + "_" + scenario + ".json"): + system_file = i + end = len("_" + scenario + ".json") + break + elif i.startswith(system_desc) and i.endswith(".json"): + system_file = i + end = len(".json") + break weight_data_types = None if system_file: From 4be1d16c781fe5a03c7d35d76844c6a4f8dd99b6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 17:10:57 +0100 Subject: [PATCH 49/61] Update submission_checker.py --- tools/submission/submission_checker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index fe504b8f9..208c90fb6 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2767,6 +2767,7 @@ def check_compliance_dir( "3d-unet-99", "3d-unet-99.9", "retinanet", + "rnnt", "gptj-99", "gptj-99.9", "llama2-70b-99", From 6475ac9b4c80cffbc1e6155f71cb3fca208311ab Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 00:56:42 +0530 Subject: [PATCH 50/61] Submission checker cleanup, support move/removal of invalid results --- tools/submission/preprocess_submission.py | 200 +++++++++++++++++----- tools/submission/submission_checker.py | 59 +++---- 2 files changed, 184 insertions(+), 75 deletions(-) diff --git a/tools/submission/preprocess_submission.py b/tools/submission/preprocess_submission.py index fee2aadde..c18d7a31d 100644 --- a/tools/submission/preprocess_submission.py +++ b/tools/submission/preprocess_submission.py @@ -26,33 +26,39 @@ cd tools/submission python3 preprocess_submission.py --input ORIGINAL_SUBMISSION_DIRECTORY --submitter MY_ORG \\ - --output NEW_SUBMISSION_DIRECTORY + --output NEW_SUBMISSION_DIRECTORY """ def get_args(): """Parse commandline.""" parser = argparse.ArgumentParser(description="Infer scenario results", - formatter_class=argparse.RawDescriptionHelpFormatter, epilog=HELP_TEXT) + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=HELP_TEXT) parser.add_argument("--input", required=True, help="orignal submission directory") parser.add_argument("--output", help="new submission directory") parser.add_argument("--noinfer_low_accuracy_results", - help="do not infer low accuracy results if a high accuracy result is present", - default=False, action="store_true") - parser.add_argument("--nodelete_empty_dirs", - help="do not delete empty dirs in submission tree", - default=False, action="store_true") + help="do not infer low accuracy results if a high accuracy result is present", + default=False, action="store_true") + parser.add_argument("--nodelete-empty-dirs", + help="do not delete empty dirs in submission tree", + default=False, action="store_true") + parser.add_argument("--nomove-failed-to-open", + help="do not move failed results to open division", + default=False, action="store_true") + parser.add_argument("--nodelete-failed", + help="do not delete failed results (submission checker will fail)", + default=False, action="store_true") parser.add_argument( - "--version", - default="v4.0", - choices=list(checker.MODEL_CONFIG.keys()), - help="mlperf version") + "--version", + default="v4.1", + choices=list(checker.MODEL_CONFIG.keys()), + help="mlperf version") parser.add_argument("--submitter", help="filter to submitter") parser.add_argument( - "--extra-model-benchmark-map", - help="File containing extra custom model mapping.\ - It is assumed to be inside the folder open/", - default="model_mapping.json") + "--extra-model-benchmark-map", + help="File containing extra custom model mapping.\ + It is assumed to be inside the folder open/", + default="model_mapping.json") args = parser.parse_args() if not args.output: @@ -96,19 +102,123 @@ def copy_submission_dir(src, dst, filter_submitter): shutil.copytree(os.path.join(src, division, submitter), os.path.join(dst, division, submitter)) +def change_first_directory_to_open(path): + # Split the path into components + parts = path.split(os.path.sep) -def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, config): + # Modify the first directory in the path + for i, part in enumerate(parts): + if part: # Skip empty parts to handle initial slashes in absolute paths + parts[i] = "open" + break + + # Join the parts back into a modified path + modified_path = os.path.join(*parts) + return modified_path + +def clean_invalid_results(args, log_path, config, system_desc, system_json, model, mlperf_model, division, is_closed_or_network): + # cleanup invalid results + for scenario in list_dir(log_path, system_desc, model): + scenario_fixed = checker.SCENARIO_MAPPING.get( + scenario, scenario) + scenario_path = os.path.join(log_path, system_desc, model, scenario) + acc_path = os.path.join(scenario_path, "accuracy") + try: + accuracy_is_valid, acc = checker.check_accuracy_dir( + config, + mlperf_model, + acc_path, + is_closed_or_network, + ) + except Exception as e: + print(e) + accuracy_is_valid = False + perf_path = os.path.join(scenario_path, "performance", "run_1") + try: + perf_is_valid, r, is_inferred = checker.check_performance_dir( + config, + mlperf_model, + perf_path, + scenario_fixed, + division, + system_json, + ) + except Exception as e: + print(e) + perf_is_valid = False + if perf_is_valid: + power_path = os.path.join(scenario_path, "performance", "power") + has_power = os.path.exists(power_path) + if has_power: + ranging_path = os.path.join(scenario_path, "performance", "ranging") + try: + ranging_r = get_performance_metric( + config, + mlperf_model, + ranging_path, + scenario_fixed, + division, + ) + ( + power_is_valid, + power_metric, + power_efficiency, + ) = check_power_dir( + power_path, + ranging_path, + perf_path, + scenario_fixed, + ranging_r, + r, + config, + ) + except Exception as e: + power_is_valid = False + if not power_is_valid: + print(f"Power result is invalid for {model} {scenario} scenario in {division} division. Removing...") + #shutil.rmtree(power_path) + #shutil.rmtree(ranging_path) + #shutil.rmtree(os.path.join(perf_path, "spl.txt")) + + is_valid = accuracy_is_valid and perf_is_valid + if not is_valid: #Remove the scenario result + if scenario in ["Offline", "MultiStream"] or division == "open":# they can be inferred + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...") + shutil.rmtree(scenario_path) + elif division in ["closed", "network"] : + model_results_path = os.path.dirname(scenario_path) + if not args.nomove_failed_to_open: + target_results_path = change_first_directory_to_open(model_results_path); + #if only accuracy failed, result is valid for open + if not perf_is_valid: + shutil.rmtree(scenario_path) + print(f"{scenario} scenario result is invalid for {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") + shutil.copytree(model_results_path, target_results_path) + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Moving other scenario results to open...") + else: + shutil.rmtree(model_results_path) + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") + + else:#delete this result, move other scenario results to open + shutil.rmtree(os.path.dirname(scenario_path)) #delete other scenario results too + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") + +def infer_scenario_results(args, config): """Walk result dir and check for singlestream (SS) folders and \ - corresponding offline and multistream (MS) ones. + corresponding offline and multistream (MS) ones. If SS exists and offline and MS are not existing, \ SS folder is copied to MS and offline folders. If SS and offline exists and MS is not existing, MS is inferred from SS. If SS and MS exists and offline is not existing, offline is inferred from MS. """ - for division in list_dir("."): + filter_submitter = args.submitter + noinfer_low_accuracy_results = args.noinfer_low_accuracy_results + + for division in sorted(list_dir(".")): #process closed and network before open # we are looking at ./$division, ie ./closed if division not in ["closed", "open", "network"]: continue + is_closed_or_network = division in ["closed", "network"] for submitter in list_dir(division): # we are looking at ./$division/$submitter, ie ./closed/mlperf_org @@ -117,7 +227,7 @@ def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, confi # process results for directory in ["results", "measurements"] + \ - (["compliance"] if division == "closed" else []): + (["compliance"] if division == "closed" else []): log_path = os.path.join(division, submitter, directory) if not os.path.exists(log_path): @@ -126,10 +236,10 @@ def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, confi for system_desc in list_dir(log_path): system_id_json = os.path.join(division, submitter, "systems", - system_desc + ".json") + system_desc + ".json") if not os.path.exists(system_id_json): log.error("no system_desc for %s/%s/%s", division, submitter, - system_desc) + system_desc) continue with open(system_id_json) as system_info: @@ -147,8 +257,8 @@ def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, confi extra_model_mapping = None if division == "open": model_mapping_path = ( - f"{division}/{submitter}/{config.extra_model_benchmark_map}" - ) + f"{division}/{submitter}/{config.extra_model_benchmark_map}" + ) if os.path.exists(model_mapping_path): with open(model_mapping_path) as fp: extra_model_mapping = json.load(fp) @@ -170,10 +280,14 @@ def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, confi continue - required_scenarios = config.get_required(model) + required_scenarios = config.get_required(mlperf_model) all_scenarios = set( - list(required_scenarios) + - list(config.get_optional(mlperf_model))) + list(required_scenarios) + + list(config.get_optional(mlperf_model))) + + + if directory == "results": + clean_invalid_results(args, log_path, config, system_desc, system_json, model, mlperf_model, division, is_closed_or_network) for scenario in list_dir(log_path, system_desc, model): @@ -187,7 +301,7 @@ def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, confi model, "multistream") if not os.path.exists(multistream_scenario_path) and \ not os.path.exists(offline_scenario_path): - #infer both the scenarios from SS + #infer both the scenarios from SS tobeinferredpaths = [ offline_scenario_path ] if "MultiStream" in all_scenarios: tobeinferredpaths.append(multistream_scenario_path) @@ -195,33 +309,33 @@ def infer_scenario_results(filter_submitter, noinfer_low_accuracy_results, confi for tobeinferredpath in tobeinferredpaths: inferred_scenario = os.path.basename(tobeinferredpath) log.info("Division %s, submitter %s, system %s, " \ - "model %s: \ - inferring %s results from %s", \ - division, submitter, system_desc, model, \ - inferred_scenario, "singlestream") + "model %s: \ + inferring %s results from %s", \ + division, submitter, system_desc, model, \ + inferred_scenario, "singlestream") shutil.copytree(scenario_path, tobeinferredpath) elif not os.path.exists(multistream_scenario_path) and \ "MultiStream" in all_scenarios: - #infer MS from SS + #infer MS from SS for tobeinferredpath in [ multistream_scenario_path ]: log.info("Division %s, submitter %s, system %s, model %s: \ - inferring %s results from %s", division, submitter, \ - system_desc, model, "multistream", "singlestream") + inferring %s results from %s", division, submitter, \ + system_desc, model, "multistream", "singlestream") shutil.copytree(scenario_path, multistream_scenario_path) elif not os.path.exists(offline_scenario_path): '''we have both MS and SS results. Inferring from MS is \ - expected to be better \ - ''' + expected to be better \ + ''' pass elif scenario.lower() == "multistream": offline_scenario_path = os.path.join(log_path, system_desc, \ model, "offline") '''Need to check if MS is indeed a measured result and not infeered.\ - But if MS is indeed inferred from SS, offline scenario will also be \ - inferred already by the inferring code above \ - ''' + But if MS is indeed inferred from SS, offline scenario will also be \ + inferred already by the inferring code above \ + ''' for tobeinferredpath in [offline_scenario_path]: if not os.path.exists(tobeinferredpath): log.info("Division %s, submitter %s, system %s, model %s: \ @@ -266,22 +380,22 @@ def main(): src_dir = args.input if os.path.exists(args.output): - print("output directory already exists") + print(f"output directory {args.output} already exists") sys.exit(1) os.makedirs(args.output) copy_submission_dir(args.input, args.output, args.submitter) src_dir = args.output config = checker.Config( - args.version, - args.extra_model_benchmark_map) + args.version, + args.extra_model_benchmark_map) if not args.nodelete_empty_dirs: delete_empty_dirs(os.path.join(src_dir)) os.chdir(src_dir) - infer_scenario_results(args.submitter, args.noinfer_low_accuracy_results, config) + infer_scenario_results(args, config) return 0 diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 208c90fb6..8ce6b57fc 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -814,7 +814,7 @@ def requires_equal_issue(self, model, division): "llama2-70b-99.9", "mixtral-8x7b" ] - and self.version in ["v4.1"] + and self.version not in ["v4.0", "v4.1"] ) @@ -1010,6 +1010,7 @@ def check_accuracy_dir(config, model, path, verbose): acc_targets.append(acc_target) acc_types.append(acc_type) acc_seen = [False for _ in acc_targets] + with open(os.path.join(path, "accuracy.txt"), "r", encoding="utf-8") as f: for line in f: for i, (pattern, acc_target, acc_type) in enumerate( @@ -1025,11 +1026,12 @@ def check_accuracy_dir(config, model, path, verbose): acc_seen[i] = True elif acc is not None: all_accuracy_valid = False - log.warning( - "%s accuracy not met: expected=%f, found=%s", - path, - acc_target, - acc) + if verbose: + log.warning( + "%s accuracy not met: expected=%f, found=%s", + path, + acc_target, + acc) if acc: result_acc[acc_type] = acc acc = None @@ -1045,11 +1047,12 @@ def check_accuracy_dir(config, model, path, verbose): if acc is not None and acc_upper_limit is not None and float( acc) > acc_limit: acc_limit_check = False - log.warning( - "%s accuracy not met: upper limit=%f, found=%s", - path, - acc_limit, - acc) + if verbose: + log.warning( + "%s accuracy not met: upper limit=%f, found=%s", + path, + acc_limit, + acc) acc = None if all(acc_seen) and hash_val: break @@ -1100,8 +1103,7 @@ def extra_check_llm(mlperf_log, scenario, model): def get_performance_metric( - config, model, path, scenario_fixed, division, system_json, has_power=False -): + config, model, path, scenario_fixed, division): # Assumes new logging format version = config.version @@ -1129,8 +1131,7 @@ def get_performance_metric( def check_performance_dir( - config, model, path, scenario_fixed, division, system_json, has_power=False -): + config, model, path, scenario_fixed, division, system_json): is_valid = False rt = {} @@ -1173,6 +1174,12 @@ def check_performance_dir( mlperf_log["effective_sample_concatenate_permutation"] == "true") if not config.requires_equal_issue(model, division): equal_issue_used_check = True + if not equal_issue_used_check: + log.error( + "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=false", path + ) + is_valid = False + sut_name = mlperf_log["sut_name"] # check if there are any errors in the detailed log @@ -1277,7 +1284,7 @@ def check_performance_dir( # Check if this run uses early stopping. If it does, get the # min_queries from the detail log, otherwise get this value # from the config - if not (uses_early_stopping or config.requires_equal_issue(model, division)): + if not uses_early_stopping: required_min_query_count = config.get_min_query_count(model, scenario) if required_min_query_count and min_query_count < required_min_query_count: log.error( @@ -1328,7 +1335,7 @@ def check_performance_dir( ) is_valid = False - return is_valid, res, inferred, equal_issue_used_check + return is_valid, res, inferred def get_inferred_result(scenario_fixed, scenario, res, @@ -2189,14 +2196,13 @@ def log_result( continue try: - is_valid, r, is_inferred, performance_equal_issue_check = check_performance_dir( + is_valid, r, is_inferred = check_performance_dir( config, mlperf_model, perf_path, scenario_fixed, division, system_json, - has_power, ) if is_inferred: inferred = 1 @@ -2204,13 +2210,6 @@ def log_result( "%s has inferred results, qps=%s", perf_path, r ) - # Check equal issue mode - if not performance_equal_issue_check: - log.error( - "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=false", - perf_path - ) - is_valid, r = False, None except Exception as e: log.error( "%s caused exception in check_performance_dir: %s", @@ -2225,16 +2224,12 @@ def log_result( ranging_path = os.path.join( name, "performance", "ranging" ) - ( - ranging_r - ) = get_performance_metric( + ranging_r = get_performance_metric( config, mlperf_model, ranging_path, scenario_fixed, division, - system_json, - has_power, ) except Exception as e: log.error( @@ -2242,7 +2237,7 @@ def log_result( ranging_path, e, ) - is_valid, r = False, None + is_valid, ranging_r = False, None try: ( From 85ab9f68249f03671ffdf77bc4cbc8dafd24093a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 02:06:34 +0530 Subject: [PATCH 51/61] Cleanup/fix of preprocess_submission code --- tools/submission/preprocess_submission.py | 416 +++++++++++++--------- tools/submission/submission_checker.py | 8 +- 2 files changed, 257 insertions(+), 167 deletions(-) diff --git a/tools/submission/preprocess_submission.py b/tools/submission/preprocess_submission.py index c18d7a31d..98ead7a99 100644 --- a/tools/submission/preprocess_submission.py +++ b/tools/submission/preprocess_submission.py @@ -30,13 +30,17 @@ """ + def get_args(): """Parse commandline.""" parser = argparse.ArgumentParser(description="Infer scenario results", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=HELP_TEXT) - parser.add_argument("--input", required=True, help="orignal submission directory") + parser.add_argument( + "--input", + required=True, + help="orignal submission directory") parser.add_argument("--output", help="new submission directory") - parser.add_argument("--noinfer_low_accuracy_results", + parser.add_argument("--noinfer-low-accuracy-results", help="do not infer low accuracy results if a high accuracy result is present", default=False, action="store_true") parser.add_argument("--nodelete-empty-dirs", @@ -49,16 +53,16 @@ def get_args(): help="do not delete failed results (submission checker will fail)", default=False, action="store_true") parser.add_argument( - "--version", - default="v4.1", - choices=list(checker.MODEL_CONFIG.keys()), - help="mlperf version") + "--version", + default="v4.1", + choices=list(checker.MODEL_CONFIG.keys()), + help="mlperf version") parser.add_argument("--submitter", help="filter to submitter") parser.add_argument( - "--extra-model-benchmark-map", - help="File containing extra custom model mapping.\ + "--extra-model-benchmark-map", + help="File containing extra custom model mapping.\ It is assumed to be inside the folder open/", - default="model_mapping.json") + default="model_mapping.json") args = parser.parse_args() if not args.output: @@ -75,6 +79,7 @@ def list_dir(*path): path = os.path.join(*path) return next(os.walk(path))[1] + def delete_empty_dirs(src): """ Deletes any empty directory in the src tree @@ -82,13 +87,15 @@ def delete_empty_dirs(src): if not os.path.isdir(src): return False - if all ([delete_empty_dirs(os.path.join(src, file)) for file in os.listdir(src)]): + if all([delete_empty_dirs(os.path.join(src, file)) + for file in os.listdir(src)]): log.info("Removing empty dir: (%s)", src) os.rmdir(src) return True return False + def copy_submission_dir(src, dst, filter_submitter): """ Copies the submission tree to output directory for processing @@ -102,6 +109,7 @@ def copy_submission_dir(src, dst, filter_submitter): shutil.copytree(os.path.join(src, division, submitter), os.path.join(dst, division, submitter)) + def change_first_directory_to_open(path): # Split the path into components parts = path.split(os.path.sep) @@ -116,92 +124,153 @@ def change_first_directory_to_open(path): modified_path = os.path.join(*parts) return modified_path -def clean_invalid_results(args, log_path, config, system_desc, system_json, model, mlperf_model, division, is_closed_or_network): + +def change_folder_name_in_path(path, old_folder_name, new_folder_name): + # Split the path into components + path_parts = path.split(os.path.sep) + + # Replace the old folder name with the new one + path_parts = [new_folder_name if part == + old_folder_name else part for part in path_parts] + + # Reassemble the path + new_path = os.path.join(*path_parts) + return new_path + + +def clean_invalid_results(args, log_path, config, system_desc, system_json, + model, mlperf_model, division, system_id_json, is_closed_or_network): # cleanup invalid results - for scenario in list_dir(log_path, system_desc, model): - scenario_fixed = checker.SCENARIO_MAPPING.get( - scenario, scenario) - scenario_path = os.path.join(log_path, system_desc, model, scenario) - acc_path = os.path.join(scenario_path, "accuracy") - try: - accuracy_is_valid, acc = checker.check_accuracy_dir( - config, - mlperf_model, - acc_path, - is_closed_or_network, - ) - except Exception as e: - print(e) - accuracy_is_valid = False - perf_path = os.path.join(scenario_path, "performance", "run_1") - try: - perf_is_valid, r, is_inferred = checker.check_performance_dir( - config, - mlperf_model, - perf_path, - scenario_fixed, - division, - system_json, - ) - except Exception as e: - print(e) - perf_is_valid = False - if perf_is_valid: - power_path = os.path.join(scenario_path, "performance", "power") - has_power = os.path.exists(power_path) - if has_power: - ranging_path = os.path.join(scenario_path, "performance", "ranging") - try: - ranging_r = get_performance_metric( - config, - mlperf_model, - ranging_path, - scenario_fixed, - division, - ) - ( - power_is_valid, - power_metric, - power_efficiency, - ) = check_power_dir( - power_path, - ranging_path, - perf_path, - scenario_fixed, - ranging_r, - r, - config, - ) - except Exception as e: - power_is_valid = False - if not power_is_valid: - print(f"Power result is invalid for {model} {scenario} scenario in {division} division. Removing...") - #shutil.rmtree(power_path) - #shutil.rmtree(ranging_path) - #shutil.rmtree(os.path.join(perf_path, "spl.txt")) - - is_valid = accuracy_is_valid and perf_is_valid - if not is_valid: #Remove the scenario result - if scenario in ["Offline", "MultiStream"] or division == "open":# they can be inferred - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...") - shutil.rmtree(scenario_path) - elif division in ["closed", "network"] : - model_results_path = os.path.dirname(scenario_path) - if not args.nomove_failed_to_open: - target_results_path = change_first_directory_to_open(model_results_path); - #if only accuracy failed, result is valid for open - if not perf_is_valid: - shutil.rmtree(scenario_path) - print(f"{scenario} scenario result is invalid for {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") - shutil.copytree(model_results_path, target_results_path) - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Moving other scenario results to open...") - else: - shutil.rmtree(model_results_path) - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") - - else:#delete this result, move other scenario results to open - shutil.rmtree(os.path.dirname(scenario_path)) #delete other scenario results too - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") + for scenario in list_dir(log_path, system_desc, model): + scenario_fixed = checker.SCENARIO_MAPPING.get( + scenario, scenario) + scenario_path = os.path.join(log_path, system_desc, model, scenario) + acc_path = os.path.join(scenario_path, "accuracy") + try: + accuracy_is_valid, acc = checker.check_accuracy_dir( + config, + mlperf_model, + acc_path, + is_closed_or_network, + ) + except Exception as e: + print(e) + accuracy_is_valid = False + perf_path = os.path.join(scenario_path, "performance", "run_1") + try: + perf_is_valid, r, is_inferred = checker.check_performance_dir( + config, + mlperf_model, + perf_path, + scenario_fixed, + division, + system_json, + ) + except Exception as e: + print(e) + perf_is_valid = False + if perf_is_valid: + power_path = os.path.join(scenario_path, "performance", "power") + has_power = os.path.exists(power_path) + if has_power: + ranging_path = os.path.join( + scenario_path, "performance", "ranging") + try: + ranging_r = get_performance_metric( + config, + mlperf_model, + ranging_path, + scenario_fixed, + division, + ) + ( + power_is_valid, + power_metric, + power_efficiency, + ) = check_power_dir( + power_path, + ranging_path, + perf_path, + scenario_fixed, + ranging_r, + r, + config, + ) + except Exception as e: + power_is_valid = False + if not power_is_valid: + print( + f"Power result is invalid for {model} {scenario} scenario in {division} division. Removing...") + shutil.rmtree(power_path) + shutil.rmtree(ranging_path) + shutil.rmtree(os.path.join(perf_path, "spl.txt")) + + is_valid = accuracy_is_valid and perf_is_valid + if not is_valid: # Remove the scenario result + scenario_measurements_path = change_folder_name_in_path( + scenario_path, "results", "measurements") + if scenario in [ + "Offline", "MultiStream"] or division == "open": # they can be inferred + scenario_compliance_path = change_folder_name_in_path( + scenario_path, "results", "compliance") + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...") + shutil.rmtree(scenario_path) + shutil.rmtree(scenario_measurements_path) + shutil.rmtree(scenario_compliance_path) + elif division in ["closed", "network"]: + model_results_path = os.path.dirname(scenario_path) + model_measurements_path = change_folder_name_in_path( + model_results_path, "results", "measurements") + model_compliance_path = change_folder_name_in_path( + model_results_path, "results", "compliance") + model_code_path = os.path.join( + change_folder_name_in_path( + log_path, "results", "code"), model) + if not args.nomove_failed_to_open: + target_code_path = change_first_directory_to_open( + model_code_path) + target_results_path = change_first_directory_to_open( + model_results_path) + target_measurements_path = change_first_directory_to_open( + model_measurements_path) + target_system_json = change_first_directory_to_open( + system_id_json) + # if only accuracy failed, result is valid for open + if not perf_is_valid: + shutil.rmtree(scenario_path) + print(f"{scenario} scenario result is invalid for {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") + if not os.path.exists(target_results_path): + shutil.copytree( + model_results_path, target_results_path) + if not os.path.exists(target_measurements_path): + shutil.copytree( + model_measurements_path, + target_measurements_path) + if not os.path.exists(target_code_path): + shutil.copytree(model_code_path, target_code_path) + if not os.path.exists(target_system_json): + dst_dir = os.path.dirname(target_system_json) + if not os.path.exists(dst_dir): + os.makedirs(dst_dir, exist_ok=True) + import copy + target_system_json_contents = copy.deepcopy( + system_json) + target_system_json_contents['division'] = 'open' + with open(target_system_json, 'w') as f: + json.dump(target_system_json_contents, f, indent=2) + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Moving other scenario results to open...") + else: + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") + shutil.rmtree(model_results_path) + shutil.rmtree(model_measurements_path) + shutil.rmtree(model_compliance_path) + else: # delete this result + # delete other scenario results too + shutil.rmtree(scenario_path) + # delete other scenario results too + shutil.rmtree(scenario_measurements_path) + print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") + def infer_scenario_results(args, config): """Walk result dir and check for singlestream (SS) folders and \ @@ -214,7 +283,8 @@ def infer_scenario_results(args, config): filter_submitter = args.submitter noinfer_low_accuracy_results = args.noinfer_low_accuracy_results - for division in sorted(list_dir(".")): #process closed and network before open + for division in sorted( + list_dir(".")): # process closed and network before open # we are looking at ./$division, ie ./closed if division not in ["closed", "open", "network"]: continue @@ -245,84 +315,102 @@ def infer_scenario_results(args, config): with open(system_id_json) as system_info: system_json = json.load(system_info) system_type = system_json.get("system_type") - valid_system_types = ["datacenter", "edge", \ - "datacenter,edge", "edge,datacenter"] + valid_system_types = ["datacenter", "edge", + "datacenter,edge", "edge,datacenter"] if system_type not in valid_system_types: - log.error("Division %s, submitter %s, "\ - "system %s has invalid system type (%s)", \ - division, submitter, system_id_json, system_type) + log.error("Division %s, submitter %s, " + "system %s has invalid system type (%s)", + division, submitter, system_id_json, system_type) config.set_type(system_type) for model in list_dir(log_path, system_desc): extra_model_mapping = None if division == "open": model_mapping_path = ( - f"{division}/{submitter}/{config.extra_model_benchmark_map}" - ) + f"{division}/{submitter}/{config.extra_model_benchmark_map}" + ) if os.path.exists(model_mapping_path): with open(model_mapping_path) as fp: extra_model_mapping = json.load(fp) - mlperf_model = config.get_mlperf_model(model, extra_model_mapping) + mlperf_model = config.get_mlperf_model( + model, extra_model_mapping) if not mlperf_model: - log.error("Division %s, submitter %s, system %s has "\ - "invalid model (%s)", division, submitter, \ - system_id_json, model) + log.error("Division %s, submitter %s, system %s has " + "invalid model (%s)", division, submitter, + system_id_json, model) continue if mlperf_model not in config.required: - log.error("Division %s, submitter %s, system %s has invalid "\ - "MLPerf model (%s) corresponding to given model (%s). "\ - "Valid ones for MLPerf inference version (%s) in (%s) "\ - "category are [%s]", division, submitter, system_id_json,\ - mlperf_model, model, config.version, system_type, \ - config.required.keys()) + log.error("Division %s, submitter %s, system %s has invalid " + "MLPerf model (%s) corresponding to given model (%s). " + "Valid ones for MLPerf inference version (%s) in (%s) " + "category are [%s]", division, submitter, system_id_json, + mlperf_model, model, config.version, system_type, + config.required.keys()) continue - required_scenarios = config.get_required(mlperf_model) all_scenarios = set( - list(required_scenarios) + - list(config.get_optional(mlperf_model))) - + list(required_scenarios) + + list(config.get_optional(mlperf_model))) if directory == "results": - clean_invalid_results(args, log_path, config, system_desc, system_json, model, mlperf_model, division, is_closed_or_network) + clean_invalid_results( + args, + log_path, + config, + system_desc, + system_json, + model, + mlperf_model, + division, + system_id_json, + is_closed_or_network) + if not os.path.exists(os.path.join( + log_path, system_desc, model)): + continue for scenario in list_dir(log_path, system_desc, model): - scenario_path = os.path.join(log_path, system_desc, model, scenario) + scenario_path = os.path.join( + log_path, system_desc, model, scenario) if scenario.lower() == "singlestream": tobeinferredpaths = [] - offline_scenario_path = os.path.join(log_path, system_desc, \ - model, "offline") - multistream_scenario_path = os.path.join(log_path, system_desc, \ - model, "multistream") + offline_scenario_path = os.path.join(log_path, system_desc, + model, "offline") + multistream_scenario_path = os.path.join(log_path, system_desc, + model, "multistream") if not os.path.exists(multistream_scenario_path) and \ not os.path.exists(offline_scenario_path): - #infer both the scenarios from SS - tobeinferredpaths = [ offline_scenario_path ] + # infer both the scenarios from SS + tobeinferredpaths = [offline_scenario_path] if "MultiStream" in all_scenarios: - tobeinferredpaths.append(multistream_scenario_path) + tobeinferredpaths.append( + multistream_scenario_path) for tobeinferredpath in tobeinferredpaths: - inferred_scenario = os.path.basename(tobeinferredpath) - log.info("Division %s, submitter %s, system %s, " \ - "model %s: \ - inferring %s results from %s", \ - division, submitter, system_desc, model, \ - inferred_scenario, "singlestream") - shutil.copytree(scenario_path, tobeinferredpath) + inferred_scenario = os.path.basename( + tobeinferredpath) + log.info("Division %s, submitter %s, system %s, " + "model %s: \ + inferring %s results from %s", + division, submitter, system_desc, model, + inferred_scenario, "singlestream") + shutil.copytree( + scenario_path, tobeinferredpath) elif not os.path.exists(multistream_scenario_path) and \ "MultiStream" in all_scenarios: - #infer MS from SS - for tobeinferredpath in [ multistream_scenario_path ]: + # infer MS from SS + for tobeinferredpath in [ + multistream_scenario_path]: log.info("Division %s, submitter %s, system %s, model %s: \ - inferring %s results from %s", division, submitter, \ - system_desc, model, "multistream", "singlestream") - shutil.copytree(scenario_path, multistream_scenario_path) + inferring %s results from %s", division, submitter, + system_desc, model, "multistream", "singlestream") + shutil.copytree( + scenario_path, multistream_scenario_path) elif not os.path.exists(offline_scenario_path): '''we have both MS and SS results. Inferring from MS is \ expected to be better \ @@ -330,45 +418,47 @@ def infer_scenario_results(args, config): pass elif scenario.lower() == "multistream": - offline_scenario_path = os.path.join(log_path, system_desc, \ - model, "offline") + offline_scenario_path = os.path.join(log_path, system_desc, + model, "offline") '''Need to check if MS is indeed a measured result and not infeered.\ But if MS is indeed inferred from SS, offline scenario will also be \ inferred already by the inferring code above \ ''' - for tobeinferredpath in [offline_scenario_path]: + for tobeinferredpath in [ + offline_scenario_path]: if not os.path.exists(tobeinferredpath): log.info("Division %s, submitter %s, system %s, model %s: \ - inferring %s results from %s", division, submitter,\ - system_desc, model, "offline", "multistream") - shutil.copytree(scenario_path, tobeinferredpath) + inferring %s results from %s", division, submitter, + system_desc, model, "offline", "multistream") + shutil.copytree( + scenario_path, tobeinferredpath) if not noinfer_low_accuracy_results: for system_desc in list_dir(log_path): for model in list_dir(log_path, system_desc): if model.endswith("-99.9"): - low_accuracy_model =model[:-2] + low_accuracy_model = model[:-2] if low_accuracy_model not in config.required: continue - high_accuracy_model_path = os.path.join(log_path, \ - system_desc, model) - low_accuracy_model_path = os.path.join(log_path, system_desc, \ - low_accuracy_model) + high_accuracy_model_path = os.path.join(log_path, + system_desc, model) + low_accuracy_model_path = os.path.join(log_path, system_desc, + low_accuracy_model) if not os.path.exists(low_accuracy_model_path): log.info("Division %s, submitter %s, system %s: \ - copying %s results to %s", division, submitter, \ - system_desc, model, low_accuracy_model) - - shutil.copytree(high_accuracy_model_path, \ - low_accuracy_model_path) - high_accuracy_model_code_path = os.path.join(log_path, "..", \ - "code", model) - low_accuracy_model_code_path = os.path.join(log_path, "..", \ - "code", low_accuracy_model) - if not os.path.exists(low_accuracy_model_code_path): - shutil.copytree(high_accuracy_model_code_path, \ - low_accuracy_model_code_path) + copying %s results to %s", division, submitter, + system_desc, model, low_accuracy_model) + shutil.copytree(high_accuracy_model_path, + low_accuracy_model_path) + high_accuracy_model_code_path = os.path.join(log_path, "..", + "code", model) + low_accuracy_model_code_path = os.path.join(log_path, "..", + "code", low_accuracy_model) + if not os.path.exists( + low_accuracy_model_code_path): + shutil.copytree(high_accuracy_model_code_path, + low_accuracy_model_code_path) def main(): @@ -387,8 +477,8 @@ def main(): src_dir = args.output config = checker.Config( - args.version, - args.extra_model_benchmark_map) + args.version, + args.extra_model_benchmark_map) if not args.nodelete_empty_dirs: delete_empty_dirs(os.path.join(src_dir)) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 8ce6b57fc..194c0dfb6 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -1103,7 +1103,7 @@ def extra_check_llm(mlperf_log, scenario, model): def get_performance_metric( - config, model, path, scenario_fixed, division): + config, model, path, scenario_fixed, division): # Assumes new logging format version = config.version @@ -1131,7 +1131,7 @@ def get_performance_metric( def check_performance_dir( - config, model, path, scenario_fixed, division, system_json): + config, model, path, scenario_fixed, division, system_json): is_valid = False rt = {} @@ -1177,7 +1177,7 @@ def check_performance_dir( if not equal_issue_used_check: log.error( "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=false", path - ) + ) is_valid = False sut_name = mlperf_log["sut_name"] @@ -2196,7 +2196,7 @@ def log_result( continue try: - is_valid, r, is_inferred = check_performance_dir( + is_valid, r, is_inferred = check_performance_dir( config, mlperf_model, perf_path, From 002a83603abf597a9702303ec13056599fbc145c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 02:44:24 +0530 Subject: [PATCH 52/61] Fix params for get_performance_metric (submission checker) --- tools/submission/preprocess_submission.py | 1 - tools/submission/submission_checker.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/submission/preprocess_submission.py b/tools/submission/preprocess_submission.py index 98ead7a99..42553ca10 100644 --- a/tools/submission/preprocess_submission.py +++ b/tools/submission/preprocess_submission.py @@ -181,7 +181,6 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, mlperf_model, ranging_path, scenario_fixed, - division, ) ( power_is_valid, diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 194c0dfb6..5050eabc2 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -1103,7 +1103,7 @@ def extra_check_llm(mlperf_log, scenario, model): def get_performance_metric( - config, model, path, scenario_fixed, division): + config, model, path, scenario_fixed): # Assumes new logging format version = config.version @@ -2229,7 +2229,6 @@ def log_result( mlperf_model, ranging_path, scenario_fixed, - division, ) except Exception as e: log.error( From b0142f603cad0217b8c5d62e0cf5800b54c2dd42 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 03:21:51 +0530 Subject: [PATCH 53/61] Fix params for check_performance_dir (submission checker) --- tools/submission/submission_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 5050eabc2..89fef43d1 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2820,7 +2820,7 @@ def check_compliance_dir( compliance_perf_dir = os.path.join( compliance_dir, test, "performance", "run_1" ) - compliance_perf_valid, r, is_inferred, _ = check_performance_dir( + compliance_perf_valid, r, is_inferred = check_performance_dir( config, model, compliance_perf_dir, scenario, division, system_json ) if is_inferred: From 6e8f527ab2c98f6a6b4bfeec1c60133e995c1051 Mon Sep 17 00:00:00 2001 From: arjunsuresh <4791823+arjunsuresh@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:52:35 +0000 Subject: [PATCH 54/61] [Automated Commit] Format Codebase --- tools/submission/submission_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 89fef43d1..70fa86d67 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2820,7 +2820,7 @@ def check_compliance_dir( compliance_perf_dir = os.path.join( compliance_dir, test, "performance", "run_1" ) - compliance_perf_valid, r, is_inferred = check_performance_dir( + compliance_perf_valid, r, is_inferred = check_performance_dir( config, model, compliance_perf_dir, scenario, division, system_json ) if is_inferred: From 90029be3cbb376d37d35ba2144f7a930b90fce1d Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Tue, 29 Oct 2024 09:32:15 +0000 Subject: [PATCH 55/61] Increment version to 4.1.23 --- loadgen/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/VERSION.txt b/loadgen/VERSION.txt index 566261e13..59fe035d9 100644 --- a/loadgen/VERSION.txt +++ b/loadgen/VERSION.txt @@ -1 +1 @@ -4.1.22 +4.1.23 From beb5c9f5bcdd3da242f4f934bde9cce19e028661 Mon Sep 17 00:00:00 2001 From: arjunsuresh <4791823+arjunsuresh@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:32:41 +0000 Subject: [PATCH 56/61] [Automated Commit] Format Codebase --- loadgen/setup.py | 1 - loadgen/version_generator.py | 1 - tools/submission/preprocess_submission.py | 3 --- tools/submission/submission_checker.py | 4 +--- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/loadgen/setup.py b/loadgen/setup.py index 1768a629a..6254eea17 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -134,4 +134,3 @@ include_package_data=True, long_description=mlperf_long_description, long_description_content_type='text/markdown') - diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index 6bd625684..2e7524330 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -32,7 +32,6 @@ def make_raw_string(str): return 'R"' + delimeter + "(" + str + ")" + delimeter + '"' - def func_def(name, string): return ( "const std::string& Loadgen" diff --git a/tools/submission/preprocess_submission.py b/tools/submission/preprocess_submission.py index f5af60282..a75065d6b 100644 --- a/tools/submission/preprocess_submission.py +++ b/tools/submission/preprocess_submission.py @@ -466,7 +466,6 @@ def infer_scenario_results(args, config): low_accuracy_model_code_path) - def main(): """ Tool to infer scenario results and cleanup submission tree @@ -486,7 +485,6 @@ def main(): args.version, args.extra_model_benchmark_map) - if not args.nodelete_empty_dirs: delete_empty_dirs(os.path.join(src_dir)) @@ -494,7 +492,6 @@ def main(): infer_scenario_results(args, config) - return 0 diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index cbfac811e..b509b7ab1 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -1456,7 +1456,6 @@ def check_performance_dir( return is_valid, res, inferred - def get_inferred_result( scenario_fixed, scenario, res, mlperf_log, config, log_error=False ): @@ -1572,7 +1571,7 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res): samples_per_query = 8 if (scenario_fixed in ["MultiStream"] - ) and scenario in ["SingleStream"]: + ) and scenario in ["SingleStream"]: power_metric = ( avg_power * power_duration * samples_per_query * 1000 / num_queries ) @@ -1831,7 +1830,6 @@ def log_result( unit = special_unit_dict.get(model_name, unit_dict)[scenario_fixed] power_unit = power_unit_dict[scenario_fixed] - if (power_metric <= 0) or ( not get_boolean(system_json.get("system_power_only")) ): From 72b5954bb6ff4effc4cad8e5c9da7262f79a7800 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 18:33:45 +0530 Subject: [PATCH 57/61] Added compliance checks to preprocess submission, use logging for debug output --- tools/submission/preprocess_submission.py | 51 +++++++++++++++++------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/tools/submission/preprocess_submission.py b/tools/submission/preprocess_submission.py index a75065d6b..1e26b81ca 100644 --- a/tools/submission/preprocess_submission.py +++ b/tools/submission/preprocess_submission.py @@ -149,6 +149,9 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, scenario_fixed = checker.SCENARIO_MAPPING.get( scenario, scenario) scenario_path = os.path.join(log_path, system_desc, model, scenario) + if not os.path.exists( + scenario_path): # can happen since scenario results may be moved to open division on failure + continue acc_path = os.path.join(scenario_path, "accuracy") try: accuracy_is_valid, acc = checker.check_accuracy_dir( @@ -158,7 +161,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, is_closed_or_network, ) except Exception as e: - print(e) + log.warning(e) accuracy_is_valid = False perf_path = os.path.join(scenario_path, "performance", "run_1") try: @@ -171,7 +174,7 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, system_json, ) except Exception as e: - print(e) + log.warning(e) perf_is_valid = False if perf_is_valid: power_path = os.path.join(scenario_path, "performance", "power") @@ -202,21 +205,37 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, except Exception as e: power_is_valid = False if not power_is_valid: - print( - f"Power result is invalid for {model} {scenario} scenario in {division} division. Removing...") + log.warning( + f"Power result is invalid for {system_desc}: {model} {scenario} scenario in {division} division. Removing...") shutil.rmtree(power_path) shutil.rmtree(ranging_path) shutil.rmtree(os.path.join(perf_path, "spl.txt")) - is_valid = accuracy_is_valid and perf_is_valid + compliance_is_valid = True + if is_closed_or_network: + compliance_dir = change_folder_name_in_path( + scenario_path, "results", "compliance") + if not checker.check_compliance_dir( + compliance_dir, + mlperf_model, + scenario_fixed, + config, + division, + system_json, + os.path.dirname(scenario_path) + ): + compliance_is_valid = False + + is_valid = accuracy_is_valid and perf_is_valid and compliance_is_valid if not is_valid: # Remove the scenario result scenario_measurements_path = change_folder_name_in_path( scenario_path, "results", "measurements") if scenario in [ - "Offline", "MultiStream"] or division == "open": # they can be inferred + "Offline", "MultiStream"] and (not accuracy_is_valid or not perf_is_valid) or division == "open": # they can be inferred scenario_compliance_path = change_folder_name_in_path( scenario_path, "results", "compliance") - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...") + log.warning( + f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing...") shutil.rmtree(scenario_path) shutil.rmtree(scenario_measurements_path) shutil.rmtree(scenario_compliance_path) @@ -238,10 +257,12 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, model_measurements_path) target_system_json = change_first_directory_to_open( system_id_json) - # if only accuracy failed, result is valid for open + # if only accuracy or compliance failed, result is valid + # for open if not perf_is_valid: shutil.rmtree(scenario_path) - print(f"{scenario} scenario result is invalid for {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") + log.warning( + f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} and open divisions. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") if not os.path.exists(target_results_path): shutil.copytree( model_results_path, target_results_path) @@ -261,9 +282,12 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, target_system_json_contents['division'] = 'open' with open(target_system_json, 'w') as f: json.dump(target_system_json_contents, f, indent=2) - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Moving other scenario results to open...") + if perf_is_valid: + log.warning(f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Compliance: {compliance_is_valid}. Moving {model} results to open...") + else: + log.warning(f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Compliance: {compliance_is_valid}. Moving other scenario results of {model} to open...") else: - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") + log.warning(f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing all dependent scenario results...") shutil.rmtree(model_results_path) shutil.rmtree(model_measurements_path) shutil.rmtree(model_compliance_path) @@ -272,7 +296,8 @@ def clean_invalid_results(args, log_path, config, system_desc, system_json, shutil.rmtree(scenario_path) # delete other scenario results too shutil.rmtree(scenario_measurements_path) - print(f"{scenario} scenario result is invalid for {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") + log.warning( + f"{scenario} scenario result is invalid for {system_desc}: {model} in {division} division. Accuracy: {accuracy_is_valid}, Performance: {perf_is_valid}. Removing it...") def infer_scenario_results(args, config): @@ -475,7 +500,7 @@ def main(): src_dir = args.input if os.path.exists(args.output): - print(f"output directory {args.output} already exists") + log.error(f"output directory {args.output} already exists") sys.exit(1) os.makedirs(args.output) copy_submission_dir(args.input, args.output, args.submitter) From f5bf1bf1742d2fe5c2415e3722d0a1ed92f4facd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 13:35:01 +0000 Subject: [PATCH 58/61] Update test-submission-checker.yml to check inference_results_v4.1 from v4.0 --- .github/workflows/test-submission-checker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-checker.yml b/.github/workflows/test-submission-checker.yml index da4192bdd..29a06248e 100644 --- a/.github/workflows/test-submission-checker.yml +++ b/.github/workflows/test-submission-checker.yml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install cm4mlops - git clone https://github.com/mlcommons/inference_results_v4.0 --depth 1 + git clone https://github.com/mlcommons/inference_results_v4.1 --depth 1 - name: Test MLPerf inference submission checker run: | - cm run script --tags=run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/inference_results_v4.0 --src_version=v4.0 --quiet + cm run script --tags=run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/inference_results_v4.1 --src_version=v4.1 --extra_args=" --skip-extra-accuracy-files-check" --quiet From 2d537edd156f3117119866cbe502b55d49382cb7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 19:54:21 +0530 Subject: [PATCH 59/61] Added skip-accuarcy-files-check option to submission checker --- tools/submission/submission_checker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index b509b7ab1..e91ea574e 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -1571,7 +1571,7 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res): samples_per_query = 8 if (scenario_fixed in ["MultiStream"] - ) and scenario in ["SingleStream"]: + ) and scenario in ["SingleStream"]: power_metric = ( avg_power * power_duration * samples_per_query * 1000 / num_queries ) @@ -1697,6 +1697,7 @@ def check_results_dir( skip_empty_files_check=False, skip_check_power_measure_files=False, skip_extra_files_in_root_check=False, + skip_extra_accuracy_files_check=False, scenarios_to_skip=[], ): """ @@ -2260,9 +2261,8 @@ def log_result( ) if mlperf_model in REQUIRED_ACC_BENCHMARK: if ( - config.version - in REQUIRED_ACC_BENCHMARK[mlperf_model] - ): + config.version + in REQUIRED_ACC_BENCHMARK[mlperf_model] and not skip_extra_accuracy_files_check:): extra_files_pass, missing_files = check_extra_files( acc_path, REQUIRED_ACC_BENCHMARK[mlperf_model][ @@ -2998,6 +2998,7 @@ def main(): args.skip_empty_files_check, args.skip_check_power_measure_files, args.skip_extra_files_in_root_check, + args.skip_extra_accuracy_files_check, scenarios_to_skip, ) From 20c8d17f999700c6c674a447beda9984aa244364 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 19:56:31 +0530 Subject: [PATCH 60/61] Added skip-accuarcy-files-check option to submission checker --- tools/submission/submission_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index e91ea574e..4bd93279b 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2262,7 +2262,7 @@ def log_result( if mlperf_model in REQUIRED_ACC_BENCHMARK: if ( config.version - in REQUIRED_ACC_BENCHMARK[mlperf_model] and not skip_extra_accuracy_files_check:): + in REQUIRED_ACC_BENCHMARK[mlperf_model] and not skip_extra_accuracy_files_check): extra_files_pass, missing_files = check_extra_files( acc_path, REQUIRED_ACC_BENCHMARK[mlperf_model][ From ee535b43e77473fe2129c721011f48ef2444c83d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 20:02:33 +0530 Subject: [PATCH 61/61] Added skip-accuarcy-files-check option to submission checker --- tools/submission/submission_checker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 4bd93279b..0215fd500 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -975,6 +975,11 @@ def get_args(): action="store_true", help="skips the check of extra files inside the root submission dir", ) + parser.add_argument( + "--skip-extra-accuracy-files-check", + action="store_true", + help="skips the check of extra accuracy files like the images folder of SDXL", + ) parser.add_argument( "--scenarios-to-skip", help="Delimited list input of scenarios to skip. i.e. if you only have Offline results, pass in 'Server'",