Sync <- Mlperf inference

.github/workflows/test-cm-based-ubmission-generation.yml → .github/workflows/test-cm-based-submission-generation.yml

@@ -6,11 +6,11 @@ on:
   pull_request:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
-      - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'
+      - '.github/workflows/test-cm-based-submission-generation.yml'
       # - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
       # - '!**.md'
 jobs:
-  Case-3:
+  submission_generation:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -19,6 +19,7 @@ jobs:
         python-version: [ "3.12" ]
         division: ["closed", "open"]
         category: ["datacenter", "edge"]
+        case: ["case-3", "case-7"]
         exclude:
           - os: macos-latest
           - os: windows-latest
@@ -36,7 +37,18 @@ jobs:
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Pull repo where test cases are uploaded
       run: |
-        cm pull repo anandhu-eng@inference --checkout=submission-generation-tests
-    - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }}
+        git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests
+    - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.category }} ${{ matrix.division }}
       run: |
-        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        if [ "${{ matrix.case }}" == "case-3" ]; then
+          #results_dir="submission_generation_tests/case-3/"
+          description="Submission generation (model_mapping.json not present but model name matches with official one)"
+        elif [ "${{ matrix.case }}" == "case-7" ]; then
+          #results_dir="submission_generation_tests/case-7/"
+          description="Submission generation (sut_info.json incomplete, SUT folder name in required format)"
+        fi
+        # Dynamically set the log group to simulate a dynamic step name
+        echo "::group::$description"
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        echo "::endgroup::"
+

automation/script/module.py

@@ -3980,7 +3980,10 @@ def parse_version(self, i):
 
         string = r['string']
 
-        version = r['match'].group(group_number)
+        if r['match'].lastindex and r['match'].lastindex >= group_number:
+            version = r['match'].group(group_number)
+        else:
+            return {'return':1, 'error': 'Invalid version detection group number. Version was not detected. Last index of match = {}. Given group number = {}'.format(r['match'].lastindex, group_number)}
 
         which_env[env_key] = version
         which_env['CM_DETECTED_VERSION'] = version # to be recorded in the cache meta

script/get-generic-sys-util/_cm.json

@@ -457,8 +457,12 @@
     },
     "md5sha1sum": {
       "env": {
-        "CM_SYS_UTIL_NAME": "md5sha1sum"
+        "CM_SYS_UTIL_NAME": "md5sha1sum",
+        "CM_SYS_UTIL_VERSION_CMD": "md5sum --version | grep sha1sum",
+        "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b",
+        "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0
       },
+      "new_env_keys": ["CM_MD5SHA1SUM_VERSION"],
       "state": {
         "md5sha1sum": {
           "apt": "",

script/get-generic-sys-util/customize.py

@@ -17,7 +17,8 @@ def preprocess(i):
     if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect":
         if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '':
             r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'})
-            if r['return'] > 0: #detection failed, do install via prehook_deps
+            if r['return'] != 0: #detection failed, do install via prehook_deps
+                print("detection failed, going for installation")
                 env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes"
                 return {'return': 0}
             else: #detection is successful, no need to install
@@ -55,6 +56,7 @@ def preprocess(i):
     if not package_name:
         if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in [ "1", "true", "yes" ]:
             print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...")
+            env['CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED'] = 'yes'
             return {'return': 0}
         else:
             return {'return': 1, 'error': f'No package name specified for {pm} and util name {util}'}
@@ -104,13 +106,14 @@ def detect_version(i):
     env = i['env']
     version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION"
     version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '')
+    group_number = env.get('CM_TMP_VERSION_DETECT_GROUP_NUMBER', 1)
 
     if version_check_re == '' or not os.path.exists("tmp-ver.out"):
         version = "undetected"
 
     else:
         r = i['automation'].parse_version({'match_text': version_check_re,
-                                       'group_number': 1,
+                                       'group_number': group_number,
                                        'env_key': version_env_key,
                                        'which_env': env})
         if r['return'] >0: return r
@@ -125,11 +128,11 @@ def postprocess(i):
 
     version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION"
 
-    if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') :
+    if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]:
         automation = i['automation']
         r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'})
-        if r['return'] > 0:
-            return r
+        if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]:
+            return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'}
 
         r = detect_version(i)
 

script/get-generic-sys-util/detect.sh

@@ -1,7 +1,14 @@
 #!/bin/bash
 
 if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then
-  ${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out
+  if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then
+    # Redirect both stdout and stderr to tmp-ver.out
+    cmd="${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out 2>&1"
+  else
+    cmd="${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out"
+  fi
+  echo $cmd
+  eval $cmd
+  test $? -eq 0 || exit $?
 fi
 
-test $? -eq 0 || exit $?