Add threshold on number of files / partition in SPMI collection (#44180)

* Add check for files count

* Fix the OS check

* decrese file limit to 1500:

* misc fix

* Do not upload to azure if mch files are zero size

eng/pipelines/coreclr/templates/run-superpmi-job.yml

@@ -105,7 +105,7 @@ jobs:
     - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml
       parameters:
         HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
-        HelixType: 'test/superpmi/$(Kind)/$(_Framework)/$(Architecture)'
+        HelixType: 'test/superpmi/$(CollectionName)/$(_Framework)/$(Architecture)'
         HelixAccessToken: $(HelixApiAccessToken)
         HelixTargetQueues: $(Queue)
         HelixPreCommands: $(HelixPreCommand)

src/coreclr/scripts/superpmi-setup.py

@@ -89,7 +89,7 @@
     "superpmi-shim-counter.dll",
     "superpmi-shim-simple.dll",
 ]
-
+MAX_FILES_COUNT = 1500
 
 def setup_args(args):
     """ Setup the args for SuperPMI to use.
@@ -206,7 +206,7 @@ def first_fit(sorted_by_size, max_size):
         if file_size < max_size:
             for p_index in partitions:
                 total_in_curr_par = sum(n for _, n in partitions[p_index])
-                if (total_in_curr_par + file_size) < max_size:
+                if (((total_in_curr_par + file_size) < max_size) and (len(partitions[p_index]) < MAX_FILES_COUNT)):
                     partitions[p_index].append(curr_file)
                     found_bucket = True
                     break
@@ -217,7 +217,7 @@ def first_fit(sorted_by_size, max_size):
     total_size = 0
     for p_index in partitions:
         partition_size = sum(n for _, n in partitions[p_index])
-        print("Partition {0}: {1} bytes.".format(p_index, partition_size))
+        print("Partition {0}: {1} files with {2} bytes.".format(p_index, len(partitions[p_index]), partition_size))
         total_size += partition_size
     print("Total {0} partitions with {1} bytes.".format(str(len(partitions)), total_size))
 

src/coreclr/scripts/superpmi.proj

@@ -1,9 +1,9 @@
 <Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
 
-  <PropertyGroup Condition="'$(AGENT_OS)' == 'windows'">
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
     <FileSeparatorChar>\</FileSeparatorChar>
   </PropertyGroup>
-  <PropertyGroup Condition="'$(AGENT_OS)' != 'windows'">
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
     <FileSeparatorChar>/</FileSeparatorChar>
   </PropertyGroup>
 
@@ -16,7 +16,7 @@
          PmiAssembliesPayload - Path that will be sent to helix machine to run collection on
          PmiAssembliesDirectory - Path on helix machine itself where superpmi.py will discover the sent assemblies.
     -->
-  <PropertyGroup Condition="'$(AGENT_OS)' == 'windows'">
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
     <Python>%HELIX_PYTHONPATH%</Python>
     <PmiAssembliesPayload>$(WorkItemDirectory)\pmiAssembliesDirectory</PmiAssembliesPayload>
     <PmiAssembliesDirectory>%HELIX_WORKITEM_PAYLOAD%\binaries</PmiAssembliesDirectory>
@@ -26,7 +26,7 @@
     <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)\artifacts\helixresults</HelixResultsDestinationDir>
     <WorkItemCommand>$(SuperPMIDirectory)\superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)\pmi.dll </WorkItemCommand>
   </PropertyGroup>
-  <PropertyGroup Condition="'$(AGENT_OS)' != 'windows'">
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
     <Python>$HELIX_PYTHONPATH</Python>
     <PmiAssembliesPayload>$(WorkItemDirectory)/pmiAssembliesDirectory</PmiAssembliesPayload>
     <PmiAssembliesDirectory>$HELIX_WORKITEM_PAYLOAD/binaries</PmiAssembliesDirectory>

src/coreclr/scripts/superpmi.py

@@ -2175,19 +2175,20 @@ def upload_blob(file, blob_name):
 
     files = []
     for item in coreclr_args.mch_files:
-        files += get_files_from_path(item, match_func=lambda path: any(path.endswith(extension) for extension in [".mch", ".mct"]))
+        files += get_files_from_path(item, match_func=lambda path: any(path.endswith(extension) for extension in [".mch"]))
 
+    files_to_upload = []
     # Special case: walk the files list and for every ".mch" file, check to see that either the associated ".mct" file is already
     # in the list, or add it if the ".mct" file exists.
     for file in files.copy():
-        if file.endswith(".mch"):
+        if file.endswith(".mch") and os.stat(file).st_size > 0:
+            files_to_upload.append(file)
             mct_file = file + ".mct"
-            if mct_file not in files:
-                if os.path.isfile(mct_file):
-                    files.append(mct_file)
+            if os.path.isfile(mct_file) and os.stat(mct_file).st_size > 0:
+                files_to_upload.append(mct_file)
 
     logging.info("Uploading:")
-    for item in files:
+    for item in files_to_upload:
         logging.info("  %s", item)
 
     try:
@@ -2205,7 +2206,7 @@ def upload_blob(file, blob_name):
     total_bytes_uploaded = 0
 
     with TempDir() as temp_location:
-        for file in files:
+        for file in files_to_upload:
             # Zip compress the file we will upload
             zip_name = os.path.basename(file) + ".zip"
             zip_path = os.path.join(temp_location, zip_name)