Enable SuperPMI collection of a libraries tests run

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -631,7 +631,7 @@ jobs:
           displayName: Create SuperPMI directories
           condition: always()
 
-      - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch  -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch
+      - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch
         displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections'
         condition: always()
 

eng/pipelines/coreclr/superpmi-collect.yml

@@ -234,3 +234,57 @@ extends:
             testGroup: outerloop
             liveLibrariesBuildConfig: Release
             SuperPmiCollect: true
+
+      #
+      # Collection of libraries test run: normal
+      # Libraries Test Run using Release libraries, and Checked CoreCLR
+      #
+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+          buildConfig: Release
+          platforms:
+          - osx_arm64
+          - linux_arm
+          - linux_arm64
+          - linux_x64
+          - windows_x64
+          - windows_x86
+          - windows_arm64
+          helixQueueGroup: superpmi
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          jobParameters:
+            testScope: innerloop
+            liveRuntimeBuildConfig: checked
+            dependsOnTestBuildConfiguration: Release
+            dependsOnTestArchitecture: x64
+            coreclrTestGroup: superpmi_collection
+            SuperPmiCollect: true
+            SuperPmiCollectionName: libraries_tests
+
+      #
+      # Collection of libraries test run: no_tiered_compilation
+      # Libraries Test Run using Release libraries, and Checked CoreCLR
+      #
+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+          buildConfig: Release
+          platforms:
+          - osx_arm64
+          - linux_arm
+          - linux_arm64
+          - linux_x64
+          - windows_x64
+          - windows_x86
+          - windows_arm64
+          helixQueueGroup: superpmi
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          jobParameters:
+            testScope: innerloop
+            liveRuntimeBuildConfig: checked
+            dependsOnTestBuildConfiguration: Release
+            dependsOnTestArchitecture: x64
+            coreclrTestGroup: superpmi_collection_no_tiered_compilation
+            SuperPmiCollect: true
+            SuperPmiCollectionName: libraries_tests_no_tiered_compilation

eng/pipelines/libraries/base-job.yml

@@ -24,6 +24,7 @@ parameters:
   pool: ''
   runTests: false
   pgoType: ''
+  SuperPmiCollect: false
 
 jobs:
   - template: /eng/common/templates/job/job.yml

eng/pipelines/libraries/helix.yml

@@ -13,6 +13,9 @@ parameters:
   extraHelixArguments: ''
   shouldContinueOnError: false
   scenarios: ''
+  SuperPmiCollect: ''
+  SuperPmiCollectionType: ''
+  SuperPmiCollectionName: ''
 
 steps:
   - script: $(_msbuildCommand) $(_warnAsErrorParamHelixOverride) -restore
@@ -34,6 +37,9 @@ steps:
     env:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
       _Scenarios: ${{ join(',', parameters.scenarios) }} # Pass scenarios to MSBuild as env var to avoid need of escaping comma separated list
+      _SuperPmiCollect: ${{ parameters.SuperPmiCollect }}
+      _SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }}
+      _SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }}
 
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         HelixAccessToken: $(HelixApiAccessToken)

eng/pipelines/libraries/run-test-job.yml

@@ -23,6 +23,9 @@ parameters:
   # stress modes that each test will be run with. This is the same usage as 'testGroup' in
   # eng/pipelines/common/templates/runtimes/run-test-job.yml.
   coreclrTestGroup: ''
+  SuperPmiCollect: false
+  SuperPmiCollectionType: 'run'
+  SuperPmiCollectionName: 'libraries_tests'
   dependsOn: []
 
 jobs:
@@ -42,13 +45,18 @@ jobs:
       container: ${{ parameters.container }}
       condition: ${{ parameters.condition }}
       testScope: ${{ parameters.testScope }}
+      SuperPmiCollect: ${{ parameters.SuperPmiCollect }}
       runTests: true
-      ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
-        displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
-        name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
-      ${{ if eq(parameters.liveRuntimeBuildConfig, '') }}:
-        displayName: 'Test Run'
-        name: test_run
+      ${{ if eq(parameters.SuperPmiCollect, true) }}:
+        displayName: ${{ format('SuperPMI collection {0} {1} {2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
+        name: ${{ format('spmi_{0}_{1}_{2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
+      ${{ else }}:
+        ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+          displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
+          name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
+        ${{ if eq(parameters.liveRuntimeBuildConfig, '') }}:
+          displayName: 'Test Run'
+          name: test_run
       ${{ if eq(parameters.interpreter, 'true') }}:
         testDisplayName: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }}
 
@@ -64,14 +72,36 @@ jobs:
           - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
         - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
           - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
+      # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed.
+      - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}:
+          - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', 'coreclr', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }}
 
       variables:
+
         - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
         - _archiveTestsParameter: /p:ArchiveTests=true
 
+        - ${{ if eq(parameters.SuperPmiCollect, true) }}:
+          - template: /eng/pipelines/libraries/superpmi-collect-variables.yml
+            parameters:
+              buildConfig: ${{ parameters.buildConfig }}
+              osGroup: ${{ parameters.osGroup }}
+              archType: ${{ parameters.archType }}
+              runtimeFlavor: ${{ parameters.runtimeFlavor }}
+
         - ${{ parameters.variables }}
 
       steps:
+
+        # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe on AzDO machine; see `SuperPmiMcsPath`.
+        - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}:
+          - template: /eng/pipelines/common/download-artifact-step.yml
+            parameters:
+              unpackFolder: $(_runtimeX64DownloadPath)
+              artifactFileName: 'CoreCLRProduct___${{ parameters.osGroup }}${{ parameters.osSubgroup }}_x64_${{ parameters.liveRuntimeBuildConfig }}$(archiveExtension)'
+              artifactName: 'CoreCLRProduct___${{ parameters.osGroup }}${{ parameters.osSubgroup }}_x64_${{ parameters.liveRuntimeBuildConfig }}'
+              displayName: 'CoreCLR product build (x64)'
+
         - template: /eng/pipelines/common/download-artifact-step.yml
           parameters:
             displayName: Build Assets
@@ -113,6 +143,9 @@ jobs:
             shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
             creator: dotnet-bot
             testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix)
+            SuperPmiCollect: ${{ parameters.SuperPmiCollect }}
+            SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }}
+            SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }}
 
             # coreclrTestGroup: The following mappings of 'coreclrTestGroup' to 'scenarios' is copied from
             # eng/pipelines/common/templates/runtimes/run-test-job.yml (with 'testGroup' replaced by 'coreclrTestGroup'
@@ -137,6 +170,12 @@ jobs:
             ${{ else }}:
               extraHelixArguments: $(_extraHelixArguments)
 
+            ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection') }}:
+              scenarios:
+              - normal
+            ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection_no_tiered_compilation') }}:
+              scenarios:
+              - no_tiered_compilation
             ${{ if in(parameters.coreclrTestGroup, 'jitstress') }}:
               scenarios:
               - no_tiered_compilation
@@ -205,3 +244,19 @@ jobs:
                 - syntheticpgo
                 - syntheticpgo_blend
 
+        - ${{ if eq(parameters.SuperPmiCollect, true) }}:
+          - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml
+            parameters:
+              buildConfig: ${{ parameters.buildConfig }}
+              buildConfigUpper: $(buildConfigUpper)
+              osGroup: ${{ parameters.osGroup }}
+              osSubgroup: ${{ parameters.osSubgroup }}
+              archType: ${{ parameters.archType }}
+              SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }}
+              SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }}
+              MergedMchFileLocation: $(MergedMchFileLocation)
+              MchFilesLocation: $(MchFilesLocation)
+              SpmiLogsLocation: $(SpmiLogsLocation)
+              SuperPmiMcsPath: $(SuperPmiMcsPath)
+              PythonScript: $(PythonScript)
+              PipScript: $(PipScript)

eng/pipelines/libraries/superpmi-collect-variables.yml

@@ -0,0 +1,46 @@
+parameters:
+  buildConfig: ''
+  osGroup: ''
+  archType: ''
+  runtimeFlavor: 'coreclr'
+
+variables:
+
+  - name: buildConfigUpper
+    ${{ if eq(parameters.buildConfig, 'debug') }}:
+      value: 'Debug'
+    ${{ if eq(parameters.buildConfig, 'release') }}:
+      value: 'Release'
+  - name: _runtimeX64DownloadPath
+    value: ''
+  # superpmi.py 'merge-mch' needs to be able to find the mcs tool. Point SuperPmiMcsPath at the downloaded CoreCLR binaries. For non-x64 targets, download an x64
+  # build and point at that. Pass this to superpmi.py as the '-core_root' argument. It's not actually a "Core_Root" directory, but all it needs is to find mcs.
+  - name: SuperPmiMcsPath
+    value: $(_runtimeDownloadPath)
+  - ${{ if ne(parameters.archType, 'x64') }}:
+    - name: _runtimeX64DownloadPath
+      value: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}.x64'
+    - name: SuperPmiMcsPath
+      value: $(_runtimeX64DownloadPath)
+  - ${{ if eq(parameters.osGroup, 'windows') }}:
+    - name: PythonScript
+      value: 'py -3'
+    - name: PipScript
+      value: 'py -3 -m pip'
+    - name: MchFilesLocation
+      value: '$(Build.SourcesDirectory)\artifacts\helixresults\'
+    - name: MergedMchFileLocation
+      value: '$(Build.SourcesDirectory)\artifacts\spmi_collection\'
+    - name: SpmiLogsLocation
+      value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\'
+  - ${{ if ne(parameters.osGroup, 'windows') }}:
+    - name: PythonScript
+      value: 'python3'
+    - name: PipScript
+      value: 'pip3'
+    - name: MchFilesLocation
+      value: '$(Build.SourcesDirectory)/artifacts/helixresults/'
+    - name: MergedMchFileLocation
+      value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/'
+    - name: SpmiLogsLocation
+      value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/'

eng/pipelines/libraries/superpmi-postprocess-step.yml

@@ -0,0 +1,87 @@
+########################################################################################################
+#
+# Finalize SuperPMI collection: (1) merge all MCH files generated by all Helix jobs, (2) upload MCH file
+# to Azure Storage, (3) upload log files. Note that the steps are "condition: always()" because we want
+# to upload as much of the collection as possible, even if there were test failures.
+#
+########################################################################################################
+
+parameters:
+  buildConfig: ''
+  buildConfigUpper: ''
+  osGroup: ''
+  osSubgroup: ''
+  archType: ''
+  SuperPmiCollectionType: 'run'
+  SuperPmiCollectionName: 'libraries_tests'
+  MergedMchFileLocation: ''
+  MchFilesLocation: ''
+  SpmiLogsLocation: ''
+  SuperPmiMcsPath: ''
+  PythonScript: ''
+  PipScript: ''
+
+steps:
+
+  # Create required directories for merged mch collection and superpmi logs
+  - ${{ if ne(parameters.osGroup, 'windows') }}:
+    - script: |
+        mkdir -p ${{ parameters.MergedMchFileLocation }}
+        mkdir -p ${{ parameters.SpmiLogsLocation }}
+      displayName: 'Create SuperPMI directories'
+      condition: always()
+  - ${{ if eq(parameters.osGroup, 'windows') }}:
+    - script: |
+        mkdir ${{ parameters.MergedMchFileLocation }}
+        mkdir ${{ parameters.SpmiLogsLocation }}
+      displayName: 'Create SuperPMI directories'
+      condition: always()
+
+  - script: ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern ${{ parameters.MchFilesLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}*.mch -output_mch_path ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root ${{ parameters.SuperPmiMcsPath }}
+    displayName: 'Merge ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} SuperPMI collections'
+    condition: always()
+
+  - template: /eng/pipelines/common/upload-artifact-step.yml
+    parameters:
+      rootFolder: ${{ parameters.MergedMchFileLocation }}
+      includeRootFolder: false
+      archiveType: $(archiveType)
+      tarCompression: $(tarCompression)
+      archiveExtension: $(archiveExtension)
+      artifactName: 'SuperPMI_Collection_${{ parameters.SuperPmiCollectionName }}_${{ parameters.SuperPmiCollectionType }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
+      displayName: 'Upload artifacts SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection'
+      condition: always()
+
+  # Add authenticated pip feed
+  - task: PipAuthenticate@1
+    displayName: 'Pip Authenticate'
+    inputs:
+      artifactFeeds: public/dotnet-public-pypi
+      onlyAddExtraIndex: false
+    condition: always()
+
+  # Ensure the Python azure-storage-blob package is installed before doing the upload.
+  - script: ${{ parameters.PipScript }} install --user --upgrade pip && ${{ parameters.PipScript }} install --user azure.storage.blob==12.5.0 --force-reinstall
+    displayName: Upgrade Pip to latest and install azure-storage-blob Python package
+    condition: always()
+
+  - script: ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch ${{ parameters.archType }} -build_type ${{ parameters.buildConfig }} -mch_files ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.${{ parameters.buildConfigUpper }}
+    displayName: 'Upload SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection to Azure Storage'
+    condition: always()
+    env:
+      CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline
+
+  - task: CopyFiles@2
+    displayName: Copying superpmi.log of all partitions
+    inputs:
+      sourceFolder: '${{ parameters.MchFilesLocation }}'
+      contents: '**/${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}*.log'
+      targetFolder: '${{ parameters.SpmiLogsLocation }}'
+    condition: always()
+
+  - task: PublishPipelineArtifact@1
+    displayName: Publish SuperPMI logs
+    inputs:
+      targetPath: ${{ parameters.SpmiLogsLocation }}
+      artifactName: 'SuperPMI_Logs_${{ parameters.SuperPmiCollectionName }}_${{ parameters.SuperPmiCollectionType }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
+    condition: always()

eng/testing/RunnerTemplate.cmd

@@ -47,6 +47,26 @@ set DOTNET_MULTILEVEL_LOOKUP=0
 :: Assume failure
 set HAS_TEST_RESULTS=0
 
+:: Support for SuperPMI collection
+REM SuperPMI collection
+if not defined spmi_enable_collection goto :skip_spmi_enable_collection
+echo SuperPMI collection enabled
+REM spmi_collect_dir and spmi_core_root need to be set before this script is run, if SuperPMI collection is enabled.
+if not defined spmi_collect_dir echo ERROR: spmi_collect_dir not defined&exit /b 1
+if not defined spmi_core_root echo ERROR: spmi_core_root not defined&exit /b 1
+if not exist %spmi_collect_dir% mkdir %spmi_collect_dir%
+set SuperPMIShimLogPath=%spmi_collect_dir%
+set SuperPMIShimPath=%spmi_core_root%\clrjit.dll
+if not exist %SuperPMIShimPath% echo ERROR: %SuperPMIShimPath% not found&exit /b 1
+set DOTNET_EnableExtraSuperPmiQueries=1
+set DOTNET_JitPath=%spmi_core_root%\superpmi-shim-collector.dll
+if not exist %DOTNET_JitPath% echo ERROR: %DOTNET_JitPath% not found&exit /b 1
+echo SuperPMIShimLogPath=%SuperPMIShimLogPath%
+echo SuperPMIShimPath=%SuperPMIShimPath%
+echo DOTNET_EnableExtraSuperPmiQueries=%DOTNET_EnableExtraSuperPmiQueries%
+echo DOTNET_JitPath=%DOTNET_JitPath%
+:skip_spmi_enable_collection
+
 :: ========================= BEGIN Test Execution =============================
 echo ----- start %DATE% %TIME% ===============  To repro directly: =====================================================
 echo pushd %EXECUTION_DIR%