Skip PCT by default on PRs (#2034)

* Skip PCT by default on PRs

* Run full tests weekly

* Empty commit should suffice

* `git branch -D` might be necessary

Co-authored-by: Joseph Petersen <me@jetersen.dev>

* Copying DB automerge workflow from #2031

* Revert #2032 for `prep` phase, retaining node pool for PCT #2034 (comment)

---------

Co-authored-by: Joseph Petersen <me@jetersen.dev>

.github/workflows/dependabot-automerge.yaml

@@ -0,0 +1,17 @@
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
+---
+name: Dependabot auto-merge
+on: pull_request
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  dependabot-automerge:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/run-full-test.yaml

@@ -0,0 +1,29 @@
+name: Run full tests
+on:
+  schedule:
+    - cron: '0 1 * * 6'
+  workflow_dispatch:
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - run: |
+        set -x
+        git fetch
+        if gh pr checkout full-tests
+        then
+          gh pr close --comment 'Recreating'
+          git checkout master
+          git branch -D full-tests
+          git push origin :full-tests
+          git reset --hard master
+        fi
+        git checkout -b full-tests
+        # GitHub apparently does not let you create a PR with no commits:
+        git commit --allow-empty --message 'Phony commit'
+        git push origin full-tests
+        # Not using --draft to ensure notifications are sent:
+        gh pr create --head --title 'Testing master (do not merge)' --body 'Close this PR if it passes; otherwise please fix failures.' --reviewer jenkinsci/bom-developers --label full-test
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Jenkinsfile

@@ -4,17 +4,17 @@ if (BRANCH_NAME == 'master' && currentBuild.buildCauses*._class == ['jenkins.bra
   error 'No longer running builds on response to master branch pushes. If you wish to cut a release, use “Re-run checks” from this failing check in https://github.com/jenkinsci/bom/commits/master'
 }
 
-def mavenEnv(Map params = [:], Closure body) {
+def mavenEnv(boolean nodePool, int jdk, Closure body) {
   def attempt = 0
   def attempts = 3
   retry(count: attempts, conditions: [kubernetesAgent(handleNonKubernetes: true), nonresumable()]) {
     echo 'Attempt ' + ++attempt + ' of ' + attempts
     // no Dockerized tests; https://github.com/jenkins-infra/documentation/blob/master/ci.adoc#container-agents
-    node('maven-bom') {
+    node(nodePool ? 'maven-bom': "maven-$jdk") {
       timeout(120) {
         infra.withArtifactCachingProxy {
           withEnv([
-            "JAVA_HOME=/opt/jdk-$params.jdk",
+            "JAVA_HOME=/opt/jdk-$jdk",
             "MAVEN_ARGS=${env.MAVEN_ARGS != null ? MAVEN_ARGS : ''} -B -ntp -Dmaven.repo.local=${WORKSPACE_TMP}/m2repo"
           ]) {
             body()
@@ -41,10 +41,9 @@ def parsePlugins(plugins) {
 
 def pluginsByRepository
 def lines
-def fullTest = env.CHANGE_ID && pullRequest.labels.contains('full-test')
 
 stage('prep') {
-  mavenEnv(jdk: 11) {
+  mavenEnv(false, 11) {
     checkout scm
     withEnv(['SAMPLE_PLUGIN_OPTS=-Dset.changelist']) {
       withCredentials([
@@ -61,10 +60,6 @@ stage('prep') {
       pluginsByRepository = parsePlugins(plugins)
 
       lines = readFile('lines.txt').split('\n')
-      if (env.CHANGE_ID && !fullTest) {
-        // run PCT only on newest and oldest lines, to save resources (but check all lines on deliberate master builds)
-        lines = [lines[0], lines[-1]]
-      }
       launchable.install()
       withCredentials([string(credentialsId: 'launchable-jenkins-bom', variable: 'LAUNCHABLE_TOKEN')]) {
         lines.each { line ->
@@ -85,35 +80,37 @@ stage('prep') {
   }
 }
 
-branches = [failFast: !fullTest]
-lines.each {line ->
-  pluginsByRepository.each { repository, plugins ->
-    branches["pct-$repository-$line"] = {
-      def jdk = line == 'weekly' ? 17 : 11
-      mavenEnv(jdk: jdk) {
-        unstash line
-        withEnv([
-          "PLUGINS=${plugins.join(',')}",
-          "LINE=$line",
-          'EXTRA_MAVEN_PROPERTIES=maven.test.failure.ignore=true:surefire.rerunFailingTestsCount=1'
-        ]) {
-          sh '''
-          mvn -v
-          bash pct.sh
-          '''
-        }
-        launchable.install()
-        withCredentials([string(credentialsId: 'launchable-jenkins-bom', variable: 'LAUNCHABLE_TOKEN')]) {
-          launchable('verify')
-          def sessionFile = "launchable-session-${line}.txt"
-          unstash sessionFile
-          def session = readFile(sessionFile).trim()
-          launchable("record tests --session ${session} --group ${repository} maven './**/target/surefire-reports' './**/target/failsafe-reports'")
+if (BRANCH_NAME == 'master' || env.CHANGE_ID && pullRequest.labels.contains('full-test')) {
+  branches = [failFast: false]
+  lines.each {line ->
+    pluginsByRepository.each { repository, plugins ->
+      branches["pct-$repository-$line"] = {
+        def jdk = line == 'weekly' ? 17 : 11
+        mavenEnv(true, jdk) {
+          unstash line
+          withEnv([
+            "PLUGINS=${plugins.join(',')}",
+            "LINE=$line",
+            'EXTRA_MAVEN_PROPERTIES=maven.test.failure.ignore=true:surefire.rerunFailingTestsCount=1'
+          ]) {
+            sh '''
+            mvn -v
+            bash pct.sh
+            '''
+          }
+          launchable.install()
+          withCredentials([string(credentialsId: 'launchable-jenkins-bom', variable: 'LAUNCHABLE_TOKEN')]) {
+            launchable('verify')
+            def sessionFile = "launchable-session-${line}.txt"
+            unstash sessionFile
+            def session = readFile(sessionFile).trim()
+            launchable("record tests --session ${session} --group ${repository} maven './**/target/surefire-reports' './**/target/failsafe-reports'")
+          }
         }
       }
     }
   }
+  parallel branches
 }
-parallel branches
 
 infra.maybePublishIncrementals()

README.md

@@ -145,7 +145,7 @@ If the build fails due to an unmanaged transitive plugin dependency, add it to
 
 ## PCT
 
-The CI build tries running the [Plugin Compatibility Tester (PCT)](https://github.com/jenkinsci/plugin-compat-tester/)
+The CI build can run the [Plugin Compatibility Tester (PCT)](https://github.com/jenkinsci/plugin-compat-tester/)
 on the particular combination of plugins being managed by the BOM.
 This catches mutual incompatibilities between plugins
 (as revealed by their `JenkinsRule` tests)
@@ -168,7 +168,10 @@ DOCKERIZED=true
 
 to reproduce image-specific failures.
 
-Note that to minimize build time, tests are run only on Linux, against JDK 8, and without Docker support.
+To minimize cloud resources, PCT is not run at all by default on pull requests, only some basic sanity checks.
+Add the label `full-test` to run PCT in a PR.
+
+To further minimize build time, tests are run only on Linux, against Java 11, and without Docker support.
 It is unusual but possible for cross-component incompatibilities to only be visible in more specialized environments (such as Windows).
 
 ## LTS lines
@@ -187,10 +190,6 @@ The UC currently maintains releases for the [past 400 days](https://groups.googl
 so it is reasonable to retire BOMs for lines older than that,
 or otherwise when the number of accumulated version overrides becomes large.
 
-Add the label `full-test` in dangerous-looking PRs to make sure you are running tests in all LTS lines;
-by default tests are only run in the oldest line and weeklies.
-This flag also allows all tests to be run even after some failures are recorded.
-
 ## Releasing
 
 You can cut a release using [JEP-229](https://jenkins.io/jep/229).

updatecli/updatecli.d/plugin-compat-tester.yml

@@ -43,3 +43,4 @@ actions:
     spec:
       labels:
         - dependencies
+        - full-test