mass-rebuild: Improve bisect functionality

The mass-rebuild-bisect workflow can now detect false positives and will
update its status on the issue.  It will also only try to bisect when
there is a regression on x86_64.

.github/workflows/mass-rebuild-bisect.yml

@@ -10,17 +10,29 @@ on:
         description: "The name of the Fedora package build that you want to bisect."
         required: true
         type: string
+      issue:
+        description: "The id of the issue to update with the bisect results."
+        required: false
+        type: string
+        default: 0
   workflow_call:
     inputs:
       pkg:
         description: "The name of the Fedora package build that you want to bisect."
         required: true
         type: string
+      issue:
+        description: "The id of the issue to update with the bisect results."
+        required: false
+        type: string
+        default: 0
 
 jobs:
   mass-rebuild-bisect:
     if: github.repository_owner == 'fedora-llvm-team'
     runs-on: ubuntu-24.04
+    permissions:
+      issues: write
     container:
       image: "ghcr.io/${{ github.repository_owner }}/llvm-snapshots-reproducer"
     steps:
@@ -36,5 +48,52 @@ jobs:
 
       - if: always()
         working-directory: /root/llvm-project
+        id: result
         run: |
-          git bisect log
+          touch bisect.log
+          if ! git bisect log; then
+            touch bisect.log
+            echo "result=False Positive" >> $GITHUB_OUTPUT
+          else
+            git bisect log > bisect.log
+            echo "result=Bisect Complete" >> $GITHUB_OUTPUT
+          fi
+
+      - if: always() && inputs.issue != 0
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const jobs = await github.rest.actions.listJobsForWorkflowRunAttempt({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+              attempt_number: process.env.GITHUB_RUN_ATTEMPT
+            });
+            var fs = require('fs');
+            const bisect_log = await fs.readFileSync('/root/llvm-project/bisect.log');
+            const result_string = "${{ steps.result.outputs.result }}";
+            const package = '${{ inputs.pkg }}'
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: '${{ inputs.issue }}',
+              body: `[Bisect](${jobs.data.jobs[0].html_url}) complete for: ${package}\n\n${result_string}\n\n${bisect_log}`
+            })
+
+            if (result_string == "False Positive") {
+              issue = await github.rest.issues.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: '${{ inputs.issue }}'
+              })
+
+              console.log(issue);
+              var body = issue.data.body.replace(`- [ ] [${package}]`,`- [x] [${package}]`)
+
+              github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: '${{ inputs.issue }}',
+                body: body
+              })
+            }

.github/workflows/mass-rebuild-reporter.yml

@@ -18,7 +18,8 @@ jobs:
     container:
       image: "registry.fedoraproject.org/fedora:41"
     outputs:
-      regressions: ${{ steps.regressions.outputs.REGRESSIONS }}
+      bisect-list: ${{ steps.report.outputs.bisect-list }}
+      issue-id: ${{ steps.report.outputs.issue-id }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -76,6 +77,7 @@ jobs:
 
       - name: Create Report
         if: steps.new-rebuild.outputs.completed == 'true'
+        id: report
         uses: actions/github-script@v7
         env:
           REGRESSIONS: ${{ steps.regressions.outputs.REGRESSIONS }}
@@ -85,11 +87,17 @@ jobs:
             const regressions = await JSON.parse(fs.readFileSync('./regressions'));
             comment = "During the last mass rebuild, some packages failed:\n";
             console.log(regressions);
+            bisect_list = [];
             if (regressions.length == 0)
               return;
             regressions.forEach(function(value){
-              comment = comment + `- [ ] [${value.name}](${value.url})\n`
+              arches = value.failed_chroots.map((value) => value.split('-')[2]);
+              comment = comment + `- [ ] [${value.name}](${value.url.replace('@','g/')}) (${arches})\n`
+              if (arches.includes('x86_64')) {
+                bisect_list.push(value.name)
+              }
             });
+
             console.log(comment);
             const issue = await github.rest.issues.create({
                owner: context.repo.owner,
@@ -98,17 +106,44 @@ jobs:
                labels: ['mass-rebuild'],
                body: comment
             });
-            console.log(issue);
+            console.log(bisect_list);
+            core.setOutput("bisect-list", JSON.stringify(bisect_list))
+            core.setOutput("issue-id", issue.data.number)
 
   bisect-failures:
     if: github.repository_owner == 'fedora-llvm-team'
     needs:
       - check-for-rebuild
+    permissions:
+      contents: read
+      issues: write
     strategy:
       max-parallel: 1
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.check-for-rebuild.outputs.regressions) }}
+        name: ${{ fromJson(needs.check-for-rebuild.outputs.bisect-list) }}
     uses: ./.github/workflows/mass-rebuild-bisect.yml
     with:
       pkg: ${{ matrix.name }}
+      issue: ${{ needs.check-for-rebuild.outputs.issue-id }}
+
+  notify-bisect-complete:
+    if: always()
+    runs-on: ubuntu-24.04
+    needs:
+      - check-for-rebuild
+      - bisect-failures
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: '${{ needs.check-for-rebuild.outputs.issue-id }}',
+              body: "Bisect Complete"
+            })
+
+

scripts/bisect.sh

@@ -36,11 +36,25 @@ srpm_name=$(basename $srpm_url)
 
 dnf builddep -y $srpm_name
 
+# Test the good commit to see if this a false positive
+git checkout $good_commit
+
+cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD=Native -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang
+
+if ! ./git-bisect-script.sh $srpm_name; then
+  echo "False Positive."
+  exit 1
+fi
+
+git checkout $bad_commit
+# Test the bad commit to see if this a false positive
+if ./git-bisect-script.sh $srpm_name; then
+  echo "False Positive."
+  exit 1
+fi
 
 git bisect start
 git bisect good $good_commit
 git bisect bad $bad_commit
 
-cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD=Native -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang
-
 git bisect run ./git-bisect-script.sh $srpm_name

scripts/rebuilder.py

@@ -237,12 +237,23 @@ def get_monthly_rebuild_regressions(
         pkgs.append(
             {
                 "name": p.name,
+                "fail_id" : p.latest.id,
                 "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{p.latest.id}/",
+                "chroots" : p.latest.chroots,
             }
         )
     return pkgs
 
 
+def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:
+    for p in pkgs:
+        p['failed_chroots'] = []
+        for c in p["chroots"]:
+            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
+            if result['state'] == 'failed':
+                p['failed_chroots'].append(c)
+
+
 def start_rebuild(
     project_owner: str,
     project_name: str,
@@ -369,6 +380,12 @@ def main():
         pkg_failures = get_monthly_rebuild_regressions(
             project_owner, project_name, start_time, copr_pkgs
         )
+        get_chroot_results(pkg_failures, copr_client)
+        # Delete attributes we don't need to print
+        for p in pkg_failures:
+            for k in ["fail_id", "chroots"]:
+                del p[k]
+
         print(json.dumps(pkg_failures))
     elif args.command == "get-snapshot-date":
         project = copr_client.project_proxy.get(project_owner, project_name)