Add some automation for doing periodic mass rebuilds of Fedora packages

.github/actions/prepare-python/action.yml

@@ -7,11 +7,16 @@ inputs:
     description: "Where this project was checked out"
     required: false
     default: "."
+  use-system-python:
+    description: "Set to true if you want to use the python installation from the OS"
+    required: false
+    default: false
 runs:
   using: "composite"
   steps:
     - name: "Setup python"
       uses: actions/setup-python@v5
+      if: "${{ inputs.use-system-python != 'true' }}"
       with:
         python-version: 3.12
         cache: 'pip'
@@ -24,4 +29,14 @@ runs:
 
     - name: "Install python dependencies for project"
       shell: bash
-      run: pip install -r ${{ inputs.checkout-path }}/requirements.txt
+      run: |
+        pip install -r ${{ inputs.checkout-path }}/requirements.txt
+
+    # The dnf module cannot be installed by pip, so it's only possible to use
+    # it when using the system pyhton interpreter.
+    - name: Install DNF module
+      if: "${{ inputs.use-system-python == 'true' }}"
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-dnf

.github/workflows/build-reproducer-container.yml

@@ -0,0 +1,27 @@
+name: "Build Reproducer Container"
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  build-reproducer-container:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Container
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install podman
+          podman build -t ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer -f Containerfile scripts/
+          podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
+          podman push ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer

.github/workflows/mass-rebuild-bisect.yml

@@ -0,0 +1,40 @@
+name: Mass Rebuild Bisect
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      pkg:
+        description: "The name of the Fedora package build that you want to bisect."
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      pkg:
+        description: "The name of the Fedora package build that you want to bisect."
+        required: true
+        type: string
+
+jobs:
+  mass-rebuild-bisect:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    container:
+      image: "ghcr.io/${{ github.repository_owner }}/llvm-snapshots-reproducer"
+    steps:
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          max-size: 8G
+          key: bisect
+      - working-directory: /root/llvm-project/
+        run: |
+          git fetch origin
+          bash bisect.sh ${{ inputs.pkg }}
+
+      - if: always()
+        working-directory: /root/llvm-project
+        run: |
+          git bisect log

.github/workflows/mass-rebuild-reporter.yml

@@ -0,0 +1,115 @@
+name: "Mass Rebuild Reporter"
+
+on:
+  schedule:
+    # Hourly at minute 40, e.g. 2024-12-18 00:40:00
+    - cron: "40 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  check-for-rebuild:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    permissions:
+      issues: write
+    container:
+      image: "registry.fedoraproject.org/fedora:41"
+    outputs:
+      regressions: ${{ steps.regressions.outputs.REGRESSIONS }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts/rebuilder.py
+          sparse-checkout-cone-mode: false
+
+
+      - name: Check for last report
+        uses: actions/github-script@v7
+        id: last-report
+        with:
+          result-encoding: string
+          script: |
+            const issues = await github.rest.search.issuesAndPullRequests({
+              q: "label:mass-rebuild+is:issue",
+              sort: "created",
+              order: "desc",
+              per_page: 1
+            });
+
+            console.log(issues)
+            if (issues.data.total_count == 0)
+              return 0;
+            const issue = issues.data.items[0];
+            console.log(issue);
+            return issue.created_at
+
+      - name: Check if a new rebuild has completed
+        id: new-rebuild
+        run: |
+          sudo dnf install -y python3-dnf python3-copr
+          if python3 scripts/rebuilder.py rebuild-in-progress; then
+            echo "completed=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          last_rebuild=$(date +%s -d "${{ steps.last-report.outputs.result }}")
+          current_snapshot=$(date +%s -d "$(python3 scripts/rebuilder.py get-snapshot-date)")
+
+          echo "last_rebuild: $last_rebuild current_snapshot: $current_snapshot"
+
+          if [ $last_rebuild -gt $current_snapshot ]; then
+            echo "completed=false" >> $GITHUB_OUTPUT
+          else
+            echo "completed=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Collect Regressions
+        if: steps.new-rebuild.outputs.completed == 'true'
+        id: regressions
+        run: |
+          python3 scripts/rebuilder.py get-regressions --start-date ${{ steps.last-report.outputs.result }} > regressions
+          echo "REGRESSIONS=$(cat regressions)" >> $GITHUB_OUTPUT
+
+      - name: Create Report
+        if: steps.new-rebuild.outputs.completed == 'true'
+        uses: actions/github-script@v7
+        env:
+          REGRESSIONS: ${{ steps.regressions.outputs.REGRESSIONS }}
+        with:
+          script: |
+            var fs = require('fs');
+            const regressions = await JSON.parse(fs.readFileSync('./regressions'));
+            comment = "During the last mass rebuild, some packages failed:\n";
+            console.log(regressions);
+            if (regressions.length == 0)
+              return;
+            regressions.forEach(function(value){
+              comment = comment.concat('\n', value.name);
+              comment = comment.concat(': ', value.url);
+            });
+            console.log(comment);
+            const issue = await github.rest.issues.create({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               title: "Mass Rebuild Report",
+               labels: ['mass-rebuild'],
+               body: comment
+            });
+            console.log(issue);
+
+  bisect-failures:
+    if: github.repository_owner == 'fedora-llvm-team'
+    needs:
+      - check-for-rebuild
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.check-for-rebuild.outputs.regressions) }}
+    uses: ./.github/workflows/mass-rebuild-bisect.yml
+    with:
+      pkg: ${{ matrix.name }}

.github/workflows/mass-rebuild-runner.yml

@@ -0,0 +1,37 @@
+name: "Mass Rebuild Runner"
+
+on:
+  schedule:
+    # Run on the first of every month.
+    - cron: 30 1 1 * *
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  start-rebuild:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    container:
+      image: "registry.fedoraproject.org/fedora:41"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts/rebuilder.py
+          sparse-checkout-cone-mode: false
+
+      - name: Setup Copr config file
+        env:
+          # You need to have those secrets in your repo.
+          # See also: https://copr.fedorainfracloud.org/api/.
+          COPR_CONFIG_FILE: ${{ secrets.COPR_CONFIG }}
+        run: |
+          mkdir -p ~/.config
+          printf "$COPR_CONFIG_FILE" > ~/.config/copr
+
+      - name: Start rebuild
+        run: |
+          sudo dnf install -y python3-dnf python3-copr
+          python3 scripts/rebuilder.py rebuild

.github/workflows/python-format-and-tests.yml

@@ -19,6 +19,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/prepare-python
+        with:
+          use-system-python: true
       - name: Run pytest with coverage
         shell: bash -e {0}
         env:

Containerfile

@@ -0,0 +1,19 @@
+FROM registry.fedoraproject.org/fedora:41
+
+ENV LLVM_SYSROOT=/opt/llvm \
+    AR=llvm-ar \
+    RANLIB=llvm-ranlib
+
+RUN dnf -y copr enable tstellar/fedora-clang-default-cc
+
+RUN dnf -y install jq cmake ninja-build git binutils-devel clang fedora-clang-default-cc rpmbuild ccache
+
+WORKDIR /root
+
+RUN git clone https://github.com/llvm/llvm-project
+
+WORKDIR /root/llvm-project
+
+ADD bisect.sh git-bisect-script.sh .
+
+COPY --from=ghcr.io/llvm/ci-ubuntu-22.04:1734145213 $LLVM_SYSROOT $LLVM_SYSROOT

README.adoc

@@ -193,6 +193,38 @@ $ cd llvm
 $ make snapshot-rpm
 ----
 
+= README
+:icons: font
+
 You might need to install missing dependencies. The build process itself probably takes quite some time.
 
 You're going to find `results/YYYYMMDD/snapshot-rpm.log` with logging everything from this makefile target.
+
+== Mass Rebuilds ==
+
+This repository uses GitHub Actions to periodically perform rebuilds of selected
+Fedora packages.  Once a mass rebuild is complete there is also automation
+that will create a new issue with the results of the rebuild.
+
+The rebuild process will attempt to automatically bisect the failures to a specific upstream
+LLVM commit.
+
+The rebuild can be started manually using the rebuilder.py script in
+`.github/workflows/`
+
+[source,console]
+---
+$ python3 rebuilder.py rebuild
+---
+
+You can also view the regression report once the rebuild is complete using
+the same script.
+
+[source,console]
+---
+$ python3 rebuilder.py get-regressions --start-date=<yyyy-mm-dd>
+---
+
+The start date should be the day the rebuild was started (In reality
+it can be any date between when the last rebuild ended and the
+new rebuild began).

pytest.ini

@@ -11,7 +11,7 @@ addopts: --doctest-modules --durations=0 -s
 
 ; Directories to search for tests when no files or directories are given on the
 ; command line
-testpaths: snapshot_manager
+testpaths: snapshot_manager scripts
 
 ; Each line specifies a pattern for warnings.filterwarnings. Processed after
 ; -W/--pythonwarnings.

scripts/bisect.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -ex
+
+function get_clang_commit {
+  buildid=$1
+  pkg=$2
+
+  curl "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/fedora-41-clang-20/fedora-41-x86_64/0$buildid-$pkg/root.log.gz" | gunzip |  grep -o 'clang[[:space:]]\+x86_64[[:space:]]\+[0-9a-g~pre.]\+' | cut -d 'g' -f 3
+}
+
+
+pkg_or_buildid=$1
+
+if echo $pkg_or_buildid | grep '^[0-9]\+'; then
+  buildid=$pkg_or_buildid
+  read -r pkg last_success_id <<<$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r '[.builds.latest.source_package.name,.builds.latest_succeeded.id] |  join(" ")')
+else
+  pkg=$pkg_or_buildid
+fi
+
+read -r buildid last_success_id <<<$(curl -X 'GET' \
+  "https://copr.fedorainfracloud.org/api_3/package/?ownername=%40fedora-llvm-team&projectname=fedora-41-clang-20&packagename=$pkg&with_latest_build=true&with_latest_succeeded_build=true" \
+  -H 'accept: application/json' | jq -r '[.builds.latest.id,.builds.latest_succeeded.id] | join(" ")' )
+
+
+good_commit=llvmorg-20-init
+bad_commit=origin/main
+
+good_commit=$(get_clang_commit $last_success_id $pkg)
+bad_commit=$(get_clang_commit $buildid $pkg)
+
+srpm_url=$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r .source_package.url)
+curl -O -L $srpm_url
+srpm_name=$(basename $srpm_url)
+
+dnf builddep -y $srpm_name
+
+
+git bisect start
+git bisect good $good_commit
+git bisect bad $bad_commit
+
+cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD=Native -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang
+
+git bisect run ./git-bisect-script.sh $srpm_name

scripts/git-bisect-script.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+srpm_name=$1
+
+ninja -C build install-clang install-clang-resource-headers install-LLVMgold install-llvm-ar install-llvm-ranlib
+
+rpmbuild -rb $srpm_name