From 351ff2b55b66b02eef2d0c429ea49e1faa13dfac Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 8 Jan 2025 13:37:10 -0800 Subject: [PATCH] Add some automation for doing periodic mass rebuilds of Fedora packages (#864) These mass rebuilds will help test the snapshot builds and catch regressions in real-world projects. The GitHub workflows are currently configured to start a new mass rebuild once per month and then when the rebuild is complete, it will create any issue reporting any regressions since the last build. The goal is to keep the rebuilder as simple as possible, so it will only rebuild packages that successfully built in the last rebuild, and it makes no attempt to work around packages that hard code gcc as their compiler. As a result, some of the 'passing' builds may actually use gcc instead of clang, but this is an acceptable trade-off to keep the rebuild process simple. Co-authored-by: Konrad Kleine --- .github/actions/prepare-python/action.yml | 17 +- .../workflows/build-reproducer-container.yml | 27 ++ .github/workflows/mass-rebuild-bisect.yml | 40 ++ .github/workflows/mass-rebuild-reporter.yml | 115 ++++++ .github/workflows/mass-rebuild-runner.yml | 37 ++ .github/workflows/python-format-and-tests.yml | 2 + Containerfile | 19 + README.adoc | 32 ++ pytest.ini | 2 +- scripts/bisect.sh | 46 +++ scripts/git-bisect-script.sh | 7 + scripts/rebuilder.py | 391 ++++++++++++++++++ 12 files changed, 733 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-reproducer-container.yml create mode 100644 .github/workflows/mass-rebuild-bisect.yml create mode 100644 .github/workflows/mass-rebuild-reporter.yml create mode 100644 .github/workflows/mass-rebuild-runner.yml create mode 100644 Containerfile create mode 100644 scripts/bisect.sh create mode 100755 scripts/git-bisect-script.sh create mode 100644 scripts/rebuilder.py diff --git a/.github/actions/prepare-python/action.yml b/.github/actions/prepare-python/action.yml index d947c7f4..7df003bd 100644 --- a/.github/actions/prepare-python/action.yml +++ b/.github/actions/prepare-python/action.yml @@ -7,11 +7,16 @@ inputs: description: "Where this project was checked out" required: false default: "." + use-system-python: + description: "Set to true if you want to use the python installation from the OS" + required: false + default: false runs: using: "composite" steps: - name: "Setup python" uses: actions/setup-python@v5 + if: "${{ inputs.use-system-python != 'true' }}" with: python-version: 3.12 cache: 'pip' @@ -24,4 +29,14 @@ runs: - name: "Install python dependencies for project" shell: bash - run: pip install -r ${{ inputs.checkout-path }}/requirements.txt + run: | + pip install -r ${{ inputs.checkout-path }}/requirements.txt + + # The dnf module cannot be installed by pip, so it's only possible to use + # it when using the system pyhton interpreter. + - name: Install DNF module + if: "${{ inputs.use-system-python == 'true' }}" + shell: bash + run: | + sudo apt-get update + sudo apt-get install python3-dnf diff --git a/.github/workflows/build-reproducer-container.yml b/.github/workflows/build-reproducer-container.yml new file mode 100644 index 00000000..02aa539f --- /dev/null +++ b/.github/workflows/build-reproducer-container.yml @@ -0,0 +1,27 @@ +name: "Build Reproducer Container" + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + build-reproducer-container: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - uses: actions/checkout@v4 + - name: Build Container + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get -y update + sudo apt-get -y install podman + podman build -t ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer -f Containerfile scripts/ + podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io + podman push ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer diff --git a/.github/workflows/mass-rebuild-bisect.yml b/.github/workflows/mass-rebuild-bisect.yml new file mode 100644 index 00000000..039d04d7 --- /dev/null +++ b/.github/workflows/mass-rebuild-bisect.yml @@ -0,0 +1,40 @@ +name: Mass Rebuild Bisect + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + pkg: + description: "The name of the Fedora package build that you want to bisect." + required: true + type: string + workflow_call: + inputs: + pkg: + description: "The name of the Fedora package build that you want to bisect." + required: true + type: string + +jobs: + mass-rebuild-bisect: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + container: + image: "ghcr.io/${{ github.repository_owner }}/llvm-snapshots-reproducer" + steps: + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + max-size: 8G + key: bisect + - working-directory: /root/llvm-project/ + run: | + git fetch origin + bash bisect.sh ${{ inputs.pkg }} + + - if: always() + working-directory: /root/llvm-project + run: | + git bisect log diff --git a/.github/workflows/mass-rebuild-reporter.yml b/.github/workflows/mass-rebuild-reporter.yml new file mode 100644 index 00000000..bd5df5e1 --- /dev/null +++ b/.github/workflows/mass-rebuild-reporter.yml @@ -0,0 +1,115 @@ +name: "Mass Rebuild Reporter" + +on: + schedule: + # Hourly at minute 40, e.g. 2024-12-18 00:40:00 + - cron: "40 * * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check-for-rebuild: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + permissions: + issues: write + container: + image: "registry.fedoraproject.org/fedora:41" + outputs: + regressions: ${{ steps.regressions.outputs.REGRESSIONS }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts/rebuilder.py + sparse-checkout-cone-mode: false + + + - name: Check for last report + uses: actions/github-script@v7 + id: last-report + with: + result-encoding: string + script: | + const issues = await github.rest.search.issuesAndPullRequests({ + q: "label:mass-rebuild+is:issue", + sort: "created", + order: "desc", + per_page: 1 + }); + + console.log(issues) + if (issues.data.total_count == 0) + return 0; + const issue = issues.data.items[0]; + console.log(issue); + return issue.created_at + + - name: Check if a new rebuild has completed + id: new-rebuild + run: | + sudo dnf install -y python3-dnf python3-copr + if python3 scripts/rebuilder.py rebuild-in-progress; then + echo "completed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + last_rebuild=$(date +%s -d "${{ steps.last-report.outputs.result }}") + current_snapshot=$(date +%s -d "$(python3 scripts/rebuilder.py get-snapshot-date)") + + echo "last_rebuild: $last_rebuild current_snapshot: $current_snapshot" + + if [ $last_rebuild -gt $current_snapshot ]; then + echo "completed=false" >> $GITHUB_OUTPUT + else + echo "completed=true" >> $GITHUB_OUTPUT + fi + + - name: Collect Regressions + if: steps.new-rebuild.outputs.completed == 'true' + id: regressions + run: | + python3 scripts/rebuilder.py get-regressions --start-date ${{ steps.last-report.outputs.result }} > regressions + echo "REGRESSIONS=$(cat regressions)" >> $GITHUB_OUTPUT + + - name: Create Report + if: steps.new-rebuild.outputs.completed == 'true' + uses: actions/github-script@v7 + env: + REGRESSIONS: ${{ steps.regressions.outputs.REGRESSIONS }} + with: + script: | + var fs = require('fs'); + const regressions = await JSON.parse(fs.readFileSync('./regressions')); + comment = "During the last mass rebuild, some packages failed:\n"; + console.log(regressions); + if (regressions.length == 0) + return; + regressions.forEach(function(value){ + comment = comment.concat('\n', value.name); + comment = comment.concat(': ', value.url); + }); + console.log(comment); + const issue = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Mass Rebuild Report", + labels: ['mass-rebuild'], + body: comment + }); + console.log(issue); + + bisect-failures: + if: github.repository_owner == 'fedora-llvm-team' + needs: + - check-for-rebuild + strategy: + max-parallel: 1 + fail-fast: false + matrix: + include: ${{ fromJson(needs.check-for-rebuild.outputs.regressions) }} + uses: ./.github/workflows/mass-rebuild-bisect.yml + with: + pkg: ${{ matrix.name }} diff --git a/.github/workflows/mass-rebuild-runner.yml b/.github/workflows/mass-rebuild-runner.yml new file mode 100644 index 00000000..adda4e60 --- /dev/null +++ b/.github/workflows/mass-rebuild-runner.yml @@ -0,0 +1,37 @@ +name: "Mass Rebuild Runner" + +on: + schedule: + # Run on the first of every month. + - cron: 30 1 1 * * + workflow_dispatch: + +permissions: + contents: read + +jobs: + start-rebuild: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + container: + image: "registry.fedoraproject.org/fedora:41" + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts/rebuilder.py + sparse-checkout-cone-mode: false + + - name: Setup Copr config file + env: + # You need to have those secrets in your repo. + # See also: https://copr.fedorainfracloud.org/api/. + COPR_CONFIG_FILE: ${{ secrets.COPR_CONFIG }} + run: | + mkdir -p ~/.config + printf "$COPR_CONFIG_FILE" > ~/.config/copr + + - name: Start rebuild + run: | + sudo dnf install -y python3-dnf python3-copr + python3 scripts/rebuilder.py rebuild diff --git a/.github/workflows/python-format-and-tests.yml b/.github/workflows/python-format-and-tests.yml index e010fd61..016b6dab 100644 --- a/.github/workflows/python-format-and-tests.yml +++ b/.github/workflows/python-format-and-tests.yml @@ -19,6 +19,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/prepare-python + with: + use-system-python: true - name: Run pytest with coverage shell: bash -e {0} env: diff --git a/Containerfile b/Containerfile new file mode 100644 index 00000000..381ba70f --- /dev/null +++ b/Containerfile @@ -0,0 +1,19 @@ +FROM registry.fedoraproject.org/fedora:41 + +ENV LLVM_SYSROOT=/opt/llvm \ + AR=llvm-ar \ + RANLIB=llvm-ranlib + +RUN dnf -y copr enable tstellar/fedora-clang-default-cc + +RUN dnf -y install jq cmake ninja-build git binutils-devel clang fedora-clang-default-cc rpmbuild ccache + +WORKDIR /root + +RUN git clone https://github.com/llvm/llvm-project + +WORKDIR /root/llvm-project + +ADD bisect.sh git-bisect-script.sh . + +COPY --from=ghcr.io/llvm/ci-ubuntu-22.04:1734145213 $LLVM_SYSROOT $LLVM_SYSROOT diff --git a/README.adoc b/README.adoc index 8fa316d5..977cbbde 100644 --- a/README.adoc +++ b/README.adoc @@ -193,6 +193,38 @@ $ cd llvm $ make snapshot-rpm ---- += README +:icons: font + You might need to install missing dependencies. The build process itself probably takes quite some time. You're going to find `results/YYYYMMDD/snapshot-rpm.log` with logging everything from this makefile target. + +== Mass Rebuilds == + +This repository uses GitHub Actions to periodically perform rebuilds of selected +Fedora packages. Once a mass rebuild is complete there is also automation +that will create a new issue with the results of the rebuild. + +The rebuild process will attempt to automatically bisect the failures to a specific upstream +LLVM commit. + +The rebuild can be started manually using the rebuilder.py script in +`.github/workflows/` + +[source,console] +--- +$ python3 rebuilder.py rebuild +--- + +You can also view the regression report once the rebuild is complete using +the same script. + +[source,console] +--- +$ python3 rebuilder.py get-regressions --start-date= +--- + +The start date should be the day the rebuild was started (In reality +it can be any date between when the last rebuild ended and the +new rebuild began). diff --git a/pytest.ini b/pytest.ini index b4270b29..94e77da3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,7 +11,7 @@ addopts: --doctest-modules --durations=0 -s ; Directories to search for tests when no files or directories are given on the ; command line -testpaths: snapshot_manager +testpaths: snapshot_manager scripts ; Each line specifies a pattern for warnings.filterwarnings. Processed after ; -W/--pythonwarnings. diff --git a/scripts/bisect.sh b/scripts/bisect.sh new file mode 100644 index 00000000..3c82cae2 --- /dev/null +++ b/scripts/bisect.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -ex + +function get_clang_commit { + buildid=$1 + pkg=$2 + + curl "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/fedora-41-clang-20/fedora-41-x86_64/0$buildid-$pkg/root.log.gz" | gunzip | grep -o 'clang[[:space:]]\+x86_64[[:space:]]\+[0-9a-g~pre.]\+' | cut -d 'g' -f 3 +} + + +pkg_or_buildid=$1 + +if echo $pkg_or_buildid | grep '^[0-9]\+'; then + buildid=$pkg_or_buildid + read -r pkg last_success_id <<<$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r '[.builds.latest.source_package.name,.builds.latest_succeeded.id] | join(" ")') +else + pkg=$pkg_or_buildid +fi + +read -r buildid last_success_id <<<$(curl -X 'GET' \ + "https://copr.fedorainfracloud.org/api_3/package/?ownername=%40fedora-llvm-team&projectname=fedora-41-clang-20&packagename=$pkg&with_latest_build=true&with_latest_succeeded_build=true" \ + -H 'accept: application/json' | jq -r '[.builds.latest.id,.builds.latest_succeeded.id] | join(" ")' ) + + +good_commit=llvmorg-20-init +bad_commit=origin/main + +good_commit=$(get_clang_commit $last_success_id $pkg) +bad_commit=$(get_clang_commit $buildid $pkg) + +srpm_url=$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r .source_package.url) +curl -O -L $srpm_url +srpm_name=$(basename $srpm_url) + +dnf builddep -y $srpm_name + + +git bisect start +git bisect good $good_commit +git bisect bad $bad_commit + +cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD=Native -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang + +git bisect run ./git-bisect-script.sh $srpm_name diff --git a/scripts/git-bisect-script.sh b/scripts/git-bisect-script.sh new file mode 100755 index 00000000..ef21d305 --- /dev/null +++ b/scripts/git-bisect-script.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +srpm_name=$1 + +ninja -C build install-clang install-clang-resource-headers install-LLVMgold install-llvm-ar install-llvm-ranlib + +rpmbuild -rb $srpm_name diff --git a/scripts/rebuilder.py b/scripts/rebuilder.py new file mode 100644 index 00000000..892d5f29 --- /dev/null +++ b/scripts/rebuilder.py @@ -0,0 +1,391 @@ +import argparse +import datetime +import json +import logging +import re +import sys +from typing import Set + +import copr.v3 +import dnf +import hawkey +from munch import Munch + + +class CoprBuild(Munch): + pass + + def is_in_progress(self) -> bool: + return self.state not in [ + "succeeded", + "forked", + "skipped", + "failed", + "canceled", + ] + + +class CoprPkg(Munch): + + @classmethod + def get_packages_from_copr( + cls, project_owner: str, project_name: str, copr_client: copr.v3.Client + ) -> list["CoprPkg"]: + return [ + CoprPkg(p) + for p in copr_client.package_proxy.get_list( + project_owner, + project_name, + with_latest_succeeded_build=True, + with_latest_build=True, + ) + ] + + def get_build(self, name: str) -> CoprBuild: + if "builds" not in self: + return None + if name not in self.builds: + return None + build = self.builds[name] + if not build: + return None + return CoprBuild(build) + + @property + def latest(self) -> CoprBuild: + return self.get_build("latest") + + @property + def latest_succeeded(self) -> CoprBuild: + return self.get_build("latest_succeeded") + + +def load_tests(loader, tests, ignore): + """We want unittest to pick up all of our doctests + + See https://docs.python.org/3/library/unittest.html#load-tests-protocol + See https://stackoverflow.com/a/27171468 + """ + import doctest + + tests.addTests(doctest.DocTestSuite()) + return tests + + +def filter_llvm_pkgs(pkgs: set[str]) -> set[str]: + """Filters out LLVM packages and returns the rest. + + Args: + pkgs (set[str]): List of package names + + Returns: + set[str]: List of package names without LLVM packages + + Example: + + >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'} + >>> filtered=list(filter_llvm_pkgs(pkgs)) + >>> filtered.sort() + >>> print(filtered) + ['firefox', 'libreoffice'] + + """ + llvm_pkgs = { + "llvm", + "clang", + "llvm-bolt", + "libomp", + "compiler-rt", + "lld", + "lldb", + "polly", + "libcxx", + "libclc", + "flang", + "mlir", + } + llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$" + return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)} + + +def get_exclusions() -> set[str]: + """ + This returns a list of packages we don't want to test. + """ + return set() + + +def get_pkgs(exclusions: set[str]) -> set[set]: + base = dnf.Base() + conf = base.conf + for c in "AppStream", "BaseOS", "CRB", "Extras": + base.repos.add_new_repo( + f"{c}-source", + conf, + baseurl=[ + f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/" + ], + ) + repos = base.repos.get_matching("*") + repos.disable() + repos = base.repos.get_matching("*-source*") + repos.enable() + + base.fill_sack() + q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES) + q = q.available() + q = q.filter(requires=["clang", "gcc", "gcc-c++"]) + pkgs = [p.name for p in list(q)] + return filter_llvm_pkgs(set(pkgs)) - exclusions + + +def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]: + """Returns the list of packages that should be built in the next rebuild. + It will select all the packages that built successfully during the last + rebuild. + + Args: + pkgs (set[str]): A list of every package that should be considered for + the rebuild. + copr_pkgs (list[dist]): A list containing the latest build results from + the COPR project. + + Returns: + set[str]: List of packages that should be rebuilt. + + Example: + + >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } } + >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } } + >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } } + >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } } + >>> pkgs = { "b", "c", "d"} + >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]] + >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs) + >>> print(rebuild_pkgs) + {'d'} + """ + + for p in copr_pkgs: + if p.name not in pkgs: + continue + if not p.latest_succeeded: + pkgs.discard(p.name) + continue + if p.latest.id != p.latest_succeeded.id: + pkgs.discard(p.name) + return pkgs + + +def get_monthly_rebuild_regressions( + project_owner: str, + project_name: str, + start_time: datetime.datetime, + copr_pkgs: list[CoprPkg], +) -> set[str]: + """Returns the list of packages that failed to build in the most recent + rebuild, but built successfully in the previous rebuild. + + Args: + start_time (datetime.datetime): The start time of the most recent mass + rebuild. This needs to be a time + before the most recent mass rebuild + and after the previous one. + copr_pkgs (list[dict]): List of built packages for the COPR project. + + Returns: + set[str]: List of packages that regressed in the most recent rebuilt. + + Example: + + >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321 } , "latest_succeeded" : None } } + >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : None } } + >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } } + >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } } + >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1 } , "latest_succeeded" : { "id" : 1 } } } + >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } } + >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]] + >>> project_owner = "@fedora-llvm-team" + >>> project_name = "fedora41-clang-20" + >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs) + >>> print(regressions) + [{'name': 'f', 'url': 'https://copr.fedorainfracloud.org/coprs/@fedora-llvm-team/fedora41-clang-20/build/2/'}] + + """ + pkgs = [] + for p in copr_pkgs: + if not p.latest: + continue + + # Don't report regressions if there are still builds in progress + if p.latest.is_in_progress(): + continue + + if not p.latest_succeeded: + continue + if p.latest.id == p.latest_succeeded.id: + continue + # latest is a successful build, but this doesn't mean it failed. + # It could be in progress. + if p.latest.state != "failed": + continue + if int(p.latest.submitted_on) < start_time.timestamp(): + continue + pkgs.append( + { + "name": p.name, + "url": f"https://copr.fedorainfracloud.org/coprs/{project_owner}/{project_name}/build/{p.latest.id}/", + } + ) + return pkgs + + +def start_rebuild( + project_owner: str, + project_name: str, + copr_client: copr.v3.Client, + pkgs: set[str], + snapshot_project_name: str, +): + + print("START", pkgs, "END") + # Update the rebuild project to use the latest snapshot + copr_client.project_proxy.edit( + project_owner, + project_name, + additional_repos=[ + "copr://tstellar/fedora-clang-default-cc", + f"copr://@fedora-llvm-team/{snapshot_project_name}", + ], + ) + + buildopts = { + "background": True, + } + logging.info("Rebuilding", len(pkgs), "packages") + for p in pkgs: + logging.info("Rebuild", p) + copr_client.build_proxy.create_from_distgit( + project_owner, project_name, p, "f41", buildopts=buildopts + ) + + +def select_snapshot_project( + copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14 +) -> str | None: + project_owner = "@fedora-llvm-team" + for i in range(max_lookback_days): + chroots = set() + day = datetime.date.today() - datetime.timedelta(days=i) + project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d") + logging.info("Trying:", project_name) + try: + p = copr_client.project_proxy.get(project_owner, project_name) + if not p: + continue + pkgs = copr_client.build_proxy.get_list( + project_owner, project_name, "llvm", status="succeeded" + ) + for pkg in pkgs: + chroots.update(pkg["chroots"]) + + logging.info(project_name, chroots) + if all(t in chroots for t in target_chroots): + logging.info("PASS", project_name) + return project_name + except: + continue + logging.warn("FAIL") + return None + + +def create_new_project( + project_owner: str, + project_name: str, + copr_client: copr.v3.Client, + target_chroots: list[str], +): + copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots) + for c in target_chroots: + copr_client.project_chroot_proxy.edit( + project_owner, + project_name, + c, + additional_packages=["fedora-clang-default-cc"], + with_opts=["toolchain_clang", "clang_lto"], + ) + + +def main(): + + logging.basicConfig(filename="rebuilder.log", level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument( + "command", + type=str, + choices=[ + "rebuild", + "get-regressions", + "get-snapshot-date", + "rebuild-in-progress", + ], + ) + parser.add_argument( + "--start-date", type=str, help="Any ISO date format is accepted" + ) + + args = parser.parse_args() + copr_client = copr.v3.Client.create_from_config_file() + + os_name = "fedora-41" + clang_version = "20" + target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"] + target_chroots = [f"{os_name}-{a}" for a in target_arches] + project_owner = "@fedora-llvm-team" + project_name = f"{os_name}-clang-{clang_version}" + + if args.command == "rebuild": + exclusions = get_exclusions() + pkgs = get_pkgs(exclusions) + print(pkgs) + try: + copr_client.project_proxy.get(project_owner, project_name) + copr_pkgs = CoprPkg.get_packages_from_copr( + project_owner, project_name, copr_client + ) + pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs) + except: + create_new_project(project_owner, project_name, copr_client, target_chroots) + snapshot_project = select_snapshot_project(copr_client, target_chroots) + start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project) + elif args.command == "get-regressions": + start_time = datetime.datetime.fromisoformat(args.start_date) + copr_pkgs = CoprPkg.get_packages_from_copr( + project_owner, project_name, copr_client + ) + pkg_failures = get_monthly_rebuild_regressions( + project_owner, project_name, start_time, copr_pkgs + ) + print(json.dumps(pkg_failures)) + elif args.command == "get-snapshot-date": + project = copr_client.project_proxy.get(project_owner, project_name) + for repo in project["additional_repos"]: + match = re.match( + r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo + ) + if match: + print(datetime.datetime.fromisoformat(match.group(1)).isoformat()) + return + elif args.command == "rebuild-in-progress": + for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[ + "packages" + ]: + for c in pkg["chroots"]: + build = CoprBuild(pkg["chroots"][c]) + if build.is_in_progress(): + sys.exit(0) + sys.exit(1) + + +if __name__ == "__main__": + main()