diff --git a/.github/actions/prepare-python/action.yml b/.github/actions/prepare-python/action.yml index d947c7f4..7df003bd 100644 --- a/.github/actions/prepare-python/action.yml +++ b/.github/actions/prepare-python/action.yml @@ -7,11 +7,16 @@ inputs: description: "Where this project was checked out" required: false default: "." + use-system-python: + description: "Set to true if you want to use the python installation from the OS" + required: false + default: false runs: using: "composite" steps: - name: "Setup python" uses: actions/setup-python@v5 + if: "${{ inputs.use-system-python != 'true' }}" with: python-version: 3.12 cache: 'pip' @@ -24,4 +29,14 @@ runs: - name: "Install python dependencies for project" shell: bash - run: pip install -r ${{ inputs.checkout-path }}/requirements.txt + run: | + pip install -r ${{ inputs.checkout-path }}/requirements.txt + + # The dnf module cannot be installed by pip, so it's only possible to use + # it when using the system pyhton interpreter. + - name: Install DNF module + if: "${{ inputs.use-system-python == 'true' }}" + shell: bash + run: | + sudo apt-get update + sudo apt-get install python3-dnf diff --git a/.github/workflows/build-reproducer-container.yml b/.github/workflows/build-reproducer-container.yml new file mode 100644 index 00000000..02aa539f --- /dev/null +++ b/.github/workflows/build-reproducer-container.yml @@ -0,0 +1,27 @@ +name: "Build Reproducer Container" + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + build-reproducer-container: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - uses: actions/checkout@v4 + - name: Build Container + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get -y update + sudo apt-get -y install podman + podman build -t ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer -f Containerfile scripts/ + podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io + podman push ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer diff --git a/.github/workflows/mass-rebuild-bisect.yml b/.github/workflows/mass-rebuild-bisect.yml new file mode 100644 index 00000000..039d04d7 --- /dev/null +++ b/.github/workflows/mass-rebuild-bisect.yml @@ -0,0 +1,40 @@ +name: Mass Rebuild Bisect + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + pkg: + description: "The name of the Fedora package build that you want to bisect." + required: true + type: string + workflow_call: + inputs: + pkg: + description: "The name of the Fedora package build that you want to bisect." + required: true + type: string + +jobs: + mass-rebuild-bisect: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + container: + image: "ghcr.io/${{ github.repository_owner }}/llvm-snapshots-reproducer" + steps: + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + max-size: 8G + key: bisect + - working-directory: /root/llvm-project/ + run: | + git fetch origin + bash bisect.sh ${{ inputs.pkg }} + + - if: always() + working-directory: /root/llvm-project + run: | + git bisect log diff --git a/.github/workflows/mass-rebuild-reporter.yml b/.github/workflows/mass-rebuild-reporter.yml new file mode 100644 index 00000000..bd5df5e1 --- /dev/null +++ b/.github/workflows/mass-rebuild-reporter.yml @@ -0,0 +1,115 @@ +name: "Mass Rebuild Reporter" + +on: + schedule: + # Hourly at minute 40, e.g. 2024-12-18 00:40:00 + - cron: "40 * * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check-for-rebuild: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + permissions: + issues: write + container: + image: "registry.fedoraproject.org/fedora:41" + outputs: + regressions: ${{ steps.regressions.outputs.REGRESSIONS }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts/rebuilder.py + sparse-checkout-cone-mode: false + + + - name: Check for last report + uses: actions/github-script@v7 + id: last-report + with: + result-encoding: string + script: | + const issues = await github.rest.search.issuesAndPullRequests({ + q: "label:mass-rebuild+is:issue", + sort: "created", + order: "desc", + per_page: 1 + }); + + console.log(issues) + if (issues.data.total_count == 0) + return 0; + const issue = issues.data.items[0]; + console.log(issue); + return issue.created_at + + - name: Check if a new rebuild has completed + id: new-rebuild + run: | + sudo dnf install -y python3-dnf python3-copr + if python3 scripts/rebuilder.py rebuild-in-progress; then + echo "completed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + last_rebuild=$(date +%s -d "${{ steps.last-report.outputs.result }}") + current_snapshot=$(date +%s -d "$(python3 scripts/rebuilder.py get-snapshot-date)") + + echo "last_rebuild: $last_rebuild current_snapshot: $current_snapshot" + + if [ $last_rebuild -gt $current_snapshot ]; then + echo "completed=false" >> $GITHUB_OUTPUT + else + echo "completed=true" >> $GITHUB_OUTPUT + fi + + - name: Collect Regressions + if: steps.new-rebuild.outputs.completed == 'true' + id: regressions + run: | + python3 scripts/rebuilder.py get-regressions --start-date ${{ steps.last-report.outputs.result }} > regressions + echo "REGRESSIONS=$(cat regressions)" >> $GITHUB_OUTPUT + + - name: Create Report + if: steps.new-rebuild.outputs.completed == 'true' + uses: actions/github-script@v7 + env: + REGRESSIONS: ${{ steps.regressions.outputs.REGRESSIONS }} + with: + script: | + var fs = require('fs'); + const regressions = await JSON.parse(fs.readFileSync('./regressions')); + comment = "During the last mass rebuild, some packages failed:\n"; + console.log(regressions); + if (regressions.length == 0) + return; + regressions.forEach(function(value){ + comment = comment.concat('\n', value.name); + comment = comment.concat(': ', value.url); + }); + console.log(comment); + const issue = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Mass Rebuild Report", + labels: ['mass-rebuild'], + body: comment + }); + console.log(issue); + + bisect-failures: + if: github.repository_owner == 'fedora-llvm-team' + needs: + - check-for-rebuild + strategy: + max-parallel: 1 + fail-fast: false + matrix: + include: ${{ fromJson(needs.check-for-rebuild.outputs.regressions) }} + uses: ./.github/workflows/mass-rebuild-bisect.yml + with: + pkg: ${{ matrix.name }} diff --git a/.github/workflows/mass-rebuild-runner.yml b/.github/workflows/mass-rebuild-runner.yml new file mode 100644 index 00000000..adda4e60 --- /dev/null +++ b/.github/workflows/mass-rebuild-runner.yml @@ -0,0 +1,37 @@ +name: "Mass Rebuild Runner" + +on: + schedule: + # Run on the first of every month. + - cron: 30 1 1 * * + workflow_dispatch: + +permissions: + contents: read + +jobs: + start-rebuild: + if: github.repository_owner == 'fedora-llvm-team' + runs-on: ubuntu-24.04 + container: + image: "registry.fedoraproject.org/fedora:41" + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts/rebuilder.py + sparse-checkout-cone-mode: false + + - name: Setup Copr config file + env: + # You need to have those secrets in your repo. + # See also: https://copr.fedorainfracloud.org/api/. + COPR_CONFIG_FILE: ${{ secrets.COPR_CONFIG }} + run: | + mkdir -p ~/.config + printf "$COPR_CONFIG_FILE" > ~/.config/copr + + - name: Start rebuild + run: | + sudo dnf install -y python3-dnf python3-copr + python3 scripts/rebuilder.py rebuild diff --git a/.github/workflows/python-format-and-tests.yml b/.github/workflows/python-format-and-tests.yml index e010fd61..016b6dab 100644 --- a/.github/workflows/python-format-and-tests.yml +++ b/.github/workflows/python-format-and-tests.yml @@ -19,6 +19,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/prepare-python + with: + use-system-python: true - name: Run pytest with coverage shell: bash -e {0} env: diff --git a/Containerfile b/Containerfile new file mode 100644 index 00000000..381ba70f --- /dev/null +++ b/Containerfile @@ -0,0 +1,19 @@ +FROM registry.fedoraproject.org/fedora:41 + +ENV LLVM_SYSROOT=/opt/llvm \ + AR=llvm-ar \ + RANLIB=llvm-ranlib + +RUN dnf -y copr enable tstellar/fedora-clang-default-cc + +RUN dnf -y install jq cmake ninja-build git binutils-devel clang fedora-clang-default-cc rpmbuild ccache + +WORKDIR /root + +RUN git clone https://github.com/llvm/llvm-project + +WORKDIR /root/llvm-project + +ADD bisect.sh git-bisect-script.sh . + +COPY --from=ghcr.io/llvm/ci-ubuntu-22.04:1734145213 $LLVM_SYSROOT $LLVM_SYSROOT diff --git a/README.adoc b/README.adoc index 8fa316d5..977cbbde 100644 --- a/README.adoc +++ b/README.adoc @@ -193,6 +193,38 @@ $ cd llvm $ make snapshot-rpm ---- += README +:icons: font + You might need to install missing dependencies. The build process itself probably takes quite some time. You're going to find `results/YYYYMMDD/snapshot-rpm.log` with logging everything from this makefile target. + +== Mass Rebuilds == + +This repository uses GitHub Actions to periodically perform rebuilds of selected +Fedora packages. Once a mass rebuild is complete there is also automation +that will create a new issue with the results of the rebuild. + +The rebuild process will attempt to automatically bisect the failures to a specific upstream +LLVM commit. + +The rebuild can be started manually using the rebuilder.py script in +`.github/workflows/` + +[source,console] +--- +$ python3 rebuilder.py rebuild +--- + +You can also view the regression report once the rebuild is complete using +the same script. + +[source,console] +--- +$ python3 rebuilder.py get-regressions --start-date= +--- + +The start date should be the day the rebuild was started (In reality +it can be any date between when the last rebuild ended and the +new rebuild began). diff --git a/pytest.ini b/pytest.ini index b4270b29..94e77da3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,7 +11,7 @@ addopts: --doctest-modules --durations=0 -s ; Directories to search for tests when no files or directories are given on the ; command line -testpaths: snapshot_manager +testpaths: snapshot_manager scripts ; Each line specifies a pattern for warnings.filterwarnings. Processed after ; -W/--pythonwarnings. diff --git a/scripts/bisect.sh b/scripts/bisect.sh new file mode 100644 index 00000000..3c82cae2 --- /dev/null +++ b/scripts/bisect.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -ex + +function get_clang_commit { + buildid=$1 + pkg=$2 + + curl "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/fedora-41-clang-20/fedora-41-x86_64/0$buildid-$pkg/root.log.gz" | gunzip | grep -o 'clang[[:space:]]\+x86_64[[:space:]]\+[0-9a-g~pre.]\+' | cut -d 'g' -f 3 +} + + +pkg_or_buildid=$1 + +if echo $pkg_or_buildid | grep '^[0-9]\+'; then + buildid=$pkg_or_buildid + read -r pkg last_success_id <<<$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r '[.builds.latest.source_package.name,.builds.latest_succeeded.id] | join(" ")') +else + pkg=$pkg_or_buildid +fi + +read -r buildid last_success_id <<<$(curl -X 'GET' \ + "https://copr.fedorainfracloud.org/api_3/package/?ownername=%40fedora-llvm-team&projectname=fedora-41-clang-20&packagename=$pkg&with_latest_build=true&with_latest_succeeded_build=true" \ + -H 'accept: application/json' | jq -r '[.builds.latest.id,.builds.latest_succeeded.id] | join(" ")' ) + + +good_commit=llvmorg-20-init +bad_commit=origin/main + +good_commit=$(get_clang_commit $last_success_id $pkg) +bad_commit=$(get_clang_commit $buildid $pkg) + +srpm_url=$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r .source_package.url) +curl -O -L $srpm_url +srpm_name=$(basename $srpm_url) + +dnf builddep -y $srpm_name + + +git bisect start +git bisect good $good_commit +git bisect bad $bad_commit + +cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD=Native -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang + +git bisect run ./git-bisect-script.sh $srpm_name diff --git a/scripts/git-bisect-script.sh b/scripts/git-bisect-script.sh new file mode 100755 index 00000000..ef21d305 --- /dev/null +++ b/scripts/git-bisect-script.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +srpm_name=$1 + +ninja -C build install-clang install-clang-resource-headers install-LLVMgold install-llvm-ar install-llvm-ranlib + +rpmbuild -rb $srpm_name diff --git a/scripts/rebuilder.py b/scripts/rebuilder.py new file mode 100644 index 00000000..892d5f29 --- /dev/null +++ b/scripts/rebuilder.py @@ -0,0 +1,391 @@ +import argparse +import datetime +import json +import logging +import re +import sys +from typing import Set + +import copr.v3 +import dnf +import hawkey +from munch import Munch + + +class CoprBuild(Munch): + pass + + def is_in_progress(self) -> bool: + return self.state not in [ + "succeeded", + "forked", + "skipped", + "failed", + "canceled", + ] + + +class CoprPkg(Munch): + + @classmethod + def get_packages_from_copr( + cls, project_owner: str, project_name: str, copr_client: copr.v3.Client + ) -> list["CoprPkg"]: + return [ + CoprPkg(p) + for p in copr_client.package_proxy.get_list( + project_owner, + project_name, + with_latest_succeeded_build=True, + with_latest_build=True, + ) + ] + + def get_build(self, name: str) -> CoprBuild: + if "builds" not in self: + return None + if name not in self.builds: + return None + build = self.builds[name] + if not build: + return None + return CoprBuild(build) + + @property + def latest(self) -> CoprBuild: + return self.get_build("latest") + + @property + def latest_succeeded(self) -> CoprBuild: + return self.get_build("latest_succeeded") + + +def load_tests(loader, tests, ignore): + """We want unittest to pick up all of our doctests + + See https://docs.python.org/3/library/unittest.html#load-tests-protocol + See https://stackoverflow.com/a/27171468 + """ + import doctest + + tests.addTests(doctest.DocTestSuite()) + return tests + + +def filter_llvm_pkgs(pkgs: set[str]) -> set[str]: + """Filters out LLVM packages and returns the rest. + + Args: + pkgs (set[str]): List of package names + + Returns: + set[str]: List of package names without LLVM packages + + Example: + + >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'} + >>> filtered=list(filter_llvm_pkgs(pkgs)) + >>> filtered.sort() + >>> print(filtered) + ['firefox', 'libreoffice'] + + """ + llvm_pkgs = { + "llvm", + "clang", + "llvm-bolt", + "libomp", + "compiler-rt", + "lld", + "lldb", + "polly", + "libcxx", + "libclc", + "flang", + "mlir", + } + llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$" + return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)} + + +def get_exclusions() -> set[str]: + """ + This returns a list of packages we don't want to test. + """ + return set() + + +def get_pkgs(exclusions: set[str]) -> set[set]: + base = dnf.Base() + conf = base.conf + for c in "AppStream", "BaseOS", "CRB", "Extras": + base.repos.add_new_repo( + f"{c}-source", + conf, + baseurl=[ + f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/" + ], + ) + repos = base.repos.get_matching("*") + repos.disable() + repos = base.repos.get_matching("*-source*") + repos.enable() + + base.fill_sack() + q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES) + q = q.available() + q = q.filter(requires=["clang", "gcc", "gcc-c++"]) + pkgs = [p.name for p in list(q)] + return filter_llvm_pkgs(set(pkgs)) - exclusions + + +def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]: + """Returns the list of packages that should be built in the next rebuild. + It will select all the packages that built successfully during the last + rebuild. + + Args: + pkgs (set[str]): A list of every package that should be considered for + the rebuild. + copr_pkgs (list[dist]): A list containing the latest build results from + the COPR project. + + Returns: + set[str]: List of packages that should be rebuilt. + + Example: + + >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } } + >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } } + >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } } + >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } } + >>> pkgs = { "b", "c", "d"} + >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]] + >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs) + >>> print(rebuild_pkgs) + {'d'} + """ + + for p in copr_pkgs: + if p.name not in pkgs: + continue + if not p.latest_succeeded: + pkgs.discard(p.name) + continue + if p.latest.id != p.latest_succeeded.id: + pkgs.discard(p.name) + return pkgs + + +def get_monthly_rebuild_regressions( + project_owner: str, + project_name: str, + start_time: datetime.datetime, + copr_pkgs: list[CoprPkg], +) -> set[str]: + """Returns the list of packages that failed to build in the most recent + rebuild, but built successfully in the previous rebuild. + + Args: + start_time (datetime.datetime): The start time of the most recent mass + rebuild. This needs to be a time + before the most recent mass rebuild + and after the previous one. + copr_pkgs (list[dict]): List of built packages for the COPR project. + + Returns: + set[str]: List of packages that regressed in the most recent rebuilt. + + Example: + + >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321 } , "latest_succeeded" : None } } + >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : None } } + >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } } + >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } } + >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1 } , "latest_succeeded" : { "id" : 1 } } } + >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } } + >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]] + >>> project_owner = "@fedora-llvm-team" + >>> project_name = "fedora41-clang-20" + >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs) + >>> print(regressions) + [{'name': 'f', 'url': 'https://copr.fedorainfracloud.org/coprs/@fedora-llvm-team/fedora41-clang-20/build/2/'}] + + """ + pkgs = [] + for p in copr_pkgs: + if not p.latest: + continue + + # Don't report regressions if there are still builds in progress + if p.latest.is_in_progress(): + continue + + if not p.latest_succeeded: + continue + if p.latest.id == p.latest_succeeded.id: + continue + # latest is a successful build, but this doesn't mean it failed. + # It could be in progress. + if p.latest.state != "failed": + continue + if int(p.latest.submitted_on) < start_time.timestamp(): + continue + pkgs.append( + { + "name": p.name, + "url": f"https://copr.fedorainfracloud.org/coprs/{project_owner}/{project_name}/build/{p.latest.id}/", + } + ) + return pkgs + + +def start_rebuild( + project_owner: str, + project_name: str, + copr_client: copr.v3.Client, + pkgs: set[str], + snapshot_project_name: str, +): + + print("START", pkgs, "END") + # Update the rebuild project to use the latest snapshot + copr_client.project_proxy.edit( + project_owner, + project_name, + additional_repos=[ + "copr://tstellar/fedora-clang-default-cc", + f"copr://@fedora-llvm-team/{snapshot_project_name}", + ], + ) + + buildopts = { + "background": True, + } + logging.info("Rebuilding", len(pkgs), "packages") + for p in pkgs: + logging.info("Rebuild", p) + copr_client.build_proxy.create_from_distgit( + project_owner, project_name, p, "f41", buildopts=buildopts + ) + + +def select_snapshot_project( + copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14 +) -> str | None: + project_owner = "@fedora-llvm-team" + for i in range(max_lookback_days): + chroots = set() + day = datetime.date.today() - datetime.timedelta(days=i) + project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d") + logging.info("Trying:", project_name) + try: + p = copr_client.project_proxy.get(project_owner, project_name) + if not p: + continue + pkgs = copr_client.build_proxy.get_list( + project_owner, project_name, "llvm", status="succeeded" + ) + for pkg in pkgs: + chroots.update(pkg["chroots"]) + + logging.info(project_name, chroots) + if all(t in chroots for t in target_chroots): + logging.info("PASS", project_name) + return project_name + except: + continue + logging.warn("FAIL") + return None + + +def create_new_project( + project_owner: str, + project_name: str, + copr_client: copr.v3.Client, + target_chroots: list[str], +): + copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots) + for c in target_chroots: + copr_client.project_chroot_proxy.edit( + project_owner, + project_name, + c, + additional_packages=["fedora-clang-default-cc"], + with_opts=["toolchain_clang", "clang_lto"], + ) + + +def main(): + + logging.basicConfig(filename="rebuilder.log", level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument( + "command", + type=str, + choices=[ + "rebuild", + "get-regressions", + "get-snapshot-date", + "rebuild-in-progress", + ], + ) + parser.add_argument( + "--start-date", type=str, help="Any ISO date format is accepted" + ) + + args = parser.parse_args() + copr_client = copr.v3.Client.create_from_config_file() + + os_name = "fedora-41" + clang_version = "20" + target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"] + target_chroots = [f"{os_name}-{a}" for a in target_arches] + project_owner = "@fedora-llvm-team" + project_name = f"{os_name}-clang-{clang_version}" + + if args.command == "rebuild": + exclusions = get_exclusions() + pkgs = get_pkgs(exclusions) + print(pkgs) + try: + copr_client.project_proxy.get(project_owner, project_name) + copr_pkgs = CoprPkg.get_packages_from_copr( + project_owner, project_name, copr_client + ) + pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs) + except: + create_new_project(project_owner, project_name, copr_client, target_chroots) + snapshot_project = select_snapshot_project(copr_client, target_chroots) + start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project) + elif args.command == "get-regressions": + start_time = datetime.datetime.fromisoformat(args.start_date) + copr_pkgs = CoprPkg.get_packages_from_copr( + project_owner, project_name, copr_client + ) + pkg_failures = get_monthly_rebuild_regressions( + project_owner, project_name, start_time, copr_pkgs + ) + print(json.dumps(pkg_failures)) + elif args.command == "get-snapshot-date": + project = copr_client.project_proxy.get(project_owner, project_name) + for repo in project["additional_repos"]: + match = re.match( + r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo + ) + if match: + print(datetime.datetime.fromisoformat(match.group(1)).isoformat()) + return + elif args.command == "rebuild-in-progress": + for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[ + "packages" + ]: + for c in pkg["chroots"]: + build = CoprBuild(pkg["chroots"][c]) + if build.is_in_progress(): + sys.exit(0) + sys.exit(1) + + +if __name__ == "__main__": + main()