From 0fbce5a21fe67cf83f7a8a5bca407b8d9d785ade Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 2 Jul 2024 22:03:18 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 1 + .ci/assets/bindings/.gitkeep | 0 .ci/assets/ci_constraints.txt | 7 + .ci/scripts/changelog.py | 70 ------- .ci/scripts/cherrypick.sh | 69 ------- .ci/scripts/tweet.py | 16 -- .ci/scripts/update_ci_branches.py | 25 --- .ci/scripts/upper_bound.py | 24 --- .github/template_gitref | 2 +- .github/workflows/build.yml | 55 +++++- ...ython_client.sh => build_python_client.sh} | 33 +--- .../workflows/scripts/build_ruby_client.sh | 35 ++++ .../scripts/create_release_from_tag.sh | 11 -- .../workflows/scripts/install_ruby_client.sh | 42 ----- .github/workflows/scripts/release.py | 177 ------------------ .github/workflows/scripts/script.sh | 89 +++++---- .github/workflows/scripts/update_ci.sh | 31 --- .github/workflows/test.yml | 64 +++---- CHANGES/.TEMPLATE.rst | 47 ----- template_config.yml | 3 +- 20 files changed, 190 insertions(+), 611 deletions(-) delete mode 100644 .ci/assets/bindings/.gitkeep create mode 100644 .ci/assets/ci_constraints.txt delete mode 100755 .ci/scripts/changelog.py delete mode 100755 .ci/scripts/cherrypick.sh delete mode 100755 .ci/scripts/tweet.py delete mode 100755 .ci/scripts/update_ci_branches.py delete mode 100755 .ci/scripts/upper_bound.py rename .github/workflows/scripts/{install_python_client.sh => build_python_client.sh} (51%) create mode 100755 .github/workflows/scripts/build_ruby_client.sh delete mode 100755 .github/workflows/scripts/create_release_from_tag.sh delete mode 100755 .github/workflows/scripts/install_ruby_client.sh delete mode 100755 .github/workflows/scripts/release.py delete mode 100755 .github/workflows/scripts/update_ci.sh delete mode 100644 CHANGES/.TEMPLATE.rst diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 338ff6974..bad75f3bb 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -22,6 +22,7 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/assets/bindings/.gitkeep b/.ci/assets/bindings/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 000000000..2617a4089 --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,7 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* + + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py deleted file mode 100755 index f8d276eba..000000000 --- a/.ci/scripts/changelog.py +++ /dev/null @@ -1,70 +0,0 @@ -import re -import os -import requests -from packaging.version import Version -from git import Repo - -repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp_rpm.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] -branches.sort(key=lambda ver: Version(ver), reverse=True) - - -def get_changelog(branch): - """ - Get changelog file for a given branch. - - """ - return requests.get( - f"https://raw.githubusercontent.com/pulp/pulp_rpm/{branch}/CHANGES.rst" - ).text - - -def get_changelog_releases(changelog): - """ - Get all versions in changelog. - - """ - versions = re.findall( - r"([0-9]+)\.([0-9]+)\.([0-9][0-9ab]*) \([0-9]{4}-[0-9]{2}-[0-9]{2}\)", changelog - ) - return {".".join(v) for v in versions} - - -def get_changelog_entry(changelog, version): - """ - Get changelog entry for a given version. - - """ - entries = changelog.split(f"{version} (")[1].split("=====\n") - header = f"{version} ({entries[0]}=====\n" - text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1]) - return header + text + "\n\n\n" - - -main_changelog = get_changelog("main") -main_entries = get_changelog_releases(main_changelog) -entries_list = list(main_entries) -to_add = {} -for branch in branches: - changelog = get_changelog(branch) - entries = get_changelog_releases(changelog) - for entry in entries.difference(main_entries): - description = get_changelog_entry(changelog, entry) - entries_list.append(entry) - print(description) - to_add[entry] = description - -entries_list.sort(key=lambda ver: Version(ver), reverse=True) -for version in sorted(to_add, key=lambda ver: Version(ver)): - next_version = entries_list[entries_list.index(version) + 1] - new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version] - new_changelog = new_changelog + f"{next_version} (" - new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1] - main_changelog = new_changelog - -with open("CHANGES.rst", "w") as f: - f.write(main_changelog) - -if to_add: - repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst") diff --git a/.ci/scripts/cherrypick.sh b/.ci/scripts/cherrypick.sh deleted file mode 100755 index 76346b788..000000000 --- a/.ci/scripts/cherrypick.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_rpm' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -e - -if [ ! -d CHANGES ]; then - echo "Error: no CHANGES directory detected. This script must be run from the project root." - exit 1 -fi - -if [ $# -lt 3 ] -then - echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]" - echo " ex: .ci/scripts/cherrypick.sh abcd1234 1234 4567" - echo "" - echo "Note: make sure you are on a fork of the release branch before running this script." - exit -fi - -commit="$(git rev-parse $1)" -issue="$2" -backport="$3" -commit_message=$(git log --format=%B -n 1 $commit) - -if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q "\[noissue\]" -then - if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q -E "(fixes|closes).*#$issue" - then - echo "Error: issue $issue not detected in commit message." && exit 1 - fi -fi - -if [ "$4" = "--continue" ] -then - echo "Continue after manually resolving conflicts..." -elif [ "$4" = "" ] -then - if ! git cherry-pick --no-commit "$commit" - then - echo "Please resolve and add merge conflicts and restart this command with appended '--continue'." - exit 1 - fi -else - exit 1 -fi - -for file in $(find CHANGES -name "$issue.*") -do - newfile="${file/$issue/$backport}" - git mv "$file" "$newfile" - sed -i -e "\$a (backported from #$issue)" "$newfile" - git add "$newfile" -done - -commit_message="$(printf "$commit_message" | sed -E 's/(fixes|closes)/backports/i')" -commit_message="$commit_message - -fixes #$backport - -(cherry picked from commit $commit)" -git commit -m "$commit_message" - -printf "\nSuccessfully backported commit $1.\n" diff --git a/.ci/scripts/tweet.py b/.ci/scripts/tweet.py deleted file mode 100755 index 9959bcc15..000000000 --- a/.ci/scripts/tweet.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import sys -from tweepy import Client - -release_version = sys.argv[1] -if release_version.endswith(".0"): - client = Client( - consumer_key=os.getenv("TWITTER_API_KEY"), - consumer_secret=os.getenv("TWITTER_API_KEY_SECRET"), - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - ) - link = "https://docs.pulpproject.org/pulp_rpm/changes.html" - msg = f"pulp_rpm-{release_version} - Check out for more details: {link}" - release_msg = f"Hey! We've just released {msg}" - client.create_tweet(text=release_msg) diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py deleted file mode 100755 index 1ba9f49e7..000000000 --- a/.ci/scripts/update_ci_branches.py +++ /dev/null @@ -1,25 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_rpm' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import os -import sys -import requests - -branches = sys.argv[1:] - -headers = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github.v3+json", -} - -github_api = "https://api.github.com" -workflow_path = "/actions/workflows/update_ci.yml/dispatches" -url = f"{github_api}/repos/pulp/pulp_rpm{workflow_path}" - -for branch in branches: - print(f"Updating {branch}") - requests.post(url, headers=headers, json={"ref": branch}) diff --git a/.ci/scripts/upper_bound.py b/.ci/scripts/upper_bound.py deleted file mode 100755 index 64d4bbbe6..000000000 --- a/.ci/scripts/upper_bound.py +++ /dev/null @@ -1,24 +0,0 @@ -import warnings -from pkg_resources import Requirement - -packages = [] - -try: - with open("requirements.txt", "r") as fd: - for line in fd.readlines(): - if not line.startswith("#"): - req = Requirement.parse(line) - spec = str(req.specs) - if "~=" in spec: - warnings.warn(f"Please avoid using ~= on {req.name}") - continue - if len(req.specs) < 2 and "==" not in spec and "<" not in spec: - packages.append(req.name) -except FileNotFoundError: - # skip this test for plugins that don't use a requirements.txt - pass - -if packages: - raise RuntimeError( - "The following packages are missing upper bound: {}".format(", ".join(packages)) - ) diff --git a/.github/template_gitref b/.github/template_gitref index d98f4b245..3a1fb7d1d 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-342-g13e7dc5 +2021.08.26-348-gb87ed3b diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20d633ba8..eb6173e2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,13 +23,18 @@ jobs: with: fetch-depth: 1 path: "pulp_rpm" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel + pip install packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | @@ -37,7 +42,7 @@ jobs: twine check dist/* - name: "Install built packages" run: | - pip install dist/pulp_rpm-*-py3-none-any.whl + pip install dist/pulp_rpm-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt - name: "Generate api specs" run: | pulpcore-manager openapi --file "api.json" @@ -60,3 +65,49 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true + - name: "Build Python bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + pulp_rpm/rpm-python-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + pulp_rpm/rpm-python-client-docs.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Ruby bindings packages" + run: | + .github/workflows/scripts/build_ruby_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v4" + with: + name: "ruby-client.tar" + path: | + pulp_rpm/rpm-ruby-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/build_python_client.sh similarity index 51% rename from .github/workflows/scripts/install_python_client.sh rename to .github/workflows/scripts/build_python_client.sh index 10472b507..372aeb839 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -12,36 +15,20 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "rpm" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - pushd ../pulp-openapi-generator -rm -rf pulp_rpm-client +rm -rf "pulp_rpm-client" -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=rpm" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_rpm python "$VERSION" -else - ./generate.sh pulp_rpm python "$VERSION" -fi +# We need to copy that over to be visible in the container. +cp "../pulp_rpm/rpm-api.json" . +./gen-client.sh "rpm-api.json" "rpm" python "pulp_rpm" pushd pulp_rpm-client python setup.py sdist bdist_wheel --python-tag py3 -twine check "dist/pulp_rpm_client-$VERSION-py3-none-any.whl" -twine check "dist/pulp_rpm-client-$VERSION.tar.gz" +twine check "dist/pulp_rpm_client-"*"-py3-none-any.whl" +twine check "dist/pulp_rpm-client-"*".tar.gz" -cmd_prefix pip3 install "/root/pulp-openapi-generator/pulp_rpm-client/dist/pulp_rpm_client-${VERSION}-py3-none-any.whl" -tar cvf ../../pulp_rpm/rpm-python-client.tar ./dist +tar cvf "../../pulp_rpm/rpm-python-client.tar" ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; find ./docs/* -exec sed -i 's/README//g' {} \; diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 000000000..651170199 --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_rpm' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_rpm-client" + +# We need to copy that over to be visible in the container. +#cp "../pulp_rpm/rpm-api.json" . +#./gen-client.sh "rpm-api.json" "rpm" ruby "pulp_rpm" + +# ------------- +# The generator still needs to have it called api.json at this time... +cp "../pulp_rpm/api.json" . +./gen-client.sh "api.json" "rpm" ruby "pulp_rpm" +# ------------- + +pushd pulp_rpm-client +gem build pulp_rpm_client +tar cvf "../../pulp_rpm/rpm-ruby-client.tar" "./pulp_rpm_client-"*".gem" +popd +popd diff --git a/.github/workflows/scripts/create_release_from_tag.sh b/.github/workflows/scripts/create_release_from_tag.sh deleted file mode 100755 index 4de7777d7..000000000 --- a/.github/workflows/scripts/create_release_from_tag.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -eu - -curl -s -X "POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases" \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "tag_name": "$1", - "name": "$1" -} -EOF diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 938ed2ca5..000000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_rpm' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "rpm" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -rm -rf pulp_rpm-client - -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=rpm" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_rpm ruby "$VERSION" -else - ./generate.sh pulp_rpm ruby "$VERSION" -fi - -pushd pulp_rpm-client -gem build pulp_rpm_client -gem install --both "./pulp_rpm_client-$VERSION.gem" -tar cvf ../../pulp_rpm/rpm-ruby-client.tar "./pulp_rpm_client-$VERSION.gem" -popd -popd diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py deleted file mode 100755 index 656484baa..000000000 --- a/.github/workflows/scripts/release.py +++ /dev/null @@ -1,177 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_rpm' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import re -import os -import textwrap -import requests -import subprocess - -from git import Repo -from pathlib import Path - - -def get_package_from_pypi(version, plugin_path): - """ - Download a package from PyPI. - - :param version: version of the package to download from PyPI - :return: True/False if download was successful - """ - os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - r = requests.get(f"https://pypi.org/pypi/pulp-rpm/{version}/json") - if r.status_code == 200: - metadata = r.json() - for url_data in metadata["urls"]: - filename = url_data["filename"] - r2 = requests.get(url_data["url"]) - if r2.status_code != 200: - raise RuntimeError(f"Failed to download released artifact {filename}") - with open(os.path.join(plugin_path, "dist", filename), "wb") as f: - f.write(r2.content) - return True - return False - - -def create_release_commits(repo, release_version, plugin_path): - """Build changelog, set version, commit, bump to next dev version, commit.""" - issues_to_close = set() - for filename in Path(f"{plugin_path}/CHANGES").rglob("*"): - if filename.stem.isdigit(): - issue = filename.stem - issues_to_close.add(issue) - - issues = ",".join(issues_to_close) - # First commit: changelog - os.system(f"towncrier build --yes --version {release_version}") - git = repo.git - git.add("CHANGES.rst") - git.add("CHANGES/*") - git.commit("-m", f"Add changelog for {release_version}\n\n[noissue]") - - # Second commit: release version - os.system("bump2version release --allow-dirty") - - git.add(f"{plugin_path}") - git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]") - sha = repo.head.object.hexsha - short_sha = git.rev_parse(sha, short=7) - - os.system("bump2version patch --allow-dirty") - - new_dev_version = None - with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - new_dev_version = re.split("\"|'", line)[1] - if not new_dev_version: - raise RuntimeError("Could not detect new dev version ... aborting.") - - git.add(f"{plugin_path}") - git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") - print(f"Release commit == {short_sha}") - print(f"All changes were committed on branch: release_{release_version}") - return sha - - -def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): - """Create a tag if one is needed and build a package if one is not on PyPI.""" - # Remove auth header config - with repo.config_writer() as conf: - conf.remove_section('http "https://github.com/"') - conf.release() - - # Determine if a tag exists and if it matches the specified commit sha - tag = None - for existing_tag in repo.tags: - if existing_tag.name == desired_tag: - if existing_tag.commit.hexsha == commit_sha: - tag = existing_tag - else: - raise RuntimeError( - f"The '{desired_tag}' tag already exists, but the commit sha does not match " - f"'{commit_sha}'." - ) - - # Create a tag if one does not exist - if not tag: - tag = repo.create_tag(desired_tag, ref=commit_sha) - - # Checkout the desired tag and reset the tree - repo.head.reference = tag.commit - repo.head.reset(index=True, working_tree=True) - - # Check if Package is available on PyPI - if not get_package_from_pypi(tag.name, plugin_path): - os.system("python3 setup.py sdist bdist_wheel --python-tag py3") - - -def main(): - helper = textwrap.dedent( - """\ - Start the release process. - - Example: - setup.py on plugin before script: - version="2.0.0.dev" - - $ python .ci/scripts/release.py - - setup.py on plugin after script: - version="2.0.1.dev" - - - """ - ) - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, description=helper - ) - - parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", - nargs="?", - ) - - args = parser.parse_args() - - release_version_arg = args.release_version - - release_path = os.path.dirname(os.path.abspath(__file__)) - plugin_path = release_path.split("/.github")[0] - - output = subprocess.check_output(["bump2version", "--dry-run", "--list", "release"]) - release_version = re.findall(r"\nnew_version=([0-9.]*)\n", output.decode())[0] - - print(f"\n\nRepo path: {plugin_path}") - repo = Repo(plugin_path) - - release_commit = None - if release_version_arg and release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break - if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at " - "HEAD. A release commit for such version does not exist." - ) - - if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) - else: - release_commit_sha = release_commit.hexsha - create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 3de32229a..ab84d9d15 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -52,49 +52,74 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Generate and install binding +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt pushd ../pulp-openapi-generator -# Use app_label to generate api.json and package to produce the proper package name. + # Use app_label to generate api.json and package to produce the proper package name. -if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] -then # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > api.json - ./gen-client.sh api.json "${COMPONENT}" python "${PACKAGE}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done -else # Sadly: Different pulpcore-versions aren't either... - for item in $(jq -r '.versions[]| select(.component!="rpm")| tojson' <<<"$REPORTED_STATUS") + # So we exclude the prebuilt ones only for domains disabled. + if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] + then + BUILT_CLIENTS="" + else + BUILT_CLIENTS=" rpm " + fi + + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" PACKAGE="${MODULE%%.*}" - cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > api.json - ./gen-client.sh api.json "${COMPONENT}" python "${PACKAGE}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] + then + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python setup.py sdist bdist_wheel --python-tag py3 + popd + else + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi + fi + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_rpm/bindings_requirements.txt" done -fi popd -# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +echo "::endgroup::" + +echo "::group::Debug bindings diffs" + +# Bindings diff for rpm +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "rpm-api.json" > "build-api.json" +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/rpm-api.json" > "test-api.json" +jsondiff --indent 2 build-api.json test-api.json || true +echo "::endgroup::" + +# Install test requirements +########################### + +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" @@ -125,7 +150,7 @@ else else cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8" cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m 'not parallel'" - fi + fi fi export PULP_FIXTURES_URL="http://pulp-fixtures:8080" pushd ../pulp-cli diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh deleted file mode 100755 index 08d062ca8..000000000 --- a/.github/workflows/scripts/update_ci.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_rpm' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -eu - -if [ ! -f "template_config.yml" ]; then - echo "No template_config.yml detected." - exit 1 -fi - -pushd ../plugin_template -./plugin-template --github pulp_rpm -popd - -# Check if only gitref file has changed, so no effect on CI workflows. -if [[ $(git diff --name-only) == ".github/template_gitref" ]]; then - echo "No changes detected in github section." - git restore ".github/template_gitref" -fi - -if [[ $(git status --porcelain) ]]; then - git add -A - git commit -m "Update CI files" -m "[noissue]" -else - echo "No updates needed" -fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b213623be..d2cb5abe1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,15 +43,36 @@ jobs: with: python-version: "3.11" - - uses: "actions/download-artifact@v4" + - name: "Download plugin package" + uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_rpm/dist/" + - name: "Download API specs" + uses: "actions/download-artifact@v4" + with: + name: "api_spec" + path: "pulp_rpm/" + + - name: "Download client packages" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp_rpm" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + mkdir -p "pulp_rpm-client" + pushd "pulp_rpm-client" + tar xvf "../../pulp_rpm/rpm-python-client.tar" + popd + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_rpm/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: @@ -94,16 +115,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - if: "${{ env.TEST == 'pulp' }}" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Script" run: | .github/workflows/scripts/script.sh @@ -126,34 +137,6 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true - - name: Upload python client packages - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "python-client.tar" - path: "pulp_rpm/rpm-python-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - - name: Upload python client docs - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_rpm/rpm-python-client-docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - name: Upload Ruby client - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "ruby-client.tar" - path: "pulp_rpm/rpm-ruby-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - name: Upload built docs if: ${{ env.TEST == 'docs' }} uses: actions/upload-artifact@v4 @@ -175,3 +158,4 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +... diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.rst deleted file mode 100644 index 49c2305d7..000000000 --- a/CHANGES/.TEMPLATE.rst +++ /dev/null @@ -1,47 +0,0 @@ -{% if render_title %} -{% if versiondata.name %} -{{ versiondata.name }} {{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}} -{% else %} -{{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} -{% endif %} -{% endif %} -{% for section, _ in sections.items() %} -{% set underline = underlines[0] %}{% if section %}{{section}} -{{ underline * section|length }}{% set underline = underlines[1] %} - -{% endif %} - -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section]%} -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category].items() %} -- {{ text }} - {{ values|join(',\n ') }} -{% endfor %} - -{% else %} -- {{ sections[section][category]['']|join(', ') }} - -{% endif %} -{% if sections[section][category]|length == 0 %} -No significant changes. - -{% else %} -{% endif %} - -{% endfor %} -{% else %} -No significant changes. - - -{% endif %} -{% endfor %} ----- - - - diff --git a/template_config.yml b/template_config.yml index 2be917f4b..8e9e7bdb7 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-326-ge5addc7 +# generated with plugin_template@2021.08.26-348-gb87ed3b api_root: /pulp/ black: true @@ -32,6 +32,7 @@ kanban: true latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 plugin_app_label: rpm plugin_default_branch: main