diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index e4b863331..d57b44ed2 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -28,8 +28,7 @@ RUN pip3 install --upgrade pip setuptools wheel && \ {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \ - pipdeptree && \ +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 14b515968..2617a4089 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -5,8 +5,3 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.* tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. - - - -# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue. -pipdeptree<=3.23.1 diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index b5be1b1f5..7f9eeae33 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -13,6 +13,11 @@ from packaging.version import Version import yaml +try: + import tomllib +except ImportError: + import tomli as tomllib + CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" @@ -59,11 +64,11 @@ def to_upper_bound(req): operator = "~=" version = Version(spec.version) if version.micro != 0: - max_version = f"{version.major}.{version.minor}.{version.micro-1}" + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" elif version.minor != 0: - max_version = f"{version.major}.{version.minor-1}" + max_version = f"{version.major}.{version.minor - 1}" elif version.major != 0: - max_version = f"{version.major-1}.0" + max_version = f"{version.major - 1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" @@ -100,18 +105,32 @@ def main(): parser.add_argument("filename", nargs="*") args = parser.parse_args() - with fileinput.input(files=args.filename) as req_file: - for line in req_file: - if line.strip().startswith("#"): - # Shortcut comment only lines - print(line.strip()) - else: - req, comment = split_comment(line) - if args.upper: - new_req = to_upper_bound(req) + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) else: - new_req = to_lower_bound(req) - print(new_req + comment) + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) if __name__ == "__main__": diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index b6057c845..0847711ae 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -86,6 +86,13 @@ def main(): ) if req_txt_diff: reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.append( + "pyproject.toml changed (PLEASE check if dependencies are affected." + ) if reasons: curr_version = Version(last_tag) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 7c8f8e48f..0edfaccc0 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-rpm-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.github/template_gitref b/.github/template_gitref index 4fb36e960..fd87a7096 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-387-ge627e91 +2021.08.26-391-g4510e3a diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb6173e2d..c5976f7e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,11 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel mkdocs jq + pip install build packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d8d920cbd..292688a00 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -34,29 +34,37 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump2version --dry-run --list release + # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3c4933ee3..3ab77ff8c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -185,4 +185,4 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index e2ef810cb..b0fdb789a 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 3605099c6..6c89e53ab 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -166,5 +166,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7175ee868..4de0bf31b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,5 +147,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true ... diff --git a/lint_requirements.txt b/lint_requirements.txt index 24a7ef0dc..cbac4367b 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -7,6 +7,7 @@ # python packages handy for developers, but not required by pulp black==24.3.0 +bump2version check-manifest flake8 flake8-black diff --git a/releasing.md b/releasing.md index 6c4d72529..3d40eae8a 100644 --- a/releasing.md +++ b/releasing.md @@ -9,19 +9,21 @@ This document outlines the steps to perform a release. ### Determine if a Release is Required +- Make sure to have GitPython python package installed - Run the release checker script: ``` python3 .ci/scripts/check_release.py ``` -### Create a New Y-branch (e.g., 3.23) -- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_rpm/actions/workflows/create-branch.yml) job via the GitHub Actions. +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_rpm/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. - Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_rpm/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. ### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) - Trigger a [Release Pipeline](https://github.com/pulp/pulp_rpm/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. -### Final Steps (Optional but Recommended) -- Ensure the new version appears on PyPI. -- Verify that the changelog has been updated by looking for the "Update Changelog" pull request. -- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_rpm/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/).