From c7eac6eb6579e8e861b207da3e34b2fd8b894ed2 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 21:47:10 +0000 Subject: [PATCH 1/7] Manage repository with cruft --- renovate.json5 => .github/renovate.json5 | 9 +-- .github/workflows/cruft.yaml | 79 ++++++++++++++++++++++ .github/workflows/lint.yaml | 46 +++++++++---- .github/workflows/pages.yaml | 17 ++--- .github/workflows/publish.yaml | 51 +++++++++++++++ .github/workflows/python-package.yaml | 52 --------------- .github/workflows/python-publish.yaml | 32 --------- .github/workflows/test.yaml | 45 +++++++++++++ .pre-commit-config.yaml | 83 ++++++++++++------------ mypy.ini | 17 ----- pyproject.toml | 26 ++++++++ requirements_dev.txt | 25 +++++++ script/run-in-env.sh | 22 ------- script/run-mypy.sh | 10 +++ 14 files changed, 324 insertions(+), 190 deletions(-) rename renovate.json5 => .github/renovate.json5 (86%) create mode 100644 .github/workflows/cruft.yaml create mode 100644 .github/workflows/publish.yaml delete mode 100644 .github/workflows/python-package.yaml delete mode 100644 .github/workflows/python-publish.yaml create mode 100644 .github/workflows/test.yaml delete mode 100644 mypy.ini create mode 100644 pyproject.toml create mode 100644 requirements_dev.txt delete mode 100755 script/run-in-env.sh create mode 100755 script/run-mypy.sh diff --git a/renovate.json5 b/.github/renovate.json5 similarity index 86% rename from renovate.json5 rename to .github/renovate.json5 index ec42befa..91aa690f 100644 --- a/renovate.json5 +++ b/.github/renovate.json5 @@ -1,7 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "config:base", + "config:base" "regexManagers:dockerfileVersions", ], "timezone": "America/Los_Angeles", @@ -12,7 +12,7 @@ "automerge": true, "automergeType": "branch", "matchUpdateTypes": ["minor", "patch"] - }, + } ], "regexManagers": [ { @@ -23,7 +23,8 @@ "extractVersionTemplate": "^v?(?.*)$", } ], - "pre-commit": { - "enabled": true, + "pip_requirements": { + "fileMatch": ["requirements_dev.txt"] }, + "pre-commit": {"enabled": true} } diff --git a/.github/workflows/cruft.yaml b/.github/workflows/cruft.yaml new file mode 100644 index 00000000..75bc576a --- /dev/null +++ b/.github/workflows/cruft.yaml @@ -0,0 +1,79 @@ +--- +name: Update repository with Cruft +permissions: + contents: write + pull-requests: write +on: + schedule: + - cron: "0 0 * * *" + +env: + PYTHON_VERSION: 3.12 + +jobs: + update: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + include: + - add-paths: . + body: Use this to merge the changes to this repository. + branch: cruft/update + commit-message: "chore: accept new Cruft update" + title: New updates detected with Cruft + - add-paths: .cruft.json + body: Use this to reject the changes in this repository. + branch: cruft/reject + commit-message: "chore: reject new Cruft update" + title: Reject new updates detected with Cruft + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Cruft + run: pip3 install cruft + + - name: Check if update is available + continue-on-error: false + id: check + run: | + CHANGES=0 + if [ -f .cruft.json ]; then + if ! cruft check; then + CHANGES=1 + fi + else + echo "No .cruft.json file" + fi + + echo "has_changes=$CHANGES" >> "$GITHUB_OUTPUT" + + - name: Run update if available + if: steps.check.outputs.has_changes == '1' + run: | + git config --global user.email "allen.porter@gmail.com" + git config --global user.name "Allen Porter" + + cruft update --skip-apply-ask --refresh-private-variables + git restore --staged . + + + - name: Create pull request + if: steps.check.outputs.has_changes == '1' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + add-paths: ${{ matrix.add-paths }} + commit-message: ${{ matrix.commit-message }} + branch: ${{ matrix.branch }} + delete-branch: true + branch-suffix: timestamp + title: ${{ matrix.title }} + body: | + This is an autogenerated PR. ${{ matrix.body }} + + [Cruft](https://cruft.github.io/cruft/) has detected updates from the Cookiecutter repository. diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e5c58519..031141d1 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -4,27 +4,45 @@ name: Lint on: push: branches: - - main + - main pull_request: branches: - - main + - main + +env: + PYTHON_VERSION: 3.12 jobs: build: - runs-on: ubuntu-latest strategy: fail-fast: false steps: - - uses: actions/checkout@v4 - - uses: codespell-project/actions-codespell@master - with: - skip: ./tests/tool/__snapshots__/test_diagnostics.ambr,./tests/testdata/cluster9/clusters/dev/flux-system/gotk-components.yaml - - uses: chartboost/ruff-action@v1.0.0 - - name: Run yamllint - uses: ibiqlik/action-yamllint@v3 - with: - file_or_dir: "./" - config_file: "./.yaml-lint.yaml" - strict: true + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1.0.0 + - uses: codespell-project/actions-codespell@v2.0 + with: + check_hidden: false + - name: Run yamllint + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: "./" + config_file: "./.yaml-lint.yaml" + strict: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "**/requirements_dev.txt" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_dev.txt + + - name: Static typing with mypy + run: | + mypy --install-types --non-interactive --no-warn-unused-ignores . diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index c1c1c13c..e097c187 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -2,28 +2,26 @@ name: Deploy static content to Pages on: - # Runs on pushes targeting the default branch push: branches: - main - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: contents: read pages: write id-token: write actions: read -# Allow one concurrent deployment concurrency: group: "pages" cancel-in-progress: true +env: + PYTHON_VERSION: 3.12 + jobs: - # Single deploy job since we're just deploying deploy: environment: name: github-pages @@ -33,13 +31,16 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "**/requirements_dev.txt" - name: Install dependencies run: | - pip install -r requirements.txt + python -m pip install --upgrade pip + pip install -r requirements_dev.txt - run: pdoc ./flux_local -o docs/ - name: Setup Pages uses: actions/configure-pages@v5 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..8d424756 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,51 @@ +--- +name: Upload Python Package + +on: + release: + types: [created] + +env: + PYTHON_VERSION: 3.12 + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/flux_local + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/python-package.yaml b/.github/workflows/python-package.yaml deleted file mode 100644 index eba5e279..00000000 --- a/.github/workflows/python-package.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: Python package - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: - - "3.10" - - "3.11" - - "3.12" - - steps: - - uses: actions/checkout@v4 - - name: Setup Flux CLI - uses: fluxcd/flux2/action@v2.2.3 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install -e . - - uses: supplypike/setup-bin@v4 - with: - uri: https://github.com/kyverno/kyverno/releases/download/v1.11.3/kyverno-cli_v1.11.3_linux_x86_64.tar.gz - name: kyverno-cli - version: v1.11.3 - - name: Test with pytest - run: | - SKIP_DIFF_TESTS=1 pytest --cov=flux_local --cov-report=term-missing --snapshot-warn-unused - - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - env_vars: OS,PYTHON - fail_ci_if_error: true - verbose: true - - name: Test flux-local diff - run: | - pytest tests/tool/test_diff.py diff --git a/.github/workflows/python-publish.yaml b/.github/workflows/python-publish.yaml deleted file mode 100644 index 19d72e48..00000000 --- a/.github/workflows/python-publish.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@v1.8.14 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..64079cc4 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,45 @@ +--- +name: Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Setup Flux CLI + uses: fluxcd/flux2/action@v2.2.3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "**/requirements_dev.txt" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + - name: Test with pytest + run: | + SKIP_DIFF_TESTS=1 pytest --cov=flux_local --cov-report=term-missing --snapshot-warn-unused + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + env_vars: OS,PYTHON + fail_ci_if_error: true + verbose: true + - name: Test flux-local diff + run: | + pytest tests/tool/test_diff.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df42cf35..3fb815a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,43 +1,44 @@ --- repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 - hooks: - - id: trailing-whitespace - exclude: '^tests/.*.ambr$' - - id: end-of-file-fixer - exclude: '^tests/.*.ambr$' - - id: check-yaml - args: - - --allow-multiple-documents - - id: check-added-large-files -- repo: https://github.com/codespell-project/codespell - rev: v2.2.6 - hooks: - - id: codespell -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.4.4' - hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix] -- repo: https://github.com/adrienverge/yamllint.git - rev: v1.35.1 - hooks: - - id: yamllint - args: - - -c - - ".yaml-lint.yaml" -- repo: https://github.com/psf/black - rev: 24.4.2 - hooks: - - id: black - exclude: '^tests/.*.ambr$' -- repo: local - hooks: - - id: mypy - name: mypy - entry: script/run-in-env.sh mypy - language: script - types: [python] - require_serial: true - files: ^(flux_local/|tests/) + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/psf/black + rev: 24.4.2 + hooks: + - id: black + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.4.4 + hooks: + - id: ruff + args: + - --fix + - --exit-non-zero-on-fix + - repo: local + hooks: + - id: mypy + name: mypy + entry: script/run-mypy.sh + language: script + types: [python] + require_serial: true + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint + exclude: '^tests/tool/testdata/.*\.yaml$' + args: + - -c + - ".yaml-lint.yaml" + - repo: https://github.com/asottile/setup-cfg-fmt + rev: v2.5.0 + hooks: + - id: setup-cfg-fmt diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 0a30e539..00000000 --- a/mypy.ini +++ /dev/null @@ -1,17 +0,0 @@ -[mypy] -ignore_missing_imports = True -exclude = (venv|build) -check_untyped_defs = True -disallow_any_generics = True -disallow_incomplete_defs = True -disallow_subclassing_any = True -disallow_untyped_calls = True -disallow_untyped_decorators = True -disallow_untyped_defs = True -no_implicit_optional = True -no_implicit_reexport = True -warn_return_any = True -warn_unreachable = True -warn_redundant_casts = True -warn_unused_ignores = True -warn_no_return = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..45a4a341 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[tool.mypy] +exclude = [ + "venv/", +] +platform = "linux" +show_error_codes = true +follow_imports = "normal" +local_partial_types = true +strict_equality = true +no_implicit_optional = true +warn_incomplete_stub = true +warn_redundant_casts = true +warn_unused_configs = true +warn_unused_ignores = true +disable_error_code = [ + "import-untyped", +] +extra_checks = false +check_untyped_defs = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true +warn_return_any = true +warn_unreachable = true diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000..bde63011 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,25 @@ +-e . +black==24.4.2 +coverage==7.5.1 +mypy==1.10.0 +pdoc==14.4.0 +pip==24.0 +pre-commit==3.6.0 +pytest-cov==5.0.0 +pytest==8.2.0 +ruff==0.4.4 + + +aiofiles==23.2.1 +GitPython==3.1.43 +mashumaro==3.13 +nest_asyncio==1.6.0 +pytest-asyncio==0.23.6 +python-slugify==8.0.4 +PyYAML==6.0.1 +types-aiofiles==23.2.0.20240403 +types-PyYAML==6.0.12.20240311 +typing-extensions==4.11.0 +types-python-slugify==8.0.2.20240310 +yamllint==1.35.1 +syrupy==4.6.1 diff --git a/script/run-in-env.sh b/script/run-in-env.sh deleted file mode 100755 index 271e7a4a..00000000 --- a/script/run-in-env.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env sh -set -eu - -# Activate pyenv and virtualenv if present, then run the specified command - -# pyenv, pyenv-virtualenv -if [ -s .python-version ]; then - PYENV_VERSION=$(head -n 1 .python-version) - export PYENV_VERSION -fi - -# other common virtualenvs -my_path=$(git rev-parse --show-toplevel) - -for venv in venv .venv .; do - if [ -f "${my_path}/${venv}/bin/activate" ]; then - . "${my_path}/${venv}/bin/activate" - break - fi -done - -exec "$@" diff --git a/script/run-mypy.sh b/script/run-mypy.sh new file mode 100755 index 00000000..fc3804b4 --- /dev/null +++ b/script/run-mypy.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -o errexit + +# Change directory to the project root directory. +cd "$(dirname "$0")" + +pip3 install -r requirements_dev.txt --no-input --quiet + +mypy . From 49940e7ef9c5f4ede8b7637e71eb3e851c75f191 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 21:47:33 +0000 Subject: [PATCH 2/7] Fix syntax error --- .github/renovate.json5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 91aa690f..a484613c 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,7 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "config:base" + "config:base", "regexManagers:dockerfileVersions", ], "timezone": "America/Los_Angeles", From f052d820062ee8f40ae291ba62ffa1f164f8969f Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 21:51:17 +0000 Subject: [PATCH 3/7] Exclude yaml files from spell check --- .github/workflows/lint.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 031141d1..e9483cc8 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -24,6 +24,8 @@ jobs: - uses: codespell-project/actions-codespell@v2.0 with: check_hidden: false + exclude_file: ./tests/testdata/cluster9/clusters/dev/flux-system/gotk-components.yaml + - name: Run yamllint uses: ibiqlik/action-yamllint@v3 with: From e51fdd087298a7e1a6338a7b6b3dd7090cc9dcdb Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 21:52:30 +0000 Subject: [PATCH 4/7] Re-install kyverno cli --- .github/workflows/test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 64079cc4..9d1cc23b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,6 +21,11 @@ jobs: - uses: actions/checkout@v4 - name: Setup Flux CLI uses: fluxcd/flux2/action@v2.2.3 + - uses: supplypike/setup-bin@v4 + with: + uri: https://github.com/kyverno/kyverno/releases/download/v1.11.3/kyverno-cli_v1.11.3_linux_x86_64.tar.gz + name: kyverno-cli + version: v1.11.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 7424a911c29bbfd5214c2250263778665a77b5c9 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 21:58:51 +0000 Subject: [PATCH 5/7] Exclude some current mypy warnings --- pyproject.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 45a4a341..4e760172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,13 @@ warn_unused_configs = true warn_unused_ignores = true disable_error_code = [ "import-untyped", + # TODO: Fix these warnings + "assignment", + "attr-defined", + "return-value", + "no-untyped-def", + "union-attr", + "assignment", ] extra_checks = false check_untyped_defs = true @@ -24,3 +31,8 @@ disallow_untyped_decorators = true disallow_untyped_defs = true warn_return_any = true warn_unreachable = true + +ignore_missing_imports = True +disallow_any_generics = True +no_implicit_reexport = True +warn_no_return = True From 2a78ccf60a6a75a2ea8e87a8cc9930f5fea2a441 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 22:00:37 +0000 Subject: [PATCH 6/7] Fix toml style --- pyproject.toml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4e760172..461d2397 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,8 +31,7 @@ disallow_untyped_decorators = true disallow_untyped_defs = true warn_return_any = true warn_unreachable = true - -ignore_missing_imports = True -disallow_any_generics = True -no_implicit_reexport = True -warn_no_return = True +ignore_missing_imports = true +disallow_any_generics = true +no_implicit_reexport = true +warn_no_return = true From b3272f633ee9eed41e036ae6fe07ec22a07e837b Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 12 May 2024 22:02:02 +0000 Subject: [PATCH 7/7] Ignore additional mypy warnings --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 461d2397..fdbf6fa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ disable_error_code = [ "no-untyped-def", "union-attr", "assignment", + "arg-type", ] extra_checks = false check_untyped_defs = true