From aeb83d4d20a09105b79b33b165367098e5c16ac3 Mon Sep 17 00:00:00 2001 From: William Dutton Date: Tue, 10 Dec 2024 10:23:59 +1000 Subject: [PATCH 1/2] chore: cicd, CKAN 2.9.x --- .github/workflows/publish.yml | 202 ++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 58 ++++++++-- README.md | 17 ++- pyproject.toml | 150 +++++++++++++++++++++++++ setup.py | 88 +-------------- 5 files changed, 413 insertions(+), 102 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..bb5f002b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,202 @@ +--- +name: Publish to pypi +on: + push: + #On versioned releases + tags: + - '*.*.*' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + inputs: + force: + type: choice + description: Retry Publish Version + options: + - No + - Yes + environment: + description: 'Deployment environment' + required: true + default: 'pypi' + type: choice + options: + - pypi + - testpypi + dryRun: + description: 'Dry Run deployment (set to false to deploy)' + required: true + type: boolean + default: true + + + +jobs: + lint: + if: github.repository == 'ckan/ckanext-validation' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --extend-exclude ckan + + test: + needs: lint + strategy: + matrix: + include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags +# - ckan-version: "2.11" +# ckan-image: "2.11-py3.10" +# - ckan-version: "2.10" +# ckan-image: "2.10-py3.10" + - ckan-version: "2.9" + ckan-image: "2.9-py3.9" + #- ckan-version: "master" Publish does not care about master + # ckan-image: "master" + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + container: + image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + options: --user root + services: + solr: + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + + steps: + - uses: actions/checkout@v4 + + - name: Pin setuptools for ckan 2.9 only + if: ${{ matrix.ckan-version == 2.9 }} + run: pip install "setuptools>=44.1.0,<71" + + - name: Install requirements + run: | + pip install -r requirements.txt + pip install -r dev-requirements.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + + - name: Setup extension + run: | + ckan -c test.ini db init + + - name: Run tests + run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv --junit-xml=/tmp/artifacts/junit/results.xml + + - name: Test Summary + uses: test-summary/action@v2 + continue-on-error: ${{ matrix.experimental }} + with: + paths: "/tmp/artifacts/junit/*.xml" + if: always() + + publishSkipped: + if: github.repository != 'ckan/ckanext-validation' + runs-on: ubuntu-latest + steps: + - run: | + echo "## Skipping PyPI publish on downstream repository" >> $GITHUB_STEP_SUMMARY + + publish: + needs: test + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + name: Publish Package + runs-on: ubuntu-latest + environment: + name: ${{ github.event.inputs.environment || 'pypi' }} + url: ${{ steps.version.outputs.url }} + concurrency: + group: ${{ github.event.inputs.environment }}-deployment + cancel-in-progress: false + env: + ENVIRONMENT: ${{ github.event.inputs.environment || 'pypi' }} + steps: + - name: Get Git Tag and set url from environment + id: version + run: | + #!/bin/bash + + TAG_VALUE=${GITHUB_REF/refs\/tags\//} + echo "version=${TAG_VALUE}" >> $GITHUB_OUTPUT + + # Extract the repository name (minus the owner/org) + reponame=$(basename $GITHUB_REPOSITORY) + echo "reponame=${reponame}" >> $GITHUB_OUTPUT + + if [ "$env.ENVIRONMENT" == "testpypi" ]; then + url="https://test.pypi.org/project/$reponame/$TAG_VALUE/" + echo "environment=${env.ENVIRONMENT}" >> $GITHUB_OUTPUT + else + url="https://pypi.org/project/$reponame/$TAG_VALUE/" + echo "environment=pypi" >> $GITHUB_OUTPUT + fi + + echo "url=${url}" >> $GITHUB_OUTPUT + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate tag version + if: ${{ startsWith(github.ref, 'refs/tags') }} + run: | + PYTHON_VERSION=$(grep -E "\bversion='[^']+'" setup.py | awk -F "'" '{print $2}') + echo "Tag version is [${{ steps.version.outputs.version }}], Python version is [$PYTHON_VERSION]" + if [ "${{ steps.version.outputs.version }}" != "$PYTHON_VERSION" ]; then + echo "Version mismatch; tag version is [${{ steps.version.outputs.version }}] but Python version is [$PYTHON_VERSION]" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + - name: Build package ${{ steps.version.outputs.reponame }} @ ${{ steps.version.outputs.version }} + run: | + pip install build + pip install twine + python -m build + - name: Publish package distributions to PyPI + if: ${{ startsWith(github.ref, 'refs/tags') && steps.version.outputs.environment == 'pypi' && github.event.inputs.dryRun != 'true' }} + uses: pypa/gh-action-pypi-publish@release/v1 +# with: +# skip-existing: true +# verbose: true +# print-hash: true + - name: Test Publish package distributions to PyPI + if: ${{ startsWith(github.ref, 'refs/tags') && steps.version.outputs.environment == 'testpypi' && github.event.inputs.dryRun == 'true' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ +# skip-existing: true +# verbose: true +# print-hash: true + - name: Summary output + if: ${{ startsWith(github.ref, 'refs/tags') && github.event.inputs.dryRun != 'true' }} + run: + echo "Published ${{ steps.version.outputs.repo_name }} @ ${{ steps.version.outputs.version }} to ${{ steps.version.outputs.url }}" >> $GITHUB_STEP_SUMMARY + + - name: (TEST RUN) Test Publish package distributions to PyPI + if: ${{ github.event.inputs.dryRun == 'true' }} + run: + echo "Dry run deployment, did not publish" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5483f1cf..9f9a47d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,38 +1,56 @@ name: Tests -on: [push, pull_request] +on: + push: + pull_request: + jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Install requirements run: pip install flake8 pycodestyle - name: Check syntax - run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --extend-exclude ckan test: needs: lint strategy: matrix: - ckan-version: [2.9] + include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags +# - ckan-version: "2.11" +# ckan-image: "2.11-py3.10" +# experimental: false +# - ckan-version: "2.10" +# ckan-image: "2.10-py3.10" +# experimental: false + - ckan-version: "2.9" + ckan-image: "2.9-py3.9" + experimental: false +# - ckan-version: "master" +# ckan-image: "master" +# experimental: true # master is unstable, good to know if we are compatible or not fail-fast: false name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + options: --user root services: solr: - image: ckan/ckan-solr:${{ matrix.ckan-version }} + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 postgres: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres + ports: + - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: image: redis:3 @@ -44,23 +62,41 @@ jobs: CKAN_REDIS_URL: redis://redis:6379/1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + continue-on-error: ${{ matrix.experimental }} + + - name: Pin setuptools for ckan 2.9 only + if: ${{ matrix.ckan-version == 2.9 }} + run: pip install "setuptools>=44.1.0,<71" + continue-on-error: ${{ matrix.experimental }} + - name: Install requirements + continue-on-error: ${{ matrix.experimental }} run: | pip install -r dev-requirements.txt pip install -r requirements.txt - pip install --no-warn-conflicts jinja2==2.10.1 - pip install --no-warn-conflicts markupsafe==2.0.1 pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Setup extension + continue-on-error: ${{ matrix.experimental }} run: | ckan -c test.ini db init + - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv + continue-on-error: ${{ matrix.experimental }} + run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv --junit-xml=/tmp/artifacts/junit/results.xml + + - name: Test Summary + uses: test-summary/action@v2 + continue-on-error: ${{ matrix.experimental }} + with: + paths: "/tmp/artifacts/junit/*.xml" + if: always() - name: Upload coverage report to codecov uses: codecov/codecov-action@v1 + continue-on-error: ${{ matrix.experimental }} with: file: ./coverage.xml diff --git a/README.md b/README.md index 3e3ba62f..618ccbb2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Data description and validation for CKAN with [Frictionless Data](https://fricti ## Table of Contents * [Overview](#overview) - * [Versions supported and requirements](#versions-supported-and-requirements) + * [Versions supported and Requirements](#versions-supported-and-requirements) * [Installation](#installation) * [Configuration](#configuration) * [How it works](#how-it-works) @@ -58,9 +58,18 @@ If you are eager to get started, jump to the [Installation](#installation) and [Configuration](#configuration) instructions. To learn more about data validation and how the extension works, read the next section. -## Versions supported and requirements +## Versions supported and Requirements + +Compatibility with core CKAN versions: + + | CKAN version | Compatibility | + | -------------- |------------------------------------------------------------| + | 2.7 | no longer supported | + | 2.8 | no longer supported (last supported 1.x) | + | 2.9 | yes (Python3) Must: `pip install "setuptools>=44.1.0,<71"` | + | 2.10 | no | + | 2.11 | no | -This extension is currently tested in **CKAN 2.9 (Python 3)**. For previous CKAN versions and Python 2 you can use the 1.x versions, but those will not be supported going forward. It is strongly recommended to use it alongside [ckanext-scheming](https://github.com/ckan/ckanext-scheming) to define the @@ -71,7 +80,7 @@ necessary extra fields in the default CKAN schema. To install ckanext-validation, activate your CKAN virtualenv and run: - git clone https://github.com/frictionlessdata/ckanext-validation.git + git clone https://github.com/ckan/ckanext-validation.git cd ckanext-validation pip install -r requirements.txt python setup.py develop diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..20837b9d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,150 @@ +[build-system] +requires = [ "setuptools",] +build-backend = "setuptools.build_meta" + +[project] +name = "ckanext-validation" +version = "2.0.0" +description = "Data description and validation for CKAN with Frictionless Data tools." +classifiers = [ "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10",] +keywords = [ "CKAN", "extension", "validation",] +dependencies = [ "typing_extensions",] +authors = [ + {name = "Open Knowledge Foundation", email = "info@okfn.org"}, + {name = "Adrià Mercader (amercader)", email = "amercadero@gmail.com"}, + {name = "Jesse Vickery (JVickery-TBS)", email = "jesse.vickery@tbs-sct.gc.ca"}, + {name = "Edgar Z. Alvarenga (aivuk)"}, + {name = "ThrawnCA", email = "carl.antuar@smartservice.qld.gov.au"}, + {name = "William Dutton (duttonw)", email = "william.dutton@qld.gov.au"}, +# {name = "", email = ""}, +] +maintainers = [ + {name = "Open Knowledge Foundation", email = "info@okfn.org"}, + {name = "Adrià Mercader (amercader)", email = "amercadero@gmail.com"}, + {name = "William Dutton (duttonw)", email = "william.dutton@qld.gov.au"}, + {name = "ThrawnCA", email = "carl.antuar@smartservice.qld.gov.au"}, +] + +[project.readme] +file = "README.md" +content-type = "text/markdown" + +[project.license] +text = "AGPL" + +[project.urls] +Homepage = "https://github.com/ckan/ckanext-validation" + +[project.optional-dependencies] +test = [ "pytest-factoryboy",] + +[project.entry-points."ckan.plugins"] +validation = "ckanext.validation.plugin:ValidationPlugin" + +[project.entry-points."ckan.test_plugins"] +test_validation_plugin = "ckanext.validation.tests.test_interfaces:TestPlugin" + +[project.entry-points."ckan.paster_command"] +validation = "ckanext.validation.commands:Validation" + +[project.entry-points."babel.extractors"] +ckan = "ckan.lib.extract:extract_ckan" + +[tool.setuptools.packages] +find = {} + +[tool.black] +line-length = 79 +preview = true + +[tool.isort] +known_ckan = "ckan" +known_ckanext = "ckanext" +known_self = "ckanext.validation" +sections = "FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,CKAN,CKANEXT,SELF,LOCALFOLDER" + +[tool.pytest.ini_options] +addopts = "--ckan-ini test.ini" +filterwarnings = [ + "ignore::sqlalchemy.exc.SADeprecationWarning", + "ignore::sqlalchemy.exc.SAWarning", + "ignore::DeprecationWarning", +] + +[tool.pyright] +pythonVersion = "3.7" +include = ["ckanext"] +exclude = [ + "**/test*", + "**/migration", +] +strict = [] + +strictParameterNoneValue = true # type must be Optional if default value is None + +# Check the meaning of rules here +# https://github.com/microsoft/pyright/blob/main/docs/configuration.md +reportFunctionMemberAccess = true # non-standard member accesses for functions +reportMissingImports = true +reportMissingModuleSource = true +reportMissingTypeStubs = false +reportImportCycles = true +reportUnusedImport = true +reportUnusedClass = true +reportUnusedFunction = true +reportUnusedVariable = true +reportDuplicateImport = true +reportOptionalSubscript = true +reportOptionalMemberAccess = true +reportOptionalCall = true +reportOptionalIterable = true +reportOptionalContextManager = true +reportOptionalOperand = true +reportTypedDictNotRequiredAccess = false # We are using Context in a way that conflicts with this check +reportConstantRedefinition = true +reportIncompatibleMethodOverride = true +reportIncompatibleVariableOverride = true +reportOverlappingOverload = true +reportUntypedFunctionDecorator = false +reportUnknownParameterType = true +reportUnknownArgumentType = false +reportUnknownLambdaType = false +reportUnknownMemberType = false +reportMissingTypeArgument = true +reportInvalidTypeVarUse = true +reportCallInDefaultInitializer = true +reportUnknownVariableType = true +reportUntypedBaseClass = true +reportUnnecessaryIsInstance = true +reportUnnecessaryCast = true +reportUnnecessaryComparison = true +reportAssertAlwaysTrue = true +reportSelfClsParameterName = true +reportUnusedCallResult = false # allow function calls for side-effect only (like logic.check_acces) +useLibraryCodeForTypes = true +reportGeneralTypeIssues = true +reportPropertyTypeMismatch = true +reportWildcardImportFromLibrary = true +reportUntypedClassDecorator = false # authenticator relies on repoze.who class-decorator +reportUntypedNamedTuple = true +reportPrivateUsage = true +reportPrivateImportUsage = true +reportInconsistentConstructor = true +reportMissingSuperCall = false +reportUninitializedInstanceVariable = true +reportInvalidStringEscapeSequence = true +reportMissingParameterType = true +reportImplicitStringConcatenation = false +reportUndefinedVariable = true +reportUnboundVariable = true +reportInvalidStubStatement = true +reportIncompleteStub = true +reportUnsupportedDunderAll = true +reportUnusedCoroutine = true +reportUnnecessaryTypeIgnoreComment = true +reportMatchNotExhaustive = true \ No newline at end of file diff --git a/setup.py b/setup.py index 9175d4ba..cb65266c 100644 --- a/setup.py +++ b/setup.py @@ -1,94 +1,8 @@ # -*- coding: utf-8 -*- -from setuptools import setup, find_packages # Always prefer setuptools over distutils -from os import path +from setuptools import setup -here = path.abspath(path.dirname(__file__)) setup( - name='''ckanext-validation''', - - # Versions should comply with PEP440. For a discussion on single-sourcing - # the version across setup.py and the project code, see - # http://packaging.python.org/en/latest/tutorial.html#version - version='2.0.0', - - description='''Data description and validation for CKAN with Frictionless Data tools.''', - - # The project's main homepage. - url='https://github.com/frictionlessdata/ckanext-validation', - - # Author details - author='''Open Knowledge Foundation''', - author_email='''info@okfn.org''', - - # Choose your license - license='AGPL', - - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 4 - Beta', - - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', - - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - - - # What does your project relate to? - keywords='''CKAN''', - - # You can just specify the packages manually here if your project is - # simple. Or you can use find_packages(). - packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - namespace_packages=['ckanext'], - - install_requires=[ - # CKAN extensions should not list dependencies here, but in a separate - # ``requirements.txt`` file. - # - # http://docs.ckan.org/en/latest/extensions/best-practices.html#add-third-party-libraries-to-requirements-txt - ], - - # If there are data files included in your packages that need to be - # installed, specify them here. If using Python 2.6 or less, then these - # have to be included in MANIFEST.in as well. - include_package_data=True, - package_data={ - }, - - # Although 'package_data' is the preferred approach, in some case you may - # need to place data files outside of your packages. - # see http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files - # In this case, 'data_file' will be installed into '/my_data' - data_files=[], - - # To provide executable scripts, use entry points in preference to the - # "scripts" keyword. Entry points provide cross-platform support and allow - # pip to create the appropriate form of executable for the target platform. - entry_points={ - 'ckan.plugins': [ - 'validation=ckanext.validation.plugin:ValidationPlugin' - ], - 'ckan.test_plugins': [ - 'test_validation_plugin = ckanext.validation.tests.test_interfaces:TestPlugin', - ], - 'paste.paster_command': [ - 'validation = ckanext.validation.commands:Validation' - ], - 'babel.extractors': [ - 'ckan = ckan.lib.extract:extract_ckan' - ] - }, - # If you are changing from the default layout of your extension, you may # have to change the message extractors, you can read more about babel # message extraction at From 63dbc41a58d9cd2e594ea8c3074d6308769128b5 Mon Sep 17 00:00:00 2001 From: William Dutton Date: Tue, 10 Dec 2024 11:01:25 +1000 Subject: [PATCH 2/2] fixed version jinja2/markupsafe for now --- .github/workflows/publish.yml | 3 +++ .github/workflows/test.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bb5f002b..2fc0a067 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -96,6 +96,9 @@ jobs: run: | pip install -r requirements.txt pip install -r dev-requirements.txt + echo "no-warn-conflicts install of jinja2 and markupsafe" + pip install --no-warn-conflicts jinja2==2.10.1 + pip install --no-warn-conflicts markupsafe==2.0.1 pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f9a47d6..fb3fd8c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,6 +75,9 @@ jobs: run: | pip install -r dev-requirements.txt pip install -r requirements.txt + echo "no-warn-conflicts install of jinja2 and markupsafe" + pip install --no-warn-conflicts jinja2==2.10.1 + pip install --no-warn-conflicts markupsafe==2.0.1 pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini