chore: Move pre-commit config to the top level of kedro-plugins (#259)

* Create .pre-commit-config.yaml * Update .pre-commit-config.yaml * remove pre-commit-config from each plugin * fix relative paths * add exclude regex * Update .pre-commit-config.yaml * rework pre-commit * test * Revert "test" This reverts commit edf731e. * Update Makefile * Update .pre-commit-config.yaml * Update .pre-commit-config.yaml * Update .pre-commit-config.yaml * Update .pre-commit-config.yaml * TEST * Update .pre-commit-config.yaml * fix isort * undo test * lint * add secret scan and bandit again * pass_filenames: true * Update check-plugin.yml * Revert "pass_filenames: true" This reverts commit 1e85806. * disable import errors * lint * Update .pre-commit-config.yaml * Update .pre-commit-config.yaml * isort * try running hooks seperatley * revert telemetry isort * fix ga yaml * Update check-plugin.yml * add hook stage * changes based on review * add features and tests for docker, airflow and telemetry * Update Makefile * indent * lint * lint for airflow and docker * Update check-plugin.yml * undo plugin lint
kedro-org · Jul 19, 2023 · 31b311e · 31b311e
1 parent e73d13b
commit 31b311e
Show file tree

Hide file tree

Showing 7 changed files with 227 additions and 365 deletions.
diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml
@@ -88,7 +88,6 @@ jobs:
             pip freeze
       - name: Install pre-commit hooks
         run: |
-            cd ${{ inputs.plugin }}
             pre-commit install --install-hooks
             pre-commit install --hook-type pre-push
       - name: Run linter

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,226 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+default_stages: [commit, manual]
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.2.3
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml # Checks yaml files for parseable syntax.
+      - id: check-added-large-files
+      - id: check-case-conflict # Check for files that would conflict in case-insensitive filesystems
+      - id: check-merge-conflict # Check for files that contain merge conflict strings.
+      - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source.
+      - id: requirements-txt-fixer # Sorts entries in requirements.txt
+      - id: flake8
+        files: ^(kedro-datasets/kedro_datasets/|kedro-airflow/kedro_airflow/|kedro-docker/kedro_docker/|kedro-telemetry/kedro_telemetry/)
+        args:
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+          - "--ignore=E203,E266,E501,W503"
+        exclude: "^kedro_airflow/dag_template.py|^template.py"
+
+  - repo: local
+    hooks:
+      # pylint quick checks
+      - id: pylint-quick-kedro-datasets
+        name: "Quick PyLint on kedro_datasets/*"
+        language: system
+        types: [file, python]
+        files: ^kedro-datasets/kedro_datasets/
+        exclude: ^(?!kedro-datasets/kedro_datasets/).*\.py$
+        entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=unnecessary-pass
+        stages: [commit]
+
+      - id: pylint-quick-kedro-airflow
+        name: "Quick PyLint on kedro_airflow/*"
+        language: system
+        types: [file, python]
+        files: ^kedro-airflow/kedro_airflow/
+        exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$
+        entry: pylint --disable=unnecessary-pass
+        stages: [commit]
+
+      - id: pylint-quick-kedro-docker
+        name: "Quick PyLint on kedro_docker/*"
+        language: system
+        types: [file, python]
+        files: ^kedro-docker/kedro_docker/
+        exclude: ^(?!kedro-docker/kedro_docker/).*\.py$
+        entry: pylint --disable=unnecessary-pass
+        stages: [commit]
+
+      - id: pylint-quick-kedro-telemetry
+        name: "Quick PyLint on kedro_telemetry/*"
+        language: system
+        types: [file, python]
+        files: ^kedro-telemetry/kedro_telemetry/
+        exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$
+        entry: pylint --disable=unnecessary-pass
+        stages: [commit]
+
+      # pylint full checks
+      - id: pylint-kedro-datasets
+        name: "PyLint on kedro_datasets/*"
+        language: system
+        files: ^kedro-datasets/kedro_datasets/.*\.py$
+        exclude: ^(?!kedro-datasets/kedro_datasets/).*\.py$
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=unnecessary-pass,E0401 kedro-datasets/kedro_datasets
+
+      - id: pylint-kedro-datasets-features
+        name: "PyLint on kedro-datasets features/*"
+        language: system
+        files: ^kedro-datasets/features/.*\.py$
+        exclude: ^(?!kedro-datasets/features/).*\.py$
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --rcfile kedro-datasets/pyproject.toml  --disable=missing-docstring,no-name-in-module,E0401 kedro-datasets/features
+
+      - id: pylint-kedro-datasets-tests
+        name: "PyLint on kedro-datasets tests/*"
+        language: system
+        files: ^kedro-datasets/tests/.*\.py$
+        exclude: ^(?!kedro-datasets/tests/).*\.py$
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --rcfile kedro-datasets/pyproject.toml  --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,E0401 kedro-datasets/tests
+
+      - id: pylint-kedro-airflow
+        name: "PyLint on kedro_airflow/*"
+        language: system
+        files: ^kedro-airflow/kedro_airflow/.*\.py$
+        exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=unnecessary-pass,E0401 kedro-airflow/kedro_airflow
+
+      - id: pylint-kedro-airflow-features
+        name: "PyLint on kedro-airflow features/*"
+        language: system
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=missing-docstring,no-name-in-module kedro-airflow/features
+
+      - id: pylint-kedro-airflow-tests
+        name: "PyLint on kedro-airflow tests/*"
+        language: system
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-airflow/tests
+
+      - id: pylint-kedro-docker
+        name: "PyLint on kedro_docker/*"
+        language: system
+        files: ^kedro-docker/kedro_docker/.*\.py$
+        exclude: ^(?!kedro-docker/kedro_docker/).*\.py$
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=unnecessary-pass,E0401 kedro-docker/kedro_docker
+
+      - id: pylint-kedro-docker-features
+        name: "PyLint on kedro-docker features/*"
+        language: system
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=missing-docstring,no-name-in-module kedro-docker/features
+
+      - id: pylint-kedro-docker-tests
+        name: "PyLint on kedro-docker tests/*"
+        language: system
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=missing-docstring,redefined-outer-name,invalid-name,protected-access,too-many-arguments kedro-docker/tests
+
+      - id: pylint-kedro-telemetry
+        name: "PyLint on kedro_telemetry/*"
+        language: system
+        files: ^kedro-telemetry/kedro_telemetry/.*\.py$
+        exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=unnecessary-pass,E0401 kedro-telemetry/kedro_telemetry
+
+      - id: pylint-kedro-telemetry-features
+        name: "PyLint on kedro-docker features/*"
+        language: system
+        stages: [ manual ]
+        entry: echo 'Not needed to run for this directory'
+        files: .*
+
+      - id: pylint-kedro-telemetry-tests
+        name: "PyLint on kedro-telemetry tests/*"
+        language: system
+        pass_filenames: false
+        stages: [manual]
+        entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-telemetry/tests
+
+      - id: isort-kedro-datasets
+        name: "Sort imports"
+        language: system
+        types: [ file, python ]
+        files: ^kedro-datasets/
+        entry: isort
+
+      - id: isort-kedro-docker
+        name: "Sort imports"
+        language: system
+        types: [ file, python ]
+        files: ^kedro-docker/
+        entry: isort
+
+      - id: isort-kedro-airflow
+        name: "Sort imports"
+        language: system
+        types: [ file, python ]
+        files: ^kedro-airflow/
+        entry: isort
+
+      - id: isort-kedro-telemetry
+        name: "Sort imports"
+        language: system
+        types: [ file, python ]
+        files: ^kedro-telemetry/
+        entry: isort
+
+      - id: black-kedro-datasets
+        name: "Black"
+        language: system
+        pass_filenames: false
+        entry: black kedro-datasets/kedro_datasets kedro-datasets/features kedro-datasets/tests
+
+      - id: black-kedro-docker
+        name: "Black"
+        language: system
+        pass_filenames: false
+        entry: black kedro-docker/kedro_docker kedro-docker/features kedro-docker/tests
+
+      - id: black-kedro-airflow
+        name: "Black"
+        language: system
+        pass_filenames: false
+        entry: black kedro-airflow/kedro_airflow kedro-airflow/features kedro-airflow/tests
+
+      - id: black-kedro-telemetry
+        name: "Black"
+        language: system
+        pass_filenames: false
+        entry: black kedro-telemetry/kedro_telemetry kedro-telemetry/tests
+
+      - id: secret_scan
+        name: "Secret scan"
+        language: system
+        pass_filenames: false
+        entry: make secret-scan
+
+      - id: bandit
+        name: "Bandit security check"
+        language: system
+        types: [file, python]
+        exclude: ^kedro-(telemetry|docker|datasets|airflow)/tests/|^kedro-(telemetry|docker|datasets|airflow)/features/
+        entry: bandit -ll
diff --git a/Makefile b/Makefile
@@ -14,7 +14,7 @@ install-pip-setuptools:
 	python -m pip install -U pip setuptools wheel
 
 lint:
-	cd $(plugin) && pre-commit run -a --hook-stage manual
+	pre-commit run trailing-whitespace --all-files && pre-commit run end-of-file-fixer --all-files && pre-commit run check-yaml --all-files && pre-commit run check-added-large-files --all-files && pre-commit run check-case-conflict --all-files && pre-commit run check-merge-conflict --all-files && pre-commit run debug-statements --all-files && pre-commit run requirements-txt-fixer --all-files && pre-commit run flake8 --all-files && pre-commit run isort-$(plugin) --all-files --hook-stage manual && pre-commit run black-$(plugin) --all-files --hook-stage manual && pre-commit run secret_scan --all-files --hook-stage manual && pre-commit run bandit --all-files --hook-stage manual && pre-commit run pylint-$(plugin) --all-files --hook-stage manual && pre-commit run pylint-$(plugin)-features --all-files --hook-stage manual && pre-commit run pylint-$(plugin)-tests --all-files --hook-stage manual
 
 test:
 	cd $(plugin) && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile

diff --git a/kedro-airflow/.pre-commit-config.yaml b/kedro-airflow/.pre-commit-config.yaml