diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml index 20d1c14bb..6926215ee 100644 --- a/.github/workflows/kedro-airflow.yml +++ b/.github/workflows/kedro-airflow.yml @@ -34,7 +34,7 @@ jobs: with: plugin: kedro-airflow os: ubuntu-latest - python-version: "3.8" + python-version: "3.11" e2e-tests: strategy: diff --git a/.github/workflows/kedro-docker.yml b/.github/workflows/kedro-docker.yml index 3ffec91a3..3fcae7b36 100644 --- a/.github/workflows/kedro-docker.yml +++ b/.github/workflows/kedro-docker.yml @@ -34,7 +34,7 @@ jobs: with: plugin: kedro-docker os: ubuntu-latest - python-version: "3.8" + python-version: "3.11" e2e-tests: strategy: diff --git a/.github/workflows/kedro-telemetry.yml b/.github/workflows/kedro-telemetry.yml index 034965230..00e9b69ee 100644 --- a/.github/workflows/kedro-telemetry.yml +++ b/.github/workflows/kedro-telemetry.yml @@ -34,7 +34,7 @@ jobs: with: plugin: kedro-telemetry os: ubuntu-latest - python-version: "3.8" + python-version: "3.11" e2e-tests: strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f45c6c8e4..e8804f2cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_stages: [commit, manual] repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v3.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -14,178 +14,44 @@ repos: - id: check-case-conflict # Check for files that would conflict in case-insensitive filesystems - id: check-merge-conflict # Check for files that contain merge conflict strings. - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source. - - id: flake8 - files: ^(kedro-datasets/kedro_datasets/|kedro-airflow/kedro_airflow/|kedro-docker/kedro_docker/|kedro-telemetry/kedro_telemetry/) - args: - - "--max-line-length=88" - - "--max-complexity=18" - - "--select=B,C,E,F,W,T4,B9" - - "--ignore=E203,E266,E501,W503" - exclude: "^kedro_airflow/dag_template.py|^template.py" - repo: local hooks: - # pylint quick checks - - id: pylint-quick-kedro-datasets - name: "Quick PyLint on kedro_datasets/*" + - id: ruff-kedro-datasets + name: "Ruff on kedro_datasets/*" language: system - types: [file, python] files: ^kedro-datasets/kedro_datasets/ exclude: ^(?!kedro-datasets/kedro_datasets/).*\.py$ - entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=unnecessary-pass - stages: [commit] - - - id: pylint-quick-kedro-airflow - name: "Quick PyLint on kedro_airflow/*" - language: system - types: [file, python] - files: ^kedro-airflow/kedro_airflow/ - exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$ - entry: pylint --disable=unnecessary-pass - stages: [commit] - - - id: pylint-quick-kedro-docker - name: "Quick PyLint on kedro_docker/*" - language: system - types: [file, python] - files: ^kedro-docker/kedro_docker/ - exclude: ^(?!kedro-docker/kedro_docker/).*\.py$ - entry: pylint --disable=unnecessary-pass - stages: [commit] - - - id: pylint-quick-kedro-telemetry - name: "Quick PyLint on kedro_telemetry/*" - language: system - types: [file, python] - files: ^kedro-telemetry/kedro_telemetry/ - exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$ - entry: pylint --disable=unnecessary-pass - stages: [commit] - - # pylint full checks - - id: pylint-kedro-datasets - name: "PyLint on kedro_datasets/*" - language: system - files: ^kedro-datasets/kedro_datasets/.*\.py$ - exclude: ^(?!kedro-datasets/kedro_datasets/).*\.py$ pass_filenames: false - stages: [manual] - entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=unnecessary-pass,E0401 kedro-datasets/kedro_datasets - - - id: pylint-kedro-datasets-features - name: "PyLint on kedro-datasets features/*" - language: system - files: ^kedro-datasets/features/.*\.py$ - exclude: ^(?!kedro-datasets/features/).*\.py$ - pass_filenames: false - stages: [manual] - entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=missing-docstring,no-name-in-module,E0401 kedro-datasets/features - - - id: pylint-kedro-datasets-tests - name: "PyLint on kedro-datasets tests/*" - language: system - files: ^kedro-datasets/tests/.*\.py$ - exclude: ^(?!kedro-datasets/tests/).*\.py$ - pass_filenames: false - stages: [manual] - entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,E0401 kedro-datasets/tests + stages: [ manual ] + entry: ruff kedro-datasets --fix --exit-non-zero-on-fix - - id: pylint-kedro-airflow - name: "PyLint on kedro_airflow/*" + - id: ruff-kedro-airflow + name: "Ruff on kedro_airflow/*" language: system - files: ^kedro-airflow/kedro_airflow/.*\.py$ + files: ^kedro-airflow/kedro_airflow/ exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$ pass_filenames: false - stages: [manual] - entry: pylint --disable=unnecessary-pass,E0401 kedro-airflow/kedro_airflow - - - id: pylint-kedro-airflow-features - name: "PyLint on kedro-airflow features/*" - language: system - pass_filenames: false - stages: [manual] - entry: pylint --disable=missing-docstring,no-name-in-module kedro-airflow/features - - - id: pylint-kedro-airflow-tests - name: "PyLint on kedro-airflow tests/*" - language: system - pass_filenames: false - stages: [manual] - entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-airflow/tests + stages: [ manual ] + entry: ruff kedro-airflow --fix --exit-non-zero-on-fix - - id: pylint-kedro-docker - name: "PyLint on kedro_docker/*" + - id: ruff-kedro-docker + name: "Ruff on kedro_docker/*" language: system - files: ^kedro-docker/kedro_docker/.*\.py$ + files: ^kedro-docker/kedro_docker/ exclude: ^(?!kedro-docker/kedro_docker/).*\.py$ pass_filenames: false - stages: [manual] - entry: pylint --disable=unnecessary-pass,E0401 kedro-docker/kedro_docker - - - id: pylint-kedro-docker-features - name: "PyLint on kedro-docker features/*" - language: system - pass_filenames: false - stages: [manual] - entry: pylint --disable=missing-docstring,no-name-in-module kedro-docker/features - - - id: pylint-kedro-docker-tests - name: "PyLint on kedro-docker tests/*" - language: system - pass_filenames: false - stages: [manual] - entry: pylint --disable=missing-docstring,redefined-outer-name,invalid-name,protected-access,too-many-arguments kedro-docker/tests - - - id: pylint-kedro-telemetry - name: "PyLint on kedro_telemetry/*" - language: system - files: ^kedro-telemetry/kedro_telemetry/.*\.py$ - exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$ - pass_filenames: false - stages: [manual] - entry: pylint --disable=unnecessary-pass,E0401 kedro-telemetry/kedro_telemetry - - - id: pylint-kedro-telemetry-features - name: "PyLint on kedro-docker features/*" - language: system stages: [ manual ] - entry: echo 'Not needed to run for this directory' - files: .* + entry: ruff kedro-docker --fix --exit-non-zero-on-fix - - id: pylint-kedro-telemetry-tests - name: "PyLint on kedro-telemetry tests/*" + - id: ruff-kedro-telemetry + name: "Ruff on kedro_telemetry/*" language: system + files: ^kedro-telemetry/kedro_telemetry/ + exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$ pass_filenames: false stages: [manual] - entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-telemetry/tests - - - id: isort-kedro-datasets - name: "Sort imports" - language: system - types: [ file, python ] - files: ^kedro-datasets/ - entry: isort - - - id: isort-kedro-docker - name: "Sort imports" - language: system - types: [ file, python ] - files: ^kedro-docker/ - entry: isort - - - id: isort-kedro-airflow - name: "Sort imports" - language: system - types: [ file, python ] - files: ^kedro-airflow/ - entry: isort - - - id: isort-kedro-telemetry - name: "Sort imports" - language: system - types: [ file, python ] - files: ^kedro-telemetry/ - entry: isort + entry: ruff kedro-telemetry --fix --exit-non-zero-on-fix - id: black-kedro-datasets name: "Black" diff --git a/Makefile b/Makefile index 1c6c7e478..03e74bec0 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ install-pip-setuptools: python -m pip install -U pip setuptools wheel lint: - pre-commit run trailing-whitespace --all-files && pre-commit run end-of-file-fixer --all-files && pre-commit run check-yaml --all-files && pre-commit run check-added-large-files --all-files && pre-commit run check-case-conflict --all-files && pre-commit run check-merge-conflict --all-files && pre-commit run debug-statements --all-files && pre-commit run flake8 --all-files && pre-commit run isort-$(plugin) --all-files --hook-stage manual && pre-commit run black-$(plugin) --all-files --hook-stage manual && pre-commit run secret_scan --all-files --hook-stage manual && pre-commit run bandit --all-files --hook-stage manual && pre-commit run pylint-$(plugin) --all-files --hook-stage manual && pre-commit run pylint-$(plugin)-features --all-files --hook-stage manual && pre-commit run pylint-$(plugin)-tests --all-files --hook-stage manual + pre-commit run -a --hook-stage manual ruff-$(plugin) && pre-commit run trailing-whitespace --all-files && pre-commit run end-of-file-fixer --all-files && pre-commit run check-yaml --all-files && pre-commit run check-added-large-files --all-files && pre-commit run check-case-conflict --all-files && pre-commit run check-merge-conflict --all-files && pre-commit run debug-statements --all-files && pre-commit run black-$(plugin) --all-files --hook-stage manual && pre-commit run secret_scan --all-files --hook-stage manual && pre-commit run bandit --all-files --hook-stage manual test: cd $(plugin) && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile diff --git a/kedro-airflow/.pylintrc b/kedro-airflow/.pylintrc deleted file mode 100644 index 6a2acae02..000000000 --- a/kedro-airflow/.pylintrc +++ /dev/null @@ -1,425 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns=.*template\.py - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint.extensions.docparams - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=ungrouped-imports,bad-continuation - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=useless-suppression - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=20 - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=1 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/kedro-airflow/CONTRIBUTING.md b/kedro-airflow/CONTRIBUTING.md index 2d3e4c020..0d081ed7f 100644 --- a/kedro-airflow/CONTRIBUTING.md +++ b/kedro-airflow/CONTRIBUTING.md @@ -84,20 +84,20 @@ pip install ".[test]" All checks run by our CI / CD pipeline can be run locally on your computer. -#### PEP-8 Standards (`isort`, `pylint` and `flake8`) +#### Linting (`ruff` and `black`) ```bash -make lint +make plugin=kedro-airflow lint ``` #### Unit tests, 100% coverage (`pytest`, `pytest-cov`) ```bash -make test +make plugin=kedro-airflow test ``` #### End-to-end tests (`behave`) ```bash -make e2e-tests +make plugin=kedro-airflow e2e-tests ``` diff --git a/kedro-airflow/features/environment.py b/kedro-airflow/features/environment.py index 0da6ac934..8f87afd7f 100644 --- a/kedro-airflow/features/environment.py +++ b/kedro-airflow/features/environment.py @@ -11,7 +11,6 @@ def before_scenario(context, scenario): - # pylint: disable=unused-argument """Environment preparation before other cli tests are run. Installs kedro by running pip in the top level directory. """ @@ -56,7 +55,6 @@ def call(cmd, print_output=False): def after_scenario(context, scenario): - # pylint: disable=unused-argument rmtree(str(context.temp_dir)) rmtree(str(context.venv_dir)) diff --git a/kedro-airflow/features/steps/cli_steps.py b/kedro-airflow/features/steps/cli_steps.py index 79dde5622..23eb58727 100644 --- a/kedro-airflow/features/steps/cli_steps.py +++ b/kedro-airflow/features/steps/cli_steps.py @@ -2,6 +2,7 @@ import yaml from behave import given, then, when + from features.steps.sh_run import run OK_EXIT_CODE = 0 diff --git a/kedro-airflow/features/steps/sh_run.py b/kedro-airflow/features/steps/sh_run.py index 634eab66e..cc8afc413 100644 --- a/kedro-airflow/features/steps/sh_run.py +++ b/kedro-airflow/features/steps/sh_run.py @@ -34,10 +34,7 @@ def run( """ if isinstance(cmd, str) and split: cmd = shlex.split(cmd) - # pylint: disable=subprocess-run-check - result = subprocess.run( - cmd, input="", stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) + result = subprocess.run(cmd, input="", capture_output=True, **kwargs) result.stdout = result.stdout.decode("utf-8") result.stderr = result.stderr.decode("utf-8") if print_output: diff --git a/kedro-airflow/kedro_airflow/plugin.py b/kedro-airflow/kedro_airflow/plugin.py index 569e91be2..921643c8e 100644 --- a/kedro-airflow/kedro_airflow/plugin.py +++ b/kedro-airflow/kedro_airflow/plugin.py @@ -22,7 +22,7 @@ @click.group(name="Kedro-Airflow") -def commands(): # pylint: disable=missing-function-docstring +def commands(): pass @@ -88,14 +88,14 @@ def _load_config(context: KedroContext, pipeline_name: str) -> dict[str, Any]: callback=_split_params, ) @click.pass_obj -def create( +def create( # noqa: PLR0913 metadata: ProjectMetadata, pipeline_name, env, target_path, jinja_file, params, -): # pylint: disable=too-many-locals,too-many-arguments +): """Create an Airflow DAG for a project""" project_path = Path.cwd().resolve() bootstrap_project(project_path) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index ca177dfbd..50f5eabee 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -28,15 +28,14 @@ test = [ "bandit", "behave", "black~=22.0", - "flake8", "kedro-datasets", - "pre-commit>=1.17.0, <2.0", - "pylint>=2.5.2, <3.0", + "pre-commit>=2.9.2", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist", "trufflehog>=2.1.0, <3.0", + "ruff~=0.0.290", "wheel" ] @@ -72,3 +71,20 @@ fail_under = 100 show_missing = true omit = ["tests/*"] exclude_lines = ["pragma: no cover", "raise NotImplementedError"] + +[tool.ruff] +line-length = 88 +show-fixes = true +select = [ + "F", # Pyflakes + "W", # pycodestyle + "E", # pycodestyle + "I", # isort + "UP", # pyupgrade + "PL", # Pylint + "T201", # Print Statement +] +ignore = ["E501"] # Black takes care of line-too-long + +[tool.ruff.per-file-ignores] +"{tests,features}/*" = ["T201", "PLR2004", "PLR0915", "PLW1510"] diff --git a/kedro-airflow/tests/conftest.py b/kedro-airflow/tests/conftest.py index ea285bb2c..4fc790668 100644 --- a/kedro-airflow/tests/conftest.py +++ b/kedro-airflow/tests/conftest.py @@ -42,7 +42,7 @@ def _create_kedro_settings_py(file_name: Path, patterns: list[str]): @fixture(scope="session") -def kedro_project(cli_runner): # pylint: disable=unused-argument +def kedro_project(cli_runner): tmp_path = Path().cwd() # From `kedro-mlflow.tests.conftest.py` config = { @@ -98,7 +98,7 @@ def register_pipelines(): @fixture(scope="session") -def metadata(kedro_project): # pylint: disable=unused-argument +def metadata(kedro_project): # cwd() depends on ^ the isolated filesystem, created by CliRunner() project_path = kedro_project return ProjectMetadata( diff --git a/kedro-airflow/tests/test_plugin.py b/kedro-airflow/tests/test_plugin.py index 4b67ff840..2bcdde472 100644 --- a/kedro-airflow/tests/test_plugin.py +++ b/kedro-airflow/tests/test_plugin.py @@ -5,6 +5,7 @@ import pytest import yaml + from kedro_airflow.plugin import commands @@ -46,9 +47,7 @@ def _create_kedro_airflow_yml(file_name: Path, content: dict[str, Any]): yaml.dump(content, fp) -def test_airflow_config_params( - cli_runner, metadata -): # pylint: disable=too-many-statements +def test_airflow_config_params(cli_runner, metadata): """Check if config variables are picked up""" dag_name = "hello_world" template_name = "airflow_params.j2" diff --git a/kedro-datasets/docs/source/conf.py b/kedro-datasets/docs/source/conf.py index 4b231efe9..c5e84732c 100644 --- a/kedro-datasets/docs/source/conf.py +++ b/kedro-datasets/docs/source/conf.py @@ -370,8 +370,8 @@ def autodoc_process_docstring(app, what, name, obj, options, lines): print( style( "Failed to check for class name mentions that can be " - "converted to reStructuredText links in docstring of {}. " - "Error is: \n{}".format(name, str(e)), + f"converted to reStructuredText links in docstring of {name}. " + f"Error is: \n{str(e)}", fg="red", ) ) @@ -430,7 +430,7 @@ def setup(app): style( "Failed to create list of (regex, reStructuredText link " "replacement) for class names and method names in docstrings. " - "Error is: \n{}".format(str(e)), + f"Error is: \n{str(e)}", fg="red", ) ) diff --git a/kedro-datasets/kedro_datasets/api/api_dataset.py b/kedro-datasets/kedro_datasets/api/api_dataset.py index 7081eaed7..8a696f456 100644 --- a/kedro-datasets/kedro_datasets/api/api_dataset.py +++ b/kedro-datasets/kedro_datasets/api/api_dataset.py @@ -91,9 +91,8 @@ class APIDataset(AbstractDataset[None, requests.Response]): "timeout": 60, "chunk_size": 100, } - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, url: str, method: str = "GET", diff --git a/kedro-datasets/kedro_datasets/biosequence/biosequence_dataset.py b/kedro-datasets/kedro_datasets/biosequence/biosequence_dataset.py index a85ff6bd9..d24d38ba0 100644 --- a/kedro-datasets/kedro_datasets/biosequence/biosequence_dataset.py +++ b/kedro-datasets/kedro_datasets/biosequence/biosequence_dataset.py @@ -42,8 +42,7 @@ class BioSequenceDataset(AbstractDataset[List, List]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/dask/parquet_dataset.py b/kedro-datasets/kedro_datasets/dask/parquet_dataset.py index 713d08651..9900e1a19 100644 --- a/kedro-datasets/kedro_datasets/dask/parquet_dataset.py +++ b/kedro-datasets/kedro_datasets/dask/parquet_dataset.py @@ -88,8 +88,7 @@ class ParquetDataset(AbstractDataset[dd.DataFrame, dd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {"write_index": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/databricks/managed_table_dataset.py b/kedro-datasets/kedro_datasets/databricks/managed_table_dataset.py index b46511ff0..e2e847484 100644 --- a/kedro-datasets/kedro_datasets/databricks/managed_table_dataset.py +++ b/kedro-datasets/kedro_datasets/databricks/managed_table_dataset.py @@ -43,7 +43,7 @@ def __post_init__(self): The validation is performed by calling a function named: `validate_(self, value) -> raises DatasetError` """ - for name in self.__dataclass_fields__.keys(): # pylint: disable=no-member + for name in self.__dataclass_fields__.keys(): method = getattr(self, f"_validate_{name}", None) if method: method() @@ -194,7 +194,7 @@ class ManagedTableDataset(AbstractVersionedDataset): # using ``ThreadRunner`` instead _SINGLE_PROCESS = True - def __init__( # pylint: disable=R0913 + def __init__( # noqa: PLR0913 self, table: str, catalog: str = None, @@ -383,9 +383,8 @@ def _save(self, data: Union[DataFrame, pd.DataFrame]) -> None: ) else: data = data.select(*cols) - else: - if self._table.dataframe_type == "pandas": - data = self._get_spark().createDataFrame(data) + elif self._table.dataframe_type == "pandas": + data = self._get_spark().createDataFrame(data) if self._table.write_mode == "overwrite": self._save_overwrite(data) elif self._table.write_mode == "upsert": diff --git a/kedro-datasets/kedro_datasets/email/message_dataset.py b/kedro-datasets/kedro_datasets/email/message_dataset.py index 573ea55dd..b81dc7804 100644 --- a/kedro-datasets/kedro_datasets/email/message_dataset.py +++ b/kedro-datasets/kedro_datasets/email/message_dataset.py @@ -50,8 +50,7 @@ class EmailMessageDataset(AbstractVersionedDataset[Message, Message]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/geopandas/geojson_dataset.py b/kedro-datasets/kedro_datasets/geopandas/geojson_dataset.py index 334b83ac5..3c5807b9a 100644 --- a/kedro-datasets/kedro_datasets/geopandas/geojson_dataset.py +++ b/kedro-datasets/kedro_datasets/geopandas/geojson_dataset.py @@ -44,8 +44,7 @@ class GeoJSONDataset( DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS = {"driver": "GeoJSON"} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/holoviews/holoviews_writer.py b/kedro-datasets/kedro_datasets/holoviews/holoviews_writer.py index 5cb1bf138..7d64b8bf6 100644 --- a/kedro-datasets/kedro_datasets/holoviews/holoviews_writer.py +++ b/kedro-datasets/kedro_datasets/holoviews/holoviews_writer.py @@ -35,8 +35,7 @@ class HoloviewsWriter(AbstractVersionedDataset[HoloViews, NoReturn]): DEFAULT_SAVE_ARGS: Dict[str, Any] = {"fmt": "png"} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, fs_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/json/json_dataset.py b/kedro-datasets/kedro_datasets/json/json_dataset.py index fcb489466..6cae55cce 100644 --- a/kedro-datasets/kedro_datasets/json/json_dataset.py +++ b/kedro-datasets/kedro_datasets/json/json_dataset.py @@ -48,8 +48,7 @@ class JSONDataset(AbstractVersionedDataset[Any, Any]): DEFAULT_SAVE_ARGS: Dict[str, Any] = {"indent": 2} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, save_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/matplotlib/matplotlib_writer.py b/kedro-datasets/kedro_datasets/matplotlib/matplotlib_writer.py index f17174c96..568928caf 100644 --- a/kedro-datasets/kedro_datasets/matplotlib/matplotlib_writer.py +++ b/kedro-datasets/kedro_datasets/matplotlib/matplotlib_writer.py @@ -103,8 +103,7 @@ class MatplotlibWriter( DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, fs_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/networkx/gml_dataset.py b/kedro-datasets/kedro_datasets/networkx/gml_dataset.py index c27978885..cc7d21bf0 100644 --- a/kedro-datasets/kedro_datasets/networkx/gml_dataset.py +++ b/kedro-datasets/kedro_datasets/networkx/gml_dataset.py @@ -36,8 +36,7 @@ class GMLDataset(AbstractVersionedDataset[networkx.Graph, networkx.Graph]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/networkx/graphml_dataset.py b/kedro-datasets/kedro_datasets/networkx/graphml_dataset.py index 1704c4a78..902b29114 100644 --- a/kedro-datasets/kedro_datasets/networkx/graphml_dataset.py +++ b/kedro-datasets/kedro_datasets/networkx/graphml_dataset.py @@ -35,8 +35,7 @@ class GraphMLDataset(AbstractVersionedDataset[networkx.Graph, networkx.Graph]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/networkx/json_dataset.py b/kedro-datasets/kedro_datasets/networkx/json_dataset.py index 91b2fbc53..3d565003d 100644 --- a/kedro-datasets/kedro_datasets/networkx/json_dataset.py +++ b/kedro-datasets/kedro_datasets/networkx/json_dataset.py @@ -36,8 +36,7 @@ class JSONDataset(AbstractVersionedDataset[networkx.Graph, networkx.Graph]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/pandas/csv_dataset.py b/kedro-datasets/kedro_datasets/pandas/csv_dataset.py index 94bf9384e..4887968cd 100644 --- a/kedro-datasets/kedro_datasets/pandas/csv_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/csv_dataset.py @@ -69,8 +69,7 @@ class CSVDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {"index": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, @@ -198,7 +197,7 @@ def _invalidate_cache(self) -> None: def _preview(self, nrows: int = 40) -> Dict: # Create a copy so it doesn't contaminate the original dataset dataset_copy = self._copy() - dataset_copy._load_args["nrows"] = nrows # pylint: disable=protected-access + dataset_copy._load_args["nrows"] = nrows data = dataset_copy.load() return data.to_dict(orient="split") diff --git a/kedro-datasets/kedro_datasets/pandas/deltatable_dataset.py b/kedro-datasets/kedro_datasets/pandas/deltatable_dataset.py index cbf1413dc..9df340c6d 100644 --- a/kedro-datasets/kedro_datasets/pandas/deltatable_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/deltatable_dataset.py @@ -14,7 +14,7 @@ from kedro_datasets._io import AbstractDataset, DatasetError -class DeltaTableDataset(AbstractDataset): # pylint:disable=too-many-instance-attributes +class DeltaTableDataset(AbstractDataset): """``DeltaTableDataset`` loads/saves delta tables from/to a filesystem (e.g.: local, S3, GCS), Databricks unity catalog and AWS Glue catalog respectively. It handles load and save using a pandas dataframe. When saving data, you can specify one of two @@ -84,7 +84,7 @@ class DeltaTableDataset(AbstractDataset): # pylint:disable=too-many-instance-at DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {"mode": DEFAULT_WRITE_MODE} - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: PLR0913 self, filepath: Optional[str] = None, catalog_type: Optional[DataCatalog] = None, diff --git a/kedro-datasets/kedro_datasets/pandas/excel_dataset.py b/kedro-datasets/kedro_datasets/pandas/excel_dataset.py index 8ffc814bd..181e6cd71 100644 --- a/kedro-datasets/kedro_datasets/pandas/excel_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/excel_dataset.py @@ -109,8 +109,7 @@ class ExcelDataset( DEFAULT_LOAD_ARGS = {"engine": "openpyxl"} DEFAULT_SAVE_ARGS = {"index": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, engine: str = "openpyxl", @@ -232,7 +231,6 @@ def _save(self, data: Union[pd.DataFrame, Dict[str, pd.DataFrame]]) -> None: output = BytesIO() save_path = get_filepath_str(self._get_save_path(), self._protocol) - # pylint: disable=abstract-class-instantiated with pd.ExcelWriter(output, **self._writer_args) as writer: if isinstance(data, dict): for sheet_name, sheet_data in data.items(): @@ -267,7 +265,7 @@ def _invalidate_cache(self) -> None: def _preview(self, nrows: int = 40) -> Dict: # Create a copy so it doesn't contaminate the original dataset dataset_copy = self._copy() - dataset_copy._load_args["nrows"] = nrows # pylint: disable=protected-access + dataset_copy._load_args["nrows"] = nrows data = dataset_copy.load() return data.to_dict(orient="split") diff --git a/kedro-datasets/kedro_datasets/pandas/feather_dataset.py b/kedro-datasets/kedro_datasets/pandas/feather_dataset.py index c409493d9..45a454dcf 100644 --- a/kedro-datasets/kedro_datasets/pandas/feather_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/feather_dataset.py @@ -70,8 +70,7 @@ class FeatherDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/pandas/gbq_dataset.py b/kedro-datasets/kedro_datasets/pandas/gbq_dataset.py index c39a37ed0..8dba87dd8 100644 --- a/kedro-datasets/kedro_datasets/pandas/gbq_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/gbq_dataset.py @@ -65,8 +65,7 @@ class GBQTableDataset(AbstractDataset[None, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {"progress_bar": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, dataset: str, table_name: str, @@ -210,8 +209,7 @@ class GBQQueryDataset(AbstractDataset[None, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, sql: str = None, project: str = None, @@ -316,7 +314,7 @@ def _load(self) -> pd.DataFrame: **load_args, ) - def _save(self, data: None) -> NoReturn: # pylint: disable=no-self-use + def _save(self, data: None) -> NoReturn: raise DatasetError("'save' is not supported on GBQQueryDataset") diff --git a/kedro-datasets/kedro_datasets/pandas/generic_dataset.py b/kedro-datasets/kedro_datasets/pandas/generic_dataset.py index eae3f9b3a..d9395b8e8 100644 --- a/kedro-datasets/kedro_datasets/pandas/generic_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/generic_dataset.py @@ -81,8 +81,7 @@ class GenericDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, file_format: str, diff --git a/kedro-datasets/kedro_datasets/pandas/hdf_dataset.py b/kedro-datasets/kedro_datasets/pandas/hdf_dataset.py index 6fb94ba23..50d33e460 100644 --- a/kedro-datasets/kedro_datasets/pandas/hdf_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/hdf_dataset.py @@ -56,8 +56,7 @@ class HDFDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, key: str, @@ -177,7 +176,6 @@ def _save(self, data: pd.DataFrame) -> None: **self._save_args, ) as store: store.put(self._key, data, format="table") - # pylint: disable=protected-access binary_data = store._handle.get_file_image() with self._fs.open(save_path, **self._fs_open_args_save) as fs_file: diff --git a/kedro-datasets/kedro_datasets/pandas/json_dataset.py b/kedro-datasets/kedro_datasets/pandas/json_dataset.py index c6c87e17f..91dd2930d 100644 --- a/kedro-datasets/kedro_datasets/pandas/json_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/json_dataset.py @@ -64,8 +64,7 @@ class JSONDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/pandas/parquet_dataset.py b/kedro-datasets/kedro_datasets/pandas/parquet_dataset.py index 96f35ff66..dc4c05618 100644 --- a/kedro-datasets/kedro_datasets/pandas/parquet_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/parquet_dataset.py @@ -75,8 +75,7 @@ class ParquetDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/pandas/sql_dataset.py b/kedro-datasets/kedro_datasets/pandas/sql_dataset.py index 59c1c20b2..59feb51b4 100644 --- a/kedro-datasets/kedro_datasets/pandas/sql_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/sql_dataset.py @@ -153,8 +153,7 @@ class SQLTableDataset(AbstractDataset[pd.DataFrame, pd.DataFrame]): # sqlalchemy.engine.Engine or sqlalchemy.engine.base.Engine engines: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, table_name: str, credentials: Dict[str, Any], @@ -376,7 +375,7 @@ class SQLQueryDataset(AbstractDataset[None, pd.DataFrame]): # sqlalchemy.engine.Engine or sqlalchemy.engine.base.Engine engines: Dict[str, Any] = {} - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: PLR0913 self, sql: str = None, credentials: Dict[str, Any] = None, @@ -509,7 +508,7 @@ def _load(self) -> pd.DataFrame: return pd.read_sql_query(con=engine, **load_args) - def _save(self, data: None) -> NoReturn: # pylint: disable=no-self-use + def _save(self, data: None) -> NoReturn: raise DatasetError("'save' is not supported on SQLQueryDataset") # For mssql only diff --git a/kedro-datasets/kedro_datasets/pandas/xml_dataset.py b/kedro-datasets/kedro_datasets/pandas/xml_dataset.py index 43dd40084..129d5e3fb 100644 --- a/kedro-datasets/kedro_datasets/pandas/xml_dataset.py +++ b/kedro-datasets/kedro_datasets/pandas/xml_dataset.py @@ -47,8 +47,7 @@ class XMLDataset(AbstractVersionedDataset[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {"index": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py b/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py index 52004f4e8..19f8072a0 100644 --- a/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py +++ b/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py @@ -68,8 +68,7 @@ class PickleDataset(AbstractVersionedDataset[Any, Any]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments,too-many-locals - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, backend: str = "pickle", diff --git a/kedro-datasets/kedro_datasets/pillow/image_dataset.py b/kedro-datasets/kedro_datasets/pillow/image_dataset.py index 99a16d572..91bae8842 100644 --- a/kedro-datasets/kedro_datasets/pillow/image_dataset.py +++ b/kedro-datasets/kedro_datasets/pillow/image_dataset.py @@ -32,8 +32,7 @@ class ImageDataset(AbstractVersionedDataset[Image.Image, Image.Image]): DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, save_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/plotly/json_dataset.py b/kedro-datasets/kedro_datasets/plotly/json_dataset.py index 97ad31e27..b21f4f9bc 100644 --- a/kedro-datasets/kedro_datasets/plotly/json_dataset.py +++ b/kedro-datasets/kedro_datasets/plotly/json_dataset.py @@ -50,8 +50,7 @@ class JSONDataset( DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/plotly/plotly_dataset.py b/kedro-datasets/kedro_datasets/plotly/plotly_dataset.py index 9a5e53b20..985588e0a 100644 --- a/kedro-datasets/kedro_datasets/plotly/plotly_dataset.py +++ b/kedro-datasets/kedro_datasets/plotly/plotly_dataset.py @@ -66,8 +66,7 @@ class PlotlyDataset(JSONDataset): """ - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, plotly_args: Dict[str, Any], diff --git a/kedro-datasets/kedro_datasets/polars/csv_dataset.py b/kedro-datasets/kedro_datasets/polars/csv_dataset.py index 1ed8ce2d5..0e87c2bb2 100644 --- a/kedro-datasets/kedro_datasets/polars/csv_dataset.py +++ b/kedro-datasets/kedro_datasets/polars/csv_dataset.py @@ -67,8 +67,7 @@ class CSVDataset(AbstractVersionedDataset[pl.DataFrame, pl.DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {"rechunk": True} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/polars/generic_dataset.py b/kedro-datasets/kedro_datasets/polars/generic_dataset.py index a7e030378..8b790e456 100644 --- a/kedro-datasets/kedro_datasets/polars/generic_dataset.py +++ b/kedro-datasets/kedro_datasets/polars/generic_dataset.py @@ -15,7 +15,6 @@ from kedro_datasets._io import AbstractVersionedDataset, DatasetError -# pylint: disable=too-many-instance-attributes class GenericDataset(AbstractVersionedDataset[pl.DataFrame, pl.DataFrame]): """``polars.GenericDataset`` loads/saves data from/to a data file using an underlying filesystem (e.g.: local, S3, GCS). It uses polars to handle the dynamically select the @@ -54,8 +53,7 @@ class GenericDataset(AbstractVersionedDataset[pl.DataFrame, pl.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, file_format: str, @@ -139,7 +137,7 @@ def __init__( self._fs_open_args_load = _fs_open_args_load self._fs_open_args_save = _fs_open_args_save - def _load(self) -> pl.DataFrame: # pylint: disable= inconsistent-return-statements + def _load(self) -> pl.DataFrame: load_path = get_filepath_str(self._get_load_path(), self._protocol) load_method = getattr(pl, f"read_{self._file_format}", None) diff --git a/kedro-datasets/kedro_datasets/redis/redis_dataset.py b/kedro-datasets/kedro_datasets/redis/redis_dataset.py index 8c2809e7a..dc04de00e 100644 --- a/kedro-datasets/kedro_datasets/redis/redis_dataset.py +++ b/kedro-datasets/kedro_datasets/redis/redis_dataset.py @@ -60,8 +60,7 @@ class PickleDataset(AbstractDataset[Any, Any]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, key: str, backend: str = "pickle", diff --git a/kedro-datasets/kedro_datasets/snowflake/snowpark_dataset.py b/kedro-datasets/kedro_datasets/snowflake/snowpark_dataset.py index 85cdc1450..d98ef2dd6 100644 --- a/kedro-datasets/kedro_datasets/snowflake/snowpark_dataset.py +++ b/kedro-datasets/kedro_datasets/snowflake/snowpark_dataset.py @@ -102,7 +102,7 @@ class SnowparkTableDataset(AbstractDataset): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: PLR0913 self, table_name: str, schema: str = None, diff --git a/kedro-datasets/kedro_datasets/spark/spark_dataset.py b/kedro-datasets/kedro_datasets/spark/spark_dataset.py index 0bf24643d..5971ba495 100644 --- a/kedro-datasets/kedro_datasets/spark/spark_dataset.py +++ b/kedro-datasets/kedro_datasets/spark/spark_dataset.py @@ -37,7 +37,7 @@ def _parse_glob_pattern(pattern: str) -> str: def _split_filepath(filepath: str) -> Tuple[str, str]: split_ = filepath.split("://", 1) - if len(split_) == 2: + if len(split_) == 2: # noqa: PLR2004 return split_[0] + "://", split_[1] return "", split_[0] @@ -80,12 +80,12 @@ def _get_dbutils(spark: SparkSession) -> Optional[Any]: return dbutils try: - from pyspark.dbutils import DBUtils # pylint: disable=import-outside-toplevel + from pyspark.dbutils import DBUtils dbutils = DBUtils(spark) except ImportError: try: - import IPython # pylint: disable=import-outside-toplevel + import IPython except ImportError: pass else: @@ -111,7 +111,7 @@ def _dbfs_exists(pattern: str, dbutils: Any) -> bool: try: dbutils.fs.ls(file) return True - except Exception: # pylint: disable=broad-except + except Exception: return False @@ -233,7 +233,7 @@ class SparkDataset(AbstractVersionedDataset[DataFrame, DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - def __init__( # pylint: disable=too-many-arguments disable=too-many-locals + def __init__( # noqa: PLR0913 self, filepath: str, file_format: str = "parquet", diff --git a/kedro-datasets/kedro_datasets/spark/spark_hive_dataset.py b/kedro-datasets/kedro_datasets/spark/spark_hive_dataset.py index 5343791ee..860855719 100644 --- a/kedro-datasets/kedro_datasets/spark/spark_hive_dataset.py +++ b/kedro-datasets/kedro_datasets/spark/spark_hive_dataset.py @@ -12,7 +12,6 @@ from kedro_datasets._io import AbstractDataset, DatasetError -# pylint:disable=too-many-instance-attributes class SparkHiveDataset(AbstractDataset[DataFrame, DataFrame]): """``SparkHiveDataset`` loads and saves Spark dataframes stored on Hive. This data set also handles some incompatible file types such as using partitioned parquet on @@ -67,8 +66,7 @@ class SparkHiveDataset(AbstractDataset[DataFrame, DataFrame]): DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint:disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, database: str, table: str, @@ -211,7 +209,6 @@ def _validate_save(self, data: DataFrame): ) def _exists(self) -> bool: - # noqa # pylint:disable=protected-access return ( self._get_spark() ._jsparkSession.catalog() diff --git a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py index 301067bb0..c062a6a70 100644 --- a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py +++ b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py @@ -65,8 +65,7 @@ class SparkJDBCDataset(AbstractDataset[DataFrame, DataFrame]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, url: str, table: str, diff --git a/kedro-datasets/kedro_datasets/svmlight/svmlight_dataset.py b/kedro-datasets/kedro_datasets/svmlight/svmlight_dataset.py index 7318cb3b0..2ea1b3be7 100644 --- a/kedro-datasets/kedro_datasets/svmlight/svmlight_dataset.py +++ b/kedro-datasets/kedro_datasets/svmlight/svmlight_dataset.py @@ -86,8 +86,7 @@ class SVMLightDataset(AbstractVersionedDataset[_DI, _DO]): DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro-datasets/kedro_datasets/tensorflow/tensorflow_model_dataset.py b/kedro-datasets/kedro_datasets/tensorflow/tensorflow_model_dataset.py index 1a283a331..18b4274c7 100644 --- a/kedro-datasets/kedro_datasets/tensorflow/tensorflow_model_dataset.py +++ b/kedro-datasets/kedro_datasets/tensorflow/tensorflow_model_dataset.py @@ -60,8 +60,7 @@ class TensorFlowModelDataset(AbstractVersionedDataset[tf.keras.Model, tf.keras.M DEFAULT_LOAD_ARGS: Dict[str, Any] = {} DEFAULT_SAVE_ARGS: Dict[str, Any] = {"save_format": "tf"} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, load_args: Dict[str, Any] = None, @@ -132,7 +131,7 @@ def _load(self) -> tf.keras.Model: with tempfile.TemporaryDirectory(prefix=self._tmp_prefix) as path: if self._is_h5: - path = str(PurePath(path) / TEMPORARY_H5_FILE) + path = str(PurePath(path) / TEMPORARY_H5_FILE) # noqa: PLW2901 self._fs.copy(load_path, path) else: self._fs.get(load_path, path, recursive=True) @@ -151,7 +150,7 @@ def _save(self, data: tf.keras.Model) -> None: with tempfile.TemporaryDirectory(prefix=self._tmp_prefix) as path: if self._is_h5: - path = str(PurePath(path) / TEMPORARY_H5_FILE) + path = str(PurePath(path) / TEMPORARY_H5_FILE) # noqa: PLW2901 tf.keras.models.save_model(data, path, **self._save_args) diff --git a/kedro-datasets/kedro_datasets/text/text_dataset.py b/kedro-datasets/kedro_datasets/text/text_dataset.py index 58c2e2a19..2c1ecff6f 100644 --- a/kedro-datasets/kedro_datasets/text/text_dataset.py +++ b/kedro-datasets/kedro_datasets/text/text_dataset.py @@ -42,8 +42,7 @@ class TextDataset(AbstractVersionedDataset[str, str]): """ - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, version: Version = None, diff --git a/kedro-datasets/kedro_datasets/video/video_dataset.py b/kedro-datasets/kedro_datasets/video/video_dataset.py index cf101de1c..b85fc1231 100644 --- a/kedro-datasets/kedro_datasets/video/video_dataset.py +++ b/kedro-datasets/kedro_datasets/video/video_dataset.py @@ -264,8 +264,7 @@ class VideoDataset(AbstractDataset[AbstractVideo, AbstractVideo]): """ - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, fourcc: Optional[str] = "mp4v", diff --git a/kedro-datasets/kedro_datasets/yaml/yaml_dataset.py b/kedro-datasets/kedro_datasets/yaml/yaml_dataset.py index 76dd94473..45350b338 100644 --- a/kedro-datasets/kedro_datasets/yaml/yaml_dataset.py +++ b/kedro-datasets/kedro_datasets/yaml/yaml_dataset.py @@ -45,8 +45,7 @@ class YAMLDataset(AbstractVersionedDataset[Dict, Dict]): DEFAULT_SAVE_ARGS: Dict[str, Any] = {"default_flow_style": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: PLR0913 self, filepath: str, save_args: Dict[str, Any] = None, diff --git a/kedro-datasets/pyproject.toml b/kedro-datasets/pyproject.toml index 96828d508..d5be97bbc 100644 --- a/kedro-datasets/pyproject.toml +++ b/kedro-datasets/pyproject.toml @@ -28,46 +28,6 @@ include = ["kedro_datasets*"] readme = {file = "README.md", content-type = "text/markdown"} version = {attr = "kedro_datasets.__version__"} -[tool.isort] -profile = "black" - -[tool.pylint] -[tool.pylint.master] -ignore = "CVS" -load-plugins = [ - "pylint.extensions.docparams", - "pylint.extensions.no_self_use" -] -extension-pkg-whitelist = "cv2" -unsafe-load-any-extension = false - -[tool.pylint.messages_control] -disable = [ - "ungrouped-imports", - "duplicate-code", - "too-many-instance-attributes", - "too-few-public-methods", # https://github.com/pylint-dev/pylint/issues/8865 -] -enable = ["useless-suppression"] - -[tool.pylint.refactoring] -max-nested-blocks = 5 - -[tool.pylint.format] -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines='^\s*(# )??$' -indent-after-paren = 4 -indent-string = " " - -[tool.pylint.miscellaneous] -notes = [ - "FIXME", - "XXX" -] - -[tool.pylint.design] -min-public-methods = 1 - [tool.coverage.report] fail_under = 100 show_missing = true @@ -84,3 +44,21 @@ addopts = """ --no-cov-on-fail \ -ra \ -W ignore""" + +[tool.ruff] +line-length = 88 +show-fixes = true +select = [ + "F", # Pyflakes + "W", # pycodestyle + "E", # pycodestyle + "I", # isort + "UP", # pyupgrade + "PL", # Pylint + "T201", # Print Statement +] +ignore = ["E501"] # Black takes care of line-too-long + +[tool.ruff.per-file-ignores] +"{tests,docs}/*" = ["PLR2004", "PLR0913", "T201"] +"*/{__init__.py}" = ["F821"] # temporarily ignore ruff undefined name errors for dataset aliases diff --git a/kedro-datasets/setup.py b/kedro-datasets/setup.py index e79d58954..1535d28dd 100644 --- a/kedro-datasets/setup.py +++ b/kedro-datasets/setup.py @@ -186,10 +186,9 @@ def _collect_requirements(requires): "Pillow~=9.0", "plotly>=4.8.0, <6.0", "polars[xlsx2csv, deltalake]~=0.18.0", - "pre-commit>=2.9.2, <3.0", # The hook `mypy` requires pre-commit version 2.9.2. + "pre-commit>=2.9.2", "pyarrow>=1.0; python_version < '3.11'", "pyarrow>=7.0; python_version >= '3.11'", # Adding to avoid numpy build errors - "pylint>=2.5.2, <3.0", "pyodbc~=4.0.35", "pyproj~=3.0", "pyspark>=2.2, <3.4; python_version < '3.11'", @@ -201,6 +200,7 @@ def _collect_requirements(requires): "redis~=4.1", "requests-mock~=1.6", "requests~=2.20", + "ruff~=0.0.290", "s3fs>=0.3.0, <0.5", # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem. "snowflake-snowpark-python~=1.0.0; python_version == '3.8'", "scikit-learn>=1.0.2,<2", diff --git a/kedro-datasets/tests/api/test_api_dataset.py b/kedro-datasets/tests/api/test_api_dataset.py index e5a0e6827..10a0baf6d 100644 --- a/kedro-datasets/tests/api/test_api_dataset.py +++ b/kedro-datasets/tests/api/test_api_dataset.py @@ -1,4 +1,3 @@ -# pylint: disable=no-member import base64 import importlib import json @@ -296,9 +295,7 @@ def test_successful_save(self, requests_mock, method, data): Then check that the response is OK and the sent data is in the correct form. """ - def json_callback( - request: requests.Request, context: Any # pylint: disable=unused-argument - ) -> dict: + def json_callback(request: requests.Request, context: Any) -> dict: """Callback that sends back the json.""" return request.json() @@ -342,9 +339,7 @@ def test_successful_save_with_json(self, requests_mock, save_methods): Then check we get a response """ - def json_callback( - request: requests.Request, context: Any # pylint: disable=unused-argument - ) -> dict: + def json_callback(request: requests.Request, context: Any) -> dict: """Callback that sends back the json.""" return request.json() diff --git a/kedro-datasets/tests/databricks/conftest.py b/kedro-datasets/tests/databricks/conftest.py index 958ee6a83..afe164adc 100644 --- a/kedro-datasets/tests/databricks/conftest.py +++ b/kedro-datasets/tests/databricks/conftest.py @@ -5,7 +5,7 @@ https://docs.pytest.org/en/latest/fixture.html """ # importlib_metadata needs backport for python 3.8 and older -import importlib_metadata as importlib_metadata # pylint: disable=useless-import-alias +import importlib_metadata import pytest from pyspark.sql import SparkSession diff --git a/kedro-datasets/tests/databricks/test_managed_table_dataset.py b/kedro-datasets/tests/databricks/test_managed_table_dataset.py index 0ae7964ec..dc2595740 100644 --- a/kedro-datasets/tests/databricks/test_managed_table_dataset.py +++ b/kedro-datasets/tests/databricks/test_managed_table_dataset.py @@ -183,7 +183,6 @@ def test_deprecation(module_name, class_name): getattr(importlib.import_module(module_name), class_name) -# pylint: disable=too-many-public-methods class TestManagedTableDataset: def test_full_table(self): unity_ds = ManagedTableDataset(catalog="test", database="test", table="test") diff --git a/kedro-datasets/tests/matplotlib/test_matplotlib_writer.py b/kedro-datasets/tests/matplotlib/test_matplotlib_writer.py index a8e83b2da..5270e13a5 100644 --- a/kedro-datasets/tests/matplotlib/test_matplotlib_writer.py +++ b/kedro-datasets/tests/matplotlib/test_matplotlib_writer.py @@ -32,7 +32,7 @@ def mock_single_plot(): def mock_list_plot(): plots_list = [] colour = "red" - for index in range(5): # pylint: disable=unused-variable + for index in range(5): plots_list.append(plt.figure()) plt.plot([1, 2, 3], [4, 5, 6], color=colour) plt.close("all") @@ -104,9 +104,7 @@ def overwrite(request): @pytest.fixture -def plot_writer( - mocked_s3_bucket, fs_args, save_args, overwrite -): # pylint: disable=unused-argument +def plot_writer(mocked_s3_bucket, fs_args, save_args, overwrite): return MatplotlibWriter( filepath=FULL_PATH, credentials=AWS_CREDENTIALS, diff --git a/kedro-datasets/tests/pandas/test_gbq_dataset.py b/kedro-datasets/tests/pandas/test_gbq_dataset.py index f392f6ae8..be4d65942 100644 --- a/kedro-datasets/tests/pandas/test_gbq_dataset.py +++ b/kedro-datasets/tests/pandas/test_gbq_dataset.py @@ -28,9 +28,7 @@ def mock_bigquery_client(mocker): @pytest.fixture -def gbq_dataset( - load_args, save_args, mock_bigquery_client -): # pylint: disable=unused-argument +def gbq_dataset(load_args, save_args, mock_bigquery_client): return GBQTableDataset( dataset=DATASET, table_name=TABLE_NAME, @@ -42,7 +40,7 @@ def gbq_dataset( @pytest.fixture(params=[{}]) -def gbq_sql_dataset(load_args, mock_bigquery_client): # pylint: disable=unused-argument +def gbq_sql_dataset(load_args, mock_bigquery_client): return GBQQueryDataset( sql=SQL_QUERY, project=PROJECT, @@ -59,9 +57,7 @@ def sql_file(tmp_path: PosixPath): @pytest.fixture(params=[{}]) -def gbq_sql_file_dataset( - load_args, sql_file, mock_bigquery_client -): # pylint: disable=unused-argument +def gbq_sql_file_dataset(load_args, sql_file, mock_bigquery_client): return GBQQueryDataset( filepath=sql_file, project=PROJECT, diff --git a/kedro-datasets/tests/pandas/test_generic_dataset.py b/kedro-datasets/tests/pandas/test_generic_dataset.py index b48e099d1..8cacaa5bc 100644 --- a/kedro-datasets/tests/pandas/test_generic_dataset.py +++ b/kedro-datasets/tests/pandas/test_generic_dataset.py @@ -33,7 +33,6 @@ def filepath_html(tmp_path): return tmp_path / "test.html" -# pylint: disable=line-too-long @pytest.fixture() def sas_binary(): return b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc2\xea\x81`\xb3\x14\x11\xcf\xbd\x92\x08\x00\t\xc71\x8c\x18\x1f\x10\x11""\x002"\x01\x022\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x01\x18\x1f\x10\x11""\x002"\x01\x022\x042\x01""\x00\x00\x00\x00\x10\x03\x01\x00\x00\x00\x00\x00\x00\x00\x00SAS FILEAIRLINE DATA \x00\x00\xc0\x95j\xbe\xd6A\x00\x00\xc0\x95j\xbe\xd6A\x00\x00\x00\x00\x00 \xbc@\x00\x00\x00\x00\x00 \xbc@\x00\x04\x00\x00\x00\x10\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x009.0000M0WIN\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00WIN\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc0\x95LN\xaf\xf0LN\xaf\xf0LN\xaf\xf0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00jIW-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00kIW-\x00\x00\x00\x00\x00\x00\x00\x00<\x04\x00\x00\x00\x02-\x00\r\x00\x00\x00 \x0e\x00\x00\xe0\x01\x00\x00\x00\x00\x00\x00\x14\x0e\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\xe4\x0c\x00\x000\x01\x00\x00\x00\x00\x00\x00H\x0c\x00\x00\x9c\x00\x00\x00\x00\x01\x00\x00\x04\x0c\x00\x00D\x00\x00\x00\x00\x01\x00\x00\xa8\x0b\x00\x00\\\x00\x00\x00\x00\x01\x00\x00t\x0b\x00\x004\x00\x00\x00\x00\x00\x00\x00@\x0b\x00\x004\x00\x00\x00\x00\x00\x00\x00\x0c\x0b\x00\x004\x00\x00\x00\x00\x00\x00\x00\xd8\n\x00\x004\x00\x00\x00\x00\x00\x00\x00\xa4\n\x00\x004\x00\x00\x00\x00\x00\x00\x00p\n\x00\x004\x00\x00\x00\x00\x00\x00\x00p\n\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00p\x9e@\x00\x00\x00@\x8bl\xf3?\x00\x00\x00\xc0\x9f\x1a\xcf?\x00\x00\x00\xa0w\x9c\xc2?\x00\x00\x00\x00\xd7\xa3\xf6?\x00\x00\x00\x00\x81\x95\xe3?\x00t\x9e@\x00\x00\x00\xe0\xfb\xa9\xf5?\x00\x00\x00\x00\xd7\xa3\xd0?\x00\x00\x00`\xb3\xea\xcb?\x00\x00\x00 \xdd$\xf6?\x00\x00\x00\x00T\xe3\xe1?\x00x\x9e@\x00\x00\x00\xc0\x9f\x1a\xf9?\x00\x00\x00\x80\xc0\xca\xd1?\x00\x00\x00\xc0m4\xd4?\x00\x00\x00\x80?5\xf6?\x00\x00\x00 \x04V\xe2?\x00|\x9e@\x00\x00\x00\x00\x02+\xff?\x00\x00\x00@\x0c\x02\xd3?\x00\x00\x00\xc0K7\xd9?\x00\x00\x00\xc0\xcc\xcc\xf8?\x00\x00\x00\xc0I\x0c\xe2?\x00\x80\x9e@\x00\x00\x00`\xb8\x1e\x02@\x00\x00\x00@\n\xd7\xd3?\x00\x00\x00\xc0\x10\xc7\xd6?\x00\x00\x00\x00\xfe\xd4\xfc?\x00\x00\x00@5^\xe2?\x00\x84\x9e@\x00\x00\x00\x80\x16\xd9\x05@\x00\x00\x00\xe0\xa5\x9b\xd4?\x00\x00\x00`\xc5\xfe\xd6?\x00\x00\x00`\xe5\xd0\xfe?\x00\x00\x00 \x83\xc0\xe6?\x00\x88\x9e@\x00\x00\x00@33\x08@\x00\x00\x00\xe0\xa3p\xd5?\x00\x00\x00`\x8f\xc2\xd9?\x00\x00\x00@\x8bl\xff?\x00\x00\x00\x00\xfe\xd4\xe8?\x00\x8c\x9e@\x00\x00\x00\xe0\xf9~\x0c@\x00\x00\x00`ff\xd6?\x00\x00\x00\xe0\xb3Y\xd9?\x00\x00\x00`\x91\xed\x00@\x00\x00\x00\xc0\xc8v\xea?\x00\x90\x9e@\x00\x00\x00\x00\xfe\xd4\x0f@\x00\x00\x00\xc0\x9f\x1a\xd7?\x00\x00\x00\x00\xf7u\xd8?\x00\x00\x00@\xe1z\x03@\x00\x00\x00\xa0\x99\x99\xe9?\x00\x94\x9e@\x00\x00\x00\x80\x14\xae\x11@\x00\x00\x00@\x89A\xd8?\x00\x00\x00\xa0\xed|\xd3?\x00\x00\x00\xa0\xef\xa7\x05@\x00\x00\x00\x00\xd5x\xed?\x00\x98\x9e@\x00\x00\x00 \x83@\x12@\x00\x00\x00\xe0$\x06\xd9?\x00\x00\x00`\x81\x04\xd5?\x00\x00\x00`\xe3\xa5\x05@\x00\x00\x00\xa0n\x12\xf1?\x00\x9c\x9e@\x00\x00\x00\x80=\x8a\x15@\x00\x00\x00\x80\x95C\xdb?\x00\x00\x00\xa0\xab\xad\xd8?\x00\x00\x00\xa0\x9b\xc4\x06@\x00\x00\x00\xc0\xf7S\xf1?\x00\xa0\x9e@\x00\x00\x00\xc0K7\x16@\x00\x00\x00 X9\xdc?\x00\x00\x00@io\xd4?\x00\x00\x00\xa0E\xb6\x08@\x00\x00\x00\x00-\xb2\xf7?\x00\xa4\x9e@\x00\x00\x00\x00)\xdc\x15@\x00\x00\x00\xe0\xa3p\xdd?\x00\x00\x00@\xa2\xb4\xd3?\x00\x00\x00 \xdb\xf9\x08@\x00\x00\x00\xe0\xa7\xc6\xfb?\x00\xa8\x9e@\x00\x00\x00\xc0\xccL\x17@\x00\x00\x00\x80=\n\xdf?\x00\x00\x00@\x116\xd8?\x00\x00\x00\x00\xd5x\t@\x00\x00\x00`\xe5\xd0\xfe?\x00\xac\x9e@\x00\x00\x00 \x06\x81\x1b@\x00\x00\x00\xe0&1\xe0?\x00\x00\x00 \x83\xc0\xda?\x00\x00\x00\xc0\x9f\x1a\n@\x00\x00\x00\xc0\xf7S\x00@\x00\xb0\x9e@\x00\x00\x00\x80\xc0J\x1f@\x00\x00\x00\xc0K7\xe1?\x00\x00\x00\xa0\x87\x85\xe0?\x00\x00\x00\xa0\xc6K\x0b@\x00\x00\x00@\xb6\xf3\xff?\x00\xb4\x9e@\x00\x00\x00\xa0p="@\x00\x00\x00\xc0I\x0c\xe2?\x00\x00\x00\xa0\x13\xd0\xe2?\x00\x00\x00`\xe7\xfb\x0c@\x00\x00\x00\x00V\x0e\x02@\x00\xb8\x9e@\x00\x00\x00\xe0$\x06%@\x00\x00\x00 \x83\xc0\xe2?\x00\x00\x00\xe0H.\xe1?\x00\x00\x00\xa0\xc6K\x10@\x00\x00\x00\xc0\x9d\xef\x05@\x00\xbc\x9e@\x00\x00\x00\x80=\n*@\x00\x00\x00\x80l\xe7\xe3?\x00\x00\x00@io\xdc?\x00\x00\x00@\n\xd7\x12@\x00\x00\x00`\x12\x83\x0c@\x00\xc0\x9e@\x00\x00\x00\xc0\xa1\x85.@\x00\x00\x00@\xdfO\xe5?\x00\x00\x00\xa0e\x88\xd3?\x00\x00\x00@5\xde\x14@\x00\x00\x00\x80h\x11\x13@\x00\xc4\x9e@\x00\x00\x00\xc0 P0@\x00\x00\x00 Zd\xe7?\x00\x00\x00`\x7f\xd9\xcd?\x00\x00\x00\xe0\xa7F\x16@\x00\x00\x00\xa0C\x0b\x1a@\x00\xc8\x9e@\x00\x00\x00 \x83\x000@\x00\x00\x00@\x8d\x97\xea?\x00\x00\x00\xe06\x1a\xc8?\x00\x00\x00@\xe1\xfa\x15@\x00\x00\x00@\x0c\x82\x1e@\x00\xcc\x9e@\x00\x00\x00 \x83\xc0/@\x00\x00\x00\xc0\xf3\xfd\xec?\x00\x00\x00`\xf7\xe4\xc9?\x00\x00\x00 \x04V\x15@\x00\x00\x00\x80\x93X!@\x00\xd0\x9e@\x00\x00\x00\xe0x\xa90@\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\xa0\xd4\t\xd0?\x00\x00\x00\xa0Ga\x15@\x00\x00\x00\xe0x\xa9 @\x00\xd4\x9e@\x00\x00\x00\x80\x95\x031@\x00\x00\x00@`\xe5\xf0?\x00\x00\x00@@\x13\xd1?\x00\x00\x00`\xe3\xa5\x16@\x00\x00\x00 /\x1d!@\x00\xd8\x9e@\x00\x00\x00\x80\x14N3@\x00\x00\x00\x80\x93\x18\xf2?\x00\x00\x00\xa0\xb2\x0c\xd1?\x00\x00\x00\x00\x7f\xea\x16@\x00\x00\x00\xa0\x18\x04#@\x00\xdc\x9e@\x00\x00\x00\x80\x93\xb82@\x00\x00\x00@\xb6\xf3\xf3?\x00\x00\x00\xc0\xeas\xcd?\x00\x00\x00\x00T\xe3\x16@\x00\x00\x00\x80\xbe\x1f"@\x00\xe0\x9e@\x00\x00\x00\x00\x00@3@\x00\x00\x00\x00\x00\x00\xf6?\x00\x00\x00\xc0\xc1\x17\xd6?\x00\x00\x00\xc0I\x0c\x17@\x00\x00\x00\xe0$\x86 @\x00\xe4\x9e@\x00\x00\x00\xc0\xa1\xa54@\x00\x00\x00`9\xb4\xf8?\x00\x00\x00@\xe8\xd9\xdc?\x00\x00\x00@\x0c\x82\x17@\x00\x00\x00@`\xe5\x1d@\x00\xe8\x9e@\x00\x00\x00 \xdb\xb96@\x00\x00\x00\xe0|?\xfb?\x00\x00\x00@p\xce\xe2?\x00\x00\x00\x80\x97n\x18@\x00\x00\x00\x00\x7fj\x1c@\x00\xec\x9e@\x00\x00\x00\xc0v\x9e7@\x00\x00\x00\xc0\xc8v\xfc?\x00\x00\x00\x80q\x1b\xe1?\x00\x00\x00\xc0rh\x1b@\x00\x00\x00\xe0\xf9~\x1b@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfb\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00p\x00\r\x00\x00\x00\x00\x00\x00\x00\xfe\xfb\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00`\x00\x0b\x00\x00\x00\x00\x00\x00\x00\xfe\xfb\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00L\x00\r\x00\x00\x00\x00\x00\x00\x00\xfe\xfb\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00<\x00\t\x00\x00\x00\x00\x00\x00\x00\xfe\xfb\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(\x00\x0f\x00\x00\x00\x00\x00\x00\x00\xfe\xfb\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x00\x04\x00\x00\x00\x00\x00\x00\x00\xfc\xff\xff\xffP\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x04\x01\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x04\x01\x00\x0c\x00\x00\x00\x08\x00\x00\x00\x00\x04\x01\x00\x14\x00\x00\x00\x08\x00\x00\x00\x00\x04\x01\x00\x1c\x00\x00\x00\x08\x00\x00\x00\x00\x04\x01\x00$\x00\x00\x00\x08\x00\x00\x00\x00\x04\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x04\x00\x00\x00\x00\x00$\x00\x01\x00\x00\x00\x00\x008\x00\x01\x00\x00\x00\x00\x00H\x00\x01\x00\x00\x00\x00\x00\\\x00\x01\x00\x00\x00\x00\x00l\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xff\xff\xff\x90\x00\x10\x00\x80\x00\x00\x00\x00\x00\x00\x00Written by SAS\x00\x00YEARyearY\x00\x00\x00level of output\x00W\x00\x00\x00wage rate\x00\x00\x00R\x00\x00\x00interest rate\x00\x00\x00L\x00\x00\x00labor input\x00K\x00\x00\x00capital input\x00\x00\x00\x01\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfc\xff\xff0\x00\x00\x00\x04\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\x07\x00\x00\x00\x00\x00\x00\xfc\xff\xff\xff\x01\x00\x00\x00\x06\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\xfd\xff\xff\xff\x01\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x04\x00\x00\x00\xff\xff\xff\xff\x01\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x05\x00\x00\x00\xfe\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfb\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfa\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf9\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\xf6\xf6\xf6\x06\x00\x00\x00\x00\x00\x00\x00\xf7\xf7\xf7\xf7\xcd\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x110\x02\x00,\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00.\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00 \x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00kIW-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x0c\x00\x00\x00\x01\x00\x00\x00\x0e\x00\x00\x00\x01\x00\x00\x00-\x00\x00\x00\x01\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x0c\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x08\x00\x00\x00\x1c\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' diff --git a/kedro-datasets/tests/pandas/test_hdf_dataset.py b/kedro-datasets/tests/pandas/test_hdf_dataset.py index 07860d745..74b3fee86 100644 --- a/kedro-datasets/tests/pandas/test_hdf_dataset.py +++ b/kedro-datasets/tests/pandas/test_hdf_dataset.py @@ -144,7 +144,7 @@ def test_thread_lock_usage(self, hdf_dataset, dummy_dataframe, mocker): hdf_dataset.save(dummy_dataframe) calls = [ - mocker.call.__enter__(), # pylint: disable=unnecessary-dunder-call + mocker.call.__enter__(), mocker.call.__exit__(None, None, None), ] mocked_lock.assert_has_calls(calls) diff --git a/kedro-datasets/tests/pandas/test_sql_dataset.py b/kedro-datasets/tests/pandas/test_sql_dataset.py index 10b9cb093..26f7e0bd4 100644 --- a/kedro-datasets/tests/pandas/test_sql_dataset.py +++ b/kedro-datasets/tests/pandas/test_sql_dataset.py @@ -1,4 +1,3 @@ -# pylint: disable=no-member import importlib from pathlib import PosixPath from unittest.mock import ANY diff --git a/kedro-datasets/tests/spark/test_spark_dataset.py b/kedro-datasets/tests/spark/test_spark_dataset.py index 010f65895..393b401f5 100644 --- a/kedro-datasets/tests/spark/test_spark_dataset.py +++ b/kedro-datasets/tests/spark/test_spark_dataset.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-lines import importlib import re import sys @@ -182,7 +181,6 @@ def test_deprecation(module_name, class_name): getattr(importlib.import_module(module_name), class_name) -# pylint: disable=too-many-public-methods class TestSparkDataset: def test_load_parquet(self, tmp_path, sample_pandas_df): temp_path = (tmp_path / "data").as_posix() @@ -537,8 +535,8 @@ def test_save_version_warning(self, tmp_path, sample_spark_df): ) pattern = ( - r"Save version '{ev.save}' did not match load version " - r"'{ev.load}' for SparkDataset\(.+\)".format(ev=exact_version) + rf"Save version '{exact_version.save}' did not match load version " + rf"'{exact_version.load}' for SparkDataset\(.+\)" ) with pytest.warns(UserWarning, match=pattern): ds_local.save(sample_spark_df) @@ -578,7 +576,7 @@ def test_versioning_existing_dataset( sys.platform.startswith("win"), reason="DBFS doesn't work on Windows" ) class TestSparkDatasetVersionedDBFS: - def test_load_latest( # pylint: disable=too-many-arguments + def test_load_latest( self, mocker, versioned_dataset_dbfs, version, tmp_path, sample_spark_df ): mocked_glob = mocker.patch.object(versioned_dataset_dbfs, "_glob_function") @@ -605,7 +603,7 @@ def test_load_exact(self, tmp_path, sample_spark_df): assert reloaded.exceptAll(sample_spark_df).count() == 0 - def test_save( # pylint: disable=too-many-arguments + def test_save( self, mocker, versioned_dataset_dbfs, version, tmp_path, sample_spark_df ): mocked_glob = mocker.patch.object(versioned_dataset_dbfs, "_glob_function") @@ -618,7 +616,7 @@ def test_save( # pylint: disable=too-many-arguments ) assert (tmp_path / FILENAME / version.save / FILENAME).exists() - def test_exists( # pylint: disable=too-many-arguments + def test_exists( self, mocker, versioned_dataset_dbfs, version, tmp_path, sample_spark_df ): mocked_glob = mocker.patch.object(versioned_dataset_dbfs, "_glob_function") @@ -750,9 +748,7 @@ def test_load_latest(self, mocker, versioned_dataset_s3): versioned_dataset_s3.load() - mocked_glob.assert_called_once_with( - "{b}/{f}/*/{f}".format(b=BUCKET_NAME, f=FILENAME) - ) + mocked_glob.assert_called_once_with(f"{BUCKET_NAME}/{FILENAME}/*/{FILENAME}") get_spark.return_value.read.load.assert_called_once_with( "s3a://{b}/{f}/{v}/{f}".format( b=BUCKET_NAME, f=FILENAME, v="mocked_version" @@ -771,7 +767,7 @@ def test_load_exact(self, mocker): ds_s3.load() get_spark.return_value.read.load.assert_called_once_with( - "s3a://{b}/{f}/{v}/{f}".format(b=BUCKET_NAME, f=FILENAME, v=ts), "parquet" + f"s3a://{BUCKET_NAME}/{FILENAME}/{ts}/{FILENAME}", "parquet" ) def test_save(self, versioned_dataset_s3, version, mocker): @@ -785,7 +781,7 @@ def test_save(self, versioned_dataset_s3, version, mocker): versioned_dataset_s3.save(mocked_spark_df) mocked_spark_df.write.save.assert_called_once_with( - "s3a://{b}/{f}/{v}/{f}".format(b=BUCKET_NAME, f=FILENAME, v=version.save), + f"s3a://{BUCKET_NAME}/{FILENAME}/{version.save}/{FILENAME}", "parquet", ) @@ -799,15 +795,13 @@ def test_save_version_warning(self, mocker): mocked_spark_df = mocker.Mock() pattern = ( - r"Save version '{ev.save}' did not match load version " - r"'{ev.load}' for SparkDataset\(.+\)".format(ev=exact_version) + rf"Save version '{exact_version.save}' did not match load version " + rf"'{exact_version.load}' for SparkDataset\(.+\)" ) with pytest.warns(UserWarning, match=pattern): ds_s3.save(mocked_spark_df) mocked_spark_df.write.save.assert_called_once_with( - "s3a://{b}/{f}/{v}/{f}".format( - b=BUCKET_NAME, f=FILENAME, v=exact_version.save - ), + f"s3a://{BUCKET_NAME}/{FILENAME}/{exact_version.save}/{FILENAME}", "parquet", ) @@ -883,7 +877,7 @@ def test_load_exact(self, mocker): versioned_hdfs.load() get_spark.return_value.read.load.assert_called_once_with( - "hdfs://{fn}/{f}/{v}/{f}".format(fn=FOLDER_NAME, f=FILENAME, v=ts), + f"hdfs://{FOLDER_NAME}/{FILENAME}/{ts}/{FILENAME}", "parquet", ) @@ -905,13 +899,11 @@ def test_save(self, mocker, version): versioned_hdfs.save(mocked_spark_df) hdfs_status.assert_called_once_with( - "{fn}/{f}/{v}/{f}".format(fn=FOLDER_NAME, v=version.save, f=FILENAME), + f"{FOLDER_NAME}/{FILENAME}/{version.save}/{FILENAME}", strict=False, ) mocked_spark_df.write.save.assert_called_once_with( - "hdfs://{fn}/{f}/{v}/{f}".format( - fn=FOLDER_NAME, v=version.save, f=FILENAME - ), + f"hdfs://{FOLDER_NAME}/{FILENAME}/{version.save}/{FILENAME}", "parquet", ) @@ -924,16 +916,14 @@ def test_save_version_warning(self, mocker): mocked_spark_df = mocker.Mock() pattern = ( - r"Save version '{ev.save}' did not match load version " - r"'{ev.load}' for SparkDataset\(.+\)".format(ev=exact_version) + rf"Save version '{exact_version.save}' did not match load version " + rf"'{exact_version.load}' for SparkDataset\(.+\)" ) with pytest.warns(UserWarning, match=pattern): versioned_hdfs.save(mocked_spark_df) mocked_spark_df.write.save.assert_called_once_with( - "hdfs://{fn}/{f}/{sv}/{f}".format( - fn=FOLDER_NAME, f=FILENAME, sv=exact_version.save - ), + f"hdfs://{FOLDER_NAME}/{FILENAME}/{exact_version.save}/{FILENAME}", "parquet", ) @@ -955,7 +945,7 @@ def test_prevent_overwrite(self, mocker, version): versioned_hdfs.save(mocked_spark_df) hdfs_status.assert_called_once_with( - "{fn}/{f}/{v}/{f}".format(fn=FOLDER_NAME, v=version.save, f=FILENAME), + f"{FOLDER_NAME}/{FILENAME}/{version.save}/{FILENAME}", strict=False, ) mocked_spark_df.write.save.assert_not_called() diff --git a/kedro-datasets/tests/spark/test_spark_hive_dataset.py b/kedro-datasets/tests/spark/test_spark_hive_dataset.py index 4a7f4c97e..e33ca5cce 100644 --- a/kedro-datasets/tests/spark/test_spark_hive_dataset.py +++ b/kedro-datasets/tests/spark/test_spark_hive_dataset.py @@ -52,8 +52,8 @@ def spark_session(): pass # remove the cached JVM vars - SparkContext._jvm = None # pylint: disable=protected-access - SparkContext._gateway = None # pylint: disable=protected-access + SparkContext._jvm = None + SparkContext._gateway = None # py4j doesn't shutdown properly so kill the actual JVM process for obj in gc.get_objects(): @@ -145,7 +145,7 @@ def test_deprecation(module_name, class_name): class TestSparkHiveDataset: def test_cant_pickle(self): - import pickle # pylint: disable=import-outside-toplevel + import pickle with pytest.raises(pickle.PicklingError): pickle.dumps( diff --git a/kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py b/kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py index 03d016e4b..ffeafe321 100644 --- a/kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py +++ b/kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py @@ -1,4 +1,3 @@ -# pylint: disable=import-outside-toplevel import importlib from pathlib import PurePosixPath @@ -125,7 +124,6 @@ def __init__(self): self.dense1 = tf.keras.layers.Dense(4, activation=tf.nn.relu) self.dense2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax) - # pylint: disable=unused-argument def call(self, inputs, training=None, mask=None): # pragma: no cover x = self.dense1(inputs) return self.dense2(x) @@ -313,7 +311,7 @@ def test_save_and_load( dummy_x_test, load_version, save_version, - ): # pylint: disable=unused-argument + ): """Test saving and reloading the versioned data set.""" predictions = dummy_tf_base_model.predict(dummy_x_test) diff --git a/kedro-docker/.pylintrc b/kedro-docker/.pylintrc deleted file mode 100644 index e1f257e03..000000000 --- a/kedro-docker/.pylintrc +++ /dev/null @@ -1,425 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint.extensions.docparams - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=ungrouped-imports,bad-continuation - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=useless-suppression - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=20 - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=1 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/kedro-docker/CONTRIBUTING.md b/kedro-docker/CONTRIBUTING.md index 7bbab860a..57e92017a 100644 --- a/kedro-docker/CONTRIBUTING.md +++ b/kedro-docker/CONTRIBUTING.md @@ -84,20 +84,20 @@ pip install ".[test]" All checks run by our CI / CD pipeline can be run locally on your computer. -#### PEP-8 Standards (`isort`, `pylint` and `flake8`) +#### Linting (`ruff` and `black`) ```bash -make lint +make plugin=kedro-docker lint ``` #### Unit tests, 100% coverage (`pytest`, `pytest-cov`) ```bash -make test +make plugin=kedro-docker test ``` #### End-to-end tests (`behave`) ```bash -make e2e-tests +make plugin=kedro-docker e2e-tests ``` diff --git a/kedro-docker/features/environment.py b/kedro-docker/features/environment.py index 930f97a7d..e006227ee 100644 --- a/kedro-docker/features/environment.py +++ b/kedro-docker/features/environment.py @@ -91,7 +91,6 @@ def after_all(context): def before_scenario(context, feature): - # pylint: disable=unused-argument context.temp_dir = Path(tempfile.mkdtemp()) diff --git a/kedro-docker/features/steps/sh_run.py b/kedro-docker/features/steps/sh_run.py index 66ef9092e..7d9f6152a 100644 --- a/kedro-docker/features/steps/sh_run.py +++ b/kedro-docker/features/steps/sh_run.py @@ -9,7 +9,7 @@ def run( cmd: Union[str, Sequence], split: bool = True, print_output: bool = False, - **kwargs: str + **kwargs: str, ) -> subprocess.CompletedProcess: """ Args: @@ -39,10 +39,7 @@ def run( """ if isinstance(cmd, str) and split: cmd = shlex.split(cmd) - # pylint: disable=subprocess-run-check - result = subprocess.run( - cmd, input="", stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) + result = subprocess.run(cmd, input="", capture_output=True, **kwargs) result.stdout = result.stdout.decode("utf-8") result.stderr = result.stderr.decode("utf-8") if print_output: diff --git a/kedro-docker/features/steps/util.py b/kedro-docker/features/steps/util.py index 2d259f2ce..dd212f6ca 100644 --- a/kedro-docker/features/steps/util.py +++ b/kedro-docker/features/steps/util.py @@ -43,7 +43,7 @@ def wait_for( while time() <= end: try: retval = func(**kwargs) - except Exception as err: # pylint: disable=broad-except + except Exception as err: if print_error: print(err) else: diff --git a/kedro-docker/kedro_docker/helpers.py b/kedro-docker/kedro_docker/helpers.py index 879ec4fab..981bfcdcb 100644 --- a/kedro-docker/kedro_docker/helpers.py +++ b/kedro-docker/kedro_docker/helpers.py @@ -57,8 +57,7 @@ def _list_docker_volumes(host_root: str, container_root: str, volumes: Sequence[ yield "-v", str(hpath) + ":" + str(cpath) -# pylint: disable=too-many-arguments -def compose_docker_run_args( +def compose_docker_run_args( # noqa: PLR0913 host_root: str = None, container_root: str = None, mount_volumes: Sequence[str] = None, diff --git a/kedro-docker/kedro_docker/plugin.py b/kedro-docker/kedro_docker/plugin.py index 27af7db96..eabd7986e 100644 --- a/kedro-docker/kedro_docker/plugin.py +++ b/kedro-docker/kedro_docker/plugin.py @@ -1,5 +1,4 @@ """ Kedro plugin for packaging a project with Docker """ -# pylint: disable=unused-argument import shlex import subprocess from pathlib import Path @@ -88,7 +87,7 @@ def _make_docker_args_option(**kwargs): @click.group(name="Kedro-Docker") -def commands(): # pylint: disable=missing-function-docstring +def commands(): pass @@ -125,7 +124,7 @@ def docker_init(spark): if KEDRO_VERSION.match(">=0.17.0"): verbose = KedroCliError.VERBOSE_ERROR else: - from kedro.framework.cli.cli import ( # noqa # pylint:disable=import-outside-toplevel, no-name-in-module + from kedro.framework.cli.cli import ( _VERBOSE as verbose, ) @@ -169,9 +168,7 @@ def docker_init(spark): help="Optional arguments to be passed to `docker build` command" ) @click.pass_context -def docker_build( - ctx, uid, gid, spark, base_image, image, docker_args -): # pylint: disable=too-many-arguments +def docker_build(ctx, uid, gid, spark, base_image, image, docker_args): # noqa: PLR0913 """Build a Docker image for the project.""" uid, gid = get_uid_gid(uid, gid) project_path = Path.cwd() @@ -210,7 +207,8 @@ def docker_run(image, docker_args, args, **kwargs): Any extra arguments unspecified in this help are passed to `docker run` as is. - **kwargs is needed to make the global `verbose` argument work and pass it through.""" + **kwargs is needed to make the global `verbose` argument work and pass it through. + """ container_name = make_container_name(image, "run") _docker_run_args = compose_docker_run_args( @@ -233,7 +231,8 @@ def docker_ipython(image, docker_args, args, **kwargs): Any extra arguments unspecified in this help are passed to `kedro ipython` command inside the container as is. - **kwargs is needed to make the global `verbose` argument work and pass it through.""" + **kwargs is needed to make the global `verbose` argument work and pass it through. + """ container_name = make_container_name(image, "ipython") _docker_run_args = compose_docker_run_args( @@ -262,7 +261,8 @@ def docker_jupyter_notebook(docker_args, port, image, args, **kwargs): Any extra arguments unspecified in this help are passed to `kedro jupyter notebook` command inside the container as is. - **kwargs is needed to make the global `verbose` argument work and pass it through.""" + **kwargs is needed to make the global `verbose` argument work and pass it through. + """ container_name = make_container_name(image, "jupyter-notebook") _docker_run_args = compose_docker_run_args( @@ -291,7 +291,8 @@ def docker_jupyter_lab(docker_args, port, image, args, **kwargs): Any extra arguments unspecified in this help are passed to `kedro jupyter lab` command inside the container as is. - **kwargs is needed to make the global `verbose` argument work and pass it through.""" + **kwargs is needed to make the global `verbose` argument work and pass it through. + """ container_name = make_container_name(image, "jupyter-lab") _docker_run_args = compose_docker_run_args( @@ -315,7 +316,8 @@ def docker_cmd(args, docker_args, image, **kwargs): """Run arbitrary command from ARGS in the Docker container. If ARGS are not specified, this will invoke `kedro run` inside the container. - **kwargs is needed to make the global `verbose` argument work and pass it through.""" + **kwargs is needed to make the global `verbose` argument work and pass it through. + """ container_name = make_container_name(image, "cmd") _docker_run_args = compose_docker_run_args( diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml index be5c89c67..e49603c6a 100644 --- a/kedro-docker/pyproject.toml +++ b/kedro-docker/pyproject.toml @@ -28,16 +28,15 @@ test = [ "behave", "black~=22.0", "docker", - "flake8>=3.5, <4.0", - "pre-commit>=1.17.0, <2.0", + "pre-commit>=2.9.2", "psutil", - "pylint>=2.4.4, <3.0", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist[psutil]~=2.2.1", "PyYAML>=5.1, <7.0", "trufflehog>=2.0.99, <3.0", + "ruff~=0.0.290", "wheel==0.32.2" ] @@ -72,16 +71,25 @@ addopts = """ --no-cov-on-fail -ra""" -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -line_length = 88 -default_section = "THIRDPARTY" - [tool.coverage.report] fail_under = 100 show_missing = true omit = ["tests/*", "*/plugin.py"] exclude_lines = ["pragma: no cover", "raise NotImplementedError"] + +[tool.ruff] +line-length = 88 +show-fixes = true +select = [ + "F", # Pyflakes + "W", # pycodestyle + "E", # pycodestyle + "I", # isort + "UP", # pyupgrade + "PL", # Pylint + "T201", # Print Statement +] +ignore = ["E501"] # Black takes care of line-too-long + +[tool.ruff.per-file-ignores] +"{tests,features}/*" = ["T201", "PLW1510"] diff --git a/kedro-telemetry/kedro_telemetry/masking.py b/kedro-telemetry/kedro_telemetry/masking.py index 53955dffc..fe5f0a3f6 100644 --- a/kedro-telemetry/kedro_telemetry/masking.py +++ b/kedro-telemetry/kedro_telemetry/masking.py @@ -84,11 +84,10 @@ def _mask_kedro_cli(cli_struct: Dict[str, Any], command_args: List[str]) -> List output.append(arg_part) elif arg_part: output.append(MASK) - else: - if arg in vocabulary: - output.append(arg) - elif arg: - output.append(MASK) + elif arg in vocabulary: + output.append(arg) + elif arg: + output.append(MASK) return output diff --git a/kedro-telemetry/kedro_telemetry/plugin.py b/kedro-telemetry/kedro_telemetry/plugin.py index 5eeb4d489..cc27731b6 100644 --- a/kedro-telemetry/kedro_telemetry/plugin.py +++ b/kedro-telemetry/kedro_telemetry/plugin.py @@ -42,7 +42,7 @@ def _get_hashed_username(): try: username = getpass.getuser() return _hash(username) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logger.warning( "Something went wrong with getting the username. Exception: %s", exc, @@ -53,8 +53,6 @@ def _get_hashed_username(): class KedroTelemetryCLIHooks: """Hook to send CLI command data to Heap""" - # pylint: disable=too-few-public-methods - @cli_hook_impl def before_command_run( self, project_metadata: ProjectMetadata, command_args: List[str] @@ -101,7 +99,7 @@ def before_command_run( identity=hashed_username, properties=generic_properties, ) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logger.warning( "Something went wrong in hook implementation to send command run data to Heap. " "Exception: %s", @@ -109,7 +107,7 @@ def before_command_run( ) -class KedroTelemetryProjectHooks: # pylint: disable=too-few-public-methods +class KedroTelemetryProjectHooks: """Hook to send project statistics data to Heap""" @hook_impl @@ -209,7 +207,7 @@ def _send_heap_event( resp = requests.post( url=HEAP_ENDPOINT, headers=HEAP_HEADERS, data=json.dumps(data), timeout=10 ) - if resp.status_code != 200: + if resp.status_code != 200: # noqa: PLR2004 logger.warning( "Failed to send data to Heap. Response code returned: %s, Response reason: %s", resp.status_code, @@ -261,7 +259,7 @@ def _confirm_consent(telemetry_file_path: Path) -> bool: ) yaml.dump({"consent": False}, telemetry_file) return False - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logger.warning( "Failed to confirm consent. No data was sent to Heap. Exception: %s", exc, diff --git a/kedro-telemetry/pyproject.toml b/kedro-telemetry/pyproject.toml index 21d80ee1a..f5ca2627b 100644 --- a/kedro-telemetry/pyproject.toml +++ b/kedro-telemetry/pyproject.toml @@ -26,15 +26,13 @@ test = [ "bandit>=1.6.2, <2.0", "behave", "black~=22.0", - "flake8", - "isort>=4.3.21, <5.0", - "pre-commit>=1.17.0, <2.0", - "pylint>=2.5.2, <3.0", + "pre-commit>=2.9.2", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist[psutil]~=2.2.1", "trufflehog>=2.1.0, <3.0", + "ruff~=0.0.290", "wheel" ] @@ -53,10 +51,19 @@ zip-safe = false readme = {file = "README.md", content-type = "text/markdown"} version = {attr = "kedro_telemetry.__version__"} -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -line_length = 88 -known_first_party = "kedro_telemetry" +[tool.ruff] +line-length = 88 +show-fixes = true +select = [ + "F", # Pyflakes + "W", # pycodestyle + "E", # pycodestyle + "I", # isort + "UP", # pyupgrade + "PL", # Pylint + "T201", # Print Statement +] +ignore = ["E501"] # Black takes care of line-too-long + +[tool.ruff.isort] +known-first-party = ["kedro_telemetry"] diff --git a/kedro-telemetry/tests/test_masking.py b/kedro-telemetry/tests/test_masking.py index e094ee1ae..b5ef954f9 100644 --- a/kedro-telemetry/tests/test_masking.py +++ b/kedro-telemetry/tests/test_masking.py @@ -1,5 +1,3 @@ -# pylint: disable=protected-access - """Testing module for CLI tools""" import shutil from collections import namedtuple diff --git a/kedro-telemetry/tests/test_plugin.py b/kedro-telemetry/tests/test_plugin.py index 222bcc914..ccbaf8afe 100644 --- a/kedro-telemetry/tests/test_plugin.py +++ b/kedro-telemetry/tests/test_plugin.py @@ -22,8 +22,6 @@ REPO_NAME = "dummy_project" PACKAGE_NAME = "dummy_package" -# pylint: disable=too-few-public-methods - @fixture def fake_metadata(tmp_path): @@ -379,7 +377,7 @@ def test_after_context_created_without_kedro_run( # The 1st call is the Project Hook without CLI assert mocked_heap_call.call_args_list[0] == expected_call - def test_after_context_created_with_kedro_run( + def test_after_context_created_with_kedro_run( # noqa: PLR0913 self, mocker, fake_context,