From cd203ed48ba0e4563c3790949f418a0a39537bbb Mon Sep 17 00:00:00 2001 From: elijahbenizzy Date: Tue, 28 Nov 2023 06:52:33 -0800 Subject: [PATCH] Updates pre-commit hooks Fixes pre-commit hooks. TBD what was happening, but this updates everything, and adds known_first_party to disambiguate the use of the hamilton package. Should make it stable across all systems, and ensures we use the latest linting/checking rules. --- .pre-commit-config.yaml | 10 +++++----- .../dagworks/text_summarization/__init__.py | 1 - .../user/zilto/text_summarization/__init__.py | 1 - contrib/setup.cfg | 18 +++++++++++++----- examples/data_quality/pandera/run_ray.py | 1 - examples/data_quality/simple/run_ray.py | 1 - examples/decoupling_io/adapters.py | 1 - examples/model_examples/time-series/utils.py | 1 - hamilton/experimental/h_async.py | 2 +- hamilton/plugins/sklearn_plot_extensions.py | 1 - plugin_tests/h_ray/test_h_ray_workflow.py | 2 +- setup.cfg | 15 ++++++++++----- tests/execution/test_executors.py | 16 ++++++++-------- tests/execution/test_node_grouping.py | 3 ++- tests/function_modifiers/test_macros.py | 2 +- tests/function_modifiers/test_recursive.py | 2 +- tests/function_modifiers/test_validation.py | 10 +++++----- tests/io/test_materialization.py | 2 +- tests/test_caching.py | 2 +- tests/test_common.py | 2 +- tests/test_default_data_quality.py | 12 ++++++------ tests/test_end_to_end.py | 2 +- tests/test_graph.py | 6 +++--- tests/test_hamilton_driver.py | 2 +- writeups/garbage_collection/memory_test.py | 1 - 25 files changed, 61 insertions(+), 55 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2bb775938..b8abef426 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,12 +6,12 @@ repos: - repo: https://github.com/ambv/black - rev: 22.3.0 + rev: 23.11.0 hooks: - id: black args: [--line-length=100] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.5.0 hooks: - id: trailing-whitespace # ensures files are either empty or end with a blank line @@ -21,12 +21,12 @@ repos: # valid python file - id: check-ast # isort python package import sorting -- repo: https://github.com/pre-commit/mirrors-isort - rev: 'v5.10.1' +- repo: https://github.com/pycqa/isort + rev: '5.12.0' hooks: - id: isort args: ["--profile", "black", --line-length=100] - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 + rev: 6.1.0 hooks: - id: flake8 diff --git a/contrib/hamilton/contrib/dagworks/text_summarization/__init__.py b/contrib/hamilton/contrib/dagworks/text_summarization/__init__.py index 35d9d64b8..0f49e3dbf 100644 --- a/contrib/hamilton/contrib/dagworks/text_summarization/__init__.py +++ b/contrib/hamilton/contrib/dagworks/text_summarization/__init__.py @@ -14,7 +14,6 @@ from tqdm import tqdm import litellm - from hamilton.function_modifiers import config diff --git a/contrib/hamilton/contrib/user/zilto/text_summarization/__init__.py b/contrib/hamilton/contrib/user/zilto/text_summarization/__init__.py index 34ef27523..04cd10170 100644 --- a/contrib/hamilton/contrib/user/zilto/text_summarization/__init__.py +++ b/contrib/hamilton/contrib/user/zilto/text_summarization/__init__.py @@ -14,7 +14,6 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tqdm import tqdm - from hamilton.function_modifiers import config diff --git a/contrib/setup.cfg b/contrib/setup.cfg index 26946c07b..6b1dae301 100644 --- a/contrib/setup.cfg +++ b/contrib/setup.cfg @@ -2,8 +2,16 @@ max-line-length = 100 exclude = build/,.git/,venv/ ignore = - E203, # whitespace before ':' \ - E402, # module level import not at top of file \ - E501, # line too long \ - W503, # line break before binary operator \ - W605 # invalid escape sequence + # whitespace before ':' + E203, + # module level import not at top of file + E402, + # line too long + E501, + # line break before binary operator + W503, + # invalid escape sequence + W605 + +[isort] +known_first_party=hamilton diff --git a/examples/data_quality/pandera/run_ray.py b/examples/data_quality/pandera/run_ray.py index 7313bab6a..e750b2545 100644 --- a/examples/data_quality/pandera/run_ray.py +++ b/examples/data_quality/pandera/run_ray.py @@ -26,7 +26,6 @@ from hamilton.plugins import h_ray if __name__ == "__main__": - logging.basicConfig(stream=sys.stdout) logger = logging.getLogger(__name__) # Setup a local cluster. diff --git a/examples/data_quality/simple/run_ray.py b/examples/data_quality/simple/run_ray.py index 0e4b595dc..7adddd896 100644 --- a/examples/data_quality/simple/run_ray.py +++ b/examples/data_quality/simple/run_ray.py @@ -26,7 +26,6 @@ from hamilton.plugins import h_ray if __name__ == "__main__": - logging.basicConfig(stream=sys.stdout) logger = logging.getLogger(__name__) # Setup a local cluster. diff --git a/examples/decoupling_io/adapters.py b/examples/decoupling_io/adapters.py index b1486d3a4..8e861356a 100644 --- a/examples/decoupling_io/adapters.py +++ b/examples/decoupling_io/adapters.py @@ -45,7 +45,6 @@ @dataclasses.dataclass class SklearnPlotSaver(DataSaver): - path: Union[str, PathLike] # kwargs dpi: float = 200 diff --git a/examples/model_examples/time-series/utils.py b/examples/model_examples/time-series/utils.py index e5632c063..5293d5e8c 100644 --- a/examples/model_examples/time-series/utils.py +++ b/examples/model_examples/time-series/utils.py @@ -19,7 +19,6 @@ def reduce_mem_usage(df: pd.DataFrame, name: str, verbose=True): for col in df.columns: col_type = df[col].dtypes if col_type in numerics: - c_min = df[col].min() c_max = df[col].max() if str(col_type)[:3] == "int": diff --git a/hamilton/experimental/h_async.py b/hamilton/experimental/h_async.py index cd84e19bf..3c3cd1be4 100644 --- a/hamilton/experimental/h_async.py +++ b/hamilton/experimental/h_async.py @@ -67,7 +67,7 @@ async def new_fn(fn=callabl, **fn_kwargs): task_dict = {key: process_value(value) for key, value in fn_kwargs.items()} fn_kwargs = await await_dict_of_tasks(task_dict) if inspect.iscoroutinefunction(fn): - return await (fn(**fn_kwargs)) + return await fn(**fn_kwargs) return fn(**fn_kwargs) coroutine = new_fn(**kwargs) diff --git a/hamilton/plugins/sklearn_plot_extensions.py b/hamilton/plugins/sklearn_plot_extensions.py index 2d4bcbd4a..9ddcacebc 100644 --- a/hamilton/plugins/sklearn_plot_extensions.py +++ b/hamilton/plugins/sklearn_plot_extensions.py @@ -46,7 +46,6 @@ @dataclasses.dataclass class SklearnPlotSaver(DataSaver): - path: Union[str, PathLike] # kwargs dpi: float = 200 diff --git a/plugin_tests/h_ray/test_h_ray_workflow.py b/plugin_tests/h_ray/test_h_ray_workflow.py index 050ebcc74..e8ba7039c 100644 --- a/plugin_tests/h_ray/test_h_ray_workflow.py +++ b/plugin_tests/h_ray/test_h_ray_workflow.py @@ -1,11 +1,11 @@ import pandas as pd import pytest import ray +from plugin_tests.h_ray.resources import example_module, smoke_screen_module from ray import workflow from hamilton import base, driver from hamilton.plugins import h_ray -from plugin_tests.h_ray.resources import example_module, smoke_screen_module @pytest.fixture(scope="module") diff --git a/setup.cfg b/setup.cfg index 26946c07b..b8abec40d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,8 +2,13 @@ max-line-length = 100 exclude = build/,.git/,venv/ ignore = - E203, # whitespace before ':' \ - E402, # module level import not at top of file \ - E501, # line too long \ - W503, # line break before binary operator \ - W605 # invalid escape sequence + # whitespace before ':' + E203, + # module level import not at top of file + E402, + # line too long + E501, + # line break before binary operator + W503, + # invalid escape sequence + W605 diff --git a/tests/execution/test_executors.py b/tests/execution/test_executors.py index d9164e6b9..7109c0afe 100644 --- a/tests/execution/test_executors.py +++ b/tests/execution/test_executors.py @@ -3,6 +3,14 @@ import numpy as np import pytest +from tests.resources.dynamic_parallelism import ( + inputs_in_collect, + no_parallel, + parallel_collect_multiple_arguments, + parallel_complex, + parallel_delayed, + parallel_linear_basic, +) import hamilton.ad_hoc_utils from hamilton import base, driver @@ -21,14 +29,6 @@ TaskImplementation, ) from hamilton.htypes import Collect, Parallelizable -from tests.resources.dynamic_parallelism import ( - inputs_in_collect, - no_parallel, - parallel_collect_multiple_arguments, - parallel_complex, - parallel_delayed, - parallel_linear_basic, -) ADAPTER = base.DefaultAdapter() diff --git a/tests/execution/test_node_grouping.py b/tests/execution/test_node_grouping.py index 3418f3910..0bf675082 100644 --- a/tests/execution/test_node_grouping.py +++ b/tests/execution/test_node_grouping.py @@ -1,3 +1,5 @@ +from tests.resources.dynamic_parallelism import no_parallel, parallel_complex, parallel_linear_basic + from hamilton import ad_hoc_utils, base, graph, node from hamilton.execution import grouping from hamilton.execution.grouping import ( @@ -9,7 +11,6 @@ ) from hamilton.graph import FunctionGraph from hamilton.node import NodeType -from tests.resources.dynamic_parallelism import no_parallel, parallel_complex, parallel_linear_basic def test_group_individually(): diff --git a/tests/function_modifiers/test_macros.py b/tests/function_modifiers/test_macros.py index a9f834b0b..7fc03bb80 100644 --- a/tests/function_modifiers/test_macros.py +++ b/tests/function_modifiers/test_macros.py @@ -3,9 +3,9 @@ import pandas as pd import pytest +import tests.resources.pipe import hamilton.function_modifiers -import tests.resources.pipe from hamilton import base, driver, function_modifiers, models, node from hamilton.function_modifiers import does from hamilton.function_modifiers.dependencies import source, value diff --git a/tests/function_modifiers/test_recursive.py b/tests/function_modifiers/test_recursive.py index 98f036ea2..198a3212f 100644 --- a/tests/function_modifiers/test_recursive.py +++ b/tests/function_modifiers/test_recursive.py @@ -3,8 +3,8 @@ from typing import Tuple import pytest - import tests.resources.reuse_subdag + from hamilton import ad_hoc_utils, graph from hamilton.function_modifiers import ( InvalidDecoratorException, diff --git a/tests/function_modifiers/test_validation.py b/tests/function_modifiers/test_validation.py index b440b10be..af06e8dfc 100644 --- a/tests/function_modifiers/test_validation.py +++ b/tests/function_modifiers/test_validation.py @@ -1,6 +1,11 @@ import numpy as np import pandas as pd import pytest +from tests.resources.dq_dummy_examples import ( + DUMMY_VALIDATORS_FOR_TESTING, + SampleDataValidator2, + SampleDataValidator3, +) from hamilton import node from hamilton.data_quality.base import DataValidationError, ValidationResult @@ -11,11 +16,6 @@ check_output_custom, ) from hamilton.node import DependencyType -from tests.resources.dq_dummy_examples import ( - DUMMY_VALIDATORS_FOR_TESTING, - SampleDataValidator2, - SampleDataValidator3, -) def test_check_output_node_transform(): diff --git a/tests/io/test_materialization.py b/tests/io/test_materialization.py index c31de6c80..e009fc461 100644 --- a/tests/io/test_materialization.py +++ b/tests/io/test_materialization.py @@ -2,9 +2,9 @@ from typing import Any, Collection, Dict, List, Optional, Tuple, Type import pytest - import tests.resources.cyclic_functions import tests.resources.test_default_args + from hamilton import base, graph, node, registry from hamilton.function_modifiers import load_from, save_to, value from hamilton.io import materialization diff --git a/tests/test_caching.py b/tests/test_caching.py index 83b1fd50f..64ca85124 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -3,11 +3,11 @@ import pandas as pd import pytest +from tests import nodes from hamilton import base from hamilton.driver import Driver from hamilton.experimental import h_cache -from tests import nodes @pytest.fixture diff --git a/tests/test_common.py b/tests/test_common.py index aeb784dbc..c904b4786 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,7 +1,7 @@ import pytest - import tests.resources.cyclic_functions import tests.resources.test_default_args + from hamilton import common, driver diff --git a/tests/test_default_data_quality.py b/tests/test_default_data_quality.py index 3df2414a3..86225ed17 100644 --- a/tests/test_default_data_quality.py +++ b/tests/test_default_data_quality.py @@ -6,6 +6,12 @@ import numpy as np import pandas as pd import pytest +from tests.resources.dq_dummy_examples import ( + DUMMY_VALIDATORS_FOR_TESTING, + SampleDataValidator1, + SampleDataValidator2, + SampleDataValidator3, +) import hamilton.data_quality.base from hamilton.data_quality import default_validators @@ -14,12 +20,6 @@ AVAILABLE_DEFAULT_VALIDATORS, resolve_default_validators, ) -from tests.resources.dq_dummy_examples import ( - DUMMY_VALIDATORS_FOR_TESTING, - SampleDataValidator1, - SampleDataValidator2, - SampleDataValidator3, -) @pytest.mark.parametrize( diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 5c7a23b4b..5ba389fa8 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -4,10 +4,10 @@ from typing import Any, Callable, Dict, List, Type import pytest - import tests.resources.data_quality import tests.resources.dynamic_config import tests.resources.overrides + from hamilton import ad_hoc_utils, base, driver, settings from hamilton.base import DefaultAdapter from hamilton.data_quality.base import DataValidationError, ValidationResult diff --git a/tests/test_graph.py b/tests/test_graph.py index 4428b6414..c1900c3af 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -5,9 +5,6 @@ import pandas as pd import pytest - -import hamilton.graph_utils -import hamilton.htypes import tests.resources.bad_functions import tests.resources.config_modifier import tests.resources.cyclic_functions @@ -22,6 +19,9 @@ import tests.resources.parametrized_nodes import tests.resources.test_default_args import tests.resources.typing_vs_not_typing + +import hamilton.graph_utils +import hamilton.htypes from hamilton import ad_hoc_utils, base, graph, node from hamilton.execution import graph_functions from hamilton.node import NodeType diff --git a/tests/test_hamilton_driver.py b/tests/test_hamilton_driver.py index 3cb825451..e545be075 100644 --- a/tests/test_hamilton_driver.py +++ b/tests/test_hamilton_driver.py @@ -2,13 +2,13 @@ import pandas as pd import pytest - import tests.resources.cyclic_functions import tests.resources.dummy_functions import tests.resources.dynamic_parallelism.parallel_linear_basic import tests.resources.tagging import tests.resources.test_default_args import tests.resources.very_simple_dag + from hamilton import base, node from hamilton.driver import ( Builder, diff --git a/writeups/garbage_collection/memory_test.py b/writeups/garbage_collection/memory_test.py index 7794a8754..5912fba58 100644 --- a/writeups/garbage_collection/memory_test.py +++ b/writeups/garbage_collection/memory_test.py @@ -56,7 +56,6 @@ def foo_i(foo_i_minus_one: pd.DataFrame) -> pd.DataFrame: if __name__ == "__main__": - import os import psutil