diff --git a/src/python/pants/backend/python/lint/isort/subsystem.py b/src/python/pants/backend/python/lint/isort/subsystem.py index b9eb5dc56d1..69e7c7abda7 100644 --- a/src/python/pants/backend/python/lint/isort/subsystem.py +++ b/src/python/pants/backend/python/lint/isort/subsystem.py @@ -18,14 +18,12 @@ def register_options(cls, register): "--skip", type=bool, default=False, - fingerprint=True, help="Don't use isort when running `./pants fmt` and `./pants lint`", ) register( "--args", type=list, member_type=shell_str, - fingerprint=True, help="Arguments to pass directly to isort, e.g. " '`--isort-args="--case-sensitive --trailing-comma"`', ) @@ -33,6 +31,5 @@ def register_options(cls, register): "--config", type=list, member_type=file_option, - fingerprint=True, help="Path to `isort.cfg` or alternative isort config file(s)", ) diff --git a/src/python/pants/backend/python/lint/mypy/BUILD b/src/python/pants/backend/python/lint/mypy/BUILD new file mode 100644 index 00000000000..b638ce329d5 --- /dev/null +++ b/src/python/pants/backend/python/lint/mypy/BUILD @@ -0,0 +1,42 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library( + dependencies=[ + '3rdparty/python:dataclasses', + 'src/python/pants/backend/python/lint', + 'src/python/pants/backend/python/subsystems', + 'src/python/pants/backend/python/rules', + 'src/python/pants/core/goals', + 'src/python/pants/core/util_rules', + 'src/python/pants/engine:fs', + 'src/python/pants/engine:process', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:selectors', + 'src/python/pants/option', + 'src/python/pants/python', + ], + tags = {"partially_type_checked"}, +) + +python_tests( + name='integration', + sources=['*_integration_test.py'], + dependencies=[ + ':mypy', + 'src/python/pants/backend/python/lint', + 'src/python/pants/backend/python/subsystems', + 'src/python/pants/core/goals', + 'src/python/pants/engine:addresses', + 'src/python/pants/engine:fs', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:selectors', + 'src/python/pants/engine:unions', + 'src/python/pants/engine/legacy:structs', + 'src/python/pants/source', + 'src/python/pants/testutil:interpreter_selection_utils', + 'src/python/pants/testutil:external_tool_test_base', + 'src/python/pants/testutil/option', + ], + tags = {'integration', 'partially_type_checked'}, +) diff --git a/src/python/pants/backend/python/lint/mypy/__init__.py b/src/python/pants/backend/python/lint/mypy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/lint/mypy/register.py b/src/python/pants/backend/python/lint/mypy/register.py new file mode 100644 index 00000000000..90ef90bc70e --- /dev/null +++ b/src/python/pants/backend/python/lint/mypy/register.py @@ -0,0 +1,14 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +"""Type checker for Python. + +See https://pants.readme.io/docs/python-linters-and-formatters and +https://mypy.readthedocs.io/en/stable/. +""" + +from pants.backend.python.lint.mypy import rules as mypy_rules + + +def rules(): + return mypy_rules.rules() diff --git a/src/python/pants/backend/python/lint/mypy/rules.py b/src/python/pants/backend/python/lint/mypy/rules.py new file mode 100644 index 00000000000..9426c576ab2 --- /dev/null +++ b/src/python/pants/backend/python/lint/mypy/rules.py @@ -0,0 +1,144 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from typing import Tuple + +from pants.backend.python.lint.mypy.subsystem import MyPy +from pants.backend.python.rules import download_pex_bin, importable_python_sources, pex +from pants.backend.python.rules.importable_python_sources import ImportablePythonSources +from pants.backend.python.rules.pex import ( + Pex, + PexInterpreterConstraints, + PexRequest, + PexRequirements, +) +from pants.backend.python.subsystems import python_native_code, subprocess_environment +from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment +from pants.backend.python.target_types import PythonSources +from pants.core.goals.lint import LintRequest, LintResult, LintResults +from pants.core.util_rules import determine_source_files, strip_source_roots +from pants.engine.addresses import Addresses +from pants.engine.fs import ( + Digest, + FileContent, + InputFilesContent, + MergeDigests, + PathGlobs, + Snapshot, +) +from pants.engine.process import FallibleProcessResult, Process +from pants.engine.rules import SubsystemRule, rule +from pants.engine.selectors import Get, MultiGet +from pants.engine.target import FieldSetWithOrigin, Targets, TransitiveTargets +from pants.engine.unions import UnionRule +from pants.option.global_options import GlobMatchErrorBehavior +from pants.python.python_setup import PythonSetup +from pants.util.strutil import pluralize + + +@dataclass(frozen=True) +class MyPyFieldSet(FieldSetWithOrigin): + required_fields = (PythonSources,) + + sources: PythonSources + + +class MyPyRequest(LintRequest): + field_set_type = MyPyFieldSet + + +def generate_args(mypy: MyPy, *, file_list_path: str) -> Tuple[str, ...]: + args = [] + if mypy.config: + args.append(f"--config-file={mypy.config}") + args.extend(mypy.args) + args.append(f"@{file_list_path}") + return tuple(args) + + +# TODO(#10131): Improve performance, e.g. by leveraging the MyPy cache. +# TODO(#10131): Support plugins and type stubs. +@rule(desc="Lint using MyPy") +async def mypy_lint( + request: MyPyRequest, + mypy: MyPy, + python_setup: PythonSetup, + subprocess_encoding_environment: SubprocessEncodingEnvironment, +) -> LintResults: + if mypy.skip: + return LintResults() + + transitive_targets = await Get( + TransitiveTargets, Addresses(fs.address for fs in request.field_sets) + ) + + prepared_sources_request = Get(ImportablePythonSources, Targets(transitive_targets.closure)) + pex_request = Get( + Pex, + PexRequest( + output_filename="mypy.pex", + requirements=PexRequirements(mypy.get_requirement_specs()), + # TODO(#10131): figure out how to robustly handle interpreter constraints. Unlike other + # linters, the version of Python used to run MyPy can be different than the version of + # the code. + interpreter_constraints=PexInterpreterConstraints(mypy.default_interpreter_constraints), + entry_point=mypy.get_entry_point(), + ), + ) + config_snapshot_request = Get( + Snapshot, + PathGlobs( + globs=[mypy.config] if mypy.config else [], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin="the option `--mypy-config`", + ), + ) + prepared_sources, pex, config_snapshot = await MultiGet( + prepared_sources_request, pex_request, config_snapshot_request + ) + + file_list_path = "__files.txt" + file_list = await Get( + Digest, + InputFilesContent( + [FileContent(file_list_path, "\n".join(prepared_sources.snapshot.files).encode())] + ), + ) + + merged_input_files = await Get( + Digest, + MergeDigests( + [file_list, prepared_sources.snapshot.digest, pex.digest, config_snapshot.digest] + ), + ) + + address_references = ", ".join(sorted(tgt.address.spec for tgt in transitive_targets.closure)) + process = pex.create_process( + python_setup=python_setup, + subprocess_encoding_environment=subprocess_encoding_environment, + pex_path=pex.output_filename, + pex_args=generate_args(mypy, file_list_path=file_list_path), + input_digest=merged_input_files, + description=( + f"Run MyPy on {pluralize(len(transitive_targets.closure), 'target')}: " + f"{address_references}." + ), + ) + result = await Get(FallibleProcessResult, Process, process) + return LintResults([LintResult.from_fallible_process_result(result, linter_name="MyPy")]) + + +def rules(): + return [ + mypy_lint, + SubsystemRule(MyPy), + UnionRule(LintRequest, MyPyRequest), + *download_pex_bin.rules(), + *determine_source_files.rules(), + *importable_python_sources.rules(), + *pex.rules(), + *python_native_code.rules(), + *strip_source_roots.rules(), + *subprocess_environment.rules(), + ] diff --git a/src/python/pants/backend/python/lint/mypy/rules_integration_test.py b/src/python/pants/backend/python/lint/mypy/rules_integration_test.py new file mode 100644 index 00000000000..57080753b7e --- /dev/null +++ b/src/python/pants/backend/python/lint/mypy/rules_integration_test.py @@ -0,0 +1,213 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pathlib import PurePath +from textwrap import dedent +from typing import List, Optional + +from pants.backend.python.lint.mypy.rules import MyPyFieldSet, MyPyRequest +from pants.backend.python.lint.mypy.rules import rules as mypy_rules +from pants.backend.python.target_types import PythonLibrary +from pants.base.specs import SingleAddress +from pants.core.goals.lint import LintResults +from pants.engine.addresses import Address +from pants.engine.fs import FileContent +from pants.engine.rules import RootRule +from pants.engine.selectors import Params +from pants.engine.target import TargetWithOrigin, WrappedTarget +from pants.testutil.external_tool_test_base import ExternalToolTestBase +from pants.testutil.option.util import create_options_bootstrapper + + +class MyPyIntegrationTest(ExternalToolTestBase): + + good_source = FileContent( + "project/good.py", + dedent( + """\ + def add(x: int, y: int) -> int: + return x + y + + result = add(3, 3) + """ + ).encode(), + ) + bad_source = FileContent( + "project/bad.py", + dedent( + """\ + def add(x: int, y: int) -> int: + return x + y + + result = add(2.0, 3.0) + """ + ).encode(), + ) + needs_config_source = FileContent( + "project/needs_config.py", + dedent( + """\ + from typing import Any, cast + + # This will fail if `--disallow-any-expr` is configured. + x = cast(Any, "hello") + """ + ).encode(), + ) + + @classmethod + def rules(cls): + return (*super().rules(), *mypy_rules(), RootRule(MyPyRequest)) + + @classmethod + def target_types(cls): + return [PythonLibrary] + + def make_target_with_origin( + self, + source_files: List[FileContent], + *, + name: str = "target", + dependencies: Optional[List[Address]] = None, + ) -> TargetWithOrigin: + for source_file in source_files: + self.create_file(source_file.path, source_file.content.decode()) + source_globs = [PurePath(source_file.path).name for source_file in source_files] + self.add_to_build_file( + "project", + dedent( + f"""\ + python_library( + name={repr(name)}, + sources={source_globs}, + dependencies={[str(dep) for dep in dependencies or ()]}, + ) + """ + ), + ) + target = self.request_single_product(WrappedTarget, Address("project", name)).target + origin = SingleAddress(directory="project", name=name) + return TargetWithOrigin(target, origin) + + def run_mypy( + self, + targets: List[TargetWithOrigin], + *, + config: Optional[str] = None, + passthrough_args: Optional[str] = None, + skip: bool = False, + additional_args: Optional[List[str]] = None, + ) -> LintResults: + args = ["--backend-packages2=pants.backend.python.lint.mypy"] + if config: + self.create_file(relpath="mypy.ini", contents=config) + args.append("--mypy-config=mypy.ini") + if passthrough_args: + args.append(f"--mypy-args='{passthrough_args}'") + if skip: + args.append("--mypy-skip") + if additional_args: + args.extend(additional_args) + return self.request_single_product( + LintResults, + Params( + MyPyRequest(MyPyFieldSet.create(tgt) for tgt in targets), + create_options_bootstrapper(args=args), + ), + ) + + def test_passing_source(self) -> None: + target = self.make_target_with_origin([self.good_source]) + result = self.run_mypy([target]) + assert len(result) == 1 + assert result[0].exit_code == 0 + assert "Success: no issues found" in result[0].stdout.strip() + + def test_failing_source(self) -> None: + target = self.make_target_with_origin([self.bad_source]) + result = self.run_mypy([target]) + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "project/bad.py:4" in result[0].stdout + + def test_mixed_sources(self) -> None: + target = self.make_target_with_origin([self.good_source, self.bad_source]) + result = self.run_mypy([target]) + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "project/good.py" not in result[0].stdout + assert "project/bad.py:4" in result[0].stdout + assert "checked 3 source files" in result[0].stdout + + def test_multiple_targets(self) -> None: + targets = [ + self.make_target_with_origin([self.good_source], name="t1"), + self.make_target_with_origin([self.bad_source], name="t2"), + ] + result = self.run_mypy(targets) + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "project/good.py" not in result[0].stdout + assert "project/bad.py:4" in result[0].stdout + assert "checked 3 source files" in result[0].stdout + + def test_respects_config_file(self) -> None: + target = self.make_target_with_origin([self.needs_config_source]) + result = self.run_mypy([target], config="[mypy]\ndisallow_any_expr = True\n") + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "project/needs_config.py:4" in result[0].stdout + + def test_respects_passthrough_args(self) -> None: + target = self.make_target_with_origin([self.needs_config_source]) + result = self.run_mypy([target], passthrough_args="--disallow-any-expr") + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "project/needs_config.py:4" in result[0].stdout + + def test_skip(self) -> None: + target = self.make_target_with_origin([self.bad_source]) + result = self.run_mypy([target], skip=True) + assert not result + + def test_transitive_dependencies(self) -> None: + self.create_file( + "project/util/lib.py", + dedent( + """\ + def capitalize(v: str) -> str: + return v.capitalize() + """ + ), + ) + self.add_to_build_file("project/util", "python_library()") + self.create_file( + "project/math/add.py", + dedent( + """\ + from project.util.lib import capitalize + + def add(x: int, y: int) -> str: + sum = x + y + return capitalize(sum) # This is the wrong type. + """ + ), + ) + self.add_to_build_file("project/math", "python_library(dependencies=['project/util'])") + source_content = FileContent( + "project/app.py", + dedent( + """\ + from project.math.add import add + + print(add(2, 4)) + """ + ).encode(), + ) + target = self.make_target_with_origin( + [source_content], dependencies=[Address.parse("project/math")] + ) + result = self.run_mypy([target]) + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "project/math/add.py:5" in result[0].stdout diff --git a/src/python/pants/backend/python/lint/mypy/subsystem.py b/src/python/pants/backend/python/lint/mypy/subsystem.py new file mode 100644 index 00000000000..9817099412e --- /dev/null +++ b/src/python/pants/backend/python/lint/mypy/subsystem.py @@ -0,0 +1,46 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from typing import Optional, Tuple, cast + +from pants.backend.python.subsystems.python_tool_base import PythonToolBase +from pants.option.custom_types import file_option, shell_str + + +class MyPy(PythonToolBase): + options_scope = "mypy" + default_version = "mypy==0.781" + default_entry_point = "mypy" + default_interpreter_constraints = ["CPython>=3.5"] + + @classmethod + def register_options(cls, register): + super().register_options(register) + register( + "--skip", type=bool, default=False, help="Don't use MyPy when running `./pants lint`." + ) + register( + "--args", + type=list, + member_type=shell_str, + help="Arguments to pass directly to mypy, e.g. " + '`--mypy-args="--python-version 3.7 --disallow-any-expr"`', + ) + register( + "--config", + type=file_option, + advanced=True, + help="Path to `mypy.ini` or alternative MyPy config file", + ) + + @property + def skip(self) -> bool: + return cast(bool, self.options.skip) + + @property + def args(self) -> Tuple[str, ...]: + return tuple(self.options.args) + + @property + def config(self) -> Optional[str]: + return cast(Optional[str], self.options.config)