Update mypy to latest and turn on everywhere

.changes/unreleased/Under the Hood-20220427-112127.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Mypy -> 0.942 + fixed import logic to allow for full mypy coverage
+time: 2022-04-27T11:21:27.499359-05:00
+custom:
+  Author: iknox-fa
+  Issue: "4805, 4089"
+  PR: "5171"

.github/workflows/main.yml

@@ -52,7 +52,7 @@ jobs:
           pip --version
           pip install pre-commit
           pre-commit --version
-          pip install mypy==0.782
+          pip install mypy==0.942
           mypy --version
           pip install -r editable-requirements.txt
           dbt --version

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
     alias: flake8-check
     stages: [manual]
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.782
+  rev: v0.942
   hooks:
   - id: mypy
     # N.B.: Mypy is... a bit fragile.

core/dbt/__init__.py

@@ -0,0 +1,7 @@
+# N.B.
+# This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters)
+# The matching statement is in plugins/postgres/dbt/__init__.py
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

core/dbt/adapters/__init__.py

@@ -0,0 +1,7 @@
+# N.B.
+# This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters)
+# The matching statement is in plugins/postgres/dbt/adapters/__init__.py
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

core/dbt/adapters/factory.py

@@ -140,8 +140,6 @@ def get_adapter_plugins(self, name: Optional[str]) -> List[AdapterPlugin]:
                 raise InternalException(f"No plugin found for {plugin_name}") from None
             plugins.append(plugin)
             seen.add(plugin_name)
-            if plugin.dependencies is None:
-                continue
             for dep in plugin.dependencies:
                 if dep not in seen:
                     plugin_names.append(dep)

core/dbt/adapters/protocol.py

@@ -7,7 +7,6 @@
     List,
     Generic,
     TypeVar,
-    ClassVar,
     Tuple,
     Union,
     Dict,
@@ -88,10 +87,13 @@ class AdapterProtocol(  # type: ignore[misc]
         Compiler_T,
     ],
 ):
-    AdapterSpecificConfigs: ClassVar[Type[AdapterConfig_T]]
-    Column: ClassVar[Type[Column_T]]
-    Relation: ClassVar[Type[Relation_T]]
-    ConnectionManager: ClassVar[Type[ConnectionManager_T]]
+    # N.B. Technically these are ClassVars, but mypy doesn't support putting type vars in a
+    # ClassVar due to the restirctiveness of PEP-526
+    # See: https://github.com/python/mypy/issues/5144
+    AdapterSpecificConfigs: Type[AdapterConfig_T]
+    Column: Type[Column_T]
+    Relation: Type[Relation_T]
+    ConnectionManager: Type[ConnectionManager_T]
     connections: ConnectionManager_T
 
     def __init__(self, config: AdapterRequiredConfig):

core/dbt/adapters/reference_keys.py

@@ -1,7 +1,7 @@
 # this module exists to resolve circular imports with the events module
 
 from collections import namedtuple
-from typing import Optional
+from typing import Any, Optional
 
 
 _ReferenceKey = namedtuple("_ReferenceKey", "database schema identifier")
@@ -14,7 +14,7 @@ def lowercase(value: Optional[str]) -> Optional[str]:
         return value.lower()
 
 
-def _make_key(relation) -> _ReferenceKey:
+def _make_key(relation: Any) -> _ReferenceKey:
     """Make _ReferenceKeys with lowercase values for the cache so we don't have
     to keep track of quoting
     """

core/dbt/clients/system.py

@@ -246,16 +246,17 @@ def _supports_long_paths() -> bool:
     # https://stackoverflow.com/a/35097999/11262881
     # I don't know exaclty what he means, but I am inclined to believe him as
     # he's pretty active on Python windows bugs!
-    try:
-        dll = WinDLL("ntdll")
-    except OSError:  # I don't think this happens? you need ntdll to run python
-        return False
-    # not all windows versions have it at all
-    if not hasattr(dll, "RtlAreLongPathsEnabled"):
-        return False
-    # tell windows we want to get back a single unsigned byte (a bool).
-    dll.RtlAreLongPathsEnabled.restype = c_bool
-    return dll.RtlAreLongPathsEnabled()
+    else:
+        try:
+            dll = WinDLL("ntdll")
+        except OSError:  # I don't think this happens? you need ntdll to run python
+            return False
+        # not all windows versions have it at all
+        if not hasattr(dll, "RtlAreLongPathsEnabled"):
+            return False
+        # tell windows we want to get back a single unsigned byte (a bool).
+        dll.RtlAreLongPathsEnabled.restype = c_bool
+        return dll.RtlAreLongPathsEnabled()
 
 
 def convert_path(path: str) -> str:

core/dbt/contracts/graph/parsed.py

@@ -586,10 +586,7 @@ def quote_columns(self) -> Optional[bool]:
 
     @property
     def columns(self) -> Sequence[UnparsedColumn]:
-        if self.table.columns is None:
-            return []
-        else:
-            return self.table.columns
+        return [] if self.table.columns is None else self.table.columns
 
     def get_tests(self) -> Iterator[Tuple[Dict[str, Any], Optional[UnparsedColumn]]]:
         for test in self.tests:

core/dbt/events/types.py

@@ -2417,9 +2417,7 @@ class GeneralWarningMsg(WarnLevel):
     code: str = "Z046"
 
     def message(self) -> str:
-        if self.log_fmt is not None:
-            return self.log_fmt.format(self.msg)
-        return self.msg
+        return self.log_fmt.format(self.msg) if self.log_fmt is not None else self.msg
 
 
 @dataclass
@@ -2429,9 +2427,7 @@ class GeneralWarningException(WarnLevel):
     code: str = "Z047"
 
     def message(self) -> str:
-        if self.log_fmt is not None:
-            return self.log_fmt.format(str(self.exc))
-        return str(self.exc)
+        return self.log_fmt.format(str(self.exc)) if self.log_fmt is not None else str(self.exc)
 
 
 @dataclass

core/dbt/graph/selector_methods.py

@@ -540,7 +540,7 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
             )
 
         current_state_sources = {
-            result.unique_id: getattr(result, "max_loaded_at", None)
+            result.unique_id: getattr(result, "max_loaded_at", 0)
             for result in self.previous_state.sources_current.results
             if hasattr(result, "max_loaded_at")
         }
@@ -552,7 +552,7 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
         }
 
         previous_state_sources = {
-            result.unique_id: getattr(result, "max_loaded_at", None)
+            result.unique_id: getattr(result, "max_loaded_at", 0)
             for result in self.previous_state.sources.results
             if hasattr(result, "max_loaded_at")
         }

core/dbt/parser/manifest.py

@@ -946,8 +946,6 @@ def _check_resource_uniqueness(
     for resource, node in manifest.nodes.items():
         if not node.is_relational:
             continue
-        # appease mypy - sources aren't refable!
-        assert not isinstance(node, ParsedSourceDefinition)
 
         name = node.name
         # the full node name is really defined by the adapter's relation

core/dbt/parser/sources.py

@@ -63,7 +63,7 @@ def construct_sources(self) -> None:
                 self.sources[unpatched.unique_id] = unpatched
                 continue
             # returns None if there is no patch
-            patch = self.get_patch_for(unpatched)
+            patch = self.get_patch_for(unpatched)  # type: ignore[unreachable] # CT-564 / GH 5169
 
             # returns unpatched if there is no patch
             patched = self.patch_source(unpatched, patch)
@@ -213,8 +213,8 @@ def get_patch_for(
         self,
         unpatched: UnpatchedSourceDefinition,
     ) -> Optional[SourcePatch]:
-        if isinstance(unpatched, ParsedSourceDefinition):
-            return None
+        if isinstance(unpatched, ParsedSourceDefinition):  # type: ignore[unreachable] # CT-564 / GH 5169
+            return None  # type: ignore[unreachable] # CT-564 / GH 5169
         key = (unpatched.package_name, unpatched.source.name)
         patch: Optional[SourcePatch] = self.manifest.source_patches.get(key)
         if patch is None:

core/dbt/profiler.py

@@ -1,10 +1,11 @@
 from contextlib import contextmanager
 from cProfile import Profile
 from pstats import Stats
+from typing import Any, Generator
 
 
 @contextmanager
-def profiler(enable, outfile):
+def profiler(enable: bool, outfile: str) -> Generator[Any, None, None]:
     try:
         if enable:
             profiler = Profile()
@@ -16,4 +17,4 @@ def profiler(enable, outfile):
             profiler.disable()
             stats = Stats(profiler)
             stats.sort_stats("tottime")
-            stats.dump_stats(outfile)
+            stats.dump_stats(str(outfile))

core/dbt/selected_resources.py

@@ -1,6 +1,8 @@
+from typing import Set, Any
+
 SELECTED_RESOURCES = []
 
 
-def set_selected_resources(selected_resources):
+def set_selected_resources(selected_resources: Set[Any]) -> None:
     global SELECTED_RESOURCES
     SELECTED_RESOURCES = list(selected_resources)

core/dbt/ui.py

@@ -18,28 +18,28 @@
 COLOR_RESET_ALL = COLORS["reset_all"]
 
 
-def color(text: str, color_code: str):
+def color(text: str, color_code: str) -> str:
     if flags.USE_COLORS:
         return "{}{}{}".format(color_code, text, COLOR_RESET_ALL)
     else:
         return text
 
 
-def printer_width():
+def printer_width() -> int:
     if flags.PRINTER_WIDTH:
         return flags.PRINTER_WIDTH
     return 80
 
 
-def green(text: str):
+def green(text: str) -> str:
     return color(text, COLOR_FG_GREEN)
 
 
-def yellow(text: str):
+def yellow(text: str) -> str:
     return color(text, COLOR_FG_YELLOW)
 
 
-def red(text: str):
+def red(text: str) -> str:
     return color(text, COLOR_FG_RED)
 
 

core/dbt/version.py

@@ -3,6 +3,7 @@
 import os
 import glob
 import json
+from pathlib import Path
 from typing import Iterator, List, Optional, Tuple
 
 import requests
@@ -224,6 +225,13 @@ def _get_adapter_plugin_names() -> Iterator[str]:
     # not be reporting plugin versions today
     if spec is None or spec.submodule_search_locations is None:
         return
+
+    # https://github.com/dbt-labs/dbt-core/pull/5171 changes how importing adapters works a bit and renders the previous discovery method useless for postgres.
+    # To solve this we manually add that path to the search path below.
+    #  I don't like this solution.  Not one bit.
+    postgres_path = Path(__file__ + "/../../../plugins/postgres/dbt/adapters").resolve()
+    spec.submodule_search_locations.append(str(postgres_path))
+
     for adapters_path in spec.submodule_search_locations:
         version_glob = os.path.join(adapters_path, "*", "__version__.py")
         for version_path in glob.glob(version_glob):

core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py

@@ -5,6 +5,7 @@
 import os
 import re
 import sys
+from typing import List
 import yaml
 
 LOGGER = logging.getLogger("upgrade_dbt_schema")
@@ -362,7 +363,7 @@ def convert_schema(initial, extra_complex_tests):
 
 
 class CustomSortedSchema(dict):
-    ITEMS_ORDER = NotImplemented
+    ITEMS_ORDER: List[str]
 
     @classmethod
     def _items_keyfunc(cls, items):
@@ -395,7 +396,7 @@ class CustomSortedColumnsSchema(CustomSortedSchema):
 
 
 for cls in (CustomSortedRootSchema, CustomSortedModelsSchema, CustomSortedColumnsSchema):
-    yaml.add_representer(cls, cls.representer)
+    yaml.add_representer(cls, cls.representer)  # type: ignore [attr-defined]
 
 
 if __name__ == "__main__":

dev-requirements.txt

@@ -4,7 +4,7 @@ flake8
 flaky
 freezegun==0.3.12
 ipdb
-mypy==0.782
+mypy==0.942
 pip-tools
 pre-commit
 pytest
@@ -17,4 +17,13 @@ pytest-xdist
 pytz
 tox>=3.13
 twine
+types-colorama
+types-PyYAML
+types-freezegun
+types-Jinja2
+types-mock
+types-python-dateutil
+types-pytz
+types-requests
+types-setuptools
 wheel

mypy.ini

@@ -1,3 +1,9 @@
 [mypy]
-mypy_path = ./third-party-stubs
+mypy_path = third-party-stubs/
 namespace_packages = True
+#explicit_package_bases = True
+exclude = plugins/*|third-party-stubs/*
+
+# Per-module options:
+#[mypy-mycode.foo.*]
+#disallow_untyped_defs = True

plugins/postgres/dbt/__init__.py

@@ -0,0 +1,7 @@
+# N.B.
+# This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters)
+# The matching statement is in core/dbt/__init__.py
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

plugins/postgres/dbt/adapters/__init__.py

@@ -0,0 +1,7 @@
+# N.B.
+# This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters)
+# The matching statement is in core/dbt/adapters/__init__.py
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)