Namespace packages (PEP 420) (python#5691)

Tentative implementation of PEP 420. Fixes python#1645.

Clarification of the implementation:

- `candidate_base_dirs` is a list of `(dir, verify)` tuples, laboriously pre-computed.
- The old code just looped over this list, looking for `dir/<module>`, and if found, it would verify that there were `__init__.py` files in all the right places (except if `verify` is false); the first success would be the hit;
- In PEP 420 mode, if the above approach finds no hits, we do something different for those paths that failed due to `__init__` verification; essentially we narrow down the list of candidate paths by checking for `__init__` files from the top down. Hopefully the last test added clarifies this.

mypy/modulefinder.py

@@ -170,6 +170,7 @@ def _find_module(self, id: str) -> Optional[str]:
         # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories.
         seplast = os.sep + components[-1]  # so e.g. '/baz'
         sepinit = os.sep + '__init__'
+        near_misses = []  # Collect near misses for namespace mode (see below).
         for base_dir, verify in candidate_base_dirs:
             base_path = base_dir + seplast  # so e.g. '/usr/lib/python3.4/foo/bar/baz'
             # Prefer package over module, i.e. baz/__init__.py* over baz.py*.
@@ -178,19 +179,49 @@ def _find_module(self, id: str) -> Optional[str]:
                 path_stubs = base_path + '-stubs' + sepinit + extension
                 if fscache.isfile_case(path):
                     if verify and not verify_module(fscache, id, path):
+                        near_misses.append(path)
                         continue
                     return path
                 elif fscache.isfile_case(path_stubs):
                     if verify and not verify_module(fscache, id, path_stubs):
+                        near_misses.append(path_stubs)
                         continue
                     return path_stubs
             # No package, look for module.
             for extension in PYTHON_EXTENSIONS:
                 path = base_path + extension
                 if fscache.isfile_case(path):
                     if verify and not verify_module(fscache, id, path):
+                        near_misses.append(path)
                         continue
                     return path
+
+        # In namespace mode, re-check those entries that had 'verify'.
+        # Assume search path entries xxx, yyy and zzz, and we're
+        # looking for foo.bar.baz.  Suppose near_misses has:
+        #
+        # - xxx/foo/bar/baz.py
+        # - yyy/foo/bar/baz/__init__.py
+        # - zzz/foo/bar/baz.pyi
+        #
+        # If any of the foo directories has __init__.py[i], it wins.
+        # Else, we look for foo/bar/__init__.py[i], etc.  If there are
+        # none, the first hit wins.  Note that this does not take into
+        # account whether the lowest-level module is a file (baz.py),
+        # a package (baz/__init__.py), or a stub file (baz.pyi) -- for
+        # these the first one encountered along the search path wins.
+        #
+        # The helper function highest_init_level() returns an int that
+        # indicates the highest level at which a __init__.py[i] file
+        # is found; if no __init__ was found it returns 0, if we find
+        # only foo/bar/__init__.py it returns 1, and if we have
+        # foo/__init__.py it returns 2 (regardless of what's in
+        # foo/bar).  It doesn't look higher than that.
+        if self.options and self.options.namespace_packages and near_misses:
+            levels = [highest_init_level(fscache, id, path) for path in near_misses]
+            index = levels.index(max(levels))
+            return near_misses[index]
+
         return None
 
     def find_modules_recursive(self, module: str) -> List[BuildSource]:
@@ -236,6 +267,19 @@ def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool:
     return True
 
 
+def highest_init_level(fscache: FileSystemCache, id: str, path: str) -> int:
+    """Compute the highest level where an __init__ file is found."""
+    if path.endswith(('__init__.py', '__init__.pyi')):
+        path = os.path.dirname(path)
+    level = 0
+    for i in range(id.count('.')):
+        path = os.path.dirname(path)
+        if any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension)))
+               for extension in PYTHON_EXTENSIONS):
+            level = i + 1
+    return level
+
+
 def mypy_path() -> List[str]:
     path_env = os.getenv('MYPYPATH')
     if not path_env:

test-data/unit/check-modules.test

@@ -1,4 +1,5 @@
 -- Type checker test cases dealing with modules and imports.
+-- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages).
 
 [case testAccessImportedDefinitions]
 import m
@@ -2525,3 +2526,97 @@ def __radd__(self) -> int: ...
 
 [case testFunctionWithInPlaceDunderName]
 def __iadd__(self) -> int: ...
+
+-- Tests for PEP 420 namespace packages.
+
+[case testClassicPackage]
+from foo.bar import x
+[file foo/__init__.py]
+# empty
+[file foo/bar.py]
+x = 0
+
+[case testClassicNotPackage]
+from foo.bar import x
+[file foo/bar.py]
+x = 0
+[out]
+main:1: error: Cannot find module named 'foo.bar'
+main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+
+[case testNamespacePackage]
+# flags: --namespace-packages
+from foo.bar import x
+reveal_type(x)  # E: Revealed type is 'builtins.int'
+[file foo/bar.py]
+x = 0
+
+[case testNamespacePackageWithMypyPath]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bax import x
+from foo.bay import y
+from foo.baz import z
+reveal_type(x)  # E: Revealed type is 'builtins.int'
+reveal_type(y)  # E: Revealed type is 'builtins.int'
+reveal_type(z)  # E: Revealed type is 'builtins.int'
+[file xx/foo/bax.py]
+x = 0
+[file yy/foo/bay.py]
+y = 0
+[file foo/baz.py]
+z = 0
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testClassicPackageIgnoresEarlierNamespacePackage]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar import y
+reveal_type(y)  # E: Revealed type is 'builtins.int'
+[file xx/foo/bar.py]
+x = ''
+[file yy/foo/bar.py]
+y = 0
+[file yy/foo/__init__.py]
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testNamespacePackagePickFirstOnMypyPath]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar import x
+reveal_type(x)  # E: Revealed type is 'builtins.int'
+[file xx/foo/bar.py]
+x = 0
+[file yy/foo/bar.py]
+x = ''
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testNamespacePackageInsideClassicPackage]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar.baz import x
+reveal_type(x)  # E: Revealed type is 'builtins.int'
+[file xx/foo/bar/baz.py]
+x = ''
+[file yy/foo/bar/baz.py]
+x = 0
+[file yy/foo/__init__.py]
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testClassicPackageInsideNamespacePackage]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar.baz.boo import x
+reveal_type(x)  # E: Revealed type is 'builtins.int'
+[file xx/foo/bar/baz/boo.py]
+x = ''
+[file xx/foo/bar/baz/__init__.py]
+[file yy/foo/bar/baz/boo.py]
+x = 0
+[file yy/foo/bar/__init__.py]
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy