fix: don't measure third-party packages

Avoid measuring code located where third-party packages get installed.
We have to take care to measure --source code even if it is installed in
a third-party location.

This also fixes #905, coverage generating warnings about coverage being
imported when it will be measured.

#876
#905

CHANGES.rst

@@ -24,10 +24,23 @@ want to know what's different in 5.0 since 4.5.x, see :ref:`whatsnew5x`.
 Unreleased
 ----------
 
+- Third-party packages are now ignored in coverage reporting.  This solves a
+  few problems:
+
+  - Coverage will no longer report about other people's code (`issue 876`_).
+    This is true even when using ``--source=.`` with a venv in the current
+    directory.
+
+  - Coverage will no longer generate "Already imported a file that will be
+    measured" warnings about coverage itself (`issue 905`_).
+
 - The JSON report now includes ``percent_covered_display``, a string with the
   total percentage, rounded to the same number of decimal places as the other
   reports' totals.
 
+.. _issue 876: https://github.com/nedbat/coveragepy/issues/876
+.. _issue 905: https://github.com/nedbat/coveragepy/issues/905
+
 
 .. _changes_55:
 

coverage/inorout.py

@@ -3,18 +3,17 @@
 
 """Determining whether files are being measured/reported or not."""
 
-# For finding the stdlib
-import atexit
 import inspect
 import itertools
 import os
 import platform
 import re
 import sys
+import sysconfig
 import traceback
 
 from coverage import env
-from coverage.backward import code_object
+from coverage.backward import code_object, importlib_util_find_spec
 from coverage.disposition import FileDisposition, disposition_init
 from coverage.files import TreeMatcher, FnmatchMatcher, ModuleMatcher
 from coverage.files import prep_patterns, find_python_files, canonical_filename
@@ -108,14 +107,53 @@ def module_has_file(mod):
     return os.path.exists(mod__file__)
 
 
+def file_for_module(modulename):
+    """Find the file for `modulename`, or return None."""
+    if importlib_util_find_spec:
+        filename = None
+        try:
+            spec = importlib_util_find_spec(modulename)
+        except ImportError:
+            pass
+        else:
+            if spec is not None:
+                filename = spec.origin
+        return filename
+    else:
+        import imp
+        openfile = None
+        glo, loc = globals(), locals()
+        try:
+            # Search for the module - inside its parent package, if any - using
+            # standard import mechanics.
+            if '.' in modulename:
+                packagename, name = modulename.rsplit('.', 1)
+                package = __import__(packagename, glo, loc, ['__path__'])
+                searchpath = package.__path__
+            else:
+                packagename, name = None, modulename
+                searchpath = None  # "top-level search" in imp.find_module()
+            openfile, pathname, _ = imp.find_module(name, searchpath)
+            return pathname
+        except ImportError:
+            return None
+        finally:
+            if openfile:
+                openfile.close()
+
+
 def add_stdlib_paths(paths):
     """Add paths where the stdlib can be found to the set `paths`."""
     # Look at where some standard modules are located. That's the
     # indication for "installed with the interpreter". In some
     # environments (virtualenv, for example), these modules may be
     # spread across a few locations. Look at all the candidate modules
     # we've imported, and take all the different ones.
-    for m in (atexit, inspect, os, platform, _pypy_irc_topic, re, _structseq, traceback):
+    modules_we_happen_to_have = [
+        inspect, itertools, os, platform, re, sysconfig, traceback,
+        _pypy_irc_topic, _structseq,
+    ]
+    for m in modules_we_happen_to_have:
         if m is not None and hasattr(m, "__file__"):
             paths.add(canonical_path(m, directory=True))
 
@@ -129,6 +167,20 @@ def add_stdlib_paths(paths):
             paths.add(canonical_path(structseq_file))
 
 
+def add_third_party_paths(paths):
+    """Add locations for third-party packages to the set `paths`."""
+    # Get the paths that sysconfig knows about.
+    scheme_names = set(sysconfig.get_scheme_names())
+
+    for scheme in scheme_names:
+        # https://foss.heptapod.net/pypy/pypy/-/issues/3433
+        better_scheme = "pypy_posix" if scheme == "pypy" else scheme
+        if os.name in better_scheme.split("_"):
+            config_paths = sysconfig.get_paths(scheme)
+            for path_name in ["platlib", "purelib"]:
+                paths.add(config_paths[path_name])
+
+
 def add_coverage_paths(paths):
     """Add paths where coverage.py code can be found to the set `paths`."""
     cover_path = canonical_path(__file__, directory=True)
@@ -156,8 +208,8 @@ def __init__(self, warn, debug):
         # The matchers for should_trace.
         self.source_match = None
         self.source_pkgs_match = None
-        self.pylib_paths = self.cover_paths = None
-        self.pylib_match = self.cover_match = None
+        self.pylib_paths = self.cover_paths = self.third_paths = None
+        self.pylib_match = self.cover_match = self.third_match = None
         self.include_match = self.omit_match = None
         self.plugins = []
         self.disp_class = FileDisposition
@@ -168,6 +220,9 @@ def __init__(self, warn, debug):
         self.source_pkgs_unmatched = []
         self.omit = self.include = None
 
+        # Is the source inside a third-party area?
+        self.source_in_third = False
+
     def configure(self, config):
         """Apply the configuration to get ready for decision-time."""
         self.source_pkgs.extend(config.source_pkgs)
@@ -191,6 +246,10 @@ def configure(self, config):
         self.cover_paths = set()
         add_coverage_paths(self.cover_paths)
 
+        # Find where third-party packages are installed.
+        self.third_paths = set()
+        add_third_party_paths(self.third_paths)
+
         def debug(msg):
             if self.debug:
                 self.debug.write(msg)
@@ -218,6 +277,24 @@ def debug(msg):
         if self.omit:
             self.omit_match = FnmatchMatcher(self.omit)
             debug("Omit matching: {!r}".format(self.omit_match))
+        if self.third_paths:
+            self.third_match = TreeMatcher(self.third_paths)
+            debug("Third-party lib matching: {!r}".format(self.third_match))
+
+        # Check if the source we want to measure has been installed as a
+        # third-party package.
+        for pkg in self.source_pkgs:
+            try:
+                modfile = file_for_module(pkg)
+                debug("Imported {} as {}".format(pkg, modfile))
+            except CoverageException as exc:
+                debug("Couldn't import {}: {}".format(pkg, exc))
+                continue
+            if modfile and self.third_match.match(modfile):
+                self.source_in_third = True
+        for src in self.source:
+            if self.third_match.match(src):
+                self.source_in_third = True
 
     def should_trace(self, filename, frame=None):
         """Decide whether to trace execution in `filename`, with a reason.
@@ -352,6 +429,9 @@ def check_include_omit_etc(self, filename, frame):
                     ok = True
             if not ok:
                 return extra + "falls outside the --source spec"
+            if not self.source_in_third:
+                if self.third_match.match(filename):
+                    return "inside --source, but in third-party"
         elif self.include_match:
             if not self.include_match.match(filename):
                 return "falls outside the --include trees"
@@ -361,6 +441,10 @@ def check_include_omit_etc(self, filename, frame):
             if self.pylib_match and self.pylib_match.match(filename):
                 return "is in the stdlib"
 
+            # Exclude anything in the third-party installation areas.
+            if self.third_match and self.third_match.match(filename):
+                return "is a third-party module"
+
             # We exclude the coverage.py code itself, since a little of it
             # will be measured otherwise.
             if self.cover_match and self.cover_match.match(filename):
@@ -485,14 +569,15 @@ def sys_info(self):
         Returns a list of (key, value) pairs.
         """
         info = [
-            ('cover_paths', self.cover_paths),
-            ('pylib_paths', self.pylib_paths),
+            ("coverage_paths", self.cover_paths),
+            ("stdlib_paths", self.pylib_paths),
+            ("third_party_paths", self.third_paths),
         ]
 
         matcher_names = [
             'source_match', 'source_pkgs_match',
             'include_match', 'omit_match',
-            'cover_match', 'pylib_match',
+            'cover_match', 'pylib_match', 'third_match',
             ]
 
         for matcher_name in matcher_names:

coverage/version.py

@@ -5,7 +5,7 @@
 # This file is exec'ed in setup.py, don't import anything!
 
 # Same semantics as sys.version_info.
-version_info = (5, 5, 1, "alpha", 0)
+version_info = (5, 6, 0, "beta", 1)
 
 
 def _make_version(major, minor, micro, releaselevel, serial):

tests/test_debug.py

@@ -183,8 +183,9 @@ def test_debug_sys(self):
         out_lines = self.f1_debug_output(["sys"])
 
         labels = """
-            version coverage cover_paths pylib_paths tracer configs_attempted config_file
-            configs_read data_file python platform implementation executable
+            version coverage coverage_paths stdlib_paths third_party_paths
+            tracer configs_attempted config_file configs_read data_file
+            python platform implementation executable
             pid cwd path environment command_line cover_match pylib_match
             """.split()
         for label in labels:

tests/test_process.py

@@ -8,6 +8,7 @@
 import os
 import os.path
 import re
+import shutil
 import stat
 import sys
 import sysconfig
@@ -24,7 +25,7 @@
 from coverage.misc import output_encoding
 
 from tests.coveragetest import CoverageTest, TESTS_DIR
-from tests.helpers import re_lines
+from tests.helpers import change_dir, make_file, nice_file, re_lines, run_command
 
 
 class ProcessTest(CoverageTest):
@@ -1640,3 +1641,100 @@ def test_dashm_pkg_sub(self):
 
     def test_script_pkg_sub(self):
         self.assert_pth_and_source_work_together('', 'pkg', 'sub')
+
+
+def run_in_venv(args):
+    """Run python with `args` in the "venv" virtualenv.
+
+    Returns the text output of the command.
+    """
+    if env.WINDOWS:
+        cmd = r".\venv\Scripts\python.exe "
+    else:
+        cmd = "./venv/bin/python "
+    cmd += args
+    status, output = run_command(cmd)
+    print(output)
+    assert status == 0
+    return output
+
+
+@pytest.fixture(scope="session", name="venv_factory")
+def venv_factory_fixture(tmp_path_factory):
+    """Produce a function which can copy a venv template to a new directory.
+
+    The function accepts one argument, the directory to use for the venv.
+    """
+    tmpdir = tmp_path_factory.mktemp("venv_template")
+    with change_dir(str(tmpdir)):
+        # Create a virtualenv.
+        run_command("python -m virtualenv venv")
+
+        # A third-party package that installs two different packages.
+        make_file("third_pkg/third/__init__.py", """\
+            import fourth
+            def third(x):
+                return 3 * x
+            """)
+        make_file("third_pkg/fourth/__init__.py", """\
+            def fourth(x):
+                return 4 * x
+            """)
+        make_file("third_pkg/setup.py", """\
+            import setuptools
+            setuptools.setup(name="third", packages=["third", "fourth"])
+            """)
+
+        # Install the third-party packages.
+        run_in_venv("-m pip install --no-index ./third_pkg")
+
+        # Install coverage.
+        coverage_src = nice_file(TESTS_DIR, "..")
+        run_in_venv("-m pip install --no-index {}".format(coverage_src))
+
+    def factory(dst):
+        """The venv factory function.
+
+        Copies the venv template to `dst`.
+        """
+        shutil.copytree(str(tmpdir / "venv"), dst, symlinks=(not env.WINDOWS))
+
+    return factory
+
+
+class VirtualenvTest(CoverageTest):
+    """Tests of virtualenv considerations."""
+
+    def setup_test(self):
+        self.make_file("myproduct.py", """\
+            import third
+            print(third.third(11))
+            """)
+        self.del_environ("COVERAGE_TESTING")    # To avoid needing contracts installed.
+        super(VirtualenvTest, self).setup_test()
+
+    def test_third_party_venv_isnt_measured(self, venv_factory):
+        venv_factory("venv")
+        out = run_in_venv("-m coverage run --source=. myproduct.py")
+        # In particular, this warning doesn't appear:
+        # Already imported a file that will be measured: .../coverage/__main__.py
+        assert out == "33\n"
+        out = run_in_venv("-m coverage report")
+        assert "myproduct.py" in out
+        assert "third" not in out
+
+    def test_us_in_venv_is_measured(self, venv_factory):
+        venv_factory("venv")
+        out = run_in_venv("-m coverage run --source=third myproduct.py")
+        assert out == "33\n"
+        out = run_in_venv("-m coverage report")
+        assert "myproduct.py" not in out
+        assert "third" in out
+
+    def test_venv_isnt_measured(self, venv_factory):
+        venv_factory("venv")
+        out = run_in_venv("-m coverage run myproduct.py")
+        assert out == "33\n"
+        out = run_in_venv("-m coverage report")
+        assert "myproduct.py" in out
+        assert "third" not in out