Combine coverage from multiple test runs within a workflow

.circleci/config.yml

@@ -45,7 +45,7 @@ jobs:
           command: ci/checks/run-all-checks.sh
 
   # Earliest versions of python/git supported by git-machete
-  "python 3_6 git 1_8_10":
+  "python 3_6 git 1_8_0":
     executor: ubuntu_executor
     steps:
       - checkout
@@ -54,6 +54,9 @@ jobs:
           command: PYTHON_VERSION=3.6 GIT_VERSION=1.8.0  bash ci/tox/ci-run.sh
       - store_test_results:
           path: test-results/
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
 
   "python 3_7 git 2_7_6":
     executor: ubuntu_executor
@@ -64,6 +67,9 @@ jobs:
           command: PYTHON_VERSION=3.7 GIT_VERSION=2.7.6  bash ci/tox/ci-run.sh
       - store_test_results:
           path: test-results/
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
 
   "python 3_8 git 2_25_0":
     executor: ubuntu_executor
@@ -74,37 +80,44 @@ jobs:
           command: PYTHON_VERSION=3.8 GIT_VERSION=2.25.0  bash ci/tox/ci-run.sh
       - store_test_results:
           path: test-results/
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
+
+  "python 3_9 git 2_30_1":
+    executor: ubuntu_executor
+    steps:
+      - checkout
+      - run:
+          name: Run tests against Python 3.9 and git 2.30.1
+          command: PYTHON_VERSION=3.9 GIT_VERSION=2.30.1  bash ci/tox/ci-run.sh
+      - store_test_results:
+          path: test-results/
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
 
   # Note that there is no particular reason for the below assignment
-  # of PEP8/coverage/docs checks to Python version, other than:
+  # of PEP8/docs checks to Python version, other than:
   #  - Python 3.6 being EOLed and hence not supporting the versions of packages used for the checks,
   #  - https://github.com/VirtusLab/git-machete/issues/936 apparently forcing the use of Python 3.11 for Sphinx HTML docs.
   # Each of these checks needs to be run against just one Python version
   # (since there's very little chance that they pass under one version but fail under another),
   # so let's spread these checks evenly across versions to reduce CI execution time
   # (when compared to the scenario where all checks are executed under the same Python version).
 
-  "pep8 and python 3_9 git 2_30_1":
+  "pep8 and python 3_10 git 2_33_1":
     executor: ubuntu_executor
     steps:
       - checkout
       - run:
-          name: Run tests against Python 3.9 and git 2.30.1 (with isort/PEP8/vulture check)
-          command: PYTHON_VERSION=3.9 GIT_VERSION=2.30.1 CHECK_PEP8=true  bash ci/tox/ci-run.sh
+          name: Run tests against Python 3.10 and git 2.33.1 (with isort/PEP8/vulture check)
+          command: PYTHON_VERSION=3.10 GIT_VERSION=2.33.1 CHECK_PEP8=true  bash ci/tox/ci-run.sh
       - store_test_results:
           path: test-results/
-
-  "coverage and python 3_10 git 2_33_1":
-    executor: ubuntu_executor
-    steps:
-      - checkout
-      - run:
-          name: Run tests against Python 3.10 and git 2.33.1 (with coverage report)
-          command: PYTHON_VERSION=3.10 GIT_VERSION=2.33.1 CHECK_COVERAGE=true  bash ci/tox/ci-run.sh
-      - store_test_results:
-          path: test-results/
-      - codecov/upload:
-          file: coverage.xml
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
 
   # Note that if docs are ever to be built under a different Python version in CI,
   # this change needs to be reflected in tox.ini and .readthedocs.yaml as well
@@ -118,26 +131,47 @@ jobs:
           command: PYTHON_VERSION=3.11 GIT_VERSION=2.38.1 BUILD_SPHINX_HTML=true CHECK_DOCS_UP_TO_DATE=true  bash ci/tox/ci-run.sh
       - store_test_results:
           path: test-results/
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
       - store_artifacts:
           path: docs/html/
           destination: docs
 
-  # Note the while we use Linux to test against specific Python&git version,
+  # Note the while we use Linux executor to test against specific Python&git version,
   # on macOS and Windows we're just checking compatibility with the OS itself,
   # relying on whatever recent versions of Python and git are provided in the image.
+
   "macos tests":
     executor: macos_executor
     steps:
       - checkout
       - run: pip install tox
-      - run: tox -e py311 -- -vv
+      - run: PYTHON_VERSION=3.11-macos  tox -e coverage-py311 -- -vv
+      - store_test_results:
+          path: test-results/
+      - persist_to_workspace:
+          root: .
+          paths: [ .coverage.* ]
 
   "windows tests":
     executor: windows_executor
     steps:
       - checkout
       - run: pip install tox
-      - run: tox -e py311 -- -vv
+      # We don't collect coverage on Windows as it massively extends execution time for some reason.
+      - run: tox -e testenv -- -vv
+
+  "coverage upload":
+    executor: ubuntu_executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run: pip3 install tox
+      - run: tox -e coverage-combine
+      - codecov/upload:
+          file: coverage.xml
 
   "dry run deploy":
     executor: ubuntu_executor
@@ -238,11 +272,11 @@ jobs:
 # it's just to introduce a reusable anchor to be referenced from `requires`.
 mandatory_jobs: &mandatory_jobs
   - general checks
-  - python 3_6 git 1_8_10
+  - python 3_6 git 1_8_0
   - python 3_7 git 2_7_6
   - python 3_8 git 2_25_0
-  - pep8 and python 3_9 git 2_30_1
-  - coverage and python 3_10 git 2_33_1
+  - python 3_9 git 2_30_1
+  - pep8 and python 3_10 git 2_33_1
   - docs and python 3_11 git 2_38_1
   - macos tests
   - windows tests
@@ -252,14 +286,17 @@ workflows:
   build:
     jobs:
       - general checks
-      - python 3_6 git 1_8_10
+      - python 3_6 git 1_8_0
       - python 3_7 git 2_7_6
       - python 3_8 git 2_25_0
-      - pep8 and python 3_9 git 2_30_1
-      - coverage and python 3_10 git 2_33_1
+      - python 3_9 git 2_30_1
+      - pep8 and python 3_10 git 2_33_1
       - docs and python 3_11 git 2_38_1
       - macos tests
       - windows tests
+      - coverage upload:
+          requires:
+            *mandatory_jobs
       - dry run deploy:
           filters:
             branches:

.gitignore

@@ -6,7 +6,7 @@
 AUTHORS
 ChangeLog
 build/
-.coverage
+.coverage*
 coverage.xml
 dist/
 docs/html/

ci/tox/.env-sample

@@ -4,6 +4,5 @@ PYTHON_VERSION=3.8
 
 # Vars for `docker-compose run`
 BUILD_SPHINX_HTML=false
-CHECK_COVERAGE=false
 CHECK_DOCS_UP_TO_DATE=false
 CHECK_PEP8=false

ci/tox/build-context/run.sh

@@ -11,11 +11,8 @@ env | sort | head -4
 
 set -x
 
-if [[ $CHECK_COVERAGE = true ]]; then
-  TOX_ENV_LIST="mypy-py${PYTHON_VERSION/./},coverage"
-else
-  TOX_ENV_LIST="mypy-py${PYTHON_VERSION/./},py${PYTHON_VERSION/./}"
-fi
+suffix=py${PYTHON_VERSION/./}
+TOX_ENV_LIST="mypy-$suffix,coverage-$suffix"
 
 if [[ $BUILD_SPHINX_HTML = true ]]; then
   TOX_ENV_LIST="$TOX_ENV_LIST,sphinx-html"

ci/tox/docker-compose.yml

@@ -10,7 +10,6 @@ services:
         - python_version=${PYTHON_VERSION:-0.0.0}
     environment:
       - BUILD_SPHINX_HTML=${BUILD_SPHINX_HTML:-false}
-      - CHECK_COVERAGE=${CHECK_COVERAGE:-false}
       - CHECK_DOCS_UP_TO_DATE=${CHECK_DOCS_UP_TO_DATE:-false}
       - CHECK_PEP8=${CHECK_PEP8:-false}
     volumes:

git_machete/client.py

@@ -2019,7 +2019,7 @@ def ask_if(
             return 'y'
         try:
             ans: str = input(fmt(msg) if apply_fmt else msg).lower()
-        except InterruptedError:  # pragma: no cover
+        except InterruptedError:
             sys.exit(1)
         return ans
 

git_machete/git_operations.py

@@ -365,9 +365,9 @@ def push(self, remote: str, branch: LocalBranchShortName, force_with_lease: bool
             opt_force = []
         elif self.get_git_version() >= (2, 30, 0):  # earliest version of git to support 'push --force-with-lease --force-if-includes'
             opt_force = ["--force-with-lease", "--force-if-includes"]
-        elif self.get_git_version() >= (1, 8, 5):  # pragma: no cover; earliest version of git to support 'push --force-with-lease'
+        elif self.get_git_version() >= (1, 8, 5):  # earliest version of git to support 'push --force-with-lease'
             opt_force = ["--force-with-lease"]
-        else:  # pragma: no cover
+        else:
             opt_force = ["--force"]
         args = [remote, branch]
         self._run_git("push", "--set-upstream", *(opt_force + args), flush_caches=True)
@@ -620,7 +620,7 @@ def get_reflog(self, branch: AnyBranchName) -> List[GitReflogEntry]:
                 self.__load_all_reflogs()
             assert self.__reflogs_cached is not None
             return self.__reflogs_cached.get(branch, [])
-        else:  # pragma: no cover
+        else:
             if self.__reflogs_cached is None:
                 self.__reflogs_cached = {}
             if branch not in self.__reflogs_cached:
@@ -847,7 +847,7 @@ def rebase(self, onto: AnyRevision, from_exclusive: AnyRevision, branch: LocalBr
         try:
             if not opt_no_interactive_rebase:
                 rebase_opts.append("--interactive")
-            if self.get_git_version() >= (2, 26, 0):  # pragma: no branch
+            if self.get_git_version() >= (2, 26, 0):
                 rebase_opts.append("--empty=drop")
             self._run_git("rebase", *rebase_opts, "--onto", onto, from_exclusive, branch, flush_caches=True)
         finally:

requirements.coverage-py36.txt

@@ -0,0 +1,5 @@
+# Deliberately using a non-standard file extension so that the GitHub dependabot doesn't try to update this file.
+# Version marks like `python_version=="3.6"` aren't supported by GitHub dependabot.
+coverage==6.2
+pytest==7.0.1
+pytest-mock==3.6.1

tox.ini

@@ -70,29 +70,53 @@ commands = vulture git_machete/ tests/
 [testenv:venv]
 commands = {posargs}
 
-[testenv:coverage]
+[testenv:coverage-py36]
 description = "Check the test coverage of the code"
 deps =
-  -r{toxinidir}/requirements.coverage.txt
+  -r{toxinidir}/requirements.coverage-py36.txt
+passenv = PYTHON_VERSION
+allowlist_externals = cp
 commands =
   coverage erase
-  # Generates .coverage binary file, used as the source of truth for the subsequent reports
+  # Generates .coverage.* binary file, used as the source of truth for the subsequent reports
   # Note that coverage (as of v7.2.3) doesn't support parallel running (`--numprocesses=...`) via python-xdist
-  coverage run --branch --omit='tests/*' -m pytest --junitxml=test-results/testenv-{envname}.xml {posargs}
+  coverage run --branch --omit='tests/*' \
+    -m pytest --junitxml=test-results/testenv-{envname}.xml {posargs}
   # Prints a report to console
   coverage report
   # Saves a report to htmlcov/
   coverage html
-  # Saves a report to coverage.xml (uploaded to Codecov from CI, as Codecov apparently does not accept .coverage binary files)
-  coverage xml
+  cp .coverage .coverage.{env:PYTHON_VERSION:bin}
+
+[testenv:coverage-{py37,py38,py39,py310,py311}]
+description = "Check the test coverage of the code"
+deps =
+  -r{toxinidir}/requirements.coverage.txt
+passenv = PYTHON_VERSION
+allowlist_externals = cp
+commands =
+  {[testenv:coverage-py36]commands}
+
+[coverage:run]
+relative_files = True
 
 [coverage:report]
 exclude_lines =
   except EOFError:
+  except InterruptedError:
   except KeyboardInterrupt:
   if __name__ == .__main__.:
   pragma: no cover
 
+[testenv:coverage-combine]
+description = "Combine coverage results"
+deps =
+  -r{toxinidir}/requirements.coverage.txt
+commands =
+  coverage combine
+  # Saves a report to coverage.xml (uploaded to Codecov from CI, as Codecov apparently does not accept .coverage binary files)
+  coverage xml
+
 # Python 3.11 is needed in envs that use Sphinx due to https://github.com/VirtusLab/git-machete/issues/936
 
 [testenv:sphinx-html]
@@ -149,4 +173,4 @@ commands =
 deps =
   -r{toxinidir}/requirements.mypy.txt
 commands =
-  mypy --config-file mypy.ini git_machete tests
+  {[testenv:mypy-py36]commands}