Compartmentalize README doc/shell tests (#261)

* Compartmentalize README doc/shell tests

They're now no longer run as part of the core test suite; only
as part of the Azure Pipelines auxiliary CI.

This will help ease the pain of #260, and also will minimize
the thrashing of the commit history during general development...
README glitches will only be surfaced on PR merge and releases.

Closes #260.

* Add --readme to 'check all tests ran' step

* Add pragma for case of failed CLI load attempt

azure-pipelines.yml

@@ -126,6 +126,26 @@ stages:
     - script: cd doc; make doctest
       displayName: Run doctests
 
+  - job: readme
+    displayName: Run README doctests/shell tests
+
+    pool:
+      vmImage: 'Ubuntu-latest'
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.10'
+
+    - script: pip install -r requirements-ci.txt
+      displayName: Install CI requirements
+
+    - script: cd doc; make html
+      displayName: Build docs
+
+    - script: pytest -k readme --readme --doctest-glob="README.rst"
+      displayName: Run README doc/shell tests
+
   - job: linkcheck
     displayName: Run docs link-check suite
 
@@ -161,7 +181,7 @@ stages:
     - script: cd doc; make html; mkdir scratch
       displayName: Build docset
 
-    - script: pytest --cov=. --nonloc --flake8_ext
+    - script: pytest --cov=. --nonloc --flake8_ext --readme
       displayName: Run pytest with coverage on the entire project tree
 
     - script: coverage report --include="tests/*" --fail-under=100

conftest.py

@@ -63,6 +63,7 @@ def pytest_addoption(parser):
     parser.addoption(
         "--flake8_ext", action="store_true", help="Include flake8 extensions test"
     )
+    parser.addoption("--readme", action="store_true", help="Include README shell tests")
 
 
 @pytest.fixture(scope="session")

src/sphobjinv/cli/load.py

@@ -170,7 +170,7 @@ def attempt_inv_load(url, params):
             print_stderr(f"  ... HTTP error: {e.code} {e.reason}.", params)
         except URLError:  # pragma: no cover
             print_stderr("  ... error attempting to retrieve URL.", params)
-        except VersionError:
+        except VersionError:  # pragma: no cover
             print_stderr("  ... no recognized inventory.", params)
         except ValueError:
             print_stderr(

tests/test_readme.py

@@ -58,6 +58,20 @@
 )
 
 
+pytestmark = [pytest.mark.readme]
+
+
+@pytest.fixture(scope="module", autouse=True)
+def skip_if_no_readme_option(pytestconfig):
+    """Skip test if --readme not provided.
+
+    Auto-applied to all functions in module, since module is dedicated to README.
+
+    """
+    if not pytestconfig.getoption("--readme"):
+        pytest.skip("'--readme' not specified")  # pragma: no cover
+
+
 @pytest.mark.skipif(
     sphinx_ver != sphinx_req,
     reason="Skip if Sphinx version mismatches current dev version.",

tox.ini

@@ -124,7 +124,7 @@ markers =
   first: Inherited marker from `pytest-ordering`
   timeout: Inherited marker from `pytest-timeout`
 
-addopts = --strict-markers --doctest-glob="README.rst" -rsxX -Werror
+addopts = --strict-markers -rsxX -Werror
 
 norecursedirs = .* env* src *.egg dist build