Reduce flakiness of CI test runs (#4653)

* test: Remove another PY2 bit; So long, pal...

* test: use api_default_testing_config fixture

* test: monkeypatch get_or_merge_config

* test: Use msumastro=1.1.6 for PyPI skeleton test

msumastro has been updated a couple of days ago and versions >=1.2 now
store the metadata in setup.cfg which is not handled by skeletons.pypi.
It is easy to fix this if we reuse the code from _load_setup_py_data but
we might as well use/advertize newer projects like grayskull instead.

setup.cfg

@@ -34,6 +34,7 @@ markers =
     serial: execute test serially (to avoid race conditions)
     slow: execute the slow tests if active
     sanity: execute the sanity tests
+    no_default_testing_config: used internally to disable monkeypatching for testing_config
 
 [versioneer]
 VCS = git

tests/conftest.py

@@ -5,8 +5,11 @@
 from collections import defaultdict
 
 import pytest
+
+import conda_build.config
 from conda_build.config import (
     Config,
+    get_or_merge_config,
     _src_cache_root_default,
     conda_pkg_format_default,
     enable_static_default,
@@ -90,7 +93,7 @@ def return_to_saved_path():
 @pytest.fixture(scope="function")
 def testing_config(testing_workdir):
     def boolify(v):
-        return True if "v" == "true" else False
+        return v == "true"
 
     result = Config(
         croot=testing_workdir,
@@ -120,6 +123,27 @@ def boolify(v):
     return result
 
 
+@pytest.fixture(scope="function", autouse=True)
+def default_testing_config(testing_config, monkeypatch, request):
+    """Monkeypatch get_or_merge_config to use testing_config by default
+
+    This requests fixture testing_config, thus implicitly testing_workdir, too.
+    """
+
+    # Allow single tests to disable this fixture even if outer scope adds it.
+    if "no_default_testing_config" in request.keywords:
+        return
+
+    def get_or_merge_testing_config(config, variant=None, **kwargs):
+        return get_or_merge_config(config or testing_config, variant, **kwargs)
+
+    monkeypatch.setattr(
+        conda_build.config,
+        "get_or_merge_config",
+        get_or_merge_testing_config,
+    )
+
+
 @pytest.fixture(scope="function")
 def testing_metadata(request, testing_config):
     d = defaultdict(dict)

tests/test_api_build.py

@@ -46,26 +46,6 @@
 
 from .utils import is_valid_dir, metadata_dir, fail_dir, add_mangling
 
-# For Python 2, backport backslashreplace error handler for decodes.
-import codecs
-codecs.register_error(
-    'backslashreplace_',
-    lambda ex: (
-        ''.join(
-            (
-                '\\x{:x}'
-                if c < 0x100
-                else '\\u{:04x}'
-                if c < 0x10000
-                else '\\U{:08x}'
-            ).format(c)
-            for c in map(ord, ex.object[ex.start : ex.end])
-        ),
-        ex.end,
-    ),
-)
-
-
 # define a few commonly used recipes - use os.path.join(metadata_dir, recipe) elsewhere
 empty_sections = os.path.join(metadata_dir, "empty_sections")
 
@@ -1283,7 +1263,7 @@ def test_failed_recipe_leaves_folders(testing_config, testing_workdir):
             any_locks = True
             dest_path = base64.b64decode(os.path.basename(lock.lock_file))
             if hasattr(dest_path, 'decode'):
-                dest_path = dest_path.decode(sys.getfilesystemencoding(), errors='backslashreplace_')
+                dest_path = dest_path.decode(sys.getfilesystemencoding(), errors='backslashreplace')
             locks_list.add((lock.lock_file, dest_path))
     assert not any_locks, "remaining locks:\n{}".format('\n'.join('->'.join((l, r))
                                                                 for (l, r) in locks_list))

tests/test_api_consistency.py

@@ -5,10 +5,14 @@
 
 import sys
 
+import pytest
+
 from conda_build import api
 
 from inspect import getfullargspec as getargspec
 
+pytestmark = pytest.mark.no_default_testing_config
+
 
 def test_api_config():
     assert hasattr(api, 'Config')

tests/test_api_skeleton.py

@@ -443,7 +443,7 @@ def test_pypi_section_order_preserved(testing_workdir):
 
 
 @pytest.mark.slow
-@pytest.mark.flaky(max_runs=5)
+@pytest.mark.flaky(rerun=5, reruns_delay=2)
 @pytest.mark.skipif(not external.find_executable("shellcheck"), reason="requires shellcheck >=0.7.0")
 @pytest.mark.parametrize(
     "package, repo", [("r-rmarkdown", "cran"), ("Perl::Lint", "cpan"), ("screen", "rpm")]

tests/test_api_skeleton_cran.py

@@ -42,7 +42,7 @@
 
 @pytest.mark.slow
 @pytest.mark.parametrize("package, license_id, license_family, license_files", cran_packages)
-@pytest.mark.flaky(max_runs=5)
+@pytest.mark.flaky(rerun=5, reruns_delay=2)
 def test_cran_license(package, license_id, license_family, license_files, testing_workdir, testing_config):
     api.skeletonize(packages=package, repo='cran', output_dir=testing_workdir,
                     config=testing_config)
@@ -66,6 +66,7 @@ def test_cran_license(package, license_id, license_family, license_files, testin
 
 
 @pytest.mark.parametrize("package, skip_text", cran_os_type_pkgs)
+@pytest.mark.flaky(rerun=5, reruns_delay=2)
 def test_cran_os_type(package, skip_text, testing_workdir, testing_config):
     api.skeletonize(packages=package, repo='cran', output_dir=testing_workdir,
                     config=testing_config)
@@ -75,6 +76,7 @@ def test_cran_os_type(package, skip_text, testing_workdir, testing_config):
 
 
 # Test cran skeleton argument --no-comments
+@pytest.mark.flaky(rerun=5, reruns_delay=2)
 def test_cran_no_comments(testing_workdir, testing_config):
     package = "data.table"
     meta_yaml_comment = '  # This is required to make R link correctly on Linux.'

tests/test_cli.py

@@ -291,7 +291,7 @@ def test_skeleton_pypi_arguments_work(testing_workdir):
 
     https://github.com/conda/conda-build/pull/1384
     """
-    args = ['pypi', 'msumastro', '--pin-numpy']
+    args = ['pypi', 'msumastro', '--version=1.1.6', '--pin-numpy']
     main_skeleton.execute(args)
     assert os.path.isdir('msumastro')