chore(py_library): add split_system_tests (#712)

synthtool/gcp/common.py

@@ -15,10 +15,13 @@
 import json
 import os
 import re
+import shutil
 import yaml
 from pathlib import Path
 from typing import Dict, List, Optional
 
+import jinja2
+
 from synthtool import _tracked_paths
 from synthtool.languages import node
 from synthtool.log import logger
@@ -105,7 +108,27 @@ def py_library(self, **kwargs) -> Path:
         if "samples" not in kwargs:
             self.excludes += ["samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md"]
 
-        return self._generic_library("python_library", **kwargs)
+        ret = self._generic_library("python_library", **kwargs)
+
+        # If split_system_tests is set to True, we disable the system
+        # test in the main presubmit build and create individual build
+        # configs for each python versions.
+        if kwargs.get("split_system_tests", False):
+            template_root = self._template_root / "py_library_split_systests"
+            # copy the main presubmit config
+            shutil.copy2(
+                template_root / ".kokoro/presubmit/presubmit.cfg",
+                ret / ".kokoro/presubmit/presubmit.cfg",
+            )
+            env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_root)))
+            tmpl = env.get_template(".kokoro/presubmit/system.cfg")
+            for v in kwargs["system_test_python_versions"]:
+                nox_session = f"system-{v}"
+                dest = ret / f".kokoro/presubmit/system-{v}.cfg"
+                content = tmpl.render(nox_session=nox_session)
+                with open(dest, "w") as f:
+                    f.write(content)
+        return ret
 
     def java_library(self, **kwargs) -> Path:
         # kwargs["metadata"] is required to load values from .repo-metadata.json

synthtool/gcp/templates/py_library_split_systests/.kokoro/presubmit/presubmit.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Disable system tests.
+env_vars: {
+    key: "RUN_SYSTEM_TESTS"
+    value: "false"
+}

synthtool/gcp/templates/py_library_split_systests/.kokoro/presubmit/system.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "{{ nox_session }}"
+}

synthtool/gcp/templates/python_library/.kokoro/build.sh

@@ -36,4 +36,10 @@ python3.6 -m pip uninstall --yes --quiet nox-automation
 python3.6 -m pip install --upgrade --quiet nox
 python3.6 -m nox --version
 
-python3.6 -m nox
+# If NOX_SESSION is set, it only runs the specified session,
+# otherwise run all the sessions.
+if [[ -n "${NOX_SESSION:-}" ]]; then
+    python3.6 -m nox -s "${NOX_SESSION:-}"
+else
+    python3.6 -m nox
+fi

synthtool/gcp/templates/python_library/noxfile.py.j2

@@ -26,7 +26,7 @@ import nox
 BLACK_VERSION = "black==19.10b0"
 BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"]
 
-DEFAULT_PYTHON_VERSION="{{ default_python_version }}" 
+DEFAULT_PYTHON_VERSION="{{ default_python_version }}"
 SYSTEM_TEST_PYTHON_VERSIONS=[{% for v in system_test_python_versions %}"{{v}}"{% if not loop.last %},{% endif %}{% endfor %}]
 UNIT_TEST_PYTHON_VERSIONS=[{% for v in unit_test_python_versions %}"{{v}}"{% if not loop.last %},{% endif %}{% endfor %}]
 
@@ -106,6 +106,10 @@ def system(session):
     """Run the system test suite."""
     system_test_path = os.path.join("tests", "system.py")
     system_test_folder_path = os.path.join("tests", "system")
+
+    # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
+    if os.environ.get("RUN_SYSTEM_TESTS", "true") == 'false':
+        session.skip("RUN_SYSTEM_TESTS is set to false, skipping")
     # Sanity check: Only run tests if the environment variable is set.
     if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
         session.skip("Credentials must be set via environment variable")
@@ -122,7 +126,7 @@ def system(session):
     # Install all test dependencies, then install this package into the
     # virtualenv's dist-packages.
     session.install("mock", "pytest", "google-cloud-testutils", {% for d in system_test_external_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %})
-    
+
     {%- if system_test_local_dependencies %}
     session.install("-e", {% for d in system_test_local_dependencies %}"{{d}}"{% if not loop.last %},{% endif %}{% endfor %})
     {%- endif %}

tests/test_python_library.py

@@ -26,3 +26,20 @@ def test_python_library():
 
     assert os.path.exists(templated_files / ".kokoro/docs/docs-presubmit.cfg")
     assert os.path.exists(templated_files / ".kokoro/docker/docs/fetch_gpg_keys.sh")
+
+
+def test_split_system_tests():
+    os.chdir(Path(__file__).parent / "fixtures/python_library")
+    template_dir = Path(__file__).parent.parent / "synthtool/gcp/templates"
+    common = gcp.CommonTemplates(template_path=template_dir)
+    templated_files = common.py_library(split_system_tests=True)
+
+    with open(templated_files / ".kokoro/presubmit/presubmit.cfg", "r") as f:
+        contents = f.read()
+        assert "RUN_SYSTEM_TESTS" in contents
+        assert "false" in contents
+
+    assert os.path.exists(templated_files / ".kokoro/presubmit/system-3.8.cfg")
+    with open(templated_files / ".kokoro/presubmit/system-3.8.cfg", "r") as f:
+        contents = f.read()
+        assert "system-3.8" in contents