Bug 1442092 [wpt PR 9726] - Test serve start config, a=testonly

Automatic update from web-platform-testsRewrite the code to parametrise over products

--
Add test to check that serve.start gets called with right host

--
Test all wptrunner environments

--
Fix #9750: bind_hostname needs to be a Python bool

db55c167d484d0e34fd618960887161522280d47 fixed this for Servo;
"true"/"false" worked when it was interpolated as JSON, but that is no
longer the case since cb2d75f (#9480).

--
Use default value of bind_hostname unless we need to override it

wpt-commits: e21bdf0b8778ac6c3467de97b09d0e88591e7ee8, 7787a28414acd7e8e75af2e7e05cbaec28f64853, 35261cc5571f5ad2c34c096eb48433497f1db82d, 8fd41644484ab9064017ee773dc96e3a881a7448, 88f08cce716ce4b97e3da5260144ce9d8a81cc33
wpt-pr: 9726
wpt-commits: e21bdf0b8778ac6c3467de97b09d0e88591e7ee8, 7787a28414acd7e8e75af2e7e05cbaec28f64853, 35261cc5571f5ad2c34c096eb48433497f1db82d, 8fd41644484ab9064017ee773dc96e3a881a7448, 88f08cce716ce4b97e3da5260144ce9d8a81cc33
wpt-pr: 9726

testing/web-platform/tests/tools/wptrunner/tox.ini

@@ -2,7 +2,7 @@
 xfail_strict=true
 
 [tox]
-envlist = {py27,pypy}-{base,chrome,firefox,sauce,servo},py27-flake8
+envlist = {py27,pypy}-{base,chrome,edge,firefox,ie,opera,safari,sauce,servo},py27-flake8
 
 [testenv]
 deps =
@@ -12,7 +12,11 @@ deps =
      mock
      -r{toxinidir}/requirements.txt
      chrome: -r{toxinidir}/requirements_chrome.txt
+     edge: -r{toxinidir}/requirements_edge.txt
      firefox: -r{toxinidir}/requirements_firefox.txt
+     ie: -r{toxinidir}/requirements_ie.txt
+     opera: -r{toxinidir}/requirements_opera.txt
+     safari: -r{toxinidir}/requirements_safari.txt
      sauce: -r{toxinidir}/requirements_sauce.txt
      servo: -r{toxinidir}/requirements_servo.txt
 

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py

@@ -60,7 +60,7 @@ def env_extras(**kwargs):
 
 
 def env_options():
-    return {"bind_hostname": "true"}
+    return {}
 
 
 class ChromeBrowser(Browser):

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome_android.py

@@ -68,7 +68,7 @@ def env_extras(**kwargs):
 
 
 def env_options():
-    return {"bind_hostname": "true"}
+    return {}
 
 
 class ChromeAndroidBrowser(Browser):

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py

@@ -38,8 +38,7 @@ def env_extras(**kwargs):
     return []
 
 def env_options():
-    return {"bind_hostname": "true",
-            "supports_debugger": False}
+    return {"supports_debugger": False}
 
 class EdgeBrowser(Browser):
     used_ports = set()

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py

@@ -124,7 +124,7 @@ def env_options():
     # https://github.com/w3c/web-platform-tests/pull/9480
     return {"host_ip": "127.0.0.1",
             "host": "web-platform.test",
-            "bind_hostname": "false",
+            "bind_hostname": False,
             "certificate_domain": "web-platform.test",
             "supports_debugger": True}
 

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/ie.py

@@ -42,8 +42,7 @@ def env_extras(**kwargs):
     return []
 
 def env_options():
-    return {"bind_hostname": "true",
-            "supports_debugger": False}
+    return {"supports_debugger": False}
 
 class InternetExplorerBrowser(Browser):
     used_ports = set()

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/opera.py

@@ -60,7 +60,7 @@ def env_extras(**kwargs):
 
 
 def env_options():
-    return {"bind_hostname": "true"}
+    return {}
 
 
 class OperaBrowser(Browser):

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/safari.py

@@ -44,7 +44,7 @@ def env_extras(**kwargs):
 
 
 def env_options():
-    return {"bind_hostname": "true"}
+    return {}
 
 
 class SafariBrowser(Browser):

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py

@@ -113,8 +113,7 @@ def env_extras(**kwargs):
 
 
 def env_options():
-    return {"bind_hostname": "true",
-            "supports_debugger": False}
+    return {"supports_debugger": False}
 
 
 def get_tar(url, dest):

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py

@@ -55,7 +55,6 @@ def env_extras(**kwargs):
 def env_options():
     return {"host_ip": "127.0.0.1",
             "host": "web-platform.test",
-            "bind_hostname": "true",
             "testharnessreport": "testharnessreport-servodriver.js",
             "supports_debugger": True}
 

testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py

@@ -144,13 +144,15 @@ def load_config(self):
                 "ws": [8888]
             },
             "check_subdomains": False,
-            "bind_hostname": self.options["bind_hostname"],
             "ssl": {}
         }
 
         if "host" in self.options:
             local_config["host"] = self.options["host"]
 
+        if "bind_hostname" in self.options:
+            local_config["bind_hostname"] = self.options["bind_hostname"]
+
         with open(default_config_path) as f:
             default_config = json.load(f)
 

testing/web-platform/tests/tools/wptrunner/wptrunner/tests/base.py

@@ -0,0 +1,60 @@
+import os
+import sys
+
+from os.path import dirname, join
+
+import pytest
+
+sys.path.insert(0, join(dirname(__file__), "..", ".."))
+
+from wptrunner import browsers
+
+
+_products = browsers.product_list
+_active_products = set()
+
+if "CURRENT_TOX_ENV" in os.environ:
+    current_tox_env_split = os.environ["CURRENT_TOX_ENV"].split("-")
+
+    tox_env_extra_browsers = {
+        "chrome": {"chrome_android"},
+        "servo": {"servodriver"},
+    }
+
+    _active_products = set(_products) & set(current_tox_env_split)
+    for product in frozenset(_active_products):
+        _active_products |= tox_env_extra_browsers.get(product, set())
+else:
+    _active_products = set(_products)
+
+
+class all_products(object):
+    def __init__(self, arg, marks={}):
+        self.arg = arg
+        self.marks = marks
+
+    def __call__(self, f):
+        params = []
+        for product in _products:
+            if product in self.marks:
+                params.append(pytest.param(product, marks=self.marks[product]))
+            else:
+                params.append(product)
+        return pytest.mark.parametrize(self.arg, params)(f)
+
+
+class active_products(object):
+    def __init__(self, arg, marks={}):
+        self.arg = arg
+        self.marks = marks
+
+    def __call__(self, f):
+        params = []
+        for product in _products:
+            if product not in _active_products:
+                params.append(pytest.param(product, marks=pytest.mark.skip(reason="wrong toxenv")))
+            elif product in self.marks:
+                params.append(pytest.param(product, marks=self.marks[product]))
+            else:
+                params.append(product)
+        return pytest.mark.parametrize(self.arg, params)(f)

testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py

@@ -3,22 +3,61 @@
 
 from os.path import join, dirname
 
+import mock
 import pytest
 
-sys.path.insert(0, join(dirname(__file__), "..", ".."))
+from .base import all_products, active_products
 
+sys.path.insert(0, join(dirname(__file__), "..", "..", "..", ".."))  # repo root
+
+from tools import localpaths
+
+import sslutils
+
+from wptrunner import environment
 from wptrunner import products
 
+test_paths = {"/": {"tests_path": join(dirname(__file__), "..", "..", "..", "..")}}  # repo root
+environment.do_delayed_imports(None, test_paths)
 
+
+@active_products("product")
 def test_load_active_product(product):
     """test we can successfully load the product of the current testenv"""
     products.load_product({}, product)
     # test passes if it doesn't throw
 
 
-def test_load_all_products(all_product):
+@all_products("product")
+def test_load_all_products(product):
     """test every product either loads or throws ImportError"""
     try:
-        products.load_product({}, all_product)
+        products.load_product({}, product)
     except ImportError:
         pass
+
+
+@active_products("product", marks={
+    "sauce": pytest.mark.skip("needs env extras kwargs"),
+})
+def test_server_start_config(product):
+    (check_args,
+     target_browser_cls, get_browser_kwargs,
+     executor_classes, get_executor_kwargs,
+     env_options, get_env_extras, run_info_extras) = products.load_product({}, product)
+
+    env_extras = get_env_extras()
+
+    with mock.patch.object(environment.serve, "start") as start:
+        with environment.TestEnvironment(test_paths,
+                                         sslutils.environments["none"](None),
+                                         False,
+                                         None,
+                                         env_options,
+                                         env_extras) as test_environment:
+            start.assert_called_once()
+            args = start.call_args
+            config = args[0][0]
+            if "host" in env_options:
+                assert config["host"] == env_options["host"]
+            assert isinstance(config["bind_hostname"], bool)