Fix manifest writing in python3

tools/manifest/item.py

@@ -205,7 +205,7 @@ def to_json(self):
         if self.jsshell:
             rv[-1]["jsshell"] = True
         if self.script_metadata:
-            rv[-1]["script_metadata"] = self.script_metadata
+            rv[-1]["script_metadata"] = [(k.decode('utf8'), v.decode('utf8')) for (k,v) in self.script_metadata]
         return rv
 
 

tools/manifest/manifest.py

@@ -391,7 +391,7 @@ def write(manifest, manifest_path):
     dir_name = os.path.dirname(manifest_path)
     if not os.path.exists(dir_name):
         os.makedirs(dir_name)
-    with open(manifest_path, "wb") as f:
+    with open(manifest_path, "w") as f:
         # Use ',' instead of the default ', ' separator to prevent trailing
         # spaces: https://docs.python.org/2/library/json.html#json.dump
         json.dump(manifest.to_json(caller_owns_obj=True), f,

tools/manifest/sourcefile.py

@@ -2,7 +2,7 @@
 import re
 import os
 from collections import deque
-from six import binary_type, PY3, iteritems
+from six import binary_type, iteritems, text_type
 from six.moves.urllib.parse import urljoin
 from fnmatch import fnmatch
 
@@ -308,11 +308,7 @@ def hash(self):
                 content = f.read()
 
             data = b"".join((b"blob ", b"%d" % len(content), b"\0", content))
-            hash_str = hashlib.sha1(data).hexdigest()  # type: str
-            if PY3:
-                self._hash = hash_str.encode("ascii")
-            else:
-                self._hash = hash_str
+            self._hash = text_type(hashlib.sha1(data).hexdigest())
 
         return self._hash
 

tools/manifest/tests/test_sourcefile.py

@@ -832,4 +832,4 @@ def test_reftest_fuzzy_multi(fuzzy, expected):
 
 def test_hash():
     s = SourceFile("/", "foo", "/", contents=b"Hello, World!")
-    assert b"b45ef6fec89518d314f546fd6c3025367b721684" == s.hash
+    assert "b45ef6fec89518d314f546fd6c3025367b721684" == s.hash

tools/manifest/typedata.py

@@ -256,14 +256,27 @@ def to_json(self):
         """
         json_rv = self._json_data.copy()
 
+        def safe_sorter(element):
+            # type: (Tuple[str,str]) -> Tuple[str,str]
+            """ key function to sort lists with None values.
+
+            Python3 is more strict typewise. Comparing None and str for example is valid
+            in python2 but throws an exception in python3.
+            """
+            if element and not element[0]:
+                return ("", element[1])
+            else:
+                return element
+
         stack = [(self._data, json_rv, tuple())]  # type: List[Tuple[Dict[Text, Any], Dict[Text, Any], Tuple[Text, ...]]]
         while stack:
             data_node, json_node, par_full_key = stack.pop()
             for k, v in iteritems(data_node):
                 full_key = par_full_key + (k,)
                 if isinstance(v, set):
                     assert k not in json_node
-                    json_node[k] = [self._hashes.get(full_key)] + [t for t in sorted(test.to_json() for test in v)]
+                    json_node[k] = [self._hashes.get(
+                        full_key)] + [t for t in sorted((test.to_json() for test in v), key=safe_sorter)]
                 else:
                     json_node[k] = json_node.get(k, {}).copy()
                     stack.append((v, json_node[k], full_key))

tools/manifest/vcs.py

@@ -63,7 +63,7 @@ def _local_changes(self):
         assert self.git is not None
         # note that git runs the command with tests_root as the cwd, which may
         # not be the root of the git repo (e.g., within a browser repo)
-        cmd = [b"diff-index", b"--relative", b"--no-renames", b"--name-only", b"-z", b"HEAD"]
+        cmd = ["diff-index", "--relative", "--no-renames", "--name-only", "-z", "HEAD"]
         data = self.git(*cmd)
         return set(data.split("\0"))
 

tools/wpt/requirements.txt

@@ -1 +1,2 @@
 requests==2.23.0
+mozinfo==1.2.1  # https://bugzilla.mozilla.org/show_bug.cgi?id=1621226

tools/wpt/tests/test_wpt.py

@@ -1,5 +1,6 @@
 import errno
 import os
+import platform
 import shutil
 import socket
 import subprocess
@@ -43,8 +44,9 @@ def get_persistent_manifest_path():
 
 @pytest.fixture(scope="module", autouse=True)
 def init_manifest():
-    if sys.version_info >= (3,):
-        pytest.xfail(reason="broken on Py3")
+    # See https://github.com/pypa/virtualenv/issues/1710
+    if sys.version_info[0] >= 3 and platform.system() == "Windows":
+        pytest.xfail(reason="virtualenv activation fails in Windows for python3")
     with pytest.raises(SystemExit) as excinfo:
         wpt.main(argv=["manifest", "--no-download",
                        "--path", get_persistent_manifest_path()])

tools/wptrunner/requirements.txt

@@ -1,5 +1,5 @@
 html5lib==1.0.1
-mozinfo==1.1.0
+mozinfo==1.2.1  # https://bugzilla.mozilla.org/show_bug.cgi?id=1621226
 mozlog==6.0
 mozdebug==0.2
 # Pillow 7 requires Python 3

tools/wptrunner/wptrunner/tests/test_products.py

@@ -1,5 +1,3 @@
-import sys
-
 from os.path import join, dirname
 
 import mock
@@ -20,11 +18,7 @@ def test_load_active_product(product):
     # test passes if it doesn't throw
 
 
-@all_products("product", marks={
-    "firefox": pytest.mark.xfail(sys.platform.startswith('linux') and
-                                 sys.version_info >= (3, 8),
-                                 reason="mozinfo doesn't support Linux 3.8+")
-})
+@all_products("product")
 def test_load_all_products(product):
     """test every product either loads or throws ImportError"""
     try: