Merge pull request #65 from w3c/py3

Test on python 3.

.travis.yml

@@ -1,4 +1,7 @@
 language: python
+python:
+  - "2.7"
+  - "3.5"
 install:
   - pip install pytest
   - pip install ./html5lib

manifest/__init__.py

@@ -1,5 +1,5 @@
-import item
-import manifest
-import sourcefile
-import tree
-import update
+from . import item
+from . import manifest
+from . import sourcefile
+from . import tree
+from . import update

manifest/item.py

@@ -1,15 +1,15 @@
 import os
-import urlparse
+from six.moves.urllib.parse import urljoin
 from abc import ABCMeta, abstractmethod, abstractproperty
 
-from utils import from_os_path, to_os_path
+from .utils import from_os_path, to_os_path
 
 item_types = ["testharness", "reftest", "manual", "stub", "wdspec"]
 
 
 def get_source_file(source_files, tests_root, manifest, path):
     def make_new():
-        from sourcefile import SourceFile
+        from .sourcefile import SourceFile
 
         return SourceFile(tests_root, path, manifest.url_base)
 
@@ -85,7 +85,7 @@ def id(self):
 
     @property
     def url(self):
-        return urlparse.urljoin(self.url_base, self._url)
+        return urljoin(self.url_base, self._url)
 
     def to_json(self):
         rv = ManifestItem.to_json(self)
@@ -137,7 +137,7 @@ def __init__(self, source_file, url, references, url_base="/", timeout=None,
         URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest)
         for _, ref_type in references:
             if ref_type not in ["==", "!="]:
-                raise ValueError, "Unrecognised ref_type %s" % ref_type
+                raise ValueError("Unrecognised ref_type %s" % ref_type)
         self.references = tuple(references)
         self.timeout = timeout
         self.viewport_size = viewport_size

manifest/manifest.py

@@ -1,11 +1,12 @@
 import json
 import os
 from collections import defaultdict, OrderedDict
+from six import iteritems
 
-from item import item_types, ManualTest, WebdriverSpecTest, Stub, RefTest, TestharnessTest
-from log import get_logger
-from sourcefile import SourceFile
-from utils import from_os_path, to_os_path
+from .item import item_types, ManualTest, WebdriverSpecTest, Stub, RefTest, TestharnessTest
+from .log import get_logger
+from .sourcefile import SourceFile
+from .utils import from_os_path, to_os_path
 
 
 CURRENT_VERSION = 3
@@ -37,7 +38,7 @@ def _included_items(self, include_types=None):
         for item_type in include_types:
             paths = self._data[item_type].copy()
             for local_types, local_paths in self.local_changes.itertypes(item_type):
-                for path, items in local_paths.iteritems():
+                for path, items in iteritems(local_paths):
                     paths[path] = items
                 for path in self.local_changes.iterdeleted():
                     if path in paths:
@@ -82,7 +83,7 @@ def itertypes(self, *types):
         if not types:
             types = None
         for item_type, items in self._included_items(types):
-            for item in sorted(items.items()):
+            for item in sorted(iteritems(items)):
                 yield item
 
     def __iter__(self):
@@ -146,7 +147,7 @@ def update(self,
         self.local_changes = LocalChanges(self)
 
         local_paths = set()
-        for rel_path, status in local_changes.iteritems():
+        for rel_path, status in iteritems(local_changes):
             local_paths.add(rel_path)
 
             if status == "modified":
@@ -206,12 +207,12 @@ def compute_reftests(self, reftest_nodes):
         reftests = set()
 
         has_inbound = set()
-        for path, items in reftest_nodes.iteritems():
+        for path, items in iteritems(reftest_nodes):
             for item in items:
                 for ref_url, ref_type in item.references:
                     has_inbound.add(ref_url)
 
-        for path, items in reftest_nodes.iteritems():
+        for path, items in iteritems(reftest_nodes):
             for item in items:
                 if item.url in has_inbound:
                     continue
@@ -223,14 +224,14 @@ def to_json(self):
         out_items = {
             item_type: sorted(
                 test.to_json()
-                for _, tests in items.iteritems()
+                for _, tests in iteritems(items)
                 for test in tests
             )
-            for item_type, items in self._data.iteritems()
+            for item_type, items in iteritems(self._data)
         }
 
         reftest_nodes = OrderedDict()
-        for key, value in sorted(self.reftest_nodes.items()):
+        for key, value in sorted(iteritems(self.reftest_nodes)):
             reftest_nodes[from_os_path(key)] = [v.to_json() for v in value]
 
         rv = {"url_base": self.url_base,
@@ -249,7 +250,7 @@ def from_json(cls, tests_root, obj):
 
         self = cls(git_rev=obj["rev"],
                    url_base=obj.get("url_base", "/"))
-        if not hasattr(obj, "iteritems"):
+        if not hasattr(obj, "items"):
             raise ManifestError
 
         item_classes = {"testharness": TestharnessTest,
@@ -260,15 +261,15 @@ def from_json(cls, tests_root, obj):
 
         source_files = {}
 
-        for k, values in obj["items"].iteritems():
+        for k, values in iteritems(obj["items"]):
             if k not in item_types:
                 raise ManifestError
             for v in values:
                 manifest_item = item_classes[k].from_json(self, tests_root, v,
                                                           source_files=source_files)
                 self._add(manifest_item)
 
-        for path, values in obj["reftest_nodes"].iteritems():
+        for path, values in iteritems(obj["reftest_nodes"]):
             path = to_os_path(path)
             for v in values:
                 item = RefTest.from_json(self, tests_root, v,
@@ -329,26 +330,26 @@ def iterdeleted(self):
             yield item
 
     def iterdeletedreftests(self):
-        for item in self._deleted_reftests.iteritems():
+        for item in iteritems(self._deleted_reftests):
             yield item
 
     def __getitem__(self, item_type):
         return self._data[item_type]
 
     def to_json(self):
         reftest_nodes = {from_os_path(key): [v.to_json() for v in value]
-                         for key, value in self.reftest_nodes.iteritems()}
+                         for key, value in iteritems(self.reftest_nodes)}
 
         deleted_reftests = {from_os_path(key): [v.to_json() for v in value]
-                            for key, value in self._deleted_reftests.iteritems()}
+                            for key, value in iteritems(self._deleted_reftests)}
 
         rv = {"items": defaultdict(dict),
               "reftest_nodes": reftest_nodes,
               "deleted": [from_os_path(path) for path in self._deleted],
               "deleted_reftests": deleted_reftests}
 
-        for test_type, paths in self._data.iteritems():
-            for path, tests in paths.iteritems():
+        for test_type, paths in iteritems(self._data):
+            for path, tests in iteritems(paths):
                 path = from_os_path(path)
                 rv["items"][test_type][path] = [test.to_json() for test in tests]
 
@@ -357,7 +358,7 @@ def to_json(self):
     @classmethod
     def from_json(cls, manifest, tests_root, obj, source_files=None):
         self = cls(manifest)
-        if not hasattr(obj, "iteritems"):
+        if not hasattr(obj, "items"):
             raise ManifestError
 
         item_classes = {"testharness": TestharnessTest,
@@ -366,16 +367,16 @@ def from_json(cls, manifest, tests_root, obj, source_files=None):
                         "stub": Stub,
                         "wdspec": WebdriverSpecTest}
 
-        for test_type, paths in obj["items"].iteritems():
-            for path, tests in paths.iteritems():
+        for test_type, paths in iteritems(obj["items"]):
+            for path, tests in iteritems(paths):
                 for test in tests:
                     manifest_item = item_classes[test_type].from_json(manifest,
                                                                       tests_root,
                                                                       test,
                                                                       source_files=source_files)
                     self.add(manifest_item)
 
-        for path, values in obj["reftest_nodes"].iteritems():
+        for path, values in iteritems(obj["reftest_nodes"]):
             path = to_os_path(path)
             for v in values:
                 item = RefTest.from_json(self.manifest, tests_root, v,
@@ -386,7 +387,7 @@ def from_json(cls, manifest, tests_root, obj, source_files=None):
         for item in obj["deleted"]:
             self.add_deleted(to_os_path(item))
 
-        for path, values in obj.get("deleted_reftests", {}).iteritems():
+        for path, values in iteritems(obj.get("deleted_reftests", {})):
             path = to_os_path(path)
             for v in values:
                 item = RefTest.from_json(self.manifest, tests_root, v,

manifest/sourcefile.py

@@ -1,5 +1,5 @@
 import os
-import urlparse
+from six.moves.urllib.parse import urljoin
 from fnmatch import fnmatch
 try:
     from xml.etree import cElementTree as ElementTree
@@ -8,9 +8,9 @@
 
 import html5lib
 
-import vcs
-from item import Stub, ManualTest, WebdriverSpecTest, RefTest, TestharnessTest
-from utils import rel_path_to_url, is_blacklisted, ContextManagerStringIO, cached_property
+from . import vcs
+from .item import Stub, ManualTest, WebdriverSpecTest, RefTest, TestharnessTest
+from .utils import rel_path_to_url, is_blacklisted, ContextManagerStringIO, cached_property
 
 wd_pattern = "*.py"
 
@@ -74,7 +74,7 @@ def open(self):
             blob = git("show", "HEAD:%s" % self.rel_path)
             file_obj = ContextManagerStringIO(blob)
         else:
-            file_obj = open(self.path)
+            file_obj = open(self.path, 'rb')
         return file_obj
 
     @property
@@ -267,7 +267,7 @@ def references(self):
         rel_map = {"match": "==", "mismatch": "!="}
         for item in self.reftest_nodes:
             if "href" in item.attrib:
-                ref_url = urlparse.urljoin(self.url, item.attrib["href"])
+                ref_url = urljoin(self.url, item.attrib["href"])
                 ref_type = rel_map[item.attrib["rel"]]
                 rv.append((ref_url, ref_type))
         return rv

manifest/tree.py

@@ -1,10 +1,10 @@
 import os
-from cStringIO import StringIO
+from six.moves import cStringIO as StringIO
 from fnmatch import fnmatch
 
-import vcs
-from log import get_logger
-from utils import is_blacklisted, rel_path_to_url
+from . import vcs
+from .log import get_logger
+from .utils import is_blacklisted, rel_path_to_url
 
 def chunks(data, n):
     for i in range(0, len(data) - 1, n):

manifest/update.py

@@ -5,9 +5,9 @@
 import sys
 
 import manifest
-import vcs
-from log import get_logger
-from tree import GitTree, NoVCSTree
+from . import vcs
+from .log import get_logger
+from .tree import GitTree, NoVCSTree
 
 here = os.path.dirname(__file__)
 localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, "localpaths.py")))

manifest/utils.py

@@ -1,6 +1,5 @@
 import os
-import urlparse
-from StringIO import StringIO
+from six import StringIO
 
 blacklist = ["/", "/tools/", "/resources/", "/common/", "/conformance-checkers/", "_certs"]