Bug 1616075 [wpt PR 21838] - Improve python3 support of test_wpt, a=t…

…estonly

Automatic update from web-platform-tests
Improve python3 support of test_wpt (#21838)

Main change is that git commands return Text now instead of bytes. Updated callsites and also mypy comments.
--

wpt-commits: b4023970550ce0d454e700fad25e957ad1366ceb
wpt-pr: 21838

testing/web-platform/tests/tools/manifest/download.py

@@ -26,6 +26,7 @@
     from typing import Callable
     from typing import List
     from typing import Optional
+    from typing import Text
 
 here = os.path.dirname(__file__)
 
@@ -50,9 +51,9 @@ def should_download(manifest_path, rebuild_time=timedelta(days=5)):
 
 
 def merge_pr_tags(repo_root, max_count=50):
-    # type: (str, int) -> List[str]
+    # type: (str, int) -> List[Text]
     gitfunc = git(repo_root)
-    tags = []  # type: List[str]
+    tags = []  # type: List[Text]
     if gitfunc is None:
         return tags
     for line in gitfunc("log", "--format=%D", "--max-count=%s" % max_count).split("\n"):
@@ -79,7 +80,7 @@ def score_name(name):
 
 
 def github_url(tags):
-    # type: (List[str]) -> Optional[List[str]]
+    # type: (List[Text]) -> Optional[List[Text]]
     for tag in tags:
         url = "https://api.github.com/repos/web-platform-tests/wpt/releases/tags/%s" % tag
         try:
@@ -111,8 +112,8 @@ def github_url(tags):
 
 def download_manifest(
         manifest_path,  # type: str
-        tags_func,  # type: Callable[[], List[str]]
-        url_func,  # type: Callable[[List[str]], Optional[List[str]]]
+        tags_func,  # type: Callable[[], List[Text]]
+        url_func,  # type: Callable[[List[Text]], Optional[List[Text]]]
         force=False  # type: bool
 ):
     # type: (...) -> bool

testing/web-platform/tests/tools/manifest/sourcefile.py

@@ -2,7 +2,7 @@
 import re
 import os
 from collections import deque
-from six import binary_type, PY3
+from six import binary_type, PY3, iteritems
 from six.moves.urllib.parse import urljoin
 from fnmatch import fnmatch
 
@@ -195,7 +195,7 @@ class SourceFile(object):
                          ("css", "common")}  # type: Set[Tuple[bytes, ...]]
 
     def __init__(self, tests_root, rel_path, url_base, hash=None, contents=None):
-        # type: (AnyStr, AnyStr, Text, Optional[bytes], Optional[bytes]) -> None
+        # type: (AnyStr, AnyStr, Text, Optional[Text], Optional[bytes]) -> None
         """Object representing a file in a source tree.
 
         :param tests_root: Path to the root of the source tree
@@ -242,9 +242,7 @@ def __getstate__(self):
 
         if "__cached_properties__" in rv:
             cached_properties = rv["__cached_properties__"]
-            for key in rv.keys():
-                if key in cached_properties:
-                    del rv[key]
+            rv = {key:value for key, value in iteritems(rv) if key not in cached_properties}
             del rv["__cached_properties__"]
         return rv
 
@@ -304,7 +302,7 @@ def url(self):
 
     @cached_property
     def hash(self):
-        # type: () -> bytes
+        # type: () -> Text
         if not self._hash:
             with self.open() as f:
                 content = f.read()

testing/web-platform/tests/tools/manifest/utils.py

@@ -55,16 +55,16 @@ def to_os_path(path):
 
 
 def git(path):
-    # type: (bytes) -> Optional[Callable[..., bytes]]
+    # type: (bytes) -> Optional[Callable[..., Text]]
     def gitfunc(cmd, *args):
-        # type: (bytes, *bytes) -> bytes
+        # type: (bytes, *bytes) -> Text
         full_cmd = ["git", cmd] + list(args)
         try:
-            return subprocess.check_output(full_cmd, cwd=path, stderr=subprocess.STDOUT)
+            return subprocess.check_output(full_cmd, cwd=path, stderr=subprocess.STDOUT).decode('utf8')
         except Exception as e:
             if platform.uname()[0] == "Windows" and isinstance(e, WindowsError):
                 full_cmd[0] = "git.bat"
-                return subprocess.check_output(full_cmd, cwd=path, stderr=subprocess.STDOUT)
+                return subprocess.check_output(full_cmd, cwd=path, stderr=subprocess.STDOUT).decode('utf8')
             else:
                 raise
 

testing/web-platform/tests/tools/manifest/vcs.py

@@ -33,7 +33,7 @@ def get_tree(tests_root, manifest, manifest_path, cache_root,
     # type: (bytes, Manifest, Optional[bytes], Optional[bytes], bool, bool) -> FileSystem
     tree = None
     if cache_root is None:
-        cache_root = os.path.join(tests_root, b".wptcache")
+        cache_root = os.path.join(tests_root, ".wptcache")
     if not os.path.exists(cache_root):
         try:
             os.makedirs(cache_root)
@@ -58,21 +58,21 @@ def __init__(self, path):
         self.git = git(path)
 
     def _local_changes(self):
-        # type: () -> Set[bytes]
+        # type: () -> Set[Text]
         """get a set of files which have changed between HEAD and working copy"""
         assert self.git is not None
         # note that git runs the command with tests_root as the cwd, which may
         # not be the root of the git repo (e.g., within a browser repo)
         cmd = [b"diff-index", b"--relative", b"--no-renames", b"--name-only", b"-z", b"HEAD"]
         data = self.git(*cmd)
-        return set(data.split(b"\0"))
+        return set(data.split("\0"))
 
     def hash_cache(self):
-        # type: () -> Dict[bytes, Optional[bytes]]
+        # type: () -> Dict[Text, Optional[Text]]
         """
         A dict of rel_path -> current git object id if the working tree matches HEAD else None
         """
-        hash_cache = {}  # type: Dict[bytes, Optional[bytes]]
+        hash_cache = {}  # type: Dict[Text, Optional[Text]]
 
         if self.git is None:
             return hash_cache
@@ -81,9 +81,9 @@ def hash_cache(self):
         # not be the root of the git repo (e.g., within a browser repo)
         cmd = ["ls-tree", "-r", "-z", "HEAD"]
         local_changes = self._local_changes()
-        for result in self.git(*cmd).split(b"\0")[:-1]:  # type: bytes
-            data, rel_path = result.rsplit(b"\t", 1)
-            hash_cache[rel_path] = None if rel_path in local_changes else data.split(b" ", 3)[2]
+        for result in self.git(*cmd).split("\0")[:-1]:  # type: Text
+            data, rel_path = result.rsplit("\t", 1)
+            hash_cache[rel_path] = None if rel_path in local_changes else data.split(" ", 3)[2]
 
         return hash_cache
 
@@ -174,7 +174,7 @@ def check_valid(self, data):
 
 
 class MtimeCache(CacheFile):
-    file_name = b"mtime.json"
+    file_name = "mtime.json"
 
     def __init__(self, cache_root, tests_root, manifest_path, rebuild=False):
         # type: (bytes, bytes, bytes, bool) -> None
@@ -222,7 +222,7 @@ def dump(self):
 
 
 class GitIgnoreCache(CacheFile, MutableMapping):  # type: ignore
-    file_name = b"gitignore.json"
+    file_name = "gitignore.json"
 
     def check_valid(self, data):
         # type: (Dict[Any, Any]) -> Dict[Any, Any]
@@ -286,7 +286,7 @@ def walk(root):
     relpath = os.path.relpath
 
     root = os.path.abspath(root)
-    stack = deque([(root, b"")])
+    stack = deque([(root, "")])
 
     while stack:
         dir_path, rel_path = stack.popleft()

testing/web-platform/tests/tools/wpt/testfiles.py

@@ -299,7 +299,7 @@ def affected_by_wdspec(test):
 
     def affected_by_interfaces(file_contents):
         # type: (Union[bytes, Text]) -> bool
-        if len(interfaces_changed_names) > 0:
+        if len(interfaces_changed) > 0:
             if 'idlharness.js' in file_contents:
                 for interface in interfaces_changed_names:
                     regex = '[\'"]' + interface + '(\\.idl)?[\'"]'
@@ -324,9 +324,9 @@ def affected_by_interfaces(file_contents):
 
             with open(test_full_path, "rb") as fh:
                 raw_file_contents = fh.read()  # type: bytes
-                if raw_file_contents.startswith("\xfe\xff"):
+                if raw_file_contents.startswith(b"\xfe\xff"):
                     file_contents = raw_file_contents.decode("utf-16be", "replace")  # type: Text
-                elif raw_file_contents.startswith("\xff\xfe"):
+                elif raw_file_contents.startswith(b"\xff\xfe"):
                     file_contents = raw_file_contents.decode("utf-16le", "replace")
                 else:
                     file_contents = raw_file_contents.decode("utf8", "replace")