Bug 1639009 - Convert 'mach xpcshell-test' to run with Python 3 r=gbrown

Differential Revision: https://phabricator.services.mozilla.com/D78522

mach

@@ -55,7 +55,6 @@ py2commands="
     wpt-test-paths
     wpt-unittest
     wpt-update
-    xpcshell-test
 "
 
 run_py() {

testing/xpcshell/mach_commands.py

@@ -132,19 +132,7 @@ def _run_xpcshell_harness(self, **kwargs):
                 raise
         kwargs['tempDir'] = temp_dir
 
-        # Python through 2.7.2 has issues with unicode in some of the
-        # arguments. Work around that.
-        filtered_args = {}
-        for k, v in kwargs.iteritems():
-            if isinstance(v, unicode_type):
-                v = v.encode('utf-8')
-
-            if isinstance(k, unicode_type):
-                k = k.encode('utf-8')
-
-            filtered_args[k] = v
-
-        result = xpcshell.runTests(filtered_args)
+        result = xpcshell.runTests(kwargs)
 
         self.log_manager.disable_unstructured()
 

testing/xpcshell/runxpcshelltests.py

@@ -19,6 +19,7 @@
 import tempfile
 import time
 import traceback
+import six
 
 from argparse import Namespace
 from collections import defaultdict, deque, namedtuple
@@ -92,6 +93,30 @@
 _cleanup_encoding_re = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]')
 
 
+def dict_keys_str(dictionary):
+    """
+      Dictionary is a dict type object which might contain keys
+      of type bytes. This function decodes those keys to str
+      using UTF-8 decoding and updates the dictionary by deleting
+      the bytes type keys and replacing them with str type keys.
+    """
+    deleteKeys = []
+    for key in dictionary:
+        if isinstance(dictionary[key], dict):
+            dictionary[key] = dict_keys_str(dictionary[key])
+        elif isinstance(dictionary[key], bytes):
+            dictionary[key] = six.ensure_str(dictionary[key])
+
+        if isinstance(key, bytes):
+            deleteKeys.append(key)
+
+    for delete in deleteKeys:
+        dictionary[delete.decode('utf-8')] = dictionary[delete]
+        del dictionary[delete]
+
+    return dictionary
+
+
 def _cleanup_encoding_repl(m):
     c = m.group(0)
     return '\\\\' if c == '\\' else '\\x{0:02X}'.format(ord(c))
@@ -103,9 +128,17 @@ def cleanup_encoding(s):
        points, etc.  If it is a byte string, it is assumed to be
        UTF-8, but it may not be *correct* UTF-8.  Return a
        sanitized unicode object."""
-    if not isinstance(s, basestring):
-        return unicode(s)
-    if not isinstance(s, unicode):
+    if six.PY2:
+        ensure = unicode
+    else:
+        ensure = str
+
+    if not isinstance(s, six.string_types):
+        if isinstance(s, six.binary_type):
+            return six.ensure_str(s)
+        else:
+            return ensure(s)
+    if isinstance(s, six.binary_type):
         s = s.decode('utf-8', 'replace')
     # Replace all C0 and C1 control characters with \xNN escapes.
     return _cleanup_encoding_re.sub(_cleanup_encoding_repl, s)
@@ -267,6 +300,12 @@ def launchProcess(self, cmd, stdout, stderr, env, cwd, timeout=None):
         """
         # timeout is needed by remote xpcshell to extend the
         # remote device timeout. It is not used in this function.
+        if six.PY3:
+            env = dict_keys_str(env)
+            cwd = six.ensure_str(cwd)
+            for i in range(len(cmd)):
+                cmd[i] = six.ensure_str(cmd[i])
+
         if HAVE_PSUTIL:
             popen_func = psutil.Popen
         else:
@@ -293,8 +332,8 @@ def logCommand(self, name, completeCmd, testdir):
         self.log.info("%s | current directory: %r" % (name, testdir))
         # Show only those environment variables that are changed from
         # the ambient environment.
-        changedEnv = (set("%s=%s" % i for i in self.env.iteritems())
-                      - set("%s=%s" % i for i in os.environ.iteritems()))
+        changedEnv = (set("%s=%s" % i for i in six.iteritems(self.env))
+                      - set("%s=%s" % i for i in six.iteritems(os.environ)))
         self.log.info("%s | environment: %s" % (name, list(changedEnv)))
         shell_command_tokens = [pipes.quote(tok) for tok in list(changedEnv) + completeCmd]
         self.log.info("%s | as shell command: (cd %s; %s)" %
@@ -588,7 +627,7 @@ def fix_text_output(self, line):
     def log_line(self, line):
         """Log a line of output (either a parser json object or text output from
         the test process"""
-        if isinstance(line, basestring):
+        if isinstance(line, six.string_types) or isinstance(line, bytes):
             line = self.fix_text_output(line).rstrip('\r\n')
             self.log.process_output(self.proc_ident,
                                     line,
@@ -953,7 +992,7 @@ def buildTestList(self, test_tags=None, test_paths=None, verify=False):
         if self.totalChunks > 1:
             filters.append(chunk_by_slice(self.thisChunk, self.totalChunks))
         try:
-            self.alltests = map(normalize, mp.active_tests(filters=filters, **mozinfo.info))
+            self.alltests = list(map(normalize, mp.active_tests(filters=filters, **mozinfo.info)))
         except TypeError:
             sys.stderr.write("*** offending mozinfo.info: %s\n" % repr(mozinfo.info))
             raise
@@ -1197,8 +1236,11 @@ def startServer(name, serverJs):
             try:
                 # We pipe stdin to node because the server will exit when its
                 # stdin reaches EOF
+                if six.PY3:
+                    self.env = dict_keys_str(self.env)
                 process = Popen([nodeBin, serverJs], stdin=PIPE, stdout=PIPE,
-                                stderr=PIPE, env=self.env, cwd=os.getcwd())
+                                stderr=PIPE, env=self.env, cwd=os.getcwd(),
+                                universal_newlines=True)
                 self.nodeProc[name] = process
 
                 # Check to make sure the server starts properly by waiting for it to
@@ -1222,7 +1264,7 @@ def shutdownNode(self):
         """
           Shut down our node process, if it exists
         """
-        for name, proc in self.nodeProc.iteritems():
+        for name, proc in six.iteritems(self.nodeProc):
             self.log.info('Node %s server shutting down ...' % name)
             if proc.poll() is not None:
                 self.log.info('Node server %s already dead %s' % (name, proc.poll()))
@@ -1268,8 +1310,11 @@ def startHttp3Server(self):
             self.log.info('Using %s' % (dbPath))
             # We pipe stdin to the server because it will exit when its stdin
             # reaches EOF
+            if six.PY3:
+                self.env = dict_keys_str(self.env)
             process = Popen([http3ServerPath, dbPath], stdin=PIPE, stdout=PIPE,
-                            stderr=PIPE, env=self.env, cwd=os.getcwd())
+                            stderr=PIPE, env=self.env, cwd=os.getcwd(),
+                            universal_newlines=True)
             self.http3ServerProc['http3Server'] = process
 
             # Check to make sure the server starts properly by waiting for it to
@@ -1288,7 +1333,7 @@ def shutdownHttp3Server(self):
         """
           Shutdown our http3Server process, if it exists
         """
-        for name, proc in self.http3ServerProc.iteritems():
+        for name, proc in six.iteritems(self.http3ServerProc):
             self.log.info('%s server shutting down ...' % name)
             if proc.poll() is not None:
                 self.log.info('Http3 server %s already dead %s' % (name, proc.poll()))
@@ -1345,8 +1390,8 @@ def updateMozinfo(self, prefs, options):
         # All of the keys in question should be ASCII.
         fixedInfo = {}
         for k, v in self.mozInfo.items():
-            if isinstance(k, unicode):
-                k = k.encode('ascii')
+            if isinstance(k, bytes):
+                k = k.decode('utf-8')
             fixedInfo[k] = v
         self.mozInfo = fixedInfo
 
@@ -1618,7 +1663,7 @@ def step1():
                 # the logging system gets confused when 2 or more tests with the same
                 # name run at the same time.
                 sequential_tests = []
-                for i in xrange(VERIFY_REPEAT):
+                for i in range(VERIFY_REPEAT):
                     self.testCount += 1
                     test = testClass(test_object, retry=False,
                                      mobileArgs=mobileArgs, **kwargs)
@@ -1631,7 +1676,7 @@ def step2():
                 # Run tests sequentially, with MOZ_CHAOSMODE enabled.
                 sequential_tests = []
                 self.env["MOZ_CHAOSMODE"] = "3"
-                for i in xrange(VERIFY_REPEAT):
+                for i in range(VERIFY_REPEAT):
                     self.testCount += 1
                     test = testClass(test_object, retry=False,
                                      mobileArgs=mobileArgs, **kwargs)

testing/xpcshell/xpcshellcommandline.py

@@ -7,7 +7,7 @@
 
 def add_common_arguments(parser):
     parser.add_argument("--app-path",
-                        type=unicode, dest="appPath", default=None,
+                        type=str, dest="appPath", default=None,
                         help="application directory (as opposed to XRE directory)")
     parser.add_argument("--interactive",
                         action="store_true", dest="interactive", default=False,
@@ -27,7 +27,7 @@ def add_common_arguments(parser):
     parser.add_argument("--dump-tests", type=str, dest="dump_tests", default=None,
                         help="Specify path to a filename to dump all the tests that will be run")
     parser.add_argument("--manifest",
-                        type=unicode, dest="manifest", default=None,
+                        type=str, dest="manifest", default=None,
                         help="Manifest of test directories to use")
     parser.add_argument("--no-logfiles",
                         action="store_false", dest="logfiles",