diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py index 20a58b50fd107..fdb17b1b834f0 100644 --- a/build/mach_bootstrap.py +++ b/build/mach_bootstrap.py @@ -247,6 +247,10 @@ def bootstrap(topsrcdir): # The resource module is UNIX only. pass + from mozbuild.util import patch_main + + patch_main() + def resolve_repository(): import mozversioncontrol diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index 9917d31fb6f56..071daecc397b0 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -1402,6 +1402,71 @@ def recurse_indented_repr(o, level): f.write(result) +def patch_main(): + """This is a hack to work around the fact that Windows multiprocessing needs + to import the original main module, and assumes that it corresponds to a file + ending in .py. + + We do this by a sort of two-level function interposing. The first + level interposes forking.get_command_line() with our version defined + in my_get_command_line(). Our version of get_command_line will + replace the command string with the contents of the fork_interpose() + function to be used in the subprocess. + + The subprocess then gets an interposed imp.find_module(), which we + hack up to find the main module name multiprocessing will assume, since we + know what this will be based on the main module in the parent. If we're not + looking for our main module, then the original find_module will suffice. + + See also: http://bugs.python.org/issue19946 + And: https://bugzilla.mozilla.org/show_bug.cgi?id=914563 + """ + # XXX In Python 3.4 the multiprocessing module was re-written and the below + # code is no longer valid. The Python issue19946 also claims to be fixed in + # this version. It's not clear whether this hack is still needed in 3.4+ or + # not, but at least some basic mach commands appear to work without it. So + # skip it in 3.4+ until we determine it's still needed. + if sys.platform == "win32" and sys.version_info < (3, 4): + import os + from multiprocessing import forking + + global orig_command_line + + # Figure out what multiprocessing will assume our main module + # is called (see python/Lib/multiprocessing/forking.py). + main_path = getattr(sys.modules["__main__"], "__file__", None) + if main_path is None: + # If someone deleted or modified __main__, there's nothing left for + # us to do. + return + main_file_name = os.path.basename(main_path) + main_module_name, ext = os.path.splitext(main_file_name) + if ext == ".py": + # If main is a .py file, everything ought to work as expected. + return + + def my_get_command_line(): + with open( + os.path.join(os.path.dirname(__file__), "fork_interpose.py"), "rU" + ) as fork_file: + fork_code = fork_file.read() + # Add our relevant globals. + fork_string = ( + "main_file_name = '%s'\n" % main_file_name + + "main_module_name = '%s'\n" % main_module_name + + fork_code + ) + cmdline = orig_command_line() + # We don't catch errors if "-c" is not found because it's not clear + # what we should do if the original command line is not of the form + # "python ... -c 'script'". + cmdline[cmdline.index("-c") + 1] = fork_string + return cmdline + + orig_command_line = forking.get_command_line + forking.get_command_line = my_get_command_line + + def ensure_bytes(value, encoding="utf-8"): if isinstance(value, six.text_type): return value.encode(encoding)