Bug 923080 - Generate xpt files into final location. r=glandium

Now that the mozbuild backend knows about FINAL_TARGET, we are able to
install generated xpt files into their final location. This saves us
from copying xpt files into their final location on every build.

Original patch by gps, rebased and comments addressed by Ms2ger

--HG--
extra : transplant_source : %E2%DC%0F%E0%AD%C2%25%A1%B8%A9%FE%B0%8C%60%FF%CB%02G%25%E5

config/makefiles/xpidl/Makefile.in

@@ -22,48 +22,39 @@ include $(topsrcdir)/config/rules.mk
 # As an optimization to reduce overall CPU usage, we process all .idl
 # belonging to a module with a single command invocation. This prevents
 # redundant parsing of .idl files and significantly reduces CPU cycles.
-#
-# Future improvement: Headers are currently written to a local directory then
-# installed in the distribution directory. It is preferable to write headers
-# directly into the distribution directory. However, PGO builds remove the dist
-# directory via rm -rf (with no regards to manifests). Since the cost of
-# processing XPIDL files is not trivial, it is preferrable to cache the headers
-# and reinstall them rather than regenerate them. Ideally the dist pruning is
-# performed with manifests. At that time we can write headers directly to the
-# dist directory.
 
 # For dependency files.
 idl_deps_dir := .deps
 
-# Where we put our final, linked .xpt files.
-idl_xpt_dir := xpt
-
 dist_idl_dir := $(DIST)/idl
 dist_include_dir := $(DIST)/include
 process_py := $(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py
 
-# TODO we should use py_action, but that would require extra directories to be
-# in the virtualenv.
-idlprocess := $(PYTHON_PATH) $(PLY_INCLUDE) -I$(IDL_PARSER_DIR) -I$(IDL_PARSER_CACHE_DIR) \
-    $(process_py) --cache-dir $(IDL_PARSER_CACHE_DIR) $(dist_idl_dir) \
-        $(dist_include_dir) $(idl_xpt_dir) $(idl_deps_dir)
-
 ifdef LIBXUL_SDK
-idlprocess += -I$(LIBXUL_SDK)/idl
+libxul_sdk_includes := -I$(LIBXUL_SDK)/idl
 endif
 
+# TODO we should use py_action, but that would require extra directories to be
+# in the virtualenv.
+%.xpt:
+	@echo "$(@F)"
+	$(PYTHON_PATH) $(PLY_INCLUDE) -I$(IDL_PARSER_DIR) -I$(IDL_PARSER_CACHE_DIR) \
+		$(process_py) --cache-dir $(IDL_PARSER_CACHE_DIR) $(dist_idl_dir) \
+		$(dist_include_dir) $(@D) $(idl_deps_dir) $(libxul_sdk_includes) \
+		$(basename $(notdir $@ $(filter %.idl,$^)))
+
 xpidl_modules := @xpidl_modules@
+xpt_files := @xpt_files@
 
 @xpidl_rules@
 
-linked_xpt_files := $(addprefix $(idl_xpt_dir)/,$(addsuffix .xpt,$(xpidl_modules)))
 depends_files := $(foreach root,$(xpidl_modules),$(idl_deps_dir)/$(root).pp)
 
-GARBAGE += $(linked_xpt_files) $(depends_files)
+GARBAGE += $(xpt_files) $(depends_files)
 
-xpidl:: $(linked_xpt_files)
+xpidl:: $(xpt_files)
 
-$(linked_xpt_files): $(process_py) $(call mkdir_deps,$(idl_deps_dir) $(dist_include_dir) $(idl_xpt_dir))
+$(xpt_files): $(process_py) $(call mkdir_deps,$(idl_deps_dir) $(dist_include_dir))
 
 $(call include_deps,$(depends_files))
 

config/rules.mk

@@ -1157,11 +1157,6 @@ endif
 ifdef XPT_NAME #{
 
 ifndef NO_DIST_INSTALL
-_XPT_NAME_FILES := $(DEPTH)/config/makefiles/xpidl/xpt/$(XPT_NAME)
-_XPT_NAME_DEST := $(FINAL_TARGET)/components
-_XPT_NAME_TARGET := misc
-INSTALL_TARGETS += _XPT_NAME
-
 ifndef NO_INTERFACES_MANIFEST
 misc:: $(call mkdir_deps,$(FINAL_TARGET)/components)
 	$(call py_action,buildlist,$(FINAL_TARGET)/components/interfaces.manifest 'interfaces $(XPT_NAME)')

js/xpconnect/tests/idl/moz.build

@@ -14,3 +14,11 @@ XPIDL_SOURCES += [
 
 XPIDL_MODULE = 'xpctest'
 
+# XXX: This relies on xpctest.xpt being created in dist/bin/components/ during
+# the export tier AND TEST_HARNESS_FILES being processed after that.
+TEST_HARNESS_FILES.xpcshell.js.xpconnect.tests.components.native += [
+    '!/dist/bin/components/xpctest.xpt',
+]
+TEST_HARNESS_FILES.xpcshell.js.xpconnect.tests.components.js += [
+    '!/dist/bin/components/xpctest.xpt',
+]

python/mozbuild/mozbuild/backend/common.py

@@ -39,7 +39,7 @@ def __init__(self, config):
         self.idls = {}
         self.modules = {}
 
-    def register_idl(self, source, module, allow_existing=False):
+    def register_idl(self, source, module, install_target, allow_existing=False):
         """Registers an IDL file with this instance.
 
         The IDL file will be built, installed, etc.
@@ -58,7 +58,8 @@ def register_idl(self, source, module, allow_existing=False):
             raise Exception('IDL already registered: %' % entry['basename'])
 
         self.idls[entry['basename']] = entry
-        self.modules.setdefault(entry['module'], set()).add(entry['root'])
+        t = self.modules.setdefault(entry['module'], (install_target, set()))
+        t[1].add(entry['root'])
 
 
 class WebIDLCollection(object):
@@ -181,7 +182,8 @@ def consume_object(self, obj):
                     topsrcdir=obj.topsrcdir)
 
         elif isinstance(obj, XPIDLFile):
-            self._idl_manager.register_idl(obj.source_path, obj.module)
+            self._idl_manager.register_idl(obj.source_path, obj.module,
+                obj.install_target)
 
         elif isinstance(obj, ConfigFileSubstitution):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -4,6 +4,7 @@
 
 from __future__ import unicode_literals
 
+import errno
 import itertools
 import json
 import logging
@@ -15,11 +16,11 @@
     defaultdict,
     namedtuple,
 )
+from StringIO import StringIO
 
 import mozwebidlcodegen
 from reftest import ReftestManifest
 
-import mozbuild.makeutil as mozmakeutil
 from mozpack.copier import FilePurger
 from mozpack.manifests import (
     InstallManifest,
@@ -751,7 +752,7 @@ def consume_finished(self):
 
         # Write out a master list of all IPDL source files.
         ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
-        mk = mozmakeutil.Makefile()
+        mk = Makefile()
 
         sorted_ipdl_sources = list(sorted(self._ipdl_sources))
         mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources))
@@ -983,8 +984,7 @@ def _process_javascript_modules(self, obj, backend_file):
     def _handle_idl_manager(self, manager):
         build_files = self._install_manifests['xpidl']
 
-        for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
-            'xpt/.mkdir.done'):
+        for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done'):
             build_files.add_optional_exists(p)
 
         for idl in manager.idls.values():
@@ -994,34 +994,44 @@ def _handle_idl_manager(self, manager):
                 % idl['root'])
 
         for module in manager.modules:
-            build_files.add_optional_exists(mozpath.join('xpt',
-                '%s.xpt' % module))
             build_files.add_optional_exists(mozpath.join('.deps',
                 '%s.pp' % module))
 
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
-        rules = []
+        xpt_files = set()
+
+        mk = Makefile()
 
         for module in xpt_modules:
-            deps = sorted(modules[module])
-            idl_deps = ['$(dist_idl_dir)/%s.idl' % dep for dep in deps]
-            rules.extend([
-                # It may seem strange to have the .idl files listed as
-                # prerequisites both here and in the auto-generated .pp files.
-                # It is necessary to list them here to handle the case where a
-                # new .idl is added to an xpt. If we add a new .idl and nothing
-                # else has changed, the new .idl won't be referenced anywhere
-                # except in the command invocation. Therefore, the .xpt won't
-                # be rebuilt because the dependencies say it is up to date. By
-                # listing the .idls here, we ensure the make file has a
-                # reference to the new .idl. Since the new .idl presumably has
-                # an mtime newer than the .xpt, it will trigger xpt generation.
-                '$(idl_xpt_dir)/%s.xpt: %s' % (module, ' '.join(idl_deps)),
-                '\t@echo "$(notdir $@)"',
-                '\t$(idlprocess) $(basename $(notdir $@)) %s' % ' '.join(deps),
-                '',
-            ])
+            install_target, sources = modules[module]
+            deps = sorted(sources)
+
+            # It may seem strange to have the .idl files listed as
+            # prerequisites both here and in the auto-generated .pp files.
+            # It is necessary to list them here to handle the case where a
+            # new .idl is added to an xpt. If we add a new .idl and nothing
+            # else has changed, the new .idl won't be referenced anywhere
+            # except in the command invocation. Therefore, the .xpt won't
+            # be rebuilt because the dependencies say it is up to date. By
+            # listing the .idls here, we ensure the make file has a
+            # reference to the new .idl. Since the new .idl presumably has
+            # an mtime newer than the .xpt, it will trigger xpt generation.
+            xpt_path = '$(DEPTH)/%s/components/%s.xpt' % (install_target, module)
+            xpt_files.add(xpt_path)
+            rule = mk.create_rule([xpt_path])
+            rule.add_dependencies(['$(call mkdir_deps,%s)' % mozpath.dirname(xpt_path)])
+            rule.add_dependencies(manager.idls['%s.idl' % dep]['source'] for dep in deps)
+
+            if install_target.startswith('dist/'):
+                path = mozpath.relpath(xpt_path, '$(DEPTH)/dist')
+                prefix, subpath = path.split('/', 1)
+                key = 'dist_%s' % prefix
+
+                self._install_manifests[key].add_optional_exists(subpath)
+
+        rules = StringIO()
+        mk.dump(rules, removal_guard=False)
 
         # Create dependency for output header so we force regeneration if the
         # header was deleted. This ideally should not be necessary. However,
@@ -1037,8 +1047,9 @@ def _handle_idl_manager(self, manager):
         obj.topobjdir = self.environment.topobjdir
         obj.config = self.environment
         self._create_makefile(obj, extra=dict(
-            xpidl_rules='\n'.join(rules),
+            xpidl_rules=rules.getvalue(),
             xpidl_modules=' '.join(xpt_modules),
+            xpt_files=' '.join(sorted(xpt_files)),
         ))
 
     def _process_program(self, program, backend_file):

python/mozbuild/mozbuild/frontend/context.py

@@ -1161,7 +1161,7 @@ def __new__(cls, obj):
         Files from topsrcdir and the objdir can also be installed by prefixing
         the path(s) with a '/' character and a '!' character, respectively::
            TEST_HARNESS_FILES.path += ['/build/bar.py', '!quux.py']
-        """, None),
+        """, 'libs'),
 }
 
 # Sanity check: we don't want any variable above to have a list as storage type.

python/mozbuild/mozbuild/frontend/data.py

@@ -157,6 +157,7 @@ class XPIDLFile(ContextDerived):
 
     __slots__ = (
         'basename',
+        'install_target',
         'source_path',
     )
 
@@ -167,6 +168,8 @@ def __init__(self, context, source, module):
         self.basename = mozpath.basename(source)
         self.module = module
 
+        self.install_target = context['FINAL_TARGET']
+
 class Defines(ContextDerived):
     """Context derived container object for DEFINES, which is an OrderedDict.
     """

python/mozbuild/mozbuild/frontend/emitter.py

@@ -345,6 +345,14 @@ def emit_from_context(self, context):
 
         This is a generator of mozbuild.frontend.data.ContextDerived instances.
         """
+
+        # We only want to emit an InstallationTarget if one of the consulted
+        # variables is defined. Later on, we look up FINAL_TARGET, which has
+        # the side-effect of populating it. So, we need to do this lookup
+        # early.
+        if any(k in context for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')):
+            yield InstallationTarget(context)
+
         # We always emit a directory traversal descriptor. This is needed by
         # the recursive make backend.
         for o in self._emit_directory_traversal_from_context(context): yield o
@@ -523,9 +531,9 @@ def canonical_suffix_for_file(f):
                 for s in strings:
                     if context.is_objdir_path(s):
                         if s.startswith('!/'):
-                            raise SandboxValidationError(
-                                'Topobjdir-relative file not allowed in TEST_HARNESS_FILES: %s' % s, context)
-                        objdir_files[path].append(s[1:])
+                            objdir_files[path].append('$(DEPTH)/%s' % s[2:])
+                        else:
+                            objdir_files[path].append(s[1:])
                     else:
                         resolved = context.resolve_path(s)
                         if '*' in s:
@@ -616,10 +624,6 @@ def canonical_suffix_for_file(f):
                     'does not exist: %s (resolved to %s)' % (local_include, actual_include), context)
             yield LocalInclude(context, local_include)
 
-        if context.get('FINAL_TARGET') or context.get('XPI_NAME') or \
-                context.get('DIST_SUBDIR'):
-            yield InstallationTarget(context)
-
         final_target_files = context.get('FINAL_TARGET_FILES')
         if final_target_files:
             yield FinalTargetFiles(context, final_target_files, context['FINAL_TARGET'])

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -443,7 +443,9 @@ def test_xpidl_generation(self):
 
         m = InstallManifest(path=mozpath.join(install_dir, 'xpidl'))
         self.assertIn('.deps/my_module.pp', m)
-        self.assertIn('xpt/my_module.xpt', m)
+
+        m = InstallManifest(path=os.path.join(install_dir, 'dist_bin'))
+        self.assertIn('components/my_module.xpt', m)
 
         m = InstallManifest(path=mozpath.join(install_dir, 'dist_include'))
         self.assertIn('foo.h', m)