Bug 896797 - Part 3: Use install manifests for managing dist/include;…

… r=glandium

Makefile.in

@@ -62,6 +62,7 @@ ifndef MOZ_PROFILE_USE
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
 	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
 	$(call py_action,purge_manifests,-d _build_manifests/purge .)
+	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER

js/src/Makefile.in

@@ -177,6 +177,7 @@ MOZILLA_DTRACE_SRC = $(srcdir)/devtools/javascript-trace.d
 endif
 
 default::
+	$(call py_action,process_install_manifest,$(if $(JS_STANDALONE),,--no-remove) $(DIST)/include _build_manifests/install/dist_include)
 
 ifneq (,$(CROSS_COMPILE)$(filter-out WINNT OS2,$(OS_ARCH)))
 # nsinstall doesn't get built until we enter config/ in the exports phase,

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -13,14 +13,14 @@
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, *paths):
+def process_manifest(destdir, paths, remove_unaccounted=True):
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     manifest.populate_registry(copier)
-    return copier.copy(destdir)
+    return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
 
 
 if __name__ == '__main__':
@@ -29,10 +29,13 @@ def process_manifest(destdir, *paths):
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
+    parser.add_argument('--no-remove', action='store_true',
+        help='Do not remove unaccounted files from destination.')
 
     args = parser.parse_args()
 
-    result = process_manifest(args.destdir, *args.manifests)
+    result = process_manifest(args.destdir, args.manifests,
+        remove_unaccounted=not args.no_remove)
 
     print(COMPLETE.format(dest=args.destdir,
         existing=result.existing_files_count,

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -149,11 +149,8 @@ def detailed(summary):
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._install_manifests = dict()
-
         self._purge_manifests = dict(
             dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_include=PurgeManifest(relpath='dist/include'),
             dist_private=PurgeManifest(relpath='dist/private'),
             dist_public=PurgeManifest(relpath='dist/public'),
             dist_sdk=PurgeManifest(relpath='dist/sdk'),
@@ -163,6 +160,7 @@ def detailed(summary):
 
         self._install_manifests = dict(
             dist_idl=InstallManifest(),
+            dist_include=InstallManifest(),
         )
 
     def _update_from_avoid_write(self, result):
@@ -368,27 +366,22 @@ def _process_directory_traversal(self, obj, backend_file):
                 ' '.join(obj.parallel_external_make_dirs))
 
     def _process_exports(self, obj, exports, backend_file, namespace=""):
+        # This may not be needed, but is present for backwards compatibility
+        # with the old make rules, just in case.
+        if not obj.dist_install:
+            return
+
         strings = exports.get_strings()
         if namespace:
-            if strings:
-                backend_file.write('EXPORTS_NAMESPACES += %s\n' % namespace)
-            export_name = 'EXPORTS_%s' % namespace
             namespace += '/'
-        else:
-            export_name = 'EXPORTS'
-
-        # Iterate over the list of export filenames, printing out an EXPORTS
-        # declaration for each.
-        if strings:
-            backend_file.write('%s += %s\n' % (export_name, ' '.join(strings)))
 
-            for s in strings:
-                source = os.path.normpath(os.path.join(obj.srcdir, s))
-                if not os.path.isfile(source):
-                    raise Exception('File listed in EXPORTS does not exist: %s' % source)
+        for s in strings:
+            source = os.path.normpath(os.path.join(obj.srcdir, s))
+            dest = '%s%s' % (namespace, os.path.basename(s))
+            self._install_manifests['dist_include'].add_symlink(source, dest)
 
-                p = '%s%s' % (namespace, s)
-                self._purge_manifests['dist_include'].add(p)
+            if not os.path.exists(source):
+                raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
@@ -405,7 +398,8 @@ def _handle_idl_manager(self, manager):
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
-            self._purge_manifests['dist_include'].add('%s.h' % idl['root'])
+            self._install_manifests['dist_include'].add_optional_exists('%s.h'
+                % idl['root'])
             build_files.add(mozpath.join('headers', '%s.h' % idl['root']))
 
         for module in manager.modules:

python/mozbuild/mozbuild/frontend/data.py

@@ -161,11 +161,13 @@ class Exports(SandboxDerived):
     this object fills that role. It just has a reference to the underlying
     HierarchicalStringList, which is created when parsing EXPORTS.
     """
-    __slots__ = ('exports')
+    __slots__ = ('exports', 'dist_install')
 
-    def __init__(self, sandbox, exports):
+    def __init__(self, sandbox, exports, dist_install=True):
         SandboxDerived.__init__(self, sandbox)
         self.exports = exports
+        self.dist_install = dist_install
+
 
 class IPDLFile(SandboxDerived):
     """Describes an individual .ipdl source file."""

python/mozbuild/mozbuild/frontend/emitter.py

@@ -154,7 +154,8 @@ def emit_from_sandbox(self, sandbox):
 
         exports = sandbox.get('EXPORTS')
         if exports:
-            yield Exports(sandbox, exports)
+            yield Exports(sandbox, exports,
+                dist_install=not sandbox.get('NO_DIST_INSTALL', False))
 
         program = sandbox.get('PROGRAM')
         if program:

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -234,33 +234,16 @@ def test_variable_passthru(self):
             self.assertEqual(found, val)
 
     def test_exports(self):
-        """Ensure EXPORTS is written out correctly."""
+        """Ensure EXPORTS is handled properly."""
         env = self._consume('exports', RecursiveMakeBackend)
 
-        backend_path = os.path.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
-
-        self.assertEqual(lines, [
-            'MOZBUILD_DERIVED := 1',
-            'NO_MAKEFILE_RULE := 1',
-            'NO_SUBMAKEFILES_RULE := 1',
-            'EXPORTS += foo.h',
-            'EXPORTS_NAMESPACES += mozilla',
-            'EXPORTS_mozilla += mozilla1.h mozilla2.h',
-            'EXPORTS_NAMESPACES += mozilla/dom',
-            'EXPORTS_mozilla/dom += dom1.h dom2.h',
-            'EXPORTS_NAMESPACES += mozilla/gfx',
-            'EXPORTS_mozilla/gfx += gfx.h',
-            'EXPORTS_NAMESPACES += nspr/private',
-            'EXPORTS_nspr/private += pprio.h',
-        ])
-
-        # EXPORTS files should appear in the dist_include purge manifest.
-        m = PurgeManifest(path=os.path.join(env.topobjdir,
-            '_build_manifests', 'purge', 'dist_include'))
-        self.assertIn('foo.h', m.entries)
-        self.assertIn('mozilla/mozilla1.h', m.entries)
-        self.assertIn('mozilla/dom/dom2.h', m.entries)
+        # EXPORTS files should appear in the dist_include install manifest.
+        m = InstallManifest(path=os.path.join(env.topobjdir,
+            '_build_manifests', 'install', 'dist_include'))
+        self.assertEqual(len(m), 7)
+        self.assertIn('foo.h', m)
+        self.assertIn('mozilla/mozilla1.h', m)
+        self.assertIn('mozilla/dom/dom2.h', m)
 
     def test_xpcshell_manifests(self):
         """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly."""
@@ -297,8 +280,8 @@ def test_xpidl_generation(self):
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
-        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_include'))
-        self.assertIn('foo.h', m.entries)
+        m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
+        self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
@@ -325,7 +308,6 @@ def test_purge_manifests_written(self):
 
         expected = [
             'dist_bin',
-            'dist_include',
             'dist_private',
             'dist_public',
             'dist_sdk',