bug 1259832 - package generated sources and upload them along with ot…

…her build artifacts. r=gps

This change makes us upload an `$(PKG_BASENAME).generated-files.tar.gz` archive
alongside other build artifacts which contains all the generated source files
from the build. A change after this will introduce an `upload-generated-sources`
task to take this artifact and upload the individual files to an S3 bucket.

This will be used to provide links to generated source files when they appear
in stack traces in crash reports.

MozReview-Commit-ID: 6yQAdlZ5q3O

--HG--
extra : rebase_source : d92fb17ae737d1360e9724997f6688e29bedef12
extra : source : 14d18d7cf454c4c3d0f6d49d1d01660e06e4be4b

Makefile.in

@@ -305,6 +305,10 @@ endif
 update-packaging:
 	$(MAKE) -C tools/update-packaging
 
+.PHONY: package-generated-sources
+package-generated-sources:
+	$(call py_action,package_generated_sources,'$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)')
+
 #XXX: this is a hack, since we don't want to clobber for MSVC
 # PGO support, but we can't do this test in client.mk
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)

build/moz-automation.mk

@@ -27,6 +27,7 @@ tier_MOZ_AUTOMATION_INSTALLER = installer
 tier_MOZ_AUTOMATION_PACKAGE = package
 tier_MOZ_AUTOMATION_PACKAGE_TESTS = package-tests
 tier_MOZ_AUTOMATION_UPDATE_PACKAGING = update-packaging
+tier_MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES = package-generated-sources
 tier_MOZ_AUTOMATION_UPLOAD_SYMBOLS = uploadsymbols
 tier_MOZ_AUTOMATION_UPLOAD = upload
 
@@ -41,6 +42,7 @@ moz_automation_symbols = \
   MOZ_AUTOMATION_PACKAGE \
   MOZ_AUTOMATION_INSTALLER \
   MOZ_AUTOMATION_UPDATE_PACKAGING \
+  MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES \
   MOZ_AUTOMATION_L10N_CHECK \
   MOZ_AUTOMATION_UPLOAD \
   $(NULL)
@@ -60,6 +62,7 @@ automation/upload: automation/package
 automation/upload: automation/package-tests
 automation/upload: automation/buildsymbols
 automation/upload: automation/update-packaging
+automation/upload: automation/package-generated-sources
 
 # The installer and packager all run stage-package, and may conflict
 # with each other.

build/mozconfig.artifact.automation

@@ -3,3 +3,4 @@
 
 MOZ_AUTOMATION_BUILD_SYMBOLS=0
 MOZ_AUTOMATION_L10N_CHECK=0
+MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0

build/mozconfig.automation

@@ -15,6 +15,7 @@ mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}"
 mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}"
 mk_add_options "export MOZ_AUTOMATION_INSTALLER=${MOZ_AUTOMATION_INSTALLER-0}"
 mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=${MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES-1}"
 mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}"
 mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}"
 

mobile/android/config/mozconfigs/android-api-15-frontend/nightly

@@ -7,6 +7,7 @@ MOZ_AUTOMATION_PACKAGE_TESTS=0
 MOZ_AUTOMATION_UPDATE_PACKAGING=0
 MOZ_AUTOMATION_UPLOAD=0
 MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
 
 NO_CACHE=1
 NO_NDK=1

mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly

@@ -7,6 +7,7 @@ MOZ_AUTOMATION_PACKAGE_TESTS=0
 MOZ_AUTOMATION_UPDATE_PACKAGING=0
 MOZ_AUTOMATION_UPLOAD=0
 MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
 
 NO_CACHE=1
 NO_NDK=1

mobile/android/config/mozconfigs/android-api-15-gradle/nightly-artifact

@@ -1,5 +1,4 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1

mobile/android/config/mozconfigs/android-api-15/debug-artifact

@@ -1,5 +1,4 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1

mobile/android/config/mozconfigs/android-api-15/nightly-artifact

@@ -1,5 +1,4 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1

mobile/android/config/mozconfigs/android-x86/nightly-artifact

@@ -1,5 +1,4 @@
-MOZ_AUTOMATION_BUILD_SYMBOLS=0
-MOZ_AUTOMATION_L10N_CHECK=0
+. "$topsrcdir/build/mozconfig.artifact.automation"
 
 NO_CACHE=1
 NO_NDK=1

python/mozbuild/mozbuild/action/package_generated_sources.py

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import json
+import os.path
+import sys
+
+import buildconfig
+from mozpack.archive import create_tar_gz_from_files
+from mozbuild.generated_sources import get_generated_sources
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Produce archive of generated sources')
+    parser.add_argument('outputfile', help='File to write output to')
+    args = parser.parse_args(argv)
+
+
+    files = dict(get_generated_sources())
+    with open(args.outputfile, 'wb') as fh:
+        create_tar_gz_from_files(fh, files, compresslevel=5)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/generated_sources.py

@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+
+from mozpack.files import FileFinder
+import mozpack.path as mozpath
+
+
+def get_generated_sources():
+    '''
+    Yield tuples of `(objdir-rel-path, file)` for generated source files
+    in this objdir, where `file` is either an absolute path to the file or
+    a `mozpack.File` instance.
+    '''
+    import buildconfig
+
+    # First, get the list of generated sources produced by the build backend.
+    gen_sources = os.path.join(buildconfig.topobjdir, 'generated-sources.json')
+    with open(gen_sources, 'rb') as f:
+        data = json.load(f)
+    for f in data['sources']:
+        yield f, mozpath.join(buildconfig.topobjdir, f)
+    # Next, return all the files in $objdir/ipc/ipdl/_ipdlheaders.
+    base = 'ipc/ipdl/_ipdlheaders'
+    finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
+    for p, f in finder.find('**/*.h'):
+        yield mozpath.join(base, p), f
+    # Next, return any Rust source files that were generated into the Rust
+    # object directory.
+    rust_build_kind = 'debug' if buildconfig.substs.get('MOZ_DEBUG_RUST') else 'release'
+    base = mozpath.join('toolkit/library',
+                        buildconfig.substs['RUST_TARGET'],
+                        rust_build_kind,
+                        'build')
+    finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
+    for p, f in finder.find('**/*.rs'):
+        yield mozpath.join(base, p), f

toolkit/mozapps/installer/package-name.mk

@@ -80,6 +80,9 @@ PKG_SRCPACK_PATH =
 SYMBOL_FULL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols-full
 SYMBOL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols
 
+# Generated file package naming
+GENERATED_SOURCE_FILE_PACKAGE = $(PKG_BASENAME).generated-files.tar.gz
+
 # Code coverage package naming
 CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno
 

toolkit/mozapps/installer/upload-files.mk

@@ -397,6 +397,7 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(WP_TEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GTEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDID_INFO_TXT_FILE)) \