windows_toolchain.py

#!/usr/bin/env python2.7
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# This script is used to create and manipulate archives containing
# files necessary to build Firefox on Windows (referred to as the
# "Windows toolchain").
#
# When updating behavior of this script, remember to update the docs
# in ``build/docs/toolchains.rst``.

from __future__ import absolute_import, unicode_literals

import hashlib
import os
import sys

import six

from mozpack.files import FileFinder
from mozpack.mozjar import JarWriter
import mozpack.path as mozpath

SDK_RELEASE = "10.0.17134.0"

PATTERNS = [
    {
        "srcdir": "%(vs_path)s/DIA SDK",
        "dstdir": "DIA SDK",
        "files": [
            {
                "pattern": "bin/**",
                "ignore": ("bin/arm/**",),
            },
            {
                "pattern": "idl/**",
            },
            {
                "pattern": "include/**",
            },
            {
                "pattern": "lib/**",
                "ignore": ("lib/arm/**",),
            },
        ],
    },
    {
        "srcdir": "%(vs_path)s/VC/Tools/MSVC/14.16.27023",
        "dstdir": "VC",
        "files": [
            # ATL is needed by Breakpad.
            {
                "pattern": "atlmfc/include/**",
            },
            {
                "pattern": "atlmfc/lib/arm64/atls.*",
            },
            {
                "pattern": "atlmfc/lib/x64/atls.*",
            },
            {
                "pattern": "atlmfc/lib/x86/atls.*",
            },
            # ARM64 PGO-instrumented builds require ARM64 pgort140.dll.
            {
                "pattern": "bin/arm64/pgort140.dll",
            },
            {
                "pattern": "bin/Hostx64/**",
            },
            # 32-bit PGO-instrumented builds require 32-bit pgort140.dll.
            {
                "pattern": "bin/Hostx86/x86/pgort140.dll",
            },
            {
                "pattern": "include/**",
            },
            {
                "pattern": "lib/**",
                "ignore": (
                    "lib/arm64/store/**",
                    "lib/onecore/**",
                    "lib/x64/store/**",
                    "lib/x86/store/**",
                ),
            },
        ],
    },
    {
        "srcdir": "%(vs_path)s/VC/Redist/MSVC/14.16.27012",
        "dstdir": "VC/redist",
        "files": [
            {
                "pattern": "arm64/Microsoft.VC141.CRT/**",
            },
            {
                "pattern": "x64/Microsoft.VC141.CRT/**",
            },
            {
                "pattern": "x86/Microsoft.VC141.CRT/**",
            },
        ],
    },
    {
        "srcdir": "%(sdk_path)s",
        "dstdir": "SDK",
        "files": [
            {
                "pattern": "bin/%s/x64/**" % SDK_RELEASE,
            },
            {
                "pattern": "Include/%s/**" % SDK_RELEASE,
            },
            {
                "pattern": "Lib/%s/ucrt/arm64/**" % SDK_RELEASE,
            },
            {
                "pattern": "Lib/%s/ucrt/x64/**" % SDK_RELEASE,
            },
            {
                "pattern": "Lib/%s/ucrt/x86/**" % SDK_RELEASE,
            },
            {
                "pattern": "Lib/%s/um/arm64/**" % SDK_RELEASE,
            },
            {
                "pattern": "Lib/%s/um/x64/**" % SDK_RELEASE,
            },
            {
                "pattern": "Lib/%s/um/x86/**" % SDK_RELEASE,
            },
            {
                "pattern": "Redist/D3D/**",
            },
            {
                "pattern": "Redist/ucrt/DLLs/x64/**",
            },
            {
                "pattern": "Redist/ucrt/DLLs/x86/**",
            },
        ],
    },
]


def find_vs_paths():
    """Resolve source locations of files.

    Returns a 2-tuple of (Visual Studio Path, SDK Path).
    """
    pf = os.environ.get("ProgramFiles(x86)")
    if not pf:
        raise Exception(
            'No "ProgramFiles(x86)" environment variable. '
            "Not running on 64-bit Windows?"
        )

    vs_path = os.path.join(pf, "Microsoft Visual Studio", "2017", "Community")
    if not os.path.exists(vs_path):
        raise Exception(
            "%s does not exist; Visual Studio 2017 not installed?" % vs_path
        )

    sdk_path = os.path.join(pf, "Windows Kits", "10")
    if not os.path.exists(sdk_path):
        raise Exception("%s does not exist; Windows 10 SDK not installed?" % sdk_path)

    sdk_fullver_path = os.path.join(sdk_path, "Include", SDK_RELEASE)
    if not os.path.exists(sdk_fullver_path):
        raise Exception(
            "%s does not exist; Wrong SDK version installed?" % sdk_fullver_path
        )

    return vs_path, sdk_path


def resolve_files():
    """Resolve the files that constitute a standalone toolchain.

    This is a generator of (dest path, file) where the destination
    path is relative and the file instance is a BaseFile from mozpack.
    """
    vs_path, sdk_path = find_vs_paths()

    for entry in PATTERNS:
        fullpath = entry["srcdir"] % {
            "vs_path": vs_path,
            "sdk_path": sdk_path,
        }
        for pattern in entry["files"]:
            finder = FileFinder(fullpath, ignore=pattern.get("ignore", []))
            for p, f in finder.find(pattern["pattern"]):
                dstpath = "%s/%s" % (entry["dstdir"], p)
                yield dstpath.encode("utf-8"), f


def resolve_files_and_hash(manifest):
    """Resolve files and hash their data.

    This is a generator of 3-tuples of (relpath, data, mode).

    As data is read, the manifest is populated with metadata.
    Keys are set to the relative file path. Values are 2-tuples
    of (data length, sha-256).
    """
    assert manifest == {}
    for p, f in resolve_files():
        data = f.read()

        sha256 = hashlib.sha256()
        sha256.update(data)
        manifest[p] = (len(data), sha256.hexdigest().encode("utf-8"))

        yield p, data, f.mode


def format_manifest(manifest):
    """Return formatted SHA-256 manifests as a byte strings."""
    sha256_lines = []
    for path, (length, sha256) in sorted(manifest.items()):
        sha256_lines.append(b"%s\t%d\t%s" % (sha256, length, path))

    # Trailing newline.
    sha256_lines.append(b"")

    return b"\n".join(sha256_lines)


def write_zip(zip_path, prefix=None):
    """Write toolchain data to a zip file."""
    prefix = six.ensure_binary(prefix, encoding="utf-8")

    with JarWriter(file=zip_path, compress_level=5) as zip:
        manifest = {}
        for p, data, mode in resolve_files_and_hash(manifest):
            print(p)
            if prefix:
                p = mozpath.join(prefix, p)

            zip.add(p, data, mode=mode)

        sha256_manifest = format_manifest(manifest)

        sdk_path = b"SDK_VERSION"
        sha256_path = b"MANIFEST.SHA256"
        if prefix:
            sdk_path = mozpath.join(prefix, sdk_path)
            sha256_path = mozpath.join(prefix, sha256_path)

        zip.add(sdk_path, SDK_RELEASE.encode("utf-8"))
        zip.add(sha256_path, sha256_manifest)


if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("usage: %s create-zip <path-prefix>" % sys.argv[0])
        sys.exit(1)

    assert sys.argv[1] == "create-zip"
    prefix = os.path.basename(sys.argv[2])
    destzip = "%s.zip" % sys.argv[2]
    write_zip(destzip, prefix=prefix)

    sha1 = hashlib.sha1()
    sha256 = hashlib.sha256()
    sha512 = hashlib.sha512()

    with open(destzip, "rb") as fh:
        data = fh.read()
        sha1.update(data)
        sha256.update(data)
        sha512.update(data)

    print("Hashes of %s (size=%d)" % (destzip, len(data)))
    print("SHA-1:   %s" % sha1.hexdigest())
    print("SHA-256: %s" % sha256.hexdigest())
    print("SHA-512: %s" % sha512.hexdigest())