collect_and_build_with_pgo.py

#!/usr/bin/env python3
"""
This script:
- Builds clang with user-defined flags
- Uses that clang to build an instrumented clang, which can be used to collect
  PGO samples
- Builds a user-defined set of sources (default: clang) to act as a
  "benchmark" to generate a PGO profile
- Builds clang once more with the PGO profile generated above

This is a total of four clean builds of clang (by default). This may take a
while. :)

This scripts duplicates https://llvm.org/docs/AdvancedBuilds.html#multi-stage-pgo
Eventually, it will be updated to instead call the cmake cache mentioned there.
"""

import argparse
import collections
import multiprocessing
import os
import shlex
import shutil
import subprocess
import sys

### User configuration


# If you want to use a different 'benchmark' than building clang, make this
# function do what you want. out_dir is the build directory for clang, so all
# of the clang binaries will live under "${out_dir}/bin/". Using clang in
# ${out_dir} will magically have the profiles go to the right place.
#
# You may assume that out_dir is a freshly-built directory that you can reach
# in to build more things, if you'd like.
def _run_benchmark(env, out_dir, include_debug_info):
    """The 'benchmark' we run to generate profile data."""
    target_dir = env.output_subdir("instrumentation_run")

    # `check-llvm` and `check-clang` are cheap ways to increase coverage. The
    # former lets us touch on the non-x86 backends a bit if configured, and the
    # latter gives us more C to chew on (and will send us through diagnostic
    # paths a fair amount, though the `if (stuff_is_broken) { diag() ... }`
    # branches should still heavily be weighted in the not-taken direction,
    # since we built all of LLVM/etc).
    _build_things_in(env, out_dir, what=["check-llvm", "check-clang"])

    # Building tblgen gets us coverage; don't skip it. (out_dir may also not
    # have them anyway, but that's less of an issue)
    cmake = _get_cmake_invocation_for_bootstrap_from(env, out_dir, skip_tablegens=False)

    if include_debug_info:
        cmake.add_flag("CMAKE_BUILD_TYPE", "RelWithDebInfo")

    _run_fresh_cmake(env, cmake, target_dir)

    # Just build all the things. The more data we have, the better.
    _build_things_in(env, target_dir, what=["all"])


### Script


class CmakeInvocation:
    _cflags = ["CMAKE_C_FLAGS", "CMAKE_CXX_FLAGS"]
    _ldflags = [
        "CMAKE_EXE_LINKER_FLAGS",
        "CMAKE_MODULE_LINKER_FLAGS",
        "CMAKE_SHARED_LINKER_FLAGS",
    ]

    def __init__(self, cmake, maker, cmake_dir):
        self._prefix = [cmake, "-G", maker, cmake_dir]

        # Map of str -> (list|str).
        self._flags = {}
        for flag in CmakeInvocation._cflags + CmakeInvocation._ldflags:
            self._flags[flag] = []

    def add_new_flag(self, key, value):
        self.add_flag(key, value, allow_overwrites=False)

    def add_flag(self, key, value, allow_overwrites=True):
        if key not in self._flags:
            self._flags[key] = value
            return

        existing_value = self._flags[key]
        if isinstance(existing_value, list):
            existing_value.append(value)
            return

        if not allow_overwrites:
            raise ValueError("Invalid overwrite of %s requested" % key)

        self._flags[key] = value

    def add_cflags(self, flags):
        # No, I didn't intend to append ['-', 'O', '2'] to my flags, thanks :)
        assert not isinstance(flags, str)
        for f in CmakeInvocation._cflags:
            self._flags[f].extend(flags)

    def add_ldflags(self, flags):
        assert not isinstance(flags, str)
        for f in CmakeInvocation._ldflags:
            self._flags[f].extend(flags)

    def to_args(self):
        args = self._prefix.copy()
        for key, value in sorted(self._flags.items()):
            if isinstance(value, list):
                # We preload all of the list-y values (cflags, ...). If we've
                # nothing to add, don't.
                if not value:
                    continue
                value = " ".join(value)

            arg = "-D" + key
            if value != "":
                arg += "=" + value
            args.append(arg)
        return args


class Env:
    def __init__(self, llvm_dir, use_make, output_dir, default_cmake_args, dry_run):
        self.llvm_dir = llvm_dir
        self.use_make = use_make
        self.output_dir = output_dir
        self.default_cmake_args = default_cmake_args.copy()
        self.dry_run = dry_run

    def get_default_cmake_args_kv(self):
        return self.default_cmake_args.items()

    def get_cmake_maker(self):
        return "Ninja" if not self.use_make else "Unix Makefiles"

    def get_make_command(self):
        if self.use_make:
            return ["make", "-j{}".format(multiprocessing.cpu_count())]
        return ["ninja"]

    def output_subdir(self, name):
        return os.path.join(self.output_dir, name)

    def has_llvm_subproject(self, name):
        if name == "compiler-rt":
            subdir = "../compiler-rt"
        elif name == "clang":
            subdir = "../clang"
        else:
            raise ValueError("Unknown subproject: %s" % name)

        return os.path.isdir(os.path.join(self.llvm_dir, subdir))

    # Note that we don't allow capturing stdout/stderr. This works quite nicely
    # with dry_run.
    def run_command(self, cmd, cwd=None, check=False, silent_unless_error=False):
        print("Running `%s` in %s" % (cmd, shlex.quote(cwd or os.getcwd())))

        if self.dry_run:
            return

        if silent_unless_error:
            stdout, stderr = subprocess.PIPE, subprocess.STDOUT
        else:
            stdout, stderr = None, None

        # Don't use subprocess.run because it's >= py3.5 only, and it's not too
        # much extra effort to get what it gives us anyway.
        popen = subprocess.Popen(
            cmd, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, cwd=cwd
        )
        stdout, _ = popen.communicate()
        return_code = popen.wait(timeout=0)

        if not return_code:
            return

        if silent_unless_error:
            print(stdout.decode("utf-8", "ignore"))

        if check:
            raise subprocess.CalledProcessError(
                returncode=return_code, cmd=cmd, output=stdout, stderr=None
            )


def _get_default_cmake_invocation(env):
    inv = CmakeInvocation(
        cmake="cmake", maker=env.get_cmake_maker(), cmake_dir=env.llvm_dir
    )
    for key, value in env.get_default_cmake_args_kv():
        inv.add_new_flag(key, value)
    return inv


def _get_cmake_invocation_for_bootstrap_from(env, out_dir, skip_tablegens=True):
    clang = os.path.join(out_dir, "bin", "clang")
    cmake = _get_default_cmake_invocation(env)
    cmake.add_new_flag("CMAKE_C_COMPILER", clang)
    cmake.add_new_flag("CMAKE_CXX_COMPILER", clang + "++")

    # We often get no value out of building new tblgens; the previous build
    # should have them. It's still correct to build them, just slower.
    def add_tablegen(key, binary):
        path = os.path.join(out_dir, "bin", binary)

        # Check that this exists, since the user's allowed to specify their own
        # stage1 directory (which is generally where we'll source everything
        # from). Dry runs should hope for the best from our user, as well.
        if env.dry_run or os.path.exists(path):
            cmake.add_new_flag(key, path)

    if skip_tablegens:
        add_tablegen("LLVM_TABLEGEN", "llvm-tblgen")
        add_tablegen("CLANG_TABLEGEN", "clang-tblgen")

    return cmake


def _build_things_in(env, target_dir, what):
    cmd = env.get_make_command() + what
    env.run_command(cmd, cwd=target_dir, check=True)


def _run_fresh_cmake(env, cmake, target_dir):
    if not env.dry_run:
        try:
            shutil.rmtree(target_dir)
        except FileNotFoundError:
            pass

        os.makedirs(target_dir, mode=0o755)

    cmake_args = cmake.to_args()
    env.run_command(cmake_args, cwd=target_dir, check=True, silent_unless_error=True)


def _build_stage1_clang(env):
    target_dir = env.output_subdir("stage1")
    cmake = _get_default_cmake_invocation(env)
    _run_fresh_cmake(env, cmake, target_dir)
    _build_things_in(env, target_dir, what=["clang", "llvm-profdata", "profile"])
    return target_dir


def _generate_instrumented_clang_profile(env, stage1_dir, profile_dir, output_file):
    llvm_profdata = os.path.join(stage1_dir, "bin", "llvm-profdata")
    if env.dry_run:
        profiles = [os.path.join(profile_dir, "*.profraw")]
    else:
        profiles = [
            os.path.join(profile_dir, f)
            for f in os.listdir(profile_dir)
            if f.endswith(".profraw")
        ]
    cmd = [llvm_profdata, "merge", "-output=" + output_file] + profiles
    env.run_command(cmd, check=True)


def _build_instrumented_clang(env, stage1_dir):
    assert os.path.isabs(stage1_dir)

    target_dir = os.path.join(env.output_dir, "instrumented")
    cmake = _get_cmake_invocation_for_bootstrap_from(env, stage1_dir)
    cmake.add_new_flag("LLVM_BUILD_INSTRUMENTED", "IR")

    # libcxx's configure step messes with our link order: we'll link
    # libclang_rt.profile after libgcc, and the former requires atexit from the
    # latter. So, configure checks fail.
    #
    # Since we don't need libcxx or compiler-rt anyway, just disable them.
    cmake.add_new_flag("LLVM_BUILD_RUNTIME", "No")

    _run_fresh_cmake(env, cmake, target_dir)
    _build_things_in(env, target_dir, what=["clang", "lld"])

    profiles_dir = os.path.join(target_dir, "profiles")
    return target_dir, profiles_dir


def _build_optimized_clang(env, stage1_dir, profdata_file):
    if not env.dry_run and not os.path.exists(profdata_file):
        raise ValueError(
            "Looks like the profdata file at %s doesn't exist" % profdata_file
        )

    target_dir = os.path.join(env.output_dir, "optimized")
    cmake = _get_cmake_invocation_for_bootstrap_from(env, stage1_dir)
    cmake.add_new_flag("LLVM_PROFDATA_FILE", os.path.abspath(profdata_file))

    # We'll get complaints about hash mismatches in `main` in tools/etc. Ignore
    # it.
    cmake.add_cflags(["-Wno-backend-plugin"])
    _run_fresh_cmake(env, cmake, target_dir)
    _build_things_in(env, target_dir, what=["clang"])
    return target_dir


Args = collections.namedtuple(
    "Args",
    [
        "do_optimized_build",
        "include_debug_info",
        "profile_location",
        "stage1_dir",
    ],
)


def _parse_args():
    parser = argparse.ArgumentParser(
        description="Builds LLVM and Clang with instrumentation, collects "
        "instrumentation profiles for them, and (optionally) builds things "
        "with these PGO profiles. By default, it's assumed that you're "
        "running this from your LLVM root, and all build artifacts will be "
        "saved to $PWD/out."
    )
    parser.add_argument(
        "--cmake-extra-arg",
        action="append",
        default=[],
        help="an extra arg to pass to all cmake invocations. Note that this "
        "is interpreted as a -D argument, e.g. --cmake-extra-arg FOO=BAR will "
        "be passed as -DFOO=BAR. This may be specified multiple times.",
    )
    parser.add_argument(
        "--dry-run", action="store_true", help="print commands instead of running them"
    )
    parser.add_argument(
        "--llvm-dir",
        default=".",
        help="directory containing an LLVM checkout (default: $PWD)",
    )
    parser.add_argument(
        "--no-optimized-build",
        action="store_true",
        help="disable the final, PGO-optimized build",
    )
    parser.add_argument(
        "--out-dir", help="directory to write artifacts to (default: $llvm_dir/out)"
    )
    parser.add_argument(
        "--profile-output",
        help="where to output the profile (default is $out/pgo_profile.prof)",
    )
    parser.add_argument(
        "--stage1-dir",
        help="instead of having an initial build of everything, use the given "
        "directory. It is expected that this directory will have clang, "
        "llvm-profdata, and the appropriate libclang_rt.profile already built",
    )
    parser.add_argument(
        "--use-debug-info-in-benchmark",
        action="store_true",
        help="use a regular build instead of RelWithDebInfo in the benchmark. "
        "This increases benchmark execution time and disk space requirements, "
        "but gives more coverage over debuginfo bits in LLVM and clang.",
    )
    parser.add_argument(
        "--use-make",
        action="store_true",
        default=shutil.which("ninja") is None,
        help="use Makefiles instead of ninja",
    )

    args = parser.parse_args()

    llvm_dir = os.path.abspath(args.llvm_dir)
    if args.out_dir is None:
        output_dir = os.path.join(llvm_dir, "out")
    else:
        output_dir = os.path.abspath(args.out_dir)

    extra_args = {
        "CMAKE_BUILD_TYPE": "Release",
        "LLVM_ENABLE_PROJECTS": "clang;compiler-rt;lld",
    }
    for arg in args.cmake_extra_arg:
        if arg.startswith("-D"):
            arg = arg[2:]
        elif arg.startswith("-"):
            raise ValueError(
                "Unknown not- -D arg encountered; you may need "
                "to tweak the source..."
            )
        split = arg.split("=", 1)
        if len(split) == 1:
            key, val = split[0], ""
        else:
            key, val = split
        extra_args[key] = val

    env = Env(
        default_cmake_args=extra_args,
        dry_run=args.dry_run,
        llvm_dir=llvm_dir,
        output_dir=output_dir,
        use_make=args.use_make,
    )

    if args.profile_output is not None:
        profile_location = args.profile_output
    else:
        profile_location = os.path.join(env.output_dir, "pgo_profile.prof")

    result_args = Args(
        do_optimized_build=not args.no_optimized_build,
        include_debug_info=args.use_debug_info_in_benchmark,
        profile_location=profile_location,
        stage1_dir=args.stage1_dir,
    )

    return env, result_args


def _looks_like_llvm_dir(directory):
    """Arbitrary set of heuristics to determine if `directory` is an llvm dir.

    Errs on the side of false-positives."""

    contents = set(os.listdir(directory))
    expected_contents = [
        "CODE_OWNERS.TXT",
        "cmake",
        "docs",
        "include",
        "utils",
    ]

    if not all(c in contents for c in expected_contents):
        return False

    try:
        include_listing = os.listdir(os.path.join(directory, "include"))
    except NotADirectoryError:
        return False

    return "llvm" in include_listing


def _die(*args, **kwargs):
    kwargs["file"] = sys.stderr
    print(*args, **kwargs)
    sys.exit(1)


def _main():
    env, args = _parse_args()

    if not _looks_like_llvm_dir(env.llvm_dir):
        _die("Looks like %s isn't an LLVM directory; please see --help" % env.llvm_dir)
    if not env.has_llvm_subproject("clang"):
        _die("Need a clang checkout at tools/clang")
    if not env.has_llvm_subproject("compiler-rt"):
        _die("Need a compiler-rt checkout at projects/compiler-rt")

    def status(*args):
        print(*args, file=sys.stderr)

    if args.stage1_dir is None:
        status("*** Building stage1 clang...")
        stage1_out = _build_stage1_clang(env)
    else:
        stage1_out = args.stage1_dir

    status("*** Building instrumented clang...")
    instrumented_out, profile_dir = _build_instrumented_clang(env, stage1_out)
    status("*** Running profdata benchmarks...")
    _run_benchmark(env, instrumented_out, args.include_debug_info)
    status("*** Generating profile...")
    _generate_instrumented_clang_profile(
        env, stage1_out, profile_dir, args.profile_location
    )

    print("Final profile:", args.profile_location)
    if args.do_optimized_build:
        status("*** Building PGO-optimized binaries...")
        optimized_out = _build_optimized_clang(env, stage1_out, args.profile_location)
        print("Final build directory:", optimized_out)


if __name__ == "__main__":
    _main()