convert_bazel_headers.py

import concurrent.futures
import json
import os
import platform
import shlex
import shutil
import subprocess
import sys
import traceback
from typing import Annotated, List

import typer


def work(target_library: str, silent: bool, cpu_count: int, cc: List[str]):
    headers = set()
    original_headers = set()

    def get_headers(line):
        nonlocal headers
        try:
            with open(target_library + ".bazel_headers") as f:
                bazel_headers = [line.strip() for line in f.readlines()]
                bazel_headers += [
                    "src/mongo/platform/basic.h",
                    "src/mongo/platform/windows_basic.h",
                ]

            with open(target_library + ".env_vars") as f:
                tmp_env_vars = json.load(f)
                env_vars = {}
                # subprocess requies only strings
                for k, v in tmp_env_vars.items():
                    env_vars[str(k)] = str(v)

            for command in cc:
                cmd_output = command["output"].replace("\\", "/").strip("'").strip('"')
                line_output = line.replace("\\", "/")

                if cmd_output == line_output:
                    os.makedirs(os.path.dirname(line), exist_ok=True)
                    if os.name == "nt":
                        header_arg = " /showIncludes"
                    else:
                        header_arg = " -H"

                    if not silent:
                        print(f"compiling {line}")

                    p = subprocess.run(
                        shlex.split((command["command"].replace("\\", "/") + header_arg)),
                        env=env_vars,
                        capture_output=True,
                        text=True,
                    )
                    if p.returncode != 0:
                        print(f"Error compiling, exitcode: {p.returncode}", file=sys.stderr)
                        print(f"STDOUT: {p.stdout}", file=sys.stderr)
                        print(f"STDERR: {p.stderr}", file=sys.stderr)
                        sys.exit(1)
                    if os.name == "nt":
                        for line in p.stdout.split("\n"):
                            line = (
                                line.replace("Note: including file:", "")
                                .strip(" ")
                                .replace("\\", "/")
                            )

                            if not line.startswith(os.getcwd().replace("\\", "/")):
                                continue

                            line = os.path.relpath(
                                line, start=os.getcwd().replace("\\", "/")
                            ).replace("\\", "/")
                            if line not in bazel_headers:
                                if line.startswith("src/") or line.startswith("bazel-out/"):
                                    original_headers.add(line)
                                    line = "//" + line
                                    line = ":".join(line.rsplit("/", 1))

                                    headers.add(line)
                    else:
                        for line in p.stderr.split("\n"):
                            if ". src/" in line or ". bazel-out/" in line:
                                while line.startswith("."):
                                    line = line[1:]
                                line = line.replace("\\", "/")

                                if line[1:] not in bazel_headers:
                                    original_headers.add(line[1:])
                                    line = "//" + line[1:]
                                    line = ":".join(line.rsplit("/", 1))

                                    headers.add(line)
        except Exception as exc:
            print(traceback.format_exc(), file=sys.stderr)
            raise exc

    sources = []
    with open(target_library + ".obj_files") as f:
        lines = f.readlines()
        for line in lines:
            line = line.strip()
            line = line.replace("build/opt", "//src")
            line = line[: line.find(".")] + ".cpp"
            src_header = os.path.splitext(line[2:])[0] + ".h"
            if os.path.exists(src_header):
                src_header = "//" + ":".join(src_header.rsplit("/", 1))
                headers.add(src_header)
            line = ":".join(line.rsplit("/", 1))
            if line.endswith("_gen.cpp"):
                line = line[:-4]
            sources.append(line)

        with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:
            jobs = {executor.submit(get_headers, line.strip()): line.strip() for line in lines}
            for completed_job in concurrent.futures.as_completed(jobs):
                if not silent:
                    print(f"finished {jobs[completed_job]}")

    with open(".bazel_include_info.json") as f:
        bazel_include_info = json.load(f)

    header_map = bazel_include_info["header_map"]
    gen_header_map = bazel_include_info["gen_header_map"]
    bazel_exec = bazel_include_info["bazel_exec"]
    bazel_config = bazel_include_info["config"]

    global_headers = (
        "src/mongo:config.h",
        "src/mongo/config.h",
        "src/mongo/platform/basic.h",
        "src/mongo/platform/windows_basic.h",
    )

    reverse_header_map = {}
    for k, v in header_map.items():
        for hdr in v:
            if not hdr or hdr.endswith(global_headers):
                continue
            bazel_header = "//" + hdr.replace("\\", "/")
            bazel_header = ":".join(bazel_header.rsplit("/", 1))
            if bazel_header.startswith("//src/third_party/SafeInt"):
                reverse_header_map[bazel_header] = ["//src/third_party/SafeInt:headers"]
            elif bazel_header.startswith("//src/third_party/immer"):
                reverse_header_map[bazel_header] = ["//src/third_party/immer:headers"]
            elif bazel_header in reverse_header_map:
                if bazel_header.startswith("//src/third_party/"):
                    continue
                reverse_header_map[bazel_header].append(k)
            else:
                reverse_header_map[bazel_header] = [k]

    for k, v in gen_header_map.items():
        for hdr in v:
            if not hdr or hdr.endswith(global_headers):
                continue
            bazel_header = "//" + hdr.replace("\\", "/")
            bazel_header = ":".join(bazel_header.rsplit("/", 1))
            if bazel_header not in reverse_header_map:
                reverse_header_map[bazel_header] = [k]
            else:
                reverse_header_map[bazel_header].append(k)

    recommended_deps = set()
    minimal_headers = set()

    basename_sources = [os.path.splitext(src.rsplit(":", 1)[1])[0] for src in sources]
    for header in headers:
        header_basename = os.path.splitext(header.rsplit(":", 1)[1])[0]
        if header_basename in basename_sources:
            minimal_headers.add(header)
            continue

        if header in reverse_header_map:
            found = False
            for lib in reverse_header_map[header]:
                recommended_deps.add(lib)
        else:
            if not header.endswith(global_headers):
                minimal_headers.add(header)

    deps_order_by_height = []
    deps_queries = {}

    with open(target_library + ".bazel_deps") as f:
        original_deps = [line.strip() for line in f.readlines()]

    for dep in recommended_deps | set(original_deps):
        p = subprocess.run(
            [bazel_exec, "cquery"]
            + bazel_config
            + [f'kind("extract_debuginfo|idl_generator|render_template", deps("@{dep}"))'],
            capture_output=True,
            text=True,
        )
        deps_queries[dep] = [
            line.split(" ")[0] for line in p.stdout.splitlines() if line.startswith("//")
        ]
        deps_order_by_height.append((dep, len(deps_queries[dep])))

    deps_order_by_height.sort(key=lambda x: x[1])

    deps_order_by_height = [dep[0] for dep in deps_order_by_height]
    optimal_header_deps = set()
    for header in headers:
        if header in minimal_headers:
            continue

        path_header = "/".join(header.rsplit(":", 1))
        path_header = path_header[2:]
        for dep in deps_order_by_height:
            if dep in header_map and path_header in header_map[dep]:
                optimal_header_deps.add(dep)
                break
            found = False
            for other_dep in deps_order_by_height:
                if other_dep in gen_header_map:
                    continue
                if dep in deps_queries[other_dep]:
                    optimal_header_deps.add(other_dep)
                    found = True
                    break
            if found:
                continue
            if dep in gen_header_map:
                minimal_headers.add(dep)
            else:
                raise Exception(
                    f"Should not happen, did not find way to add dep {dep} for {target_library}"
                )

    optimal_header_deps = list(optimal_header_deps)

    working_deps = optimal_header_deps.copy()
    for dep in optimal_header_deps:
        if dep in working_deps:
            for test_dep in optimal_header_deps:
                if test_dep == dep:
                    continue
                if test_dep in working_deps and test_dep in deps_queries[dep]:
                    working_deps.remove(test_dep)

    link_deps = []
    header_deps = []
    for dep in sorted(list(set(list(working_deps) + list(set(original_deps))))):
        if dep in original_deps:
            link_deps.append(dep)
        else:
            header_deps.append(dep)

    target_name = os.path.splitext(os.path.basename(target_library))[0]
    if target_name.startswith("lib"):
        target_name = target_name[3:]

    bazel_target = f"{target_library}\n"
    bazel_target += "=" * 50 + "\n"
    local_bazel_path = os.path.dirname(target_library.replace("build/opt", "//src")) + ":"
    bazel_target += "mongo_cc_library(\n"
    bazel_target += f'    name = "{target_name}",\n'
    if sources:
        bazel_target += "    srcs = [\n"
        for src in sorted([src.replace(local_bazel_path, "") for src in sources]):
            bazel_target += f'        "{src}",\n'
        bazel_target += "    ],\n"
    if minimal_headers:
        bazel_target += "    hdrs = [\n"
        for header in sorted([header.replace(local_bazel_path, "") for header in minimal_headers]):
            bazel_target += f'        "{header}",\n'
        bazel_target += "    ],\n"
    if header_deps:
        bazel_target += "    header_deps = [\n"
        for dep in sorted([dep.strip().replace(local_bazel_path, "") for dep in header_deps]):
            bazel_target += f'        "{dep}",\n'
        bazel_target += "    ],\n"
    if link_deps:
        bazel_target += "    deps = [\n"
        for dep in sorted([dep.strip().replace(local_bazel_path, "") for dep in link_deps]):
            bazel_target += f'        "{dep}",\n'
        bazel_target += "    ],\n"
    bazel_target += ")\n"
    return bazel_target


def main(
    target_libraries: Annotated[List[str], typer.Argument()],
    silent: Annotated[bool, typer.Option()] = False,
    skip_scons: Annotated[bool, typer.Option()] = False,
    debug_mode: Annotated[bool, typer.Option()] = False,
):
    extra_args = []
    if os.name == "nt":
        extra_args += [
            "CPPPATH=C:\sasl\include",
            "LIBPATH=C:\sasl\lib",
        ]
        target_library = os.path.join(
            os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib"
        )

    path = shutil.which("icecc")
    if path is None:
        extra_args += ["ICECC="]

    # Define separate functions instead of using lambdas
    def target_fmt_nt(target_library: str) -> str:
        return os.path.join(
            os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib"
        )

    def target_fmt_darwin(target_library: str) -> str:
        return target_library[:-2] + "a"

    def target_fmt_default(x: str) -> None:
        return None

    if os.name == "nt":
        target_fmt = target_fmt_nt
    elif platform.system() == "Darwin":
        target_fmt = target_fmt_darwin
    else:
        target_fmt = target_fmt_default

    map(target_fmt, target_libraries)

    cmd = [
        sys.executable,
        "buildscripts/scons.py",
        "--build-profile=opt",
        " ".join(
            [f"--bazel-includes-info={target_library}" for target_library in target_libraries]
        ),
        "--libdeps-linting=off",
        "--ninja=disabled",
        "compiledb",
    ] + extra_args

    if not skip_scons:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)

        while True:
            line = p.stdout.readline()
            if not line:
                break
            print(line.strip(), file=sys.stderr)

        _, _ = p.communicate()

        if p.returncode != 0:
            print(f"SCons build failed, exit code {p.returncode}", file=sys.stderr)
            sys.exit(1)

    with open("compile_commands.json") as f:
        cc = json.load(f)
    if platform.system() == "Linux":
        cpu_count = len(os.sched_getaffinity(0)) + 4
    else:
        cpu_count = os.cpu_count() + 4

    # Process pool makes it harder to debug what is happening
    # so for debug mode, we disabled process pool so things happen in order
    # you can just print from the process.
    if debug_mode:
        bazel_targets = []
        for target_library in target_libraries:
            bazel_targets += [work(target_library, silent, cpu_count, cc)]
    else:
        with concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count) as executor:
            jobs = {
                executor.submit(work, target_library, silent, cpu_count, cc): target_library
                for target_library in target_libraries
            }
            bazel_targets = [job.result() for job in concurrent.futures.as_completed(jobs)]

    print("====== Bazel Targets ======\n")
    print("\n".join(bazel_targets))


if __name__ == "__main__":
    typer.run(main)