Migrate configure.sh to python for better Windows usability

Summary: Pull Request resolved: facebook#62 Reviewed By: ridiculousfish Differential Revision: D16419848 Pulled By: dulinriley fbshipit-source-id: 5ade6a8dd8e8cf90adff9a4c7009c2506605abb6
gaodeng · Jul 24, 2019 · f35a8ec · f35a8ec
1 parent db373cd
commit f35a8ec
Show file tree

Hide file tree

Showing 8 changed files with 288 additions and 308 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -126,6 +126,8 @@ endif()
 #
 # cmake -G Ninja -DLLVM_BUILD_DIR=$XXX -DLLVM_SRC_DIR=$YYY
 #
+file(TO_CMAKE_PATH "${LLVM_BUILD_DIR}" LLVM_BUILD_DIR)
+file(TO_CMAKE_PATH "${LLVM_SRC_DIR}" LLVM_SRC_DIR)
 list(APPEND CMAKE_MODULE_PATH "${LLVM_BUILD_DIR}/lib/cmake/llvm/")
 
 # Enable debug mode by default

diff --git a/README.md b/README.md
@@ -20,10 +20,9 @@ To build a local debug version of the Hermes CLI tools the following steps shoul
 ```shell
 mkdir hermes_workingdir
 cd hermes_workingdir
-export HERMES_WS_DIR="$PWD"
 git clone [email protected]:facebook/hermes.git
-hermes/utils/build_llvm.py llvm llvm_build
-hermes/utils/configure.sh
+hermes/utils/build/build_llvm.py
+hermes/utils/build/configure.py
 cd build
 ninja
 ```
@@ -33,10 +32,9 @@ Or if you're using Windows, the following should get you going in a Git Bash she
 ```shell
 mkdir hermes_workingdir
 cd hermes_workingdir
-export HERMES_WS_DIR="$PWD"
 git -c core.autocrlf=false clone [email protected]:facebook/hermes.git
-BUILD_SYSTEM='Visual Studio 16 2019' CMAKE_FLAGS='-A x64' hermes/utils/build_llvm.py --distribute llvm llvm_build
-CMAKE_FLAGS='-A x64 -DLLVM_ENABLE_LTO=OFF' hermes/utils/configure.sh 'Visual Studio 16 2019'
+hermes/utils/build/build_llvm.py --build-system='Visual Studio 16 2019' --cmake-flags='-A x64' --distribute
+hermes/utils/build/configure.py --build-system='Visual Studio 16 2019' --cmake-flags='-A x64 -DLLVM_ENABLE_LTO=OFF' --distribute
 cd build
 MSBuild.exe ALL_BUILD.vcxproj /p:Configuration=Release
 ```

diff --git a/doc/BuildingAndRunning.md b/doc/BuildingAndRunning.md
@@ -21,14 +21,14 @@ On Mac via Homebrew:
 
 ## Building on Linux and macOS
 
-Hermes will place its build files in a directory given by the `HERMES_WS_DIR` environment variable. Note that Hermes will download and build LLVM as part of its own build.
+Hermes will place its build files in the current directory by default. Note that Hermes will download and build LLVM as part of its own build.
+You can also give explicit source and build directories, use `--help` on the build scripts to see how.
 
 Create a base directory to work in, e.g. ~/workspace, and cd into it. Follow the steps below to build LLVM and generate the Hermes build system:
 
-    export HERMES_WS_DIR="$PWD"
     git clone [email protected]:facebook/hermes.git
-    hermes/utils/build_llvm.py
-    hermes/utils/configure.sh
+    hermes/utils/build/build_llvm.py
+    hermes/utils/build/configure.py
 
 The build system has now been generated in the `build` directory. To perform the build:
 
@@ -38,11 +38,9 @@ The build system has now been generated in the `build` directory. To perform the
 
 The Windows build depends on which particular combination of GitBash/Cygwin/WSL and Visual Studio is used.
 
-    export HERMES_WS_DIR='c:/ws'
-    cd "$HERMES_WS_DIR"
     git -c core.autocrlf=false clone [email protected]:facebook/hermes.git
-    BUILD_SYSTEM='Visual Studio 16 2019' CMAKE_FLAGS='-A x64' DISTRIBUTE=1 hermes/utils/build_llvm.py
-    CMAKE_FLAGS='-A x64 -DLLVM_ENABLE_LTO=OFF' DISTRIBUTE=1 hermes/utils/configure.sh 'Visual Studio 16 2019'
+    hermes/utils/build/build_llvm.py --build-system='Visual Studio 16 2019' --cmake-flags='-A x64' --distribute
+    hermes/utils/build/configure.py --build-system='Visual Studio 16 2019' --cmake-flags='-A x64 -DLLVM_ENABLE_LTO=OFF' --distribute
     cd build_release && MSBuild.exe ALL_BUILD.vcxproj /p:Configuration=Release
 
 ## Running Hermes
@@ -68,10 +66,10 @@ To run the Hermes test suite:
 
 ## Release Build
 
-The above instructions create an unoptimized debug build. The `DISTRIBUTE=1` environment variable will enable a release build, in the `build_release` directory. Example:
+The above instructions create an unoptimized debug build. The `--distribute` flag will enable a release build, in the `build_release` directory. Example:
 
-    env DISTRIBUTE=1 hermes/utils/build_llvm.py
-    env DISTRIBUTE=1 hermes/utils/configure.sh
+    hermes/utils/build/build_llvm.py --distribute
+    hermes/utils/build/configure.py --distribute
     cd build_release && ninja
 
 ### Other Tools

diff --git a/utils/build_llvm.py → utils/build/build_llvm.py b/utils/build_llvm.py → utils/build/build_llvm.py
@@ -6,132 +6,59 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import argparse
 import os
 import platform
-import shutil
 import subprocess
 import sys
 
+from common import build_dir_suffix, get_parser, run_command, which
+
 
 # It references the commit day so we can shallow clone
 # and still manage to checkout this specific revision.
 # NOTE: The revision date must be the day before the
 # actual commit date.
 _LLVM_REV = "c179d7b006348005d2da228aed4c3c251590baa3"
-_LLVM_REV_DATE = "2018-10-08" 
+_LLVM_REV_DATE = "2018-10-08"
 
 
 def parse_args():
-    if platform.system() == "Linux":
-        default_platform = "linux"
-    elif platform.system() == "macos":
-        default_platform = "macosx"
-    elif platform.system() == "Windows":
-        default_platform = "windows"
-    else:
-        default_platform = "unknown"
+    def default_build_command(args):
+        # Choose a default based on the build system chosen
+        if args.build_system == "Ninja":
+            return "ninja"
+        elif args.build_system == "Unix Makefiles":
+            return "make"
+        elif "Visual Studio" in args.build_system:
+            return "MSBuild.exe LLVM.sln -target:build -maxcpucount -verbosity:normal"
+        else:
+            raise Exception("Unrecognized build system: {}".format(args.build_system))
 
-    parser = argparse.ArgumentParser()
+    parser = get_parser()
     parser.add_argument("llvm_src_dir", type=str, nargs="?", default="llvm")
     parser.add_argument("llvm_build_dir", type=str, nargs="?", default="llvm_build")
-    parser.add_argument(
-        "--target-platform",
-        type=str,
-        dest="target_platform",
-        choices=["linux", "macosx", "windows", "iphoneos", "iphonesimulator"],
-        default=default_platform,
-    )
-    parser.add_argument(
-        "--build-system",
-        type=str,
-        dest="build_system",
-        default="Ninja",
-        help="Generator to pass into CMake",
-    )
-    parser.add_argument(
-        "--cmake-flags",
-        type=str,
-        dest="cmake_flags",
-        default="",
-        help="Additional flags to pass to CMake",
-    )
-    parser.add_argument(
-        "--build-type",
-        type=str,
-        dest="build_type",
-        choices=["MinSizeRel", "Debug"],
-        default=None,
-        help="Optimization level of build",
-    )
     parser.add_argument(
         "--build-command",
         type=str,
         dest="build_command",
         default=None,
         help="Command to run once cmake finishes",
     )
-    parser.add_argument(
-        "--http-proxy",
-        type=str,
-        dest="http_proxy",
-        default=os.environ.get("HTTP_PROXY", ""),
-    )
-    parser.add_argument("--distribute", action="store_true")
-    parser.add_argument("--32-bit", dest="is_32_bit", action="store_true")
-    parser.add_argument("--enable-asan", dest="enable_asan", action="store_true")
     parser.add_argument(
         "--cross-compile-only", dest="cross_compile_only", action="store_true"
     )
     args = parser.parse_args()
     args.llvm_src_dir = os.path.realpath(args.llvm_src_dir)
     args.llvm_build_dir = os.path.realpath(args.llvm_build_dir)
     if not args.build_command:
-        # Choose a default based on the build system chosen
-        if args.build_system == "Ninja":
-            args.build_command = "ninja"
-        elif args.build_system == "Unix Makefiles":
-            args.build_command = "make"
-        elif "Visual Studio" in args.build_system:
-            args.build_command = (
-                "MSBuild.exe LLVM.sln -target:build -maxcpucount -verbosity:normal"
-            )
-        else:
-            raise Exception("Unrecognized build system: {}".format(args.build_system))
-
+        args.build_command = default_build_command(args)
     if args.cross_compile_only:
         args.build_command += " " + os.path.join("bin", "llvm-tblgen")
     args.build_type = args.build_type or ("MinSizeRel" if args.distribute else "Debug")
-
+    args.llvm_build_dir += build_dir_suffix(args)
     return args
 
 
-def which(cmd):
-    if sys.version_info >= (3, 3):
-        # On Python 3.3 and above, use shutil.which for a quick error message.
-        resolved = shutil.which(cmd)
-        if not resolved:
-            raise Exception("{} not found on PATH".format(cmd))
-        return os.path.realpath(resolved)
-    else:
-        # Manually check PATH
-        for p in os.environ["PATH"].split(os.path.pathsep):
-            p = os.path.join(p, cmd)
-            if "PATHEXT" in os.environ:
-                # try out adding each extension to the PATH as well
-                for ext in os.environ["PATHEXT"].split(os.path.pathsep):
-                    # Add the extension.
-                    p_and_extension = p + ext
-                    if os.path.exists(p_and_extension) and os.access(
-                        p_and_extension, os.X_OK
-                    ):
-                        return os.path.realpath(p_and_extension)
-            else:
-                if os.path.isfile(p) and os.access(p, os.X_OK):
-                    return os.path.realpath(p)
-        raise Exception("{} not found on PATH".format(cmd))
-
-
 def build_git_command(http_proxy):
     # Otherwise, trust that the user has git on the path.
     command = [which("git")]
@@ -143,60 +70,47 @@ def build_git_command(http_proxy):
     return command
 
 
-def run_command(cmd, **kwargs):
-    print("+ " + " ".join(cmd))
-    return subprocess.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr, **kwargs)
-
-
-def main():
-    args = parse_args()
-    build_dir_suffix = ""
-    if args.enable_asan:
-        build_dir_suffix += "_asan"
-    if args.distribute:
-        build_dir_suffix += "_release"
-    if args.is_32_bit:
-        build_dir_suffix += "_32"
-
-    print("Source Dir: {}".format(args.llvm_src_dir))
-    print("Using Build system: {}".format(args.build_system))
-    print("Using Build command: {}".format(args.build_command))
-    print("Build Dir: {}".format(args.llvm_build_dir))
-    print("Build Type: {}".format(args.build_type))
-
+def clone_and_patch_llvm(args):
     git = build_git_command(args.http_proxy)
-
     if not os.path.exists(args.llvm_src_dir):
         # If the directory doesn't exist, clone LLVM there.
         print("Cloning LLVM into {}".format(args.llvm_src_dir))
         run_command(
             git
-            + ["clone", "--shallow-since", _LLVM_REV_DATE, "https://github.com/llvm-mirror/llvm.git", args.llvm_src_dir]
+            + [
+                "clone",
+                "--shallow-since",
+                _LLVM_REV_DATE,
+                "https://github.com/llvm-mirror/llvm.git",
+                args.llvm_src_dir,
+            ]
         )
 
     # Checkout a specific revision in LLVM.
     run_command(git + ["checkout", _LLVM_REV], cwd=args.llvm_src_dir)
 
-    # Apply small edits to LLVM from patch files.
     # Check that the respository is clean.
     try:
         run_command(git + ["diff-index", "--quiet", "HEAD"], cwd=args.llvm_src_dir)
     except subprocess.CalledProcessError:
         raise Exception("llvm dir is dirty (contains uncommitted changes)")
 
+    # Apply small edits to LLVM from patch files.
     run_command(
         git
         + [
             "apply",
             "--ignore-space-change",
             "--ignore-whitespace",
-            os.path.realpath(
-                os.path.join(__file__, "..", "llvm-changes-for-hermes.patch")
+            os.path.join(
+                os.path.dirname(os.path.realpath(__file__)),
+                "llvm-changes-for-hermes.patch",
             ),
         ],
         cwd=args.llvm_src_dir,
     )
 
+    # Commit the patch.
     run_command(
         git
         + [
@@ -212,6 +126,18 @@ def main():
         cwd=args.llvm_src_dir,
     )
 
+
+def main():
+    args = parse_args()
+
+    print("Source Dir: {}".format(args.llvm_src_dir))
+    print("Using Build system: {}".format(args.build_system))
+    print("Using Build command: {}".format(args.build_command))
+    print("Build Dir: {}".format(args.llvm_build_dir))
+    print("Build Type: {}".format(args.build_type))
+
+    clone_and_patch_llvm(args)
+
     cmake_flags = args.cmake_flags.split() + [
         "-DLLVM_TARGETS_TO_BUILD=",
         "-DCMAKE_BUILD_TYPE={}".format(args.build_type),
@@ -266,13 +192,7 @@ def main():
 
     print("CMake flags: {}".format(" ".join(cmake_flags)))
     run_command(
-        [
-            which("cmake"),
-            "-G",
-            "{}".format(args.build_system),
-            "{}".format(args.llvm_src_dir),
-        ]
-        + cmake_flags,
+        [which("cmake"), "-G", args.build_system, args.llvm_src_dir] + cmake_flags,
         env=os.environ,
         cwd=args.llvm_build_dir,
     )