Parallelize compilation of LDC D unittests

This serial step (unless using explicit -DCOMPILE_D_MODULES_SEPARATELY=ON,
which may decrease LDC performance due to missing cross-module
optimizations) used to take more than 4 minutes for Shippable...

.azure-pipelines/posix.yml

@@ -80,9 +80,9 @@ steps:
       -DCMAKE_INSTALL_PREFIX=$installDir \
       -DINCLUDE_INSTALL_DIR=$installDir/import \
       $EXTRA_CMAKE_FLAGS
-    ninja -j$PARALLEL_JOBS all all-test-runners
+    ninja -j$PARALLEL_JOBS all ldc2-unittest all-test-runners
     bin/ldc2 -version
-  displayName: Build LDC & defaultlib unittest runners
+  displayName: Build LDC & LDC D unittests & defaultlib unittest runners
 - script: |
     set -e
     cd ..
@@ -105,7 +105,7 @@ steps:
     set -e
     cd ../build
     ctest --output-on-failure -R "ldc2-unittest"
-  displayName: Build & run LDC D unittests
+  displayName: Run LDC D unittests
   condition: succeededOrFailed()
 - script: |
     set -e

.azure-pipelines/windows.yml

@@ -92,15 +92,15 @@ steps:
     mkdir build
     cd build
     cmake -G Ninja %BUILD_SOURCESDIRECTORY% -DCMAKE_C_COMPILER:PATH=clang-cl.exe -DCMAKE_CXX_COMPILER:PATH=clang-cl.exe -DCMAKE_BUILD_TYPE=Release %EXTRA_CMAKE_FLAGS% -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DINCLUDE_INSTALL_DIR=%INSTALL_DIR%/import -DLLVM_ROOT_DIR=%CD%/../llvm -DD_COMPILER=%CD%/../bootstrap-ldc/bin/ldmd2
-    ninja -j2 all all-test-runners
+    ninja -j2 all ldc2-unittest all-test-runners
     bin\ldc2 --version
-  displayName: Build LDC & defaultlib unittest runners
+  displayName: Build LDC & LDC D unittests & defaultlib unittest runners
 - script: |
     cd ..
     call "%VSINSTALLDIR%Common7\Tools\VsDevCmd.bat" -arch=%ARCH%
     cd build
     ctest --output-on-failure -R "ldc2-unittest"
-  displayName: Build & run LDC D unittests
+  displayName: Run LDC D unittests
   condition: succeededOrFailed()
 - script: |
     cd ..

.circleci/config.yml

@@ -53,7 +53,7 @@ commonSteps: &commonSteps
         name: Checkout git submodules
         command: git submodule update --init
     - run:
-        name: Build LDC & defaultlib unittest runners
+        name: Build LDC & LDC D unittests & defaultlib unittest runners
         command: |
           set -x
           cd ..
@@ -65,10 +65,10 @@ commonSteps: &commonSteps
             -DCMAKE_BUILD_TYPE=Release \
             -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
             $EXTRA_CMAKE_FLAGS
-          ninja -j2 all all-test-runners
+          ninja -j2 all ldc2-unittest all-test-runners
           bin/ldc2 -version
     - run:
-        name: Build & run LDC D unittests
+        name: Run LDC D unittests
         when: always
         command: cd ../build && ctest --output-on-failure -R ldc2-unittest
     - run:

.semaphoreci/job1.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 free -m
 cd build
-ninja -j2
+ninja -j2 all ldc2-unittest
 bin/ldc2 -version
 
 EC=0

.travis.yml

@@ -87,14 +87,14 @@ install:
 script:
   # Invoke CMake to generate the Ninja build files.
   - cmake -G Ninja -DLLVM_ROOT_DIR=$LLVM_ROOT_DIR -DLDC_INSTALL_LLVM_RUNTIME_LIBS=ON $OPTS .
-  # Build LDC and defaultlib unittest runners.
+  # Build LDC, LDC D unittests and defaultlib unittest runners.
   # [Some unittest modules eat large amounts of memory during compilation => -j2.]
-  - ninja -j2 all runtime/objects-unittest/std/algorithm/searching.o runtime/objects-unittest-debug/std/algorithm/searching.o runtime/objects-unittest/std/range/package.o runtime/objects-unittest-debug/std/range/package.o runtime/objects-unittest/std/regex/internal/tests.o runtime/objects-unittest-debug/std/regex/internal/tests.o
+  - ninja -j2 all ldc2-unittest runtime/objects-unittest/std/algorithm/searching.o runtime/objects-unittest-debug/std/algorithm/searching.o runtime/objects-unittest/std/range/package.o runtime/objects-unittest-debug/std/range/package.o runtime/objects-unittest/std/regex/internal/tests.o runtime/objects-unittest-debug/std/regex/internal/tests.o
   - ninja -j3 all-test-runners
   # Output some environment info, plus make sure we only run the test suite
   # if we could actually build the executable.
   - bin/ldc2 -version || exit 1
-  # Build and run LDC D unittests.
+  # Run LDC D unittests.
   - ctest --output-on-failure -R "ldc2-unittest"
   # Run LIT testsuite.
   - ctest -V -R "lit-tests"

CMakeLists.txt

@@ -706,6 +706,7 @@ else()
         "$<TARGET_LINKER_FILE:${LDC_LIB}>"
         "${FE_RES}"
         "${LDC_LIB}"
+        ${COMPILE_D_MODULES_SEPARATELY}
     )
 endif()
 
@@ -747,6 +748,7 @@ build_d_executable(
     "$<TARGET_LINKER_FILE:LDMD_CXX_LIB>"
     ""
     "LDMD_CXX_LIB"
+    ${COMPILE_D_MODULES_SEPARATELY}
 )
 
 # Little helper.
@@ -902,6 +904,7 @@ build_d_executable(
     "$<TARGET_LINKER_FILE:${LDC_LIB}>"
     ""
     "${LDC_LIB}"
+    ON # always build separately (faster with parallelization)
 )
 add_custom_target(ldc2-unittest DEPENDS ${LDC_UNITTEST_EXE_FULL})
 add_test(NAME build-ldc2-unittest COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target ldc2-unittest)
@@ -928,6 +931,7 @@ build_d_executable(
     ""
     "${PROJECT_SOURCE_DIR}/runtime/ldc-build-runtime.d.in"
     ""
+    ${COMPILE_D_MODULES_SEPARATELY}
 )
 add_custom_target(ldc-build-runtime ALL DEPENDS ${LDC_BUILD_RUNTIME_EXE_FULL})
 

cmake/Modules/BuildDExecutable.cmake

@@ -1,13 +1,12 @@
 # Depends on these global variables:
-# - COMPILE_D_MODULES_SEPARATELY
 # - D_COMPILER
 # - D_COMPILER_FLAGS
 # - DDMD_DFLAGS
 # - DDMD_LFLAGS
 # - LDC_LINK_MANUALLY
 # - LDC_LINKERFLAG_LIST
 # - LDC_TRANSLATED_LINKER_FLAGS
-function(build_d_executable output_exe d_src_files compiler_args linker_args extra_compile_deps link_deps)
+function(build_d_executable output_exe d_src_files compiler_args linker_args extra_compile_deps link_deps compile_separately)
     set(dflags "${D_COMPILER_FLAGS} ${DDMD_DFLAGS}")
     if(UNIX)
       separate_arguments(dflags UNIX_COMMAND "${dflags}")
@@ -18,7 +17,7 @@ function(build_d_executable output_exe d_src_files compiler_args linker_args ext
     get_filename_component(exe_basename ${output_exe} NAME_WE) # no path, no extension
 
     set(object_files)
-    if(NOT COMPILE_D_MODULES_SEPARATELY)
+    if(NOT compile_separately)
         # Compile all D modules to a single object.
         set(object_file ${PROJECT_BINARY_DIR}/obj/${exe_basename}${CMAKE_CXX_OUTPUT_EXTENSION})
         add_custom_command(
@@ -48,7 +47,7 @@ function(build_d_executable output_exe d_src_files compiler_args linker_args ext
     # Use a response file on Windows when compiling separately, in order not to
     # exceed the max command-line length.
     set(objects_args "${object_files}")
-    if(WIN32 AND COMPILE_D_MODULES_SEPARATELY)
+    if(WIN32 AND compile_separately)
         string(REPLACE ";" " " objects_args "${object_files}")
         file(WRITE ${output_exe}.rsp ${objects_args})
         set(objects_args "@${output_exe}.rsp")

shippable.yml

@@ -60,7 +60,7 @@ build:
     - ninja -j32
     - bin/ldc2 --version
     - cd ..
-    # Build LDC
+    # Build LDC & LDC D unittests
     - |
       set -x
       installDir=$PWD/install
@@ -75,11 +75,11 @@ build:
         -DCOMPILE_ALL_D_FILES_AT_ONCE=OFF \
         $EXTRA_CMAKE_FLAGS
       set +x
-    - ninja -j32
+    - ninja -j32 all ldc2-unittest
     - bin/ldc2 --version
     # Build defaultlib unittest runners
     - ninja -j16 all-test-runners
-    # Build & run LDC D unittests
+    # Run LDC D unittests
     - ctest --output-on-failure -R "ldc2-unittest"
     # Run LIT testsuite, ignore the errors
     - PATH=$PWD/../llvm/bin:$PATH ctest -V -R "lit-tests" || true