Skip to content

Commit

Permalink
Azure CI: Test std.* and the rest separately
Browse files Browse the repository at this point in the history 
On CI, we have been running into OOM issues when running the test
suite on Windows for quite some time.

Unfortunately, we are very close to having the same issues on Linux
as well. Some additional comptime work immediately makes these builds
fail as well.

Add a new `test-toolchain` step, that tests everything except `std.*`
and documentation.

On CI, call `test-toolchain`, `test-std` and `docs` separately
instead of the `test` big hammer that emcompasses all of them.

Change the special case we made for Windows to the same code as other
platforms.

This is a stopgap measure that stage2 will eventually make useless.

Until then, it gives us some headroom.

Change `linux_script` by the way to only output the log of failing
steps. This shrinks the Linux CI log from a bazilion lines down to
something more humanely manageable.
  • Loading branch information
@jedisct1
jedisct1 committed Apr 30, 2021
1 parent d3f5578 commit b49d99b
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 37 deletions.
46 changes: 25 additions & 21 deletions build.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub fn build(b: *Builder) !void {
const docs_step = b.step("docs", "Build documentation");
docs_step.dependOn(&docgen_cmd.step);

const test_step = b.step("test", "Run all the tests");
const toolchain_step = b.step("test-toolchain", "Run the tests for the toolchain");

var test_stage2 = b.addTest("src/test.zig");
test_stage2.setBuildMode(mode);
Expand Down Expand Up @@ -90,7 +90,7 @@ pub fn build(b: *Builder) !void {
exe.install();
exe.setBuildMode(mode);
exe.setTarget(target);
test_step.dependOn(&exe.step);
toolchain_step.dependOn(&exe.step);
b.default_step.dependOn(&exe.step);

exe.addBuildOption(bool, "skip_non_native", skip_non_native);
Expand Down Expand Up @@ -233,7 +233,7 @@ pub fn build(b: *Builder) !void {
const test_stage2_step = b.step("test-stage2", "Run the stage2 compiler tests");
test_stage2_step.dependOn(&test_stage2.step);
if (!skip_stage2_tests) {
test_step.dependOn(test_stage2_step);
toolchain_step.dependOn(test_stage2_step);
}

var chosen_modes: [4]builtin.Mode = undefined;
Expand All @@ -257,33 +257,37 @@ pub fn build(b: *Builder) !void {
const modes = chosen_modes[0..chosen_mode_index];

// run stage1 `zig fmt` on this build.zig file just to make sure it works
test_step.dependOn(&fmt_build_zig.step);
toolchain_step.dependOn(&fmt_build_zig.step);
const fmt_step = b.step("test-fmt", "Run zig fmt against build.zig to make sure it works");
fmt_step.dependOn(&fmt_build_zig.step);

// TODO for the moment, skip wasm32-wasi until bugs are sorted out.
test_step.dependOn(tests.addPkgTests(b, test_filter, "test/stage1/behavior.zig", "behavior", "Run the behavior tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));

test_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/std.zig", "std", "Run the standard library tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));

test_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/compiler_rt.zig", "compiler-rt", "Run the compiler_rt tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));
test_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/c.zig", "minilibc", "Run the mini libc tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));

test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes));
test_step.dependOn(tests.addStackTraceTests(b, test_filter, modes));
test_step.dependOn(tests.addCliTests(b, test_filter, modes));
test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes));
test_step.dependOn(tests.addRuntimeSafetyTests(b, test_filter, modes));
test_step.dependOn(tests.addTranslateCTests(b, test_filter));
toolchain_step.dependOn(tests.addPkgTests(b, test_filter, "test/stage1/behavior.zig", "behavior", "Run the behavior tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));

toolchain_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/compiler_rt.zig", "compiler-rt", "Run the compiler_rt tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));
toolchain_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/c.zig", "minilibc", "Run the mini libc tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir));

toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addStandaloneTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addStackTraceTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addCliTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addRuntimeSafetyTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addTranslateCTests(b, test_filter));
if (!skip_run_translated_c) {
test_step.dependOn(tests.addRunTranslatedCTests(b, test_filter, target));
toolchain_step.dependOn(tests.addRunTranslatedCTests(b, test_filter, target));
}
// tests for this feature are disabled until we have the self-hosted compiler available
// test_step.dependOn(tests.addGenHTests(b, test_filter));
// toolchain_step.dependOn(tests.addGenHTests(b, test_filter));
if (!skip_compile_errors) {
test_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes));
}

const std_step = tests.addPkgTests(b, test_filter, "lib/std/std.zig", "std", "Run the standard library tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir);

const test_step = b.step("test", "Run all the tests");
test_step.dependOn(toolchain_step);
test_step.dependOn(std_step);
test_step.dependOn(docs_step);
}

Expand Down
12 changes: 11 additions & 1 deletion ci/azure/linux_script
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,17 @@ make $JOBS install
cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig"
make $JOBS install

release/bin/zig build test -Denable-qemu -Denable-wasmtime
set +x
LOG=$(mktemp)
for step in test-toolchain test-std docs; do
echo "* Running step: [$step]"
if ! release/bin/zig build $step -Denable-qemu -Denable-wasmtime 2>"$LOG" >&2; then
cat "$LOG" >&2
exit 1
fi
echo " Done."
done
set -x

# Look for HTML errors.
tidy -qe ../zig-cache/langref.html
Expand Down
14 changes: 12 additions & 2 deletions ci/azure/macos_script
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ tar xf "$CACHE_BASENAME.tar.xz"

ZIG="$PREFIX/bin/zig"
NATIVE_LIBC_TXT="$HOME/native_libc.txt"
$ZIG libc > "$NATIVE_LIBC_TXT"
$ZIG libc >"$NATIVE_LIBC_TXT"
export ZIG_LIBC="$NATIVE_LIBC_TXT"
export CC="$ZIG cc"
export CXX="$ZIG c++"
Expand Down Expand Up @@ -55,7 +55,17 @@ make $JOBS install
cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig" -DZIG_TARGET_MCPU="x86_64_v2"
make $JOBS install

release/bin/zig build test
set +x
LOG=$(mktemp)
for step in test-toolchain test-std docs; do
echo "* Running step: [$step]"
if ! release/bin/zig build $step -Denable-qemu -Denable-wasmtime 2>"$LOG" >&2; then
cat "$LOG" >&2
exit 1
fi
echo " Done."
done
set -x

if [ "${BUILD_REASON}" != "PullRequest" ]; then
mv ../LICENSE release/
Expand Down
14 changes: 1 addition & 13 deletions ci/azure/windows_msvc_script.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,8 @@ cd %ZIGBUILDDIR%
cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release -DZIG_OMIT_STAGE2=ON || exit /b
msbuild /maxcpucount /p:Configuration=Release INSTALL.vcxproj || exit /b

"%ZIGINSTALLDIR%\bin\zig.exe" build test-behavior -Dskip-non-native || exit /b
REM Disabled to prevent OOM
REM "%ZIGINSTALLDIR%\bin\zig.exe" build test-stage2 -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-fmt -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-toolchain -Dskip-non-native -Dskip-stage2-tests || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-std -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-compiler-rt -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-compare-output -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-standalone -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-stack-traces -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-cli -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-asm-link -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-runtime-safety -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-translate-c -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build test-run-translated-c -Dskip-non-native || exit /b
"%ZIGINSTALLDIR%\bin\zig.exe" build docs || exit /b

set "PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem"
Expand Down

0 comments on commit b49d99b

Please sign in to comment.