From bfed6278b5d3fdbb44abf464a4a7eef77b1fe9af Mon Sep 17 00:00:00 2001 From: "Rick Poyner (rico)" Date: Thu, 12 Nov 2020 18:34:19 -0500 Subject: [PATCH] kcov: Implement parallel testing schedule (#14294) * kcov: Implement parallel testing schedule Relevant to: #10617 * Update the kcov* bazel configs to run tests in parallel. * Expand test timeouts for slowdowns from parallel schedule. * Add kcov_tool to merge and clean up kcov data. * Update documentation and mentions of bazel-kcov. --- doc/bazel.rst | 17 ++++-- doc/sample_vimrc | 2 +- tools/dynamic_analysis/bazel.rc | 12 ++--- tools/dynamic_analysis/kcov.sh | 2 +- tools/dynamic_analysis/kcov_tool | 88 ++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 11 deletions(-) create mode 100755 tools/dynamic_analysis/kcov_tool diff --git a/doc/bazel.rst b/doc/bazel.rst index a5fbb6eeee17..cf6dd84e0d78 100644 --- a/doc/bazel.rst +++ b/doc/bazel.rst @@ -264,7 +264,7 @@ kcov ---- ``kcov`` can analyze coverage for any binary that contains DWARF format -debuggging symbols, and produce nicely formatted browse-able coverage reports. +debugging symbols, and produce nicely formatted browse-able coverage reports. Drake's ``kcov`` build system integration is only supported on Ubuntu, not macOS. @@ -282,5 +282,16 @@ Note that it disables compiler-optimization (``-O0``) to have a better and more precise coverage report. If you have trouble with kcov and unoptimized programs, you can turn it back on by also supplying ``--copt -O2``. -The coverage report is written to the ``drake/bazel-kcov`` directory. To -view it, browse to ``drake/bazel-kcov/index.html``. +For each test program, individual coverage reports are written to per-target +directories. Use the ``kcov_tool`` to merge coverage data into a new directory:: + + tools/dynamic_analysis/kcov_tool merge [OUTPUT-DIR] + +To view the merged data, browse to ``index.html`` in the OUTPUT-DIR. + +In a local developer workspace, coverage data may accumulate over successive +build jobs, even if source files or other dependencies have changed. The stale +data would be scattered within the directory tree linked as +``bazel-testlogs``. To clear out old data, use ``kcov_tool clean``:: + + tools/dynamic_analysis/kcov_tool clean diff --git a/doc/sample_vimrc b/doc/sample_vimrc index cbd9830b45f0..b3f96baacaee 100644 --- a/doc/sample_vimrc +++ b/doc/sample_vimrc @@ -112,7 +112,7 @@ let g:lt_quickfix_list_toggle_map = 'lq' " ctrlp.vim "---------- let g:ctrlp_working_path_mode = 'rw' -let g:ctrlp_custom_ignore = '\v[\/](build|doc|bazel-kcov)$' +let g:ctrlp_custom_ignore = '\v[\/](build|doc)$' " Launch CtrlP. Use this and then start typing fragments of the file path. nnoremap ff :CtrlP diff --git a/tools/dynamic_analysis/bazel.rc b/tools/dynamic_analysis/bazel.rc index a10b0fbf048f..6c1dd4822f92 100644 --- a/tools/dynamic_analysis/bazel.rc +++ b/tools/dynamic_analysis/bazel.rc @@ -7,14 +7,14 @@ build:kcov --copt -g build:kcov --copt -O0 build:kcov --spawn_strategy=standalone build:kcov --run_under //tools/dynamic_analysis:kcov -build:kcov --local_test_jobs=1 +build:kcov --local_test_jobs=HOST_CPUS*0.5 build:kcov --test_tag_filters=-lint,-gurobi,-mosek,-snopt,-no_kcov build:kcov --nocache_test_results # These increased timeouts were set through experimentation. Because kcov runs # in a separate process from the main program, the OS has to context-switch # between the processes every time a line is hit, slowing down execution -# significantly. Timeouts are 3.5x default values. -build:kcov --test_timeout=210,1050,3150,12600 +# significantly. Timeouts are 10x default values. +build:kcov --test_timeout=600,3000,9000,36000 ### Kcov everything build. ### build:kcov_everything --build_tests_only @@ -25,11 +25,11 @@ build:kcov_everything --copt -g build:kcov_everything --copt -O0 build:kcov_everything --spawn_strategy=standalone build:kcov_everything --run_under=//tools/dynamic_analysis:kcov -build:kcov_everything --local_test_jobs=1 +build:kcov_everything --local_test_jobs=HOST_CPUS*0.5 build:kcov_everything --test_tag_filters=-lint,-no_kcov build:kcov_everything --nocache_test_results -# See timeout note above. Timeouts are 5x default values. -build:kcov_everything --test_timeout=300,1500,4500,18000 +# See timeout note above. Timeouts are 10x default values. +build:kcov_everything --test_timeout=600,3000,9000,36000 ### ASan build. Clang only. ### build:asan --build_tests_only diff --git a/tools/dynamic_analysis/kcov.sh b/tools/dynamic_analysis/kcov.sh index a35e2cea0644..2e9e59962a4b 100755 --- a/tools/dynamic_analysis/kcov.sh +++ b/tools/dynamic_analysis/kcov.sh @@ -16,6 +16,6 @@ kcov \ "--include-path=${WORKSPACE}" \ --verify \ --exclude-pattern=third_party \ - "${WORKSPACE}/bazel-kcov" \ + "${TEST_UNDECLARED_OUTPUTS_DIR}/kcov" \ "--replace-src-path=/proc/self/cwd:${WORKSPACE}" \ "$@" diff --git a/tools/dynamic_analysis/kcov_tool b/tools/dynamic_analysis/kcov_tool new file mode 100755 index 000000000000..a47267b43a0d --- /dev/null +++ b/tools/dynamic_analysis/kcov_tool @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +""" +Manages the code coverage data generated by bazel kcov and kcov_everything +configurations. See instructions here: +https://drake.mit.edu/bazel.html?highlight=kcov#kcov +""" + +import argparse +import glob +import shutil +import subprocess +import os +import sys + +# This bit may need adjusting if this file moves to a new directory. +WORKSPACE = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__)))) + + +def find_kcov_data(): + """Finds the kcov data of individual test runs within the tree linked by + bazel-testlogs. + """ + return glob.glob(os.path.join(WORKSPACE, 'bazel-testlogs/**/kcov'), + recursive=True) + + +def clean(args): + """Implements the clean subcommand.""" + for d in find_kcov_data(): + shutil.rmtree(d) + + +def do_merge(output_dir): + """Merges the kcov data of individual test runs into output_dir.""" + dirs = find_kcov_data() + if not dirs: + sys.exit("Error: no kcov data to merge") + + # Smooth over kcov location change from Bionic to Focal. + newenv = os.environ.copy() + newenv['PATH'] = "/opt/kcov/35/bin:" + newenv['PATH'] + + cmd = ['kcov', '--merge', output_dir] + dirs + subprocess.run(cmd, env=newenv) + + +def ci_merge(args): + """Implements the ci_merge subcommand.""" + do_merge(os.path.join(WORKSPACE, "bazel-kcov")) + + +def merge(args): + """Implements the merge subcommand.""" + do_merge(args.output_dir) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + help='subcommand to run', dest='subcommand') + subparsers.required = True + + parser_ci_merge = subparsers.add_parser( + 'ci_merge', help='like merge, but verbose, and always to bazel-kcov.' + ' For now, succeeds even if no data is available; see #10617.') + parser_ci_merge.set_defaults(func=ci_merge) + + parser_clean = subparsers.add_parser( + 'clean', help='remove all kcov data from the bazel-testlogs tree') + parser_clean.set_defaults(func=clean) + + parser_merge = subparsers.add_parser( + 'merge', + help='merge all kcov data from the bazel-testlogs tree to OUTPUT-DIR') + parser_merge.add_argument( + 'output_dir', metavar='OUTPUT-DIR', + help='output directory for merged kcov data;' + ' if not existing it will be created.') + parser_merge.set_defaults(func=merge) + + args = parser.parse_args() + args.func(args) + + +if __name__ == '__main__': + main()