From f789501e942ea2935a0d56c2bd42576b0df8589b Mon Sep 17 00:00:00 2001 From: "Rick Poyner (rico)" Date: Mon, 31 Aug 2020 14:05:05 -0400 Subject: [PATCH] cassie_bench: Pass optional arguments to benchmark program (#13963) * cassie_bench: Pass optional arguments to benchmark program Relevant to: #13902 This patch allows passing additional cassie_bench arguments when using the :record_results target, or the conduct experiment script. In addition, it exploits the new capability to waste less build time running the :record_results test in debug mode. --- .../multibody/cassie_benchmark/BUILD.bazel | 6 +++ examples/multibody/cassie_benchmark/README.md | 39 ++++++++++++++----- .../cassie_benchmark/conduct_experiment | 3 +- .../cassie_benchmark/record_results.sh | 1 + 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/examples/multibody/cassie_benchmark/BUILD.bazel b/examples/multibody/cassie_benchmark/BUILD.bazel index c13e2266875d..3d94adbde4bf 100644 --- a/examples/multibody/cassie_benchmark/BUILD.bazel +++ b/examples/multibody/cassie_benchmark/BUILD.bazel @@ -29,6 +29,12 @@ sh_test( name = "record_results", size = "small", srcs = ["record_results.sh"], + # Debug-configured test runs are nice for coverage, but not very useful + # otherwise. Don't waste too much time on them. + args = select({ + "//tools/cc_toolchain:debug": ["--benchmark_repetitions=1"], + "//conditions:default": [], + }), data = [ ":cassie_bench", "//tools/workspace/cc:identify_compiler", diff --git a/examples/multibody/cassie_benchmark/README.md b/examples/multibody/cassie_benchmark/README.md index 8ec96ff81412..efe04e3fa5d7 100644 --- a/examples/multibody/cassie_benchmark/README.md +++ b/examples/multibody/cassie_benchmark/README.md @@ -22,15 +22,19 @@ On the default supported platform, the following command will build code and save result data to a user supplied directory, under relatively controlled conditions. - $ examples/multibody/cassie_benchmark/conduct_experiment [DIRECTORY] + $ examples/multibody/cassie_benchmark/conduct_experiment [DIRECTORY] [ARGS...] -The conduct_experiment script will attempt to reduce result variance by -controlling cpu throttling and by setting a cpu affinity mask. +The `conduct_experiment` script will attempt to reduce result variance +by controlling cpu throttling and by setting a cpu affinity mask. It is still up to the user to make sure the machine is appropriate (not a virtual machine, for example), and relatively unloaded. Close as many running programs as is practical. +Optionally, it is possible to pass command line arguments through to the +`cassie_bench` executable. See 'Configuring details of benchmark runs' +below. + ## Comparing experiment data Since experiment results are highly machine dependent, any stored @@ -49,10 +53,10 @@ discussion of methods are being tracked in Github issue #13902. The following command will perform a benchmark experiment (with environment controls) and collect context information. It is not -confined to a specific platform, like :conduct_experiment, and it +confined to a specific platform, like `conduct_experiment`, and it controls fewer environment conditions. - $ bazel run //examples/multibody/cassie_benchmark:record_results + $ bazel run //examples/multibody/cassie_benchmark:record_results [-- [ARGS...]] For best results, some conditions will need to be manually controlled. These include: @@ -61,14 +65,18 @@ controlled. These include: * cpu throttling -- varies with platform * Linux: https://github.com/google/benchmark#disabling-cpu-frequency-scaling -The :record_results target does try to mitigate costs of rescheduling to -a different processor (Linux only), by setting a processor +The `:record_results` target does try to mitigate costs of rescheduling +to a different processor (Linux only), by setting a processor affinity. However, it does not do full-featured processor isolation, so it is still important to limit the number of running programs. As of this writing, the benchmark is assigned to processor #0; it may be worth monitoring the system to ensure that processor will be fully available to the experiment. +Optionally, it is possible to pass command line arguments through to the +`cassie_bench` executable. See 'Configuring details of benchmark runs' +below. + ## Results data files Bazel buries the results deep in its output tree. To copy them @@ -85,10 +93,21 @@ complete. Still missing: ## Configuring details of benchmark runs -To run individual benchmarks, alter output, etc., run the -benchmark program directly: +In addition to the above forms, it is possible run the benchmark program +directly: + + $ bazel run //examples/multibody/cassie_benchmark:cassie_bench [-- [ARGS...]] + +or + + $ bazel-bin/examples/multibody/cassie_benchmark/cassie_bench [ARGS...] + +if the `:cassie_bench` target is built. The direct forms may be useful +for combining debugging or profiling tools. - $ bazel run //examples/multibody/cassie_benchmark:cassie_bench -- [ARGS] +All of these forms take the same set of command line arguments, which +can select which cases are run, control iterations and repetitions, +alter output formatting, etc. Documentation for command line arguments is here: https://github.com/google/benchmark#command-line diff --git a/examples/multibody/cassie_benchmark/conduct_experiment b/examples/multibody/cassie_benchmark/conduct_experiment index 02c4640144f3..440786cfb5f3 100755 --- a/examples/multibody/cassie_benchmark/conduct_experiment +++ b/examples/multibody/cassie_benchmark/conduct_experiment @@ -61,6 +61,7 @@ clean () { say Validate input. [[ "$#" -ge 1 ]] || die "missing argument: destination directory" DESTINATION="$1" +shift say Validate environment. is_default_ubuntu || die "experiments only supported on default platform" @@ -88,7 +89,7 @@ set_cpu_governor performance set_no_turbo 1 say Run the experiment. -bazel run "$TARGET" +bazel run "$TARGET" -- "$@" say Save data. "$HERE"/copy_results_to "$DESTINATION" diff --git a/examples/multibody/cassie_benchmark/record_results.sh b/examples/multibody/cassie_benchmark/record_results.sh index 311b2ab22cd3..fbe7f6c7a0d9 100755 --- a/examples/multibody/cassie_benchmark/record_results.sh +++ b/examples/multibody/cassie_benchmark/record_results.sh @@ -35,6 +35,7 @@ ${TEST_SRCDIR}/drake/examples/multibody/cassie_benchmark/cassie_bench \ --benchmark_repetitions=9 \ --benchmark_out_format=json \ --benchmark_out=${TEST_UNDECLARED_OUTPUTS_DIR}/results.json \ + "$@" \ >& ${TEST_UNDECLARED_OUTPUTS_DIR}/summary.txt echo Full results are in: