From d3b8ca8fae917e0a6d9ee7bbf795f433ebf8a117 Mon Sep 17 00:00:00 2001 From: Chris White Date: Wed, 21 Aug 2019 15:48:38 -0700 Subject: [PATCH] Clean up benchmarks and add new example of how parameters should be documented --- cmake/BLTMacros.cmake | 79 ++++++++++------------------ cmake/BLTOptions.cmake | 2 + docs/api/index.rst | 2 +- docs/api/target.rst | 86 ++++++++++++++++++++++++------- docs/index.rst | 2 +- thirdparty_builtin/CMakeLists.txt | 43 +++++++++------- 6 files changed, 122 insertions(+), 92 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index e95c01e7f..579efe88b 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -571,17 +571,18 @@ endmacro(blt_add_executable) ##------------------------------------------------------------------------------ -## blt_add_test( NAME [name] -## COMMAND [command] -## NUM_MPI_TASKS [n] ) +## blt_add_test( NAME [name] +## COMMAND [command] +## NUM_MPI_TASKS [n] +## CONFIGURATIONS [config1 [config2...]]) ## -## Adds a CMake test to the project. +## Adds a test to the project. ##------------------------------------------------------------------------------ macro(blt_add_test) set(options ) set(singleValueArgs NAME NUM_MPI_TASKS) - set(multiValueArgs COMMAND) + set(multiValueArgs COMMAND CONFIGURATIONS) # Parse the arguments to the macro cmake_parse_arguments(arg @@ -634,62 +635,36 @@ macro(blt_add_test) set(test_command ${_mpiexec} ${MPIEXEC_NUMPROC_FLAG} ${arg_NUM_MPI_TASKS} ${BLT_MPI_COMMAND_APPEND} ${test_command} ) endif() - add_test(NAME ${arg_NAME} - COMMAND ${test_command} ) + add_test(NAME ${arg_NAME} + COMMAND ${test_command} + CONFIGURATIONS ${arg_CONFIGURATIONS}) endmacro(blt_add_test) ##------------------------------------------------------------------------------ -## blt_add_benchmark( NAME [name] -## COMMAND [command]) +## blt_add_benchmark( NAME [name] +## COMMAND [command] +## NUM_MPI_TASKS [n]) ## -## Adds a (google) benchmark test to the project. +## Adds a benchmark to the project. ##------------------------------------------------------------------------------ macro(blt_add_benchmark) - if(ENABLE_BENCHMARKS) - - set(options) - set(singleValueArgs NAME) - set(multiValueArgs COMMAND) - - ## parse the arguments to the macro - cmake_parse_arguments(arg - "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN} ) - - if ( NOT DEFINED arg_NAME ) - message(FATAL_ERROR "NAME is a required parameter to blt_add_benchmark") - endif() - - if ( NOT DEFINED arg_COMMAND ) - message(FATAL_ERROR "COMMAND is a required parameter to blt_add_benchmark") - endif() - - # Generate command - if ( NOT TARGET ${arg_NAME} ) - # Handle case of running multiple tests against one executable, - # the NAME will not be the target - list(GET arg_COMMAND 0 executable) - get_target_property(runtime_output_directory ${executable} RUNTIME_OUTPUT_DIRECTORY ) - else() - get_target_property(runtime_output_directory ${arg_NAME} RUNTIME_OUTPUT_DIRECTORY ) - endif() - set(test_command ${runtime_output_directory}/${arg_COMMAND} ) - - # Note: No MPI handling for now. If desired, see how this is handled in blt_add_test macro - - # The 'CONFIGURATIONS Benchmark' line excludes benchmarks - # from the general list of tests - add_test( NAME ${arg_NAME} - COMMAND ${test_command} - CONFIGURATIONS Benchmark - ) - - if(ENABLE_TESTS) - add_dependencies(run_benchmarks ${arg_NAME}) - endif() - endif() + set(options) + set(singleValueArgs NAME NUM_MPI_TASKS) + set(multiValueArgs COMMAND) + + ## parse the arguments to the macro + cmake_parse_arguments(arg + "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN} ) + + # The 'CONFIGURATIONS Benchmark' line excludes benchmarks + # from the general list of tests + blt_add_test( NAME ${arg_NAME} + COMMAND ${arg_COMMAND} + NUM_MPI_TASKS ${arg_NUM_MPI_TASKS} + CONFIGURATIONS Benchmark) endmacro(blt_add_benchmark) diff --git a/cmake/BLTOptions.cmake b/cmake/BLTOptions.cmake index 52b93c46d..d215e08d5 100644 --- a/cmake/BLTOptions.cmake +++ b/cmake/BLTOptions.cmake @@ -64,6 +64,8 @@ endif() option(ENABLE_GTEST "Enable Google Test testing support (if ENABLE_TESTS=ON)" ${_CXX_enabled}) option(ENABLE_GMOCK "Enable Google Mock testing support (if ENABLE_TESTS=ON)" OFF) option(ENABLE_FRUIT "Enable Fruit testing support (if ENABLE_TESTS=ON and ENABLE_FORTRAN=ON)" ON) +option(ENABLE_GBENCHMARK "Enable Google Benchmark support (if ENABLE_TESTS=ON)" ${ENABLE_BENCHMARKS}) + if( (NOT _CXX_enabled) AND ENABLE_GTEST ) message( FATAL_ERROR diff --git a/docs/api/index.rst b/docs/api/index.rst index c6731e468..cee8969f4 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -3,7 +3,7 @@ .. # .. # SPDX-License-Identifier: (BSD-3-Clause) -API documentation +API Documentation ================= .. toctree:: diff --git a/docs/api/target.rst b/docs/api/target.rst index 69e7b0bed..997cea4dd 100644 --- a/docs/api/target.rst +++ b/docs/api/target.rst @@ -6,23 +6,39 @@ Target Macros ============= +.. _blt_add_benchmark: + blt_add_benchmark ~~~~~~~~~~~~~~~~~ .. code-block:: cmake - blt_add_benchmark( NAME [name] - COMMAND [command]) + blt_add_benchmark( NAME [name] + COMMAND [command] + NUM_MPI_TASKS [n]) + +Adds a benchmark to the project. + +NAME + Name that CTest reports. -Adds a (google) benchmark test to the project. +COMMAND + Command line that will be used to run the test and can include arguments. -NAME is used for the name that CTest reports and should include the string 'benchmark'. +NUM_MPI_TASKS + Indicates this is an MPI test and how many MPI tasks to use. -COMMAND is the command line that will be used to run the test and can include arguments. +This macro adds the benchmark to the ``run_benchmarks`` build target. +The underlying executable, previously added with :ref:`blt_add_executable`, +should include necessary benchmarking library in its DEPENDS_ON list. -This will have the RUNTIME_OUTPUT_DIRECTORY prepended to it to fully qualify the path. -The underlying executable (added with blt_add_executable) should include gbenchmark -as one of its dependencies. +BLT provides a built-in Google Benchmark that is enabled by default if you set +ENABLE_BENCHMARKS=ON and can be turned off with the option ENABLE_GBENCHMARK. + +.. note:: + This is just a thin wrapper around :ref:`blt_add_test` that sets the CTest configuration + to "Benchmark" to filter these benchmarks out of the ``test`` build target. It also assists + with building up the correct command line. For more info see :ref:`blt_add_test`. .. code-block:: cmake :caption: **Example** @@ -36,6 +52,8 @@ as one of its dependencies. COMMAND component_benchmark "--benchmark_min_time=0.0 --v=3 --benchmark_format=json") +.. _blt_add_executable: + blt_add_executable ~~~~~~~~~~~~~~~~~~ @@ -72,6 +90,8 @@ This is available when ENABLE_FOLDERS is ON and when using a cmake generator that supports this feature and will otherwise be ignored. +.. _blt_add_library: + blt_add_library ~~~~~~~~~~~~~~~ @@ -138,30 +158,60 @@ Note: Do not use with header-only (INTERFACE) libraries, as this will generate a CMake configuration error. +.. _blt_add_test: + blt_add_test ~~~~~~~~~~~~ .. code-block:: cmake - blt_add_test( NAME [name] - COMMAND [command] - NUM_MPI_TASKS [n]) + blt_add_test( NAME [name] + COMMAND [command] + NUM_MPI_TASKS [n] + CONFIGURATIONS [config1 [config2...]]) + +Adds a test to the project. + +NAME + Name that CTest reports. -Adds a CMake test to the project. +COMMAND + Command line that will be used to run the test and can include arguments. -NAME is used for the name that CTest reports with. +NUM_MPI_TASKS + Indicates this is an MPI test and how many MPI tasks to use. -COMMAND is the command line that will be used to run the test. This will -have the RUNTIME_OUTPUT_DIRECTORY prepended to it to fully qualify the path. +CONFIGURATIONS + Set the CTest configuration for this test. Do not specify if you want the + test to run every ``test`` build target. -NUM_MPI_TASKS indicates this is an MPI test and how many tasks to use. The -command line will use MPIEXEC, MPIEXEC_NUMPROC_FLAG, and BLT_MPI_COMMAND_APPEND -to create the MPI run line. +This macro adds the named test to CTest and the build target ``test``. This macro +does not build the executable and requires a call to :ref:`blt_add_executable` first. + +This macro assists with building up the correct command line. It will prepend +the RUNTIME_OUTPUT_DIRECTORY target property to the executable. If NUM_MPI_TASKS +is given, the macro will appropiately use MPIEXEC, MPIEXEC_NUMPROC_FLAG, and +BLT_MPI_COMMAND_APPEND to create the MPI run line. MPIEXEC and MPIEXEC_NUMPROC_FLAG are filled in by CMake's FindMPI.cmake but can be overwritten in your host-config specific to your platform. BLT_MPI_COMMAND_APPEND is useful on machines that require extra arguments to MPIEXEC. +.. note:: + If you do not want the command line assistance, for example you already have a script + you wish to run as a test, then just call CMake's ``add_test()``. + +.. code-block:: cmake + :caption: **Example** + :linenos: + + blt_add_executable(NAME my_test + SOURCES my_test.cpp) + blt_add_test(NAME my_test + COMMAND my_test --with-some-argument) + + +.. _blt_register_library: blt_register_library ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/index.rst b/docs/index.rst index de46bd956..aad69cf40 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -66,4 +66,4 @@ Documentation :maxdepth: 2 User Tutorial - API Documenation + API Documentation diff --git a/thirdparty_builtin/CMakeLists.txt b/thirdparty_builtin/CMakeLists.txt index c92c38bbc..926e9115f 100644 --- a/thirdparty_builtin/CMakeLists.txt +++ b/thirdparty_builtin/CMakeLists.txt @@ -121,33 +121,36 @@ if(ENABLE_TESTS) endif() if(ENABLE_BENCHMARKS) - if(WIN32 AND BUILD_SHARED_LIBS) - message(FATAL_ERROR "Benchmarks cannot be built when BUILD_SHARED_LIBS=On") + if(NOT ENABLE_TESTS) + message(FATAL_ERROR "ENABLE_BENCHMARKS requires ENABLE_TESTS to be ON") endif() - ## google benchmark support - add_subdirectory(gbenchmark-master-2017-05-19 - ${BLT_BUILD_DIR}/thirdparty_builtin/gbenchmark-master-2017-05-19) + message(STATUS "Google Benchmark Support is ${ENABLE_GBENCHMARK}") - if (UNIX AND NOT APPLE) - find_library(RT_LIBRARIES rt) - endif() + if(ENABLE_GBENCHMARK) + if(WIN32 AND BUILD_SHARED_LIBS) + message(FATAL_ERROR "Google Benchmark cannot be built when BUILD_SHARED_LIBS=ON or on Windows") + endif() - blt_register_library(NAME gbenchmark - INCLUDES ${benchmark_SOURCE_DIR}/include ${benchmark_SOURCE_DIR} - LIBRARIES benchmark ${RT_LIBRARIES} - TREAT_INCLUDES_AS_SYSTEM ON - ) + add_subdirectory(gbenchmark-master-2017-05-19 + ${BLT_BUILD_DIR}/thirdparty_builtin/gbenchmark-master-2017-05-19) - list(APPEND _blt_tpl_targets benchmark) + if (UNIX AND NOT APPLE) + find_library(RT_LIBRARIES rt) + endif() + + blt_register_library(NAME gbenchmark + INCLUDES ${benchmark_SOURCE_DIR}/include ${benchmark_SOURCE_DIR} + LIBRARIES benchmark ${RT_LIBRARIES} + TREAT_INCLUDES_AS_SYSTEM ON) - if(ENABLE_TESTS) - # This sets up a target to run the benchmarks - add_custom_target(run_benchmarks - COMMAND ctest -C Benchmark -VV -R benchmark - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - ) + list(APPEND _blt_tpl_targets gbenchmark) endif() + + # This sets up a target to run the benchmarks + add_custom_target(run_benchmarks + COMMAND ctest -C Benchmark -VV + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) endif() # Set the folder property of the blt thirdparty libraries