Clean up benchmarks and add new example of how parameters should be d…

…ocumented
xjrc · Aug 22, 2019 · d3b8ca8 · d3b8ca8
1 parent 3884f38
commit d3b8ca8
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 92 deletions.
diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake
@@ -571,17 +571,18 @@ endmacro(blt_add_executable)
 
 
 ##------------------------------------------------------------------------------
-## blt_add_test( NAME          [name]
-##               COMMAND       [command] 
-##               NUM_MPI_TASKS [n] )
+## blt_add_test( NAME           [name]
+##               COMMAND        [command] 
+##               NUM_MPI_TASKS  [n]
+##               CONFIGURATIONS [config1 [config2...]])
 ##
-## Adds a CMake test to the project.
+## Adds a test to the project.
 ##------------------------------------------------------------------------------
 macro(blt_add_test)
 
     set(options )
     set(singleValueArgs NAME NUM_MPI_TASKS)
-    set(multiValueArgs COMMAND)
+    set(multiValueArgs COMMAND CONFIGURATIONS)
 
     # Parse the arguments to the macro
     cmake_parse_arguments(arg
@@ -634,62 +635,36 @@ macro(blt_add_test)
         set(test_command ${_mpiexec} ${MPIEXEC_NUMPROC_FLAG} ${arg_NUM_MPI_TASKS} ${BLT_MPI_COMMAND_APPEND} ${test_command} )
     endif()
 
-    add_test(NAME ${arg_NAME}
-             COMMAND ${test_command} )
+    add_test(NAME           ${arg_NAME}
+             COMMAND        ${test_command}
+             CONFIGURATIONS ${arg_CONFIGURATIONS})
 
 endmacro(blt_add_test)
 
 
 ##------------------------------------------------------------------------------
-## blt_add_benchmark( NAME [name] 
-##                    COMMAND [command])
+## blt_add_benchmark( NAME          [name] 
+##                    COMMAND       [command]
+##                    NUM_MPI_TASKS [n])
 ##
-## Adds a (google) benchmark test to the project.
+## Adds a benchmark to the project.
 ##------------------------------------------------------------------------------
 macro(blt_add_benchmark)
 
-   if(ENABLE_BENCHMARKS)
-
-      set(options)
-      set(singleValueArgs NAME)      
-      set(multiValueArgs COMMAND)
-
-      ## parse the arguments to the macro
-      cmake_parse_arguments(arg
-         "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN} )
-
-      if ( NOT DEFINED arg_NAME )
-          message(FATAL_ERROR "NAME is a required parameter to blt_add_benchmark")
-      endif()
-
-      if ( NOT DEFINED arg_COMMAND )
-          message(FATAL_ERROR "COMMAND is a required parameter to blt_add_benchmark")
-      endif()
-
-      # Generate command
-      if ( NOT TARGET ${arg_NAME} )
-          # Handle case of running multiple tests against one executable, 
-          # the NAME will not be the target
-          list(GET arg_COMMAND 0 executable)
-          get_target_property(runtime_output_directory ${executable} RUNTIME_OUTPUT_DIRECTORY )
-      else()
-          get_target_property(runtime_output_directory ${arg_NAME} RUNTIME_OUTPUT_DIRECTORY )
-      endif()
-      set(test_command ${runtime_output_directory}/${arg_COMMAND} )
-
-      # Note: No MPI handling for now.  If desired, see how this is handled in blt_add_test macro
-
-      # The 'CONFIGURATIONS Benchmark' line excludes benchmarks 
-      # from the general list of tests
-      add_test( NAME ${arg_NAME}
-                COMMAND ${test_command}
-                CONFIGURATIONS Benchmark   
-                )
-
-      if(ENABLE_TESTS)
-        add_dependencies(run_benchmarks ${arg_NAME})
-      endif()
-   endif()
+    set(options)
+    set(singleValueArgs NAME NUM_MPI_TASKS)      
+    set(multiValueArgs COMMAND)
+
+    ## parse the arguments to the macro
+    cmake_parse_arguments(arg
+     "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN} )
+
+    # The 'CONFIGURATIONS Benchmark' line excludes benchmarks 
+    # from the general list of tests
+    blt_add_test( NAME           ${arg_NAME}
+                  COMMAND        ${arg_COMMAND}
+                  NUM_MPI_TASKS  ${arg_NUM_MPI_TASKS}
+                  CONFIGURATIONS Benchmark)
 
 endmacro(blt_add_benchmark)
 

diff --git a/cmake/BLTOptions.cmake b/cmake/BLTOptions.cmake
@@ -64,6 +64,8 @@ endif()
 option(ENABLE_GTEST        "Enable Google Test testing support (if ENABLE_TESTS=ON)" ${_CXX_enabled})
 option(ENABLE_GMOCK        "Enable Google Mock testing support (if ENABLE_TESTS=ON)" OFF)
 option(ENABLE_FRUIT        "Enable Fruit testing support (if ENABLE_TESTS=ON and ENABLE_FORTRAN=ON)" ON)
+option(ENABLE_GBENCHMARK   "Enable Google Benchmark support (if ENABLE_TESTS=ON)" ${ENABLE_BENCHMARKS})
+
 
 if( (NOT _CXX_enabled) AND ENABLE_GTEST )
   message( FATAL_ERROR

diff --git a/docs/api/index.rst b/docs/api/index.rst
@@ -3,7 +3,7 @@
 .. # 
 .. # SPDX-License-Identifier: (BSD-3-Clause)
 
-API documentation
+API Documentation
 =================
 
 .. toctree::

diff --git a/docs/api/target.rst b/docs/api/target.rst
@@ -6,23 +6,39 @@
 Target Macros
 =============
 
+.. _blt_add_benchmark:
+
 blt_add_benchmark
 ~~~~~~~~~~~~~~~~~
 
 .. code-block:: cmake
 
-    blt_add_benchmark( NAME [name]
-                       COMMAND [command])
+    blt_add_benchmark( NAME          [name]
+                       COMMAND       [command]
+                       NUM_MPI_TASKS [n])
+
+Adds a benchmark to the project.
+
+NAME
+  Name that CTest reports.
 
-Adds a (google) benchmark test to the project.
+COMMAND
+  Command line that will be used to run the test and can include arguments.  
 
-NAME is used for the name that CTest reports and should include the string 'benchmark'.
+NUM_MPI_TASKS
+  Indicates this is an MPI test and how many MPI tasks to use.
 
-COMMAND is the command line that will be used to run the test and can include arguments.  
+This macro adds the benchmark to the ``run_benchmarks`` build target.
+The underlying executable, previously added with :ref:`blt_add_executable`,
+should include necessary benchmarking library in its DEPENDS_ON list.
 
-This will have the RUNTIME_OUTPUT_DIRECTORY prepended to it to fully qualify the path.
-The underlying executable (added with blt_add_executable) should include gbenchmark
-as one of its dependencies.
+BLT provides a built-in Google Benchmark that is enabled by default if you set
+ENABLE_BENCHMARKS=ON and can be turned off with the option ENABLE_GBENCHMARK.
+
+.. note::
+  This is just a thin wrapper around :ref:`blt_add_test` that sets the CTest configuration
+  to "Benchmark" to filter these benchmarks out of the ``test`` build target. It also assists
+  with building up the correct command line.  For more info see :ref:`blt_add_test`.
 
 .. code-block:: cmake
     :caption: **Example**
@@ -36,6 +52,8 @@ as one of its dependencies.
          COMMAND component_benchmark "--benchmark_min_time=0.0 --v=3 --benchmark_format=json")
 
 
+.. _blt_add_executable:
+
 blt_add_executable
 ~~~~~~~~~~~~~~~~~~
 
@@ -72,6 +90,8 @@ This is available when ENABLE_FOLDERS is ON and when using a cmake generator
 that supports this feature and will otherwise be ignored.
 
 
+.. _blt_add_library:
+
 blt_add_library
 ~~~~~~~~~~~~~~~
 
@@ -138,30 +158,60 @@ Note: Do not use with header-only (INTERFACE) libraries, as this will generate
 a CMake configuration error.
 
 
+.. _blt_add_test:
+
 blt_add_test
 ~~~~~~~~~~~~
 
 .. code-block:: cmake
 
-    blt_add_test( NAME          [name]
-                  COMMAND       [command] 
-                  NUM_MPI_TASKS [n])
+    blt_add_test( NAME           [name]
+                  COMMAND        [command]
+                  NUM_MPI_TASKS  [n]
+                  CONFIGURATIONS [config1 [config2...]])
+
+Adds a test to the project.
+
+NAME
+  Name that CTest reports.
 
-Adds a CMake test to the project.
+COMMAND
+  Command line that will be used to run the test and can include arguments.
 
-NAME is used for the name that CTest reports with.
+NUM_MPI_TASKS
+  Indicates this is an MPI test and how many MPI tasks to use.
 
-COMMAND is the command line that will be used to run the test. This will
-have the RUNTIME_OUTPUT_DIRECTORY prepended to it to fully qualify the path.
+CONFIGURATIONS
+  Set the CTest configuration for this test.  Do not specify if you want the
+  test to run every ``test`` build target.
 
-NUM_MPI_TASKS indicates this is an MPI test and how many tasks to use. The
-command line will use MPIEXEC, MPIEXEC_NUMPROC_FLAG, and BLT_MPI_COMMAND_APPEND
-to create the MPI run line.
+This macro adds the named test to CTest and the build target ``test``. This macro
+does not build the executable and requires a call to :ref:`blt_add_executable` first.
+
+This macro assists with building up the correct command line. It will prepend
+the RUNTIME_OUTPUT_DIRECTORY target property to the executable. If NUM_MPI_TASKS
+is given, the macro will appropiately use MPIEXEC, MPIEXEC_NUMPROC_FLAG, and 
+BLT_MPI_COMMAND_APPEND to create the MPI run line.
 
 MPIEXEC and MPIEXEC_NUMPROC_FLAG are filled in by CMake's FindMPI.cmake but can
 be overwritten in your host-config specific to your platform. BLT_MPI_COMMAND_APPEND
 is useful on machines that require extra arguments to MPIEXEC.
 
+.. note::
+  If you do not want the command line assistance, for example you already have a script
+  you wish to run as a test, then just call CMake's ``add_test()``.
+
+.. code-block:: cmake
+    :caption: **Example**
+    :linenos:
+
+    blt_add_executable(NAME    my_test
+                       SOURCES my_test.cpp)
+    blt_add_test(NAME    my_test
+                 COMMAND my_test --with-some-argument)
+
+
+.. _blt_register_library:
 
 blt_register_library
 ~~~~~~~~~~~~~~~~~~~~

diff --git a/docs/index.rst b/docs/index.rst
@@ -66,4 +66,4 @@ Documentation
     :maxdepth: 2
 
     User Tutorial <tutorial/index>
-    API Documenation <api/index>
+    API Documentation <api/index>
diff --git a/thirdparty_builtin/CMakeLists.txt b/thirdparty_builtin/CMakeLists.txt
@@ -121,33 +121,36 @@ if(ENABLE_TESTS)
 endif()
 
 if(ENABLE_BENCHMARKS)
-    if(WIN32 AND BUILD_SHARED_LIBS)
-      message(FATAL_ERROR "Benchmarks cannot be built when BUILD_SHARED_LIBS=On")
+    if(NOT ENABLE_TESTS)
+        message(FATAL_ERROR "ENABLE_BENCHMARKS requires ENABLE_TESTS to be ON")
     endif()
 
-    ## google benchmark support
-    add_subdirectory(gbenchmark-master-2017-05-19
-                     ${BLT_BUILD_DIR}/thirdparty_builtin/gbenchmark-master-2017-05-19)
+    message(STATUS "Google Benchmark Support is ${ENABLE_GBENCHMARK}")
 
-    if (UNIX AND NOT APPLE)
-      find_library(RT_LIBRARIES rt)
-    endif()
+    if(ENABLE_GBENCHMARK)
+        if(WIN32 AND BUILD_SHARED_LIBS)
+          message(FATAL_ERROR "Google Benchmark cannot be built when BUILD_SHARED_LIBS=ON or on Windows")
+        endif()
 
-    blt_register_library(NAME gbenchmark
-                         INCLUDES ${benchmark_SOURCE_DIR}/include ${benchmark_SOURCE_DIR}
-                         LIBRARIES benchmark ${RT_LIBRARIES}
-                         TREAT_INCLUDES_AS_SYSTEM ON
-                         )
+        add_subdirectory(gbenchmark-master-2017-05-19
+                         ${BLT_BUILD_DIR}/thirdparty_builtin/gbenchmark-master-2017-05-19)
 
-    list(APPEND _blt_tpl_targets benchmark)
+        if (UNIX AND NOT APPLE)
+          find_library(RT_LIBRARIES rt)
+        endif()
+
+        blt_register_library(NAME      gbenchmark
+                             INCLUDES  ${benchmark_SOURCE_DIR}/include ${benchmark_SOURCE_DIR}
+                             LIBRARIES benchmark ${RT_LIBRARIES}
+                             TREAT_INCLUDES_AS_SYSTEM ON)
 
-    if(ENABLE_TESTS)
-      # This sets up a target to run the benchmarks
-      add_custom_target(run_benchmarks 
-                        COMMAND ctest -C Benchmark -VV -R benchmark
-                        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-                        )
+        list(APPEND _blt_tpl_targets gbenchmark)
     endif()
+
+    # This sets up a target to run the benchmarks
+    add_custom_target(run_benchmarks
+                      COMMAND ctest -C Benchmark -VV
+                      WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
 endif()
 
 # Set the folder property of the blt thirdparty libraries