Add BufferedRegionProfile for Fortran and C (LLNL#268)

* Add a Fortran RegionProfile wrapper

* Add cali_begin_region() / cali_end_region()

* Example updates

* Enable Fortran in gitlab configs

* Skip Fortran option with xlc for now

* Add test for BufferedRegionProfile

cmake/hostconfig/gitlab-butte-xlc.cmake

@@ -1,17 +1,15 @@
 set(CMAKE_C_COMPILER   "/usr/tce/packages/xl/xl-2019.02.07/bin/xlc" CACHE PATH "")
 set(CMAKE_CXX_COMPILER "/usr/tce/packages/xl/xl-2019.02.07/bin/xlC" CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "/usr/tce/packages/xl/xl-2019.02.07/bin/xlf" CACHE PATH "")
 
 # set(MPI_C_COMPILER     "/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2019.02.07/bin/mpicc" CACHE PATH "")
 # set(MPI_CXX_COMPILER   "/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2019.02.07/bin/mpicxx" CACHE PATH "")
 
 set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-9.2.148" CACHE PATH "")
 set(CUPTI_PREFIX "/usr/tce/packages/cuda/cuda-9.2.148/extras/CUPTI" CACHE PATH "")
 
-# DBO 2019-02-28: dyninst-10/boost installation on toss3 is broken, leading to
-# undefined references or non-existing include files. Turn it off for now.
-# Build with spack if dyninst is needed.
-#set(CMAKE_PREFIX_PATH "/usr/tce/packages/dyninst/dyninst-10.0.0/lib/cmake" CACHE PATH "")
-
+# Our f2003 requires some extra options for xlf. Skip it for now. 
+set(WITH_FORTRAN  Off CACHE BOOL "")
 set(WITH_CALLPATH Off CACHE BOOL "")
 set(WITH_NVPROF   On  CACHE BOOL "")
 set(WITH_CUPTI    On  CACHE BOOL "")

cmake/hostconfig/gitlab-quartz-gcc.cmake

@@ -1,5 +1,6 @@
 set(CMAKE_C_COMPILER   "/usr/tce/packages/gcc/gcc-8.1.0/bin/gcc" CACHE PATH "")
 set(CMAKE_CXX_COMPILER "/usr/tce/packages/gcc/gcc-8.1.0/bin/g++" CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran" CACHE PATH "")
 
 set(CMAKE_BUILD_TYPE   "RelWithDebInfo" CACHE STRING "")
 
@@ -16,6 +17,7 @@ set(adiak_DIR "/g/g90/boehme3/local/adiak/toss3-0.1.1/lib/cmake/adiak" CACHE PAT
 # Build with spack if dyninst is needed.
 #set(CMAKE_PREFIX_PATH "/usr/tce/packages/dyninst/dyninst-10.0.0/lib/cmake" CACHE PATH "")
 
+set(WITH_FORTRAN  On  CACHE BOOL "")
 set(WITH_ADIAK    On  CACHE BOOL "")
 set(WITH_CALLPATH On  CACHE BOOL "")
 set(WITH_NVPROF   Off CACHE BOOL "")

cmake/hostconfig/gitlab-quartz-intel.cmake

@@ -1,5 +1,6 @@
 set(CMAKE_C_COMPILER   "/usr/tce/packages/intel/intel-19.0.4/bin/icc" CACHE PATH "")
 set(CMAKE_CXX_COMPILER "/usr/tce/packages/intel/intel-19.0.4/bin/icpc" CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "/usr/tce/packages/intel/intel-19.0.4/bin/ifort" CACHE PATH "")
 
 set(MPI_C_COMPILER     "/usr/tce/packages/mvapich2/mvapich2-2.3-intel-19.0.4/bin/mpicc" CACHE PATH "")
 set(MPI_CXX_COMPILER   "/usr/tce/packages/mvapich2/mvapich2-2.3-intel-19.0.4/bin/mpicxx" CACHE PATH "")
@@ -14,6 +15,7 @@ set(adiak_DIR "/g/g90/boehme3/local/adiak/toss3-0.1.1/lib/cmake/adiak" CACHE PAT
 # Build with spack if dyninst is needed.
 #set(CMAKE_PREFIX_PATH "/usr/tce/packages/dyninst/dyninst-10.0.0/lib/cmake" CACHE PATH "")
 
+set(WITH_FORTRAN  On  CACHE BOOL "")
 set(WITH_ADIAK    On  CACHE BOOL "")
 set(WITH_CALLPATH On  CACHE BOOL "")
 set(WITH_NVPROF   Off CACHE BOOL "")

cmake/hostconfig/gitlab-quartz-pgi.cmake

@@ -1,5 +1,6 @@
 set(CMAKE_C_COMPILER   "/usr/tce/packages/pgi/pgi-19.7/bin/pgcc" CACHE PATH "")
 set(CMAKE_CXX_COMPILER "/usr/tce/packages/pgi/pgi-19.7/bin/pgc++" CACHE PATH "")
+set(CMAKE_CXX_COMPILER "/usr/tce/packages/pgi/pgi-19.7/bin/pgfortran" CACHE PATH "")
 
 set(CMAKE_CXX_FLAGS    "-std=c++11" CACHE STRING "")
 
@@ -13,7 +14,8 @@ set(ITT_PREFIX  "/usr/tce/packages/vtune/default" CACHE PATH "")
 # undefined references or non-existing include files. Turn it off for now.
 # Build with spack if dyninst is needed.
 #set(CMAKE_PREFIX_PATH "/usr/tce/packages/dyninst/dyninst-10.0.0/lib/cmake" CACHE PATH "")
-
+
+set(WITH_FORTRAN  On  CACHE BOOL "")
 set(WITH_CALLPATH On  CACHE BOOL "")
 set(WITH_NVPROF   Off CACHE BOOL "")
 set(WITH_CUPTI    Off CACHE BOOL "")

examples/apps/CMakeLists.txt

@@ -23,7 +23,8 @@ set(CALIPER_C_EXAMPLE_APPS
   cali-basic-annotations-c
   cali-print-snapshot)
 set(CALIPER_Fortran_EXAMPLE_APPS
-  fortran-example)
+  fortran-example
+  fortran-regionprofile)
 
 foreach(app ${CALIPER_CXX_EXAMPLE_APPS})
   add_executable(${app} ${app}.cpp)
@@ -40,7 +41,6 @@ if (WITH_FORTRAN)
   foreach(app ${CALIPER_Fortran_EXAMPLE_APPS})
     add_executable(${app} ${app}.f)
     set_source_files_properties(${app}.f PROPERTIES Fortran_FORMAT FREE)
-    set_target_properties(${app} PROPERTIES LINKER_LANGUAGE CXX)
     target_include_directories(${app} PRIVATE ${CMAKE_Fortran_MODULE_DIRECTORY})
     target_link_libraries(${app} caliper)
   endforeach()

examples/apps/fortran-example.f

@@ -3,7 +3,6 @@ program fortran_example
 
     implicit none
 
-    type(ScopeAnnotation) :: m_ann, i_ann
     type(ConfigManager)   :: mgr
 
     integer               :: i, count, argc
@@ -34,16 +33,16 @@ program fortran_example
     call mgr%start
 
     ! A scope annotation. Start region 'main'
-    m_ann = ScopeAnnotation_begin('main')
+    call cali_begin_region('main')
 
     ! Add another region 'inner' nested under 'main'
-    i_ann = ScopeAnnotation_begin('inner')
+    call cali_begin_region('inner')
     count = 4
     ! End the inner region
-    call ScopeAnnotation_end(i_ann)
+    call cali_end_region('inner')
 
     ! End 'main'
-    call ScopeAnnotation_end(m_ann)
+    call cali_end_region('main')
 
     ! Compute and flush output for the ConfigManager profiles.
     call mgr%flush

examples/apps/fortran-regionprofile.f

@@ -0,0 +1,78 @@
+program fortran_example
+    use caliper_mod
+
+    implicit none
+
+    type(BufferedRegionProfile) :: rp
+    type(ConfigManager)   :: mgr
+
+    integer               :: i, count, argc
+    real                  :: innertime, outertime, tottime
+
+    logical               :: ret
+    character(len=:), allocatable :: errmsg
+    character(len=256)    :: arg
+
+    ! Initialize Caliper. Use cali_mpi_init() in an MPI program.
+    call cali_init()
+
+    ! (Optional) create a ConfigManager object to control profiling.
+    ! Users can provide a configuration string (e.g., 'runtime-report')
+    ! on the command line.
+    mgr = ConfigManager_new()
+    argc = command_argument_count()
+    if (argc .ge. 1) then
+        call get_command_argument(1, arg)
+        call mgr%add(arg)
+        ret = mgr%error()
+        if (ret) then
+            errmsg = mgr%error_msg()
+            write(*,*) 'ConfigManager: ', errmsg
+        endif
+    endif
+
+    ! Start configured profiling channels
+    call mgr%start
+
+    ! A scope annotation. Start region 'main'
+    call cali_begin_region('main')
+
+    ! Create a BufferedRegionProfile instance to query Caliper region times
+    rp = BufferedRegionProfile_new()
+
+    do i = 1, 4
+        ! Start the region profile
+        call rp%start()
+
+        ! Add region 'outer' using cali_begin_region()
+        call cali_begin_region('outer')
+        ! Add another region 'inner' nested under 'outer'
+        call cali_begin_region('inner')
+
+        call cali_end_region('inner')
+        call cali_end_region('outer')
+
+        ! Stop the region profile and fetch region times 
+        call rp%stop
+        call rp%fetch_inclusive_region_times
+        innertime = rp%region_time('inner')
+        outertime = rp%region_time('outer')
+        tottime   = rp%total_region_time()
+
+        write(*,*) 'Cycle ', i, ': ', tottime, ' sec'
+        write(*,*) 'outer:   ', outertime, ' sec'
+        write(*,*) '  inner: ', innertime, ' sec'
+
+        ! Reset the profile
+        call rp%clear
+    end do
+
+    ! Delete the RegionProfile instance
+    call BufferedRegionProfile_delete(rp)
+
+    call cali_end_region('main')
+
+    ! Compute and flush output for the ConfigManager profiles.
+    call mgr%flush
+    call ConfigManager_delete(mgr)
+end program fortran_example

include/caliper/RegionProfile.h

@@ -44,6 +44,8 @@ class RegionProfile : public ChannelController
     /// Note that profiling must be started explicitly with the start() method.
     RegionProfile();
 
+    virtual ~RegionProfile();
+
     /// \brief A tuple containing the computed time profiles.
     /// 
     /// The first member is a string -> double STL map that stores the times 

include/caliper/cali.h

@@ -335,6 +335,29 @@ cali_channel_get(cali_id_t chn_id, cali_id_t attr_id);
  * \{
  */
 
+/**
+ * \brief Begin nested region \a name
+ *
+ * Begins nested region \a name using the built-in \a annotation attribute.
+ * Equivalent to the macro CALI_MARK_REGION_BEGIN.
+ *
+ * \see cali_end_region()
+ */
+void
+cali_begin_region(const char* name);
+
+/**
+ * \brief End nested region \a name
+ *
+ * Ends nested region \a name using the built-in \a annotation attribute.
+ * Prints an error if \a name does not match the currently open region.
+ * Equivalent to the macro CALI_MARK_REGION_END.
+ *
+ * \see cali_begin_region()
+ */
+void
+cali_end_region(const char* name);
+
 /**
  * \brief Begin region where the value for \a attr is `true` on the blackboard.
  */
@@ -394,6 +417,8 @@ cali_set_string(cali_id_t attr, const char* val);
 /**
  * \brief  Begin region where the value for the attribute named \a attr_name
  *   is set to `true` on the blackboard.
+ *
+ * \deprecated Use cali_begin_region()
  */
 void
 cali_begin_byname(const char* attr_name);

include/caliper/cali_macros.h

@@ -195,9 +195,7 @@ extern cali_id_t cali_annotation_attr_id;
 /// \param name The region name. Must be convertible to `const char*`.
 /// \sa CALI_MARK_END
 #define CALI_MARK_BEGIN(name) \
-    if (cali_annotation_attr_id == CALI_INV_ID) \
-        cali_init(); \
-    cali_begin_string(cali_annotation_attr_id, (name))
+    cali_begin_region(name)
 
 /// \brief Mark end of a user-defined code region.
 ///
@@ -209,7 +207,7 @@ extern cali_id_t cali_annotation_attr_id;
 ///   if it doesn't.
 /// \sa CALI_MARK_BEGIN
 #define CALI_MARK_END(name) \
-    cali_safe_end_string(cali_annotation_attr_id, (name))
+    cali_end_region(name)
 
 /**
  * \} (group)

src/caliper/RegionProfile.cpp

@@ -36,6 +36,8 @@ RegionProfile::RegionProfile()
     , mP { new RegionProfileImpl }
 { }
 
+RegionProfile::~RegionProfile()
+{ }
 
 RegionProfile::region_profile_t
 RegionProfile::exclusive_region_times(const std::string& region_type)

src/caliper/cali.cpp

@@ -26,6 +26,13 @@
 
 #define SNAP_MAX 120
 
+namespace cali
+{
+
+extern Attribute annotation_attr;
+
+}
+
 using namespace cali;
 
 //
@@ -356,6 +363,28 @@ cali_channel_get(cali_id_t chn_id, cali_id_t attr_id)
 // --- Annotation interface
 //
 
+void
+cali_begin_region(const char* name)
+{
+    Caliper c;
+    c.begin(cali::annotation_attr, Variant(name));
+}
+
+void
+cali_end_region(const char* name)
+{
+    Caliper c;
+    Variant v_n(name);
+    Variant v_s = c.get(cali::annotation_attr).value();
+
+    if (!(v_n == v_s))
+        Log(0).stream() << "region nesting error: trying to end \"" << v_n
+                        << "\" but current region is \""            << v_s << "\""
+                        << std::endl;
+
+    c.end(cali::annotation_attr);
+}
+
 void
 cali_begin(cali_id_t attr_id)
 {
@@ -653,7 +682,7 @@ struct emplace_helper<
            decltype(std::declval<Container>().emplace(std::make_pair("","")))
        >::value
        , void
-   >::type 
+   >::type
 > {
     template<typename Emplaced>
     static void emplace(Container& emplace_into, Emplaced&& object){

src/caliper/test/CMakeLists.txt

@@ -2,6 +2,7 @@ set(CALIPER_TEST_SOURCES
   test_attribute.cpp
   test_blackboard.cpp
   test_c_api.cpp
+  test_c_wrapper.cpp
   test_channel_api.cpp
   test_channel_controller.cpp
   test_configmanager.cpp

src/caliper/test/test_attribute.cpp

@@ -143,4 +143,8 @@ TEST(AttributeAPITest, NestedAttribute) {
     node = node->parent();
     ASSERT_NE(node, nullptr);
     EXPECT_EQ(node->attribute(), CALI_INV_ID);
+
+    c.end(nested_b);
+    c.end(nomerge);
+    c.end(nested_a);
 }

src/caliper/test/test_c_wrapper.cpp

@@ -0,0 +1,55 @@
+#include "caliper/caliper-config.h"
+
+#include "../../interface/c_fortran/wrapBufferedRegionProfile.h"
+
+#include "caliper/cali.h"
+
+#define _XOPEN_SOURCE
+#include <unistd.h>
+
+#include <gtest/gtest.h>
+
+TEST(C_Wrapper, BufferedRegionProfile) {
+    cali_BufferedRegionProfile rp;
+    cali_BufferedRegionProfile_new(&rp);
+
+    cali_BufferedRegionProfile_start(&rp);
+
+    CALI_MARK_BEGIN("wrap.rp.outer");
+    CALI_MARK_BEGIN("wrap.rp.inner");
+    usleep(20000);
+    CALI_MARK_END("wrap.rp.inner");
+    usleep(10000);
+    CALI_MARK_END("wrap.rp.outer");
+
+    cali_BufferedRegionProfile_stop(&rp);
+
+    cali_BufferedRegionProfile_fetch_exclusive_region_times(&rp);
+    double e_tot = cali_BufferedRegionProfile_total_profiling_time(&rp);
+    double e_reg = cali_BufferedRegionProfile_total_region_time(&rp);
+    double e_out = cali_BufferedRegionProfile_region_time(&rp, "wrap.rp.outer");
+    double e_inn = cali_BufferedRegionProfile_region_time(&rp, "wrap.rp.inner");
+
+    cali_BufferedRegionProfile_fetch_inclusive_region_times(&rp);
+    double i_tot = cali_BufferedRegionProfile_total_profiling_time(&rp);
+    double i_reg = cali_BufferedRegionProfile_total_region_time(&rp);
+    double i_out = cali_BufferedRegionProfile_region_time(&rp, "wrap.rp.outer");
+    double i_inn = cali_BufferedRegionProfile_region_time(&rp, "wrap.rp.inner");
+
+    cali_BufferedRegionProfile_delete(&rp);
+
+    EXPECT_GT(e_inn, 0.0);
+    EXPECT_GT(e_out, 0.0);
+    EXPECT_GT(e_inn, e_out);
+    EXPECT_GE(e_reg, e_inn + e_out);
+    EXPECT_GE(e_tot, e_reg);
+
+    EXPECT_FLOAT_EQ(e_inn, i_inn);
+    EXPECT_FLOAT_EQ(e_reg, i_reg);
+    EXPECT_FLOAT_EQ(e_tot, i_tot);
+
+    EXPECT_GT(i_inn, 0.0);
+    EXPECT_GT(i_out, i_inn);
+    EXPECT_GE(i_reg, i_out);
+    EXPECT_GE(i_tot, i_reg);
+}