Add libdw option for symbollookup (LLNL#308)

* Add libdw symbollookup module

* Fix libdw usage

* Fix build w/o libdw

* Callpath.cpp cleanup, add libdw CMake message

* Enable libdw support in travis config

.travis.yml

@@ -10,6 +10,7 @@ addons:
       - openmpi-bin
       - libopenmpi-dev
       - libunwind-dev
+      - libdw-dev
       - gfortran
 
 # The base build matrix

CMakeLists.txt

@@ -194,6 +194,21 @@ if (WITH_DYNINST)
   # CMake already provides useful error message if DyninstConfig.cmake isn't found
 endif()
 
+if (WITH_LIBDW)
+# For some reason Dyninst crashes when libdw is linked.
+# Detect this here and disable libdw.
+  if (Dyninst_FOUND)
+    message(WARNING "Libdw option is incompatible with dyninst -- disabling libdw.")
+  else()
+    include(FindLibDw)
+    if (LIBDW_FOUND)
+      list(APPEND CALIPER_EXTERNAL_LIBS ${LIBDW_LIBRARY})
+      set(CALIPER_HAVE_LIBDW TRUE)
+      set(CALIPER_Libdw_CMAKE_MSG "Yes, using ${LIBDW_LIBRARY}")
+    endif()
+  endif()
+endif()
+
 # Find libunwind
 if (WITH_CALLPATH)
   include(FindLibunwind)
@@ -204,20 +219,6 @@ if (WITH_CALLPATH)
   else()
     message(WARNING "Callpath support was requested but libunwind was not found!")
   endif()
-  if (WITH_LIBDW)
-    # For some reason Dyninst crashes when libdw is linked.
-    # Detect this here and disable libdw.
-    if (Dyninst_FOUND)
-      message(WARNING "Libdw option is incompatible with dyninst -- disabling libdw.")
-    else()
-      include(FindLibDw)
-      if (LIBDW_FOUND)
-        message(STATUS "Found libdw in " ${LIBDW_LIBRARY})
-        list(APPEND CALIPER_EXTERNAL_LIBS ${LIBDW_LIBRARY})
-        set(CALIPER_HAVE_LIBDW TRUE)
-      endif()
-    endif()
-  endif()
 endif()
 
 # Find Gotcha
@@ -357,6 +358,9 @@ if (WITH_SYMBOLLOOKUP)
   if (CALIPER_HAVE_DYNINST)
     set(CALIPER_HAVE_SYMBOLLOOKUP TRUE)
     set(CALIPER_SymbolLookup_CMAKE_MSG "Yes, using Dyninst")
+  elseif(CALIPER_HAVE_LIBDW)
+    set(CALIPER_HAVE_SYMBOLLOOKUP TRUE)
+    set(CALIPER_SymbolLookup_CMAKE_MSG "Yes, using libdw")
   else()
     find_library(DL_LIBRARY
       dl)
@@ -472,6 +476,7 @@ set(CALIPER_MODULES
   adiak
   GOTCHA
   PAPI
+  Libdw
   Libpfm
   Libunwind
   Dyninst

cmake/hostconfig/travis.cmake

@@ -5,6 +5,7 @@ set(WITH_CALLPATH     On  CACHE BOOL "")
 set(WITH_NVPROF       Off CACHE BOOL "")
 set(WITH_CUPTI        Off CACHE BOOL "")
 set(WITH_PAPI         Off CACHE BOOL "")
+set(WITH_LIBDW        On  CACHE BOOL "")
 set(WITH_LIBPFM       Off CACHE BOOL "")
 set(WITH_DYNINST      Off CACHE BOOL "")
 set(WITH_SAMPLER      On  CACHE BOOL "")

src/services/callpath/Callpath.cpp

@@ -15,6 +15,7 @@
 
 #include <cstring>
 #include <string>
+#include <sstream>
 #include <type_traits>
 
 #define UNW_LOCAL_ONLY
@@ -31,27 +32,26 @@
 using namespace cali;
 using namespace std;
 
-namespace 
+namespace
 {
 
 class Callpath
 {
     static const ConfigSet::Entry s_configdata[];
-    
+
     Attribute callpath_name_attr { Attribute::invalid};
     Attribute callpath_addr_attr { Attribute::invalid };
 
     bool      use_name { false };
     bool      use_addr { false };
+    bool      skip_internal { false };
 
     unsigned  skip_frames { 0 };
 
     Node      callpath_root_node;
 
-#ifdef CALIPER_HAVE_LIBDW
-    Dwfl* dwfl;
-    Dwfl_Module* caliper_module;
-#endif
+    uintptr_t caliper_start_addr { 0 };
+    uintptr_t caliper_end_addr   { 0 };
 
     void snapshot_cb(Caliper* c, Channel* chn, int scope, const SnapshotRecord*, SnapshotRecord* snapshot) {
         Variant v_addr[MAX_PATH];
@@ -82,23 +82,15 @@ class Callpath
 
         while (n < MAX_PATH && unw_step(&unw_cursor) > 0) {
 
-#ifdef CALIPER_HAVE_LIBDW
             // skip stack frames inside caliper
             unw_word_t ip;
             unw_get_reg(&unw_cursor, UNW_REG_IP, &ip);
 
-            Dwfl_Module* module=dwfl_addrmodule (dwfl, ip);
-
-            if (module == caliper_module)
+            if (skip_internal && (ip >= caliper_start_addr && ip < caliper_end_addr))
                 continue;
-#endif
 
             // store path from top to bottom
             if (use_addr) {
-#ifndef CALIPER_HAVE_LIBDW
-                unw_word_t ip;
-                unw_get_reg(&unw_cursor, UNW_REG_IP, &ip);
-#endif
                 uint64_t uint = ip;
                 v_addr[MAX_PATH-(n+1)] = Variant(CALI_TYPE_ADDR, &uint, sizeof(uint64_t));
             }
@@ -122,20 +114,21 @@ class Callpath
             if (use_name)
                 snapshot->append(
                     c->make_tree_entry(callpath_name_attr, n, v_name+(MAX_PATH-n),
-                                       &callpath_root_node));        
+                                       &callpath_root_node));
         }
     }
 
-    void initialize_dw() {
+    void get_caliper_module_addresses() {
 #ifdef CALIPER_HAVE_LIBDW
         // initialize dwarf
-        char *debuginfo_path=nullptr;
+        char* debuginfopath = nullptr;
         Dwfl_Callbacks callbacks;
+
         callbacks.find_elf = dwfl_linux_proc_find_elf;
         callbacks.find_debuginfo = dwfl_standard_find_debuginfo;
-        callbacks.debuginfo_path = &debuginfo_path;
+        callbacks.debuginfo_path = &debuginfopath;
 
-        dwfl=dwfl_begin(&callbacks);
+        Dwfl* dwfl = dwfl_begin(&callbacks);
 
         dwfl_linux_proc_report(dwfl, getpid());
         dwfl_report_end(dwfl, nullptr, nullptr);
@@ -147,15 +140,33 @@ class Callpath
         unw_getcontext(&unw_ctx);
 
         if (unw_init_local(&unw_cursor, &unw_ctx) < 0) {
-            Log(0).stream() << "callpath::measure_cb: error: unable to init libunwind cursor" << endl;
+            Log(0).stream() << "callpath::measure_cb: error: unable to init libunwind" << endl;
             return;
         }
 
         // Get current (caliper) module
         unw_word_t ip;
         unw_get_reg(&unw_cursor, UNW_REG_IP, &ip);
 
-        caliper_module = dwfl_addrmodule(dwfl, ip);
+        Dwfl_Module* mod = dwfl_addrmodule(dwfl, ip);
+        Dwarf_Addr start = 0;
+        Dwarf_Addr end = 0;
+
+        dwfl_module_info(mod, nullptr, &start, &end, nullptr, nullptr, nullptr, nullptr);
+
+        caliper_start_addr = start;
+        caliper_end_addr = end;
+
+        if (Log::verbosity() >= 2) {
+            std::ostringstream os;
+            os << std::hex << caliper_start_addr << ":" << caliper_end_addr;
+
+            Log(2).stream() << "callpath: skipping internal caliper frames ("
+                            << os.str() << ")"
+                            << std::endl;
+        }
+
+        dwfl_end(dwfl);
 #endif
     }
 
@@ -165,24 +176,32 @@ class Callpath
             ConfigSet config =
                 chn->config().init("callpath", s_configdata);
 
-            use_name    = config.get("use_name").to_bool();
-            use_addr    = config.get("use_address").to_bool();
-            skip_frames = config.get("skip_frames").to_uint();
+            use_name      = config.get("use_name").to_bool();
+            use_addr      = config.get("use_address").to_bool();
+            skip_frames   = config.get("skip_frames").to_uint();
+            skip_internal = config.get("skip_internal").to_bool();
 
             Attribute symbol_class_attr = c->get_attribute("class.symboladdress");
             Variant v_true(true);
 
-            callpath_addr_attr = 
+            callpath_addr_attr =
                 c->create_attribute("callpath.address", CALI_TYPE_ADDR,
-                                    CALI_ATTR_SCOPE_THREAD | 
+                                    CALI_ATTR_SCOPE_THREAD |
                                     CALI_ATTR_SKIP_EVENTS  |
                                     CALI_ATTR_NOMERGE,
                                     1, &symbol_class_attr, &v_true);
-            callpath_name_attr = 
+            callpath_name_attr =
                 c->create_attribute("callpath.regname", CALI_TYPE_STRING,
                                     CALI_ATTR_SCOPE_THREAD |
                                     CALI_ATTR_SKIP_EVENTS  |
                                     CALI_ATTR_NOMERGE);
+
+#ifdef CALIPER_HAVE_LIBDW
+            if (skip_internal)
+                get_caliper_module_addresses();
+#else
+            skip_internal = false;
+#endif
         }
 
 public:
@@ -201,7 +220,7 @@ class Callpath
 
         Log(1).stream() << chn->name() << ": Registered callpath service" << std::endl;
     }
-    
+
 }; // class Callpath
 
 const ConfigSet::Entry Callpath::s_configdata[] = {
@@ -218,6 +237,10 @@ const ConfigSet::Entry Callpath::s_configdata[] = {
       "Skip this number of stack frames.\n"
       "Avoids recording stack frames within the caliper library"
     },
+    { "skip_internal", CALI_TYPE_BOOL, "true",
+      "Skip caliper-internal stack frames",
+      "Skip caliper-internal stack frames. Requires libdw support.\n"
+    },
     ConfigSet::Terminator
 };
 

src/services/symbollookup/CMakeLists.txt

@@ -4,6 +4,9 @@ set(CALIPER_SYMBOLLOOKUP_SOURCES
 if (CALIPER_HAVE_DYNINST)
   list(APPEND CALIPER_SYMBOLLOOKUP_SOURCES
     LookupDyninst.cpp)
+elseif(CALIPER_HAVE_LIBDW)
+  list(APPEND CALIPER_SYMBOLLOOKUP_SOURCES
+    LookupLibdw.cpp)
 else()
   list(APPEND CALIPER_SYMBOLLOOKUP_SOURCES
     LookupDlAddr.cpp)