Skip to content

Commit

Permalink
Add mem.highwatermark profiling option (LLNL#197)
Browse files Browse the repository at this point in the history 
* Add memusage service and profile config

* Add malloc.bytes attribute

* sysalloc: Get instance only when there are active channels

* Fix sysalloc service

* Simplify AllocService

* alloc: Track region high-water mark

* Add mem.highwatermark profiling option
  • Loading branch information
@daboehme
daboehme authored Aug 13, 2019
1 parent 94d761b commit dd2ca84
Show file tree
Hide file tree
Showing 11 changed files with 386 additions and 154 deletions.
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,9 @@ endif(BUILD_SHARED_LIBS)

if (${CALIPER_HAVE_LINUX})
set(CALIPER_HAVE_CPUINFO TRUE)
set(CALIPER_HAVE_MEMUSAGE TRUE)
set(CALIPER_cpuinfo_CMAKE_MSG "Yes")
set(CALIPER_memusage_CMAKE_MSG "Yes")
endif()

if(WITH_VTUNE)
Expand Down Expand Up @@ -396,6 +398,7 @@ message(STATUS "Build tests : ${BUILD_TESTING}")

set(CALIPER_MODULES
cpuinfo
memusage
adiak
GOTCHA
PAPI
Expand Down
92 changes: 61 additions & 31 deletions src/caliper/controllers/RuntimeReportController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,18 @@ using namespace cali;
namespace
{

enum Wrapper {
WrapMpi = 1,
WrapCuda = 2
enum ProfileConfig {
WrapMpi = 1,
WrapCuda = 2,
MallocInfo = 4,
MemHighWaterMark = 8
};

class RuntimeReportController : public cali::ChannelController
{
public:

RuntimeReportController(bool use_mpi, const char* output, int wrappers)
RuntimeReportController(bool use_mpi, const char* output, int profile)
: ChannelController("runtime-report", 0, {
{ "CALI_CHANNEL_FLUSH_ON_EXIT", "false" },
{ "CALI_SERVICES_ENABLE", "aggregate,event,timestamp" },
Expand All @@ -35,35 +37,58 @@ class RuntimeReportController : public cali::ChannelController
{ "CALI_TIMER_UNIT", "sec" }
})
{
std::string select;
std::string groupby = "prop:nested";

if (use_mpi) {
config()["CALI_SERVICES_ENABLE" ].append(",mpireport");
config()["CALI_MPIREPORT_FILENAME"] = output;
config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false";
config()["CALI_MPIREPORT_CONFIG" ] =
"select min(sum#time.duration) as \"Min time/rank\""
select =
" min(sum#time.duration) as \"Min time/rank\""
",max(sum#time.duration) as \"Max time/rank\""
",avg(sum#time.duration) as \"Avg time/rank\""
",percent_total(sum#time.duration) as \"Time % (total)\""
" group by prop:nested format tree";
",percent_total(sum#time.duration) as \"Time % (total)\"";
} else {
config()["CALI_SERVICES_ENABLE" ].append(",report");
config()["CALI_REPORT_FILENAME" ] = output;
config()["CALI_REPORT_CONFIG" ] =
"select inclusive_sum(sum#time.duration) as \"Inclusive time\""
select =
" inclusive_sum(sum#time.duration) as \"Inclusive time\""
",sum(sum#time.duration) as \"Exclusive time\""
",percent_total(sum#time.duration) as \"Time %\""
" group by prop:nested format tree";
",percent_total(sum#time.duration) as \"Time %\"";
}

if (wrappers & WrapMpi) {
if (profile & MallocInfo) {
select +=
",sum(sum#malloc.bytes) as \"Heap Allocations\""
",max(max#malloc.total.bytes) as \"Max Total Bytes on Heap\"";

config()["CALI_SERVICES_ENABLE" ].append(",mallinfo");
}
if (profile & MemHighWaterMark) {
select +=
",max(max#alloc.region.highwatermark) as \"Max Alloc'd Mem\"";

config()["CALI_SERVICES_ENABLE" ].append(",alloc,sysalloc");
config()["CALI_ALLOC_RECORD_HIGHWATERMARK"] = "true";
config()["CALI_ALLOC_TRACK_ALLOCATIONS" ] = "false";
}
if (profile & WrapMpi) {
config()["CALI_SERVICES_ENABLE" ].append(",mpi");
config()["CALI_MPI_BLACKLIST" ] =
"MPI_Comm_rank,MPI_Comm_size,MPI_Wtick,MPI_Wtime";
}

if (wrappers & WrapCuda) {
if (profile & WrapCuda) {
config()["CALI_SERVICES_ENABLE" ].append(",cupti");
}

if (use_mpi) {
config()["CALI_SERVICES_ENABLE" ].append(",mpireport");
config()["CALI_MPIREPORT_FILENAME"] = output;
config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false";
config()["CALI_MPIREPORT_CONFIG" ] =
std::string("select ") + select + " group by " + groupby + " format tree";
} else {
config()["CALI_SERVICES_ENABLE" ].append(",report");
config()["CALI_REPORT_FILENAME" ] = output;
config()["CALI_REPORT_CONFIG" ] =
std::string("select ") + select + " group by " + groupby + " format tree";
}
}
};

Expand Down Expand Up @@ -96,27 +121,32 @@ use_mpi(const cali::ConfigManager::argmap_t& args)

// Parse the "profile=" argument
int
profile_cfg(const cali::ConfigManager::argmap_t& args)
get_profile_cfg(const cali::ConfigManager::argmap_t& args)
{
auto argit = args.find("profile");

if (argit == args.end())
return 0;

int wrappers = 0;
// make sure default services are loaded
Services::add_default_services();
auto srvcs = Services::get_available_services();

const std::vector< std::tuple<const char*, Wrapper, const char*> > wrapinfo {
{ std::make_tuple( "mpi", WrapMpi, "mpi") },
{ std::make_tuple( "cuda", WrapCuda, "cupti") }
const std::vector< std::tuple<const char*, ProfileConfig, const char*> > profinfo {
{ std::make_tuple( "mpi", WrapMpi, "mpi") },
{ std::make_tuple( "cuda", WrapCuda, "cupti") },
{ std::make_tuple( "malloc", MallocInfo, "memusage") },
{ std::make_tuple( "mem.highwatermark", MemHighWaterMark, "sysalloc" )}
};

int profile = 0;

for (const std::string& s : StringConverter(argit->second).to_stringlist(",:")) {
auto it = std::find_if(wrapinfo.begin(), wrapinfo.end(), [s](decltype(wrapinfo.front()) tpl){
auto it = std::find_if(profinfo.begin(), profinfo.end(), [s](decltype(profinfo.front()) tpl){
return s == std::get<0>(tpl);
});

if (it == wrapinfo.end())
if (it == profinfo.end())
Log(0).stream() << "runtime-report: Unknown profile option \"" << s << "\"" << std::endl;
else {
if (std::find(srvcs.begin(), srvcs.end(), std::get<2>(*it)) == srvcs.end())
Expand All @@ -125,11 +155,11 @@ profile_cfg(const cali::ConfigManager::argmap_t& args)
<< " service is not available."
<< std::endl;
else
wrappers |= std::get<1>(*it);
profile |= std::get<1>(*it);
}
}

return wrappers;
return profile;
}

cali::ChannelController*
Expand All @@ -138,7 +168,7 @@ make_runtime_report_controller(const cali::ConfigManager::argmap_t& args)
auto it = args.find("output");
std::string output = (it == args.end() ? "stderr" : it->second);

return new RuntimeReportController(use_mpi(args), output.c_str(), profile_cfg(args));
return new RuntimeReportController(use_mpi(args), output.c_str(), get_profile_cfg(args));
}

} // namespace [anonymous]
Expand All @@ -148,7 +178,7 @@ namespace cali

ConfigManager::ConfigInfo runtime_report_controller_info
{
"runtime-report", "runtime-report(output=<filename>,mpi=true|false,profile=[mpi:cupti]): Print region time profile", ::runtime_report_args, ::make_runtime_report_controller
"runtime-report", "runtime-report(output=<filename>,mpi=true|false,profile=[mpi:cupti:memory]): Print region time profile", ::runtime_report_args, ::make_runtime_report_controller
};

}
53 changes: 40 additions & 13 deletions src/mpi-rt/controllers/SpotController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ using namespace cali;
namespace
{

enum ProfileConfig {
MemHighWaterMark = 1
};

std::string
make_filename()
{
Expand Down Expand Up @@ -79,6 +83,7 @@ write_adiak(CaliperMetadataDB& db, Aggregator& output_agg)
class SpotController : public cali::ChannelController
{
std::string m_output;
int m_profilecfg;
bool m_use_mpi;

public:
Expand All @@ -88,13 +93,16 @@ class SpotController : public cali::ChannelController
Log(1).stream() << "[spot controller]: Flushing Caliper data" << std::endl;

// --- Setup output reduction aggregator
std::string select =
" *"
",min(inclusive#sum#time.duration)"
",max(inclusive#sum#time.duration)"
",avg(inclusive#sum#time.duration)";

if (m_profilecfg & MemHighWaterMark)
select.append(",max(max#max#alloc.region.highwatermark)");

QuerySpec output_spec =
CalQLParser("select *,"
" min(inclusive#sum#time.duration)"
",max(inclusive#sum#time.duration)"
",avg(inclusive#sum#time.duration)"
" format cali").spec();
QuerySpec output_spec = CalQLParser((std::string(" select ") + select + " format cali").c_str()).spec();

Aggregator output_agg(output_spec);

Expand All @@ -105,10 +113,13 @@ class SpotController : public cali::ChannelController
// inclusive times

{
QuerySpec inclusive_spec =
CalQLParser("aggregate"
" inclusive_sum(sum#time.duration)"
" group by prop:nested").spec();
std::string aggcfg = "inclusive_sum(sum#time.duration)";

if (m_profilecfg & MemHighWaterMark)
aggcfg.append(",max(max#alloc.region.highwatermark)");

QuerySpec inclusive_spec =
CalQLParser(std::string("aggregate " + aggcfg + " group by prop:nested").c_str()).spec();

Aggregator inclusive_agg(inclusive_spec);

Expand Down Expand Up @@ -156,6 +167,9 @@ class SpotController : public cali::ChannelController
",min#inclusive#sum#time.duration"
",max#inclusive#sum#time.duration";

if (m_profilecfg & MemHighWaterMark)
spot_metrics.append(",max#max#max#alloc.region.highwatermark");

// set the spot.metrics value
db.set_global(db.create_attribute("spot.metrics", CALI_TYPE_STRING, CALI_ATTR_GLOBAL),
Variant(CALI_TYPE_STRING, spot_metrics.data(), spot_metrics.length()));
Expand Down Expand Up @@ -183,7 +197,7 @@ class SpotController : public cali::ChannelController
}
}

SpotController(bool use_mpi, const char* output)
SpotController(bool use_mpi, int profilecfg, const char* output)
: ChannelController("spot", 0, {
{ "CALI_SERVICES_ENABLE", "aggregate,event,timestamp" },
{ "CALI_EVENT_ENABLE_SNAPSHOT_INFO", "false" },
Expand All @@ -194,19 +208,26 @@ class SpotController : public cali::ChannelController
{ "CALI_CHANNEL_CONFIG_CHECK", "false" }
}),
m_output(output),
m_profilecfg(profilecfg),
m_use_mpi(use_mpi)
{
#ifdef CALIPER_HAVE_ADIAK
if (output != "adiak")
config()["CALI_SERVICES_ENABLE"].append(",adiak_import");
#endif
if (profilecfg & MemHighWaterMark) {
config()["CALI_SERVICES_ENABLE"].append(",alloc,sysalloc");

config()["CALI_ALLOC_TRACK_ALLOCATIONS" ] = "false";
config()["CALI_ALLOC_RECORD_HIGHWATERMARK"] = "true";
}
}

~SpotController()
{ }
};

const char* spot_args[] = { "output", nullptr };
const char* spot_args[] = { "output", "profile", nullptr };

cali::ChannelController*
make_spot_controller(const cali::ConfigManager::argmap_t& args) {
Expand All @@ -222,7 +243,13 @@ make_spot_controller(const cali::ConfigManager::argmap_t& args) {
if (it != args.end())
use_mpi = StringConverter(it->second).to_bool();

return new SpotController(use_mpi, output.c_str());
int profilecfg = 0;

it = args.find("profile");
if (it != args.end() && it->second == "mem.highwatermark")
profilecfg |= MemHighWaterMark;

return new SpotController(use_mpi, profilecfg, output.c_str());
}

} // namespace [anonymous]
Expand Down
3 changes: 3 additions & 0 deletions src/services/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ endif()
if (CALIPER_HAVE_CPUINFO)
add_subdirectory(cpuinfo)
endif()
if (CALIPER_HAVE_MEMUSAGE)
add_subdirectory(memusage)
endif()
if (CALIPER_HAVE_PAPI)
add_subdirectory(papi)
endif()
Expand Down
4 changes: 4 additions & 0 deletions src/services/Services.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ void Services::add_services(const CaliperService* services)

void Services::add_default_services()
{
for (const ServicesList* lp = ::s_services_list; lp; lp = lp->next)
if (lp->services == caliper_services)
return;

add_services(caliper_services);
}

Expand Down
Loading

0 comments on commit dd2ca84

Please sign in to comment.