Skip to content

Commit

Permalink
Add percent_total kernel (#89)
Browse files Browse the repository at this point in the history
  • Loading branch information
@daboehme
daboehme authored Dec 20, 2017
1 parent 6cb4c18 commit c7b8b02
Show file tree
Hide file tree
Showing 2 changed files with 266 additions and 15 deletions.
165 changes: 150 additions & 15 deletions src/reader/Aggregator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,11 +486,11 @@ class PercentageKernel : public AggregateKernel {
CALI_TYPE_DOUBLE, CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE);

m_sum1_attr =
db.create_attribute("sum#" + m_target_attr1_name,
db.create_attribute("pc.sum#" + m_target_attr1_name,
CALI_TYPE_DOUBLE, CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE | CALI_ATTR_HIDDEN);

m_sum2_attr =
db.create_attribute("sum#" + m_target_attr2_name,
db.create_attribute("pc.sum#" + m_target_attr2_name,
CALI_TYPE_DOUBLE, CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE | CALI_ATTR_HIDDEN);

percentage_attr = m_percentage_attr;
Expand Down Expand Up @@ -571,23 +571,154 @@ class PercentageKernel : public AggregateKernel {
Config* m_config;
};

//
// --- PercentTotalKernel
//

class PercentTotalKernel : public AggregateKernel {
public:

class Config : public AggregateKernelConfig {
std::string m_target_attr_name;
Attribute m_target_attr;
Attribute m_sum_attr;

Attribute m_percentage_attr;

std::mutex m_total_lock;
double m_total;

public:

Attribute get_target_attr(CaliperMetadataAccessInterface& db) {
if (m_target_attr == Attribute::invalid)
m_target_attr = db.get_attribute(m_target_attr_name);

return m_target_attr;
}

bool get_percentage_attribute(CaliperMetadataAccessInterface& db,
Attribute& percentage_attr,
Attribute& sum_attr) {
if (m_target_attr == Attribute::invalid)
return false;
if (m_percentage_attr != Attribute::invalid) {
percentage_attr = m_percentage_attr;
sum_attr = m_sum_attr;
return true;
}

m_percentage_attr =
db.create_attribute("percent_total#" + m_target_attr_name,
CALI_TYPE_DOUBLE,
CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE);

m_sum_attr =
db.create_attribute("pct.sum#" + m_target_attr_name,
CALI_TYPE_DOUBLE,
CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE | CALI_ATTR_HIDDEN);

percentage_attr = m_percentage_attr;
sum_attr = m_sum_attr;

return true;
}

AggregateKernel* make_kernel() {
return new PercentTotalKernel(this);
}

void add(double val) {
std::lock_guard<std::mutex>
g(m_total_lock);

m_total += val;
}

double get_total() {
return m_total;
}

Config(const std::vector<std::string>& names)
: m_target_attr_name(names.front()),
m_target_attr(Attribute::invalid),
m_total(0)
{
Log(2).stream() << "aggregate: creating percent_total kernel for attribute "
<< m_target_attr_name << std::endl;
}

static AggregateKernelConfig* create(const std::vector<std::string>& cfg) {
return new Config(cfg);
}
};

PercentTotalKernel(Config* config)
: m_sum(0), m_config(config)
{ }

virtual void aggregate(CaliperMetadataAccessInterface& db, const EntryList& list) {
Attribute target_attr = m_config->get_target_attr(db);
Attribute percentage_attr, sum_attr;

if (!m_config->get_percentage_attribute(db, percentage_attr, sum_attr))
return;

cali_id_t target_id = target_attr.id();
cali_id_t sum_id = sum_attr.id();

for (const Entry& e : list) {
cali_id_t id = e.attribute();

if (id == target_id || id == sum_id) {
double val = e.value().to_double();
m_sum += val;
m_config->add(val);
}
}
}

virtual void append_result(CaliperMetadataAccessInterface& db, EntryList& list) {
double total = m_config->get_total();

if (total > 0) {
Attribute percentage_attr, sum_attr;

if (!m_config->get_percentage_attribute(db, percentage_attr, sum_attr))
return;

list.push_back(Entry(sum_attr, Variant(m_sum)));
list.push_back(Entry(percentage_attr, Variant(100.0 * m_sum / total)));
}
}

private:

double m_sum;

std::mutex m_lock;
Config* m_config;
};

enum KernelID {
Count = 0,
Sum = 1,
Statistics = 2,
Percentage = 3,
Count = 0,
Sum = 1,
Statistics = 2,
Percentage = 3,
PercentTotal = 4
};

#define MAX_KERNEL_ID 3
#define MAX_KERNEL_ID 4

const char* kernel_args[] = { "attribute" };
const char* kernel_2args[] = { "numerator", "denominator" };

const QuerySpec::FunctionSignature kernel_signatures[] = {
{ KernelID::Count, "count", 0, 0, nullptr },
{ KernelID::Sum, "sum", 1, 1, kernel_args },
{ KernelID::Statistics, "statistics", 1, 1, kernel_args },
{ KernelID::Percentage, "percentage", 2, 2, kernel_2args },
{ KernelID::Count, "count", 0, 0, nullptr },
{ KernelID::Sum, "sum", 1, 1, kernel_args },
{ KernelID::Statistics, "statistics", 1, 1, kernel_args },
{ KernelID::Percentage, "percentage", 2, 2, kernel_2args },
{ KernelID::PercentTotal, "percent_total", 1, 1, kernel_args },

QuerySpec::FunctionSignatureTerminator
};
Expand All @@ -596,10 +727,11 @@ const struct KernelInfo {
const char* name;
AggregateKernelConfig* (*create)(const std::vector<std::string>& cfg);
} kernel_list[] = {
{ "count", CountKernel::Config::create },
{ "sum", SumKernel::Config::create },
{ "statistics", StatisticsKernel::Config::create },
{ "percentage", PercentageKernel::Config::create },
{ "count", CountKernel::Config::create },
{ "sum", SumKernel::Config::create },
{ "statistics", StatisticsKernel::Config::create },
{ "percentage", PercentageKernel::Config::create },
{ "percent_total", PercentTotalKernel::Config::create },
{ 0, 0 }
};

Expand Down Expand Up @@ -1037,6 +1169,9 @@ Aggregator::aggregation_attribute_names(const QuerySpec& spec)
case KernelID::Percentage:
ret.push_back(op.args[0] + std::string("/") + op.args[1]);
break;
case KernelID::PercentTotal:
ret.push_back(std::string("percent_total#") + op.args[0]);
break;
}
}
}
Expand Down
116 changes: 116 additions & 0 deletions src/reader/test/test_aggregator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,3 +551,119 @@ TEST(AggregatorTest, StatisticsKernel) {
EXPECT_EQ(dict[attr_sum.id()].value().to_int(), 66);
EXPECT_DOUBLE_EQ(dict[attr_avg.id()].value().to_double(), 16.5);
}

TEST(AggregatorTest, PercentTotalKernel) {
CaliperMetadataDB db;
IdMap idmap;

// create some context attributes

Attribute ctx =
db.create_attribute("ctx", CALI_TYPE_INT, CALI_ATTR_DEFAULT);
Attribute val_attr =
db.create_attribute("val", CALI_TYPE_INT, CALI_ATTR_ASVALUE);

// make some nodes

const struct NodeInfo {
cali_id_t node_id;
cali_id_t attr_id;
cali_id_t prnt_id;
Variant data;
} test_nodes[] = {
{ 100, ctx.id(), CALI_INV_ID, Variant(-1) },
{ 101, ctx.id(), 100, Variant(42) },
{ 102, ctx.id(), 100, Variant(24) }
};

const Node* node = nullptr;

for ( const NodeInfo& nI : test_nodes )
node = db.merge_node(nI.node_id, nI.attr_id, nI.prnt_id, nI.data, idmap);

//
// --- Make spec with default key and statistics kernel for "val"
//

QuerySpec spec;

spec.aggregation_key.selection = QuerySpec::SelectionList<std::string>::Default;

spec.aggregation_ops.selection = QuerySpec::SelectionList<QuerySpec::AggregationOp>::List;
spec.aggregation_ops.list.push_back(::make_op("statistics", "val"));
spec.aggregation_ops.list.push_back(::make_op("percent_total", "val"));

// perform recursive aggregation from two aggregators

Aggregator a(spec), b(spec);

Variant v_ints[4] = { Variant(4), Variant(24), Variant(16), Variant(36) };
cali_id_t nodea_id = 101;
cali_id_t nodeb_id = 102;
cali_id_t val_id = val_attr.id();

a.add(db, db.merge_snapshot(1, &nodea_id, 1, &val_id, &v_ints[0], idmap));
a.add(db, db.merge_snapshot(1, &nodeb_id, 1, &val_id, &v_ints[1], idmap));

b.add(db, db.merge_snapshot(1, &nodea_id, 1, &val_id, &v_ints[2], idmap));
b.add(db, db.merge_snapshot(1, &nodeb_id, 1, &val_id, &v_ints[3], idmap));

// merge b into a
b.flush(db, a);

Attribute attr_min = db.get_attribute("min#val");
Attribute attr_max = db.get_attribute("max#val");
Attribute attr_sum = db.get_attribute("sum#val");
Attribute attr_avg = db.get_attribute("avg#val");
Attribute attr_pct = db.get_attribute("percent_total#val");

ASSERT_NE(attr_min, Attribute::invalid);
ASSERT_NE(attr_max, Attribute::invalid);
ASSERT_NE(attr_sum, Attribute::invalid);
ASSERT_NE(attr_avg, Attribute::invalid);
ASSERT_NE(attr_pct, Attribute::invalid);

std::vector<EntryList> resdb;

a.flush(db, [&resdb](CaliperMetadataAccessInterface&, const EntryList& list) {
resdb.push_back(list);
});

// check results

EXPECT_EQ(resdb.size(), 2); // two entries

auto it = std::find_if(resdb.begin(), resdb.end(), [ctx](const EntryList& list){
for (const Entry& e : list)
if (e.value(ctx).to_int() == 42)
return true;
return false;
});

ASSERT_NE(it, resdb.end());

auto dict = make_dict_from_entrylist(*it);

EXPECT_EQ(dict[attr_min.id()].value().to_int(), 4);
EXPECT_EQ(dict[attr_max.id()].value().to_int(), 16);
EXPECT_EQ(dict[attr_sum.id()].value().to_int(), 20);
EXPECT_DOUBLE_EQ(dict[attr_avg.id()].value().to_double(), 10.0);
EXPECT_DOUBLE_EQ(dict[attr_pct.id()].value().to_double(), 25.0);

it = std::find_if(resdb.begin(), resdb.end(), [ctx](const EntryList& list){
for (const Entry& e : list)
if (e.value(ctx).to_int() == 24)
return true;
return false;
});

ASSERT_NE(it, resdb.end());

dict = make_dict_from_entrylist(*it);

EXPECT_EQ(dict[attr_min.id()].value().to_int(), 24);
EXPECT_EQ(dict[attr_max.id()].value().to_int(), 36);
EXPECT_EQ(dict[attr_sum.id()].value().to_int(), 60);
EXPECT_DOUBLE_EQ(dict[attr_avg.id()].value().to_double(), 30.0);
EXPECT_DOUBLE_EQ(dict[attr_pct.id()].value().to_double(), 75.0);
}

0 comments on commit c7b8b02

Please sign in to comment.