diff --git a/doc/sphinx/calql.rst b/doc/sphinx/calql.rst index 2538449fe..ce4701113 100644 --- a/doc/sphinx/calql.rst +++ b/doc/sphinx/calql.rst @@ -28,6 +28,7 @@ This table contains a quick reference of all CalQL statements: =scale(,) # computes = *S =truncate(,) # computes = - mod(, S) =first(,, ...) # is the first of , , ... found in the input record + ... IF # apply only if input record meets condition SELECT # Select attributes and define aggregations (i.e., select columns) * # select all attributes @@ -56,8 +57,7 @@ This table contains a quick reference of all CalQL statements: = # records where = < # records where > > # records where < - NOT # records where does not exist - NOT = # records where != + NOT # negate the filter clause FORMAT # Define output format cali # .cali format @@ -83,25 +83,55 @@ can then be used in subsequent SELECT, GROUP BY, or FORMAT statements. For example, we can use the scale() operator to scale a value before subsequent aggregations:: - LET sec=scale(time.duration,1e-6) SELECT prop:nested,sum(sec) + LET + sec=scale(time.duration,1e-6) + SELECT + prop:nested,sum(sec) -For example, we can use the truncate() operator on an iteration counter to +We can use the truncate() operator on an iteration counter to aggregate blocks of 10 iterations in a time-series profile:: - LET block=truncate(iteration#mainloop,10) SELECT block,sum(time.duration) GROUP BY block + LET + block=truncate(iteration#mainloop,10) + SELECT + block,sum(time.duration) + GROUP BY + block The first() operator returns the first attribute out of a list of attribute names found in an input record. It can also be used to rename attributes:: - LET time=first(time.duration,sum#time.duration) SELECT sum(time) AS Time GROUP BY prop:nested + LET + time=first(time.duration,sum#time.duration) + SELECT + sum(time) AS Time + GROUP BY + prop:nested LET terms have the general form - a = f(...) + a = f(...) [IF ] where f is one of the operators and `a` is the name of the result attribute. The result is added to the input records before the record is processed further. -Result entries are only added to a record if all required input operands are present. +Result entries are only added to a record if all required input operands are +present. + +With the optional IF condition, the operation is only applied for input records +that meet a condition. One can use this to compute values for a specific +subset of records. The condition clauses use the same syntax as WHERE filter +clauses. The example below defines a "work" attribute with the time in +records that contain "omp.work" regions, and then uses that to compute +efficiency from the total and "work" time: + + LET + work=first(time.duration) IF omp.work + SELECT + sum(time.duration) AS Total, + sum(work) AS Work, + ratio(work,time.duration) AS Efficiency + GROUP BY + prop:nested SELECT -------------------------------- @@ -140,7 +170,14 @@ applies to the hierarchy defined by attributes with the A more complex example:: - SELECT *, scale(time.duration,1e-6) AS Time, inclusive_percent_total(time.duration) AS "Time %" GROUP BY prop:nested FORMAT tree + SELECT + *, + scale(time.duration,1e-6) AS Time, + inclusive_percent_total(time.duration) AS "Time %" + GROUP BY + prop:nested + FORMAT + tree The computes the (exclusive) sum of `time.duration` divided by 100000 and the inclusive percent-of-total for `time.duration`. Example output:: @@ -234,8 +271,14 @@ The following example prints a iteration/function profile ordered by time and iteration number. Note that one must use the original attribute name and not an alias assigned with ``AS``: :: - SELECT *, sum(time.inclusive.duration) AS Time FORMAT table \ - ORDER BY sum#time.inclusive.duration DESC, iteration#mainloop + SELECT + *, + sum(time.inclusive.duration) AS Time + FORMAT + table + ORDER BY + sum#time.inclusive.duration DESC, + iteration#mainloop function loop iteration#mainloop Time main 100000 diff --git a/include/caliper/reader/QuerySpec.h b/include/caliper/reader/QuerySpec.h index 471bd473d..19833df52 100644 --- a/include/caliper/reader/QuerySpec.h +++ b/include/caliper/reader/QuerySpec.h @@ -89,6 +89,14 @@ struct QuerySpec } op; std::string attr_name; std::string value; + + Condition() + : op(Op::None) + { } + + Condition(Condition::Op o, const std::string& name, const std::string& val) + : op(o), attr_name(name), value(val) + { } }; /// \brief Output formatter specification. @@ -103,6 +111,7 @@ struct QuerySpec struct PreprocessSpec { std::string target; AggregationOp op; + Condition cond; }; // diff --git a/include/caliper/reader/RecordSelector.h b/include/caliper/reader/RecordSelector.h index ab57746b1..7afa0d3ea 100644 --- a/include/caliper/reader/RecordSelector.h +++ b/include/caliper/reader/RecordSelector.h @@ -29,6 +29,7 @@ class RecordSelector RecordSelector(const std::string& filter_string); RecordSelector(const QuerySpec& spec); + RecordSelector(const QuerySpec::Condition& cond); ~RecordSelector(); diff --git a/src/reader/CalQLParser.cpp b/src/reader/CalQLParser.cpp index fc6b745e5..1b88c60d1 100644 --- a/src/reader/CalQLParser.cpp +++ b/src/reader/CalQLParser.cpp @@ -362,7 +362,7 @@ struct CalQLParser::CalQLParserImpl parse_clause_from_word(next_keyword, is); } - void + QuerySpec::Condition parse_filter_clause(std::istream& is) { std::string w = util::read_word(is, ",;=<>()\n"); std::string wl(w); @@ -376,16 +376,15 @@ struct CalQLParser::CalQLParserImpl w = util::read_word(is, ",;=<>()\n"); } - if (w.empty()) { - set_error("Condition term expected", is); - return; - } - QuerySpec::Condition cond; - cond.op = QuerySpec::Condition::None; cond.attr_name = w; + if (w.empty()) { + set_error("Condition term expected", is); + return cond; + } + char c = util::read_char(is); switch (c) { @@ -427,12 +426,11 @@ struct CalQLParser::CalQLParserImpl cond.op = (negate ? QuerySpec::Condition::NotExist : QuerySpec::Condition::Exist); } - if (cond.op != QuerySpec::Condition::None) { - spec.filter.selection = QuerySpec::FilterSelection::List; - spec.filter.list.push_back(cond); - } else { + if (cond.op == QuerySpec::Condition::None) { set_error("Condition term expected", is); } + + return cond; } void @@ -440,7 +438,13 @@ struct CalQLParser::CalQLParserImpl char c = '\0'; do { - parse_filter_clause(is); + QuerySpec::Condition cond = parse_filter_clause(is); + + if (!error && cond.op != QuerySpec::Condition::None) { + spec.filter.selection = QuerySpec::FilterSelection::List; + spec.filter.list.push_back(cond); + } + c = util::read_char(is); } while (!error && is.good() && c == ','); @@ -450,10 +454,12 @@ struct CalQLParser::CalQLParserImpl void parse_let(std::istream& is) { + std::string next_keyword; char c = 0; do { const QuerySpec::FunctionSignature* defs = Preprocessor::preprocess_defs(); + QuerySpec::PreprocessSpec pspec; std::string target = util::read_word(is, ",;=<>()\n"); @@ -489,14 +495,32 @@ struct CalQLParser::CalQLParserImpl }); if (it == spec.preprocess_ops.end()) { - QuerySpec::PreprocessSpec pspec; - pspec.target = target; pspec.op = QuerySpec::AggregationOp(defs[i], args); - - spec.preprocess_ops.emplace_back(std::move(pspec)); - } else + } else { set_error(target + " defined twice", is); + return; + } + } + + // parse condition (... IF ... ) + + next_keyword.clear(); + std::string tmp = util::read_word(is, ",;=<>()\n"); + std::transform(tmp.begin(), tmp.end(), std::back_inserter(next_keyword), ::tolower); + + if (next_keyword == "if") { + pspec.cond = parse_filter_clause(is); + next_keyword.clear(); + } + + if (!error) { + spec.preprocess_ops.emplace_back(std::move(pspec)); + + if (!next_keyword.empty()) { + c = 0; + break; + } } c = util::read_char(is); @@ -504,6 +528,8 @@ struct CalQLParser::CalQLParserImpl if (c) is.unget(); + if (!next_keyword.empty()) + parse_clause_from_word(next_keyword, is); } void diff --git a/src/reader/Preprocessor.cpp b/src/reader/Preprocessor.cpp index e9ed9da1e..d10292b6c 100644 --- a/src/reader/Preprocessor.cpp +++ b/src/reader/Preprocessor.cpp @@ -6,6 +6,7 @@ #include "caliper/reader/Preprocessor.h" #include "caliper/reader/QuerySpec.h" +#include "caliper/reader/RecordSelector.h" #include "caliper/common/Attribute.h" #include "caliper/common/CaliperMetadataAccessInterface.h" @@ -14,13 +15,9 @@ #include "caliper/common/cali_types.h" -#include -#include -#include #include -#include -#include -#include +#include +#include using namespace cali; @@ -221,14 +218,14 @@ class FirstKernel : public Kernel { m_tgt_attrs.assign(args.size(), Attribute::invalid); } - + void process(CaliperMetadataAccessInterface& db, EntryList& rec) { for (size_t i = 0; i < m_tgt_attrs.size(); ++i) { Variant v_tgt = get_value(db, m_tgt_attr_names[i], m_tgt_attrs[i], rec); if (v_tgt.empty()) continue; - + cali_attr_type type = m_tgt_attrs[i].type(); if (m_res_attr == Attribute::invalid) @@ -257,9 +254,9 @@ enum KernelID { const char* sratio_args[] = { "numerator", "denominator", "scale" }; const char* scale_args[] = { "attribute", "scale" }; -const char* first_args[] = { - "attribute0", "attribute1", "attribute2", - "attribute3", "attribute4", "attribute5", +const char* first_args[] = { + "attribute0", "attribute1", "attribute2", + "attribute3", "attribute4", "attribute5", "attribute6", "attribute7", "attribute8" }; @@ -267,7 +264,7 @@ const QuerySpec::FunctionSignature kernel_signatures[] = { { KernelID::ScaledRatio, "ratio", 2, 3, sratio_args }, { KernelID::Scale, "scale", 2, 2, scale_args }, { KernelID::Truncate, "truncate", 1, 2, scale_args }, - { KernelID::First, "first", 2, 8, first_args }, + { KernelID::First, "first", 1, 8, first_args }, QuerySpec::FunctionSignatureTerminator }; @@ -287,14 +284,20 @@ constexpr int MAX_KERNEL_ID = 3; struct Preprocessor::PreprocessorImpl { - std::vector kernels; + std::vector< std::pair > kernels; void configure(const QuerySpec& spec) { for (const auto &pspec : spec.preprocess_ops) { int index = pspec.op.op.id; - if (index >= 0 && index <= MAX_KERNEL_ID) - kernels.push_back((*::kernel_create_fn[index])(pspec.target, pspec.op.args)); + if (index >= 0 && index <= MAX_KERNEL_ID) { + kernels.push_back( + std::make_pair( + RecordSelector(pspec.cond), + (*::kernel_create_fn[index])(pspec.target, pspec.op.args) + ) + ); + } } } @@ -302,7 +305,8 @@ struct Preprocessor::PreprocessorImpl EntryList ret = rec; for (auto &k : kernels) - k->process(db, ret); + if (k.first.pass(db, ret)) + k.second->process(db, ret); return ret; } @@ -313,7 +317,7 @@ struct Preprocessor::PreprocessorImpl ~PreprocessorImpl() { for (auto &k : kernels) - delete k; + delete k.second; } }; diff --git a/src/reader/RecordSelector.cpp b/src/reader/RecordSelector.cpp index 899de2265..d3fa663e8 100644 --- a/src/reader/RecordSelector.cpp +++ b/src/reader/RecordSelector.cpp @@ -24,13 +24,13 @@ namespace QuerySpec::Condition parse_clause(const std::string& str) { - QuerySpec::Condition clause { QuerySpec::Condition::None, "", "" }; - + QuerySpec::Condition clause; + // parse "[-]attribute[(<>=)value]" string if (str.empty()) return clause; - + std::string::size_type spos = 0; bool negate = false; @@ -74,7 +74,7 @@ parse_clause(const std::string& str) } else { clause.op = (negate ? QuerySpec::Condition::Op::NotExist : QuerySpec::Condition::Op::Exist); } - + if (clause.attr_name.empty() || (opos < std::string::npos && clause.value.empty())) clause.op = QuerySpec::Condition::Op::None; @@ -87,21 +87,28 @@ using namespace cali; struct RecordSelector::RecordSelectorImpl { - std::vector m_filters; + std::vector m_filters; struct Clause { QuerySpec::Condition::Op op; Attribute attr; Variant value; - }; + }; void configure(const QuerySpec& spec) { m_filters.clear(); - + if (spec.filter.selection == QuerySpec::FilterSelection::List) m_filters = spec.filter.list; } + void configure(const QuerySpec::Condition& cond) { + m_filters.clear(); + + if (cond.op != QuerySpec::Condition::None) + m_filters.push_back(cond); + } + Clause make_clause(const CaliperMetadataAccessInterface& db, const QuerySpec::Condition& f) { Clause clause { f.op, Attribute::invalid, Variant() }; @@ -109,7 +116,7 @@ struct RecordSelector::RecordSelectorImpl if (clause.attr != Attribute::invalid) clause.value = Variant::from_string(clause.attr.type(), f.value.c_str(), nullptr); - + return clause; } @@ -124,7 +131,7 @@ struct RecordSelector::RecordSelectorImpl } else if (match(entry.attribute(), entry.value())) return true; - return false; + return false; } bool pass(const CaliperMetadataAccessInterface& db, const EntryList& list) { @@ -135,7 +142,7 @@ struct RecordSelector::RecordSelectorImpl case QuerySpec::Condition::Op::Exist: { bool m = false; - + for (const Entry& e : list) if (have_match(e, [&clause](cali_id_t attr_id, const Variant&){ return attr_id == clause.attr.id(); @@ -152,11 +159,11 @@ struct RecordSelector::RecordSelectorImpl return attr_id == clause.attr.id(); })) return false; - break; + break; case QuerySpec::Condition::Op::Equal: { bool m = false; - + for (const Entry& e : list) if (have_match(e, [&clause](cali_id_t attr_id, const Variant& val){ return attr_id == clause.attr.id() && val == clause.value; @@ -232,7 +239,7 @@ struct RecordSelector::RecordSelectorImpl break; case QuerySpec::Condition::Op::None: break; - } + } } return true; @@ -252,6 +259,12 @@ RecordSelector::RecordSelector(const QuerySpec& spec) mP->configure(spec); } +RecordSelector::RecordSelector(const QuerySpec::Condition& cond) + : mP { new RecordSelectorImpl } +{ + mP->configure(cond); +} + RecordSelector::~RecordSelector() { mP.reset(); @@ -263,7 +276,7 @@ RecordSelector::pass(const CaliperMetadataAccessInterface& db, const EntryList& return mP->pass(db, list); } -void +void RecordSelector::operator()(CaliperMetadataAccessInterface& db, const EntryList& list, SnapshotProcessFn push) const { if (mP->pass(db, list)) @@ -273,7 +286,7 @@ RecordSelector::operator()(CaliperMetadataAccessInterface& db, const EntryList& std::vector RecordSelector::parse(const std::string& str) { - std::vector clauses; + std::vector clauses; std::vector clause_strings; util::split(str, ',', std::back_inserter(clause_strings)); diff --git a/src/reader/test/test_calqlparser.cpp b/src/reader/test/test_calqlparser.cpp index 83ae81b11..396824bfc 100644 --- a/src/reader/test/test_calqlparser.cpp +++ b/src/reader/test/test_calqlparser.cpp @@ -489,17 +489,47 @@ TEST(CalQLParserTest, LetClause) { ASSERT_EQ(q1.preprocess_ops[0].op.args.size(), 2); EXPECT_STREQ(q1.preprocess_ops[0].op.args[0].c_str(), "a"); EXPECT_STREQ(q1.preprocess_ops[0].op.args[1].c_str(), "b"); + EXPECT_EQ(q1.preprocess_ops[0].cond.op, QuerySpec::Condition::None); EXPECT_STREQ(q1.preprocess_ops[1].target.c_str(), "y"); EXPECT_STREQ(q1.preprocess_ops[1].op.op.name, "scale"); ASSERT_EQ(q1.preprocess_ops[1].op.args.size(), 2); EXPECT_STREQ(q1.preprocess_ops[1].op.args[0].c_str(), "c"); EXPECT_STREQ(q1.preprocess_ops[1].op.args[1].c_str(), "42"); + EXPECT_EQ(q1.preprocess_ops[1].cond.op, QuerySpec::Condition::None); EXPECT_STREQ(q1.preprocess_ops[2].target.c_str(), "z"); EXPECT_STREQ(q1.preprocess_ops[2].op.op.name, "truncate"); ASSERT_EQ(q1.preprocess_ops[2].op.args.size(), 1); EXPECT_STREQ(q1.preprocess_ops[2].op.args[0].c_str(), "yy"); + EXPECT_EQ(q1.preprocess_ops[2].cond.op, QuerySpec::Condition::None); +} + +TEST(CalQLParserTest, LetIfClause) { + CalQLParser p1("let x= ratio( a, \"b\" ) if not X, y=scale(c,42) if Y = foo let z=truncate ( yy ) if not Z>1"); + + EXPECT_FALSE(p1.error()) << "Unexpected parse error: " << p1.error_msg(); + + QuerySpec q1 = p1.spec(); + + EXPECT_STREQ(q1.preprocess_ops[0].target.c_str(), "x"); + EXPECT_STREQ(q1.preprocess_ops[0].op.op.name, "ratio"); + EXPECT_EQ(q1.preprocess_ops[0].cond.op, QuerySpec::Condition::NotExist); + EXPECT_STREQ(q1.preprocess_ops[0].cond.attr_name.c_str(), "X"); + + EXPECT_STREQ(q1.preprocess_ops[1].target.c_str(), "y"); + EXPECT_STREQ(q1.preprocess_ops[1].op.op.name, "scale"); + EXPECT_EQ(q1.preprocess_ops[1].cond.op, QuerySpec::Condition::Equal); + EXPECT_STREQ(q1.preprocess_ops[1].cond.attr_name.c_str(), "Y"); + EXPECT_STREQ(q1.preprocess_ops[1].cond.value.c_str(), "foo"); + + EXPECT_STREQ(q1.preprocess_ops[2].target.c_str(), "z"); + EXPECT_STREQ(q1.preprocess_ops[2].op.op.name, "truncate"); + ASSERT_EQ(q1.preprocess_ops[2].op.args.size(), 1); + EXPECT_STREQ(q1.preprocess_ops[2].op.args[0].c_str(), "yy"); + EXPECT_EQ(q1.preprocess_ops[2].cond.op, QuerySpec::Condition::LessOrEqual); + EXPECT_STREQ(q1.preprocess_ops[2].cond.attr_name.c_str(), "Z"); + EXPECT_STREQ(q1.preprocess_ops[2].cond.value.c_str(), "1"); } TEST(CalQLParserTest, LetClauseErrors) { diff --git a/src/reader/test/test_preprocessor.cpp b/src/reader/test/test_preprocessor.cpp index b255d35e7..a4b494f22 100644 --- a/src/reader/test/test_preprocessor.cpp +++ b/src/reader/test/test_preprocessor.cpp @@ -41,6 +41,23 @@ make_spec(const std::string& target, const QuerySpec::AggregationOp& op) spec.target = target; spec.op = op; + spec.cond.op = QuerySpec::Condition::None; + + return spec; +} + +QuerySpec::PreprocessSpec +make_spec_with_cond(const std::string& target, const QuerySpec::AggregationOp& op, const QuerySpec::Condition::Op cond, const char* cond_attr, const char* cond_val = nullptr) +{ + QuerySpec::PreprocessSpec spec; + + spec.target = target; + spec.op = op; + spec.cond.op = cond; + if (cond_attr) + spec.cond.attr_name = cond_attr; + if (cond_val) + spec.cond.value = cond_val; return spec; } @@ -319,3 +336,51 @@ TEST(PreprocessorTest, Chain) { EXPECT_DOUBLE_EQ(d_it->second.value().to_double(), 11.0); EXPECT_DOUBLE_EQ(t_it->second.value().to_double(), 10.0); } + +TEST(PreprocessorTest, Conditions) { + // + // --- setup + // + + CaliperMetadataDB db; + IdMap idmap; + + Attribute ctx_a = + db.create_attribute("ctx.1", CALI_TYPE_STRING, CALI_ATTR_DEFAULT); + Attribute val_a = + db.create_attribute("val.a", CALI_TYPE_INT, CALI_ATTR_ASVALUE); + Attribute val_b = + db.create_attribute("val.b", CALI_TYPE_INT, CALI_ATTR_ASVALUE); + + EntryList rec; + + rec.push_back(Entry(db.merge_node(100, ctx_a.id(), CALI_INV_ID, Variant("test.preprocessor.first"), idmap))); + rec.push_back(Entry(val_a, Variant(42))); + rec.push_back(Entry(val_b, Variant(24))); + + QuerySpec spec; + + spec.preprocess_ops.push_back(::make_spec_with_cond("val.a.out", ::make_op("first", "val.a"), QuerySpec::Condition::Exist, "ctx.1")); + spec.preprocess_ops.push_back(::make_spec_with_cond("val.b.out", ::make_op("first", "val.b"), QuerySpec::Condition::NotExist, "ctx.1")); + + // + // --- run + // + + Preprocessor pp(spec); + EntryList out = pp.process(db, rec); + + Attribute vao_attr = db.get_attribute("val.a.out"); + Attribute vbo_attr = db.get_attribute("val.b.out"); + + EXPECT_NE(vao_attr, Attribute::invalid); + EXPECT_EQ(vbo_attr, Attribute::invalid); + EXPECT_EQ(vao_attr.type(), CALI_TYPE_INT); + + auto res = ::make_dict_from_entrylist(out); + auto a_it = res.find(vao_attr.id()); + + ASSERT_NE(a_it, res.end()) << "val.a.out attribute not found\n"; + + EXPECT_EQ(a_it->second.value().to_int(), 42); +}