calql: Add LET ... IF (LLNL#326)

* calql: Parse let ... if

* calql: Add let ... if

* Add documentation for LET ... IF

* Initialize condition structs properly

doc/sphinx/calql.rst

@@ -28,6 +28,7 @@ This table contains a quick reference of all CalQL statements:
     <a>=scale(<b>,<S>)         # computes <a> = <b>*S
     <a>=truncate(<b>,<S>)      # computes <a> = <b> - mod(<b>, S)
     <a>=first(<a0>,<a1>, ...)  # <a> is the first of <a0>, <a1>, ... found in the input record
+    ... IF <condition>         # apply only if input record meets condition
 
   SELECT <list>                # Select attributes and define aggregations (i.e., select columns)
     *                          # select all attributes
@@ -56,8 +57,7 @@ This table contains a quick reference of all CalQL statements:
     <attribute> = <value>      # records where <attribute> = <value>
     <attribute> < <value>      # records where <attribute> > <value>
     <attribute> > <value>      # records where <attribute> < <value>
-    NOT <attribute>            # records where <attribute> does not exist
-    NOT <attribute>=<value>    # records where <attribute>!=<value>
+    NOT <clause>               # negate the filter clause
 
   FORMAT <formatter>           # Define output format
     cali                       # .cali format
@@ -83,25 +83,55 @@ can then be used in subsequent SELECT, GROUP BY, or FORMAT statements.
 For example, we can use the scale() operator to scale a value before
 subsequent aggregations::
 
-  LET sec=scale(time.duration,1e-6) SELECT prop:nested,sum(sec)
+  LET
+    sec=scale(time.duration,1e-6)
+  SELECT
+    prop:nested,sum(sec)
 
-For example, we can use the truncate() operator on an iteration counter to
+We can use the truncate() operator on an iteration counter to
 aggregate blocks of 10 iterations in a time-series profile::
 
-  LET block=truncate(iteration#mainloop,10) SELECT block,sum(time.duration) GROUP BY block
+  LET
+    block=truncate(iteration#mainloop,10)
+  SELECT
+    block,sum(time.duration)
+  GROUP BY
+    block
 
 The first() operator returns the first attribute out of a list of attribute
 names found in an input record. It can also be used to rename attributes::
 
-  LET time=first(time.duration,sum#time.duration) SELECT sum(time) AS Time GROUP BY prop:nested
+  LET
+    time=first(time.duration,sum#time.duration)
+  SELECT
+    sum(time) AS Time
+  GROUP BY
+    prop:nested
 
 LET terms have the general form
 
-  a = f(...)
+  a = f(...) [IF <condition>]
 
 where f is one of the operators and `a` is the name of the result attribute.
 The result is added to the input records before the record is processed further.
-Result entries are only added to a record if all required input operands are present.
+Result entries are only added to a record if all required input operands are
+present.
+
+With the optional IF condition, the operation is only applied for input records
+that meet a condition. One can use this to compute values for a specific
+subset of records. The condition clauses use the same syntax as WHERE filter
+clauses. The example below defines a "work" attribute with the time in
+records that contain "omp.work" regions, and then uses that to compute
+efficiency from the total and "work" time:
+
+  LET
+    work=first(time.duration) IF omp.work
+  SELECT
+    sum(time.duration)        AS Total,
+    sum(work)                 AS Work,
+    ratio(work,time.duration) AS Efficiency
+  GROUP BY
+    prop:nested
 
 SELECT
 --------------------------------
@@ -140,7 +170,14 @@ applies to the hierarchy defined by attributes with the
 
 A more complex example::
 
-  SELECT *, scale(time.duration,1e-6) AS Time, inclusive_percent_total(time.duration) AS "Time %" GROUP BY prop:nested FORMAT tree
+  SELECT
+    *,
+    scale(time.duration,1e-6) AS Time,
+    inclusive_percent_total(time.duration) AS "Time %"
+  GROUP BY
+    prop:nested
+  FORMAT
+    tree
 
 The computes the (exclusive) sum of `time.duration` divided by 100000 and the inclusive
 percent-of-total for `time.duration`. Example output::
@@ -234,8 +271,14 @@ The following example prints a iteration/function profile ordered by
 time and iteration number. Note that one must use the original
 attribute name and not an alias assigned with ``AS``: ::
 
-  SELECT *, sum(time.inclusive.duration) AS Time FORMAT table \
-    ORDER BY sum#time.inclusive.duration DESC, iteration#mainloop
+  SELECT
+    *,
+    sum(time.inclusive.duration) AS Time
+  FORMAT
+    table
+  ORDER BY
+    sum#time.inclusive.duration DESC,
+    iteration#mainloop
 
   function loop     iteration#mainloop     Time
   main                                   100000

include/caliper/reader/QuerySpec.h

@@ -89,6 +89,14 @@ struct QuerySpec
         }           op;
         std::string attr_name;
         std::string value;
+
+        Condition()
+            : op(Op::None)
+        { }
+
+        Condition(Condition::Op o, const std::string& name, const std::string& val)
+            : op(o), attr_name(name), value(val)
+        { }
     };
 
     /// \brief Output formatter specification.
@@ -103,6 +111,7 @@ struct QuerySpec
     struct PreprocessSpec {
         std::string   target;
         AggregationOp op;
+        Condition     cond;
     };
 
     //

include/caliper/reader/RecordSelector.h

@@ -29,6 +29,7 @@ class RecordSelector
 
     RecordSelector(const std::string& filter_string);
     RecordSelector(const QuerySpec& spec);
+    RecordSelector(const QuerySpec::Condition& cond);
 
     ~RecordSelector();
 

src/reader/CalQLParser.cpp

@@ -362,7 +362,7 @@ struct CalQLParser::CalQLParserImpl
             parse_clause_from_word(next_keyword, is);
     }
 
-    void
+    QuerySpec::Condition
     parse_filter_clause(std::istream& is) {
         std::string w = util::read_word(is, ",;=<>()\n");
         std::string wl(w);
@@ -376,16 +376,15 @@ struct CalQLParser::CalQLParserImpl
             w = util::read_word(is, ",;=<>()\n");
         }
 
-        if (w.empty()) {
-            set_error("Condition term expected", is);
-            return;
-        }
-
         QuerySpec::Condition cond;
-
         cond.op        = QuerySpec::Condition::None;
         cond.attr_name = w;
 
+        if (w.empty()) {
+            set_error("Condition term expected", is);
+            return cond;
+        }
+
         char c = util::read_char(is);
 
         switch (c) {
@@ -427,20 +426,25 @@ struct CalQLParser::CalQLParserImpl
             cond.op = (negate ? QuerySpec::Condition::NotExist : QuerySpec::Condition::Exist);
         }
 
-        if (cond.op != QuerySpec::Condition::None) {
-            spec.filter.selection = QuerySpec::FilterSelection::List;
-            spec.filter.list.push_back(cond);
-        } else {
+        if (cond.op == QuerySpec::Condition::None) {
             set_error("Condition term expected", is);
         }
+
+        return cond;
     }
 
     void
     parse_where(std::istream& is) {
         char c = '\0';
 
         do {
-            parse_filter_clause(is);
+            QuerySpec::Condition cond = parse_filter_clause(is);
+
+            if (!error && cond.op != QuerySpec::Condition::None) {
+                spec.filter.selection = QuerySpec::FilterSelection::List;
+                spec.filter.list.push_back(cond);
+            }
+
             c = util::read_char(is);
         } while (!error && is.good() && c == ',');
 
@@ -450,10 +454,12 @@ struct CalQLParser::CalQLParserImpl
 
     void
     parse_let(std::istream& is) {
+        std::string next_keyword;
         char c = 0;
 
         do {
             const QuerySpec::FunctionSignature* defs = Preprocessor::preprocess_defs();
+            QuerySpec::PreprocessSpec pspec;
 
             std::string target = util::read_word(is, ",;=<>()\n");
 
@@ -489,21 +495,41 @@ struct CalQLParser::CalQLParserImpl
                             });
 
                 if (it == spec.preprocess_ops.end()) {
-                    QuerySpec::PreprocessSpec pspec;
-
                     pspec.target = target;
                     pspec.op = QuerySpec::AggregationOp(defs[i], args);
-
-                    spec.preprocess_ops.emplace_back(std::move(pspec));
-                } else
+                } else {
                     set_error(target + " defined twice", is);
+                    return;
+                }
+            }
+
+            // parse condition (... IF ... )
+
+            next_keyword.clear();
+            std::string tmp = util::read_word(is, ",;=<>()\n");
+            std::transform(tmp.begin(), tmp.end(), std::back_inserter(next_keyword), ::tolower);
+
+            if (next_keyword == "if") {
+                pspec.cond = parse_filter_clause(is);
+                next_keyword.clear();
+            }
+
+            if (!error) {
+                spec.preprocess_ops.emplace_back(std::move(pspec));
+
+                if (!next_keyword.empty()) {
+                    c = 0;
+                    break;
+                }
             }
 
             c = util::read_char(is);
         } while (!error && is.good() && c == ',');
 
         if (c)
             is.unget();
+        if (!next_keyword.empty())
+            parse_clause_from_word(next_keyword, is);
     }
 
     void

src/reader/Preprocessor.cpp

@@ -6,6 +6,7 @@
 #include "caliper/reader/Preprocessor.h"
 
 #include "caliper/reader/QuerySpec.h"
+#include "caliper/reader/RecordSelector.h"
 
 #include "caliper/common/Attribute.h"
 #include "caliper/common/CaliperMetadataAccessInterface.h"
@@ -14,13 +15,9 @@
 
 #include "caliper/common/cali_types.h"
 
-#include <algorithm>
-#include <cassert>
-#include <cstring>
 #include <cmath>
-#include <iostream>
-#include <iterator>
-#include <mutex>
+#include <vector>
+#include <utility>
 
 using namespace cali;
 
@@ -221,14 +218,14 @@ class FirstKernel : public Kernel
         {
             m_tgt_attrs.assign(args.size(), Attribute::invalid);
         }
-    
+
     void process(CaliperMetadataAccessInterface& db, EntryList& rec) {
         for (size_t i = 0; i < m_tgt_attrs.size(); ++i) {
             Variant v_tgt = get_value(db, m_tgt_attr_names[i], m_tgt_attrs[i], rec);
 
             if (v_tgt.empty())
                 continue;
-                        
+
             cali_attr_type type = m_tgt_attrs[i].type();
 
             if (m_res_attr == Attribute::invalid)
@@ -257,17 +254,17 @@ enum KernelID {
 
 const char* sratio_args[]  = { "numerator", "denominator", "scale" };
 const char* scale_args[]   = { "attribute", "scale" };
-const char* first_args[]   = { 
-    "attribute0", "attribute1", "attribute2", 
-    "attribute3", "attribute4", "attribute5", 
+const char* first_args[]   = {
+    "attribute0", "attribute1", "attribute2",
+    "attribute3", "attribute4", "attribute5",
     "attribute6", "attribute7", "attribute8"
 };
 
 const QuerySpec::FunctionSignature kernel_signatures[] = {
     { KernelID::ScaledRatio,   "ratio",         2, 3, sratio_args  },
     { KernelID::Scale,         "scale",         2, 2, scale_args   },
     { KernelID::Truncate,      "truncate",      1, 2, scale_args   },
-    { KernelID::First,         "first",         2, 8, first_args   },
+    { KernelID::First,         "first",         1, 8, first_args   },
 
     QuerySpec::FunctionSignatureTerminator
 };
@@ -287,22 +284,29 @@ constexpr int MAX_KERNEL_ID = 3;
 
 struct Preprocessor::PreprocessorImpl
 {
-    std::vector<Kernel*> kernels;
+    std::vector< std::pair<RecordSelector, Kernel*> > kernels;
 
     void configure(const QuerySpec& spec) {
         for (const auto &pspec : spec.preprocess_ops) {
             int index = pspec.op.op.id;
 
-            if (index >= 0 && index <= MAX_KERNEL_ID)
-                kernels.push_back((*::kernel_create_fn[index])(pspec.target, pspec.op.args));
+            if (index >= 0 && index <= MAX_KERNEL_ID) {
+                kernels.push_back(
+                        std::make_pair(
+                            RecordSelector(pspec.cond),
+                            (*::kernel_create_fn[index])(pspec.target, pspec.op.args)
+                        )
+                    );
+            }
         }
     }
 
     EntryList process(CaliperMetadataAccessInterface& db, const EntryList& rec) {
         EntryList ret = rec;
 
         for (auto &k : kernels)
-            k->process(db, ret);
+            if (k.first.pass(db, ret))
+                k.second->process(db, ret);
 
         return ret;
     }
@@ -313,7 +317,7 @@ struct Preprocessor::PreprocessorImpl
 
     ~PreprocessorImpl() {
         for (auto &k : kernels)
-            delete k;
+            delete k.second;
     }
 };