Changing forcedFilter to forceFilter

core/src/main/scala/com/yahoo/maha/core/BaseTable.scala

@@ -3,6 +3,7 @@
 package com.yahoo.maha.core
 
 import com.yahoo.maha.core.ddl.DDLAnnotation
+import com.yahoo.maha.core.fact.ForceFilter
 
 /**
  * Created by hiral on 11/6/15.
@@ -17,7 +18,7 @@ trait BaseTable {
 
 trait PublicTable {
   def name: String
-  def forcedFilters: Set[ForcedFilter]
+  def forceFilters: Set[ForceFilter]
   def forcedFiltersByAliasMap: Map[String, ForcedFilter]
   def allColumnsByAlias: Set[String]
   def columnsByAliasMap: Map[String, PublicColumn]
@@ -27,10 +28,9 @@ trait PublicTable {
   def restrictedSchemasMap: Map[String, Set[Schema]]
   def requiredFilterAliases : Set[String]
   def validateForcedFilters() : Unit = {
-    forcedFilters.foreach {
-      filter =>
-        require(allColumnsByAlias(filter.field), s"Forced filter on non-existing column : $filter")
-        require(filter.isForceFilter, s"Forced Filter boolean ${filter.isForceFilter}, expected true")
+    forceFilters.foreach { forcedFilter =>
+        require(allColumnsByAlias(forcedFilter.filter.field), s"Forced filter on non-existing column : $forcedFilter")
+        require(forcedFilter.filter.isForceFilter, s"Forced Filter boolean ${forcedFilter.filter.isForceFilter}, expected true")
     }
   }
 }

core/src/main/scala/com/yahoo/maha/core/FilterOperation.scala

@@ -85,6 +85,7 @@ object BaseEquality {
 sealed trait ForcedFilter extends Filter {
   def isForceFilter: Boolean = false
   def isOverridable: Boolean = false
+  def isForceFilterOverridable: Boolean = false
 }
 
 case class PushDownFilter(f: Filter) extends Filter {

core/src/main/scala/com/yahoo/maha/core/RequestModel.scala

@@ -543,10 +543,10 @@ object RequestModel extends Logging {
           }
 
           // populate all forced filters from fact
-          publicFact.forcedFilters.foreach { filter =>
-            if(!allFilterAliases(filter.field)) {
-              allFilterAliases += filter.field
-              filterMap.put(filter.field, filter)
+          publicFact.forceFilters.foreach { forcedFilter =>
+            if(!allFilterAliases(forcedFilter.filter.field)) {
+              allFilterAliases += forcedFilter.filter.field
+              filterMap.put(forcedFilter.filter.field, forcedFilter.filter)
             }
           }
 
@@ -645,6 +645,9 @@ object RequestModel extends Logging {
                   publicDim.name
               }
 
+//              fact.forceFilters.foreach( forceFilter => {
+//                forceFilter.filterCondition
+//              })
               val missingFields = schemaRequiredFilterAliases.filterNot(allFilterAliases.apply)
               require(missingFields.isEmpty,
                 s"required filter for cube=${publicFact.name}, schema=${request.schema}, fact=${fact.name} not found = $missingFields , found = $allFilterAliases")
@@ -778,10 +781,10 @@ object RequestModel extends Logging {
                   }
 
                   // populate all forced filters from dim
-                  val hasForcedFilters = publicDim.forcedFilters.foldLeft(false) {
-                    (b, filter) =>
-                      val result = if(!allNonFactFilterAliases(filter.field) && !filterPostProcess(filter.field)) {
-                        filters += filter
+                  val hasForcedFilters = publicDim.forceFilters.foldLeft(false) {
+                    (b, forcedFilter) =>
+                      val result = if(!allNonFactFilterAliases(forcedFilter.filter.field) && !filterPostProcess(forcedFilter.filter.field)) {
+                        filters += forcedFilter.filter
                         true
                       } else false
                       b || result

core/src/main/scala/com/yahoo/maha/core/dimension/Dimension.scala

@@ -7,7 +7,9 @@ import com.google.common.annotations.VisibleForTesting
 import com.yahoo.maha.core.DruidDerivedFunction.GET_INTERVAL_DATE
 import com.yahoo.maha.core.ddl.{DDLAnnotation, HiveDDLAnnotation}
 import com.yahoo.maha.core._
+import com.yahoo.maha.core.fact.ForceFilter
 import com.yahoo.maha.core.request.RequestType
+
 import scala.collection.{SortedSet, mutable}
 import scala.collection.immutable.TreeSet
 
@@ -524,20 +526,20 @@ case class DimTable private[dimension](name: String
           }}}
       factAliasMap.toMap
     }
-    val forcedFiltersByBasenameMap: Map[String, Filter] = publicDimension.forcedFilters.map{
+    val forcedFiltersByBasenameMap: Map[String, Filter] = publicDimension.forceFilters.map{
       val backwardColByAlias: Map[String, String] =
         publicDimension.columns.map(col => col.alias -> col.name).toMap
-      forceFilter =>
-        val wrappedField = forceFilter.field
+      forcedFilter =>
+        val wrappedField = forcedFilter.filter.field
         if(backwardColByAlias.contains(wrappedField) && baseNameAliasMap.contains(backwardColByAlias(wrappedField))){
-          baseNameAliasMap(backwardColByAlias(wrappedField)) -> forceFilter
+          baseNameAliasMap(backwardColByAlias(wrappedField)) -> forcedFilter.filter
         }
         else{
-          throw new IllegalStateException(s"requirement failed: ${forceFilter.field} doesn't have a known alias")
+          throw new IllegalStateException(s"requirement failed: ${forcedFilter.filter.field} doesn't have a known alias")
         }
     }.toMap
 
-    require(forcedFiltersByBasenameMap.size == publicDimension.forcedFilters.size, "Forced Filters public fact and map of forced base cols differ in size")
+    require(forcedFiltersByBasenameMap.size == publicDimension.forceFilters.size, "Forced Filters public fact and map of forced base cols differ in size")
 
   }
 
@@ -714,7 +716,7 @@ case class DimensionBuilder private[dimension](private val baseDim: Dimension, p
     this
   }
 
-  def toPublicDimension(name: String, grainKey:String, columns: Set[PublicDimColumn], forcedFilters: Set[ForcedFilter] = Set.empty, revision: Int = 0, highCardinalityFilters: Set[Filter] = Set.empty) : PublicDimension = {
+  def toPublicDimension(name: String, grainKey:String, columns: Set[PublicDimColumn], forceFilters: Set[ForceFilter] = Set.empty, revision: Int = 0, highCardinalityFilters: Set[Filter] = Set.empty) : PublicDimension = {
     var ts: TreeSet[Filter] = TreeSet.empty(Filter.baseFilterOrdering)
     ts ++= highCardinalityFilters
     val schemaTableColMap : Map[Schema, Map[String, String]] = {
@@ -731,7 +733,7 @@ case class DimensionBuilder private[dimension](private val baseDim: Dimension, p
       }
       require(groupFlattened.size == 1, s"Schema $schema requires different columns across underlying tables : $groupFlattened")
     }
-    new PublicDim(name, grainKey, baseDim, columns, tableMap, forcedFilters, ts, revision)
+    new PublicDim(name, grainKey, baseDim, columns, tableMap, forceFilters, ts, revision)
   }
 
   @VisibleForTesting
@@ -821,7 +823,7 @@ case class PublicDim (name: String
                       , baseDim: Dimension
                       , columns: Set[PublicDimColumn]
                       , dims: Map[String, Dimension]
-                      , forcedFilters: Set[ForcedFilter]
+                      , forceFilters: Set[ForceFilter]
                       , private val highCardinalityFilters: Set[Filter]
                       , revision: Int = 0) extends PublicDimension {
 
@@ -830,7 +832,7 @@ case class PublicDim (name: String
   validate()
 
   val schemas: Set[Schema] = dims.values.map(_.schemas).flatten.toSet
-  val forcedFiltersByAliasMap: Map[String, ForcedFilter] = forcedFilters.map(f => f.field -> f).toMap
+  val forcedFiltersByAliasMap: Map[String, ForcedFilter] = forceFilters.map(f => f.filter.field -> f.filter).toMap
   val columnsByAliasMap: Map[String, PublicColumn] = columns.map(pdc => pdc.alias -> pdc).toMap
   val nameToDataTypeMap: Map[String, DataType] = dims.values.flatMap(_.columns).map(d=> (d.name,d.dataType)).toMap
 

core/src/main/scala/com/yahoo/maha/core/fact/Fact.scala

@@ -334,7 +334,17 @@ object CostMultiplier {
   )
 }
 
-case class ForceFilter(filter: ForcedFilter) {
+sealed trait FilterCondition {
+  def apply(requestModel: RequestModel) : Boolean
+}
+
+case class OnSelect(fields: Set[String]) extends FilterCondition {
+  override def apply(requestModel: RequestModel): Boolean = {
+    requestModel.requestColsSet.intersect(fields).size > 0
+  }
+}
+
+case class ForceFilter(filter: ForcedFilter, filterCondition: Option[FilterCondition] = None, isForceFilterOverridable: Boolean = false) {
   require(filter.isForceFilter, "Filter must be declared with isForceFilter = true")
   def isOverridable: Boolean = filter.isOverridable
 }
@@ -692,21 +702,21 @@ case class FactTable private[fact](name: String
           }}}
       factAliasMap.toMap
     }
-    val forcedFiltersByBasenameMap: Map[String, Filter] = publicFact.forcedFilters.map{
+    val forcedFiltersByBasenameMap: Map[String, Filter] = publicFact.forceFilters.map{
       val backwardColByAlias: Map[String, String] =
         publicFact.dimCols.map(col => col.alias -> col.name).toMap ++
           publicFact.factCols.map(col => col.alias -> col.name).toMap
       forceFilter =>
-        val wrappedField = forceFilter.field
+        val wrappedField = forceFilter.filter.field
         if(backwardColByAlias.contains(wrappedField) && baseNameAliasMap.contains(backwardColByAlias(wrappedField))){
-          baseNameAliasMap(backwardColByAlias(wrappedField)) -> forceFilter
+          baseNameAliasMap(backwardColByAlias(wrappedField)) -> forceFilter.filter
         }
         else{
-          throw new IllegalStateException(s"requirement failed: ${forceFilter.field} doesn't have a known alias")
+          throw new IllegalStateException(s"requirement failed: ${forceFilter.filter.field} doesn't have a known alias")
         }
     }.toMap
 
-    require(forcedFiltersByBasenameMap.size == publicFact.forcedFilters.size, "Forced Filters public fact and map of forced base cols differ in size")
+    require(forcedFiltersByBasenameMap.size == publicFact.forceFilters.size, "Forced Filters public fact and map of forced base cols differ in size")
 
   }
 
@@ -1454,7 +1464,7 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
   def toPublicFact(name: String
                    , dimCols: Set[PublicDimColumn]
                    , factCols: Set[PublicFactColumn]
-                   , forcedFilters: Set[ForcedFilter]
+                   , forceFilters: Set[ForceFilter]
                    , maxDaysWindow: Map[(RequestType, Grain), Int]
                    , maxDaysLookBack: Map[(RequestType, Grain), Int]
                    , enableUTCTimeConversion: Boolean = true
@@ -1467,7 +1477,7 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
       , dimCols
       , factCols
       , tableMap
-      , forcedFilters
+      , forceFilters
       , maxDaysWindow
       , maxDaysLookBack
       , dimCardinalityLookup
@@ -1606,7 +1616,7 @@ case class PublicFactTable private[fact](name: String
                                          , dimCols: Set[PublicDimColumn]
                                          , factCols: Set[PublicFactColumn]
                                          , facts: Map[String, Fact]
-                                         , forcedFilters: Set[ForcedFilter]
+                                         , forceFilters: Set[ForceFilter]
                                          , maxDaysWindow: Map[(RequestType, Grain), Int]
                                          , maxDaysLookBack: Map[(RequestType, Grain), Int]
                                          , dimCardinalityLookup: Option[LongRangeLookup[Map[RequestType, Map[Engine, Int]]]]
@@ -1620,7 +1630,7 @@ case class PublicFactTable private[fact](name: String
   //generate column list by alias
   val columnsByAlias: Set[String] = dimCols.map(col => col.alias) ++ factCols.map(col => col.alias)
   val allColumnsByAlias: Set[String] = columnsByAlias
-  val forcedFiltersByAliasMap: Map[String, ForcedFilter] = forcedFilters.map(f => f.field -> f).toMap
+  val forcedFiltersByAliasMap: Map[String, ForcedFilter] = forceFilters.map(f => f.filter.field -> f.filter).toMap
 
   val foreignKeyAliases: Set[String] = dimCols.filter(col => facts
     .values

core/src/main/scala/com/yahoo/maha/core/query/QueryGenerator.scala

@@ -4,7 +4,7 @@ package com.yahoo.maha.core.query
 
 import com.yahoo.maha.core._
 import com.yahoo.maha.core.dimension._
-import com.yahoo.maha.core.fact.FactBestCandidate
+import com.yahoo.maha.core.fact.{FactBestCandidate, ForceFilter}
 import com.yahoo.maha.core.query.Version.{v0, v1, v2}
 
 import scala.collection.mutable
@@ -181,7 +181,7 @@ trait QueryGenerator[T <: EngineRequirement] {
 
 trait BaseQueryGenerator[T <: EngineRequirement] extends QueryGenerator[T] {
 
-  def removeDuplicateIfForced(localFilters: Seq[Filter], forcedFilters: Seq[ForcedFilter], inputContext: FactualQueryContext): Array[Filter] = {
+  def removeDuplicateIfForced(localFilters: Seq[Filter], forceFilters: Seq[ForceFilter], inputContext: FactualQueryContext): Array[Filter] = {
     val queryContext = inputContext
 
     val fact = queryContext.factBestCandidate.fact
@@ -193,8 +193,9 @@ trait BaseQueryGenerator[T <: EngineRequirement] extends QueryGenerator[T] {
         val real_name = column.alias.getOrElse(name)
         returnedFilters(real_name) = filter
     }
-    forcedFilters.foreach {
-      filter =>
+    forceFilters.foreach {
+      forcedFilter =>
+        val filter = forcedFilter.filter
         val name = queryContext.factBestCandidate.publicFact.aliasToNameColumnMap(filter.field)
         val column = fact.columnsByNameMap(name)
         val real_name = column.alias.getOrElse(name)

core/src/main/scala/com/yahoo/maha/core/query/druid/DruidQueryGenerator.scala

@@ -1231,7 +1231,7 @@ class DruidQueryGenerator(queryOptimizer: DruidQueryOptimizer
     val fact = queryContext.factBestCandidate.fact
     val publicFact = queryContext.factBestCandidate.publicFact
     val filters = queryContext.factBestCandidate.filters
-    val factForcedFilters = queryContext.factBestCandidate.publicFact.forcedFilters
+    val factForcedFilters = queryContext.factBestCandidate.publicFact.forceFilters
     val allFilters = factForcedFilters // ++ filters need to append regular filters or pass in
     val whereFilters = new ArrayBuffer[DimFilter](filters.size)
     val havingFilters = new ArrayBuffer[HavingSpec](filters.size)

core/src/main/scala/com/yahoo/maha/core/query/hive/HiveQueryGeneratorCommon.scala

@@ -171,7 +171,7 @@ abstract class HiveQueryGeneratorCommon(partitionColumnRenderer:PartitionColumnR
     val hasPartitioningScheme = fact.annotations.contains(HiveQueryGenerator.ANY_PARTITIONING_SCHEME)
 
     val factFilters = queryContext.factBestCandidate.filters
-    val factForcedFilters = queryContext.factBestCandidate.publicFact.forcedFilters
+    val factForcedFilters = queryContext.factBestCandidate.publicFact.forceFilters
     val aliasToNameMapFull = queryContext.factBestCandidate.publicFact.aliasToNameColumnMap
     val allFilters = factForcedFilters // ++ factFilters need to append non-forced filters, or otherwise pass them in separately
 

core/src/main/scala/com/yahoo/maha/core/query/oracle/OracleQueryGenerator.scala

@@ -1027,7 +1027,7 @@ b. Dim Driven
       val fact = queryContext.factBestCandidate.fact
       val publicFact = queryContext.factBestCandidate.publicFact
       val filters = queryContext.factBestCandidate.filters
-      val allFilters = publicFact.forcedFilters //++ filters  need to append regular filters or pass in
+      val allFilters = publicFact.forceFilters //++ filters  need to append regular filters or pass in
       val whereFilters = new mutable.LinkedHashSet[String]
       val havingFilters = new mutable.LinkedHashSet[String]
       var escaped = false

core/src/main/scala/com/yahoo/maha/core/query/oracle/OuterGroupByQueryGenerator.scala

@@ -75,7 +75,7 @@ abstract class OuterGroupByQueryGenerator(partitionColumnRenderer:PartitionColum
       val fact = queryContext.factBestCandidate.fact
       val publicFact = queryContext.factBestCandidate.publicFact
       val filters = queryContext.factBestCandidate.filters
-      val allFilters = publicFact.forcedFilters //++ filters  need to append regular filters or pass in
+      val allFilters = publicFact.forceFilters //++ filters  need to append regular filters or pass in
       val whereFilters = new mutable.LinkedHashSet[String]
       val havingFilters = new mutable.LinkedHashSet[String]
       var escaped = false

core/src/main/scala/com/yahoo/maha/core/query/presto/PrestoQueryGenerator.scala

@@ -336,7 +336,7 @@ class PrestoQueryGenerator(partitionColumnRenderer:PartitionColumnRenderer, udfS
       val hasPartitioningScheme = fact.annotations.contains(PrestoQueryGenerator.ANY_PARTITIONING_SCHEME)
 
       val factFilters = queryContext.factBestCandidate.filters
-      val factForcedFilters = queryContext.factBestCandidate.publicFact.forcedFilters
+      val factForcedFilters = queryContext.factBestCandidate.publicFact.forceFilters
       val aliasToNameMapFull = queryContext.factBestCandidate.publicFact.aliasToNameColumnMap
       val allFilters = factForcedFilters // ++ factFilters need to append non-forced filters, or otherwise pass them in separately