Skip to content

Commit

Permalink
Fix outer group by query, reduce registry creation to speed up tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@patelh
patelh committed Aug 23, 2018
1 parent 9ed8ee6 commit de492fa
Show file tree
Hide file tree
Showing 9 changed files with 355 additions and 353 deletions.
4 changes: 2 additions & 2 deletions core/src/main/scala/com/yahoo/maha/core/fact/Fact.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,14 @@ case class FactCol(name: String,
case DruidFilteredRollup(filter, de, delegateAggregatorRollupExpression) =>
columnContext.render(de.fieldNamePlaceHolder, Map.empty)
columnContext.render(filter.field, Map.empty)
case PrestoCustomRollup(de) =>
de.sourceColumns.foreach((name: String) => columnContext.render(name, Map.empty))
case DruidFilteredListRollup(filters, de, delegateAggregatorRollupExpression) =>
columnContext.render(de.fieldNamePlaceHolder, Map.empty)
for( filter <- filters) {
columnContext.render(filter.field, Map.empty)
}
case DruidThetaSketchRollup =>
case PrestoCustomRollup(de) =>
de.sourceColumns.foreach((name: String) => columnContext.render(name, Map.empty))
case customRollup: CustomRollup =>
//error, we missed a check on custom rollup
throw new IllegalArgumentException(s"Need a check on custom rollup")
Expand Down
8 changes: 6 additions & 2 deletions core/src/main/scala/com/yahoo/maha/core/query/oracle/OuterGroupByQueryGenerator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ abstract class OuterGroupByQueryGenerator(partitionColumnRenderer:PartitionColum
val fact = queryContext.factBestCandidate.fact

val dimCols = queryContext.factBestCandidate.dimColMapping.toList.collect {
case (dimCol, alias) if queryContext.factBestCandidate.requestCols(dimCol) =>
case (dimCol, alias) if (queryContext.factBestCandidate.requestCols(dimCol)
|| queryContext.factBestCandidate.publicFact.foreignKeyAliases(alias)) =>
val column = fact.columnsByNameMap(dimCol)
(column, alias)
}
Expand Down Expand Up @@ -217,7 +218,10 @@ abstract class OuterGroupByQueryGenerator(partitionColumnRenderer:PartitionColum
groupedFactCols.foreach {
case (f:FactCol, colAlias: String) if f.rollupExpression.isInstanceOf[OracleCustomRollup] =>
customRollupSet.add((f,colAlias))
queryBuilderContext.setFactColAlias(f.alias.getOrElse(f.name), colAlias, f)
//if custom rollup depends on itself as a primitive col, then don't add to context
if(!f.rollupExpression.asInstanceOf[OracleCustomRollup].expression.sourceColumns(f.name)) {
queryBuilderContext.setFactColAlias(f.alias.getOrElse(f.name), colAlias, f)
}
case _=> // ignore for all the other cases as they are handled
}
customRollupSet
Expand Down
87 changes: 44 additions & 43 deletions core/src/test/scala/com/yahoo/maha/core/query/DefaultQueryPipelineFactoryTest.scala
View file

Large diffs are not rendered by default.

104 changes: 53 additions & 51 deletions core/src/test/scala/com/yahoo/maha/core/query/druid/DruidQueryGeneratorTest.scala
View file

Large diffs are not rendered by default.

52 changes: 27 additions & 25 deletions core/src/test/scala/com/yahoo/maha/core/query/hive/HiveQueryGeneratorTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import com.yahoo.maha.core.request.ReportingRequest
*/
class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {

lazy val defaultRegistry = getDefaultRegistry()

test("registering Hive query generation multiple times should fail") {
intercept[IllegalArgumentException] {
val dummyQueryGenerator = new QueryGenerator[WithHiveEngine] {
Expand All @@ -36,7 +38,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_generator_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)

assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))
Expand All @@ -50,7 +52,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_generator_test_custom_rollups.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)

assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))
Expand All @@ -67,7 +69,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "user_stats_hourly.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -85,7 +87,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_date_field.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -100,7 +102,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_escaping_required_field.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -115,7 +117,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_generator_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -130,7 +132,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_generator_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -144,7 +146,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_generator_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -159,7 +161,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_generator_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -179,7 +181,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_string_field.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -196,7 +198,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_min_max_dec_field.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -211,7 +213,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_constant_field.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -227,7 +229,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_fact_n_factder_field.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -243,7 +245,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_request_with_multiple_dimension.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -258,7 +260,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "ce_stats.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -273,7 +275,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_nooprollup_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand Down Expand Up @@ -305,7 +307,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_with_non_fk_dim_filters_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -323,7 +325,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_with_wo_non_fk_dim_filters_test.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -341,7 +343,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_with_dim_like_filter.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand Down Expand Up @@ -375,7 +377,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
val jsonString = scala.io.Source.fromFile(getBaseDir + "hive_query_with_dim_like_filter_injection_testing.json")
.getLines().mkString.replace("{from_date}", fromDate).replace("{to_date}", toDate)
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand Down Expand Up @@ -431,7 +433,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
|}
""".stripMargin
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand Down Expand Up @@ -492,7 +494,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
|}
""".stripMargin
val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand All @@ -518,7 +520,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
}"""

val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand Down Expand Up @@ -565,7 +567,7 @@ class HiveQueryGeneratorTest extends BaseHiveQueryGeneratorTest {
}"""

val request: ReportingRequest = getReportingRequestAsync(jsonString)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))

Expand Down Expand Up @@ -1413,7 +1415,7 @@ GROUP BY a2.mang_ad_status,c1.mang_campaign_name,af0.campaign_id) outergroupby

def generateHiveQuery(requestJson: String): String = {
val requestRaw = ReportingRequest.deserializeAsync(requestJson.getBytes(StandardCharsets.UTF_8), AdvertiserSchema)
val registry = getDefaultRegistry()
val registry = defaultRegistry
val request = ReportingRequest.forceHive(requestRaw.toOption.get)
val requestModel = RequestModel.from(request, registry)
assert(requestModel.isSuccess, requestModel.errorMessage("Building request model failed"))
Expand Down
Loading

0 comments on commit de492fa

Please sign in to comment.