Skip to content

Commit

Permalink
Allow public columns to be renamed while registering a new publicFact…
Browse files Browse the repository at this point in the history 
… alias
  • Loading branch information
Surabhi Pandit committed Jun 26, 2020
1 parent 36723f1 commit da68867
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 35 deletions.
2 changes: 2 additions & 0 deletions core/src/main/scala/com/yahoo/maha/core/Column.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ trait PublicColumn {
def filteringRequired: Boolean
def restrictedSchemas: Set[Schema]
def isImageColumn: Boolean
def isReplacement: Boolean

private val jUtils = JsonUtils

Expand All @@ -36,6 +37,7 @@ trait PublicColumn {
,("hiddenFromJson" -> toJSON(hiddenFromJson))
,("filteringRequired" -> toJSON(filteringRequired))
,("isImageColumn" -> toJSON(isImageColumn))
,("isReplacement" -> toJSON(isReplacement))
)
)
}
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/scala/com/yahoo/maha/core/dimension/Dimension.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -871,7 +871,8 @@ case class PubCol(name: String
, hiddenFromJson: Boolean = false
, filteringRequired: Boolean = false
, restrictedSchemas: Set[Schema] = Set.empty
, isImageColumn: Boolean = false) extends PublicDimColumn
, isImageColumn: Boolean = false
, isReplacement: Boolean = false) extends PublicDimColumn

case class RequiredAlias(name: String, alias: String, isKey: Boolean)
trait PublicDimension extends PublicTable {
Expand Down
5 changes: 3 additions & 2 deletions core/src/main/scala/com/yahoo/maha/core/fact/Fact.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1589,6 +1589,7 @@ case class PublicFactCol (name: String
, hiddenFromJson: Boolean = false
, filteringRequired: Boolean = false
, isImageColumn: Boolean = false
, isReplacement: Boolean = false
, restrictedSchemas: Set[Schema] = Set.empty) extends PublicFactColumn

case class FactBestCandidate(fkCols: SortedSet[String]
Expand Down Expand Up @@ -2012,13 +2013,13 @@ case class PublicFactTable private[fact](name: String
require(maxDaysLookBack.nonEmpty, "No max days look back window defined, public fact supports no request types!")
require(facts.nonEmpty, s"public fact $name has no underlying facts")
require(columnsByAlias.size == (dimCols.size + factCols.size), "Column names size mismatch with dimCols + factCols")
require(lowerCaseAliases.size == (dimCols.size + factCols.size), "Column aliases size mismatch with dimCols + factCols")
require(lowerCaseAliases.size == (dimCols.filterNot(c => c.isReplacement).size + factCols.size), "Column aliases size mismatch with dimCols + factCols")

//run post validate on each fact
facts.values.foreach(_.postValidate(this))

var dimColNames: Set[String] = Set()
dimCols.foreach { col =>
dimCols.filterNot(c => c.isReplacement).foreach { col =>
require(!dimColNames.contains(col.name), "dim column names should be unique")
dimColNames += col.name
}
Expand Down
64 changes: 32 additions & 32 deletions core/src/test/scala/com/yahoo/maha/core/RequestModelTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5665,38 +5665,38 @@ class RequestModelTest extends FunSuite with Matchers {
//println(s"""All tables with fact ${pubFact.get.name}: ${fkTableNames.mkString(",")}""")

val allPubCols: List[String] = List(
"""{"name":"device_type","alias":"Device Type","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set(Device ID)","filters":"Set(In)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"name","alias":"Advertiser Name","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"network_type","alias":"Network Type","schemas":"List(advertiser)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =, IsNotNull, Not In)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"status","alias":"Product Ad Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"id","alias":"Campaign ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"ad_format_id","alias":"Ad Format Name","schemas":"List(reseller)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(=)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"status","alias":"Campaign Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"stats_date","alias":"Day","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Between, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"landing_page_url","alias":"Destination URL","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set()","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"id","alias":"Product Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"status","alias":"Ad Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"description","alias":"Product Ad Description","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"clicks","alias":"Clicks","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"id","alias":"Ad Group ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"Ad Group Start Date Full","alias":"Ad Group Start Date Full","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"name","alias":"Ad Group Name","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"stats_source","alias":"Source","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(=)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"status","alias":"Advertiser Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"advertiser_id","alias":"Advertiser ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Not In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"status","alias":"Ad Group Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"email","alias":"Advertiser Email","schemas":"List(internal)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"product_ad_id","alias":"Product Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"id","alias":"Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"ad_group_id","alias":"Ad Group ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"campaign_id","alias":"Campaign ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"id","alias":"Advertiser ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Not In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"device_id","alias":"Device ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set(Device Type)","filters":"Set(In)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"ad_id","alias":"Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"name","alias":"Campaign Name","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =, Like)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"price_type","alias":"Pricing Type","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(<>, In, =, <, Between, Not In, >)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"spend","alias":"Spend","schemas":"List(reseller, advertiser)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Between, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
,"""{"name":"impressions","alias":"Impressions","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Between, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false}"""
"""{"name":"device_type","alias":"Device Type","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set(Device ID)","filters":"Set(In)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"name","alias":"Advertiser Name","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"network_type","alias":"Network Type","schemas":"List(advertiser)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =, IsNotNull, Not In)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"status","alias":"Product Ad Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"id","alias":"Campaign ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"ad_format_id","alias":"Ad Format Name","schemas":"List(reseller)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(=)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"status","alias":"Campaign Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"stats_date","alias":"Day","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Between, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"landing_page_url","alias":"Destination URL","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set()","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"id","alias":"Product Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"status","alias":"Ad Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"description","alias":"Product Ad Description","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"clicks","alias":"Clicks","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"id","alias":"Ad Group ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"Ad Group Start Date Full","alias":"Ad Group Start Date Full","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"name","alias":"Ad Group Name","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"stats_source","alias":"Source","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(=)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"status","alias":"Advertiser Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"advertiser_id","alias":"Advertiser ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Not In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"status","alias":"Ad Group Status","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"email","alias":"Advertiser Email","schemas":"List(internal)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"product_ad_id","alias":"Product Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"id","alias":"Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"ad_group_id","alias":"Ad Group ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"campaign_id","alias":"Campaign ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"id","alias":"Advertiser ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Not In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"device_id","alias":"Device ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set(Device Type)","filters":"Set(In)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"ad_id","alias":"Ad ID","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"name","alias":"Campaign Name","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, =, Like)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"price_type","alias":"Pricing Type","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(<>, In, =, <, Between, Not In, >)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"spend","alias":"Spend","schemas":"List(reseller, advertiser)","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Between, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
,"""{"name":"impressions","alias":"Impressions","schemas":"List()","dependsOnColumns":"Set()","incompatibleColumns":"Set()","filters":"Set(In, Between, =)","required":false,"hiddenFromJson":false,"filteringRequired":false,"isImageColumn":false,"isReplacement":false}"""
)

val allBaseCols: List[String] = List(
Expand Down
19 changes: 19 additions & 0 deletions core/src/test/scala/com/yahoo/maha/core/registry/RegistryTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,25 @@ class RegistryTest extends FunSuite with Matchers {

}

test("Should allow aliasing of a publicFact with column replacements") {
val pubFact1: PublicFact = pubfact
val registryBuilder = new RegistryBuilder

registryBuilder.register(base_dim)
registryBuilder.register(pubFact1)
registryBuilder.registerAlias(
Set((pubFact1.name, Some(3)), ("alias2", None)),
pubFact1,
dimColOverrides = Set(PubCol("price_type", "Pricing type", InEquality, isReplacement = true)))
val registry = registryBuilder.build()

assert(
registry.factMap.keys.toSet.contains(("alias2", 0))
&& registry.factMap.keys.toSet.contains(("publicFact", 0))
&& registry.factMap.keys.toSet.contains(("publicFact", 3))
&& registry.factMap.get(("alias2", 0)).get.columnsByAlias.contains("Pricing type"))
}

test("Should fail with invalid secondary alias (duplicate)") {
val pubFact1 = pubfact
val registryBuilder = new RegistryBuilder
Expand Down

0 comments on commit da68867

Please sign in to comment.