Open-sourcing Unified User Actions

Unified User Action (UUA) is a centralized, real-time stream of user actions on Twitter, consumed by various product, ML, and marketing teams. UUA makes sure all internal teams consume the uniformed user actions data in an accurate and fast way.

unified_user_actions/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+CONFIG.ini
+PROJECT
+docs

unified_user_actions/BUILD.bazel

@@ -0,0 +1 @@
+# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD

unified_user_actions/README.md

@@ -0,0 +1,10 @@
+# Unified User Actions (UUA)
+
+**Unified User Actions** (UUA) is a centralized, real-time stream of user actions on Twitter, consumed by various product, ML, and marketing teams. UUA reads client-side and server-side event streams that contain the user's actions and generates a unified real-time user actions Kafka stream. The Kafka stream is replicated to HDFS, GCP Pubsub, GCP GCS, GCP BigQuery.  The user actions include public actions such as favorites, retweets, replies and implicit actions like bookmark, impression, video view.
+
+## Components 
+
+- adapter: transform the raw inputs to UUA Thrift output
+- client: Kafka client related utils
+- kafka: more specific Kafka utils like customized serde
+- service: deployment, modules and services

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/AbstractAdapter.scala

@@ -0,0 +1,19 @@
+package com.twitter.unified_user_actions.adapter
+
+import com.twitter.finagle.stats.NullStatsReceiver
+import com.twitter.finagle.stats.StatsReceiver
+
+trait AbstractAdapter[INPUT, OUTK, OUTV] extends Serializable {
+
+  /**
+   * The basic input -> seq[output] adapter which concrete adapters should extend from
+   * @param input a single INPUT
+   * @return A list of (OUTK, OUTV) tuple. The OUTK is the output key mainly for publishing to Kafka (or Pubsub).
+   *         If other processing, e.g. offline batch processing, doesn't require the output key then it can drop it
+   *         like source.adaptOneToKeyedMany.map(_._2)
+   */
+  def adaptOneToKeyedMany(
+    input: INPUT,
+    statsReceiver: StatsReceiver = NullStatsReceiver
+  ): Seq[(OUTK, OUTV)]
+}

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/BUILD

@@ -0,0 +1,11 @@
+scala_library(
+    name = "base",
+    sources = [
+        "AbstractAdapter.scala",
+    ],
+    compiler_option_sets = ["fatal_warnings"],
+    tags = ["bazel-compatible"],
+    dependencies = [
+        "util/util-stats/src/main/scala/com/twitter/finagle/stats",
+    ],
+)

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements/AdsCallbackEngagement.scala

@@ -0,0 +1,125 @@
+package com.twitter.unified_user_actions.adapter.ads_callback_engagements
+
+import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
+import com.twitter.unified_user_actions.thriftscala._
+
+object AdsCallbackEngagement {
+  object PromotedTweetFav extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetFav)
+
+  object PromotedTweetUnfav extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetUnfav)
+
+  object PromotedTweetReply extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetReply)
+
+  object PromotedTweetRetweet
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetRetweet)
+
+  object PromotedTweetBlockAuthor
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetBlockAuthor)
+
+  object PromotedTweetUnblockAuthor
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetUnblockAuthor)
+
+  object PromotedTweetComposeTweet
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetComposeTweet)
+
+  object PromotedTweetClick extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetClick)
+
+  object PromotedTweetReport extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetReport)
+
+  object PromotedProfileFollow
+      extends ProfileAdsCallbackEngagement(ActionType.ServerPromotedProfileFollow)
+
+  object PromotedProfileUnfollow
+      extends ProfileAdsCallbackEngagement(ActionType.ServerPromotedProfileUnfollow)
+
+  object PromotedTweetMuteAuthor
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetMuteAuthor)
+
+  object PromotedTweetClickProfile
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetClickProfile)
+
+  object PromotedTweetClickHashtag
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetClickHashtag)
+
+  object PromotedTweetOpenLink
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetOpenLink) {
+    override def getItem(input: SpendServerEvent): Option[Item] = {
+      input.engagementEvent.flatMap { e =>
+        e.impressionData.flatMap { i =>
+          getPromotedTweetInfo(
+            i.promotedTweetId,
+            i.advertiserId,
+            tweetActionInfoOpt = Some(
+              TweetActionInfo.ServerPromotedTweetOpenLink(
+                ServerPromotedTweetOpenLink(url = e.url))))
+        }
+      }
+    }
+  }
+
+  object PromotedTweetCarouselSwipeNext
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetCarouselSwipeNext)
+
+  object PromotedTweetCarouselSwipePrevious
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetCarouselSwipePrevious)
+
+  object PromotedTweetLingerImpressionShort
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetLingerImpressionShort)
+
+  object PromotedTweetLingerImpressionMedium
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetLingerImpressionMedium)
+
+  object PromotedTweetLingerImpressionLong
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetLingerImpressionLong)
+
+  object PromotedTweetClickSpotlight
+      extends BaseTrendAdsCallbackEngagement(ActionType.ServerPromotedTweetClickSpotlight)
+
+  object PromotedTweetViewSpotlight
+      extends BaseTrendAdsCallbackEngagement(ActionType.ServerPromotedTweetViewSpotlight)
+
+  object PromotedTrendView
+      extends BaseTrendAdsCallbackEngagement(ActionType.ServerPromotedTrendView)
+
+  object PromotedTrendClick
+      extends BaseTrendAdsCallbackEngagement(ActionType.ServerPromotedTrendClick)
+
+  object PromotedTweetVideoPlayback25
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerPromotedTweetVideoPlayback25)
+
+  object PromotedTweetVideoPlayback50
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerPromotedTweetVideoPlayback50)
+
+  object PromotedTweetVideoPlayback75
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerPromotedTweetVideoPlayback75)
+
+  object PromotedTweetVideoAdPlayback25
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerPromotedTweetVideoAdPlayback25)
+
+  object PromotedTweetVideoAdPlayback50
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerPromotedTweetVideoAdPlayback50)
+
+  object PromotedTweetVideoAdPlayback75
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerPromotedTweetVideoAdPlayback75)
+
+  object TweetVideoAdPlayback25
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerTweetVideoAdPlayback25)
+
+  object TweetVideoAdPlayback50
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerTweetVideoAdPlayback50)
+
+  object TweetVideoAdPlayback75
+      extends BaseVideoAdsCallbackEngagement(ActionType.ServerTweetVideoAdPlayback75)
+
+  object PromotedTweetDismissWithoutReason
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetDismissWithoutReason)
+
+  object PromotedTweetDismissUninteresting
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetDismissUninteresting)
+
+  object PromotedTweetDismissRepetitive
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetDismissRepetitive)
+
+  object PromotedTweetDismissSpam
+      extends BaseAdsCallbackEngagement(ActionType.ServerPromotedTweetDismissSpam)
+}

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements/AdsCallbackEngagementsAdapter.scala

@@ -0,0 +1,28 @@
+package com.twitter.unified_user_actions.adapter.ads_callback_engagements
+
+import com.twitter.finagle.stats.NullStatsReceiver
+import com.twitter.finagle.stats.StatsReceiver
+import com.twitter.finatra.kafka.serde.UnKeyed
+import com.twitter.unified_user_actions.adapter.AbstractAdapter
+import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
+import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
+
+class AdsCallbackEngagementsAdapter
+    extends AbstractAdapter[SpendServerEvent, UnKeyed, UnifiedUserAction] {
+
+  import AdsCallbackEngagementsAdapter._
+
+  override def adaptOneToKeyedMany(
+    input: SpendServerEvent,
+    statsReceiver: StatsReceiver = NullStatsReceiver
+  ): Seq[(UnKeyed, UnifiedUserAction)] =
+    adaptEvent(input).map { e => (UnKeyed, e) }
+}
+
+object AdsCallbackEngagementsAdapter {
+  def adaptEvent(input: SpendServerEvent): Seq[UnifiedUserAction] = {
+    val baseEngagements: Seq[BaseAdsCallbackEngagement] =
+      EngagementTypeMappings.getEngagementMappings(Option(input).flatMap(_.engagementEvent))
+    baseEngagements.flatMap(_.getUUA(input))
+  }
+}

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements/BUILD

@@ -0,0 +1,18 @@
+scala_library(
+    sources = [
+        "*.scala",
+    ],
+    compiler_option_sets = ["fatal_warnings"],
+    tags = [
+        "bazel-compatible",
+        "bazel-only",
+    ],
+    dependencies = [
+        "kafka/finagle-kafka/finatra-kafka/src/main/scala",
+        "src/thrift/com/twitter/ads/billing/spendserver:spendserver_thrift-scala",
+        "src/thrift/com/twitter/ads/eventstream:eventstream-scala",
+        "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base",
+        "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common",
+        "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
+    ],
+)

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements/BaseAdsCallbackEngagement.scala

@@ -0,0 +1,68 @@
+package com.twitter.unified_user_actions.adapter.ads_callback_engagements
+
+import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
+import com.twitter.unified_user_actions.adapter.common.AdapterUtils
+import com.twitter.unified_user_actions.thriftscala.ActionType
+import com.twitter.unified_user_actions.thriftscala.AuthorInfo
+import com.twitter.unified_user_actions.thriftscala.EventMetadata
+import com.twitter.unified_user_actions.thriftscala.Item
+import com.twitter.unified_user_actions.thriftscala.SourceLineage
+import com.twitter.unified_user_actions.thriftscala.TweetInfo
+import com.twitter.unified_user_actions.thriftscala.TweetActionInfo
+import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
+import com.twitter.unified_user_actions.thriftscala.UserIdentifier
+
+abstract class BaseAdsCallbackEngagement(actionType: ActionType) {
+
+  protected def getItem(input: SpendServerEvent): Option[Item] = {
+    input.engagementEvent.flatMap { e =>
+      e.impressionData.flatMap { i =>
+        getPromotedTweetInfo(i.promotedTweetId, i.advertiserId)
+      }
+    }
+  }
+
+  protected def getPromotedTweetInfo(
+    promotedTweetIdOpt: Option[Long],
+    advertiserId: Long,
+    tweetActionInfoOpt: Option[TweetActionInfo] = None
+  ): Option[Item] = {
+    promotedTweetIdOpt.map { promotedTweetId =>
+      Item.TweetInfo(
+        TweetInfo(
+          actionTweetId = promotedTweetId,
+          actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(advertiserId))),
+          tweetActionInfo = tweetActionInfoOpt)
+      )
+    }
+  }
+
+  def getUUA(input: SpendServerEvent): Option[UnifiedUserAction] = {
+    val userIdentifier: UserIdentifier =
+      UserIdentifier(
+        userId = input.engagementEvent.flatMap(e => e.clientInfo.flatMap(_.userId64)),
+        guestIdMarketing = input.engagementEvent.flatMap(e => e.clientInfo.flatMap(_.guestId)),
+      )
+
+    getItem(input).map { item =>
+      UnifiedUserAction(
+        userIdentifier = userIdentifier,
+        item = item,
+        actionType = actionType,
+        eventMetadata = getEventMetadata(input),
+      )
+    }
+  }
+
+  protected def getEventMetadata(input: SpendServerEvent): EventMetadata =
+    EventMetadata(
+      sourceTimestampMs = input.engagementEvent
+        .map { e => e.engagementEpochTimeMilliSec }.getOrElse(AdapterUtils.currentTimestampMs),
+      receivedTimestampMs = AdapterUtils.currentTimestampMs,
+      sourceLineage = SourceLineage.ServerAdsCallbackEngagements,
+      language = input.engagementEvent.flatMap { e => e.clientInfo.flatMap(_.languageCode) },
+      countryCode = input.engagementEvent.flatMap { e => e.clientInfo.flatMap(_.countryCode) },
+      clientAppId =
+        input.engagementEvent.flatMap { e => e.clientInfo.flatMap(_.clientId) }.map { _.toLong },
+    )
+}

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements/BaseTrendAdsCallbackEngagement.scala

@@ -0,0 +1,18 @@
+package com.twitter.unified_user_actions.adapter.ads_callback_engagements
+
+import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
+import com.twitter.unified_user_actions.thriftscala._
+
+abstract class BaseTrendAdsCallbackEngagement(actionType: ActionType)
+    extends BaseAdsCallbackEngagement(actionType = actionType) {
+
+  override protected def getItem(input: SpendServerEvent): Option[Item] = {
+    input.engagementEvent.flatMap { e =>
+      e.impressionData.flatMap { i =>
+        i.promotedTrendId.map { promotedTrendId =>
+          Item.TrendInfo(TrendInfo(actionTrendId = promotedTrendId))
+        }
+      }
+    }
+  }
+}

unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements/BaseVideoAdsCallbackEngagement.scala

@@ -0,0 +1,54 @@
+package com.twitter.unified_user_actions.adapter.ads_callback_engagements
+
+import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
+import com.twitter.unified_user_actions.thriftscala.ActionType
+import com.twitter.unified_user_actions.thriftscala.AuthorInfo
+import com.twitter.unified_user_actions.thriftscala.TweetVideoWatch
+import com.twitter.unified_user_actions.thriftscala.Item
+import com.twitter.unified_user_actions.thriftscala.TweetActionInfo
+import com.twitter.unified_user_actions.thriftscala.TweetInfo
+
+abstract class BaseVideoAdsCallbackEngagement(actionType: ActionType)
+    extends BaseAdsCallbackEngagement(actionType = actionType) {
+
+  override def getItem(input: SpendServerEvent): Option[Item] = {
+    input.engagementEvent.flatMap { e =>
+      e.impressionData.flatMap { i =>
+        getTweetInfo(i.promotedTweetId, i.organicTweetId, i.advertiserId, input)
+      }
+    }
+  }
+
+  private def getTweetInfo(
+    promotedTweetId: Option[Long],
+    organicTweetId: Option[Long],
+    advertiserId: Long,
+    input: SpendServerEvent
+  ): Option[Item] = {
+    val actionedTweetIdOpt: Option[Long] =
+      if (promotedTweetId.isEmpty) organicTweetId else promotedTweetId
+    actionedTweetIdOpt.map { actionTweetId =>
+      Item.TweetInfo(
+        TweetInfo(
+          actionTweetId = actionTweetId,
+          actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(advertiserId))),
+          tweetActionInfo = Some(
+            TweetActionInfo.TweetVideoWatch(
+              TweetVideoWatch(
+                isMonetizable = Some(true),
+                videoOwnerId = input.engagementEvent
+                  .flatMap(e => e.cardEngagement).flatMap(_.amplifyDetails).flatMap(_.videoOwnerId),
+                videoUuid = input.engagementEvent
+                  .flatMap(_.cardEngagement).flatMap(_.amplifyDetails).flatMap(_.videoUuid),
+                prerollOwnerId = input.engagementEvent
+                  .flatMap(e => e.cardEngagement).flatMap(_.amplifyDetails).flatMap(
+                    _.prerollOwnerId),
+                prerollUuid = input.engagementEvent
+                  .flatMap(_.cardEngagement).flatMap(_.amplifyDetails).flatMap(_.prerollUuid)
+              ))
+          )
+        ),
+      )
+    }
+  }
+}