throttle and monitor youtube stream api calls

jtwires · Mar 8, 2020 · fef7c3d · fef7c3d
1 parent dc1eead
commit fef7c3d
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 37 deletions.
diff --git a/app/controllers/Game.scala b/app/controllers/Game.scala
@@ -43,11 +43,13 @@ final class Game(
         lila.mon.export.pgn.game.increment()
         env.api.gameApiV2.exportOne(game, config) flatMap { content =>
           env.api.gameApiV2.filename(game, config.format) map { filename =>
-            Ok(content).withHeaders(
-              CONTENT_DISPOSITION -> s"attachment; filename=$filename"
-            ).withHeaders(
-              lila.app.http.ResponseHeaders.headersForApiOrApp(ctx.req): _*
-            ) as gameContentType(config)
+            Ok(content)
+              .withHeaders(
+                CONTENT_DISPOSITION -> s"attachment; filename=$filename"
+              )
+              .withHeaders(
+                lila.app.http.ResponseHeaders.headersForApiOrApp(ctx.req): _*
+              ) as gameContentType(config)
           }
         }
       }

diff --git a/modules/common/src/main/mon.scala b/modules/common/src/main/mon.scala
@@ -316,6 +316,8 @@ object mon {
   object tv {
     object streamer {
       def present(n: String) = gauge("tv.streamer.present").withTag("name", n)
+      def youTube            = future("tv.streamer.youtube")
+      def twitch             = future("tv.streamer.twitch")
     }
   }
   object crosstable {

diff --git a/modules/streamer/src/main/Stream.scala b/modules/streamer/src/main/Stream.scala
@@ -1,6 +1,7 @@
 package lila.streamer
 
 import play.api.libs.json._
+import org.joda.time.DateTime
 
 import lila.user.User
 import lila.common.String.html.unescapeHtml
@@ -81,5 +82,7 @@ object Stream {
       implicit private val youtubeItemReads    = Json.reads[Item]
       implicit val youtubeResultReads          = Json.reads[Result]
     }
+
+    case class StreamsFetched(list: List[YouTube.Stream], at: DateTime)
   }
 }
diff --git a/modules/streamer/src/main/Streaming.scala b/modules/streamer/src/main/Streaming.scala
@@ -1,6 +1,7 @@
 package lila.streamer
 
 import akka.actor._
+import org.joda.time.DateTime
 import play.api.libs.json._
 import play.api.libs.ws.WSClient
 import scala.concurrent.duration._
@@ -116,45 +117,62 @@ final private class Streaming(
           .withHttpHeaders(
             "Client-ID" -> twitchClientId.value
           )
-        url.get().map { res =>
-          res.json.validate[Twitch.Result](twitchResultReads) match {
-            case JsSuccess(data, _) =>
-              data.streams(
-                keyword,
-                streamers,
-                alwaysFeatured().value.map(_.toLowerCase)
-              )
-            case JsError(err) =>
-              logger.warn(s"twitch ${res.status} $err ${~res.body.linesIterator.toList.headOption}")
-              Nil
+        url
+          .get()
+          .map { res =>
+            res.json.validate[Twitch.Result](twitchResultReads) match {
+              case JsSuccess(data, _) =>
+                data.streams(
+                  keyword,
+                  streamers,
+                  alwaysFeatured().value.map(_.toLowerCase)
+                )
+              case JsError(err) =>
+                logger.warn(s"twitch ${res.status} $err ${~res.body.linesIterator.toList.headOption}")
+                Nil
+            }
           }
-        }
+          .monSuccess(_.tv.streamer.twitch)
       }
     }
   }
 
-  def fetchYouTubeStreams(streamers: List[Streamer]): Fu[List[YouTube.Stream]] =
-    googleApiKey.value.nonEmpty ?? {
-      val youtubeStreamers = streamers.filter(_.youTube.isDefined)
-      youtubeStreamers.nonEmpty ?? ws
-        .url("https://www.googleapis.com/youtube/v3/search")
-        .withQueryStringParameters(
-          "part"      -> "snippet",
-          "type"      -> "video",
-          "eventType" -> "live",
-          "q"         -> keyword.value,
-          "key"       -> googleApiKey.value
-        )
-        .get()
-        .map { res =>
-          res.json.validate[YouTube.Result](youtubeResultReads) match {
-            case JsSuccess(data, _) => data.streams(keyword, youtubeStreamers)
-            case JsError(err) =>
-              logger.warn(s"youtube ${res.status} $err ${res.body.take(500)}")
-              Nil
-          }
+  private var prevYouTubeStreams = YouTube.StreamsFetched(Nil, DateTime.now)
+
+  def fetchYouTubeStreams(streamers: List[Streamer]): Fu[List[YouTube.Stream]] = {
+    val youtubeStreamers = streamers.filter(_.youTube.isDefined)
+    (youtubeStreamers.nonEmpty && googleApiKey.value.nonEmpty) ?? {
+      val now = DateTime.now
+      val res =
+        if (prevYouTubeStreams.list.isEmpty && prevYouTubeStreams.at.isAfter(now minusMinutes 2))
+          fuccess(prevYouTubeStreams)
+        else {
+          ws.url("https://www.googleapis.com/youtube/v3/search")
+            .withQueryStringParameters(
+              "part"      -> "snippet",
+              "type"      -> "video",
+              "eventType" -> "live",
+              "q"         -> keyword.value,
+              "key"       -> googleApiKey.value
+            )
+            .get()
+            .map { res =>
+              res.json.validate[YouTube.Result](youtubeResultReads) match {
+                case JsSuccess(data, _) =>
+                  YouTube.StreamsFetched(data.streams(keyword, youtubeStreamers), now)
+                case JsError(err) =>
+                  logger.warn(s"youtube ${res.status} $err ${res.body.take(500)}")
+                  YouTube.StreamsFetched(Nil, now)
+              }
+            }
+            .monSuccess(_.tv.streamer.youTube)
         }
+      res dmap { r =>
+        prevYouTubeStreams = r
+        r.list
+      }
     }
+  }
 
   def dedupStreamers(streams: List[Stream]): List[Stream] =
     streams