improvements

RichardMcSorley · Nov 21, 2018 · 9213bf0 · 9213bf0
1 parent 9632a3f
commit 9213bf0
Show file tree

Hide file tree

Showing 9 changed files with 376 additions and 243 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ logs
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+.vscode
 
 # Runtime data
 pids

diff --git a/api/firebase/index.js b/api/firebase/index.js
@@ -2,41 +2,84 @@
 const db = require("./firebase").database;
 const cache = require("../../cache");
 const moment = require("moment");
+const videoDBRes = "videos";
+let YOUTUBE_CRON;
+let YOUTUBE_API_KEY;
+let found = false;
+db.ref("cron/runners").on("value", v => {
+  const runners = v.val();
+  if (runners && !YOUTUBE_CRON) {
+    runners.forEach((runnerItem, index) => {
+      if (found) return;
+      if (!runnerItem.inUse) {
+        // we can use this runner
+        //update runner.inUse = true
+        YOUTUBE_CRON = index;
+        found = true;
+        db.ref(`cron/runners/${index}`).update({
+          inUse: true,
+          instance: YOUTUBE_CRON
+        });
+        require("../../cron")(index);
+      }
+    });
+  }
+  if (YOUTUBE_CRON) {
+    console.log("already connected");
+  } else {
+    console.log("could not connect");
+  }
+});
+const disconnectRunner = async () => {
+  await db.ref(`cron/runners/${YOUTUBE_CRON}`).update({
+    inUse: null,
+    instance: null
+  });
+  process.exit(0);
+};
+const cronRunnerData = (ref, callback) => {
+  return db.ref(ref).on("value", v => {
+    const value = v.val();
+    if (value) {
+      callback(value);
+    }
+  });
+};
+// keep video cache in sync
+cronRunnerData(videoDBRes + "/processed", value => {
+  cache.videos = value;
+});
 
-const sendVideoToDB = async video => {
-  if (video.videoId in cache.videos) {
+const sendVideoToDB = async ({ videoId, liveBroadcastContent, ...rest }) => {
+  const video = {
+    ...rest,
+    videoId,
+    liveBroadcastContent,
+    timestamp: moment().format()
+  };
+  if (videoId in cache.videos) {
     // video is in local cache, dont do anything
     return;
   }
-  video.timestamp = moment().format();
-  const dbBaseRef = db.ref(process.env.YOUTUBE_DB + "/tasks");
-  // Video does not exist, we should update
+  if (liveBroadcastContent === "live" || liveBroadcastContent === "upcoming") {
+    const puppetRef = db.ref("livechat/puppet/video/tasks");
+    const key = await puppetRef.push().key;
+    puppetRef.child(key).update(video);
+  }
+
+  const dbBaseRef = db.ref(videoDBRes + "/tasks");
   const key = await dbBaseRef.push().key;
-  //moment(video.publishedAt, "YYYY-MM-DDThh:mm:ss.sZ").format("x")
   dbBaseRef.child(key).update(video);
-  cache.videos[video.videoId] = 1; // add to cache
+  cache.videos[videoId] = 1; // add to cache
 };
 
 const sendInfoToDB = async info => {
-  const dbBaseRef = db.ref(process.env.YOUTUBE_DB + "/channel");
+  const dbBaseRef = db.ref(videoDBRes + "/channel/" + info.id);
   await dbBaseRef.set(info);
 };
 
-// const clearOldChat = async () => {
-//   const ref = db.ref("livechat");
-//   ref.once("value", async videoId => {
-//     videoId.ref.remove();
-//   });
-// };
-
-// const clearOldVideos = async () => {
-//   const dbBaseRef = db.ref(process.env.YOUTUBE_DB + "/videos");
-//   dbBaseRef.once("value", async videoId => {
-//     videoId.ref.remove();
-//   });
-// };
-
 module.exports.sendVideoToDB = sendVideoToDB;
 module.exports.sendInfoToDB = sendInfoToDB;
-// module.exports.clearOldChat = clearOldChat;
-// module.exports.clearOldVideos = clearOldVideos;
+module.exports.cronRunnerData = cronRunnerData;
+module.exports.disconnectRunner = disconnectRunner;
+module.exports.YOUTUBE_API_KEY = YOUTUBE_API_KEY;
diff --git a/api/index.js b/api/index.js
@@ -1,4 +1,4 @@
-const youtube = require('./youtube');
+const firebase = require("./firebase");
+module.exports.firebase = firebase;
+const youtube = require("./youtube");
 module.exports.youtube = youtube;
-const firebase = require('./firebase');
-module.exports.firebase = firebase;
diff --git a/api/youtube/index.js b/api/youtube/index.js
@@ -1,135 +1,131 @@
-// methods to call youtube's api
-const moment = require("moment");
-var request = require("request-promise");
-const cache = require("../../cache");
-const firebase = require("../firebase");
-const logger = require("../logger");
+module.exports = YOUTUBE_CRON => {
+  var request = require("request-promise");
+  const cache = require("../../cache");
+  const firebase = require("../firebase");
+  const logger = require("../logger");
+  const { cronRunnerData } = require("../firebase");
+  let YOUTUBE_API_KEY;
+  cronRunnerData(`cron/runners/${YOUTUBE_CRON}`, value => {
+    YOUTUBE_API_KEY = value.key;
+    console.log("using new key", YOUTUBE_API_KEY);
+  });
 
-const infoConstructor = item => {
-  const { id, snippet, statistics } = item;
-  const { title, customUrl, description, thumbnails } = snippet;
-  const smallImage = thumbnails.default.url;
-  const mediumImage = thumbnails.medium.url;
-  const largeImage = thumbnails.high.url;
-  const { viewCount, subscriberCount, videoCount } = statistics;
+  const infoConstructor = item => {
+    const { id, snippet, statistics, brandingSettings } = item;
+    const { title, customUrl = null, description = null, thumbnails } = snippet;
+    const { channel, image } = brandingSettings;
+    const {
+      keywords = null,
+      featuredChannelsUrls = null,
+      profileColor = null,
+      unsubscribedTrailer = null
+    } = channel;
+    const smallImage = thumbnails.default.url;
+    const mediumImage = thumbnails.medium.url;
+    const largeImage = thumbnails.high.url;
+    const { viewCount, subscriberCount, videoCount } = statistics;
 
-  return {
-    id,
-    title,
-    url: "https://youtube.com/" + customUrl,
-    description,
-    smallImage,
-    mediumImage,
-    largeImage,
-    viewCount,
-    subscriberCount,
-    viewCount,
-    videoCount
+    return {
+      id,
+      title,
+      url: "https://youtube.com/channel/" + id,
+      description,
+      smallImage,
+      mediumImage,
+      largeImage,
+      viewCount,
+      subscriberCount,
+      viewCount,
+      videoCount,
+      image,
+      keywords,
+      featuredChannelsUrls,
+      profileColor,
+      unsubscribedTrailer,
+      customUrl
+    };
   };
-};
 
-const videoConstructor = item => {
-  const { id, snippet } = item;
-  const { videoId } = id;
-  const {
-    title,
-    channelTitle,
-    liveBroadcastContent,
-    publishedAt,
-    thumbnails
-  } = snippet;
-  const { high } = thumbnails;
-  const youtubeVideoURLBase = "https://youtu.be/";
-  const videoUrl = youtubeVideoURLBase + videoId;
-  const { url } = high;
-  const thumbnailUrl = url;
+  const videoConstructor = item => {
+    const { id, snippet } = item;
+    const { videoId } = id;
+    const {
+      channelId,
+      title,
+      description,
+      channelTitle,
+      liveBroadcastContent,
+      publishedAt,
+      thumbnails
+    } = snippet;
+    const { high } = thumbnails;
+    const youtubeVideoURLBase = "https://youtu.be/";
+    const videoUrl = youtubeVideoURLBase + videoId;
+    const { url } = high;
+    const thumbnailUrl = url;
 
-  return {
-    videoId,
-    title,
-    channelTitle,
-    liveBroadcastContent,
-    videoUrl,
-    thumbnailUrl,
-    publishedAt
+    return {
+      videoId,
+      title,
+      channelTitle,
+      liveBroadcastContent,
+      videoUrl,
+      thumbnailUrl,
+      publishedAt,
+      description,
+      channelId
+    };
   };
-};
 
-const getChannelVideos = async () => {
-  cache.timesHitYoutube += 1;
-  const YOUTUBE_API_KEY = process.env.youtube_api_key;
-  const YOUTUBE_CHANNEL_ID = process.env.YOUTUBE_CHANNEL_ID;
-  let result;
-  try {
-    result = await request({
-      url: `https://www.googleapis.com/youtube/v3/search?key=${YOUTUBE_API_KEY}&channelId=${YOUTUBE_CHANNEL_ID}&part=snippet&type=video&order=date&maxResults=5`,
-      method: "GET",
-      json: true
-    });
-    return result;
-  } catch (error) {
-    logger.error(error);
-    return error;
-  }
-};
-const getLiveVideos = async () => {
-  cache.timesHitYoutube += 1;
-  const YOUTUBE_API_KEY = process.env.youtube_api_key;
-  const YOUTUBE_CHANNEL_ID = process.env.YOUTUBE_CHANNEL_ID;
-  let result;
-  try {
-    result = await request({
-      url: `https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=${YOUTUBE_CHANNEL_ID}&type=video&eventType=live&key=${YOUTUBE_API_KEY}`,
-      method: "GET",
-      json: true
-    });
-    return result;
-  } catch (error) {
-    logger.error(error);
-    return error;
-  }
-};
+  const getBySearch = async query => {
+    query = `${query}&key=${YOUTUBE_API_KEY}`;
+    let result;
+    try {
+      result = await request({
+        url: `https://www.googleapis.com/youtube/v3/search${query}`,
+        method: "GET",
+        json: true
+      });
+      return result;
+    } catch (error) {
+      logger.error(error);
+      return error;
+    }
+  };
 
-const getChannelDetails = async () => {
-  cache.timesHitYoutube += 1;
-  const YOUTUBE_API_KEY = process.env.youtube_api_key;
-  const YOUTUBE_CHANNEL_ID = process.env.YOUTUBE_CHANNEL_ID;
-  let result;
-  try {
-    result = await request({
-      url: `https://www.googleapis.com/youtube/v3/channels?part=snippet%2CcontentDetails%2Cstatistics&id=${YOUTUBE_CHANNEL_ID}&key=${YOUTUBE_API_KEY}`,
-      method: "GET",
-      json: true
-    });
+  const getByChannels = async query => {
+    query = `${query}&key=${YOUTUBE_API_KEY}`;
+    let result;
+    try {
+      result = await request({
+        url: `https://www.googleapis.com/youtube/v3/channels${query}`,
+        method: "GET",
+        json: true
+      });
 
-    return result;
-  } catch (error) {
-    logger.error(error);
-    return error;
-  }
-};
+      return result;
+    } catch (error) {
+      logger.error(error);
+      return error;
+    }
+  };
 
-const getChannelDetailsAndUpdateDB = async () => {
-  const value = await getChannelDetails();
-  const { items } = value;
-  if (items && items.length === 1) {
-    firebase.sendInfoToDB(infoConstructor(items[0]));
-  } else {
-    logger.warn("No items in channel search");
-  }
-};
+  const getChannelDetailsAndUpdateDB = async ({ query, ...rest }) => {
+    const { items = [] } = await getByChannels(query);
+    items.forEach(item => {
+      firebase.sendInfoToDB({ ...infoConstructor(item), ...rest });
+    });
+  };
 
-const updateDBwithVideos = async ({ items }) => {
-  if (items && items.length > 0) {
+  const getVideosAndUpdateDB = async ({ query, ...rest }) => {
+    const { items = [] } = await getBySearch(query);
     items.forEach(item => {
-      firebase.sendVideoToDB(videoConstructor(item));
+      firebase.sendVideoToDB({ ...videoConstructor(item), ...rest });
     });
-  } else {
-    logger.warn("No items in search");
-  }
-};
+  };
 
-module.exports.updateDBwithVideos = updateDBwithVideos;
-module.exports.getChannelDetailsAndUpdateDB = getChannelDetailsAndUpdateDB;
-module.exports.getChannelVideos = getChannelVideos;
-module.exports.getLiveVideos = getLiveVideos;
+  return {
+    getChannelDetailsAndUpdateDB,
+    getVideosAndUpdateDB
+  };
+};
diff --git a/cache.js b/cache.js
@@ -1,4 +1,3 @@
 module.exports = {
-    "videos": {},
-    "timesHitYoutube": 0
-};
+  videos: {}
+};