init

.gitignore

@@ -0,0 +1,60 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+.next

api/firebase/creds.js

@@ -0,0 +1,12 @@
+module.exports = {
+    "type": process.env.F_type,
+    "project_id": process.env.F_project_id,
+    "private_key_id": process.env.F_private_key_id,
+    "private_key": Buffer.from(process.env.F_private_key, 'base64').toString(),
+    "client_email": process.env.F_client_email,
+    "client_id": process.env.F_client_id,
+    "auth_uri": process.env.F_auth_uri,
+    "token_uri": process.env.F_token_uri,
+    "auth_provider_x509_cert_url": process.env.F_auth_provider_x509_cert_url,
+    "client_x509_cert_url": process.env.F_client_x509_cert_url
+};

api/firebase/firebase.js

@@ -0,0 +1,15 @@
+var firebase = require("firebase");
+var admin = require("firebase-admin");
+var serviceAccount = require("./creds");
+
+const config = {
+  credential: admin.credential.cert(serviceAccount),
+  databaseURL: process.env.FIREBASE_DATABASE_URL
+};
+
+admin.initializeApp(config);
+
+const database = admin.database();
+
+exports.firebase = firebase;
+exports.database = database;

api/firebase/index.js

@@ -0,0 +1,34 @@
+// methods to call firebase db
+const db = require("./firebase").database;
+const cache = require("../../cache");
+const moment = require('moment');
+
+const sendVideoToDB = async video => {
+  if (video.videoId in cache.videos) {
+    // video is in local cache, dont do anything
+    return;
+  }
+  video.timestamp = moment().format();
+  const dbBaseRef = db.ref(process.env.YOUTUBE_DB + "/videos");
+      // Video does not exist, we should update
+      const key = await dbBaseRef.push().key;
+      dbBaseRef.child(key).update(video);
+      cache.videos[video.videoId] = 1; // add to cache
+
+};
+
+const sendInfoToDB = async info => {
+  const dbBaseRef = db.ref(process.env.YOUTUBE_DB + "/channel");
+  await dbBaseRef.set(info);
+};
+
+const clearOldChat = async () => {
+  const ref = db.ref("livechat");
+  ref.once("value", async videoId => {
+    videoId.ref.remove()
+  });
+};
+
+module.exports.sendVideoToDB = sendVideoToDB;
+module.exports.sendInfoToDB = sendInfoToDB;
+module.exports.clearOldChat = clearOldChat;

api/index.js

@@ -0,0 +1,4 @@
+const youtube = require('./youtube');
+module.exports.youtube = youtube;
+const firebase = require('./firebase');
+module.exports.firebase = firebase;

api/logger/index.js

@@ -0,0 +1,36 @@
+const moment = require("moment");
+const chalk = require("chalk");
+
+function cmd(cmd, suffix) {
+  console.log(
+    chalk.cyan(`[${moment().format("YYYY-MM-DD HH:mm:ss")}]`),
+    chalk.bold.green("[COMMAND]"),
+    chalk.bold.green(cmd),
+    suffix
+  );
+}
+
+function info(msg) {
+  console.log(chalk.cyan(`[${moment().format("YYYY-MM-DD HH:mm:ss")}]`),chalk.bold.cyan("[INFO]"), msg);
+}
+
+function warn(msg) {
+  console.log(
+    chalk.cyan(`[${moment().format("YYYY-MM-DD HH:mm:ss")}]`),
+    chalk.yellow(`[WARN] ${msg}`)
+  );
+}
+
+function error(msg) {
+  console.log(
+    chalk.cyan(`[${moment().format("YYYY-MM-DD HH:mm:ss")}]`),
+    chalk.red(`[ERROR] ${msg}`)
+  );
+}
+
+module.exports = {
+  cmd,
+  info,
+  warn,
+  error
+};

api/youtube/index.js

@@ -0,0 +1,140 @@
+// methods to call youtube's api
+const moment = require("moment");
+var request = require("request-promise");
+const cache = require("../../cache");
+const firebase = require("../firebase");
+const logger = require("../logger");
+
+const infoConstructor = item => {
+  const { id, snippet, statistics } = item;
+  const {
+    title,
+    customUrl,
+    description,
+    thumbnails
+  } = snippet;
+  const smallImage = thumbnails.default.url;
+  const mediumImage = thumbnails.medium.url;
+  const largeImage = thumbnails.high.url;
+  const { viewCount, subscriberCount, videoCount } = statistics;
+
+  return {
+    id,
+    title,
+    url: 'https://youtube.com/' + customUrl,
+    description,
+    smallImage,
+    mediumImage,
+    largeImage,
+    viewCount,
+    subscriberCount,
+    viewCount,
+    videoCount
+  };
+};
+
+const videoConstructor = item => {
+  const { id, snippet } = item;
+  const { videoId } = id;
+  const {
+    title,
+    channelTitle,
+    liveBroadcastContent,
+    publishedAt,
+    thumbnails
+  } = snippet;
+  const { high } = thumbnails;
+  const youtubeVideoURLBase = "https://youtu.be/";
+  const videoUrl = youtubeVideoURLBase + videoId;
+  const { url } = high;
+  const thumbnailUrl = url;
+
+  return {
+    videoId,
+    title,
+    channelTitle,
+    liveBroadcastContent,
+    videoUrl,
+    thumbnailUrl,
+    publishedAt
+  };
+};
+
+const getChannelVideos = async () => {
+  cache.timesHitYoutube += 1;
+  const YOUTUBE_API_KEY = process.env.youtube_api_key;
+  const YOUTUBE_CHANNEL_ID = process.env.YOUTUBE_CHANNEL_ID;
+  let result;
+  try {
+    result = await request({
+      url: `https://www.googleapis.com/youtube/v3/search?key=${YOUTUBE_API_KEY}&channelId=${YOUTUBE_CHANNEL_ID}&part=snippet&type=video&eventType=completed&order=date&maxResults=5`,
+      method: "GET",
+      json: true
+    });
+    return result;
+  } catch (error) {
+    logger.error(error);
+    return error;
+  }
+};
+const getLiveVideos = async () => {
+  cache.timesHitYoutube += 1;
+  const YOUTUBE_API_KEY = process.env.youtube_api_key;
+  const YOUTUBE_CHANNEL_ID = process.env.YOUTUBE_CHANNEL_ID;
+  let result;
+  try {
+    result = await request({
+      url: `https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=${YOUTUBE_CHANNEL_ID}&type=video&eventType=live&key=${YOUTUBE_API_KEY}`,
+      method: "GET",
+      json: true
+    });
+    return result;
+  } catch (error) {
+    logger.error(error);
+    return error;
+  }
+};
+
+const getChannelDetails = async () => {
+  cache.timesHitYoutube += 1;
+  const YOUTUBE_API_KEY = process.env.youtube_api_key;
+  const YOUTUBE_CHANNEL_ID = process.env.YOUTUBE_CHANNEL_ID;
+  let result;
+  try {
+    result = await request({
+      url: `https://www.googleapis.com/youtube/v3/channels?part=snippet%2CcontentDetails%2Cstatistics&id=${YOUTUBE_CHANNEL_ID}&key=${YOUTUBE_API_KEY}`,
+      method: "GET",
+      json: true
+    });
+
+    return result;
+  } catch (error) {
+    logger.error(error);
+    return error;
+  }
+};
+
+const getChannelDetailsAndUpdateDB = async () => {
+  const value = await getChannelDetails();
+  const { items } = value;
+  if (items && items.length === 1) {
+    firebase.sendInfoToDB(infoConstructor(items[0]));
+  } else {
+    logger.warn("No items in channel search");
+  }
+};
+
+const updateDBwithVideos = async ({items}) => {
+  if (items && items.length > 0) {
+    items.forEach(item => {
+      firebase.sendVideoToDB(videoConstructor(item));
+    });
+  } else {
+    logger.warn("No items in search");
+  }
+};
+
+module.exports.updateDBwithVideos = updateDBwithVideos;
+module.exports.getChannelDetailsAndUpdateDB = getChannelDetailsAndUpdateDB;
+module.exports.getChannelVideos = getChannelVideos;
+module.exports.getLiveVideos = getLiveVideos;

cache.js

@@ -0,0 +1,4 @@
+module.exports = {
+    "videos": {},
+    "timesHitYoutube": 0
+};

cron/index.js

@@ -0,0 +1,61 @@
+// long running tasks
+const schedule = require("node-schedule");
+const cache = require("../cache");
+const api = require("../api");
+const logger = require('../api/logger');
+
+const everyHour = () => {
+  // log cache and count every hour
+  const rule = new schedule.RecurrenceRule();
+  rule.minute = 0; // every time the clock reaches 0 minutes
+  schedule.scheduleJob(rule, () => {
+    api.youtube.getChannelDetailsAndUpdateDB();
+    logger.info(
+      `HOURLY: Ran youtube api ${
+        cache.timesHitYoutube
+      } | videoId's are ${JSON.stringify(
+        cache.videos
+      )}, every day max should be 1440`
+    );
+  });
+};
+
+const everyMidnight = () => {
+  // Garbage Collection
+  const rule = new schedule.RecurrenceRule();
+  rule.minute = 0; // every time the clock reaches 0 minutes
+  rule.hour = 0; // every time the clock reaches 0 hours
+  schedule.scheduleJob(rule, () => {
+    logger.info("MIDNIGHT: garbage collection");
+    cache.videos = {};
+    cache.timesHitYoutube = 0;
+  });
+};
+
+const everyMin = () => {
+  // batch running every 1 mins
+  const rule = new schedule.RecurrenceRule();
+  rule.minute = new schedule.Range(0, 59, 1);
+  schedule.scheduleJob(rule, async () => {
+    const regular = await api.youtube.getChannelVideos();
+    const live = await api.youtube.getLiveVideos();
+    api.youtube.updateDBwithVideos(regular);
+    api.youtube.updateDBwithVideos(live);
+    api.firebase.clearOldChat()
+    logger.info(
+      `EVERY MIN: Ran youtube api ${
+        cache.timesHitYoutube
+      } | videoId's are ${JSON.stringify(
+        cache.videos
+      )}`
+    );
+  });
+};
+
+const run = () => {
+  everyMidnight();
+  everyHour();
+  everyMin();
+};
+
+run();

http/index.js

@@ -0,0 +1,6 @@
+// used for deployment with ziet's now platform
+const http = require("http");
+const server = http.createServer((req, res) => {
+  res.end("Still running jobs...");
+});
+server.listen(process.env.PORT);