Merge branch 'release/0.6.3'

WHATSNEW.md

@@ -5,6 +5,36 @@ This is the history of the [Iframely Gateway](http://iframely.com/gateway) chang
 Stay tuned, either by watching [Iframely on GitHub](https://github.com/itteco/iframely) or following [Iframely on Twitter](https://twitter.com/iframely).
 
 
+### 2014.05.15, Version 0.6.3
+
+
+Big ones:
+
+ + added Twitter Stream parser (mp4 videos). It is now in whitelist also. [Get your domains whitelist here](http://iframely.com/qa).
+ * re-enable proper domain plugins for Pinterest (proper titles, thumbnails, etc)
+
+
+Domains maintenance: 
+
+ * Fixed Vube (they switched to iFrames. Yay.)
+ * White-list and properly handle all YouTube players in Twitter Cards
+ * Proper thumnnails for Tumblr articles
+ * Instagram videos no longer mistakenly return `rel=image`
+ * Switched to better (and more responsive) embeds for Vine
+ * Fixed Behance to support their new URL scheme
+ * Fixes for other domains plugins: Dribbble, Telly, Fiddler
+ * Handle more URLs for: Flickr (including videos and smaller images), CodePen, Someecards, Globe and Mail, Guardian
+ - Removed plugins: QiK (retired as of April 30) and EventBrite (embeds support is inconsistent)
+
+
+Tech matters:
+
+ * add received cookie jar to options to prevent some redirect loops
+ * better handle whitelist re-loads (at some times, whitelist records were lost in transition)
+ * return 415 http code in cases where the charset's encoding is not supported yet
+
+
+
 
 ### 2014.04.16, Version 0.6.2
 

lib/core.js

@@ -988,6 +988,10 @@
             options = {};
         }
 
+        if (!options.jar) {
+            options.jar = request.jar();
+        }
+
         if (options.redirectsCount && options.redirectsCount > 3) {
             return cb('redirect loop');
         }
@@ -1059,7 +1063,7 @@
                 }
 
                 if (options.whitelist) {
-                    allResults.whitelist = context.whitelistRecord;
+                    allResults.whitelist = options.getWhitelistRecord && options.getWhitelistRecord(uri, true);
                 }
 
                 if (options.fetchParam) {
@@ -1165,8 +1169,5 @@
     };
 
     exports.getOembed = oembedUtils.getOembed;
-    exports.getWhitelist = function() {
-        return require('./whitelist');
-    };
 
 })(exports);

lib/plugins/system/meta/meta.js

@@ -14,6 +14,13 @@ module.exports = {
                 if (error) {
                     return cb(error);
                 }
+
+                if (meta.charset.match(/^(euc|ISO2022|shift|X\-SJIS|MS949)/gi)) {
+                    return cb({
+                        responseStatusCode: 415
+                    });
+                }
+
                 cb(null, {
                     meta: meta
                 });

lib/plugins/validators/00_init/01_qa_rels.js

@@ -16,7 +16,7 @@ module.exports = {
         var rawMeta = {};
 
         // Proxy QA tags, but ignore the ones from the config wildcard
-        if (whitelistRecord && whitelistRecord.getQATags && !whitelistRecord.isDefault) {
+        if (whitelistRecord && whitelistRecord.getQATags && !whitelistRecord.isDefault && link.rel.indexOf('allow') == -1) {
             var tags = whitelistRecord.getQATags(rawMeta, link.rel);
             link.rel = link.rel.concat(tags);
         }

lib/plugins/validators/media.js

@@ -16,6 +16,10 @@ function makeMediaResponsive(link) {
     } else {
         m["aspect-ratio"] = 4 / 3;
     }
+
+    if (typeof m["aspect-ratio"] === 'number') {
+        m["aspect-ratio"] = Math.round(m["aspect-ratio"] * 1000) / 1000;
+    }
 }
 
 function isEmpty(obj) {

lib/utils.js

@@ -7,7 +7,9 @@ var imagesize = require('imagesize');
 var moment = require('moment');
 var cache = require('./cache');
 
-GLOBAL.CONFIG = require('../config');
+if (!GLOBAL.CONFIG) {
+    GLOBAL.CONFIG = require('../config');
+}
 
 /**
  * @private

lib/whitelist.js

@@ -69,13 +69,13 @@
         return result;
     }
 
-    whitelist.findWhitelistRecordFor = function(uri) {
+    whitelist.findWhitelistRecordFor = function(uri, disableWildcard) {
 
         if (!whitelistObject) {
             return null;
         }
 
-        var patterns = extractDomainPatterns(uri);
+        var patterns = extractDomainPatterns(uri, disableWildcard);
 
         var record, i = 0;
         while(!record && i < patterns.length) {
@@ -106,7 +106,7 @@
                 }, record);
 
                 // Override empty data with wildcard.
-                if (CONFIG.WHITELIST_WILDCARD) {
+                if (!disableWildcard && CONFIG.WHITELIST_WILDCARD) {
 
                     for(var protocolId in CONFIG.WHITELIST_WILDCARD) {
 
@@ -116,6 +116,8 @@
 
                         if (!recordProtocol) {
                             recordProtocol = record[protocolId] = {};
+                        } else {
+                            recordProtocol = _.extend({}, recordProtocol);
                         }
 
                         for(var type in protocol) {
@@ -145,7 +147,7 @@
         }
     }
 
-    function extractDomainPatterns(uri) {
+    function extractDomainPatterns(uri, disableWildcard) {
 
         var patterns = [];
 
@@ -178,8 +180,10 @@
             patterns.push('*.' + domain);
         }
 
-        // System-wide top-level wildcard, taken from config.
-        patterns.push('*');
+        if (!disableWildcard) {
+            // System-wide top-level wildcard, taken from config.
+            patterns.push('*');
+        }
 
         return patterns;
     }

modules/api/views.js

@@ -6,6 +6,7 @@ var cache = require('../../lib/cache');
 var oembedUtils = require('../../lib/oembed');
 var whitelist = require('../../lib/whitelist');
 var pluginLoader = require('../../lib/loader/pluginLoader');
+var jsonxml = require('jsontoxml');
 
 function prepareUri(uri) {
 

package.json

@@ -1,6 +1,6 @@
 {
     "name": "iframely",
-    "version": "0.6.2",
+    "version": "0.6.3",
 
     "description": "oEmbed/2 gateway endpoint. Get embed data for various http links through one self-hosted API",
     "keywords": ["oembed", "embed", "open graph", "og", "twitter cards"],

plugins/domains/56.com.js

@@ -1,8 +1,8 @@
 module.exports = {
 
     re: [
-        /^http:\/\/www\.56\.com\/\w{3}\/v_([a-zA-Z0-9]+).html/i,
-        /^http:\/\/www\.56\.com\/\w{3}\/play_[a-zA-Z0-9\-]+_vid\-([a-zA-Z0-9]+).html/i
+        /^https?:\/\/www\.56\.com\/\w{3}\/v_([a-zA-Z0-9]+).html/i,
+        /^https?:\/\/www\.56\.com\/\w{3}\/play_[a-zA-Z0-9\-]+_vid\-([a-zA-Z0-9]+).html/i
     ],
 
     mixins: [

plugins/domains/9gag.com.js

@@ -1,7 +1,7 @@
 module.exports = {
 
     re: [
-        /^http:\/\/9gag\.com\/gag\/([a-z0-9\-]+)/i
+        /^https?:\/\/9gag\.com\/gag\/([a-z0-9\-]+)/i
     ],
 
     mixins: [

plugins/domains/about.me.js

@@ -1,6 +1,6 @@
 module.exports = {
 
-    re: /http:\/\/about\.me\/([a-zA-Z0-9\-]+)/i,
+    re: /https?:\/\/about\.me\/([a-zA-Z0-9\-]+)/i,
 
     mixins: [
         "twitter-image",

plugins/domains/angel.co/angel.co.js

@@ -1,6 +1,6 @@
 module.exports = {
 
-    re: /^https:\/\/angel\.co\/([a-z-]+)/i,
+    re: /^https?:\/\/angel\.co\/([a-z-]+)/i,
 
     mixins: [
         "keywords",

plugins/domains/bandcamp.com.js

@@ -1,8 +1,8 @@
 module.exports = {
 
     re: [
-        /^http:\/\/([a-z0-9-]+\.bandcamp\.com)\/(album|track)\/(\w+)/i,
-        /^http:\/\/([a-z-\.]+)\/(album|track)\/([a-z-]+)/
+        /^https?:\/\/([a-z0-9-]+\.bandcamp\.com)\/(album|track)\/(\w+)/i,
+        /^https?:\/\/([a-z-\.]+)\/(album|track)\/([a-z-]+)/
     ],
 
     mixins: [

plugins/domains/behance.net.js

@@ -3,8 +3,9 @@ var jquery = require('jquery');
 module.exports = {
 
     re: [
-        /http:\/\/www\.behance\.net\/gallery\/([a-zA-Z0-9\-\(\)]+)\/([0-9]+)/i,
-        /http:\/\/([a-z-\.]+)\/gallery\/([a-zA-Z0-9\-\(\)]+)\/([0-9]+)/i,
+        /https?:\/\/www\.behance\.net\/gallery\/([a-zA-Z0-9\-\(\)]+)\/([0-9]+)/i,
+        /https?:\/\/www\.behance\.net\/gallery\/([0-9]+)\/([a-zA-Z0-9\-\(\)]+)/i,
+        /https?:\/\/([a-z-\.]+)\/gallery\/([a-zA-Z0-9\-\(\)]+)\/([0-9]+)/i,
     ],
 
     mixins: [

plugins/domains/brightcove.com.js

@@ -1,7 +1,7 @@
 module.exports = {
 
     re: [
-        /^http:\/\/link\.brightcove\.com\/services\/player\/bcpid(\d+)\?/i,
+        /^https?:\/\/link\.brightcove\.com\/services\/player\/bcpid(\d+)\?/i,
         /\.(com|org)\/services\/player\/bcpid(\d+)\?/i  //the hosted players
     ],
 

plugins/domains/cnn.com/cnn.video.js

@@ -1,6 +1,6 @@
 module.exports = {
 
-    re: /^http:\/\/((edition|www)\.)?cnn\.com\/video\//i,
+    re: /^https?:\/\/((edition|www)\.)?cnn\.com\/video\//i,
 
     mixins: [
         "canonical",

plugins/domains/codepen.io.js

@@ -2,7 +2,7 @@ var jquery = require('jquery');
 
 module.exports = {
 
-    re: /http:\/\/codepen\.io\/([a-z0-9\-]+)\/(pen|details)\/([a-z0-9\-]+)/i,
+    re: /https?:\/\/codepen\.io\/([a-z0-9\-]+)\/(pen|details|full)\/([a-z0-9\-]+)/i,
 
     mixins: [
         "oembed-thumbnail",

plugins/domains/collegehumor.com/collegehumor.picture.js

@@ -1,4 +1,4 @@
-var re = /http:\/\/www\.collegehumor\.com\/(picture|post)\/([0-9]+)\.*/i;
+var re = /https?:\/\/www\.collegehumor\.com\/(picture|post)\/([0-9]+)\.*/i;
 
 module.exports = {
 

plugins/domains/collegehumor.com/collegehumor.video.js

@@ -1,6 +1,6 @@
 var jquery = require('jquery');
 
-var re = /http:\/\/www\.collegehumor\.com\/(video|embed)\/([0-9]+)\.*/;
+var re = /https?:\/\/www\.collegehumor\.com\/(video|embed)\/([0-9]+)\.*/;
 
 module.exports = {
 

plugins/domains/dribbble.com.js

@@ -1,7 +1,7 @@
 module.exports = {
 
     re: [
-        /^http:\/\/dribbble\.com\/shots\/([a-zA-Z0-9\-]+)/i
+        /^https?:\/\/dribbble\.com\/shots\/([a-zA-Z0-9\-]+)/i
     ],
 
     mixins: [

plugins/domains/ebaumsworld.com.js

@@ -1,7 +1,7 @@
 module.exports = {
 
     re: [
-        /^http:\/\/www\.ebaumsworld\.com\/video\/watch\/(\d+)/i
+        /^https?:\/\/www\.ebaumsworld\.com\/video\/watch\/(\d+)/i
     ],
 
     mixins: [

plugins/domains/ec.europa.eu.js

@@ -1,7 +1,7 @@
 module.exports = {
 
     re: [
-        /http:\/\/ec\.europa\.eu\/avservices\/(video|focus)/i
+        /https?:\/\/ec\.europa\.eu\/avservices\/(video|focus)/i
     ],
 
     mixins: [

plugins/domains/flickr.com/flickr.photo.js

@@ -20,13 +20,28 @@ module.exports = {
             }
 
             var result = sizes && sizes.map(function(size) {
-                return {
-                    href: size.source.replace(/^https?:/i, ""),
-                    width: size.width,
-                    height: size.height,
-                    type: "image/jpeg",
-                    rel: size.width > 500 ? CONFIG.R.image : CONFIG.R.thumbnail
-                };
+
+                if (size.media == "photo") { 
+
+                    return {
+                        href: size.source.replace(/^https?:/i, ""),
+                        width: size.width,
+                        height: size.height,
+                        type: CONFIG.T.image_jpeg,
+                        rel: size.width >= 800 || /original/i.test(size.label) ? CONFIG.R.image : CONFIG.R.thumbnail
+                    }
+
+                } else if (size.media == "video") {
+
+                    return {
+                        href: size.source,
+                        "aspect-ratio": size.width / size.height,
+                        type: /mp4/i.test(size.label) ? CONFIG.T.video_mp4 : CONFIG.T.flash,
+                        rel: CONFIG.R.player
+                    }
+
+                } 
+
             }) || [];
 
             result.push({

plugins/domains/funnyordie.com/funnyordie.video.js

@@ -1,6 +1,6 @@
 var jquery = require('jquery');
 
-var re = /^http:\/\/www\.funnyordie\.com\/videos\//i;
+var re = /^https?:\/\/www\.funnyordie\.com\/videos\//i;
 
 module.exports = {