Skip to content

Commit

Permalink
Adds in memory cache to avoid repeated work (keyed to input file size…
Browse files Browse the repository at this point in the history
… for now). Adds dryRun and useCache options.
  • Loading branch information
zachleat committed Jan 2, 2021
1 parent 6f36e7a commit a0bbb9a
Showing 5 changed files with 172 additions and 71 deletions.
32 changes: 32 additions & 0 deletions filesize-cache.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
const fs = require("fs");

class FileSizeCache {
constructor() {
this.cache = {};
}

getSize(path) {
let stats = fs.statSync(path);
return stats.size;
}

add(options, results) {
let key = JSON.stringify(options);

this.cache[key] = {
results
};
}

get(options) {
let key = JSON.stringify(options);
if(this.cache[key]) {
// may return promise
return this.cache[key].results;
}

return false;
}
}

module.exports = FileSizeCache;
112 changes: 71 additions & 41 deletions img.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
// TODO use `srcsethelp` project to improve output (blurry lowsrc?)

const path = require("path");
const fs = require("fs-extra");
const { URL } = require("url");

const shorthash = require("short-hash");
const {default: PQueue} = require("p-queue");
const getImageSize = require("image-size");
@@ -12,18 +9,10 @@ const debug = require("debug")("EleventyImg");

const svgHook = require("./format-hooks/svg");

const CacheAsset = require("@11ty/eleventy-cache-assets");

function filenameFormat(id, src, width, format) { // and options
if (width) {
return `${id}-${width}.${format}`;
}

return `${id}.${format}`;
}
const {RemoteAssetCache, queue} = require("@11ty/eleventy-cache-assets");
const FileSizeCache = require("./filesize-cache");

const globalOptions = {
src: null,
widths: [null],
formats: ["webp", "jpeg"], // "png", "svg", "avif"
concurrency: 10,
@@ -42,13 +31,16 @@ const globalOptions = {
svg: svgHook,
},
cacheDuration: "1d", // deprecated, use cacheOptions.duration
// disk cache for remote assets
cacheOptions: {
// duration: "1d",
// directory: ".cache",
// removeUrlQueryParams: false,
// fetchOptions: {},
},
filenameFormat,
useCache: true, // in-memory cache
dryRun: false,
};

const MIME_TYPES = {
@@ -59,6 +51,25 @@ const MIME_TYPES = {
"avif": "image/avif",
};

/* Size Cache */
let sizeCache = new FileSizeCache();

/* Queue */
let processingQueue = new PQueue({
concurrency: globalOptions.concurrency
});
processingQueue.on("active", () => {
debug( `Concurrency: ${processingQueue.concurrency}, Size: ${processingQueue.size}, Pending: ${processingQueue.pending}` );
});

function filenameFormat(id, src, width, format) { // and options
if (width) {
return `${id}-${width}.${format}`;
}

return `${id}.${format}`;
}

function getFormatsArray(formats) {
if(formats && formats.length) {
if(typeof formats === "string") {
@@ -247,6 +258,8 @@ async function resizeImage(src, options = {}) {
sharpInstance.resize(resizeOptions);
}

await fs.ensureDir(options.outputDir);

if(options.formatHooks && options.formatHooks[outputFormat]) {
let hookResult = await options.formatHooks[outputFormat].call(stat, sharpInstance);
if(hookResult) {
@@ -260,7 +273,7 @@ async function resizeImage(src, options = {}) {
sharpInstance.toFormat(outputFormat, sharpFormatOptions);
}

outputFilePromises.push(sharpInstance.toFile(stat.outputPath).then(data => {
outputFilePromises.push(sharpInstance[options.dryRun ? "toBuffer" : "toFile"](stat.outputPath).then(data => {
stat.size = data.size;
return stat;
}));
@@ -282,52 +295,69 @@ function isFullUrl(url) {
}
}

/* Combine it all together */
async function image(src, opts) {
function queueImage(src, opts) {
let options = Object.assign({}, globalOptions, opts);

if(!src) {
throw new Error("`src` is a required argument to the eleventy-img utility (can be a string file path, string URL, or buffer).");
}

options.__originalSrc = src;

let assetCache;
let cacheOptions = Object.assign({
duration: options.cacheDuration, // deprecated
type: "buffer"
}, options.cacheOptions);

if(typeof src === "string" && isFullUrl(src)) {
// fetch remote image
let buffer = await CacheAsset(src, Object.assign({
duration: opts.cacheDuration,
type: "buffer"
}, opts.cacheOptions));

opts.sourceUrl = src;
return resizeImage(buffer, opts);
options.sourceUrl = src;

assetCache = new RemoteAssetCache(src, cacheOptions.directory, cacheOptions);

// valid only if asset cached file is still valid
options.__validAssetCache = assetCache.isCacheValid(cacheOptions.duration);
} else {
options.__originalSize = fs.statSync(src).size;
}

// use file path to local image
return resizeImage(src, opts);
}
let cached = sizeCache.get(options);
if(options.useCache && cached) {
return cached;
}

/* Queue */
let queue = new PQueue({
concurrency: globalOptions.concurrency
});
queue.on("active", () => {
debug( `Concurrency: ${queue.concurrency}, Size: ${queue.size}, Pending: ${queue.pending}` );
});
let promise = processingQueue.add(async () => {
let input;

async function queueImage(src, opts) {
let options = Object.assign({}, globalOptions, opts);
if(typeof src === "string" && isFullUrl(src)) {
// fetch remote image
if(queue) {
// eleventy-cache-assets 2.0.4 and up
input = await queue(src, () => assetCache.fetch());
} else {
// eleventy-cache-assets 2.0.3 and below
input = await assetCache.fetch(cacheOptions);
}
} else {
input = src;
}

return resizeImage(input, options);
});

// create the output dir
await fs.ensureDir(options.outputDir);
sizeCache.add(options, promise);

return queue.add(() => image(src, options));
return promise;
}

module.exports = queueImage;

Object.defineProperty(module.exports, "concurrency", {
get: function() {
return queue.concurrency;
return processingQueue.concurrency;
},
set: function(concurrency) {
queue.concurrency = concurrency;
processingQueue.concurrency = concurrency;
},
});

3 changes: 1 addition & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -36,8 +36,7 @@
"@11ty/eleventy": ">=0.10.0"
},
"dependencies": {
"@11ty/eleventy-cache-assets": "^2.0.3",
"@saschazar/wasm-avif": "^1.0.0",
"@11ty/eleventy-cache-assets": "^2.0.4",
"debug": "^4.3.1",
"fs-extra": "^9.0.1",
"image-size": "^0.9.3",
39 changes: 11 additions & 28 deletions sample/sample.js
Original file line number Diff line number Diff line change
@@ -1,47 +1,30 @@
const eleventyImage = require("../");

(async () => {
// Twitter removed this URL
// await eleventyImage(`https://twitter.com/zachleat/profile_image?size=bigger`)
// await eleventyImage(`https://twitter.com/eleven_ty/profile_image?size=bigger`, {
// widths: [48]
// });

// await eleventyImage(`https://unavatar.now.sh/twitter/zachleat?fallback=false`, {
// widths: [75, null],
// formats: [null]
// });

// await eleventyImage(`https://unavatar.now.sh/twitter/zachleat?fallback=false`, {
// widths: [null],
// formats: ["svg"],
// });

// upscale svg issue #32
console.log( await eleventyImage(`https://www.netlify.com/v3/img/components/leaves.svg`, {
let leaves1 = await eleventyImage(`https://www.netlify.com/v3/img/components/leaves.svg`, {
formats: ["png", "avif"],
widths: [2000],
svgShortCircuit: true,
}));
});
console.log( { leaves1 } );

let leaves = await eleventyImage(`https://www.netlify.com/v3/img/components/leaves.svg`, {
let leaves2 = await eleventyImage(`https://www.netlify.com/v3/img/components/leaves.svg`, {
formats: ["svg", "webp", "jpeg", "png"],
// formats: [null],
widths: [400, 800, null],
svgShortCircuit: true,
});
console.log( leaves );
console.log( { leaves2 } );

let mexicoFlag = await eleventyImage("../test/Flag_of_Mexico.svg", {
formats: ["svg", "avif"],
widths: [600, null],
});
console.log( mexicoFlag );
console.log( { mexicoFlag } );

// let results = await eleventyImage("../test/bio-2017.jpg", {
// formats: ["avif", "jpeg"],
// widths: [400, 1280],
// });
let bioImage = await eleventyImage("../test/bio-2017.jpg", {
formats: ["avif", "jpeg"],
widths: [400, 1280],
});

// console.log( results );
console.log( bioImage );
})();
57 changes: 57 additions & 0 deletions test/test.js
Original file line number Diff line number Diff line change
@@ -341,3 +341,60 @@ test("Sync by dimension with jpeg input (wrong dimensions, supplied are larger t
t.is(stats.jpeg[0].outputPath, "img/97854483-164.jpeg");
t.is(stats.jpeg[1].outputPath, "img/97854483-328.jpeg");
});

test("Keep a cache, reuse with same file names and options", async t => {
let promise1 = eleventyImage("./test/bio-2017.jpg", { dryRun: true });
let promise2 = eleventyImage("./test/bio-2017.jpg", { dryRun: true });
t.is(promise1, promise2);

let stats1 = await promise1;
let stats2 = await promise2;
t.deepEqual(stats1, stats2);
});

test("Keep a cache, reuse with same remote url and options", async t => {
let promise1 = eleventyImage("https://www.zachleat.com/img/avatar-2017-big.png", { dryRun: true });
let promise2 = eleventyImage("https://www.zachleat.com/img/avatar-2017-big.png", { dryRun: true });
t.is(promise1, promise2);

let stats1 = await promise1;
let stats2 = await promise2;
t.deepEqual(stats1, stats2);
});

test("Keep a cache, don’t reuse with same file names and different options", async t => {
let promise1 = eleventyImage("./test/bio-2017.jpg", {
widths: [null],
dryRun: true,
});
let promise2 = eleventyImage("./test/bio-2017.jpg", {
widths: [300],
dryRun: true,
});
t.not(promise1, promise2);

let stats1 = await promise1;
let stats2 = await promise2;
t.notDeepEqual(stats1, stats2);

t.is(stats1.jpeg.length, 1);
t.is(stats2.jpeg.length, 1);
});

test.skip("Keep a cache, don’t reuse with if the image changes", async t => {
let promise1 = eleventyImage("./test/bio-2017.jpg", {
dryRun: true,
});
// TODO modify image
let promise2 = eleventyImage("./test/bio-2017.jpg", {
dryRun: true,
});
t.not(promise1, promise2);

let stats1 = await promise1;
let stats2 = await promise2;
t.notDeepEqual(stats1, stats2);

t.is(stats1.jpeg.length, 1);
t.is(stats2.jpeg.length, 1);
});

0 comments on commit a0bbb9a

Please sign in to comment.