diff --git a/.gitignore b/.gitignore index 0450a28..37b6b85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.pem *.pem.pub *.zip +bin/chromium-*.br node_modules/ nodejs/ package-lock.json diff --git a/Makefile b/Makefile index 5c5f2db..b6375b5 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,8 @@ -.PHONY: inflated - %.zip: mkdir -p nodejs/node_modules/chrome-aws-lambda/ - cd nodejs/ && npm install puppeteer-core@~1.20.0 --no-bin-links --no-optional --no-package-lock --no-save --no-shrinkwrap && cd - + cd nodejs/ && npm install lambdafs@~1.3.0 puppeteer-core@~1.20.0 --no-bin-links --no-optional --no-package-lock --no-save --no-shrinkwrap && cd - npm pack tar --directory nodejs/node_modules/chrome-aws-lambda/ --extract --file chrome-aws-lambda-*.tgz --strip-components=1 rm chrome-aws-lambda-*.tgz mkdir -p $(dir $@) zip -9 --filesync --move --recurse-paths $@ nodejs/ - -inflated: bin/chromium-*.br - brotli --decompress --rm bin/chromium-*.br diff --git a/README.md b/README.md index 6c987d2..4097f4f 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,7 @@ If you wish to install an older version of Chromium, take a look at [Versioning] ## Usage -This package works with the `nodejs8.10` AWS Lambda runtime out of the box. - -The `nodejs10.x` AWS Lambda runtime is supported, but requires a [polyfill layer](_/nodejs10.x.zip) (due to a lighter execution environment). +This package works with the `nodejs8.10` and `nodejs10.x` AWS Lambda runtimes out of the box. ```javascript const chromium = require('chrome-aws-lambda'); @@ -60,7 +58,7 @@ exports.handler = async (event, context) => { }; ``` -You should allocate at least 512 MB of RAM to your Lambda, 1600 MB is recommended. +You should allocate at least 512 MB of RAM to your Lambda, however 1600 MB (or more) is recommended. ### Running Locally @@ -87,6 +85,8 @@ To use it, simply pass a **HTTPS** URL to a custom font face _before_ launching await chromium.font('https://raw.githack.com/googlei18n/noto-emoji/master/fonts/NotoColorEmoji.ttf'); ``` +> The above font is needed if you want to [render emojis](https://getemoji.com/). + Fonts with the same basename will only be downloaded if they are not already cached. > On non-serverless environments, the `font()` method is a no-op to avoid polluting the user space. @@ -135,7 +135,7 @@ This package is versioned based on the underlying `puppeteer` minor version: | `puppeteer` Version | `chrome-aws-lambda` Version | Chromium Revision | | ------------------- | --------------------------------- | ---------------------------------------------------- | -| `1.20.*` | `npm i chrome-aws-lambda@~1.20.1` | [`686378`](https://crrev.com/686378) (`78.0.3882.0`) | +| `1.20.*` | `npm i chrome-aws-lambda@~1.20.2` | [`686378`](https://crrev.com/686378) (`78.0.3882.0`) | | `1.19.*` | `npm i chrome-aws-lambda@~1.19.0` | [`674921`](https://crrev.com/674921) (`77.0.3844.0`) | | `1.18.*` | `npm i chrome-aws-lambda@~1.18.1` | [`672088`](https://crrev.com/672088) (`77.0.3835.0`) | | `1.18.*` | `npm i chrome-aws-lambda@~1.18.0` | [`669486`](https://crrev.com/669486) (`77.0.3827.0`) | @@ -177,12 +177,6 @@ make chrome_aws_lambda.zip The above will create a `chrome-aws-lambda.zip` file, which can be uploaded to your Layers console. -Alternatively, if have `brotli` installed and wish to create a layer with the Chromium binary already decompressed: - -```shell -make inflated chrome_aws_lambda.zip -``` - ## Google Cloud Functions Since version `1.11.2`, it's also possible to use this package on Google/Firebase Cloud Functions. @@ -231,8 +225,6 @@ This allows us to get the best compression ratio and faster decompression times. | chromium.br | Brotli | 10 | 36090087 | 34.42 | 73.65% | 0.765s | | chromium.br | Brotli | 11 | 34820408 | **33.21** | **74.58%** | 0.712s | -For this reason, a stripped-down version of [`iltorb`](https://github.com/MayhemYDG/iltorb) is bundled as a dependency. - ## License MIT diff --git a/_/.gitignore b/_/.gitignore deleted file mode 100644 index 0f2bed5..0000000 --- a/_/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!nodejs10.x.zip diff --git a/_/ansible/Makefile b/_/ansible/Makefile index a0ebd06..710c621 100755 --- a/_/ansible/Makefile +++ b/_/ansible/Makefile @@ -1,22 +1,7 @@ -.PHONY: ansible chromium iltorb +.PHONY: ansible chromium ansible: sudo pip install boto boto3 chromium: ansible-playbook plays/chromium.yml -i inventory.ini - -iltorb: - rm --force --recursive "$$PWD/../../node_modules/iltorb/" - rm --force --recursive "$$PWD/../../source/iltorb/" - docker run \ - --rm \ - --user $$(id -u "$$USER"):$$(id -g "$$USER") \ - --volume "$$PWD/../../":/srv/chrome-aws-lambda \ - --workdir /srv/chrome-aws-lambda node:8.10 \ - npm install --no-shrinkwrap --silent - mkdir --parents "$$PWD/../../source/iltorb/build/" - mv --target-directory="$$PWD/../../source/iltorb/" \ - "$$PWD/../../node_modules/iltorb/build/" \ - "$$PWD/../../node_modules/iltorb/index.js" \ - "$$PWD/../../node_modules/iltorb/LICENSE" diff --git a/_/ansible/plays/chromium.yml b/_/ansible/plays/chromium.yml index a6887ad..32cf98b 100644 --- a/_/ansible/plays/chromium.yml +++ b/_/ansible/plays/chromium.yml @@ -9,7 +9,7 @@ ssh-keygen -b 2048 -t rsa -f ansible.pem -q -N '' && \ chmod 0600 ansible.pem.pub args: - chdir: ../ + chdir: .. creates: ansible.pem - name: Creating EC2 Key Pair @@ -39,12 +39,12 @@ ec2: group: Chromium image: "{{ image }}" - instance_type: c5d.9xlarge + instance_type: c5d.4xlarge instance_initiated_shutdown_behavior: terminate key_name: ansible wait: yes spot_type: one-time - spot_price: "1.00" + spot_price: "0.75" spot_wait_timeout: 300 spot_launch_group: chromium region: "{{ region }}" @@ -81,7 +81,7 @@ environment: LANG: en_US.UTF-8 LC_ALL: en_US.UTF-8 - PATH: "{{ ansible_env.PATH }}:/srv/source/depot_tools" + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/srv/source/depot_tools" tasks: - name: Installing Packages @@ -294,21 +294,12 @@ - name: Compressing Chromium shell: | - /usr/local/bin/brotli --best --force {{ item }} + brotli --best --force {{ item }} args: chdir: /srv/build/chromium with_items: - "chromium-{{ version.stdout }}" - - name: Compressing SwiftShader - shell: | - /usr/local/bin/brotli --best --force {{ item }} - args: - chdir: /srv/source/chromium/src/out/Headless/swiftshader - with_items: - - libEGL.so - - libGLESv2.so - - name: Downloading Chromium fetch: src: "/srv/build/chromium/{{ item }}" @@ -318,15 +309,27 @@ with_items: - "chromium-{{ version.stdout }}.br" + - name: Archiving SwiftShader + shell: | + tar --directory /srv/source/chromium/src/out/Headless/swiftshader --create --file swiftshader.tar libEGL.so libGLESv2.so + args: + chdir: /srv/build/chromium + creates: /srv/build/chromium/swiftshader.tar + warn: false + + - name: Compressing SwiftShader + shell: | + brotli --best --force swiftshader.tar + args: + chdir: /srv/build/chromium + creates: /srv/build/chromium/swiftshader.tar.br + - name: Downloading SwiftShader fetch: - src: "/srv/source/chromium/src/out/Headless/swiftshader/{{ item }}" - dest: ../../../bin/swiftshader/ + src: /srv/build/chromium/swiftshader.tar.br + dest: ../../../bin/ flat: yes fail_on_missing: true - with_items: - - libEGL.so.br - - libGLESv2.so.br - name: Teardown AWS hosts: localhost diff --git a/_/nodejs10.x.zip b/_/nodejs10.x.zip deleted file mode 100644 index 53a518f..0000000 Binary files a/_/nodejs10.x.zip and /dev/null differ diff --git a/bin/aws.tar.br b/bin/aws.tar.br new file mode 100644 index 0000000..b025abd Binary files /dev/null and b/bin/aws.tar.br differ diff --git a/bin/chromium-78.0.3882.0.br b/bin/chromium.br similarity index 100% rename from bin/chromium-78.0.3882.0.br rename to bin/chromium.br diff --git a/bin/swiftshader.tar.br b/bin/swiftshader.tar.br new file mode 100644 index 0000000..72209c0 Binary files /dev/null and b/bin/swiftshader.tar.br differ diff --git a/bin/swiftshader/libEGL.so.br b/bin/swiftshader/libEGL.so.br deleted file mode 100644 index e0a8463..0000000 Binary files a/bin/swiftshader/libEGL.so.br and /dev/null differ diff --git a/bin/swiftshader/libGLESv2.so.br b/bin/swiftshader/libGLESv2.so.br deleted file mode 100644 index 152503e..0000000 Binary files a/bin/swiftshader/libGLESv2.so.br and /dev/null differ diff --git a/package.json b/package.json index 0cdfc0b..b303dfb 100644 --- a/package.json +++ b/package.json @@ -19,10 +19,11 @@ "scripts": { "postversion": "git push && git push --tags && npm publish" }, - "dependencies": {}, + "dependencies": { + "lambdafs": "^1.3.0" + }, "devDependencies": { - "@types/puppeteer": "~1.20.1", - "iltorb": "^2.4.3" + "@types/puppeteer": "~1.20.1" }, "peerDependencies": { "puppeteer-core": "1.20.x" diff --git a/source/iltorb/LICENSE b/source/iltorb/LICENSE deleted file mode 100644 index de63936..0000000 --- a/source/iltorb/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Nicolas Stepien - Hung Tran - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/source/iltorb/build/bindings/iltorb.node b/source/iltorb/build/bindings/iltorb.node deleted file mode 100755 index c957f8b..0000000 Binary files a/source/iltorb/build/bindings/iltorb.node and /dev/null differ diff --git a/source/iltorb/index.js b/source/iltorb/index.js deleted file mode 100644 index 1fe96c5..0000000 --- a/source/iltorb/index.js +++ /dev/null @@ -1,253 +0,0 @@ -'use strict'; - -exports.compress = compress; -exports.decompress = decompress; -exports.compressSync = compressSync; -exports.decompressSync = decompressSync; -exports.compressStream = compressStream; -exports.decompressStream = decompressStream; - -const { StreamEncode, StreamDecode } = require('./build/bindings/iltorb.node'); -const { Transform } = require('stream'); - -class TransformStreamEncode extends Transform { - constructor(params = {}, async = true) { - super(); - this.encoding = false; - this.corked = false; - this.flushing = false; - this.encoder = new StreamEncode(async, params); - } - - _transform(chunk, encoding, next) { - this.encoding = true; - this.encoder.transform(chunk, (err, output) => { - this.encoding = false; - if (err) { - return next(err); - } - this._push(output); - next(); - if (this.flushing) { - this.flush(true); - } - }); - } - - _flush(done) { - this.encoder.flush(true, (err, output) => { - if (err) { - return done(err); - } - this._push(output); - done(); - }); - } - - _push(output) { - if (output) { - for (let i = 0; i < output.length; i++) { - this.push(output[i]); - } - } - } - - flush(force) { - if (this.flushing && !force) { - return; - } - - if (!this.corked) { - this.cork(); - } - this.corked = true; - this.flushing = true; - - if (this.encoding) { - return; - } - - this.encoder.flush(false, (err, output) => { - if (err) { - this.emit('error', err); - } else { - this._push(output); - } - this.corked = false; - this.flushing = false; - this.uncork(); - }); - } -} - -class TransformStreamDecode extends Transform { - constructor(async = true) { - super(); - this.decoder = new StreamDecode(async); - } - - _transform(chunk, encoding, next) { - this.decoder.transform(chunk, (err, output) => { - if (err) { - return next(err); - } - this._push(output); - next(); - }); - } - - _flush(done) { - this.decoder.flush((err, output) => { - if (err) { - return done(err); - } - this._push(output); - done(); - }); - } - - _push(output) { - if (output) { - for (let i = 0; i < output.length; i++) { - this.push(output[i]); - } - } - } -} - -function compress(input, params, cb) { - if (typeof params === 'function') { - cb = params; - params = {}; - } - - const gotCallback = typeof cb === 'function'; - - if (!Buffer.isBuffer(input)) { - const err = new Error('Brotli input is not a buffer.'); - if (gotCallback) { - return process.nextTick(cb, err); - } - return Promise.reject(err); - } - - params = Object.assign({}, params, { size_hint: input.length }); - - if (gotCallback) { - return compressBuffer(input, params, cb); - } - - return new Promise(function(resolve, reject) { - compressBuffer(input, params, function(err, output) { - if (err) { - reject(err); - } else { - resolve(output); - } - }); - }); -} - -function compressBuffer(input, params, cb) { - const stream = new TransformStreamEncode(params); - const chunks = []; - let length = 0; - stream.on('error', cb); - stream.on('data', function(c) { - chunks.push(c); - length += c.length; - }); - stream.on('end', function() { - cb(null, Buffer.concat(chunks, length)); - }); - stream.end(input); -} - -function decompress(input, cb) { - const gotCallback = typeof cb === 'function'; - - if (!Buffer.isBuffer(input)) { - const err = new Error('Brotli input is not a buffer.'); - if (gotCallback) { - return process.nextTick(cb, err); - } - return Promise.reject(err); - } - - if (gotCallback) { - return decompressBuffer(input, cb); - } - - return new Promise(function(resolve, reject) { - decompressBuffer(input, function(err, output) { - if (err) { - reject(err); - } else { - resolve(output); - } - }); - }); -} - -function decompressBuffer(input, cb) { - const stream = new TransformStreamDecode(); - const chunks = []; - let length = 0; - stream.on('error', cb); - stream.on('data', function(c) { - chunks.push(c); - length += c.length; - }); - stream.on('end', function() { - cb(null, Buffer.concat(chunks, length)); - }); - stream.end(input); -} - -function compressSync(input, params) { - if (!Buffer.isBuffer(input)) { - throw new Error('Brotli input is not a buffer.'); - } - if (typeof params !== 'object') { - params = {}; - } - params = Object.assign({}, params, { size_hint: input.length }); - const stream = new TransformStreamEncode(params, false); - const chunks = []; - let length = 0; - stream.on('error', function(e) { - throw e; - }); - stream.on('data', function(c) { - chunks.push(c); - length += c.length; - }); - stream.end(input); - return Buffer.concat(chunks, length); -} - -function decompressSync(input) { - if (!Buffer.isBuffer(input)) { - throw new Error('Brotli input is not a buffer.'); - } - const stream = new TransformStreamDecode(false); - const chunks = []; - let length = 0; - stream.on('error', function(e) { - throw e; - }); - stream.on('data', function(c) { - chunks.push(c); - length += c.length; - }); - stream.end(input); - return Buffer.concat(chunks, length); -} - -function compressStream(params) { - return new TransformStreamEncode(params); -} - -function decompressStream() { - return new TransformStreamDecode(); -} diff --git a/source/index.js b/source/index.js index c299c21..c37f55c 100644 --- a/source/index.js +++ b/source/index.js @@ -1,17 +1,7 @@ -let { createBrotliDecompress } = require('zlib'); -let { createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync, unlinkSync } = require('fs'); -let { get } = require('https'); -let { URL } = require('url'); - -if (process.env.AWS_EXECUTION_ENV === 'AWS_Lambda_nodejs10.x') { - if (process.env.FONTCONFIG_PATH === undefined) { - process.env.FONTCONFIG_PATH = '/opt/lib'; - } - - if (process.env.LD_LIBRARY_PATH.startsWith('/opt/lib:') !== true) { - process.env.LD_LIBRARY_PATH = [...new Set(['/opt/lib', ...process.env.LD_LIBRARY_PATH.split(':')])].join(':'); - } -} +const { createWriteStream, existsSync, mkdirSync, readdirSync, unlinkSync } = require('fs'); +const { get } = require('https'); +const { inflate } = require('lambdafs'); +const { URL } = require('url'); class Chromium { /** @@ -91,6 +81,7 @@ class Chromium { '--disable-translate', '--disable-voice-input', '--disable-wake-on-wifi', + '--disk-cache-size=33554432', '--enable-async-dns', '--enable-simple-cache-backend', '--enable-tcp-fast-open', @@ -157,20 +148,23 @@ class Chromium { return '/tmp/chromium'; } - if (existsSync('/tmp/swiftshader') !== true) { - mkdirSync('/tmp/swiftshader'); - } + let input = `${__dirname}/../bin`; + let promises = [ + inflate(`${input}/chromium.br`), + inflate(`${input}/swiftshader.tar.br`), + ]; - const input = `${__dirname}/../bin`; - const binary = readdirSync(input).find((file) => { - return file.startsWith('chromium-'); - }); + if (process.env.AWS_EXECUTION_ENV === 'AWS_Lambda_nodejs10.x') { + promises.push(inflate(`${input}/aws.tar.br`)); - const promises = [ - inflate(`${input}/${binary}`, '/tmp/chromium'), - inflate(`${input}/swiftshader/libEGL.so.br`, '/tmp/swiftshader/libEGL.so'), - inflate(`${input}/swiftshader/libGLESv2.so.br`, '/tmp/swiftshader/libGLESv2.so'), - ]; + if (process.env.FONTCONFIG_PATH === undefined) { + process.env.FONTCONFIG_PATH = '/tmp/aws'; + } + + if (process.env.LD_LIBRARY_PATH.startsWith('/tmp/aws/lib') !== true) { + process.env.LD_LIBRARY_PATH = [...new Set(['/tmp/aws/lib', ...process.env.LD_LIBRARY_PATH.split(':')])].join(':'); + } + } return Promise.all(promises).then((result) => { return result.shift(); @@ -204,39 +198,4 @@ class Chromium { } } -function inflate(input, output, mode = 0o700) { - if (createBrotliDecompress === undefined) { - let iltorb = 'iltorb'; - - if (process.env.AWS_EXECUTION_ENV === 'AWS_Lambda_nodejs8.10') { - iltorb = `${__dirname}/iltorb`; - } - - createBrotliDecompress = require(iltorb).decompressStream; - } - - return new Promise((resolve, reject) => { - const source = createReadStream(input, { highWaterMark: 8 * 1024 * 1024 }); - const target = createWriteStream(output, { mode: mode }); - - source.once('error', (error) => { - return reject(error); - }); - - target.once('error', (error) => { - return reject(error); - }); - - target.once('close', () => { - return resolve(output); - }); - - if (input.endsWith('.br') === true) { - source.pipe(createBrotliDecompress({ chunkSize: 2 * 1024 * 1024 })).pipe(target); - } else { - source.pipe(target); - } - }); -} - module.exports = Chromium;