Thumbnailer with AWS Lambda on containers (pulumi#850)

khan23153 · Dec 2, 2020 · b5c7e8b · b5c7e8b
1 parent 04e1ed7
commit b5c7e8b
Show file tree

Hide file tree

Showing 9 changed files with 248 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -76,6 +76,7 @@ Example   | Description |
 [EKS - Hello World](aws-ts-eks-hello-world) | Deploy an EKS Kubernetes cluster with an EBS-backed StorageClass, then a Kubernetes namespace and nginx deployment into the cluster.
 [EKS - Migrate Node Groups](aws-ts-migrate-nodegroups) | Create an EKS cluster and node group to use for workload migration with zero downtime.
 [Fargate](aws-ts-hello-fargate) | Build, deploy, and run a Dockerized app using ECS, ECR, and Fargate.
+[Lambda Thumbnailer](aws-ts-lambda-thumbnailer) | Create a video thumbnail extractor using serverless functions.
 [Miniflux](aws-ts-pulumi-miniflux) | Stand up an RSS Service using Fargate and RDS.
 [Pulumi Webhooks](aws-ts-pulumi-webhooks) | Create a Pulumi `cloud.HttpEndpoint` that receives webhook events delivered by the Pulumi Service, then echos the event to Slack.
 [RDS and Airflow](aws-ts-airflow) | Deploy a RDS Postgres instance and containerized Airflow.

diff --git a/aws-ts-eks-distro/index.ts b/aws-ts-eks-distro/index.ts
@@ -1,6 +1,8 @@
-import * as pulumi from "@pulumi/pulumi";
+// Copyright 2016-2020, Pulumi Corporation.  All rights reserved.
+
 import * as aws from "@pulumi/aws";
 import * as k8s from "@pulumi/kubernetes";
+import * as pulumi from "@pulumi/pulumi";
 import * as eksdistro from "./eksdistro";
 
 const store = new aws.s3.Bucket("kops-state-store");
@@ -15,7 +17,7 @@ const k8sProvider = new k8s.Provider("provider", { kubeconfig: cluster.kubeconfi
 const pod = new k8s.core.v1.Pod("mypod", {
     spec: {
         containers: [{ name: "echo", image: "k8s.gcr.io/echoserver:1.4" }],
-    }
+    },
 }, { provider: k8sProvider });
 
 export const kubeconfig = cluster.kubeconfig;
diff --git a/aws-ts-lambda-thumbnailer/Pulumi.yaml b/aws-ts-lambda-thumbnailer/Pulumi.yaml
@@ -0,0 +1,8 @@
+name: video-thumbnailer-lambda
+runtime: nodejs
+description: A video thumbnail extractor using serverless functions packaged with containers
+template:
+  config:
+    aws:region:
+      description: The AWS region to deploy into
+      default: us-west-2
diff --git a/aws-ts-lambda-thumbnailer/README.md b/aws-ts-lambda-thumbnailer/README.md
@@ -0,0 +1,121 @@
+[![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new)
+
+# Video Thumbnailer Using AWS Lambda
+
+A video thumbnail extractor using serverless functions. The video processing function is packaged as a Docker container.
+
+Navigate to [Running Container Images in AWS Lambda](https://www.pulumi.com/blog/aws-lambda-container-support/) for a full walkthrough.
+
+## Prerequisites
+
+To run this example, make sure [Docker](https://docs.docker.com/engine/installation/) is installed and running.
+
+## Running the App
+
+1.  Create a new stack:
+
+    ```
+    pulumi stack init dev
+    ```
+
+1.  Configure Pulumi to use an AWS region of your choice, for example:
+
+    ```
+    pulumi config set aws:region us-west-2
+    ```
+
+1.  Restore NPM modules via `npm install` or `yarn install`.
+
+1.  Preview and deploy the app via `pulumi up`. The preview will take some time, as it builds a Docker container. A total of 16 resources are created.
+
+    ```
+    $ pulumi up
+    Previewing update (dev)
+
+    ...
+
+    Do you want to perform this update? yes
+    Updating (dev)
+
+         Type                                  Name                          Status      
+    +   pulumi:pulumi:Stack                   video-thumbnailer-lambda-dev  created     
+    +   ├─ awsx:ecr:Repository                sampleapp                     created     
+    +   │  ├─ aws:ecr:Repository              sampleapp                     created     
+    +   │  └─ aws:ecr:LifecyclePolicy         sampleapp                     created     
+    +   ├─ aws:s3:Bucket                      bucket                        created     
+    +   │  ├─ aws:s3:BucketEventSubscription  onNewThumbnail                created     
+    +   │  │  └─ aws:lambda:Permission        onNewThumbnail                created     
+    +   │  ├─ aws:s3:BucketEventSubscription  onNewVideo                    created     
+    +   │  │  └─ aws:lambda:Permission        onNewVideo                    created     
+    +   │  └─ aws:s3:BucketNotification       onNewVideo                    created     
+    +   ├─ aws:iam:Role                       onNewThumbnail                created     
+    +   ├─ aws:iam:Role                       thumbnailerRole               created     
+    +   ├─ aws:lambda:Function                onNewThumbnail                created     
+    +   ├─ aws:iam:RolePolicyAttachment       onNewThumbnail-32be53a2       created     
+    +   ├─ aws:iam:RolePolicyAttachment       lambdaFullAccess              created     
+    +   └─ aws:lambda:Function                thumbnailer                   created     
+    
+    Outputs:
+        bucketName: "bucket-7c6b55a"
+
+    Resources:
+        + 16 created
+
+    Duration: 1m41s
+    ```
+
+1.  View the stack outputs:
+
+    ```
+    $ pulumi stack output
+    Current stack outputs (1):
+        OUTPUT                                           VALUE
+        bucketName                                       bucket-7c6b55a
+    ```
+
+1.  Upload a video, embedding the timestamp in the filename:
+
+    ```
+    $ aws s3 cp ./sample/cat.mp4 s3://$(pulumi stack output bucketName)/cat_00-01.mp4
+    upload: sample/cat.mp4 to s3://***/cat_00-01.mp4
+    ```
+
+1.  View the logs from both Lambda functions:
+
+    ```
+    $ pulumi logs -f
+    Collecting logs for stack dev since 2020-12-02T08:58:43.000+01:00.
+
+    2020-12-02T09:58:39.747+01:00[           thumbnailer-dbb2a35] START RequestId: 3ec2886e-e739-4764-be3b-a8e5a48a4986 Version: $LATEST
+    2020-12-02T09:58:39.750+01:00[           thumbnailer-dbb2a35] 2020-12-02T08:58:39.748Z	3ec2886e-e739-4764-be3b-a8e5a48a4986	INFO	Video handler called
+    2020-12-02T09:58:39.750+01:00[           thumbnailer-dbb2a35] 2020-12-02T08:58:39.750Z	3ec2886e-e739-4764-be3b-a8e5a48a4986	INFO	aws s3 cp s3://bucket-33b87c2/cat_00-01.mp4 /tmp/cat_00-01.mp4
+    download: s3://bucket-33b87c2/cat_00-01.mp4 to ../../tmp/cat_00-01.mp4ed 256.0 KiB/666.5 KiB (1.2 MiB/s) with 1 file(s) remaining
+    2020-12-02T09:58:53.068+01:00[           thumbnailer-dbb2a35] 2020-12-02T08:58:53.068Z	3ec2886e-e739-4764-be3b-a8e5a48a4986	INFO	ffmpeg -v error -i /tmp/cat_00-01.mp4 -ss 00:01 -vframes 1 -f image2 -an -y /tmp/cat.jpg
+    2020-12-02T09:59:01.701+01:00[           thumbnailer-dbb2a35] 2020-12-02T08:59:01.701Z	3ec2886e-e739-4764-be3b-a8e5a48a4986	INFO	aws s3 cp /tmp/cat.jpg s3://bucket-33b87c2/cat.jpg
+    upload: ../../tmp/cat.jpg to s3://bucket-33b87c2/cat.jpg          pleted 86.6 KiB/86.6 KiB (315.8 KiB/s) with 1 file(s) remaining
+    2020-12-02T09:59:11.628+01:00[           thumbnailer-dbb2a35] 2020-12-02T08:59:11.627Z	3ec2886e-e739-4764-be3b-a8e5a48a4986	INFO	*** New thumbnail: file cat_00-01.mp4 was saved at 2020-12-02T08:58:33.845Z.
+    2020-12-02T09:59:11.668+01:00[           thumbnailer-dbb2a35] END RequestId: 3ec2886e-e739-4764-be3b-a8e5a48a4986
+    2020-12-02T09:59:11.668+01:00[           thumbnailer-dbb2a35] REPORT RequestId: 3ec2886e-e739-4764-be3b-a8e5a48a4986	Duration: 31920.84 ms	Billed Duration: 32733 ms	Memory Size: 128 MB	Max Memory Used: 128 MB	Init Duration: 811.55 ms	
+    2020-12-02T09:59:11.777+01:00[        onNewThumbnail-2f969e0] START RequestId: 07c13039-eccb-4e38-a3cf-c7fa11982b84 Version: $LATEST
+    2020-12-02T09:59:11.788+01:00[        onNewThumbnail-2f969e0] 2020-12-02T08:59:11.782Z	07c13039-eccb-4e38-a3cf-c7fa11982b84	INFO	onNewThumbnail called
+    2020-12-02T09:59:11.788+01:00[        onNewThumbnail-2f969e0] 2020-12-02T08:59:11.788Z	07c13039-eccb-4e38-a3cf-c7fa11982b84	INFO	*** New thumbnail: file cat.jpg was saved at 2020-12-02T08:59:06.333Z.
+    2020-12-02T09:59:11.809+01:00[        onNewThumbnail-2f969e0] END RequestId: 07c13039-eccb-4e38-a3cf-c7fa11982b84
+    2020-12-02T09:59:11.809+01:00[        onNewThumbnail-2f969e0] REPORT RequestId: 07c13039-eccb-4e38-a3cf-c7fa11982b84	Duration: 31.96 ms	Billed Duration: 32 ms	Memory Size: 128 MB	Max Memory Used: 65 MB	Init Duration: 171.22 ms
+        ```
+
+1.  Download the key frame:
+
+    ```
+    $ aws s3 cp s3://$(pulumi stack output bucketName)/cat.jpg .
+    download: s3://***/cat.jpg to ./cat.jpg
+    ```
+
+## Clean up
+
+To clean up the resources, you will first need to clear the contents of the bucket.
+
+```bash
+aws s3 rm s3://$(pulumi stack output bucketName) --recursive
+```
+
+Then, run `pulumi destroy` and answer the confirmation question at the prompt.
diff --git a/aws-ts-lambda-thumbnailer/docker-ffmpeg-thumb/Dockerfile b/aws-ts-lambda-thumbnailer/docker-ffmpeg-thumb/Dockerfile
@@ -0,0 +1,24 @@
+FROM amazon/aws-lambda-nodejs:12
+ARG FUNCTION_DIR="/var/task"
+
+# Install tar and xz
+RUN yum install tar xz unzip -y
+
+# Install awscli
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -s
+RUN unzip -q awscliv2.zip
+RUN ./aws/install
+
+# Install ffmpeg
+RUN curl https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz -s
+RUN tar -xf ffmpeg.tar.xz
+RUN mv ffmpeg-4.3.1-amd64-static/ffmpeg /usr/bin
+
+# Create function directory
+RUN mkdir -p ${FUNCTION_DIR}
+
+# Copy handler function and package.json
+COPY index.js ${FUNCTION_DIR}
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "index.handler" ]
diff --git a/aws-ts-lambda-thumbnailer/docker-ffmpeg-thumb/index.js b/aws-ts-lambda-thumbnailer/docker-ffmpeg-thumb/index.js
@@ -0,0 +1,31 @@
+'use strict';
+const { execSync } = require('child_process');
+
+function run(command) {
+    console.log(command);
+    const result = execSync(command, {stdio: 'inherit'});
+    if (result != null) {
+        console.log(result.toString());
+    }
+}
+
+exports.handler = async (event) => {
+    console.log("Video handler called");
+
+    if (!event.Records) {
+        return;
+    }
+
+    for (const record of event.Records) {
+        const fileName = record.s3.object.key;
+        const bucketName = record.s3.bucket.name;
+        const thumbnailFile = fileName.substring(0, fileName.indexOf("_")) + ".jpg";
+        const framePos = fileName.substring(fileName.indexOf("_")+1, fileName.indexOf(".")).replace("-", ":");
+
+        run(`aws s3 cp s3://${bucketName}/${fileName} /tmp/${fileName}`);
+        run(`ffmpeg -v error -i /tmp/${fileName} -ss ${framePos} -vframes 1 -f image2 -an -y /tmp/${thumbnailFile}`);
+        run(`aws s3 cp /tmp/${thumbnailFile} s3://${bucketName}/${thumbnailFile}`);
+
+        console.log(`*** New thumbnail: file ${fileName} was saved at ${record.eventTime}.`);
+    }    
+};
diff --git a/aws-ts-lambda-thumbnailer/index.ts b/aws-ts-lambda-thumbnailer/index.ts
@@ -0,0 +1,49 @@
+// Copyright 2016-2020, Pulumi Corporation.  All rights reserved.
+
+import * as aws from "@pulumi/aws";
+import * as awsx from "@pulumi/awsx";
+
+// A bucket to store videos and thumbnails.
+const bucket = new aws.s3.Bucket("bucket");
+
+const image = awsx.ecr.buildAndPushImage("sampleapp", {
+    context: "./docker-ffmpeg-thumb",
+});
+const role = new aws.iam.Role("thumbnailerRole", {
+    assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({ Service: "lambda.amazonaws.com" }),
+});
+new aws.iam.RolePolicyAttachment("lambdaFullAccess", {
+    role: role.name,
+    policyArn: aws.iam.ManagedPolicies.AWSLambdaFullAccess
+});
+
+const thumbnailer = new aws.lambda.Function("thumbnailer", {
+    packageType: "Image",
+    imageUri: image.imageValue,
+    role: role.arn,
+    timeout: 900,
+});
+
+// When a new video is uploaded, run the FFMPEG task on the video file.
+// Use the time index specified in the filename (e.g. cat_00-01.mp4 uses timestamp 00:01)
+bucket.onObjectCreated("onNewVideo", thumbnailer, { filterSuffix: ".mp4" });
+
+// Export the bucket name.
+export const bucketName = bucket.id;
+
+// When a new thumbnail is created, log a message.
+bucket.onObjectCreated("onNewThumbnail", new aws.lambda.CallbackFunction<aws.s3.BucketEvent, void>("onNewThumbnail", {
+    callback: async bucketArgs => {
+        console.log("onNewThumbnail called");
+        if (!bucketArgs.Records) {
+            return;
+        }
+
+        for (const record of bucketArgs.Records) {
+            console.log(`*** New thumbnail: file ${record.s3.object.key} was saved at ${record.eventTime}.`);
+        }
+    },
+    policies: [
+        aws.iam.ManagedPolicies.AWSLambdaFullAccess,                 // Provides wide access to "serverless" services (Dynamo, S3, etc.)
+    ],
+}), { filterSuffix: ".jpg" });
diff --git a/aws-ts-lambda-thumbnailer/package.json b/aws-ts-lambda-thumbnailer/package.json
@@ -0,0 +1,10 @@
+{
+    "name": "video-thumbnailer-lambda",
+    "version": "0.1.0",
+    "main": "index.js",
+    "dependencies": {
+        "@pulumi/pulumi": "^2.0.0",
+        "@pulumi/aws": "^3.17.0",
+        "@pulumi/awsx": "^0.20.0"
+    }
+}
diff --git a/aws-ts-lambda-thumbnailer/sample/cat.mp4 b/aws-ts-lambda-thumbnailer/sample/cat.mp4