Support collections of albums

mariquit · Nov 11, 2020 · f53d7f5 · f53d7f5
1 parent a81013a
commit f53d7f5
Show file tree

Hide file tree

Showing 47 changed files with 4,292 additions and 1,810 deletions.
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 > An AWS CloudFormation stack to run a serverless password-protected photo
   gallery
 
-**Demo:** <https://awspics.net>  
+**Demo:** <https://awspics.net>
 **Credentials:** "username" / "password"
 
 ![](assets/awspics.gif)
@@ -93,7 +93,7 @@ A video walkthrough [is available on YouTube](https://youtu.be/010AGcY4uoE).
    ```
    Click Deploy (currently the orange button on the upper left).
 
-   Then click on your Lambda - Layers and you will see a version ARN that looks like: 
+   Then click on your Lambda - Layers and you will see a version ARN that looks like:
    ```
    Name          Version  Version ARN
    image-magick  1        arn:aws:lambda:us-east-1:000000000000:layer:image-magick:1
@@ -132,6 +132,44 @@ It should contain the following info - minus the comments:
   // note that the cookies are session cookies, and will get deleted when the
   // browser is closed anyway
   "sessionDuration=86400",
+  // Optional tracking ID for Google Analytics, if specified then a GA JS
+  // snippet will be outputted in the site's HTML, or leave blank for no GA
+  "googleanalytics=",
+  // Optionally override the path prefix for where original albums and their
+  // pictures live, or leave blank to have this default to "pics/original/"
+  "picsOriginalPath=",
+  // Optionally sort albums by name when building the homepage (if
+  // groupAlbumsIntoCollections is disabled), or when building collection pages
+  // (if groupAlbumsIntoCollections is enabled), specify either "asc" or "desc",
+  // or leave blank to output albums in the order that they're returned from
+  // the S3 list objects call
+  "albumSort=",
+  // Optionally sort pictures in all albums by name when building album pages,
+  // specify either "asc" or "desc", or leave blank to output pictures in the
+  // reverse order that they're returned from the S3 list objects call
+  "pictureSort=",
+  // Optionally sort collections by name when building the homepage (if
+  // groupAlbumsIntoCollections is enabled), specify either "asc" or "desc", or
+  // leave blank to output collections in the order that they're returned from
+  // the S3 list objects call
+  "collectionSort=",
+  // Optionally specify "true" to indicate that the pictures have two grouping
+  // levels, collections (first-level folders in the bucket) and albums
+  // (second-level folders in the bucket), or leave blank to indicate that
+  // the pictures just have one grouping level, albums
+  "groupAlbumsIntoCollections=",
+  // Indent homepage and album HTML output with spaces, specify "true" to
+  // enable, or leave blank to instead indent HTML output with tabs
+  "spacesInsteadOfTabs=",
+  // Optionally show the specified custom HTML instead of a design credits link
+  // to HTML5 UP on the home page, recommended that this be a link in the form:
+  // <a href="https://site123.com">Site 123</a>
+  // Note: a design credits link to HTML5 UP will still show on album pages
+  "homePageCreditsOverride=",
+  // Optionally hide the design credits link to HTML5 UP on the home page,
+  // specify "true" to enable, or leave blank to show the link
+  // Note: a design credits link to HTML5 UP will still show on album pages
+  "hideHomePageCredits=",
   // KMS key ID created in step 2
   "kmsKeyId=00000000-0000-0000-0000-000000000000",
   // CloudFront key pair ID from step 3
@@ -154,7 +192,7 @@ It should contain the following info - minus the comments:
 
   // encrypted contents of the <htpasswd> file from step 6
   "encryptedHtpasswd=AQICAH...",
-  
+
   // ------------------
   // SSL Certificate ARN
   // - provide this if you want to use an existing ACM Certificate.
@@ -185,14 +223,14 @@ You will want to update the frequency of the Cloudwatch Events Rule from its def
 in the app.yml file or after the fact in the AWS Management console.
 
 ### Note on ImageMagick Layer for Lambda
-When Amazon deprecated Node.js 8.10, they removed ImageMagick from the Amazon Linux 2 AMIs that are required to run Node.js 10.x. Again, ImageMagick is no longer bundled with the Node.js 10.x runtime. This fix may also help with running on Node.js 12.x in the future. This provides a Lambda Layer (essentially a library) for your Lambda function that makes the existing code work with Node.js 10.x. 
+When Amazon deprecated Node.js 8.10, they removed ImageMagick from the Amazon Linux 2 AMIs that are required to run Node.js 10.x. Again, ImageMagick is no longer bundled with the Node.js 10.x runtime. This fix may also help with running on Node.js 12.x in the future. This provides a Lambda Layer (essentially a library) for your Lambda function that makes the existing code work with Node.js 10.x.
 
 
 ##### Note on SSL Cert
 AWS Certificate Manager now supports SSL cert verification via DNS validation.
 It is recommended that you manually request the certificate for your hosted zone and
-chose DNS validation method for much faster validation. Then use the resulting ARN 
-in your config. You can also leave this config key empty to create the certificate as 
+chose DNS validation method for much faster validation. Then use the resulting ARN
+in your config. You can also leave this config key empty to create the certificate as
 normal.
 
 Once the initial deployment is done, you'll need to point your domain's DNS
@@ -252,40 +290,40 @@ rely on the SSL certificate being created in CloudFormation. Create it manually
 reference.
 
 GeoRestriction is commented out in the CloudFront configuration in the app.yaml. If you are sharing
-with friends and family in a specific geographic area, this is a slight improvement to security and 
-cost reduction. The US is provided as an example, but additional countries can be added to a 
+with friends and family in a specific geographic area, this is a slight improvement to security and
+cost reduction. The US is provided as an example, but additional countries can be added to a
 (whitelist/blacklist) based on their two letter ISO 3166-1 alpha-2 country code.
 
-S3 Server Side AES256 encryption is enabled for the source and resized photo buckets and encrypts files 
-using the AWS S3 Master key. Each bucket is configured to force encryption of any file it receives 
-(you will need to check the upload box or specify it in the CLI when uploading photo files to the buckets) 
-and you will get access denied messages if you don't. The Resize function re-encrypts the resized photos 
-with AES256 SSE before uploading them into the resized bucket. Cloudfront with an OAI is able to access 
-files using the S3 Master Key without any issue. One cannot at this time use a KMS key for encrypting 
+S3 Server Side AES256 encryption is enabled for the source and resized photo buckets and encrypts files
+using the AWS S3 Master key. Each bucket is configured to force encryption of any file it receives
+(you will need to check the upload box or specify it in the CLI when uploading photo files to the buckets)
+and you will get access denied messages if you don't. The Resize function re-encrypts the resized photos
+with AES256 SSE before uploading them into the resized bucket. Cloudfront with an OAI is able to access
+files using the S3 Master Key without any issue. One cannot at this time use a KMS key for encrypting
 bucket data to be accessed via Cloudfront without more complexity.
 
-The EventInvoke config is included for SiteBuilder to prevent it from queueing up invocations and causing 
-multiple cloudfront invalidations at the same time. If you need to run sitebuilder more frequently, adjust 
+The EventInvoke config is included for SiteBuilder to prevent it from queueing up invocations and causing
+multiple cloudfront invalidations at the same time. If you need to run sitebuilder more frequently, adjust
 the rate of events by editing the CloudWatch Events rule in the Management console or the app.yml file.
 
-Also, you can reduce compute costs and lock down the application several ways: 1) by manually throttling 
-the Resize Function and the SiteBuilder Function in Lambda in the Management console or 2) disabling the 
-CloudWatch Events rule that runs SiteBuilder or 3) manually disabling the trigger for a Lambda function 
+Also, you can reduce compute costs and lock down the application several ways: 1) by manually throttling
+the Resize Function and the SiteBuilder Function in Lambda in the Management console or 2) disabling the
+CloudWatch Events rule that runs SiteBuilder or 3) manually disabling the trigger for a Lambda function
 in the Management console.
 
 Default directory for photos is S3://BUCKET_NAME/pics/original/YOUR_ALBUMS_GO_HERE
 
 
 ## Troubleshooting
 
-If the project deploys and the login is entered correctly, but you are receiving access 
-denied messages, review your DNS settings. You only need a single DNS A record 
+If the project deploys and the login is entered correctly, but you are receiving access
+denied messages, review your DNS settings. You only need a single DNS A record
 pointing to the CloudFront Alias for your domain, and time for it to propagate.
 
 If SiteBuilder is hanging or having trouble completing, you may need to adjust the rate limiting delay block in index.js.
-The current S3 rate limit is 3500 writes a second, and 5500 reads/sec. If you're writing 30 files per album, 
-if you have more than 116 albums, you will hit the rate limit - and SiteBuilder will just hang and you will see 
-the files as a partial listing in the web directory. 
+The current S3 rate limit is 3500 writes a second, and 5500 reads/sec. If you're writing 30 files per album,
+if you have more than 116 albums, you will hit the rate limit - and SiteBuilder will just hang and you will see
+the files as a partial listing in the web directory.
 
 
 ## Credits

diff --git a/app.yml b/app.yml
@@ -48,8 +48,40 @@ Parameters:
   googleanalytics:
     Description: Google tracking id (gtag)
     Type: String
+  picsOriginalPath:
+    Description: Path prefix to original albums and pictures
+    Type: String
+    Default: ''
+  albumSort:
+    Description: Album sort order (asc or desc, or blank)
+    Type: String
+    Default: ''
+  pictureSort:
+    Description: Picture sort order (asc or desc, or blank)
+    Type: String
+    Default: ''
+  collectionSort:
+    Description: Collection sort order (asc or desc, or blank)
+    Type: String
+    Default: ''
+  groupAlbumsIntoCollections:
+    Description: Group first by collections then by albums (true or blank)
+    Type: String
+    Default: ''
+  spacesInsteadOfTabs:
+    Description: Indent HTML output with spaces instead of tabs (true or blank)
+    Type: String
+    Default: ''
+  homePageCreditsOverride:
+    Description: Show this HTML instead of the default credits link on the home page
+    Type: String
+    Default: ''
+  hideHomePageCredits:
+    Description: His te default credits link on the home page (true or blank)
+    Type: String
+    Default: ''
   ImageMagickLayer:
-    Description: layer for nodejs10.x and nodejs12.x for imagemagick here https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:145266761615:applications~image-magick-lambda-layer 
+    Description: layer for nodejs10.x and nodejs12.x for imagemagick here https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:145266761615:applications~image-magick-lambda-layer
     Type: String
   LambdaRate:
     Description: The rate (frequency) that determines when CloudWatch Events runs the rule that triggers the SiteBuilderFunction.
@@ -135,6 +167,7 @@ Resources:
       Environment:
         Variables:
           RESIZED_BUCKET: !Ref resizedBucket
+          PICS_ORIGINAL_PATH: !Ref picsOriginalPath
       Timeout: 30
       MemorySize: 1024
 
@@ -185,7 +218,7 @@ Resources:
       Targets:
         - Arn: !Sub ${SiteBuilderFunction.Arn}
           Id: LambdaSchedule
-  #        
+  #
   # Permission to invoke a lambda function with the CloudWatch Event
   #
   LambdaSchedulePermission:
@@ -223,6 +256,14 @@ Resources:
           SITE_BUCKET: !Ref webBucket
           WEBSITE: !Ref website
           GOOGLEANALYTICS: !Ref googleanalytics
+          PICS_ORIGINAL_PATH: !Ref picsOriginalPath
+          ALBUM_SORT: !Ref albumSort
+          PICTURE_SORT: !Ref pictureSort
+          COLLECTION_SORT: !Ref collectionSort
+          GROUP_ALBUMS_INTO_COLLECTIONS: !Ref groupAlbumsIntoCollections
+          SPACES_INSTEAD_OF_TABS: !Ref spacesInsteadOfTabs
+          HOME_PAGE_CREDITS_OVERRIDE: !Ref homePageCreditsOverride
+          HIDE_HOME_PAGE_CREDITS: !Ref hideHomePageCredits
       Role: !GetAtt SiteBuilderLambdaRole.Arn
       Timeout: 900
       MemorySize: 3008
@@ -306,8 +347,8 @@ Resources:
         BlockPublicPolicy     : true
         IgnorePublicAcls      : true
         RestrictPublicBuckets : true
-      BucketEncryption: 
-        ServerSideEncryptionConfiguration: 
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
           - ServerSideEncryptionByDefault:
               SSEAlgorithm: AES256
       LifecycleConfiguration:
@@ -361,8 +402,8 @@ Resources:
         BlockPublicPolicy     : true
         IgnorePublicAcls      : true
         RestrictPublicBuckets : true
-      BucketEncryption: 
-        ServerSideEncryptionConfiguration: 
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
           - ServerSideEncryptionByDefault:
               SSEAlgorithm: AES256
       LifecycleConfiguration:
@@ -543,6 +584,5 @@ Resources:
         # Restrictions:
         #   GeoRestriction:
         #     RestrictionType: 'whitelist'
-        #     Locations: 
+        #     Locations:
         #       - 'US'
-
diff --git a/config.example.json b/config.example.json
@@ -7,6 +7,14 @@
   "originAccessIdentity=EJG...",
   "sessionDuration=86400",
   "googleanalytics=",
+  "picsOriginalPath=",
+  "albumSort=",
+  "pictureSort=",
+  "collectionSort=",
+  "groupAlbumsIntoCollections=",
+  "spacesInsteadOfTabs=",
+  "homePageCreditsOverride=",
+  "hideHomePageCredits=",
   "ImageMagickLayer=arn:aws:lambda:us-east-1:........:layer:image-magick:...",
   "kmsKeyId=00000000-0000-0000-0000-000000000000",
   "cloudFrontKeypairId=APK...",

diff --git a/generate_random_albums b/generate_random_albums
@@ -29,6 +29,6 @@ for (( albumIndex = 0; albumIndex < numberOfAlbums; albumIndex++ )); do
       curl -sL https://source.unsplash.com/random -o "$pic"
     done
     prevMD5+=("$(md5sum "$pic" | awk '{print $1}')")
-    aws s3 cp "$pic" "s3://$sourceBucket/pics/original/Album $albumNumber/0$picIndex.jpg" >> /dev/null 2>&1
+    aws s3 cp "$pic" "s3://$sourceBucket/pics/original/Album $albumNumber/0$picIndex.jpg" --sse >> /dev/null 2>&1
   done
 done
diff --git a/generate_random_collections b/generate_random_collections
@@ -0,0 +1,38 @@
+#!/bin/bash -e
+
+if [ -z "$1" ]; then
+  echo "Usage:"
+  echo "  generate_random_collections <number of collections>"
+  echo "Removes all bucket contents, downloads 6 pictures for each album from unsplash.com, creates 6 albums for each collection, giving each collection and each album a random name, uploads them to the original bucket."
+  exit 1
+else
+  webBucket="$(sed -n 's/  "webBucket=\(.*\)"\,/\1/p' dist/config.json)"
+  sourceBucket="$(sed -n 's/  "sourceBucket=\(.*\)"\,/\1/p' dist/config.json)"
+  resizedBucket="$(sed -n 's/  "resizedBucket=\(.*\)"\,/\1/p' dist/config.json)"
+  numberOfCollections=$1
+fi
+
+echo "Removing all files in web, source & resized buckets"
+for bucket in "$webBucket" "$sourceBucket" "$resizedBucket"; do
+  aws s3 rm --recursive "s3://$bucket" >> /dev/null 2>&1 || true
+done
+
+prevMD5=()
+for (( collectionIndex = 0; collectionIndex < numberOfCollections; collectionIndex++ )); do
+  collectionNumber=$RANDOM
+  let "collectionNumber %= 1000"
+  for (( albumIndex = 0; albumIndex < 6; albumIndex++ )); do
+    albumNumber=$RANDOM
+    let "albumNumber %= 1000"
+    for (( picIndex = 0; picIndex < 6; picIndex++ )); do
+      echo "Downloading pic $picIndex for collection $collectionIndex -> album $albumIndex"
+      pic="$(mktemp).jpg"
+      curl -sL https://source.unsplash.com/random -o "$pic"
+      while [[ " ${prevMD5[@]} " =~ " $(md5sum "$pic" | awk '{print $1}') " ]]; do
+        curl -sL https://source.unsplash.com/random -o "$pic"
+      done
+      prevMD5+=("$(md5sum "$pic" | awk '{print $1}')")
+      aws s3 cp "$pic" "s3://$sourceBucket/pics/original/Collection $collectionNumber/Album $albumNumber/0$picIndex.jpg" --sse >> /dev/null 2>&1
+    done
+  done
+done
diff --git a/resize/index.js b/resize/index.js
@@ -4,6 +4,18 @@ var AWS = require("aws-sdk");
 var im = require("gm").subClass({imageMagick: true});
 var s3 = new AWS.S3({signatureVersion: 'v4'});
 
+
+const DEFAULT_PICS_ORIGINAL_PATH = 'pics/original/';
+
+
+function getPicsOriginalPath() {
+  if (process.env.PICS_ORIGINAL_PATH) {
+    return process.env.PICS_ORIGINAL_PATH;
+  }
+
+  return DEFAULT_PICS_ORIGINAL_PATH;
+}
+
 function getImageType(objectContentType) {
   if (objectContentType === "image/jpeg") {
     return "jpeg";
@@ -64,7 +76,10 @@ exports.handler = function(event, context) {
           } else {
             s3.putObject({
               "Bucket": process.env.RESIZED_BUCKET,
-              "Key": "pics/resized/" + config + "/" + image.originalKey.replace("pics/original/", ""),
+              "Key": (
+                "pics/resized/" + config + "/" +
+                image.originalKey.replace(getPicsOriginalPath(), "")
+              ),
               "Body": buffer,
               "ServerSideEncryption": "AES256",
               "ContentType": image.contentType

diff --git a/site-builder/Dockerfile b/site-builder/Dockerfile
@@ -10,10 +10,12 @@ RUN npm install --production
 
 # add source code
 COPY index.js /build/index.js
+COPY lib /build/lib
 
 # add HTML templates
 COPY homepage /build/homepage
 COPY album /build/album
+COPY shared /build/shared
 
 # zip entire context and stream output
 RUN zip -r /build/dist.zip . > /dev/null