From f53d7f5c9d9d53b1716102f27e2a0ad1ed4e6ddd Mon Sep 17 00:00:00 2001 From: Jeremy Epstein Date: Wed, 11 Nov 2020 15:45:57 +1100 Subject: [PATCH] Support collections of albums --- README.md | 86 +- app.yml | 56 +- config.example.json | 8 + generate_random_albums | 2 +- generate_random_collections | 38 + resize/index.js | 17 +- site-builder/Dockerfile | 2 + site-builder/album/index.html | 84 +- site-builder/album/snippets/picture.html | 14 + site-builder/homepage/index.html | 57 +- site-builder/homepage/snippets/album.html | 6 + site-builder/homepage/snippets/backto.html | 7 + site-builder/index.js | 420 +----- site-builder/lib/album.js | 151 ++ site-builder/lib/albumPage.js | 229 +++ site-builder/lib/cloudfrontUtils.js | 40 + site-builder/lib/collectionPage.js | 238 +++ site-builder/lib/delayUtils.js | 12 + site-builder/lib/fileUtils.js | 41 + site-builder/lib/homePage.js | 162 ++ site-builder/lib/metadataUtils.js | 30 + site-builder/lib/miscUtils.js | 29 + site-builder/lib/pathUtils.js | 15 + site-builder/lib/picture.js | 18 + site-builder/lib/sorters.js | 13 + site-builder/lib/uploadContents.js | 60 + site-builder/shared/snippets/ga.html | 9 + test/site-builder/album.spec.js | 119 ++ test/site-builder/albumPage.spec.js | 242 +++ .../cloudfrontUtils.spec.js} | 11 +- test/site-builder/fileUtils.spec.js | 85 ++ test/site-builder/homePage.spec.js | 530 +++++++ test/site-builder/index.spec.js | 352 +++++ test/site-builder/metadataUtils.spec.js | 105 ++ test/site-builder/miscUtils.spec.js | 113 ++ test/site-builder/pathUtils.spec.js | 47 + test/site-builder/sorters.spec.js | 61 + test/site-builder/uploadContents.spec.js | 1312 +++++++++++++++++ test/sitebuilder/folderName.spec.js | 49 - test/sitebuilder/getAlbumMetadata.spec.js | 93 -- test/sitebuilder/getAlbums.spec.js | 77 - test/sitebuilder/isEmpty.spec.js | 46 - test/sitebuilder/listAllContents.spec.js | 338 ----- test/sitebuilder/stripPrefix.spec.js | 58 - test/sitebuilder/uploadAlbumSite.spec.js | 267 ---- test/sitebuilder/uploadHomepageSite.spec.js | 266 ---- test/sitebuilder/walk.spec.js | 87 -- 47 files changed, 4292 insertions(+), 1810 deletions(-) create mode 100755 generate_random_collections create mode 100644 site-builder/album/snippets/picture.html create mode 100644 site-builder/homepage/snippets/album.html create mode 100644 site-builder/homepage/snippets/backto.html create mode 100644 site-builder/lib/album.js create mode 100644 site-builder/lib/albumPage.js create mode 100644 site-builder/lib/cloudfrontUtils.js create mode 100644 site-builder/lib/collectionPage.js create mode 100644 site-builder/lib/delayUtils.js create mode 100644 site-builder/lib/fileUtils.js create mode 100644 site-builder/lib/homePage.js create mode 100644 site-builder/lib/metadataUtils.js create mode 100644 site-builder/lib/miscUtils.js create mode 100644 site-builder/lib/pathUtils.js create mode 100644 site-builder/lib/picture.js create mode 100644 site-builder/lib/sorters.js create mode 100644 site-builder/lib/uploadContents.js create mode 100644 site-builder/shared/snippets/ga.html create mode 100644 test/site-builder/album.spec.js create mode 100644 test/site-builder/albumPage.spec.js rename test/{sitebuilder/invalidateCloudFront.spec.js => site-builder/cloudfrontUtils.spec.js} (85%) create mode 100644 test/site-builder/fileUtils.spec.js create mode 100644 test/site-builder/homePage.spec.js create mode 100644 test/site-builder/index.spec.js create mode 100644 test/site-builder/metadataUtils.spec.js create mode 100644 test/site-builder/miscUtils.spec.js create mode 100644 test/site-builder/pathUtils.spec.js create mode 100644 test/site-builder/sorters.spec.js create mode 100644 test/site-builder/uploadContents.spec.js delete mode 100644 test/sitebuilder/folderName.spec.js delete mode 100644 test/sitebuilder/getAlbumMetadata.spec.js delete mode 100644 test/sitebuilder/getAlbums.spec.js delete mode 100644 test/sitebuilder/isEmpty.spec.js delete mode 100644 test/sitebuilder/listAllContents.spec.js delete mode 100644 test/sitebuilder/stripPrefix.spec.js delete mode 100644 test/sitebuilder/uploadAlbumSite.spec.js delete mode 100644 test/sitebuilder/uploadHomepageSite.spec.js delete mode 100644 test/sitebuilder/walk.spec.js diff --git a/README.md b/README.md index 7c9a686..be38cd1 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ > An AWS CloudFormation stack to run a serverless password-protected photo gallery -**Demo:** +**Demo:** **Credentials:** "username" / "password" ![](assets/awspics.gif) @@ -93,7 +93,7 @@ A video walkthrough [is available on YouTube](https://youtu.be/010AGcY4uoE). ``` Click Deploy (currently the orange button on the upper left). - Then click on your Lambda - Layers and you will see a version ARN that looks like: + Then click on your Lambda - Layers and you will see a version ARN that looks like: ``` Name Version Version ARN image-magick 1 arn:aws:lambda:us-east-1:000000000000:layer:image-magick:1 @@ -132,6 +132,44 @@ It should contain the following info - minus the comments: // note that the cookies are session cookies, and will get deleted when the // browser is closed anyway "sessionDuration=86400", + // Optional tracking ID for Google Analytics, if specified then a GA JS + // snippet will be outputted in the site's HTML, or leave blank for no GA + "googleanalytics=", + // Optionally override the path prefix for where original albums and their + // pictures live, or leave blank to have this default to "pics/original/" + "picsOriginalPath=", + // Optionally sort albums by name when building the homepage (if + // groupAlbumsIntoCollections is disabled), or when building collection pages + // (if groupAlbumsIntoCollections is enabled), specify either "asc" or "desc", + // or leave blank to output albums in the order that they're returned from + // the S3 list objects call + "albumSort=", + // Optionally sort pictures in all albums by name when building album pages, + // specify either "asc" or "desc", or leave blank to output pictures in the + // reverse order that they're returned from the S3 list objects call + "pictureSort=", + // Optionally sort collections by name when building the homepage (if + // groupAlbumsIntoCollections is enabled), specify either "asc" or "desc", or + // leave blank to output collections in the order that they're returned from + // the S3 list objects call + "collectionSort=", + // Optionally specify "true" to indicate that the pictures have two grouping + // levels, collections (first-level folders in the bucket) and albums + // (second-level folders in the bucket), or leave blank to indicate that + // the pictures just have one grouping level, albums + "groupAlbumsIntoCollections=", + // Indent homepage and album HTML output with spaces, specify "true" to + // enable, or leave blank to instead indent HTML output with tabs + "spacesInsteadOfTabs=", + // Optionally show the specified custom HTML instead of a design credits link + // to HTML5 UP on the home page, recommended that this be a link in the form: + // Site 123 + // Note: a design credits link to HTML5 UP will still show on album pages + "homePageCreditsOverride=", + // Optionally hide the design credits link to HTML5 UP on the home page, + // specify "true" to enable, or leave blank to show the link + // Note: a design credits link to HTML5 UP will still show on album pages + "hideHomePageCredits=", // KMS key ID created in step 2 "kmsKeyId=00000000-0000-0000-0000-000000000000", // CloudFront key pair ID from step 3 @@ -154,7 +192,7 @@ It should contain the following info - minus the comments: // encrypted contents of the file from step 6 "encryptedHtpasswd=AQICAH...", - + // ------------------ // SSL Certificate ARN // - provide this if you want to use an existing ACM Certificate. @@ -185,14 +223,14 @@ You will want to update the frequency of the Cloudwatch Events Rule from its def in the app.yml file or after the fact in the AWS Management console. ### Note on ImageMagick Layer for Lambda -When Amazon deprecated Node.js 8.10, they removed ImageMagick from the Amazon Linux 2 AMIs that are required to run Node.js 10.x. Again, ImageMagick is no longer bundled with the Node.js 10.x runtime. This fix may also help with running on Node.js 12.x in the future. This provides a Lambda Layer (essentially a library) for your Lambda function that makes the existing code work with Node.js 10.x. +When Amazon deprecated Node.js 8.10, they removed ImageMagick from the Amazon Linux 2 AMIs that are required to run Node.js 10.x. Again, ImageMagick is no longer bundled with the Node.js 10.x runtime. This fix may also help with running on Node.js 12.x in the future. This provides a Lambda Layer (essentially a library) for your Lambda function that makes the existing code work with Node.js 10.x. ##### Note on SSL Cert AWS Certificate Manager now supports SSL cert verification via DNS validation. It is recommended that you manually request the certificate for your hosted zone and -chose DNS validation method for much faster validation. Then use the resulting ARN -in your config. You can also leave this config key empty to create the certificate as +chose DNS validation method for much faster validation. Then use the resulting ARN +in your config. You can also leave this config key empty to create the certificate as normal. Once the initial deployment is done, you'll need to point your domain's DNS @@ -252,25 +290,25 @@ rely on the SSL certificate being created in CloudFormation. Create it manually reference. GeoRestriction is commented out in the CloudFront configuration in the app.yaml. If you are sharing -with friends and family in a specific geographic area, this is a slight improvement to security and -cost reduction. The US is provided as an example, but additional countries can be added to a +with friends and family in a specific geographic area, this is a slight improvement to security and +cost reduction. The US is provided as an example, but additional countries can be added to a (whitelist/blacklist) based on their two letter ISO 3166-1 alpha-2 country code. -S3 Server Side AES256 encryption is enabled for the source and resized photo buckets and encrypts files -using the AWS S3 Master key. Each bucket is configured to force encryption of any file it receives -(you will need to check the upload box or specify it in the CLI when uploading photo files to the buckets) -and you will get access denied messages if you don't. The Resize function re-encrypts the resized photos -with AES256 SSE before uploading them into the resized bucket. Cloudfront with an OAI is able to access -files using the S3 Master Key without any issue. One cannot at this time use a KMS key for encrypting +S3 Server Side AES256 encryption is enabled for the source and resized photo buckets and encrypts files +using the AWS S3 Master key. Each bucket is configured to force encryption of any file it receives +(you will need to check the upload box or specify it in the CLI when uploading photo files to the buckets) +and you will get access denied messages if you don't. The Resize function re-encrypts the resized photos +with AES256 SSE before uploading them into the resized bucket. Cloudfront with an OAI is able to access +files using the S3 Master Key without any issue. One cannot at this time use a KMS key for encrypting bucket data to be accessed via Cloudfront without more complexity. -The EventInvoke config is included for SiteBuilder to prevent it from queueing up invocations and causing -multiple cloudfront invalidations at the same time. If you need to run sitebuilder more frequently, adjust +The EventInvoke config is included for SiteBuilder to prevent it from queueing up invocations and causing +multiple cloudfront invalidations at the same time. If you need to run sitebuilder more frequently, adjust the rate of events by editing the CloudWatch Events rule in the Management console or the app.yml file. -Also, you can reduce compute costs and lock down the application several ways: 1) by manually throttling -the Resize Function and the SiteBuilder Function in Lambda in the Management console or 2) disabling the -CloudWatch Events rule that runs SiteBuilder or 3) manually disabling the trigger for a Lambda function +Also, you can reduce compute costs and lock down the application several ways: 1) by manually throttling +the Resize Function and the SiteBuilder Function in Lambda in the Management console or 2) disabling the +CloudWatch Events rule that runs SiteBuilder or 3) manually disabling the trigger for a Lambda function in the Management console. Default directory for photos is S3://BUCKET_NAME/pics/original/YOUR_ALBUMS_GO_HERE @@ -278,14 +316,14 @@ Default directory for photos is S3://BUCKET_NAME/pics/original/YOUR_ALBUMS_GO_HE ## Troubleshooting -If the project deploys and the login is entered correctly, but you are receiving access -denied messages, review your DNS settings. You only need a single DNS A record +If the project deploys and the login is entered correctly, but you are receiving access +denied messages, review your DNS settings. You only need a single DNS A record pointing to the CloudFront Alias for your domain, and time for it to propagate. If SiteBuilder is hanging or having trouble completing, you may need to adjust the rate limiting delay block in index.js. -The current S3 rate limit is 3500 writes a second, and 5500 reads/sec. If you're writing 30 files per album, -if you have more than 116 albums, you will hit the rate limit - and SiteBuilder will just hang and you will see -the files as a partial listing in the web directory. +The current S3 rate limit is 3500 writes a second, and 5500 reads/sec. If you're writing 30 files per album, +if you have more than 116 albums, you will hit the rate limit - and SiteBuilder will just hang and you will see +the files as a partial listing in the web directory. ## Credits diff --git a/app.yml b/app.yml index 0d15128..0884cfe 100644 --- a/app.yml +++ b/app.yml @@ -48,8 +48,40 @@ Parameters: googleanalytics: Description: Google tracking id (gtag) Type: String + picsOriginalPath: + Description: Path prefix to original albums and pictures + Type: String + Default: '' + albumSort: + Description: Album sort order (asc or desc, or blank) + Type: String + Default: '' + pictureSort: + Description: Picture sort order (asc or desc, or blank) + Type: String + Default: '' + collectionSort: + Description: Collection sort order (asc or desc, or blank) + Type: String + Default: '' + groupAlbumsIntoCollections: + Description: Group first by collections then by albums (true or blank) + Type: String + Default: '' + spacesInsteadOfTabs: + Description: Indent HTML output with spaces instead of tabs (true or blank) + Type: String + Default: '' + homePageCreditsOverride: + Description: Show this HTML instead of the default credits link on the home page + Type: String + Default: '' + hideHomePageCredits: + Description: His te default credits link on the home page (true or blank) + Type: String + Default: '' ImageMagickLayer: - Description: layer for nodejs10.x and nodejs12.x for imagemagick here https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:145266761615:applications~image-magick-lambda-layer + Description: layer for nodejs10.x and nodejs12.x for imagemagick here https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:145266761615:applications~image-magick-lambda-layer Type: String LambdaRate: Description: The rate (frequency) that determines when CloudWatch Events runs the rule that triggers the SiteBuilderFunction. @@ -135,6 +167,7 @@ Resources: Environment: Variables: RESIZED_BUCKET: !Ref resizedBucket + PICS_ORIGINAL_PATH: !Ref picsOriginalPath Timeout: 30 MemorySize: 1024 @@ -185,7 +218,7 @@ Resources: Targets: - Arn: !Sub ${SiteBuilderFunction.Arn} Id: LambdaSchedule - # + # # Permission to invoke a lambda function with the CloudWatch Event # LambdaSchedulePermission: @@ -223,6 +256,14 @@ Resources: SITE_BUCKET: !Ref webBucket WEBSITE: !Ref website GOOGLEANALYTICS: !Ref googleanalytics + PICS_ORIGINAL_PATH: !Ref picsOriginalPath + ALBUM_SORT: !Ref albumSort + PICTURE_SORT: !Ref pictureSort + COLLECTION_SORT: !Ref collectionSort + GROUP_ALBUMS_INTO_COLLECTIONS: !Ref groupAlbumsIntoCollections + SPACES_INSTEAD_OF_TABS: !Ref spacesInsteadOfTabs + HOME_PAGE_CREDITS_OVERRIDE: !Ref homePageCreditsOverride + HIDE_HOME_PAGE_CREDITS: !Ref hideHomePageCredits Role: !GetAtt SiteBuilderLambdaRole.Arn Timeout: 900 MemorySize: 3008 @@ -306,8 +347,8 @@ Resources: BlockPublicPolicy : true IgnorePublicAcls : true RestrictPublicBuckets : true - BucketEncryption: - ServerSideEncryptionConfiguration: + BucketEncryption: + ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LifecycleConfiguration: @@ -361,8 +402,8 @@ Resources: BlockPublicPolicy : true IgnorePublicAcls : true RestrictPublicBuckets : true - BucketEncryption: - ServerSideEncryptionConfiguration: + BucketEncryption: + ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LifecycleConfiguration: @@ -543,6 +584,5 @@ Resources: # Restrictions: # GeoRestriction: # RestrictionType: 'whitelist' - # Locations: + # Locations: # - 'US' - \ No newline at end of file diff --git a/config.example.json b/config.example.json index 444ab86..81c40a2 100644 --- a/config.example.json +++ b/config.example.json @@ -7,6 +7,14 @@ "originAccessIdentity=EJG...", "sessionDuration=86400", "googleanalytics=", + "picsOriginalPath=", + "albumSort=", + "pictureSort=", + "collectionSort=", + "groupAlbumsIntoCollections=", + "spacesInsteadOfTabs=", + "homePageCreditsOverride=", + "hideHomePageCredits=", "ImageMagickLayer=arn:aws:lambda:us-east-1:........:layer:image-magick:...", "kmsKeyId=00000000-0000-0000-0000-000000000000", "cloudFrontKeypairId=APK...", diff --git a/generate_random_albums b/generate_random_albums index 8a96286..c37a26d 100755 --- a/generate_random_albums +++ b/generate_random_albums @@ -29,6 +29,6 @@ for (( albumIndex = 0; albumIndex < numberOfAlbums; albumIndex++ )); do curl -sL https://source.unsplash.com/random -o "$pic" done prevMD5+=("$(md5sum "$pic" | awk '{print $1}')") - aws s3 cp "$pic" "s3://$sourceBucket/pics/original/Album $albumNumber/0$picIndex.jpg" >> /dev/null 2>&1 + aws s3 cp "$pic" "s3://$sourceBucket/pics/original/Album $albumNumber/0$picIndex.jpg" --sse >> /dev/null 2>&1 done done diff --git a/generate_random_collections b/generate_random_collections new file mode 100755 index 0000000..99444f6 --- /dev/null +++ b/generate_random_collections @@ -0,0 +1,38 @@ +#!/bin/bash -e + +if [ -z "$1" ]; then + echo "Usage:" + echo " generate_random_collections " + echo "Removes all bucket contents, downloads 6 pictures for each album from unsplash.com, creates 6 albums for each collection, giving each collection and each album a random name, uploads them to the original bucket." + exit 1 +else + webBucket="$(sed -n 's/ "webBucket=\(.*\)"\,/\1/p' dist/config.json)" + sourceBucket="$(sed -n 's/ "sourceBucket=\(.*\)"\,/\1/p' dist/config.json)" + resizedBucket="$(sed -n 's/ "resizedBucket=\(.*\)"\,/\1/p' dist/config.json)" + numberOfCollections=$1 +fi + +echo "Removing all files in web, source & resized buckets" +for bucket in "$webBucket" "$sourceBucket" "$resizedBucket"; do + aws s3 rm --recursive "s3://$bucket" >> /dev/null 2>&1 || true +done + +prevMD5=() +for (( collectionIndex = 0; collectionIndex < numberOfCollections; collectionIndex++ )); do + collectionNumber=$RANDOM + let "collectionNumber %= 1000" + for (( albumIndex = 0; albumIndex < 6; albumIndex++ )); do + albumNumber=$RANDOM + let "albumNumber %= 1000" + for (( picIndex = 0; picIndex < 6; picIndex++ )); do + echo "Downloading pic $picIndex for collection $collectionIndex -> album $albumIndex" + pic="$(mktemp).jpg" + curl -sL https://source.unsplash.com/random -o "$pic" + while [[ " ${prevMD5[@]} " =~ " $(md5sum "$pic" | awk '{print $1}') " ]]; do + curl -sL https://source.unsplash.com/random -o "$pic" + done + prevMD5+=("$(md5sum "$pic" | awk '{print $1}')") + aws s3 cp "$pic" "s3://$sourceBucket/pics/original/Collection $collectionNumber/Album $albumNumber/0$picIndex.jpg" --sse >> /dev/null 2>&1 + done + done +done diff --git a/resize/index.js b/resize/index.js index 1acf6f3..510c57f 100644 --- a/resize/index.js +++ b/resize/index.js @@ -4,6 +4,18 @@ var AWS = require("aws-sdk"); var im = require("gm").subClass({imageMagick: true}); var s3 = new AWS.S3({signatureVersion: 'v4'}); + +const DEFAULT_PICS_ORIGINAL_PATH = 'pics/original/'; + + +function getPicsOriginalPath() { + if (process.env.PICS_ORIGINAL_PATH) { + return process.env.PICS_ORIGINAL_PATH; + } + + return DEFAULT_PICS_ORIGINAL_PATH; +} + function getImageType(objectContentType) { if (objectContentType === "image/jpeg") { return "jpeg"; @@ -64,7 +76,10 @@ exports.handler = function(event, context) { } else { s3.putObject({ "Bucket": process.env.RESIZED_BUCKET, - "Key": "pics/resized/" + config + "/" + image.originalKey.replace("pics/original/", ""), + "Key": ( + "pics/resized/" + config + "/" + + image.originalKey.replace(getPicsOriginalPath(), "") + ), "Body": buffer, "ServerSideEncryption": "AES256", "ContentType": image.contentType diff --git a/site-builder/Dockerfile b/site-builder/Dockerfile index 466ecd0..6be8899 100644 --- a/site-builder/Dockerfile +++ b/site-builder/Dockerfile @@ -10,10 +10,12 @@ RUN npm install --production # add source code COPY index.js /build/index.js +COPY lib /build/lib # add HTML templates COPY homepage /build/homepage COPY album /build/album +COPY shared /build/shared # zip entire context and stream output RUN zip -r /build/dist.zip . > /dev/null diff --git a/site-builder/album/index.html b/site-builder/album/index.html index efe5f2c..09b45cc 100644 --- a/site-builder/album/index.html +++ b/site-builder/album/index.html @@ -1,42 +1,42 @@ - - - - {googletracking} - {title} - - - - - - - - - -
- -
-{pictures} -
- -
- - - - - - - - + + + + {googletracking} + {title} + + + + + + + + + +
+ +
+{pictures} +
+ +
+ + + + + + + + diff --git a/site-builder/album/snippets/picture.html b/site-builder/album/snippets/picture.html new file mode 100644 index 0000000..cd365cd --- /dev/null +++ b/site-builder/album/snippets/picture.html @@ -0,0 +1,14 @@ + diff --git a/site-builder/homepage/index.html b/site-builder/homepage/index.html index 08f6a7d..427e776 100644 --- a/site-builder/homepage/index.html +++ b/site-builder/homepage/index.html @@ -1,28 +1,29 @@ - - - - {googletracking} - {title} - - - - - - - - -
- -
-{pictures} -
-
- - - - - - - + + + +{googletracking} + {title} + + + + + + + + +
+ +
+{pictures} +
+
+ + + + + + + diff --git a/site-builder/homepage/snippets/album.html b/site-builder/homepage/snippets/album.html new file mode 100644 index 0000000..8f76b31 --- /dev/null +++ b/site-builder/homepage/snippets/album.html @@ -0,0 +1,6 @@ + diff --git a/site-builder/homepage/snippets/backto.html b/site-builder/homepage/snippets/backto.html new file mode 100644 index 0000000..362f696 --- /dev/null +++ b/site-builder/homepage/snippets/backto.html @@ -0,0 +1,7 @@ + diff --git a/site-builder/index.js b/site-builder/index.js index f781a46..006963d 100644 --- a/site-builder/index.js +++ b/site-builder/index.js @@ -1,424 +1,8 @@ -const AWS = require("aws-sdk"); -const s3 = new AWS.S3({signatureVersion: 'v4'}); -const cloudfront = new AWS.CloudFront(); +const uploadContents = require('./lib/uploadContents'); -const async = require('async'); -const fs = require('fs'); -const mime = require('mime'); -const path = require('path'); -const yaml = require('js-yaml'); - -function walk(dir, done) { - let results = []; - - fs.readdir(dir, function(err, list) { - /* istanbul ignore next */ - if (err) { - return done(err); - } - - let pending = list.length; - if (!pending) { - return done(null, results); - } - - list.forEach(function(file) { - file = path.resolve(dir, file); - fs.stat(file, function(err, stat) { - if (stat && stat.isDirectory()) { - walk(file, function(err, res) { - results = results.concat(res); - - if (!--pending) { - done(null, results); - } - }); - } - else { - results.push(file); - - if (!--pending) { - done(null, results); - } - } - }); - }); - }); -}; - -function stripPrefix(object) { - return object.Key.replace('pics/original/', ''); -} - -function folderName(path) { - return path.split('/')[0]; -} - -// Test if object is empty or not -function isEmpty(obj) { - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - return false; - } - } - - return true; -} - -// Google Analytics gtag code -const ga = "\n" + - "\t\t\n" + - "\t\t"; - -function getAlbums(data) { - const objects = data - .sort(function(a,b) { - return b.LastModified - a.LastModified; - }) - .map(stripPrefix); - - const albums = objects - .map(folderName) - // Deduplicate albums - .filter(function(item, pos, self) { - return self.indexOf(item) == pos; - }); - - const pictures = albums.map(function(album) { - return objects.filter(function(object) { - return ( - object.startsWith(album + "/") && - ( - object.toLowerCase().endsWith('.jpg') || - object.toLowerCase().endsWith('.png') - ) - ); - }); - }); - - return {albums: albums, pictures: pictures}; -} - -function uploadHomepageSite(albums, pictures, metadata) { - const dir = 'homepage'; - walk(dir, function(err, files) { - /* istanbul ignore next */ - if (err) { - throw err; - } - - async.map(files, function(f, cb) { - let body = fs.readFileSync(f); - - if (path.basename(f) == 'error.html') { - // Test if "googleanalytics" is set or not - if (!isEmpty(process.env.GOOGLEANALYTICS)) { - body = body - .toString() - .replace(/\{website\}/g, process.env.WEBSITE) - .replace(/\{googletracking\}/g, ga) - .replace(/\{gtag\}/g, process.env.GOOGLEANALYTICS); - } else { - body = body - .toString() - .replace(/\{website\}/g, process.env.WEBSITE) - .replace('{googletracking}', ''); - } - } else if (path.basename(f) == 'index.html') { - let picturesHTML = ''; - - for (let i = 0; i < albums.length; i++) { - let albumTitle = albums[i]; - - if (metadata[i] && metadata[i].title) { - albumTitle = metadata[i].title; - } - - picturesHTML += ( - "\t\t\t\t\t\t
\n" + - "\t\t\t\t\t\t\t\"\"\n" + - "\t\t\t\t\t\t\t

" + albumTitle + "

\n" + - "\t\t\t\t\t\t
" - ); - } - - // Test if "googleanalytics" is set or not - if (!isEmpty(process.env.GOOGLEANALYTICS)) { - body = body - .toString() - .replace(/\{title\}/g, process.env.WEBSITE_TITLE) - .replace(/\{pictures\}/g, picturesHTML) - .replace(/\{googletracking\}/g, ga) - .replace(/\{gtag\}/g, process.env.GOOGLEANALYTICS); - } else { - body = body - .toString() - .replace(/\{title\}/g, process.env.WEBSITE_TITLE) - .replace(/\{pictures\}/g, picturesHTML) - .replace('{googletracking}', ''); - } - } - - const options = { - Bucket: process.env.SITE_BUCKET, - Key: path.relative(dir, f), - Body: body, - ContentType: mime.getType(path.extname(f)) - }; - - s3.putObject(options, cb); - }, - /* istanbul ignore next */ - function(err, results) { - if (err) { - console.log(err, err.stack); - } - }); - }); -} - -/* istanbul ignore next */ -function delayBlock() { - // This is a computationally expensive way to do a delay block - - // this delay is ~100ms using processor allocated to 3008mb memory usage on a - // Lambda instance. - // Surely this could be done in a more node.js way, but if you're reading this, - // Lambda computational power is likely cheaper than your time. - let a = 0; - for (let j = 5*10e6; j >= 0; j--) { - a++; - } -} - -function uploadAlbumSite(title, pictures, metadata) { - console.log("Writing ALBUM " + title); - - // This delay block is inserted to prevent S3 from being flooded and killing - // the sitebuild. - // This happens when you have more than 115 albums, writing about 30 files per - // album saturates the 3500 writes/sec current rate limit for S3, and this - // function doesn't handle those rejections well & hangs without some delay - // block to force a rate limit. - delayBlock(); - - const dir = 'album'; - walk(dir, function(err, files) { - /* istanbul ignore next */ - if (err) { - throw err; - } - - async.map(files, function(f, cb) { - let body = fs.readFileSync(f); - - if (path.basename(f) == 'index.html') { - // Defaults - let renderedTitle = title, - comment1 = '', - comment2 = ''; - - // Metadata - if (metadata) { - if (metadata.title) { - renderedTitle = metadata.title; - } - if (metadata.comment1) { - comment1 = metadata.comment1; - } - if (metadata.comment2) { - comment2 = metadata.comment2; - } - } - - // Pictures - let picturesHTML = ''; - for (let i = pictures.length - 1; i >= 0; i--) { - picturesHTML += ( - "\t\t\t\t\t\t" - ); - } - - // Test if "googleanalytics" is set or not - if (!isEmpty(process.env.GOOGLEANALYTICS)) { - body = body - .toString() - .replace(/\{title\}/g, renderedTitle) - .replace(/\{comment1\}/g, comment1) - .replace(/\{comment2\}/g, comment2) - .replace(/\{pictures\}/g, picturesHTML) - .replace(/\{googletracking\}/g, ga) - .replace(/\{gtag\}/g, process.env.GOOGLEANALYTICS); - } else { - body = body - .toString() - .replace(/\{title\}/g, renderedTitle) - .replace(/\{comment1\}/g, comment1) - .replace(/\{comment2\}/g, comment2) - .replace(/\{pictures\}/g, picturesHTML) - .replace('{googletracking}', ''); - } - } - - const options = { - Bucket: process.env.SITE_BUCKET, - Key: title + "/" + path.relative(dir, f), - Body: body, - ContentType: mime.getType(path.extname(f)) - }; - - s3.putObject(options, cb); - }, - /* istanbul ignore next */ - function(err, results) { - if (err) { - console.log(err, err.stack); - } - }); - }); -} - -function invalidateCloudFront() { - cloudfront.listDistributions(function(err, data) { - /* istanbul ignore next */ - if (err) { - console.log(err, err.stack); - return; - } - - // Get distribution ID from domain name - const distributionID = data.Items.find(function (d) { - return d.DomainName == process.env.CLOUDFRONT_DISTRIBUTION_DOMAIN; - }).Id; - - // Create invalidation - cloudfront.createInvalidation({ - DistributionId: distributionID, - InvalidationBatch: { - CallerReference: 'site-builder-' + Date.now(), - Paths: { - Quantity: 1, - Items: [ - '/*' - ] - } - } - }, - /* istanbul ignore next */ - function(err, data) { - if (err) { - console.log(err, err.stack); - } - }); - }); -} - -function getAlbumMetadata(album, cb) { - s3.getObject({ - "Bucket": process.env.ORIGINAL_BUCKET, - "Key": "pics/original/" + album + "/metadata.yml" - }, function(err, data) { - if (err) { - cb(null, null); - } else { - try { - const doc = yaml.safeLoad(data.Body.toString()); - cb(null, doc); - } catch (err) { - cb(null, null); - } - } - }); -} - -function listAllContents(params) { - // List all bucket objects - s3.listObjectsV2(params, function (err, data) { - /* istanbul ignore if */ - if (err) { - console.log(err, err.stack); // an error occurred - } - else { - const allContents = [], - contents = data.Contents; - - contents.forEach(function (content) { - allContents.push(content); - }); - - if (data.IsTruncated) { - const newParams = {}; - Object.assign(newParams, params); - - const token = data.NextContinuationToken - newParams.ContinuationToken = token; - console.log( - "S3 listing was truncated. Pausing 2 sec before continuing " + - token - ); - // Rate limiting for reads - this is a tested safe value - setTimeout( - function() { - listAllContents(newParams); - }, - 2000 - ); - } - else { - // Parse albums - const albumsAndPictures = getAlbums(allContents); - console.log( - "First Album: " + - albumsAndPictures.albums[albumsAndPictures.albums.length - 1] - ); - console.log("Last Album: " + albumsAndPictures.albums[0]); - - // Get metadata for all albums - async.map( - albumsAndPictures.albums, - getAlbumMetadata, - function(err, metadata) { - // Upload homepage site - uploadHomepageSite( - albumsAndPictures.albums, albumsAndPictures.pictures, metadata - ); - - // Upload album sites - for (let i = albumsAndPictures.albums.length - 1; i >= 0; i--) { - uploadAlbumSite( - albumsAndPictures.albums[i], - albumsAndPictures.pictures[i], - metadata[i] - ); - } - - // Invalidate CloudFront - invalidateCloudFront(); - } - ); - } - } - }); -} exports.handler = function(event, context) { - listAllContents({ + uploadContents.listAndUploadAllContents({ Bucket: process.env.ORIGINAL_BUCKET }); }; diff --git a/site-builder/lib/album.js b/site-builder/lib/album.js new file mode 100644 index 0000000..e906e3f --- /dev/null +++ b/site-builder/lib/album.js @@ -0,0 +1,151 @@ +const miscUtils = require('./miscUtils'); +const pathUtils = require('./pathUtils'); +const picture = require('./picture'); +const sorters = require('./sorters'); + + +function getUniqueFirstLevelObjects(objects) { + return objects + .map(pathUtils.firstLevelFolderName) + // Make unique + .filter(function(item, pos, self) { + return self.indexOf(item) === pos; + }); +} + +function getUniqueSecondLevelObjects(objects) { + return objects + .map(pathUtils.firstAndSecondLevelFolderName) + // Make unique + .filter(function(item, pos, self) { + return self.indexOf(item) === pos; + }); +} + +function getFilteredSortedBucketData(data) { + return data + .sort(sorters.lastModifiedDescSorter) + .filter(function(object) { + return ( + object.Key.lastIndexOf(miscUtils.getPicsOriginalPath(), 0) === 0 && + !object.Key.includes('metadata.yml') + ); + }) + .map(miscUtils.stripPrefix); +} + +exports.getAlbums = function(data) { + const objects = getFilteredSortedBucketData(data); + + const albums = getUniqueFirstLevelObjects(objects); + const pictures = picture.getPictures(albums, objects); + + return {albums: albums, pictures: pictures}; +}; + +exports.getAlbumsByCollection = function(data) { + const objects = getFilteredSortedBucketData(data); + + const collections = getUniqueFirstLevelObjects(objects); + const albums = getUniqueSecondLevelObjects(objects); + + let album, coll, collIndex; + const albumsByCollection = []; + const collIndexMap = {}; + + for (let i = 0; i < albums.length; i++) { + album = albums[i]; + coll = pathUtils.firstLevelFolderName(album); + + if (collIndexMap[coll] != null) { + collIndex = collIndexMap[coll]; + } + else { + collIndex = albumsByCollection.length; + collIndexMap[coll] = collIndex; + albumsByCollection.push({collection: coll, albums: []}); + } + + albumsByCollection[collIndex].albums.push(album); + } + + const pictures = picture.getPicturesByCollection(albumsByCollection, objects); + + return {albumsByCollection: albumsByCollection, pictures: pictures}; +}; + +exports.getAlbumMarkup = function(albumName, pictures, metadata, albumMarkup) { + let albumTitle = (albumName && albumName.includes('/')) ? + pathUtils.secondLevelFolderName(albumName) : + albumName; + let albumPicture = pictures[0]; + + if (metadata) { + if (metadata.title) { + albumTitle = metadata.title; + } + + if (metadata.cover_image) { + const coverPicture = albumName + '/' + metadata.cover_image; + + if (pictures.includes(coverPicture)) { + albumPicture = coverPicture; + } + } + } + + return albumMarkup + .replace(/\{albumName\}/g, albumName) + .replace(/\{albumPicture\}/g, albumPicture) + .replace(/\{albumTitle\}/g, albumTitle); +}; + +exports.getAlbumSorter = function(title) { + // Collection page, sorting its albums + if (title) { + if ( + process.env.ALBUM_SORT && + process.env.ALBUM_SORT.toLowerCase() === 'asc' + ) { + return sorters.albumAscSorter; + } + else if ( + process.env.ALBUM_SORT && + process.env.ALBUM_SORT.toLowerCase() === 'desc' + ) { + return sorters.albumDescSorter; + } + } + // Home page, sorting its collections + else if (process.env.GROUP_ALBUMS_INTO_COLLECTIONS) { + if ( + process.env.COLLECTION_SORT && + process.env.COLLECTION_SORT.toLowerCase() === 'asc' + ) { + return sorters.albumAscSorter; + } + else if ( + process.env.COLLECTION_SORT && + process.env.COLLECTION_SORT.toLowerCase() === 'desc' + ) { + return sorters.albumDescSorter; + } + } + // Home page, sorting its albums + else { + if ( + process.env.ALBUM_SORT && + process.env.ALBUM_SORT.toLowerCase() === 'asc' + ) { + return sorters.albumAscSorter; + } + else if ( + process.env.ALBUM_SORT && + process.env.ALBUM_SORT.toLowerCase() === 'desc' + ) { + return sorters.albumDescSorter; + } + } + + return null; +}; diff --git a/site-builder/lib/albumPage.js b/site-builder/lib/albumPage.js new file mode 100644 index 0000000..ae6b8ef --- /dev/null +++ b/site-builder/lib/albumPage.js @@ -0,0 +1,229 @@ +const AWS = require("aws-sdk"); +const async = require('async'); +const fs = require('fs'); +const mime = require('mime'); +const path = require('path'); + +const album = require('./album'); +const cloudfrontUtils = require('./cloudfrontUtils'); +const delayUtils = require('./delayUtils'); +const fileUtils = require('./fileUtils'); +const homePage = require('./homePage'); +const metadataUtils = require('./metadataUtils'); +const miscUtils = require('./miscUtils'); +const pathUtils = require('./pathUtils'); + + +const s3 = new AWS.S3({signatureVersion: 'v4'}); + + +function getAlbumPageBody( + data, title, metadata, pictures, pictureMarkup, ga, parentLink, parentTitle +) { + // Defaults + let renderedTitle = title, + comment1 = '', + comment2 = ''; + + // Metadata + if (metadata) { + if (metadata.title) { + renderedTitle = metadata.title; + } + if (metadata.comment1) { + comment1 = metadata.comment1; + } + if (metadata.comment2) { + comment2 = metadata.comment2; + } + } + + // Pictures + let picturesHTML = ''; + const picturesSorted = pictures.slice(); + + if (process.env.PICTURE_SORT && + process.env.PICTURE_SORT.toLowerCase() === 'asc') { + picturesSorted.sort(); + } + else if ( + process.env.PICTURE_SORT && + process.env.PICTURE_SORT.toLowerCase() === 'desc') { + picturesSorted.sort(); + picturesSorted.reverse(); + } + else { + picturesSorted.reverse(); + } + + for (let i = 0; i < picturesSorted.length; i++) { + const pictureFileName = picturesSorted[i]; + const pictureHTML = pictureMarkup + .replace(/\{picsOriginalPath\}/g, miscUtils.getPicsOriginalPath()) + .replace(/\{pictureFileName\}/g, pictureFileName); + + picturesHTML += pictureHTML; + } + + body = data + .toString() + .replace(/\{title\}/g, renderedTitle) + .replace(/\{comment1\}/g, comment1) + .replace(/\{comment2\}/g, comment2) + .replace(/\{pictures\}/g, picturesHTML) + .replace(/\{parentLink\}/g, (parentLink ? parentLink : '')) + .replace(/\{parentTitle\}/g, (parentTitle ? parentTitle : 'albums')); + + // Test if "googleanalytics" is set or not + if (!miscUtils.isEmpty(process.env.GOOGLEANALYTICS)) { + body = body + .replace(/\{googletracking\}/g, ga) + .replace(/\{gtag\}/g, process.env.GOOGLEANALYTICS); + } else { + body = body.replace(/\{googletracking\}/g, ''); + } + + if (!process.env.SPACES_INSTEAD_OF_TABS) { + body = miscUtils.spacesToTabs(body); + } + + return body; +} + +const uploadAlbumPage = exports.uploadAlbumPage = function( + albumName, pictures, metadata, parentLink, parentTitle, isFirstAlbum +) { + const title = (albumName && albumName.includes('/')) ? + pathUtils.secondLevelFolderName(albumName) : + albumName; + console.log("Writing album " + title); + + // This delay block is inserted to prevent S3 from being flooded and killing + // the sitebuild. + // This happens when you have more than 115 albums, writing about 30 files per + // album saturates the 3500 writes/sec current rate limit for S3, and this + // function doesn't handle those rejections well & hangs without some delay + // block to force a rate limit. + delayUtils.delayBlock(); + + const dir = 'album'; + + fileUtils.walk(dir, function(err, files) { + /* istanbul ignore next */ + if (err) { + throw err; + } + + // Google Analytics gtag code + const ga = fs.readFileSync('shared/snippets/ga.html').toString(); + + const pictureMarkup = fs.readFileSync('album/snippets/picture.html').toString(); + + async.map(files, function(f, cb) { + const filePath = path.relative(dir, f); + + if ( + !filePath.includes('snippets') && + (!filePath.includes('assets/') || isFirstAlbum) + ) { + let data = fs.readFileSync(f), + body; + + if (path.basename(f) === 'index.html') { + body = getAlbumPageBody( + data, + title, + metadata, + pictures, + pictureMarkup, + ga, + parentLink, + parentTitle + ); + } + else { + body = data; + } + + let fileKey; + + if (filePath.includes('assets/')) { + fileKey = filePath.replace(/assets\//g, 'assets/album/'); + } + else { + fileKey = albumName + "/" + filePath; + } + + const options = { + Bucket: process.env.SITE_BUCKET, + Key: fileKey, + Body: body, + ContentType: mime.getType(path.extname(f)) + }; + + s3.putObject(options, cb); + } + }, + /* istanbul ignore next */ + function(err, results) { + if (err) { + console.log(err, err.stack); + } + }); + }); +}; + +function uploadAlbums(albumsAndPictures, metadata) { + // Upload home page + homePage.uploadHomePage( + albumsAndPictures.albums, + albumsAndPictures.pictures, + metadata + ); + + let isFirstAlbum = true; + + // Upload album pages + for (let i = albumsAndPictures.albums.length - 1; i >= 0; i--) { + uploadAlbumPage( + albumsAndPictures.albums[i], + albumsAndPictures.pictures[i], + metadata[i], + null, + null, + isFirstAlbum + ); + + if (isFirstAlbum) { + isFirstAlbum = false; + } + } + + // Invalidate CloudFront + cloudfrontUtils.invalidateCloudFront(); +} + +exports.uploadAllContentsByAlbum = function(allContents) { + // Parse albums + const albumsAndPictures = album.getAlbums(allContents); + console.log( + "First album: " + + albumsAndPictures.albums[albumsAndPictures.albums.length - 1] + ); + console.log("Last album: " + albumsAndPictures.albums[0]); + + // Get metadata for all albums + async.map( + albumsAndPictures.albums, + metadataUtils.getAlbumOrCollectionMetadata, + function(err, metadata) { + /* istanbul ignore next */ + if (err) { + console.log(err, err.stack); + return; + } + + uploadAlbums(albumsAndPictures, metadata); + } + ); +}; diff --git a/site-builder/lib/cloudfrontUtils.js b/site-builder/lib/cloudfrontUtils.js new file mode 100644 index 0000000..4c4da4c --- /dev/null +++ b/site-builder/lib/cloudfrontUtils.js @@ -0,0 +1,40 @@ +const AWS = require("aws-sdk"); + + +const cloudfront = new AWS.CloudFront(); + + +exports.invalidateCloudFront = function() { + cloudfront.listDistributions(function(err, data) { + /* istanbul ignore next */ + if (err) { + console.log(err, err.stack); + return; + } + + // Get distribution ID from domain name + const distributionID = data.Items.find(function (d) { + return d.DomainName === process.env.CLOUDFRONT_DISTRIBUTION_DOMAIN; + }).Id; + + // Create invalidation + cloudfront.createInvalidation({ + DistributionId: distributionID, + InvalidationBatch: { + CallerReference: 'site-builder-' + Date.now(), + Paths: { + Quantity: 1, + Items: [ + '/*' + ] + } + } + }, + /* istanbul ignore next */ + function(err, data) { + if (err) { + console.log(err, err.stack); + } + }); + }); +}; diff --git a/site-builder/lib/collectionPage.js b/site-builder/lib/collectionPage.js new file mode 100644 index 0000000..54954e7 --- /dev/null +++ b/site-builder/lib/collectionPage.js @@ -0,0 +1,238 @@ +const AWS = require("aws-sdk"); +const async = require('async'); +const fs = require('fs'); +const mime = require('mime'); +const path = require('path'); + +const album = require('./album'); +const albumPage = require('./albumPage'); +const cloudfrontUtils = require('./cloudfrontUtils'); +const fileUtils = require('./fileUtils'); +const homePage = require('./homePage'); +const metadataUtils = require('./metadataUtils'); +const pathUtils = require('./pathUtils'); + + +const s3 = new AWS.S3({signatureVersion: 'v4'}); + + +function getAlbumListFromAlbumsByCollAndPictures(albumsByCollAndPictures) { + const albums = []; + let coll; + + for (let i = 0; i < albumsByCollAndPictures.albumsByCollection.length; i++) { + coll = albumsByCollAndPictures.albumsByCollection[i]; + + for (let j = 0; j < coll.albums.length; j++) { + albums.push(coll.albums[j]); + } + } + + return albums; +} + +function unflattenAlbumMetadataForCollections( + albumMetadataFlat, albumsByCollAndPictures +) { + const albumMetadata = []; + let amd, + coll; + let albumMetadataIndex = 0; + + for (let i = 0; i < albumsByCollAndPictures.albumsByCollection.length; i++) { + amd = []; + coll = albumsByCollAndPictures.albumsByCollection[i]; + + for (let j = 0; j < coll.albums.length; j++) { + amd.push(albumMetadataFlat[albumMetadataIndex]); + albumMetadataIndex += 1; + } + + albumMetadata.push(amd); + } + + return albumMetadata; +} + +function uploadCollectionPage(collectionName, title, albums, pictures, metadata) { + const dir = 'homepage'; + console.log("Writing collection " + collectionName); + + fileUtils.walk(dir, function(err, files) { + /* istanbul ignore next */ + if (err) { + throw err; + } + + // Google Analytics gtag code + const ga = fs.readFileSync('shared/snippets/ga.html').toString(); + + const albumMarkup = fs.readFileSync('homepage/snippets/album.html').toString(); + const backTo = fs.readFileSync('homepage/snippets/backto.html').toString(); + + async.map(files, function(f, cb) { + const filePath = path.relative(dir, f); + if (filePath.includes('index.html')) { + const data = fs.readFileSync(f); + const body = homePage.getHomePageBody( + data, albums, pictures, metadata, albumMarkup, ga, backTo, title + ); + + const options = { + Bucket: process.env.SITE_BUCKET, + Key: collectionName + '/' + filePath, + Body: body, + ContentType: mime.getType(path.extname(f)) + }; + + s3.putObject(options, cb); + } + }, + /* istanbul ignore next */ + function(err, results) { + if (err) { + console.log(err, err.stack); + } + }); + }); +} + +function uploadCollection( + collection, + collAlbums, + collPictures, + metadataForColl, + metadataForAlbums, + isFirstAlbum +) { + console.log( + "First album in " + collection + ": " + + pathUtils.secondLevelFolderName(collAlbums[collAlbums.length - 1]) + ); + console.log( + "Last album in " + collection + ": " + + pathUtils.secondLevelFolderName(collAlbums[0]) + ); + + const collTitle = (metadataForColl && metadataForColl.title) ? + metadataForColl.title : + collection; + + uploadCollectionPage( + collection, collTitle, collAlbums, collPictures, metadataForAlbums + ); + + let newIsFirstAlbum = isFirstAlbum; + + // Upload album pages + for (let i = collAlbums.length - 1; i >= 0; i--) { + albumPage.uploadAlbumPage( + collAlbums[i], + collPictures[i], + metadataForAlbums[i], + collection + '/index.html', + collTitle, + newIsFirstAlbum + ); + + if (newIsFirstAlbum) { + newIsFirstAlbum = false; + } + } +} + +function uploadCollections( + collections, albumMetadataFlat, pictures, collMetadata, albumsByCollAndPictures +) { + console.log("First collection: " + collections[collections.length - 1]); + console.log("Last collection: " + collections[0]); + + const albumMetadata = unflattenAlbumMetadataForCollections( + albumMetadataFlat, albumsByCollAndPictures + ); + + // Upload home page + homePage.uploadHomePage( + collections, + pictures, + collMetadata + ); + + let isFirstAlbum = true; + + // Upload collection pages + for (let i = collections.length - 1; i >= 0; i--) { + uploadCollection( + collections[i], + albumsByCollAndPictures.albumsByCollection[i].albums, + albumsByCollAndPictures.pictures[i], + collMetadata[i], + albumMetadata[i], + isFirstAlbum + ); + + isFirstAlbum = false; + } + + // Invalidate CloudFront + cloudfrontUtils.invalidateCloudFront(); +} + +function getAndUploadCollections( + albumsByCollAndPictures, collections, pictures, collMetadata +) { + const albums = getAlbumListFromAlbumsByCollAndPictures( + albumsByCollAndPictures + ); + + // Get metadata for all albums + async.map( + albums, + metadataUtils.getAlbumOrCollectionMetadata, + function(err, albumMetadataFlat) { + /* istanbul ignore next */ + if (err) { + console.log(err, err.stack); + return; + } + + uploadCollections( + collections, albumMetadataFlat, pictures, collMetadata, albumsByCollAndPictures + ); + } + ); +} + +exports.uploadAllContentsByCollection = function(allContents) { + // Parse albums by collection + const albumsByCollAndPictures = album.getAlbumsByCollection(allContents); + + const collections = albumsByCollAndPictures.albumsByCollection.map( + function(coll) { + return coll.collection; + } + ); + + const pictures = albumsByCollAndPictures.pictures.map(function(collPictures) { + return collPictures.reduce(function(albumPictures, moreAlbumPictures) { + return albumPictures.concat(moreAlbumPictures); + }); + }); + + // Get metadata for all collections + async.map( + collections, + metadataUtils.getAlbumOrCollectionMetadata, + function(err, collMetadata) { + /* istanbul ignore next */ + if (err) { + console.log(err, err.stack); + return; + } + + getAndUploadCollections( + albumsByCollAndPictures, collections, pictures, collMetadata + ); + } + ); +}; diff --git a/site-builder/lib/delayUtils.js b/site-builder/lib/delayUtils.js new file mode 100644 index 0000000..53acc5f --- /dev/null +++ b/site-builder/lib/delayUtils.js @@ -0,0 +1,12 @@ +/* istanbul ignore next */ +exports.delayBlock = function() { + // This is a computationally expensive way to do a delay block - + // this delay is ~100ms using processor allocated to 3008mb memory usage on a + // Lambda instance. + // Surely this could be done in a more node.js way, but if you're reading this, + // Lambda computational power is likely cheaper than your time. + let a = 0; + for (let j = 5*10e6; j >= 0; j--) { + a++; + } +}; diff --git a/site-builder/lib/fileUtils.js b/site-builder/lib/fileUtils.js new file mode 100644 index 0000000..b028e0a --- /dev/null +++ b/site-builder/lib/fileUtils.js @@ -0,0 +1,41 @@ +const fs = require('fs'); +const path = require('path'); + + +const walk = exports.walk = function(dir, done) { + let results = []; + + fs.readdir(dir, function(err, list) { + /* istanbul ignore next */ + if (err) { + return done(err); + } + + let pending = list.length; + if (!pending) { + return done(null, results); + } + + list.forEach(function(file) { + file = path.resolve(dir, file); + fs.stat(file, function(err, stat) { + if (stat && stat.isDirectory()) { + walk(file, function(err, res) { + results = results.concat(res); + + if (!--pending) { + done(null, results); + } + }); + } + else { + results.push(file); + + if (!--pending) { + done(null, results); + } + } + }); + }); + }); +}; diff --git a/site-builder/lib/homePage.js b/site-builder/lib/homePage.js new file mode 100644 index 0000000..6a01f2c --- /dev/null +++ b/site-builder/lib/homePage.js @@ -0,0 +1,162 @@ +const AWS = require("aws-sdk"); +const async = require('async'); +const fs = require('fs'); +const mime = require('mime'); +const path = require('path'); + +const album = require('./album'); +const fileUtils = require('./fileUtils'); +const miscUtils = require('./miscUtils'); + + +const s3 = new AWS.S3({signatureVersion: 'v4'}); + + +function getErrorPageBody(data, ga) { + let body = data.toString().replace(/\{website\}/g, process.env.WEBSITE); + + // Test if "googleanalytics" is set or not + if (!miscUtils.isEmpty(process.env.GOOGLEANALYTICS)) { + body = body + .replace(/\{googletracking\}/g, ga) + .replace(/\{gtag\}/g, process.env.GOOGLEANALYTICS); + } else { + body = body.replace(/\{googletracking\}/g, ''); + } + + if (!process.env.SPACES_INSTEAD_OF_TABS) { + body = miscUtils.spacesToTabs(body); + } + + return body; +} + +const getHomePageBody = exports.getHomePageBody = function( + data, albums, pictures, metadata, albumMarkup, ga, backTo, title +) { + let picturesHTML = ''; + const albumsPicturesMetadata = albums.map(function(album, i) { + return {album: album, pictures: pictures[i], metadata: metadata[i]}; + }); + + const sorter = album.getAlbumSorter(title); + + if (sorter) { + albumsPicturesMetadata.sort(sorter); + } + + for (let i = 0; i < albumsPicturesMetadata.length; i++) { + picturesHTML += album.getAlbumMarkup( + albumsPicturesMetadata[i].album, + albumsPicturesMetadata[i].pictures, + albumsPicturesMetadata[i].metadata, + albumMarkup + ); + } + + const pageTitle = title ? title : process.env.WEBSITE_TITLE; + let backToMarkup; + + if (title) { + backToMarkup = backTo.replace( + /\{backLink\}/g, + 'Back to ' + process.env.WEBSITE_TITLE + '' + ); + } + else if (process.env.HOME_PAGE_CREDITS_OVERRIDE) { + backToMarkup = backTo.replace( + /\{backLink\}/g, + process.env.HOME_PAGE_CREDITS_OVERRIDE + ); + } + else if (!process.env.HIDE_HOME_PAGE_CREDITS) { + backToMarkup = backTo.replace( + /\{backLink\}/g, + 'Design: HTML5 UP' + ); + } + else { + backToMarkup = ''; + } + + let body = data + .toString() + .replace(/\{title\}/g, pageTitle) + .replace(/\{pictures\}/g, picturesHTML) + .replace(/\{backTo\}/g, backToMarkup); + + // Test if "googleanalytics" is set or not + if (!miscUtils.isEmpty(process.env.GOOGLEANALYTICS)) { + body = body + .replace(/\{googletracking\}/g, ga) + .replace(/\{gtag\}/g, process.env.GOOGLEANALYTICS); + } else { + body = body.replace(/\{googletracking\}/g, ''); + } + + if (!process.env.SPACES_INSTEAD_OF_TABS) { + body = miscUtils.spacesToTabs(body); + } + + return body; +}; + +exports.uploadHomePage = function(albums, pictures, metadata) { + const dir = 'homepage'; + + fileUtils.walk(dir, function(err, files) { + /* istanbul ignore next */ + if (err) { + throw err; + } + + // Google Analytics gtag code + const ga = fs.readFileSync('shared/snippets/ga.html').toString(); + + const albumMarkup = fs.readFileSync('homepage/snippets/album.html').toString(); + const backTo = fs.readFileSync('homepage/snippets/backto.html').toString(); + + async.map(files, function(f, cb) { + if (!f.includes('snippets')) { + let data = fs.readFileSync(f), + body; + + if (path.basename(f) === 'error.html') { + body = getErrorPageBody(data, ga); + } else if (path.basename(f) === 'index.html') { + body = getHomePageBody( + data, albums, pictures, metadata, albumMarkup, ga, backTo, null + ); + } + else { + body = data; + } + + const filePath = path.relative(dir, f); + let fileKey; + + if (filePath.includes('assets/')) { + fileKey = filePath.replace(/assets\//g, 'assets/homepage/'); + } + else { + fileKey = filePath; + } + + const options = { + Bucket: process.env.SITE_BUCKET, + Key: fileKey, + Body: body, + ContentType: mime.getType(path.extname(f)) + }; + + s3.putObject(options, cb); + } + }, + /* istanbul ignore next */ + function(err, results) { + if (err) { + console.log(err, err.stack); + } + }); + }); +}; diff --git a/site-builder/lib/metadataUtils.js b/site-builder/lib/metadataUtils.js new file mode 100644 index 0000000..44df203 --- /dev/null +++ b/site-builder/lib/metadataUtils.js @@ -0,0 +1,30 @@ +const AWS = require("aws-sdk"); +const yaml = require('js-yaml'); + +const miscUtils = require('./miscUtils'); + + +const s3 = new AWS.S3({signatureVersion: 'v4'}); + + +exports.getAlbumOrCollectionMetadata = function(albumOrCollection, cb) { + s3.getObject({ + "Bucket": process.env.ORIGINAL_BUCKET, + "Key": miscUtils.getPicsOriginalPath() + albumOrCollection + "/metadata.yml" + }, function(err, data) { + if (err) { + cb(null, null); + } else { + let doc; + + try { + doc = yaml.safeLoad(data.Body.toString()); + } catch (err) { + cb(null, null); + return; + } + + cb(null, doc); + } + }); +}; diff --git a/site-builder/lib/miscUtils.js b/site-builder/lib/miscUtils.js new file mode 100644 index 0000000..1cf32ef --- /dev/null +++ b/site-builder/lib/miscUtils.js @@ -0,0 +1,29 @@ +const DEFAULT_PICS_ORIGINAL_PATH = 'pics/original/'; + + +const getPicsOriginalPath = exports.getPicsOriginalPath = function() { + if (process.env.PICS_ORIGINAL_PATH) { + return process.env.PICS_ORIGINAL_PATH; + } + + return DEFAULT_PICS_ORIGINAL_PATH; +}; + +exports.stripPrefix = function(object) { + return object.Key.replace(getPicsOriginalPath(), ''); +}; + +// Test if object is empty or not +exports.isEmpty = function(obj) { + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + return false; + } + } + + return true; +}; + +exports.spacesToTabs = function(text) { + return text.replace(/ /g, '\t'); +}; diff --git a/site-builder/lib/pathUtils.js b/site-builder/lib/pathUtils.js new file mode 100644 index 0000000..fd2f970 --- /dev/null +++ b/site-builder/lib/pathUtils.js @@ -0,0 +1,15 @@ +const path = require('path'); + + +exports.firstLevelFolderName = function(path) { + return path.split('/')[0]; +}; + +exports.secondLevelFolderName = function(path) { + return path.split('/')[1]; +}; + +exports.firstAndSecondLevelFolderName = function(path) { + const splits = path.split('/'); + return splits[0] + '/' + splits[1]; +}; diff --git a/site-builder/lib/picture.js b/site-builder/lib/picture.js new file mode 100644 index 0000000..bd33489 --- /dev/null +++ b/site-builder/lib/picture.js @@ -0,0 +1,18 @@ +const getPictures = exports.getPictures = function(albums, objects) { + return albums.map(function(album) { + return objects.filter(function(object) { + const objectLower = object.toLowerCase(); + + return ( + object.startsWith(album + "/") && + (objectLower.endsWith('.jpg') || objectLower.endsWith('.png')) + ); + }); + }); +}; + +exports.getPicturesByCollection = function(albumsByCollection, objects) { + return albumsByCollection.map(function(collAndAlbums) { + return getPictures(collAndAlbums.albums, objects); + }); +}; diff --git a/site-builder/lib/sorters.js b/site-builder/lib/sorters.js new file mode 100644 index 0000000..0ed8cab --- /dev/null +++ b/site-builder/lib/sorters.js @@ -0,0 +1,13 @@ +exports.lastModifiedDescSorter = function(a, b) { + return (a.LastModified < b.LastModified) ? + 1 : + ((b.LastModified < a.LastModified) ? -1 : 0); +}; + +exports.albumAscSorter = function(a, b) { + return (a.album > b.album) ? 1 : ((b.album > a.album) ? -1 : 0); +}; + +exports.albumDescSorter = function(a, b) { + return (a.album < b.album) ? 1 : ((b.album < a.album) ? -1 : 0); +}; diff --git a/site-builder/lib/uploadContents.js b/site-builder/lib/uploadContents.js new file mode 100644 index 0000000..5d4edaa --- /dev/null +++ b/site-builder/lib/uploadContents.js @@ -0,0 +1,60 @@ +const AWS = require("aws-sdk"); + +const albumPage = require('./albumPage'); +const collectionPage = require('./collectionPage'); + + +const s3 = new AWS.S3({signatureVersion: 'v4'}); + + +const uploadAllContents = exports.uploadAllContents = function(allContents) { + if (process.env.GROUP_ALBUMS_INTO_COLLECTIONS) { + collectionPage.uploadAllContentsByCollection(allContents); + } + else { + albumPage.uploadAllContentsByAlbum(allContents); + } +}; + +function handleTruncatedListObjectsResponse(params, token) { + const newParams = {}; + Object.assign(newParams, params); + + newParams.ContinuationToken = token; + console.log( + "S3 listing was truncated. Pausing 2 seconds before continuing " + + token + ); + // Rate limiting for reads - this is a tested safe value + setTimeout( + function() { + listAndUploadAllContents(newParams); + }, + 2000 + ); +} + +const listAndUploadAllContents = exports.listAndUploadAllContents = function(params) { + // List all bucket objects + s3.listObjectsV2(params, function (err, data) { + /* istanbul ignore if */ + if (err) { + console.log(err, err.stack); // an error occurred + } + else { + const allContents = [], + contents = data.Contents; + + contents.forEach(function (content) { + allContents.push(content); + }); + + if (data.IsTruncated) { + handleTruncatedListObjectsResponse(params, data.NextContinuationToken); + } + else { + uploadAllContents(allContents); + } + } + }); +}; diff --git a/site-builder/shared/snippets/ga.html b/site-builder/shared/snippets/ga.html new file mode 100644 index 0000000..6b37930 --- /dev/null +++ b/site-builder/shared/snippets/ga.html @@ -0,0 +1,9 @@ + + + diff --git a/test/site-builder/album.spec.js b/test/site-builder/album.spec.js new file mode 100644 index 0000000..98878e2 --- /dev/null +++ b/test/site-builder/album.spec.js @@ -0,0 +1,119 @@ +const chai = require('chai'); +const mock = require('mock-require'); + +const album = require('../../site-builder/lib/album'); + +const expect = chai.expect; + +describe('album', function() { + describe('getAlbums', function() { + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + }); + + it('returns albums and pictures from input data', function() { + const data = [ + {Key: 'pics/original/california2020/disneyland.jpg', LastModified: 20200106090000}, + {Key: 'pics/original/california2020/napa.jpg', LastModified: 20200101090000}, + {Key: 'pics/original/bluemtns2020/threesisters.png'}, + {Key: 'pics/original/bluemtns2020/blackheath.jpg', LastModified: 20200102090000}, + {Key: 'pics/original/bluemtns2020/jenolancaves.jpg', LastModified: null}, + {Key: 'pics/original/funnyalbum/funnyfile.txt', LastModified: 20200109090000} + ]; + const expectedAlbumsAndPictures = { + albums: [ + 'california2020', + 'bluemtns2020', + 'funnyalbum' + ], + pictures: [ + [ + 'california2020/disneyland.jpg', + 'california2020/napa.jpg' + ], + [ + 'bluemtns2020/threesisters.png', + 'bluemtns2020/blackheath.jpg', + 'bluemtns2020/jenolancaves.jpg' + ], + [] + ] + }; + expect(album.getAlbums(data)).to.eql(expectedAlbumsAndPictures); + }); + + it('returns empty lists from empty input', function() { + expect(album.getAlbums([])).to.eql({albums: [], pictures: []}); + }); + + it('raises error if passed null', function() { + expect(function() { album.getAlbums(null); }).to.throw(TypeError); + }); + + it('raises error if array element is null', function() { + expect(function() { album.getAlbums([null]); }).to.throw(TypeError); + }); + + it('raises error if array element is empty object', function() { + expect(function() { album.getAlbums([{}]); }).to.throw(TypeError); + }); + + it('raises error if array element missing Key', function() { + expect(function() { album.getAlbums([{LastModified: 'yesterday'}]); }) + .to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); + + describe('getAlbumsByCollection', function() { + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + }); + + it('returns albums and pictures by collection from input data', function() { + const data = [ + {Key: 'pics/original/cruisy2019/borisbirthday/cake.jpg'}, + {Key: 'pics/original/whatayear2020/california2020/disneyland.jpg'}, + {Key: 'pics/original/whatayear2020/bluemtns2020/threesisters.png'}, + {Key: 'pics/original/whatayear2020/bluemtns2020/blackheath.jpg'} + ]; + const expectedAlbumsPicturesCollections = { + albumsByCollection: [ + {collection: 'cruisy2019', albums: ['cruisy2019/borisbirthday']}, + { + collection: 'whatayear2020', + albums: [ + 'whatayear2020/california2020', + 'whatayear2020/bluemtns2020' + ] + } + ], + pictures: [ + [['cruisy2019/borisbirthday/cake.jpg']], + [ + ['whatayear2020/california2020/disneyland.jpg'], + [ + 'whatayear2020/bluemtns2020/threesisters.png', + 'whatayear2020/bluemtns2020/blackheath.jpg' + ] + ] + ] + }; + expect(album.getAlbumsByCollection(data)) + .to.eql(expectedAlbumsPicturesCollections); + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); +}); diff --git a/test/site-builder/albumPage.spec.js b/test/site-builder/albumPage.spec.js new file mode 100644 index 0000000..7f910b1 --- /dev/null +++ b/test/site-builder/albumPage.spec.js @@ -0,0 +1,242 @@ +const chai = require('chai'); +const mock = require('mock-require'); +const rewire = require('rewire'); +const sinon = require('sinon'); +const sinonChai = require('sinon-chai'); + +chai.use(sinonChai); +const expect = chai.expect; + +describe('albumPage', function() { + describe('uploadAlbumPage', function() { + let putObjectFake; + let albumPage; + + before(function() { + putObjectFake = sinon.fake(); + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() { return {putObject: putObjectFake}; } + }); + + mock('fs', {readFileSync: function(f) { + if (f.includes('index.html')) { + return ( + '\n' + + ' \n' + + '{googletracking}\n' + + ' {title}\n' + + ' \n' + + ' \n' + + '

{title}

\n' + + '

{comment1}

\n' + + '

{comment2}

\n' + + '{pictures}\n' + + ' \n' + + '\n' + ); + } + else if (f.includes('picture.html')) { + return ( + ' \n' + ); + } + else if (f.includes('ga.html')) { + return " \n"; + } + else { + return 'lotsatext'; + } + }}); + + mock('../../site-builder/lib/fileUtils', { + walk: function(dir, done) { + return done(null, [ + 'album/foo/boo.txt', + 'album/index.html', + 'album/snippets/picture.html' + ]); + } + }); + + mock('../../site-builder/lib/delayUtils', { + delayBlock: function() {} + }); + + albumPage = rewire('../../site-builder/lib/albumPage'); + + sinon.stub(console, 'log'); + + process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; + process.env.SITE_BUCKET = 'johnnyphotos'; + }); + + it('uploads album files to s3', function() { + sinon.resetHistory(); + + albumPage.uploadAlbumPage( + 'summerinsicily', + [ + 'summerinsicily/agrigento.jpg', + 'summerinsicily/taormina.jpg' + ], + { + title: 'Summer in Sicily', + comment1: 'With my cat', + comment2: 'And my llama' + } + ); + + const picture1Markup = ( + "\t\t\n" + ); + const picture2Markup = ( + "\t\t\n" + ); + + const expectedIndexBody = ( + '\n' + + '\t\n' + + "\t\t\n\n" + + '\t\tSummer in Sicily\n' + + '\t\n' + + '\t\n' + + '\t\t

Summer in Sicily

\n' + + '\t\t

With my cat

\n' + + '\t\t

And my llama

\n' + + picture1Markup + + picture2Markup + "\n" + + '\t\n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(2); + expect(console.log) + .to.have.been.calledWith('Writing album summerinsicily'); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'summerinsicily/foo/boo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'summerinsicily/index.html' + }); + }); + + it('omits google analytics markup if no tracking code configured', function() { + sinon.resetHistory(); + delete process.env.GOOGLEANALYTICS; + process.env.SPACES_INSTEAD_OF_TABS = true; + process.env.PICTURE_SORT = 'desc'; + + albumPage.uploadAlbumPage( + null, + [ + 'summerinsicily/taormina.jpg', + 'summerinsicily/agrigento.jpg' + ], + {someIgnoredMetadata: 123} + ); + + const picture1Markup = ( + " \n" + ); + const picture2Markup = ( + " \n" + ); + + const expectedIndexBody = ( + '\n' + + ' \n\n' + + ' null\n' + + ' \n' + + ' \n' + + '

null

\n' + + '

\n' + + '

\n' + + picture1Markup + + picture2Markup + "\n" + + ' \n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(2); + expect(console.log) + .to.have.been.calledWith('Writing album null'); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'null/foo/boo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'null/index.html' + }); + + process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; + delete process.env.SPACES_INSTEAD_OF_TABS; + delete process.env.PICTURE_SORT; + }); + + it('raises error if pictures is null', function() { + expect(function() { + albumPage.uploadAlbumPage( + 'summerinsicily', null, + { + title: 'Summer in Sicily', + comment1: 'With my cat', + comment2: 'And my llama' + } + ); + }).to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + mock.stop('fs'); + mock.stop('../../site-builder/lib/fileUtils'); + mock.stop('../../site-builder/lib/delayUtils'); + + sinon.restore(); + + delete process.env.GOOGLEANALYTICS; + delete process.env.SITE_BUCKET; + }); + }); +}); diff --git a/test/sitebuilder/invalidateCloudFront.spec.js b/test/site-builder/cloudfrontUtils.spec.js similarity index 85% rename from test/sitebuilder/invalidateCloudFront.spec.js rename to test/site-builder/cloudfrontUtils.spec.js index 1e8700c..0b96baa 100644 --- a/test/sitebuilder/invalidateCloudFront.spec.js +++ b/test/site-builder/cloudfrontUtils.spec.js @@ -7,10 +7,9 @@ const sinonChai = require('sinon-chai'); chai.use(sinonChai); const expect = chai.expect; -describe('siteBuilder invalidateCloudFront', function() { +describe('cloudfrontUtils', function() { let createInvalidationFake; - let siteBuilder; - let invalidateCloudFront; + let cloudfrontUtils; before(function() { createInvalidationFake = sinon.fake(); @@ -31,9 +30,7 @@ describe('siteBuilder invalidateCloudFront', function() { S3: function() {} }); - siteBuilder = rewire('../../site-builder/index'); - - invalidateCloudFront = siteBuilder.__get__('invalidateCloudFront'); + cloudfrontUtils = rewire('../../site-builder/lib/cloudfrontUtils'); process.env.CLOUDFRONT_DISTRIBUTION_DOMAIN = 'johnnyphotos.com'; }); @@ -45,7 +42,7 @@ describe('siteBuilder invalidateCloudFront', function() { return 12345; }; - invalidateCloudFront(); + cloudfrontUtils.invalidateCloudFront(); expect(createInvalidationFake).to.have.callCount(1); diff --git a/test/site-builder/fileUtils.spec.js b/test/site-builder/fileUtils.spec.js new file mode 100644 index 0000000..adc9de7 --- /dev/null +++ b/test/site-builder/fileUtils.spec.js @@ -0,0 +1,85 @@ +const chai = require('chai'); +const mock = require('mock-require'); + +const expect = chai.expect; + +describe('fileUtils', function() { + describe('walk', function() { + let fileUtils; + + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + + mock('fs', { + readdir: function(dir, cb) { + if (dir == null || !dir.includes('foo')) { + cb(null, []); + } + else if (dir.includes('oink')) { + cb(null, ['quack.html']); + } + else if (dir.includes('baa')) { + cb(null, ['woof.csv']); + } + else { + cb(null, ['oink', 'baa.txt', 'moo.pdf', 'baa']); + } + }, + stat: function(file, cb) { + cb(null, {isDirectory: function() { return file.indexOf('.') === -1; }}); + } + }); + + fileUtils = require('../../site-builder/lib/fileUtils'); + }); + + it('lists all files in a directory tree', function() { + let foundFiles; + + fileUtils.walk('/foo', function(err, files) { + foundFiles = files; + }); + + const expectedFiles = [ + '/foo/oink/quack.html', + '/foo/baa.txt', + '/foo/moo.pdf', + '/foo/baa/woof.csv', + ]; + + expect(foundFiles).to.eql(expectedFiles); + }); + + it('lists no files when directory is empty', function() { + let foundFiles; + + fileUtils.walk('/oonga', function(err, files) { + foundFiles = files; + }); + + expect(foundFiles).to.be.empty; + }); + + it('lists no files if dir is null', function() { + let foundFiles; + + fileUtils.walk(null, function(err, files) { + foundFiles = files; + }); + + expect(foundFiles).to.be.empty; + }); + + it('raises error if done is null', function() { + expect(function() { fileUtils.walk('/daffodils', null); }).to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + mock.stop('fs'); + }); + }); +}); diff --git a/test/site-builder/homePage.spec.js b/test/site-builder/homePage.spec.js new file mode 100644 index 0000000..c741ea9 --- /dev/null +++ b/test/site-builder/homePage.spec.js @@ -0,0 +1,530 @@ +const chai = require('chai'); +const mock = require('mock-require'); +const rewire = require('rewire'); +const sinon = require('sinon'); +const sinonChai = require('sinon-chai'); + +chai.use(sinonChai); +const expect = chai.expect; + +describe('homePage', function() { + describe('uploadHomePage', function() { + let putObjectFake; + let homePage; + + before(function() { + putObjectFake = sinon.fake(); + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() { return {putObject: putObjectFake}; } + }); + + mock('fs', {readFileSync: function(f) { + if (f.includes('index.html')) { + return ( + '\n' + + ' \n' + + '{googletracking}\n' + + ' {title}\n' + + ' \n' + + ' \n' + + '

{title}

\n' + + '{backTo}\n' + + '{pictures}\n' + + ' \n' + + '\n' + ); + } + else if (f.includes('error.html')) { + return ( + '\n' + + ' \n' + + '{googletracking}\n' + + ' Error\n' + + ' \n' + + ' \n' + + '

Error

\n' + + '
\n' + + ' \n' + + '\n' + ); + } + else if (f.includes('album.html')) { + return ( + ' \n' + ); + } + else if (f.includes('ga.html')) { + return " \n"; + } + else if (f.includes('backto.html')) { + return '{backLink}\n'; + } + else { + return 'lotsatext'; + } + }}); + + mock('../../site-builder/lib/fileUtils', {walk: function(dir, done) { + return done(null, [ + 'homepage/foo/hoo.txt', + 'homepage/snippets/album.html', + 'homepage/index.html', + 'homepage/error.html' + ]); + }}); + + homePage = rewire('../../site-builder/lib/homePage'); + + process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; + process.env.SITE_BUCKET = 'johnnyphotos'; + process.env.WEBSITE = "johnnyphotos.com"; + process.env.WEBSITE_TITLE = "Johnny's Awesome Photos"; + }); + + it('uploads homepage files to s3', function() { + sinon.resetHistory(); + process.env.ALBUM_SORT = 'bygiraffe'; + + homePage.uploadHomePage( + [ + 'california2020', + 'bluemtns2020', + 'queenstown2020' + ], + [ + [ + 'california2020/disneyland.jpg', + 'california2020/napa.jpg' + ], + [ + 'bluemtns2020/threesisters.png', + 'bluemtns2020/blackheath.jpg', + 'bluemtns2020/jenolancaves.jpg' + ], + ['queenstown2020/rafting.jpg'] + ], + [ + {title: 'California 2020'}, + {cover_image: 'blackheath.jpg'} + ] + ); + + const album1Markup = ( + "\t\t\n" + ); + const album2Markup = ( + "\t\t\n" + ); + const album3Markup = ( + "\t\t\n" + ); + + const expectedIndexBody = ( + '\n' + + '\t\n' + + "\t\t\n\n" + + "\t\tJohnny's Awesome Photos\n" + + '\t\n' + + '\t\n' + + "\t\t

Johnny's Awesome Photos

\n" + + 'Design: HTML5 UP\n\n' + + album1Markup + + album2Markup + + album3Markup + "\n" + + '\t\n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + '\t\n' + + "\t\t\n\n" + + '\t\tError\n' + + '\t\n' + + '\t\n' + + '\t\t

Error

\n' + + '\t\t
\n' + + '\t\n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(3); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/hoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + delete process.env.ALBUM_SORT; + }); + + it('omits google analytics markup if no tracking code configured', function() { + sinon.resetHistory(); + delete process.env.GOOGLEANALYTICS; + process.env.SPACES_INSTEAD_OF_TABS = true; + + homePage.uploadHomePage( + ['bluemtns2020'], + [['bluemtns2020/jenolancaves.jpg']], + [{cover_image: 'hydromajestic.jpg'}] + ); + + const albumMarkup = ( + " \n" + ); + + const expectedIndexBody = ( + '\n' + + ' \n\n' + + " Johnny's Awesome Photos\n" + + ' \n' + + ' \n' + + "

Johnny's Awesome Photos

\n" + + 'Design: HTML5 UP\n\n' + + albumMarkup + "\n" + + ' \n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + ' \n\n' + + ' Error\n' + + ' \n' + + ' \n' + + '

Error

\n' + + '
\n' + + ' \n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(3); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/hoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; + delete process.env.SPACES_INSTEAD_OF_TABS; + }); + + it('lists albums in ascending order if configured to do so', function() { + sinon.resetHistory(); + process.env.SPACES_INSTEAD_OF_TABS = true; + process.env.ALBUM_SORT = 'asc'; + process.env.HOME_PAGE_CREDITS_OVERRIDE = ( + 'Johnny is Awesome') + ; + + homePage.uploadHomePage( + [ + 'california2020', + 'bluemtns2020', + 'queenstown2020' + ], + [ + [ + 'california2020/disneyland.jpg', + 'california2020/napa.jpg' + ], + [ + 'bluemtns2020/threesisters.png', + 'bluemtns2020/blackheath.jpg', + 'bluemtns2020/jenolancaves.jpg' + ], + ['queenstown2020/rafting.jpg'] + ], + [ + {title: 'California 2020'}, + {cover_image: 'blackheath.jpg'} + ] + ); + + const album1Markup = ( + " \n" + ); + const album2Markup = ( + " \n" + ); + const album3Markup = ( + " \n" + ); + + const expectedIndexBody = ( + '\n' + + ' \n' + + " \n\n" + + " Johnny's Awesome Photos\n" + + ' \n' + + ' \n' + + "

Johnny's Awesome Photos

\n" + + 'Johnny is Awesome\n\n' + + album1Markup + + album2Markup + + album3Markup + "\n" + + ' \n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + ' \n' + + " \n\n" + + ' Error\n' + + ' \n' + + ' \n' + + '

Error

\n' + + '
\n' + + ' \n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(3); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/hoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + delete process.env.SPACES_INSTEAD_OF_TABS; + delete process.env.ALBUM_SORT; + delete process.env.HOME_PAGE_CREDITS_OVERRIDE; + }); + + it('lists albums in descending order if configured to do so', function() { + sinon.resetHistory(); + process.env.SPACES_INSTEAD_OF_TABS = true; + process.env.ALBUM_SORT = 'desc'; + process.env.HIDE_HOME_PAGE_CREDITS = true; + + homePage.uploadHomePage( + [ + 'california2020', + 'bluemtns2020', + 'queenstown2020' + ], + [ + [ + 'california2020/disneyland.jpg', + 'california2020/napa.jpg' + ], + [ + 'bluemtns2020/threesisters.png', + 'bluemtns2020/blackheath.jpg', + 'bluemtns2020/jenolancaves.jpg' + ], + ['queenstown2020/rafting.jpg'] + ], + [ + {title: 'California 2020'}, + {cover_image: 'blackheath.jpg'} + ] + ); + + const album1Markup = ( + " \n" + ); + const album2Markup = ( + " \n" + ); + const album3Markup = ( + " \n" + ); + + const expectedIndexBody = ( + '\n' + + ' \n' + + " \n\n" + + " Johnny's Awesome Photos\n" + + ' \n' + + ' \n' + + "

Johnny's Awesome Photos

\n\n" + + album1Markup + + album2Markup + + album3Markup + "\n" + + ' \n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + ' \n' + + " \n\n" + + ' Error\n' + + ' \n' + + ' \n' + + '

Error

\n' + + '
\n' + + ' \n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(3); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/hoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + delete process.env.SPACES_INSTEAD_OF_TABS; + delete process.env.ALBUM_SORT; + }); + + it('raises error if albums is null', function() { + expect(function() { + homePage.uploadHomePage( + null, [['bluemtns2020/jenolancaves.jpg']], [] + ); + }).to.throw(TypeError); + }); + + it('raises error if pictures is null', function() { + expect(function() { + homePage.uploadHomePage( + ['bluemtns2020'], null, [] + ); + }).to.throw(TypeError); + }); + + it('raises error if metadata is null', function() { + expect(function() { + homePage.uploadHomePage( + ['bluemtns2020'], [['bluemtns2020/jenolancaves.jpg']], null + ); + }).to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + mock.stop('fs'); + mock.stop('../../site-builder/lib/fileUtils'); + + delete process.env.GOOGLEANALYTICS; + delete process.env.SITE_BUCKET; + delete process.env.WEBSITE; + delete process.env.WEBSITE_TITLE; + delete process.env.HIDE_HOME_PAGE_CREDITS; + }); + }); +}); diff --git a/test/site-builder/index.spec.js b/test/site-builder/index.spec.js new file mode 100644 index 0000000..2499ec6 --- /dev/null +++ b/test/site-builder/index.spec.js @@ -0,0 +1,352 @@ +const chai = require('chai'); +const mock = require('mock-require'); +const rewire = require('rewire'); +const sinon = require('sinon'); +const sinonChai = require('sinon-chai'); + +chai.use(sinonChai); +const expect = chai.expect; + +describe('index', function() { + describe('handler', function() { + let siteBuilder; + let putObjectFake; + let clock; + + before(function() { + putObjectFake = sinon.fake(); + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() { return { + listObjectsV2: function(params, cb) { + if (params.ContinuationToken) { + cb(null, {Contents: [ + {Key: 'originaljohnny/bananasinbahamas/bananas.jpg', LastModified: 20180808080000}, + {Key: 'originaljohnny/bananasinbahamas/bahamas.jpg', LastModified: 20180808030000}, + {Key: 'originaljohnny/carrotsincuba/carrots.jpg', LastModified: 20180808050000}, + {Key: 'originaljohnny/carrotsincuba/cuba.jpg', LastModified: 20180808020000}, + {Key: 'originaljohnny/carrotsincuba/havana.jpg', LastModified: 20180808090000} + ]}); + } + else { + cb(null, { + Contents: [], + IsTruncated: true, + NextContinuationToken: 999 + }); + } + }, + getObject: function(params, cb) { + const key = params.Key; + + if (key.includes('bananasinbahamas')) { + cb(null, {Body: ( + 'title: Bananas in Bahamas\n' + + 'comment1: Oh the sunshine\n' + + 'comment2: Upon my pineapple\n' + )}); + } + else { + cb(true, null); + } + }, + putObject: putObjectFake + }; } + }); + + mock('fs', {readFileSync: function(f) { + if (f.includes('homepage/index.html')) { + return ( + '\n' + + '\n' + + '{googletracking}\n' + + '{title}\n' + + '\n' + + '\n' + + '

{title}

\n' + + '{pictures}\n' + + '\n' + + '\n' + ); + } + else if (f.includes('error.html')) { + return ( + '\n' + + '\n' + + '{googletracking}\n' + + 'Error\n' + + '\n' + + '\n' + + '

Error

\n' + + '
\n' + + '\n' + + '\n' + ); + } + else if (f.includes('album/index.html')) { + return ( + '\n' + + '\n' + + '{googletracking}\n' + + '{title}\n' + + '\n' + + '\n' + + '

{title}

\n' + + '

{comment1}

\n' + + '

{comment2}

\n' + + '{pictures}\n' + + '\n' + + '\n' + ); + } + else if (f.includes('album.html')) { + return ( + ' \n' + ); + } + else if (f.includes('picture.html')) { + return ( + ' \n' + ); + } + else { + return 'lotsatext'; + } + }}); + + clock = sinon.useFakeTimers(); + + mock('../../site-builder/lib/cloudfrontUtils', { + invalidateCloudFront: function() {} + }); + + mock('../../site-builder/lib/delayUtils', { + delayBlock: function() {} + }); + + mock('../../site-builder/lib/fileUtils', { + walk: function(dir, done) { + if (dir === 'homepage') { + return done(null, [ + 'homepage/snippets/album.html', + 'homepage/index.html', + 'homepage/error.html' + ]); + } + else { + return done(null, [ + 'album/index.html', + 'album/snippets/picture.html' + ]); + } + } + }); + + mock.reRequire('../../site-builder/lib/metadataUtils'); + mock.reRequire('../../site-builder/lib/homePage'); + + siteBuilder = rewire('../../site-builder/index'); + + sinon.stub(console, 'log'); + + process.env.SITE_BUCKET = 'johnnyphotos'; + process.env.WEBSITE = "johnnyphotos.com"; + process.env.WEBSITE_TITLE = "Johnny's Awesome Photos"; + process.env.PICS_ORIGINAL_PATH = 'originaljohnny/'; + }); + + it('publishes albums site based on pictures in source bucket', function() { + sinon.resetHistory(); + process.env.PICTURE_SORT = 'asc'; + + siteBuilder.handler(null, null); + clock.tick(2000); + + const album1Markup = ( + "\t\t\n" + ); + const album2Markup = ( + "\t\t\n" + ); + + const expectedIndexBody = ( + '\n' + + '\n\n' + + "Johnny's Awesome Photos\n" + + '\n' + + '\n' + + "

Johnny's Awesome Photos

\n" + + album1Markup + + album2Markup + "\n" + + '\n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + '\n\n' + + 'Error\n' + + '\n' + + '\n' + + '

Error

\n' + + '
\n' + + '\n' + + '\n' + ); + + const picture1Markup = ( + "\t\t\n" + ); + const picture2Markup = ( + "\t\t\n" + ); + + const expectedAlbum1IndexBody = ( + '\n' + + '\n\n' + + 'Bananas in Bahamas\n' + + '\n' + + '\n' + + '

Bananas in Bahamas

\n' + + '

Oh the sunshine

\n' + + '

Upon my pineapple

\n' + + picture1Markup + + picture2Markup + "\n" + + '\n' + + '\n' + ); + + const picture3Markup = ( + "\t\t\n" + ); + const picture4Markup = ( + "\t\t\n" + ); + const picture5Markup = ( + "\t\t\n" + ); + + const expectedAlbum2IndexBody = ( + '\n' + + '\n\n' + + 'carrotsincuba\n' + + '\n' + + '\n' + + '

carrotsincuba

\n' + + '

\n' + + '

\n' + + picture3Markup + + picture4Markup + + picture5Markup + "\n" + + '\n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(4); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedAlbum1IndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'bananasinbahamas/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedAlbum2IndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'carrotsincuba/index.html' + }); + + expect(console.log).to.have.callCount(5); + expect(console.log) + .to.have.been.calledWith( + 'S3 listing was truncated. Pausing 2 seconds before continuing 999' + ); + expect(console.log) + .to.have.been.calledWith('First album: bananasinbahamas'); + expect(console.log) + .to.have.been.calledWith('Last album: carrotsincuba'); + expect(console.log) + .to.have.been.calledWith('Writing album bananasinbahamas'); + expect(console.log) + .to.have.been.calledWith('Writing album carrotsincuba'); + }); + + after(function() { + mock.stop('aws-sdk'); + mock.stop('fs'); + mock.stop('../../site-builder/lib/fileUtils'); + mock.stop('../../site-builder/lib/cloudfrontUtils'); + mock.stop('../../site-builder/lib/delayUtils'); + + sinon.restore(); + clock.restore(); + + delete process.env.SITE_BUCKET; + delete process.env.WEBSITE; + delete process.env.WEBSITE_TITLE; + delete process.env.PICTURE_SORT; + delete process.env.PICS_ORIGINAL_PATH; + }); + }); +}); diff --git a/test/site-builder/metadataUtils.spec.js b/test/site-builder/metadataUtils.spec.js new file mode 100644 index 0000000..adc1ac5 --- /dev/null +++ b/test/site-builder/metadataUtils.spec.js @@ -0,0 +1,105 @@ +const chai = require('chai'); +const mock = require('mock-require'); +const rewire = require('rewire'); + +const expect = chai.expect; + +describe('metadataUtils', function() { + describe('getAlbumOrCollectionMetadata', function() { + let metadataUtils; + + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() { + return { + getObject: function(params, cb) { + const bucket = params.Bucket; + const key = params.Key; + + if (bucket !== 'johnnyphotos') { + cb(null, null); + } + else if (key.includes('funkyfauna/daisies')) { + cb(null, {Body: "title: Daisies"}); + } + else if (key.includes('bobsbirthday')) { + cb(null, {Body: "title: Bob's Birthday"}); + } + else if (key.includes('megsbirthday')) { + cb(null, 'badyaml'); + } + else { + cb(true, null); + } + } + }; + } + }); + + metadataUtils = rewire('../../site-builder/lib/metadataUtils'); + + process.env.ORIGINAL_BUCKET = 'johnnyphotos'; + }); + + it('gets metadata for specified album', function() { + let metadata; + + metadataUtils.getAlbumOrCollectionMetadata('bobsbirthday', function(err, doc) { + metadata = doc; + }); + + expect(metadata).to.eql({title: "Bob's Birthday"}); + }); + + it('gets metadata for specified collection', function() { + let metadata; + + metadataUtils.getAlbumOrCollectionMetadata('funkyfauna/daisies', function(err, doc) { + metadata = doc; + }); + + expect(metadata).to.eql({title: "Daisies"}); + }); + + it('gets nothing for album with invalid metadata', function() { + let metadata; + + metadataUtils.getAlbumOrCollectionMetadata('megsbirthday', function(err, doc) { + metadata = doc; + }); + + expect(metadata).to.be.a('null'); + }); + + it('gets nothing for album with no metadata', function() { + let metadata; + + metadataUtils.getAlbumOrCollectionMetadata('suesbirthday', function(err, doc) { + metadata = doc; + }); + + expect(metadata).to.be.a('null'); + }); + + it('gets nothing for wrong bucket', function() { + let metadata; + + process.env.ORIGINAL_BUCKET = 'jimmyphotos'; + + metadataUtils.getAlbumOrCollectionMetadata('bobsbirthday', function(err, doc) { + metadata = doc; + }); + + expect(metadata).to.be.a('null'); + + process.env.ORIGINAL_BUCKET = 'johnnyphotos'; + }); + + after(function() { + mock.stop('aws-sdk'); + + delete process.env.ORIGINAL_BUCKET; + }); + }); +}); diff --git a/test/site-builder/miscUtils.spec.js b/test/site-builder/miscUtils.spec.js new file mode 100644 index 0000000..aa8a3f6 --- /dev/null +++ b/test/site-builder/miscUtils.spec.js @@ -0,0 +1,113 @@ +const chai = require('chai'); +const mock = require('mock-require'); + +const miscUtils = require('../../site-builder/lib/miscUtils'); + +const expect = chai.expect; + +describe('miscUtils', function() { + describe('isEmpty', function() { + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + }); + + it('returns true if object is empty', function() { + expect(miscUtils.isEmpty({})).to.be.true; + }); + + it('returns false if object is not empty', function() { + expect(miscUtils.isEmpty({foo: 'hoo'})).to.be.false; + }); + + it('returns true if hasOwnProperty is hacked up', function() { + expect(miscUtils.isEmpty({ + hasOwnProperty: function() { + return false; + }, + bar: 'Here be dragons' + })).to.be.true; + }); + + it('returns true if passed null', function() { + expect(miscUtils.isEmpty(null)).to.be.true; + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); + + describe('spacesToTabs', function() { + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + }); + + it('converts each 2 spaces to a tab', function() { + expect(miscUtils.spacesToTabs('hi there mate', 2)).to.equal('hi\tthere\tmate'); + }); + + it('returns empty string if passed empty string', function() { + expect(miscUtils.spacesToTabs('')).to.equal(''); + }); + + it('raises error if passed null', function() { + expect(function() { miscUtils.spacesToTabs(null); }).to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); + + describe('stripPrefix', function() { + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + }); + + it('removes pics/original/ if present at start of string', function() { + expect(miscUtils.stripPrefix({Key: 'pics/original/bla'})).to.equal('bla'); + }); + + it('removes pics/original/ if present at end of string', function() { + expect(miscUtils.stripPrefix({Key: 'bla/pics/original/'})).to.equal('bla/'); + }); + + it('removes pics/original/ if present in middle of string', function() { + expect(miscUtils.stripPrefix({Key: 'bla/pics/original/bla'})).to.equal('bla/bla'); + }); + + it('removes only first instance of pics/original/ from string', function() { + expect(miscUtils.stripPrefix({Key: 'hoo/pics/original/haa/pics/original/bla'})) + .to.equal('hoo/haa/pics/original/bla'); + }); + + it('leaves string un-modified if pics/original/ not present', function() { + expect(miscUtils.stripPrefix({Key: 'hoohaa'})).to.equal('hoohaa'); + }); + + it('raises error if passed null', function() { + expect(function() { miscUtils.stripPrefix(null); }).to.throw(TypeError); + }); + + it('raises error if missing Key', function() { + expect(function() { miscUtils.stripPrefix({}); }).to.throw(TypeError); + }); + + it('raises error if Key is null', function() { + expect(function() { miscUtils.stripPrefix({Key: null}); }).to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); +}); diff --git a/test/site-builder/pathUtils.spec.js b/test/site-builder/pathUtils.spec.js new file mode 100644 index 0000000..50394f2 --- /dev/null +++ b/test/site-builder/pathUtils.spec.js @@ -0,0 +1,47 @@ +const chai = require('chai'); +const mock = require('mock-require'); + +const expect = chai.expect; + +describe('pathUtils', function() { + describe('firstLevelFolderName', function() { + let pathUtils; + + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + + pathUtils = require('../../site-builder/lib/pathUtils'); + }); + + it('gets string before first slash', function() { + expect(pathUtils.firstLevelFolderName('folderfoo/filefoo')).to.equal('folderfoo'); + }); + + it('gets string before first of many slashes', function() { + expect(pathUtils.firstLevelFolderName('folder1/folder2/folder3/filefoo')).to.equal('folder1'); + }); + + it('gets nothing if nothing before first slash', function() { + expect(pathUtils.firstLevelFolderName('/somepath')).to.equal(''); + }); + + it('gets nothing if passed nothing', function() { + expect(pathUtils.firstLevelFolderName('')).to.equal(''); + }); + + it('leaves string un-modified if no slash', function() { + expect(pathUtils.firstLevelFolderName('somepath')).to.equal('somepath'); + }); + + it('raises error if passed null', function() { + expect(function() { pathUtils.firstLevelFolderName(null); }).to.throw(TypeError); + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); +}); diff --git a/test/site-builder/sorters.spec.js b/test/site-builder/sorters.spec.js new file mode 100644 index 0000000..585dd3b --- /dev/null +++ b/test/site-builder/sorters.spec.js @@ -0,0 +1,61 @@ +const chai = require('chai'); +const mock = require('mock-require'); + +const sorters = require('../../site-builder/lib/sorters'); + +const expect = chai.expect; + +describe('sorters', function() { + describe('albumSorter', function() { + before(function() { + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() {} + }); + }); + + it('sorts by album in ascending order', function() { + const albumsAndStuff = [ + {album: 'ccc'}, + {album: 'bbb'}, + {album: 'ddd'}, + {album: 'bbb'}, + {album: 'aaa'} + ]; + + albumsAndStuff.sort(sorters.albumAscSorter); + + expect(albumsAndStuff).to.eql([ + {album: 'aaa'}, + {album: 'bbb'}, + {album: 'bbb'}, + {album: 'ccc'}, + {album: 'ddd'} + ]); + }); + + it('sorts by album in descending order', function() { + const albumsAndStuff = [ + {album: 'ccc'}, + {album: 'bbb'}, + {album: 'ddd'}, + {album: 'bbb'}, + {album: 'aaa'} + ]; + + albumsAndStuff.sort(sorters.albumDescSorter); + + expect(albumsAndStuff).to.eql([ + {album: 'ddd'}, + {album: 'ccc'}, + {album: 'bbb'}, + {album: 'bbb'}, + {album: 'aaa'} + ]); + }); + + after(function() { + mock.stop('aws-sdk'); + }); + }); +}); diff --git a/test/site-builder/uploadContents.spec.js b/test/site-builder/uploadContents.spec.js new file mode 100644 index 0000000..a089826 --- /dev/null +++ b/test/site-builder/uploadContents.spec.js @@ -0,0 +1,1312 @@ +const chai = require('chai'); +const mock = require('mock-require'); +const rewire = require('rewire'); +const sinon = require('sinon'); +const sinonChai = require('sinon-chai'); + +chai.use(sinonChai); +const expect = chai.expect; + +describe('uploadContents', function() { + describe('uploadAllContentsByCollection', function() { + let uploadContents; + let putObjectFake; + + before(function() { + putObjectFake = sinon.fake(); + mock('aws-sdk', { + CloudFront: function() {}, + S3: function() { return { + getObject: function(params, cb) { + const key = params.Key; + + if (key === 'pics/original/whitewinter/soggysundays/metadata.yml') { + cb(null, {Body: ( + 'title: Soggy Sundays\n' + )}); + } + else if (key === 'pics/original/whitewinter/metadata.yml') { + cb(null, {Body: ( + 'title: White Winter\n' + + 'cover_image: soggysundays/rainy.jpg\n' + )}); + } + else { + cb(true, null); + } + }, + putObject: putObjectFake + }; } + }); + + mock('fs', {readFileSync: function(f) { + if (f.includes('homepage/index.html')) { + return ( + '\n' + + '\n' + + '{googletracking}\n' + + '{title}\n' + + '\n' + + '\n' + + '

{title}

\n' + + '{backTo}\n' + + '{pictures}\n' + + '\n' + + '\n' + ); + } + else if (f.includes('error.html')) { + return ( + '\n' + + '\n' + + '{googletracking}\n' + + 'Error\n' + + '\n' + + '\n' + + '

Error

\n' + + '
\n' + + '\n' + + '\n' + ); + } + else if (f.includes('album/index.html')) { + return ( + '\n' + + '\n' + + '{googletracking}\n' + + '{title}\n' + + '\n' + + '\n' + + '

{title}

\n' + + '

{comment1}

\n' + + '

{comment2}

\n' + + '{pictures}\n' + + '\n' + + '\n' + ); + } + else if (f.includes('album.html')) { + return ( + ' \n' + ); + } + else if (f.includes('picture.html')) { + return ( + ' \n' + ); + } + else if (f.includes('backto.html')) { + return '{backLink}\n'; + } + else if (f.includes('main.css')) { + return 'cssgoeshere'; + } + else if (f.includes('album.css')) { + return 'albumcssgoeshere'; + } + else { + return 'lotsatext'; + } + }}); + + mock('../../site-builder/lib/fileUtils', { + walk: function(dir, done) { + if (dir === 'homepage') { + return done(null, [ + 'homepage/snippets/album.html', + 'homepage/index.html', + 'homepage/error.html', + 'homepage/foo/shoo.txt', + 'homepage/assets/css/main.css' + ]); + } + else { + return done(null, [ + 'album/index.html', + 'album/snippets/picture.html', + 'album/assets/css/album.css' + ]); + } + } + }); + + mock('../../site-builder/lib/cloudfrontUtils', { + invalidateCloudFront: function() {} + }); + + mock('../../site-builder/lib/delayUtils', { + delayBlock: function() {} + }); + + mock.reRequire('../../site-builder/lib/metadataUtils'); + mock.reRequire('../../site-builder/lib/homePage'); + mock.reRequire('../../site-builder/lib/albumPage'); + mock.reRequire('../../site-builder/lib/collectionPage'); + + uploadContents = rewire('../../site-builder/lib/uploadContents'); + + sinon.stub(console, 'log'); + + process.env.SITE_BUCKET = 'johnnyphotos'; + process.env.WEBSITE = "johnnyphotos.com"; + process.env.WEBSITE_TITLE = "Johnny's Awesome Photos"; + process.env.GROUP_ALBUMS_INTO_COLLECTIONS = true; + }); + + it('publishes albums grouped into collections', function() { + sinon.resetHistory(); + + const allContents = [ + {Key: 'pics/original/whitewinter/metadata.yml'}, + { + Key: 'pics/original/whitewinter/muggymondays/mist.jpg', + LastModified: 20180404000000 + }, + { + Key: 'pics/original/whitewinter/muggymondays/fog.jpg', + LastModified: 20180403000000 + }, + {Key: 'pics/original/whitewinter/soggysundays/metadata.yml'}, + { + Key: 'pics/original/whitewinter/soggysundays/wet.jpg', + LastModified: 20180406000000 + }, + { + Key: 'pics/original/whitewinter/soggysundays/rainy.jpg', + LastModified: 20180402000000 + }, + { + Key: 'pics/original/whitewinter/soggysundays/dreary.jpg', + LastModified: 20180401000000 + }, + {Key: 'pics/original/artyautumn/terrifictrees/oak.jpg'}, + {Key: 'pics/original/artyautumn/terrifictrees/birch.jpg'}, + {Key: 'pics/original/artyautumn/boisterousbirds/galah.jpg'}, + ]; + + uploadContents.uploadAllContents(allContents); + + const coll1Markup = ( + "\t\t\n" + ); + const coll2Markup = ( + "\t\t\n" + ); + + const expectedIndexBody = ( + '\n' + + '\n\n' + + "Johnny's Awesome Photos\n" + + '\n' + + '\n' + + "

Johnny's Awesome Photos

\n" + + 'Design: HTML5 UP\n\n' + + coll1Markup + + coll2Markup + "\n" + + '\n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + '\n\n' + + 'Error\n' + + '\n' + + '\n' + + '

Error

\n' + + '
\n' + + '\n' + + '\n' + ); + + const coll1album1Markup = ( + "\t\t\n" + ); + const coll1album2Markup = ( + "\t\t\n" + ); + + const expectedColl1Body = ( + '\n' + + '\n\n' + + "artyautumn\n" + + '\n' + + '\n' + + "

artyautumn

\n" + + 'Back to Johnny\'s Awesome Photos\n\n' + + coll1album1Markup + + coll1album2Markup + "\n" + + '\n' + + '\n' + ); + + const picture1Markup = ( + "\t\t\n" + ); + + const expectedColl1Album1Body = ( + '\n' + + '\n\n' + + 'boisterousbirds\n' + + '\n' + + '\n' + + '

boisterousbirds

\n' + + '

\n' + + '

\n' + + picture1Markup + "\n" + + '\n' + + '\n' + ); + + const picture2Markup = ( + "\t\t\n" + ); + const picture3Markup = ( + "\t\t\n" + ); + + const expectedColl1Album2Body = ( + '\n' + + '\n\n' + + 'terrifictrees\n' + + '\n' + + '\n' + + '

terrifictrees

\n' + + '

\n' + + '

\n' + + picture2Markup + + picture3Markup + "\n" + + '\n' + + '\n' + ); + + const coll2album1Markup = ( + "\t\t\n" + ); + const coll2album2Markup = ( + "\t\t\n" + ); + + const expectedColl2Body = ( + '\n' + + '\n\n' + + "White Winter\n" + + '\n' + + '\n' + + "

White Winter

\n" + + 'Back to Johnny\'s Awesome Photos\n\n' + + coll2album1Markup + + coll2album2Markup + "\n" + + '\n' + + '\n' + ); + + const picture4Markup = ( + "\t\t\n" + ); + const picture5Markup = ( + "\t\t\n" + ); + const picture6Markup = ( + "\t\t\n" + ); + + const expectedColl2Album1Body = ( + '\n' + + '\n\n' + + 'Soggy Sundays\n' + + '\n' + + '\n' + + '

Soggy Sundays

\n' + + '

\n' + + '

\n' + + picture4Markup + + picture5Markup + + picture6Markup + "\n" + + '\n' + + '\n' + ); + + const picture7Markup = ( + "\t\t\n" + ); + const picture8Markup = ( + "\t\t\n" + ); + + const expectedColl2Album2Body = ( + '\n' + + '\n\n' + + 'muggymondays\n' + + '\n' + + '\n' + + '

muggymondays

\n' + + '

\n' + + '

\n' + + picture7Markup + + picture8Markup + "\n" + + '\n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(11); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/shoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'cssgoeshere', + Bucket: 'johnnyphotos', + ContentType: 'text/css', + Key: 'assets/homepage/css/main.css' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'albumcssgoeshere', + Bucket: 'johnnyphotos', + ContentType: 'text/css', + Key: 'assets/album/css/album.css' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Album1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/boisterousbirds/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Album2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/terrifictrees/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Album1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/soggysundays/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Album2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/muggymondays/index.html' + }); + + expect(console.log).to.have.callCount(12); + + expect(console.log) + .to.have.been.calledWith('First collection: artyautumn'); + expect(console.log) + .to.have.been.calledWith('Last collection: whitewinter'); + expect(console.log) + .to.have.been.calledWith('First album in artyautumn: boisterousbirds'); + expect(console.log) + .to.have.been.calledWith('Last album in artyautumn: terrifictrees'); + expect(console.log) + .to.have.been.calledWith('Writing collection artyautumn'); + expect(console.log) + .to.have.been.calledWith('Writing album boisterousbirds'); + expect(console.log) + .to.have.been.calledWith('Writing album terrifictrees'); + expect(console.log) + .to.have.been.calledWith('First album in whitewinter: soggysundays'); + expect(console.log) + .to.have.been.calledWith('Last album in whitewinter: muggymondays'); + expect(console.log) + .to.have.been.calledWith('Writing collection whitewinter'); + expect(console.log) + .to.have.been.calledWith('Writing album soggysundays'); + expect(console.log) + .to.have.been.calledWith('Writing album muggymondays'); + }); + + it('publishes albums grouped into collections descending', function() { + sinon.resetHistory(); + + process.env.COLLECTION_SORT = 'desc'; + process.env.ALBUM_SORT = 'asc'; + + const allContents = [ + {Key: 'pics/original/whitewinter/metadata.yml'}, + { + Key: 'pics/original/whitewinter/muggymondays/mist.jpg', + LastModified: 20180404000000 + }, + { + Key: 'pics/original/whitewinter/muggymondays/fog.jpg', + LastModified: 20180403000000 + }, + {Key: 'pics/original/whitewinter/soggysundays/metadata.yml'}, + { + Key: 'pics/original/whitewinter/soggysundays/wet.jpg', + LastModified: 20180406000000 + }, + { + Key: 'pics/original/whitewinter/soggysundays/rainy.jpg', + LastModified: 20180402000000 + }, + { + Key: 'pics/original/whitewinter/soggysundays/dreary.jpg', + LastModified: 20180401000000 + }, + {Key: 'pics/original/artyautumn/terrifictrees/oak.jpg'}, + {Key: 'pics/original/artyautumn/terrifictrees/birch.jpg'}, + {Key: 'pics/original/artyautumn/boisterousbirds/galah.jpg'}, + ]; + + uploadContents.uploadAllContents(allContents); + + const coll1Markup = ( + "\t\t\n" + ); + const coll2Markup = ( + "\t\t\n" + ); + + const expectedIndexBody = ( + '\n' + + '\n\n' + + "Johnny's Awesome Photos\n" + + '\n' + + '\n' + + "

Johnny's Awesome Photos

\n" + + 'Design: HTML5 UP\n\n' + + coll1Markup + + coll2Markup + "\n" + + '\n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + '\n\n' + + 'Error\n' + + '\n' + + '\n' + + '

Error

\n' + + '
\n' + + '\n' + + '\n' + ); + + const coll1album1Markup = ( + "\t\t\n" + ); + const coll1album2Markup = ( + "\t\t\n" + ); + + const expectedColl1Body = ( + '\n' + + '\n\n' + + "artyautumn\n" + + '\n' + + '\n' + + "

artyautumn

\n" + + 'Back to Johnny\'s Awesome Photos\n\n' + + coll1album1Markup + + coll1album2Markup + "\n" + + '\n' + + '\n' + ); + + const picture1Markup = ( + "\t\t\n" + ); + + const expectedColl1Album1Body = ( + '\n' + + '\n\n' + + 'boisterousbirds\n' + + '\n' + + '\n' + + '

boisterousbirds

\n' + + '

\n' + + '

\n' + + picture1Markup + "\n" + + '\n' + + '\n' + ); + + const picture2Markup = ( + "\t\t\n" + ); + const picture3Markup = ( + "\t\t\n" + ); + + const expectedColl1Album2Body = ( + '\n' + + '\n\n' + + 'terrifictrees\n' + + '\n' + + '\n' + + '

terrifictrees

\n' + + '

\n' + + '

\n' + + picture2Markup + + picture3Markup + "\n" + + '\n' + + '\n' + ); + + const coll2album1Markup = ( + "\t\t\n" + ); + const coll2album2Markup = ( + "\t\t\n" + ); + + const expectedColl2Body = ( + '\n' + + '\n\n' + + "White Winter\n" + + '\n' + + '\n' + + "

White Winter

\n" + + 'Back to Johnny\'s Awesome Photos\n\n' + + coll2album1Markup + + coll2album2Markup + "\n" + + '\n' + + '\n' + ); + + const picture4Markup = ( + "\t\t\n" + ); + const picture5Markup = ( + "\t\t\n" + ); + const picture6Markup = ( + "\t\t\n" + ); + + const expectedColl2Album1Body = ( + '\n' + + '\n\n' + + 'Soggy Sundays\n' + + '\n' + + '\n' + + '

Soggy Sundays

\n' + + '

\n' + + '

\n' + + picture4Markup + + picture5Markup + + picture6Markup + "\n" + + '\n' + + '\n' + ); + + const picture7Markup = ( + "\t\t\n" + ); + const picture8Markup = ( + "\t\t\n" + ); + + const expectedColl2Album2Body = ( + '\n' + + '\n\n' + + 'muggymondays\n' + + '\n' + + '\n' + + '

muggymondays

\n' + + '

\n' + + '

\n' + + picture7Markup + + picture8Markup + "\n" + + '\n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(11); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/shoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'cssgoeshere', + Bucket: 'johnnyphotos', + ContentType: 'text/css', + Key: 'assets/homepage/css/main.css' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'albumcssgoeshere', + Bucket: 'johnnyphotos', + ContentType: 'text/css', + Key: 'assets/album/css/album.css' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Album1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/boisterousbirds/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Album2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/terrifictrees/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Album1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/soggysundays/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Album2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/muggymondays/index.html' + }); + + expect(console.log).to.have.callCount(12); + + expect(console.log) + .to.have.been.calledWith('First collection: artyautumn'); + expect(console.log) + .to.have.been.calledWith('Last collection: whitewinter'); + expect(console.log) + .to.have.been.calledWith('First album in artyautumn: boisterousbirds'); + expect(console.log) + .to.have.been.calledWith('Last album in artyautumn: terrifictrees'); + expect(console.log) + .to.have.been.calledWith('Writing collection artyautumn'); + expect(console.log) + .to.have.been.calledWith('Writing album boisterousbirds'); + expect(console.log) + .to.have.been.calledWith('Writing album terrifictrees'); + expect(console.log) + .to.have.been.calledWith('First album in whitewinter: soggysundays'); + expect(console.log) + .to.have.been.calledWith('Last album in whitewinter: muggymondays'); + expect(console.log) + .to.have.been.calledWith('Writing collection whitewinter'); + expect(console.log) + .to.have.been.calledWith('Writing album soggysundays'); + expect(console.log) + .to.have.been.calledWith('Writing album muggymondays'); + + delete process.env.COLLECTION_SORT; + delete process.env.ALBUM_SORT; + }); + + it('publishes albums grouped into collections ascending', function() { + sinon.resetHistory(); + + process.env.COLLECTION_SORT = 'asc'; + process.env.ALBUM_SORT = 'desc'; + + const allContents = [ + {Key: 'pics/original/whitewinter/metadata.yml'}, + { + Key: 'pics/original/whitewinter/muggymondays/mist.jpg', + LastModified: 20180404000000 + }, + { + Key: 'pics/original/whitewinter/muggymondays/fog.jpg', + LastModified: 20180403000000 + }, + {Key: 'pics/original/whitewinter/soggysundays/metadata.yml'}, + { + Key: 'pics/original/whitewinter/soggysundays/wet.jpg', + LastModified: 20180406000000 + }, + { + Key: 'pics/original/whitewinter/soggysundays/rainy.jpg', + LastModified: 20180402000000 + }, + { + Key: 'pics/original/whitewinter/soggysundays/dreary.jpg', + LastModified: 20180401000000 + }, + {Key: 'pics/original/artyautumn/terrifictrees/oak.jpg'}, + {Key: 'pics/original/artyautumn/terrifictrees/birch.jpg'}, + {Key: 'pics/original/artyautumn/boisterousbirds/galah.jpg'}, + ]; + + uploadContents.uploadAllContents(allContents); + + const coll1Markup = ( + "\t\t\n" + ); + const coll2Markup = ( + "\t\t\n" + ); + + const expectedIndexBody = ( + '\n' + + '\n\n' + + "Johnny's Awesome Photos\n" + + '\n' + + '\n' + + "

Johnny's Awesome Photos

\n" + + 'Design: HTML5 UP\n\n' + + coll1Markup + + coll2Markup + "\n" + + '\n' + + '\n' + ); + + const expectedErrorBody = ( + '\n' + + '\n\n' + + 'Error\n' + + '\n' + + '\n' + + '

Error

\n' + + '
\n' + + '\n' + + '\n' + ); + + const coll1album1Markup = ( + "\t\t\n" + ); + const coll1album2Markup = ( + "\t\t\n" + ); + + const expectedColl1Body = ( + '\n' + + '\n\n' + + "artyautumn\n" + + '\n' + + '\n' + + "

artyautumn

\n" + + 'Back to Johnny\'s Awesome Photos\n\n' + + coll1album1Markup + + coll1album2Markup + "\n" + + '\n' + + '\n' + ); + + const picture1Markup = ( + "\t\t\n" + ); + + const expectedColl1Album1Body = ( + '\n' + + '\n\n' + + 'boisterousbirds\n' + + '\n' + + '\n' + + '

boisterousbirds

\n' + + '

\n' + + '

\n' + + picture1Markup + "\n" + + '\n' + + '\n' + ); + + const picture2Markup = ( + "\t\t\n" + ); + const picture3Markup = ( + "\t\t\n" + ); + + const expectedColl1Album2Body = ( + '\n' + + '\n\n' + + 'terrifictrees\n' + + '\n' + + '\n' + + '

terrifictrees

\n' + + '

\n' + + '

\n' + + picture2Markup + + picture3Markup + "\n" + + '\n' + + '\n' + ); + + const coll2album1Markup = ( + "\t\t\n" + ); + const coll2album2Markup = ( + "\t\t\n" + ); + + const expectedColl2Body = ( + '\n' + + '\n\n' + + "White Winter\n" + + '\n' + + '\n' + + "

White Winter

\n" + + 'Back to Johnny\'s Awesome Photos\n\n' + + coll2album1Markup + + coll2album2Markup + "\n" + + '\n' + + '\n' + ); + + const picture4Markup = ( + "\t\t\n" + ); + const picture5Markup = ( + "\t\t\n" + ); + const picture6Markup = ( + "\t\t\n" + ); + + const expectedColl2Album1Body = ( + '\n' + + '\n\n' + + 'Soggy Sundays\n' + + '\n' + + '\n' + + '

Soggy Sundays

\n' + + '

\n' + + '

\n' + + picture4Markup + + picture5Markup + + picture6Markup + "\n" + + '\n' + + '\n' + ); + + const picture7Markup = ( + "\t\t\n" + ); + const picture8Markup = ( + "\t\t\n" + ); + + const expectedColl2Album2Body = ( + '\n' + + '\n\n' + + 'muggymondays\n' + + '\n' + + '\n' + + '

muggymondays

\n' + + '

\n' + + '

\n' + + picture7Markup + + picture8Markup + "\n" + + '\n' + + '\n' + ); + + expect(putObjectFake).to.have.callCount(11); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedIndexBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedErrorBody, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'error.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'lotsatext', + Bucket: 'johnnyphotos', + ContentType: 'text/plain', + Key: 'foo/shoo.txt' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'cssgoeshere', + Bucket: 'johnnyphotos', + ContentType: 'text/css', + Key: 'assets/homepage/css/main.css' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: 'albumcssgoeshere', + Bucket: 'johnnyphotos', + ContentType: 'text/css', + Key: 'assets/album/css/album.css' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Album1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/boisterousbirds/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl1Album2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'artyautumn/terrifictrees/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Album1Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/soggysundays/index.html' + }); + + expect(putObjectFake).to.have.been.calledWith({ + Body: expectedColl2Album2Body, + Bucket: 'johnnyphotos', + ContentType: 'text/html', + Key: 'whitewinter/muggymondays/index.html' + }); + + expect(console.log).to.have.callCount(12); + + expect(console.log) + .to.have.been.calledWith('First collection: artyautumn'); + expect(console.log) + .to.have.been.calledWith('Last collection: whitewinter'); + expect(console.log) + .to.have.been.calledWith('First album in artyautumn: boisterousbirds'); + expect(console.log) + .to.have.been.calledWith('Last album in artyautumn: terrifictrees'); + expect(console.log) + .to.have.been.calledWith('Writing collection artyautumn'); + expect(console.log) + .to.have.been.calledWith('Writing album boisterousbirds'); + expect(console.log) + .to.have.been.calledWith('Writing album terrifictrees'); + expect(console.log) + .to.have.been.calledWith('First album in whitewinter: soggysundays'); + expect(console.log) + .to.have.been.calledWith('Last album in whitewinter: muggymondays'); + expect(console.log) + .to.have.been.calledWith('Writing collection whitewinter'); + expect(console.log) + .to.have.been.calledWith('Writing album soggysundays'); + expect(console.log) + .to.have.been.calledWith('Writing album muggymondays'); + + delete process.env.COLLECTION_SORT; + delete process.env.ALBUM_SORT; + }); + + after(function() { + mock.stop('aws-sdk'); + mock.stop('fs'); + mock.stop('../../site-builder/lib/fileUtils'); + mock.stop('../../site-builder/lib/cloudfrontUtils'); + mock.stop('../../site-builder/lib/delayUtils'); + + sinon.restore(); + + delete process.env.SITE_BUCKET; + delete process.env.WEBSITE; + delete process.env.WEBSITE_TITLE; + delete process.env.GROUP_ALBUMS_INTO_COLLECTIONS; + }); + }); +}); diff --git a/test/sitebuilder/folderName.spec.js b/test/sitebuilder/folderName.spec.js deleted file mode 100644 index a55cb57..0000000 --- a/test/sitebuilder/folderName.spec.js +++ /dev/null @@ -1,49 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); - -const expect = chai.expect; - -describe('siteBuilder folderName', function() { - let siteBuilder; - let folderName; - - before(function() { - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() {} - }); - - siteBuilder = rewire('../../site-builder/index'); - - folderName = siteBuilder.__get__('folderName'); - }); - - it('gets string before first slash', function() { - expect(folderName('folderfoo/filefoo')).to.equal('folderfoo'); - }); - - it('gets string before first of many slashes', function() { - expect(folderName('folder1/folder2/folder3/filefoo')).to.equal('folder1'); - }); - - it('gets nothing if nothing before first slash', function() { - expect(folderName('/somepath')).to.equal(''); - }); - - it('gets nothing if passed nothing', function() { - expect(folderName('')).to.equal(''); - }); - - it('leaves string un-modified if no slash', function() { - expect(folderName('somepath')).to.equal('somepath'); - }); - - it('raises error if passed null', function() { - expect(function() { folderName(null); }).to.throw(TypeError); - }); - - after(function() { - mock.stop('aws-sdk'); - }); -}); diff --git a/test/sitebuilder/getAlbumMetadata.spec.js b/test/sitebuilder/getAlbumMetadata.spec.js deleted file mode 100644 index f9339b7..0000000 --- a/test/sitebuilder/getAlbumMetadata.spec.js +++ /dev/null @@ -1,93 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); - -const expect = chai.expect; - -describe('siteBuilder getAlbumMetadata', function() { - let siteBuilder; - let getAlbumMetadata; - - before(function() { - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() { - return { - getObject: function(params, cb) { - const bucket = params.Bucket; - const key = params.Key; - - if (bucket !== 'johnnyphotos') { - cb(null, null); - } - else if (key.includes('bobsbirthday')) { - cb(null, {Body: "title: Bob's Birthday"}); - } - else if (key.includes('megsbirthday')) { - cb(null, 'badyaml'); - } - else { - cb(true, null); - } - } - }; - } - }); - - siteBuilder = rewire('../../site-builder/index'); - - getAlbumMetadata = siteBuilder.__get__('getAlbumMetadata'); - - process.env.ORIGINAL_BUCKET = 'johnnyphotos'; - }); - - it('gets metadata for specified album', function() { - let metadata; - - getAlbumMetadata('bobsbirthday', function(err, doc) { - metadata = doc; - }); - - expect(metadata).to.eql({title: "Bob's Birthday"}); - }); - - it('gets nothing for album with invalid metadata', function() { - let metadata; - - getAlbumMetadata('megsbirthday', function(err, doc) { - metadata = doc; - }); - - expect(metadata).to.be.a('null'); - }); - - it('gets nothing for album with no metadata', function() { - let metadata; - - getAlbumMetadata('suesbirthday', function(err, doc) { - metadata = doc; - }); - - expect(metadata).to.be.a('null'); - }); - - it('gets nothing for wrong bucket', function() { - let metadata; - - process.env.ORIGINAL_BUCKET = 'jimmyphotos'; - - getAlbumMetadata('bobsbirthday', function(err, doc) { - metadata = doc; - }); - - expect(metadata).to.be.a('null'); - - process.env.ORIGINAL_BUCKET = 'johnnyphotos'; - }); - - after(function() { - mock.stop('aws-sdk'); - - delete process.env.ORIGINAL_BUCKET; - }); -}); diff --git a/test/sitebuilder/getAlbums.spec.js b/test/sitebuilder/getAlbums.spec.js deleted file mode 100644 index 6cff575..0000000 --- a/test/sitebuilder/getAlbums.spec.js +++ /dev/null @@ -1,77 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); - -const expect = chai.expect; - -describe('siteBuilder getAlbums', function() { - let siteBuilder; - let getAlbums; - - before(function() { - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() {} - }); - - siteBuilder = rewire('../../site-builder/index'); - - getAlbums = siteBuilder.__get__('getAlbums'); - }); - - it('returns albums and pictures from input data', function() { - const data = [ - {Key: 'california2020/disneyland.jpg', LastModified: 20200106090000}, - {Key: 'california2020/napa.jpg', LastModified: 20200101090000}, - {Key: 'bluemtns2020/threesisters.png'}, - {Key: 'bluemtns2020/blackheath.jpg', LastModified: 20200102090000}, - {Key: 'bluemtns2020/jenolancaves.jpg', LastModified: null}, - {Key: 'funnyalbum/funnyfile.txt', LastModified: 20200109090000} - ]; - const expectedAlbumsAndPictures = { - albums: [ - 'california2020', - 'bluemtns2020', - 'funnyalbum' - ], - pictures: [ - [ - 'california2020/disneyland.jpg', - 'california2020/napa.jpg' - ], - [ - 'bluemtns2020/threesisters.png', - 'bluemtns2020/blackheath.jpg', - 'bluemtns2020/jenolancaves.jpg' - ], - [] - ] - }; - expect(getAlbums(data)).to.eql(expectedAlbumsAndPictures); - }); - - it('returns empty lists from empty input', function() { - expect(getAlbums([])).to.eql({albums: [], pictures: []}); - }); - - it('raises error if passed null', function() { - expect(function() { getAlbums(null); }).to.throw(TypeError); - }); - - it('raises error if array element is null', function() { - expect(function() { getAlbums([null]); }).to.throw(TypeError); - }); - - it('raises error if array element is empty object', function() { - expect(function() { getAlbums([{}]); }).to.throw(TypeError); - }); - - it('raises error if array element missing Key', function() { - expect(function() { getAlbums([{LastModified: 'yesterday'}]); }) - .to.throw(TypeError); - }); - - after(function() { - mock.stop('aws-sdk'); - }); -}); diff --git a/test/sitebuilder/isEmpty.spec.js b/test/sitebuilder/isEmpty.spec.js deleted file mode 100644 index 0c45962..0000000 --- a/test/sitebuilder/isEmpty.spec.js +++ /dev/null @@ -1,46 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); - -const expect = chai.expect; - -describe('siteBuilder isEmpty', function() { - let siteBuilder; - let isEmpty; - - before(function() { - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() {} - }); - - siteBuilder = rewire('../../site-builder/index'); - - isEmpty = siteBuilder.__get__('isEmpty'); - }); - - it('returns true if object is empty', function() { - expect(isEmpty({})).to.be.true; - }); - - it('returns false if object is not empty', function() { - expect(isEmpty({foo: 'hoo'})).to.be.false; - }); - - it('returns true if hasOwnProperty is hacked up', function() { - expect(isEmpty({ - hasOwnProperty: function() { - return false; - }, - bar: 'Here be dragons' - })).to.be.true; - }); - - it('returns true if passed null', function() { - expect(isEmpty(null)).to.be.true; - }); - - after(function() { - mock.stop('aws-sdk'); - }); -}); diff --git a/test/sitebuilder/listAllContents.spec.js b/test/sitebuilder/listAllContents.spec.js deleted file mode 100644 index 5a4763f..0000000 --- a/test/sitebuilder/listAllContents.spec.js +++ /dev/null @@ -1,338 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); -const sinon = require('sinon'); -const sinonChai = require('sinon-chai'); - -chai.use(sinonChai); -const expect = chai.expect; - -describe('siteBuilder listAllContents', function() { - let siteBuilder; - let handler; - let putObjectFake; - let clock; - - before(function() { - putObjectFake = sinon.fake(); - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() { return { - listObjectsV2: function(params, cb) { - if (params.ContinuationToken) { - cb(null, {Contents: [ - {Key: 'bananasinbahamas/bananas.jpg', LastModified: 20180808080000}, - {Key: 'bananasinbahamas/bahamas.jpg', LastModified: 20180808030000}, - {Key: 'carrotsincuba/carrots.jpg', LastModified: 20180808050000}, - {Key: 'carrotsincuba/cuba.jpg', LastModified: 20180808020000}, - {Key: 'carrotsincuba/havana.jpg', LastModified: 20180808090000} - ]}); - } - else { - cb(null, { - Contents: [], - IsTruncated: true, - NextContinuationToken: 999 - }); - } - }, - getObject: function(params, cb) { - const key = params.Key; - - if (key.includes('bananasinbahamas')) { - cb(null, {Body: ( - 'title: Bananas in Bahamas\n' + - 'comment1: Oh the sunshine\n' + - 'comment2: Upon my pineapple\n' - )}); - } - else { - cb(true, null); - } - }, - putObject: putObjectFake - }; } - }); - - mock('fs', {readFileSync: function(f) { - if (f.includes('homepage/index.html')) { - return ( - '\n' + - '\n' + - '{googletracking}\n' + - '{title}\n' + - '\n' + - '\n' + - '

{title}

\n' + - '{pictures}\n' + - '\n' + - '\n' - ); - } - else if (f.includes('error.html')) { - return ( - '\n' + - '\n' + - '{googletracking}\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - } - else if (f.includes('album/index.html')) { - return ( - '\n' + - '\n' + - '{googletracking}\n' + - '{title}\n' + - '\n' + - '\n' + - '

{title}

\n' + - '

{comment1}

\n' + - '

{comment2}

\n' + - '{pictures}\n' + - '\n' + - '\n' - ); - } - else { - return 'lotsatext'; - } - }}); - - clock = sinon.useFakeTimers(); - - siteBuilder = rewire('../../site-builder/index'); - - handler = siteBuilder.handler; - - siteBuilder.__set__({ - walk: function(dir, done) { - if (dir === 'homepage') { - return done(null, [ - 'homepage/index.html', - 'homepage/error.html' - ]); - } - else { - return done(null, ['album/index.html']); - } - }, - invalidateCloudFront: function() {}, - delayBlock: function() {} - }); - - sinon.stub(console, 'log'); - - process.env.SITE_BUCKET = 'johnnyphotos'; - process.env.WEBSITE = "johnnyphotos.com"; - process.env.WEBSITE_TITLE = "Johnny's Awesome Photos"; - }); - - it('publishes albums site based on pictures in source bucket', function() { - sinon.resetHistory(); - - handler(null, null); - clock.tick(2000); - - const album1Markup = ( - "\t\t\t\t\t\t
\n" + - "\t\t\t\t\t\t\t" + - "\"\"" + - "\n" + - "\t\t\t\t\t\t\t

carrotsincuba

\n" + - "\t\t\t\t\t\t
" - ); - const album2Markup = ( - "\t\t\t\t\t\t
\n" + - "\t\t\t\t\t\t\t" + - "\"\"" + - "\n" + - "\t\t\t\t\t\t\t

Bananas in Bahamas

\n" + - "\t\t\t\t\t\t
" - ); - - const expectedIndexBody = ( - '\n' + - '\n\n' + - "Johnny's Awesome Photos\n" + - '\n' + - '\n' + - "

Johnny's Awesome Photos

\n" + - album1Markup + - album2Markup + "\n" + - '\n' + - '\n' - ); - - const expectedErrorBody = ( - '\n' + - '\n\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - - const picture1Markup = ( - "\t\t\t\t\t\t" - ); - const picture2Markup = ( - "\t\t\t\t\t\t" - ); - - const expectedAlbum1IndexBody = ( - '\n' + - '\n\n' + - 'Bananas in Bahamas\n' + - '\n' + - '\n' + - '

Bananas in Bahamas

\n' + - '

Oh the sunshine

\n' + - '

Upon my pineapple

\n' + - picture1Markup + - picture2Markup + "\n" + - '\n' + - '\n' - ); - - const picture3Markup = ( - "\t\t\t\t\t\t" - ); - const picture4Markup = ( - "\t\t\t\t\t\t" - ); - const picture5Markup = ( - "\t\t\t\t\t\t" - ); - - const expectedAlbum2IndexBody = ( - '\n' + - '\n\n' + - 'carrotsincuba\n' + - '\n' + - '\n' + - '

carrotsincuba

\n' + - '

\n' + - '

\n' + - picture3Markup + - picture4Markup + - picture5Markup + "\n" + - '\n' + - '\n' - ); - - expect(putObjectFake).to.have.callCount(4); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedIndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'index.html' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedErrorBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'error.html' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedAlbum1IndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'bananasinbahamas/index.html' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedAlbum2IndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'carrotsincuba/index.html' - }); - - expect(console.log).to.have.callCount(5); - expect(console.log) - .to.have.been.calledWith( - 'S3 listing was truncated. Pausing 2 sec before continuing 999' - ); - expect(console.log) - .to.have.been.calledWith('First Album: bananasinbahamas'); - expect(console.log) - .to.have.been.calledWith('Last Album: carrotsincuba'); - expect(console.log) - .to.have.been.calledWith('Writing ALBUM bananasinbahamas'); - expect(console.log) - .to.have.been.calledWith('Writing ALBUM carrotsincuba'); - }); - - after(function() { - mock.stop('aws-sdk'); - mock.stop('fs'); - - sinon.restore(); - clock.restore(); - - delete process.env.SITE_BUCKET; - delete process.env.WEBSITE; - delete process.env.WEBSITE_TITLE; - }); -}); diff --git a/test/sitebuilder/stripPrefix.spec.js b/test/sitebuilder/stripPrefix.spec.js deleted file mode 100644 index 89dbb9f..0000000 --- a/test/sitebuilder/stripPrefix.spec.js +++ /dev/null @@ -1,58 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); - -const expect = chai.expect; - -describe('siteBuilder stripPrefix', function() { - let siteBuilder; - let stripPrefix; - - before(function() { - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() {} - }); - - siteBuilder = rewire('../../site-builder/index'); - - stripPrefix = siteBuilder.__get__('stripPrefix'); - }); - - it('removes pics/original/ if present at start of string', function() { - expect(stripPrefix({Key: 'pics/original/bla'})).to.equal('bla'); - }); - - it('removes pics/original/ if present at end of string', function() { - expect(stripPrefix({Key: 'bla/pics/original/'})).to.equal('bla/'); - }); - - it('removes pics/original/ if present in middle of string', function() { - expect(stripPrefix({Key: 'bla/pics/original/bla'})).to.equal('bla/bla'); - }); - - it('removes only first instance of pics/original/ from string', function() { - expect(stripPrefix({Key: 'hoo/pics/original/haa/pics/original/bla'})) - .to.equal('hoo/haa/pics/original/bla'); - }); - - it('leaves string un-modified if pics/original/ not present', function() { - expect(stripPrefix({Key: 'hoohaa'})).to.equal('hoohaa'); - }); - - it('raises error if passed null', function() { - expect(function() { stripPrefix(null); }).to.throw(TypeError); - }); - - it('raises error if missing Key', function() { - expect(function() { stripPrefix({}); }).to.throw(TypeError); - }); - - it('raises error if Key is null', function() { - expect(function() { stripPrefix({Key: null}); }).to.throw(TypeError); - }); - - after(function() { - mock.stop('aws-sdk'); - }); -}); diff --git a/test/sitebuilder/uploadAlbumSite.spec.js b/test/sitebuilder/uploadAlbumSite.spec.js deleted file mode 100644 index 4684252..0000000 --- a/test/sitebuilder/uploadAlbumSite.spec.js +++ /dev/null @@ -1,267 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); -const sinon = require('sinon'); -const sinonChai = require('sinon-chai'); - -chai.use(sinonChai); -const expect = chai.expect; - -describe('siteBuilder uploadAlbumSite', function() { - let ga; - let putObjectFake; - let siteBuilder; - let uploadAlbumSite; - - before(function() { - putObjectFake = sinon.fake(); - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() { return {putObject: putObjectFake}; } - }); - - mock('fs', {readFileSync: function(f) { - if (f.includes('index.html')) { - return ( - '\n' + - '\n' + - '{googletracking}\n' + - '{title}\n' + - '\n' + - '\n' + - '

{title}

\n' + - '

{comment1}

\n' + - '

{comment2}

\n' + - '{pictures}\n' + - '\n' + - '\n' - ); - } - else { - return 'lotsatext'; - } - }}); - - siteBuilder = rewire('../../site-builder/index'); - - ga = siteBuilder.__get__('ga'); - uploadAlbumSite = siteBuilder.__get__('uploadAlbumSite'); - - siteBuilder.__set__({ - walk: function(dir, done) { - return done(null, [ - 'album/foo/boo.txt', - 'album/index.html' - ]); - }, - delayBlock: function() {} - }); - - sinon.stub(console, 'log'); - - process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; - process.env.SITE_BUCKET = 'johnnyphotos'; - }); - - it('uploads album files to s3', function() { - sinon.resetHistory(); - - uploadAlbumSite( - 'summerinsicily', - [ - 'summerinsicily/agrigento.jpg', - 'summerinsicily/taormina.jpg' - ], - { - title: 'Summer in Sicily', - comment1: 'With my cat', - comment2: 'And my llama' - } - ); - - const picture1Markup = ( - "\t\t\t\t\t\t" - ); - const picture2Markup = ( - "\t\t\t\t\t\t" - ); - - const expectedIndexBody = ( - '\n' + - '\n' + - ga.replace(/\{gtag\}/g, 'googleanalyticsfunkycode') + '\n' + - 'Summer in Sicily\n' + - '\n' + - '\n' + - '

Summer in Sicily

\n' + - '

With my cat

\n' + - '

And my llama

\n' + - picture1Markup + - picture2Markup + "\n" + - '\n' + - '\n' - ); - - const expectedErrorBody = ( - '\n' + - '\n' + - ga.replace(/\{gtag\}/g, 'googleanalyticsfunkycode') + '\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - - expect(putObjectFake).to.have.callCount(2); - expect(console.log) - .to.have.been.calledWith('Writing ALBUM summerinsicily'); - - expect(putObjectFake).to.have.been.calledWith({ - Body: 'lotsatext', - Bucket: 'johnnyphotos', - ContentType: 'text/plain', - Key: 'summerinsicily/foo/boo.txt' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedIndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'summerinsicily/index.html' - }); - }); - - it('omits google analytics markup if no tracking code configured', function() { - sinon.resetHistory(); - delete process.env.GOOGLEANALYTICS; - - uploadAlbumSite( - null, - [ - 'summerinsicily/agrigento.jpg', - 'summerinsicily/taormina.jpg' - ], - {someIgnoredMetadata: 123} - ); - - const picture1Markup = ( - "\t\t\t\t\t\t" - ); - const picture2Markup = ( - "\t\t\t\t\t\t" - ); - - const expectedIndexBody = ( - '\n' + - '\n\n' + - 'null\n' + - '\n' + - '\n' + - '

null

\n' + - '

\n' + - '

\n' + - picture1Markup + - picture2Markup + "\n" + - '\n' + - '\n' - ); - - const expectedErrorBody = ( - '\n' + - '\n\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - - expect(putObjectFake).to.have.callCount(2); - expect(console.log) - .to.have.been.calledWith('Writing ALBUM null'); - - expect(putObjectFake).to.have.been.calledWith({ - Body: 'lotsatext', - Bucket: 'johnnyphotos', - ContentType: 'text/plain', - Key: 'null/foo/boo.txt' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedIndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'null/index.html' - }); - - process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; - }); - - it('raises error if pictures is null', function() { - expect(function() { - uploadAlbumSite( - 'summerinsicily', null, - { - title: 'Summer in Sicily', - comment1: 'With my cat', - comment2: 'And my llama' - } - ); - }).to.throw(TypeError); - }); - - after(function() { - mock.stop('aws-sdk'); - mock.stop('fs'); - - sinon.restore(); - - delete process.env.GOOGLEANALYTICS; - delete process.env.SITE_BUCKET; - }); -}); diff --git a/test/sitebuilder/uploadHomepageSite.spec.js b/test/sitebuilder/uploadHomepageSite.spec.js deleted file mode 100644 index 4b19bc8..0000000 --- a/test/sitebuilder/uploadHomepageSite.spec.js +++ /dev/null @@ -1,266 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); -const sinon = require('sinon'); -const sinonChai = require('sinon-chai'); - -chai.use(sinonChai); -const expect = chai.expect; - -describe('siteBuilder uploadHomepageSite', function() { - let ga; - let putObjectFake; - let siteBuilder; - let uploadHomepageSite; - - before(function() { - putObjectFake = sinon.fake(); - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() { return {putObject: putObjectFake}; } - }); - - mock('fs', {readFileSync: function(f) { - if (f.includes('index.html')) { - return ( - '\n' + - '\n' + - '{googletracking}\n' + - '{title}\n' + - '\n' + - '\n' + - '

{title}

\n' + - '{pictures}\n' + - '\n' + - '\n' - ); - } - else if (f.includes('error.html')) { - return ( - '\n' + - '\n' + - '{googletracking}\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - } - else { - return 'lotsatext'; - } - }}); - - siteBuilder = rewire('../../site-builder/index'); - - ga = siteBuilder.__get__('ga'); - uploadHomepageSite = siteBuilder.__get__('uploadHomepageSite'); - - siteBuilder.__set__({walk: function(dir, done) { - return done(null, [ - 'homepage/foo/hoo.txt', - 'homepage/index.html', - 'homepage/error.html' - ]); - }}); - - process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; - process.env.SITE_BUCKET = 'johnnyphotos'; - process.env.WEBSITE = "johnnyphotos.com"; - process.env.WEBSITE_TITLE = "Johnny's Awesome Photos"; - }); - - it('uploads homepage files to s3', function() { - sinon.resetHistory(); - - uploadHomepageSite( - [ - 'california2020', - 'bluemtns2020' - ], - [ - [ - 'california2020/disneyland.jpg', - 'california2020/napa.jpg' - ], - [ - 'bluemtns2020/threesisters.png', - 'bluemtns2020/blackheath.jpg', - 'bluemtns2020/jenolancaves.jpg' - ] - ], - [{title: 'California 2020'}] - ); - - const album1Markup = ( - "\t\t\t\t\t\t
\n" + - "\t\t\t\t\t\t\t" + - "\"\"" + - "\n" + - "\t\t\t\t\t\t\t

California 2020

\n" + - "\t\t\t\t\t\t
" - ); - const album2Markup = ( - "\t\t\t\t\t\t
\n" + - "\t\t\t\t\t\t\t" + - "\"\"" + - "\n" + - "\t\t\t\t\t\t\t

bluemtns2020

\n" + - "\t\t\t\t\t\t
" - ); - - const expectedIndexBody = ( - '\n' + - '\n' + - ga.replace(/\{gtag\}/g, 'googleanalyticsfunkycode') + '\n' + - "Johnny's Awesome Photos\n" + - '\n' + - '\n' + - "

Johnny's Awesome Photos

\n" + - album1Markup + - album2Markup + "\n" + - '\n' + - '\n' - ); - - const expectedErrorBody = ( - '\n' + - '\n' + - ga.replace(/\{gtag\}/g, 'googleanalyticsfunkycode') + '\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - - expect(putObjectFake).to.have.callCount(3); - - expect(putObjectFake).to.have.been.calledWith({ - Body: 'lotsatext', - Bucket: 'johnnyphotos', - ContentType: 'text/plain', - Key: 'foo/hoo.txt' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedIndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'index.html' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedErrorBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'error.html' - }); - }); - - it('omits google analytics markup if no tracking code configured', function() { - sinon.resetHistory(); - delete process.env.GOOGLEANALYTICS; - - uploadHomepageSite( - ['bluemtns2020'], [['bluemtns2020/jenolancaves.jpg']], [] - ); - - const albumMarkup = ( - "\t\t\t\t\t\t
\n" + - "\t\t\t\t\t\t\t" + - "\"\"" + - "\n" + - "\t\t\t\t\t\t\t

bluemtns2020

\n" + - "\t\t\t\t\t\t
" - ); - - const expectedIndexBody = ( - '\n' + - '\n\n' + - "Johnny's Awesome Photos\n" + - '\n' + - '\n' + - "

Johnny's Awesome Photos

\n" + - albumMarkup + "\n" + - '\n' + - '\n' - ); - - const expectedErrorBody = ( - '\n' + - '\n\n' + - 'Error\n' + - '\n' + - '\n' + - '

Error

\n' + - '
\n' + - '\n' + - '\n' - ); - - expect(putObjectFake).to.have.callCount(3); - - expect(putObjectFake).to.have.been.calledWith({ - Body: 'lotsatext', - Bucket: 'johnnyphotos', - ContentType: 'text/plain', - Key: 'foo/hoo.txt' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedIndexBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'index.html' - }); - - expect(putObjectFake).to.have.been.calledWith({ - Body: expectedErrorBody, - Bucket: 'johnnyphotos', - ContentType: 'text/html', - Key: 'error.html' - }); - - process.env.GOOGLEANALYTICS = 'googleanalyticsfunkycode'; - }); - - it('raises error if albums is null', function() { - expect(function() { - uploadHomepageSite( - null, [['bluemtns2020/jenolancaves.jpg']], [] - ); - }).to.throw(TypeError); - }); - - it('raises error if pictures is null', function() { - expect(function() { - uploadHomepageSite( - ['bluemtns2020'], null, [] - ); - }).to.throw(TypeError); - }); - - it('raises error if metadata is null', function() { - expect(function() { - uploadHomepageSite( - ['bluemtns2020'], [['bluemtns2020/jenolancaves.jpg']], null - ); - }).to.throw(TypeError); - }); - - after(function() { - mock.stop('aws-sdk'); - mock.stop('fs'); - - delete process.env.GOOGLEANALYTICS; - delete process.env.SITE_BUCKET; - delete process.env.WEBSITE; - delete process.env.WEBSITE_TITLE; - }); -}); diff --git a/test/sitebuilder/walk.spec.js b/test/sitebuilder/walk.spec.js deleted file mode 100644 index 38048a5..0000000 --- a/test/sitebuilder/walk.spec.js +++ /dev/null @@ -1,87 +0,0 @@ -const chai = require('chai'); -const mock = require('mock-require'); -const rewire = require('rewire'); - -const expect = chai.expect; - -describe('siteBuilder walk', function() { - let siteBuilder; - let walk; - - before(function() { - mock('aws-sdk', { - CloudFront: function() {}, - S3: function() {} - }); - - mock('fs', { - readdir: function(dir, cb) { - if (dir == null || !dir.includes('foo')) { - cb(null, []); - } - else if (dir.includes('oink')) { - cb(null, ['quack.html']); - } - else if (dir.includes('baa')) { - cb(null, ['woof.csv']); - } - else { - cb(null, ['oink', 'baa.txt', 'moo.pdf', 'baa']); - } - }, - stat: function(file, cb) { - cb(null, {isDirectory: function() { return file.indexOf('.') === -1; }}); - } - }); - - siteBuilder = rewire('../../site-builder/index'); - - walk = siteBuilder.__get__('walk'); - }); - - it('lists all files in a directory tree', function() { - let foundFiles; - - walk('/foo', function(err, files) { - foundFiles = files; - }); - - const expectedFiles = [ - '/foo/oink/quack.html', - '/foo/baa.txt', - '/foo/moo.pdf', - '/foo/baa/woof.csv', - ]; - - expect(foundFiles).to.eql(expectedFiles); - }); - - it('lists no files when directory is empty', function() { - let foundFiles; - - walk('/oonga', function(err, files) { - foundFiles = files; - }); - - expect(foundFiles).to.be.empty; - }); - - it('lists no files if dir is null', function() { - let foundFiles; - - walk(null, function(err, files) { - foundFiles = files; - }); - - expect(foundFiles).to.be.empty; - }); - - it('raises error if done is null', function() { - expect(function() { walk('/daffodils', null); }).to.throw(TypeError); - }); - - after(function() { - mock.stop('aws-sdk'); - mock.stop('fs'); - }); -});