Skip to content

Commit

Permalink
Liftoff!
Browse files Browse the repository at this point in the history
  • Loading branch information
srikanthmanda committed Oct 30, 2020
1 parent c796856 commit 95d2b0c
Show file tree
Hide file tree
Showing 9 changed files with 183 additions and 25 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ Following are the components of this [Rube Goldberg application](https://en.wiki
- [x] SNS (Destination)
- [x] Lambda function createAttributeFiles
- [x] SQS ~~(Destination)~~
- [ ] Lambda function createCheatSheet
- [ ] S3 static site
- [x] Lambda function createCheatSheet
- [x] S3 static site

## Notes

Expand Down
4 changes: 3 additions & 1 deletion TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,6 @@
- [x] Move `aws-sdk` package to layer
- [x] Automate creation of S3 bucket for SAM CLI
- [x] Add Lambda functions test events for [CLI](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/lambda/invoke.html) (not [console](https://github.com/aws/serverless-application-model/issues/475))
- [x] Script lambda tests
- [x] Script lambda tests
- [ ] Review notes
- [ ] Cleanup templates
13 changes: 8 additions & 5 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
{
"name": "beginning-aws-lambda",
"description": "A repo to learn basics of AWS Lambda",
"version": "0.0.1",
"private": true,
"description": "A package built to learn basics of AWS Lambda",
"version": "1.0.0",
"repository": {
"type" : "git",
"url" : "https://github.com/srikanthmanda/beginning-aws-lambda.git"
},
"dependencies": {
"@octokit/request": "^5.4.9",
"yauzl": "^2.10.0"
Expand All @@ -11,7 +14,7 @@
"aws-sdk": "2.771.0"
},
"scripts": {
"build-aws-sdk-layer": "cd lib-aws-sdk && zip -r ../sam-build/aws-sdk.zip nodejs",
"build": "zip -r sam-build/beginning-lambda.zip src node_modules"
"sampkg-aws-sdk-layer": "cd lib-aws-sdk && zip -r ../sam-build/aws-sdk.zip nodejs",
"sampkg": "zip -r sam-build/beginning-lambda.zip src node_modules"
}
}
50 changes: 50 additions & 0 deletions sam-config/template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ Resources:
FunctionName: unzipRepoArchive
Handler: src/unzip-repo-archive.handler
Description: Reads AWS Docs repo from S3 and extracts it to EFS.
ReservedConcurrentExecutions: 1
Environment:
Variables:
EFS_PATH: "/mnt/efs"
Expand Down Expand Up @@ -90,13 +91,15 @@ Resources:
Properties:
QueueName: createAttrFilesSuccessEvents
VisibilityTimeout: 200
MessageRetentionPeriod: 300

createAttributeFiles:
Type: AWS::Serverless::Function
Properties:
FunctionName: createAttributeFiles
Handler: src/create-attribute-files.handler
Description: Reads AWS Docs repo folder from EFS and creates CloudFormation attribute files in another EFS folder.
ReservedConcurrentExecutions: 1
Environment:
Variables:
EFS_PATH: "/mnt/efs"
Expand Down Expand Up @@ -130,6 +133,49 @@ Resources:
FunctionName: !Ref createAttributeFiles
Principal: sns.amazonaws.com

generateCheatSheet:
Type: AWS::Serverless::Function
Properties:
FunctionName: generateCheatSheet
Handler: src/generate-cheat-sheet.handler
Description: Reads CloudFormation attribute files from EFS, creates a cheatsheet and uploads it to S3.
ReservedConcurrentExecutions: 1
Environment:
Variables:
EFS_PATH: "/mnt/efs"
AWS_CFN_ATTRS_CHEATSHEET_SITE: !Ref awsCfnAttrsCheatSheetSiteName
Policies:
- AWSLambdaBasicExecutionRole
- AmazonS3FullAccess
- AWSLambdaVPCAccessExecutionRole
- AmazonElasticFileSystemClientFullAccess
- AWSLambdaSQSQueueExecutionRole
VpcConfig:
SecurityGroupIds:
- Fn::ImportValue: !Sub "${lambdaEfsStack}-secgroup-id"
SubnetIds:
- Fn::ImportValue: !Sub "${lambdaEfsStack}-subnet-id"
FileSystemConfigs:
- Arn:
Fn::ImportValue: !Sub "${lambdaEfsStack}-access-pt-arn"
LocalMountPath: /mnt/efs

generateCheatSheetEventSource:
Type: "AWS::Lambda::EventSourceMapping"
Properties:
BatchSize: 1
EventSourceArn: !GetAtt createAttrFilesSuccessEvents.Arn
FunctionName: !GetAtt generateCheatSheet.Arn

awsCfnAttrsCheatSheetSite:
Type: "AWS::S3::Bucket"
Properties:
AccessControl: PublicRead
BucketName: !Ref awsCfnAttrsCheatSheetSiteName
WebsiteConfiguration:
ErrorDocument: 404.html
IndexDocument: index.md

Globals:
Function:
CodeUri: ../sam-build/beginning-lambda.zip
Expand All @@ -152,3 +198,7 @@ Parameters:
Description: Lambda EFS stack name
Type: String
Default: lambda-efs-vpc
awsCfnAttrsCheatSheetSiteName:
Description: Name of the S3 website bucket to store AWS CloudFormation Attributes cheatsheet.
Type: String
Default: aws-cloudformation-attributes-cheatsheet
8 changes: 4 additions & 4 deletions scripts/0-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ function installProdDependencies {
test -d $1 || (echo "Directory $1 does not exist" && exit 1)
cd $1 && echo "In directory: $(pwd)"
(test -f package.json || -f package-lock.json) || (echo "package.json or package-lock.json do not exist" && exit 1)
test -d node_modules && echo "Pruning node_modules for production ..." && npm prune --production
npm install --production && echo "Installed production node_modules."
# test -d node_modules && echo "Pruning node_modules for production ..." && npm prune --production
npm ci --production && echo "Installed production node_modules."
cd ${DIR_REPO_ROOT} && echo "Back in root directory $(pwd)"
}

Expand All @@ -24,7 +24,7 @@ installProdDependencies ${DIR_REPO_ROOT}

rm ${DIR_SAM_BUILD}/*.zip || echo "No build files to delete"

npm run build-aws-sdk-layer
npm run build
npm run sampkg-aws-sdk-layer
npm run sampkg

echo "Layer and function build successfully."
14 changes: 3 additions & 11 deletions scripts/2-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,16 @@ declare -A lambdaFns

function prepareLambdaFnsList {
templateFile=$1
numFunctions=0
numHandlers=0
#grep -E "FunctionName|Handler" ${templateFile} | while IFS= read -r line
## http://mywiki.wooledge.org/BashFAQ/024
while IFS= read -r line
do
if [[ ${line} =~ "FunctionName" ]]
then
lastFunction=$(echo ${line} | cut -d ' ' -f 2)
(( numFunctions+=1 ))
function=$(echo ${line} | cut -d ' ' -f 2)
else
lastHandler=$(echo ${line} | cut -d '/' -f 2 | cut -d '.' -f 1)
(( numHandlers+=1 ))
fi

if [[ numFunctions -eq numHandlers ]]
then
lambdaFns["${lastFunction}"]="${lastHandler}"
handler=$(echo ${line} | cut -d '/' -f 2 | cut -d '.' -f 1)
lambdaFns["${function}"]="${handler}"
fi
done <<< $(grep -E "FunctionName|Handler" ${templateFile})
}
Expand Down
5 changes: 3 additions & 2 deletions src/create-attribute-files.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ exports.handler = async function (event) {
fs.mkdirSync(dataDir);
console.info(`Created directory ${dataDir}.`);

await createAttributeFiles(inputDir, dataDir);
const indexFile = await createAttributeFiles(inputDir, dataDir);
console.log("Attribute files created. Sending message to SQS ...");
return sqs
.sendMessage({
MessageBody: dataDir,
MessageBody: indexFile,
QueueUrl: successNotificationSqs,
})
.promise();
Expand Down Expand Up @@ -148,4 +148,5 @@ async function createAttributeFiles(inputDir, outputDir) {
);
batchStart = batchEnd;
}
return INDEX_FILE;
}
103 changes: 103 additions & 0 deletions src/generate-cheat-sheet.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
const fs = require("fs");
const readline = require("readline");

const AWS = require("aws-sdk");
const s3 = new AWS.S3();

const rootDir = process.env.EFS_PATH ? process.env.EFS_PATH : "/tmp";
const cfnAttrFilesDir = process.env.CFN_ATTR_FILES_PATH
? process.env.CFN_ATTR_FILES_PATH
: "aws-cloudformation-attributes";
const cfnAttrCheatSheet = process.env.AWS_CFN_ATTRS_CHEATSHEET
? process.env.AWS_CFN_ATTRS_CHEATSHEET
: "aws-cloudformation-attributes.md";
const cfnAttrCheatSheetSite = process.env.AWS_CFN_ATTRS_CHEATSHEET_SITE
? process.env.AWS_CFN_ATTRS_CHEATSHEET_SITE
: "aws-cloudformation-attributes-cheatsheet";
const dataDir = rootDir + "/" + cfnAttrFilesDir;

exports.handler = async function (event) {
const indexFile = dataDir + "/" + event.Records[0].body;
const cheatSheet = dataDir + "/" + cfnAttrCheatSheet;
await generateCheatsheet(indexFile, cheatSheet);
const s3Params = {
Bucket: cfnAttrCheatSheetSite,
Key: "index.md",
Body: fs.createReadStream(cheatSheet),
ACL: "public-read",
ContentType: "text/plain",
};
return s3
.putObject(s3Params, function (err, data) {
console.log(err, data);
})
.promise();
};

async function generateCheatsheet(indexFile, cheatSheet) {
const FOOTER =
"- - -\n" +
"This document was generated from [AWS CloudFormation User Guide]" +
"(https://github.com/awsdocs/aws-cloudformation-user-guide) " +
"using [`aws-cloudformation-attributes`]" +
"(https://github.com/srikanthmanda/aws-cloudformation-attributes).";
const dataMap = {};

const rl = readline.createInterface({
input: fs.createReadStream(indexFile),
crlfDelay: Infinity,
});

for await (const line of rl) {
const fields = line.split(",");
const entityNames = fields[0].split("::");
const api = entityNames[1];
const entity = entityNames[2];
const attributesFile = fields[1];

if (dataMap[api]) {
dataMap[api][entity] = attributesFile;
} else {
dataMap[api] = {};
dataMap[api][entity] = attributesFile;
}
}

const cheatSheetBody = cheatSheet.replace(/\.md$/, "_body.md");
fs.writeFileSync(
cheatSheet,
"# AWS CloudFormation Attributes\n\n## Table of Contents\n"
);

for (const api of Object.keys(dataMap).sort()) {
fs.appendFileSync(cheatSheet, "\n### AWS::" + api + "\n");
for (const entity of Object.keys(dataMap[api]).sort()) {
fs.appendFileSync(
cheatSheetBody,
"\n#" + fs.readFileSync(dataMap[api][entity])
);
fs.appendFileSync(
cheatSheet,
"* [AWS::" +
api +
"::" +
entity +
"](#aws" +
(api + entity).toLowerCase() +
")\n"
);
}
}

fs.appendFileSync(cheatSheet, fs.readFileSync(cheatSheetBody));
fs.appendFileSync(cheatSheet, FOOTER);
console.log("cheatSheet created: " + cheatSheet);
fs.unlink(cheatSheetBody, (error) => {
if (error) {
console.error("Failed to delete cheatSheet body: " + cheatSheetBody);
console.error("Error: " + JSON.stringify(error));
} else {
console.debug("Deleted cheatSheet body: " + cheatSheetBody);
}
});
}
7 changes: 7 additions & 0 deletions test-events/generate-cheat-sheet-test-event.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"Records": [
{
"body": "aws-cloudformation-attributes.md"
}
]
}

0 comments on commit 95d2b0c

Please sign in to comment.