update terraform script and use ec2 in pipelines

crawl.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+# go to the spider directory
+cd /opt/project/project
+# run the spider
+scrapy crawl product

project/project/pipelines.py

@@ -1,5 +1,7 @@
 import datetime
 import boto3
+from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher
+
 
 
 def default_encoder(value):
@@ -17,8 +19,11 @@ class DynamoDbPipeline(object):
 
     def __init__(self, aws_access_key_id, aws_secret_access_key, region_name,
                  table_name, encoder=default_encoder):
-        self.aws_access_key_id = aws_access_key_id
-        self.aws_secret_access_key = aws_secret_access_key
+        provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2))
+        creds = provider.load().get_frozen_credentials()
+
+        self.aws_access_key_id = creds.access_key if not creds.secret_key else aws_access_key_id
+        self.aws_secret_access_key = creds.access_key if not creds.access_key else aws_secret_access_key
         self.region_name = region_name
         self.table_name = table_name
         self.encoder = encoder

terraform/main.tf

@@ -13,6 +13,7 @@ provider "aws" {
 resource "aws_instance" "project_ec2" {
   ami           = "ami-0b93ce03dcbcb10f6"
   instance_type = "t2.micro"
+  iam_instance_profile = aws_iam_instance_profile.scrapy_profile.name
   vpc_security_group_ids = ["${aws_security_group.project_sg.id}"]
   key_name = "terraform_ec2_key"
 
@@ -23,19 +24,19 @@ resource "aws_instance" "project_ec2" {
 apt-get update
 apt-get install -y python3-pip git curl
 
-# Replace this with our script and requirements.txt
-git clone https://github.com/TanZng/ot6-cloud.git
-pip3 install -r requirements.txt
+# Create directory
+mkdir -p /opt/project
 
-# Copy main script
-mkdir -p /opt/app
-cp final_project_scraper.py /opt/app/
+# Clone scraper and install requirements.txt
+git clone [email protected]:TanZng/ot6-cloud.git /opt/project
+cd /opt/project
+pip3 install -r requirements.txt
 
-# Create a cron job to run the main.py script every minute
-# TODO: change how often run the script
+# !!! Create a cron job to run the main.py script every minute
+# !!! TODO: change how often run the script
 sudo service cron start
 
-echo "* * * * * /usr/bin/python3 /opt/app/final_project_scraper.py >> /scrapy.log" | crontab
+echo "* * * * * /opt/project/crawl.sh >> > $HOME/project-`date +\%Y\%m\%d\%H\%M\%S`-cron.log 2>&1" | crontab
 
 EOF
 
@@ -82,8 +83,6 @@ resource "aws_s3_bucket" "project_s3" {
   bucket_prefix = "project-s3"
 }
 
-resource "random_uuid" "uuid" {}
-
 # Create DynamoDB table 
 resource "aws_dynamodb_table" "project-dynamodb" {
   name           = "project-dynamodb"
@@ -96,4 +95,43 @@ resource "aws_dynamodb_table" "project-dynamodb" {
     name = "ProductId"
     type = "S"
   }
+}
+
+# Create an instance profile
+resource "aws_iam_instance_profile" "scrapy_profile" {
+  name = "scrapy_profile"
+  role = aws_iam_role.ec2_role.name
+}
+
+# Create a role that can be assumed by an EC2 instance
+resource "aws_iam_role" "ec2_role" {
+  name = "ec2_role"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+# Attach policies to the role
+resource "aws_iam_role_policy_attachment" "role-policy-attachment" {
+  for_each = toset([
+    "arn:aws:iam::aws:policy/AmazonEC2FullAccess", 
+    "arn:aws:iam::aws:policy/AmazonS3FullAccess",
+    "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess"
+  ])
+
+  role       = aws_iam_role.ec2_role.name
+  policy_arn = each.value
 }