Add logger for scrapy

TanZng · Jan 12, 2023 · a4346c9 · a4346c9
1 parent 94edb4c
commit a4346c9
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 2 deletions.
diff --git a/crawl.sh b/crawl.sh
@@ -2,4 +2,4 @@
 # go to the spider directory
 cd /opt/repo/project
 # run the spider
-/usr/local/bin/scrapy crawl product -a location=$1
+/usr/local/bin/scrapy crawl product --logfile $2 -a location=$1
diff --git a/project/project/items.py b/project/project/items.py
@@ -22,3 +22,4 @@ class ProjectItem(Item):
     rating_value= Field()
     rating_count= Field()
     sold_by = Field()
+    location = Field()
diff --git a/project/project/spiders/product.py b/project/project/spiders/product.py
@@ -22,6 +22,7 @@
 }
 
 class ProductSpider(scrapy.Spider):
+    name = 'product'
 
     def __init__(self, **kwargs):
         self.name = 'product'

diff --git a/terraform/main.tf b/terraform/main.tf
@@ -36,7 +36,7 @@ pip3 install -r requirements.txt
 # Create a cron job to run the scraper at 12am and 12pm
 sudo service cron start
 
-echo "0 */12 * * * (/bin/bash /opt/repo/crawl.sh Paris 2>&1) > /var/log/cron/spider_log.log" | crontab
+echo "0 */12 * * * (/bin/bash /opt/repo/crawl.sh Paris /var/log/cron/spider_log.log)" | crontab
 
 EOF