Merge branch 'master' into fix-12

README.md

@@ -44,10 +44,6 @@ Create an s3 bucket to store current antivirus definitions.  This
 provides the fastest download speeds for the scanner.  This bucket can
 be kept as private.  
 
-In this bucket, create a directory named clamav_defs (this is the default name of the directory, can be changed at the environment variables)
- copy main.cvd | daily.cvd | bytecode.cvd from [https://www.clamav.net/downloads](https://www.clamav.net/downloads)
- this is needed for the first run of the script.
-
 To allow public access, useful for other accounts,
 add the following policy to the bucket.
 

clamav.py

@@ -32,11 +32,13 @@ def update_defs_from_s3(bucket, prefix):
     for filename in AV_DEFINITION_FILENAMES:
         s3_path = os.path.join(AV_DEFINITION_S3_PREFIX, filename)
         local_path = os.path.join(AV_DEFINITION_PATH, filename)
-        if os.path.exists(local_path) and md5_from_file(local_path) == md5_from_s3_tags(bucket, s3_path):
+        s3_md5 = md5_from_s3_tags(bucket, s3_path)
+        if os.path.exists(local_path) and md5_from_file(local_path) == s3_md5:
             print("Not downloading %s because local md5 matches s3." % filename)
             continue
-        print("Downloading definition file %s from s3://%s" % (filename, os.path.join(bucket, prefix)))
-        s3.Bucket(bucket).download_file(s3_path, local_path)
+        if s3_md5:
+            print("Downloading definition file %s from s3://%s" % (filename, os.path.join(bucket, prefix)))
+            s3.Bucket(bucket).download_file(s3_path, local_path)
 
 
 def upload_defs_to_s3(bucket, prefix, local_path):
@@ -93,7 +95,8 @@ def md5_from_s3_tags(bucket, key):
     try:
         tags = s3_client.get_object_tagging(Bucket=bucket, Key=key)["TagSet"]
     except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] == "404":
+        expected_errors = {'404', 'AccessDenied'}
+        if e.response['Error']['Code'] in expected_errors:
             return ""
         else:
             raise

common.py

@@ -28,7 +28,7 @@
 CLAMSCAN_PATH = os.getenv("CLAMSCAN_PATH", "./bin/clamscan")
 FRESHCLAM_PATH = os.getenv("FRESHCLAM_PATH", "./bin/freshclam")
 
-AV_DEFINITION_FILENAMES = ["main.cvd", "daily.cvd", "bytecode.cvd"]
+AV_DEFINITION_FILENAMES = ["main.cvd","daily.cvd", "daily.cud", "bytecode.cvd", "bytecode.cud"]
 
 s3 = boto3.resource('s3')
 s3_client = boto3.client('s3')

update.py

@@ -15,6 +15,7 @@
 import clamav
 from common import *
 from datetime import datetime
+import os
 
 
 def lambda_handler(event, context):
@@ -23,6 +24,14 @@ def lambda_handler(event, context):
           (start_time.strftime("%Y/%m/%d %H:%M:%S UTC")))
     clamav.update_defs_from_s3(AV_DEFINITION_S3_BUCKET, AV_DEFINITION_S3_PREFIX)
     clamav.update_defs_from_freshclam(AV_DEFINITION_PATH, CLAMAVLIB_PATH)
+    # If main.cvd gets updated (very rare), we will need to force freshclam
+    # to download the compressed version to keep file sizes down.
+    # The existence of main.cud is the trigger to know this has happened.
+    if os.path.exists(os.path.join(AV_DEFINITION_PATH, "main.cud")):
+        os.remove(os.path.join(AV_DEFINITION_PATH, "main.cud"))
+        if os.path.exists(os.path.join(AV_DEFINITION_PATH, "main.cvd")):
+            os.remove(os.path.join(AV_DEFINITION_PATH, "main.cvd"))
+        clamav.update_defs_from_freshclam(AV_DEFINITION_PATH, CLAMAVLIB_PATH)
     clamav.upload_defs_to_s3(AV_DEFINITION_S3_BUCKET, AV_DEFINITION_S3_PREFIX, AV_DEFINITION_PATH)
     print("Script finished at %s\n" %
           datetime.utcnow().strftime("%Y/%m/%d %H:%M:%S UTC"))