Merge pull request bluesentry#107 from Georepublic/for_python3

Python3 Support

.circleci/config.yml

@@ -15,7 +15,7 @@ executors:
       - image: docker:17.05.0-ce
   python_test:
     docker:
-      - image: circleci/python:2.7-stretch
+      - image: circleci/python:3.7-stretch
   pre_commit_test:
     docker:
       - image: circleci/python:3.7-stretch

Dockerfile

@@ -12,11 +12,11 @@ COPY requirements.txt /opt/app/requirements.txt
 
 # Install packages
 RUN yum update -y
-RUN yum install -y cpio python2-pip yum-utils zip unzip less
+RUN yum install -y cpio python3-pip yum-utils zip unzip less
 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 
 # This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
-RUN pip install -r requirements.txt
+RUN pip3 install -r requirements.txt
 RUN rm -rf /root/.cache/pip
 
 # Download libraries we need to run in lambda
@@ -39,7 +39,7 @@ RUN echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
 WORKDIR /opt/app
 RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin
 
-WORKDIR /usr/lib/python2.7/site-packages
+WORKDIR /usr/local/lib/python3.7/site-packages
 RUN zip -r9 /opt/app/build/lambda.zip *
 
 WORKDIR /opt/app

README.md

@@ -78,7 +78,7 @@ this every 3 hours to stay protected from the latest threats.
 3. Choose **Author from scratch** on the *Create function* page
 4. Name your function `bucket-antivirus-update` when prompted on the
 *Configure function* step.
-5. Set *Runtime* to `Python 2.7`
+5. Set *Runtime* to `Python 3.7`
 6. Create a new role name `bucket-antivirus-update` that uses the
 following policy document
 
@@ -144,7 +144,7 @@ the default provided.
 2. From the AWS Lambda Dashboard, click **Create function**
 3. Choose **Author from scratch** on the *Create function* page
 4. Name your function `bucket-antivirus-function`
-5. Set *Runtime* to `Python 2.7`
+5. Set *Runtime* to `Python 3.7`
 6. Create a new role name `bucket-antivirus-function` that uses the
 following policy document
 
@@ -363,6 +363,7 @@ The python tests in this repository use `unittest` and are run via the `nose` ut
 to install the developer resources and then run the tests:
 
 ```sh
+pip install -r requirements.txt
 pip install -r requirements-dev.txt
 make test
 ```

clamav.py

@@ -42,7 +42,7 @@
 
 
 def current_library_search_path():
-    ld_verbose = subprocess.check_output(["ld", "--verbose"])
+    ld_verbose = subprocess.check_output(["ld", "--verbose"]).decode("utf-8")
     rd_ld = re.compile(RE_SEARCH_DIR)
     return rd_ld.findall(ld_verbose)
 
@@ -194,7 +194,7 @@ def scan_file(path):
         stdout=subprocess.PIPE,
         env=av_env,
     )
-    output = av_proc.communicate()[0]
+    output = av_proc.communicate()[0].decode()
     print("clamscan output:\n%s" % output)
 
     # Turn the output into a data source we can read

scan.py

@@ -16,7 +16,7 @@
 import copy
 import json
 import os
-import urllib
+from urllib.parse import unquote_plus
 from distutils.util import strtobool
 
 import boto3
@@ -66,7 +66,7 @@ def event_object(event, event_source="s3"):
     key_name = s3_obj["object"].get("key", None)
 
     if key_name:
-        key_name = urllib.unquote_plus(key_name.encode("utf8"))
+        key_name = unquote_plus(key_name)
 
     # Ensure both bucket and key exist
     if (not bucket_name) or (not key_name):

scan_test.py

@@ -85,28 +85,28 @@ def test_s3_event_object_missing_bucket(self):
         event = {"Records": [{"s3": {"object": {"key": self.s3_key_name}}}]}
         with self.assertRaises(Exception) as cm:
             event_object(event)
-        self.assertEquals(cm.exception.message, "No bucket found in event!")
+            self.assertEquals(cm.exception.message, "No bucket found in event!")
 
     def test_s3_event_object_missing_key(self):
         event = {"Records": [{"s3": {"bucket": {"name": self.s3_bucket_name}}}]}
         with self.assertRaises(Exception) as cm:
             event_object(event)
-        self.assertEquals(cm.exception.message, "No key found in event!")
+            self.assertEquals(cm.exception.message, "No key found in event!")
 
     def test_s3_event_object_bucket_key_missing(self):
         event = {"Records": [{"s3": {"bucket": {}, "object": {}}}]}
         with self.assertRaises(Exception) as cm:
             event_object(event)
-        self.assertEquals(
-            cm.exception.message,
-            "Unable to retrieve object from event.\n{}".format(event),
-        )
+            self.assertEquals(
+                cm.exception.message,
+                "Unable to retrieve object from event.\n{}".format(event),
+            )
 
     def test_s3_event_object_no_records(self):
         event = {"Records": []}
         with self.assertRaises(Exception) as cm:
             event_object(event)
-        self.assertEquals(cm.exception.message, "No records found in event!")
+            self.assertEquals(cm.exception.message, "No records found in event!")
 
     def test_verify_s3_object_version(self):
         s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
@@ -165,10 +165,12 @@ def test_verify_s3_object_versioning_not_enabled(self):
         with self.assertRaises(Exception) as cm:
             with s3_stubber_resource:
                 verify_s3_object_version(self.s3, s3_obj)
-        self.assertEquals(
-            cm.exception.message,
-            "Object versioning is not enabled in bucket {}".format(self.s3_bucket_name),
-        )
+            self.assertEquals(
+                cm.exception.message,
+                "Object versioning is not enabled in bucket {}".format(
+                    self.s3_bucket_name
+                ),
+            )
 
     def test_verify_s3_object_version_multiple_versions(self):
         s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
@@ -218,12 +220,12 @@ def test_verify_s3_object_version_multiple_versions(self):
         with self.assertRaises(Exception) as cm:
             with s3_stubber_resource:
                 verify_s3_object_version(self.s3, s3_obj)
-        self.assertEquals(
-            cm.exception.message,
-            "Detected multiple object versions in {}.{}, aborting processing".format(
-                self.s3_bucket_name, self.s3_key_name
-            ),
-        )
+            self.assertEquals(
+                cm.exception.message,
+                "Detected multiple object versions in {}.{}, aborting processing".format(
+                    self.s3_bucket_name, self.s3_key_name
+                ),
+            )
 
     def test_sns_start_scan(self):
         sns_stubber = Stubber(self.sns_client)
@@ -422,9 +424,9 @@ def test_delete_s3_object_exception(self):
             with s3_stubber:
                 s3_obj = self.s3.Object(self.s3_bucket_name, self.s3_key_name)
                 delete_s3_object(s3_obj)
-        self.assertEquals(
-            cm.exception.message,
-            "Failed to delete infected file: {}.{}".format(
-                self.s3_bucket_name, self.s3_key_name
-            ),
-        )
+            self.assertEquals(
+                cm.exception.message,
+                "Failed to delete infected file: {}.{}".format(
+                    self.s3_bucket_name, self.s3_key_name
+                ),
+            )

scripts/run-scan-lambda

@@ -49,4 +49,4 @@ docker run --rm \
   --memory-swap="${MEM}" \
   --cpus="${CPUS}" \
   --name="${NAME}" \
-  lambci/lambda:python2.7 scan.lambda_handler "${EVENT}"
+  lambci/lambda:python3.7 scan.lambda_handler "${EVENT}"

scripts/run-update-lambda

@@ -26,4 +26,4 @@ docker run --rm \
   --memory-swap="${MEM}" \
   --cpus="${CPUS}" \
   --name="${NAME}" \
-  lambci/lambda:python2.7 update.lambda_handler
+  lambci/lambda:python3.7 update.lambda_handler