Merge pull request mozilla-services#2872 from peterbe/bug-1177131-sto…

…re-symbols-with-gzip-compression

fixes bug 1177131 - Store symbols with gzip compression

webapp-django/crashstats/settings/base.py

@@ -412,6 +412,11 @@ def COMPRESS_JINJA2_GET_ENVIRONMENT():
 SYMBOLS_MIME_OVERRIDES = {
     'sym': 'text/plain'
 }
+SYMBOLS_COMPRESS_EXTENSIONS = config(
+    'SYMBOLS_COMPRESS_EXTENSIONS',
+    'sym',
+    cast=Csv()
+)
 
 # ------------------------------------------------
 # Below are settings that can be overridden using

webapp-django/crashstats/symbols/tests/test_views.py

@@ -50,11 +50,17 @@ def setUp(self):
 
         self.patcher = mock.patch('crashstats.symbols.views.boto.connect_s3')
         self.uploaded_keys = {}
+        self.uploaded_headers = {}
         self.known_bucket_keys = {}
         self.created_buckets = []
         self.created_keys = []
         mocked_connect_s3 = self.patcher.start()
 
+        self.symbols_compress_extensions = settings.SYMBOLS_COMPRESS_EXTENSIONS
+        # Forcibly set this for all tests so it doesn't matter what's in
+        # settings/base.py or any local overrides
+        settings.SYMBOLS_COMPRESS_EXTENSIONS = ('sym',)
+
         def mocked_get_bucket(*a, **k):
             raise boto.exception.S3ResponseError(404, "Not found")
 
@@ -63,8 +69,9 @@ def mocked_create_bucket(name, location):
             def mocked_new_key(key_name):
                 mocked_key = mock.Mock()
 
-                def mocked_set(string):
+                def mocked_set(string, headers=None):
                     self.uploaded_keys[key_name] = string
+                    self.uploaded_headers[key_name] = headers
                     return len(string)
 
                 mocked_key.set_contents_from_string.side_effect = mocked_set
@@ -98,6 +105,7 @@ def tearDown(self):
         super(TestViews, self).tearDown()
         shutil.rmtree(self.tmp_dir)
         self.patcher.stop()
+        settings.SYMBOLS_COMPRESS_EXTENSIONS = self.symbols_compress_extensions
 
     def _login(self):
         user = User.objects.create_user('test', '[email protected]', 'secret')
@@ -230,6 +238,11 @@ def test_web_upload(self):
             settings.SYMBOLS_FILE_PREFIX,
             'south-africa-flag.jpeg'
         )
+        line += "+%s,%s/%s\n" % (
+            settings.SYMBOLS_BUCKET_DEFAULT_NAME,
+            settings.SYMBOLS_FILE_PREFIX,
+            'xpcshell.sym'
+        )
         eq_(symbol_upload.content, line)
         eq_(symbol_upload.content_type, 'text/plain')
         ok_(self.uploaded_keys)
@@ -267,6 +280,11 @@ def test_web_upload_different_bucket_by_wildcard(self):
                 settings.SYMBOLS_FILE_PREFIX,
                 'south-africa-flag.jpeg'
             )
+            line += "+%s,%s/%s\n" % (
+                'my-special-bucket-name',
+                settings.SYMBOLS_FILE_PREFIX,
+                'xpcshell.sym'
+            )
             eq_(symbol_upload.content, line)
             eq_(self.created_buckets, [
                 (
@@ -360,6 +378,11 @@ def test_web_upload_different_bucket_by_user_different_location(self):
                 settings.SYMBOLS_FILE_PREFIX,
                 'south-africa-flag.jpeg'
             )
+            line += "+%s,%s/%s\n" % (
+                'my-special-bucket-name',
+                settings.SYMBOLS_FILE_PREFIX,
+                'xpcshell.sym'
+            )
             eq_(symbol_upload.content, line)
             eq_(self.created_buckets, [
                 (
@@ -401,6 +424,11 @@ def test_upload_different_bucket_by_user_different_location(self):
                 settings.SYMBOLS_FILE_PREFIX,
                 'south-africa-flag.jpeg'
             )
+            line += "+%s,%s/%s\n" % (
+                'my-special-bucket-name',
+                settings.SYMBOLS_FILE_PREFIX,
+                'xpcshell.sym'
+            )
             eq_(symbol_upload.content, line)
             eq_(self.created_buckets, [
                 (
@@ -434,8 +462,17 @@ def test_web_upload_existing_upload(self):
             settings.SYMBOLS_FILE_PREFIX,
             'south-africa-flag.jpeg'
         )
+        line += "+%s,%s/%s\n" % (
+            settings.SYMBOLS_BUCKET_DEFAULT_NAME,
+            settings.SYMBOLS_FILE_PREFIX,
+            'xpcshell.sym'
+        )
         eq_(symbol_upload.content, line)
-        ok_(not self.uploaded_keys)  # nothing was uploaded
+        # only the xpcshell.sym was uploaded
+        eq_(
+            self.uploaded_keys.keys(),
+            ['%s/xpcshell.sym' % (settings.SYMBOLS_FILE_PREFIX,)]
+        )
 
     def test_web_upload_existing_upload_but_different_size(self):
         """what if the file already is uploaded"""
@@ -463,6 +500,11 @@ def test_web_upload_existing_upload_but_different_size(self):
             settings.SYMBOLS_FILE_PREFIX,
             'south-africa-flag.jpeg'
         )
+        line += "+%s,%s/%s\n" % (
+            settings.SYMBOLS_BUCKET_DEFAULT_NAME,
+            settings.SYMBOLS_FILE_PREFIX,
+            'xpcshell.sym'
+        )
         eq_(symbol_upload.content, line)
         ok_(key_name in self.uploaded_keys)
 
@@ -662,11 +704,27 @@ def test_upload(self):
                 ok_(symbol_upload.content)
 
         # the ZIP_FILE contains a file called south-africa-flag.jpeg
-        expected_key = os.path.join(
+        key = os.path.join(
             settings.SYMBOLS_FILE_PREFIX,
             'south-africa-flag.jpeg'
         )
-        ok_(self.uploaded_keys[expected_key])
+        ok_(self.uploaded_keys[key])
+        eq_(self.uploaded_headers[key], {'Content-Type': 'image/jpeg'})
+
+        # and a file called xpcshell.sym
+        key = os.path.join(
+            settings.SYMBOLS_FILE_PREFIX,
+            'xpcshell.sym'
+        )
+        ok_(self.uploaded_keys[key])
+        eq_(self.uploaded_headers[key], {
+            'Content-Type': 'text/plain',
+            'Content-Encoding': 'gzip'
+        })
+        # The sample.zip file contains the file xpcshell.sym and it's
+        # 1156 bytes when un-archived. But the file is compressed so
+        # the bytes we upload is less
+        ok_(len(self.uploaded_keys[key]) < 1156, len(self.uploaded_keys[key]))
 
     def test_upload_without_multipart_file(self):
         user = User.objects.create(username='user')

webapp-django/crashstats/symbols/views.py

@@ -1,5 +1,6 @@
 import re
 import os
+import mimetypes
 from functools import wraps
 from cStringIO import StringIO
 from zipfile import BadZipfile
@@ -98,12 +99,27 @@ def unpack_and_upload(iterator, symbols_upload, bucket_name, bucket_location):
             file = StringIO()
             file.write(member.extractor().read())
 
+            content_type = mimetypes.guess_type(key_name)[0]  # default guess
             for ext in settings.SYMBOLS_MIME_OVERRIDES:
-                if key_name.endswith('.{0}'.format(ext)):
-                    key.content_type = settings.SYMBOLS_MIME_OVERRIDES[ext]
+                if key_name.lower().endswith('.{0}'.format(ext)):
+                    content_type = settings.SYMBOLS_MIME_OVERRIDES[ext]
+                    key.content_type = content_type
                     symbols_upload.content_type = key.content_type
 
-            uploaded = key.set_contents_from_string(file.getvalue())
+            compress = False
+            for ext in settings.SYMBOLS_COMPRESS_EXTENSIONS:
+                if key_name.lower().endswith('.{0}'.format(ext)):
+                    compress = True
+                    break
+            headers = {
+                'Content-Type': content_type,
+            }
+            if compress:
+                headers['Content-Encoding'] = 'gzip'
+                value = file.getvalue().encode('zlib')
+            else:
+                value = file.getvalue()
+            uploaded = key.set_contents_from_string(value, headers)
             total_uploaded += uploaded
 
         symbols_upload.content += '%s%s,%s\n' % (