fixes bug 1262252 - Crash data pure implementation (mozilla-services#…

…3452)

* fixes bug 1262252 - Crash data pure implementation

* mocking on old tests

* test that was not important

* leftover comment

* leftover comment

* import cleanup

* fixing test as per new wording

* dont test crash_data through middleware in integration tests

* remove all use of middleware in integration-test.sh

* review nits

* remove unncessary rabbitmq connection

* nit fixes

* unset bad reference_value_from

scripts/integration-test.sh

@@ -131,7 +131,7 @@ function cleanup() {
   echo "INFO: Terminating background jobs"
   echo "  any kill usage errors below may be ignored"
 
-  for p in collector processor middleware
+  for p in collector processor
   do
     # destroy any running processes started by this shell
     kill $(jobs -p) > /dev/null 2>&1
@@ -206,7 +206,7 @@ function retry() {
 }
 
 #------------------------------------------------------------------------------
-# setup and run the collector, processor and middleware
+# setup and run the collector and processor
 # The collector should be configured using the 2015 method of having the
 # ability to collect multiple crash types using different end points.
 #    breakpad crashes on /submit
@@ -255,30 +255,13 @@ function start_2015_socorro_apps() {
       > processor.log 2>&1 &
   echo '   processor started'
 
-  sleep 1
-  socorro middleware \
-      --admin.conf=./config/middleware.ini \
-      --database.database_hostname=$database_hostname \
-      --database.database_username=$database_username \
-      --database.database_password=$database_password \
-      --rabbitmq.host=$rmq_host \
-      --rabbitmq.rabbitmq_user=$rmq_user \
-      --rabbitmq.rabbitmq_password=$rmq_password \
-      --rabbitmq.virtual_host=$rmq_virtual_host \
-      --rabbitmq.standard_queue_name=$rmq_normal_queue_name \
-      --rabbitmq.priority_queue_name=$rmq_priority_queue_name \
-      --rabbitmq.reprocessing_queue_name=$rmq_reprocessing_queue_name \
-      --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy \
-      > middleware.log 2>&1 &
-  echo '   middleware started'
-
   # tell the test routine to use the extra submission test
   extra_submission_test=1
   echo " Done."
 }
 
 #------------------------------------------------------------------------------
-# setup and run the collector, processor and middleware
+# setup and run the collector and processor
 # The collector will use the traditional wsgi function that can only receive
 # breakpad crashes on the endpoint /submit
 #------------------------------------------------------------------------------
@@ -320,20 +303,6 @@ function start_standard_socorro_apps() {
       --processor.processor_class=socorro.processor.mozilla_processor_2015.MozillaProcessorAlgorithm2015 \
       > processor.log 2>&1 &
   sleep 1
-  socorro middleware \
-      --admin.conf=./config/middleware.ini \
-      --database.database_hostname=$database_hostname \
-      --database.database_username=$database_username \
-      --database.database_password=$database_password \
-      --rabbitmq.host=$rmq_host \
-      --rabbitmq.rabbitmq_user=$rmq_user \
-      --rabbitmq.rabbitmq_password=$rmq_password \
-      --rabbitmq.virtual_host=$rmq_virtual_host \
-      --rabbitmq.standard_queue_name=$rmq_normal_queue_name \
-      --rabbitmq.priority_queue_name=$rmq_priority_queue_name \
-      --rabbitmq.reprocessing_queue_name=$rmq_reprocessing_queue_name \
-      --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy \
-      > middleware.log 2>&1 &
 
   # tell the test routine NOT to use the extra submission test
   extra_submission_test=0
@@ -342,7 +311,7 @@ function start_standard_socorro_apps() {
 }
 
 #------------------------------------------------------------------------------
-# setup and run the collector, processor and middleware WITHOUT RabbitMQ
+# setup and run the collector and processor WITHOUT RabbitMQ
 # The collector will use the traditional wsgi function that can only receive
 # breakpad crashes on the endpoint /submit
 # The collector saves in
@@ -368,21 +337,6 @@ function start_minimal_socorro_apps() {
       --destination.fs_root=./processedCrashStore \
       > processor.log 2>&1 &
   sleep 1
-  socorro middleware \
-      --admin.conf=./config/middleware.ini \
-      --database.database_hostname=$database_hostname \
-      --database.database_username=$database_username \
-      --database.database_password=$database_password \
-      --filesystem.fs_root=./processedCrashStore \
-      --rabbitmq.host=$rmq_host \
-      --rabbitmq.rabbitmq_user=$rmq_user \
-      --rabbitmq.rabbitmq_password=$rmq_password \
-      --rabbitmq.virtual_host=$rmq_virtual_host \
-      --rabbitmq.standard_queue_name=$rmq_normal_queue_name \
-      --rabbitmq.priority_queue_name=$rmq_priority_queue_name \
-      --rabbitmq.reprocessing_queue_name=$rmq_reprocessing_queue_name \
-      --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy \
-      > middleware.log 2>&1 &
   # tell the test routine NOT to use the extra submission test
   extra_submission_test=0
 
@@ -423,7 +377,7 @@ echo " Done."
 
 #******************************************************************************
 # Here's where we actually start testing
-# Iterate through some combinations of collector/crashmover/processor/middleware/setups
+# Iterate through some combinations of collector/crashmover/processor/setups
 # These setups are defined in functions with their names list in the for loop:
 for an_app_set in start_2015_socorro_apps start_standard_socorro_apps start_minimal_socorro_apps
 do
@@ -466,28 +420,6 @@ do
   retry 'collector' "$CRASHID"
   retry 'processor' "saved - $CRASHID"
 
-  #----------------------------------------------------------------------------
-  # check that mware has raw crash using curl to hit the HTTP endpoint
-  curl -s -D middleware_headers.log "http://localhost:8883/crash_data/?datatype=meta&uuid=$CRASHID" > /dev/null
-  err=$?
-  echo "  looking for errors in hitting the middleware for $CRASHID"
-  check_for_logged_fatal_errors $err middleware
-
-  echo "  looking for "200 OK" in hitting the middleware for $CRASHID"
-  grep '200 OK' middleware_headers.log > /dev/null
-  fatal $? "middleware test failed, no raw data for crash ID $CRASHID"
-
-  echo "  looking for processed crash through middleware for $CRASHID"
-  function find_crash_in_middleware() {
-    curl -s "http://localhost:8883/crash_data/?datatype=processed&uuid=$CRASHID" | grep date_processed
-    echo "http://localhost:8883/crash_data/?datatype=processed&uuid=$CRASHID"
-    return $?
-  }
-  retry_command middleware find_crash_in_middleware
-
-  # check that mware logs the request for the crash, and logs no errors
-  retry 'middleware' "/crash_data"
-
   #----------------------------------------------------------------------------
   # EXTRA submission test
   if [ $extra_submission_test = 1 ]

socorro/external/boto/crash_data.py

@@ -2,9 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# this is a temporary hack to coerse the middleware to talk to boto S3
-# instead of HBase.
-
+from socorrolib.lib import external_common, MissingArgumentError
+from socorro.external.boto.crashstorage import (
+    BotoS3CrashStorage,
+    CrashIDNotFound,
+)
 from socorro.external.crash_data_base import CrashDataBase
 
 
@@ -24,3 +26,53 @@ def get_storage(self):
         # implementation details with boto S3.
         return self.config.hbase.hbase_class(self.config.hbase)
 
+
+class SimplifiedCrashData(BotoS3CrashStorage):
+    """The difference between this and the base CrashData class is that
+    this one only makes the get() and if it fails it does NOT
+    try to put the crash ID back into the priority jobs queue.
+    Also, it returns a python dict instead of a DotDict which
+    makes this easier to work with from the webapp's model bridge.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(SimplifiedCrashData, self).__init__(*args, **kwargs)
+        # Forcibly set this to override the default in the base
+        # crash storage class for boto. We're confident that at this
+        # leaf point we want to NOT return a DotDict but just a plain
+        # python dict.
+        self.config.json_object_hook = dict
+
+    def get(self, **kwargs):
+        """Return JSON data of a crash report, given its uuid. """
+        filters = [
+            ('uuid', None, str),
+            ('datatype', None, str),
+            ('name', None, str)  # only applicable if datatype == 'raw'
+        ]
+        params = external_common.parse_arguments(filters, kwargs, modern=True)
+
+        if not params.uuid:
+            raise MissingArgumentError('uuid')
+
+        if not params.datatype:
+            raise MissingArgumentError('datatype')
+
+        datatype_method_mapping = {
+            'raw': 'get_raw_dump',
+            'meta': 'get_raw_crash',
+            'processed': 'get_processed',
+            'unredacted': 'get_unredacted_processed',
+        }
+        get = self.__getattribute__(datatype_method_mapping[params.datatype])
+        try:
+            if params.datatype == 'raw':
+                return get(params.uuid, name=params.name)
+            else:
+                return get(params.uuid)
+        except CrashIDNotFound:
+            # The CrashIDNotFound exception that happens inside the
+            # crashstorage is too revealing as exception message
+            # contains information about buckets and prefix keys.
+            # Re-wrap it here so the message is just the crash ID.
+            raise CrashIDNotFound(params.uuid)

socorro/external/boto/crashstorage.py

@@ -7,7 +7,6 @@
 
 import json_schema_reducer
 from socorrolib.lib.converters import change_default
-from socorrolib.lib.util import DotDict
 
 from configman import Namespace
 from configman.converters import class_converter, py_obj_to_str
@@ -70,6 +69,11 @@ class BotoCrashStorage(CrashStorageBase):
         default='.dump',
         reference_value_from='resource.boto',
     )
+    required_config.add_option(
+        'json_object_hook',
+        default='configman.dotdict.DotDict',
+        from_string_converter=class_converter,
+    )
 
     def is_operational_exception(self, x):
         if "not found, no value returned" in str(x):
@@ -174,20 +178,27 @@ def save_raw_and_processed(
         self.save_processed(processed_crash)
 
     @staticmethod
-    def do_get_raw_crash(boto_connection, crash_id):
+    def do_get_raw_crash(boto_connection, crash_id, json_object_hook):
         try:
             raw_crash_as_string = boto_connection.fetch(
                 crash_id,
                 "raw_crash"
             )
-            return json.loads(raw_crash_as_string, object_hook=DotDict)
+            return json.loads(
+                raw_crash_as_string,
+                object_hook=json_object_hook
+            )
         except boto_connection.ResponseError, x:
             raise CrashIDNotFound(
                 '%s not found: %s' % (crash_id, x)
             )
 
     def get_raw_crash(self, crash_id):
-        return self.transaction_for_get(self.do_get_raw_crash, crash_id)
+        return self.transaction_for_get(
+            self.do_get_raw_crash,
+            crash_id,
+            self.config.json_object_hook
+        )
 
     @staticmethod
     def do_get_raw_dump(boto_connection, crash_id, name=None):
@@ -244,15 +255,19 @@ def get_raw_dumps_as_files(self, crash_id):
         )
 
     @staticmethod
-    def _do_get_unredacted_processed(boto_connection, crash_id):
+    def _do_get_unredacted_processed(
+        boto_connection,
+        crash_id,
+        json_object_hook,
+    ):
         try:
             processed_crash_as_string = boto_connection.fetch(
                 crash_id,
                 "processed_crash"
             )
             return json.loads(
                 processed_crash_as_string,
-                object_hook=DotDict
+                object_hook=json_object_hook,
             )
         except boto_connection.ResponseError, x:
             raise CrashIDNotFound(
@@ -262,7 +277,8 @@ def _do_get_unredacted_processed(boto_connection, crash_id):
     def get_unredacted_processed(self, crash_id):
         return self.transaction_for_get(
             self._do_get_unredacted_processed,
-            crash_id
+            crash_id,
+            self.config.json_object_hook,
         )
 
 

socorro/external/crash_data_base.py

@@ -2,6 +2,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# XXX this is now deprecated and can be deleted.
+# See https://bugzilla.mozilla.org/show_bug.cgi?id=1299465
+
 from socorrolib.lib import (
     MissingArgumentError,
     ResourceNotFound,

socorro/external/rabbitmq/crashstorage.py

@@ -105,11 +105,13 @@ def __init__(self, config, quit_check_callback=None):
         if config.throttle == 100:
             self.dont_queue_this_crash = lambda: False
         else:
-            self.dont_queue_this_crash = lambda: randint(1, 100) > config.throttle
+            self.dont_queue_this_crash = (
+                lambda: randint(1, 100) > config.throttle
+            )
 
     #--------------------------------------------------------------------------
     def save_raw_crash(self, raw_crash, dumps, crash_id):
-        if  self.dont_queue_this_crash():
+        if self.dont_queue_this_crash():
             self.config.logger.info(
                 'Crash %s filtered out of RabbitMQ queue %s',
                 crash_id,
@@ -118,8 +120,7 @@ def save_raw_crash(self, raw_crash, dumps, crash_id):
             return
         try:
             this_crash_should_be_queued = (
-                (not self.config.filter_on_legacy_processing)
-                or
+                not self.config.filter_on_legacy_processing or
                 raw_crash.legacy_processing == 0
             )
         except KeyError:
@@ -170,7 +171,6 @@ def new_crashes(self):
         # queues the crash_id. The '_consume_acknowledgement_queue' function
         # is run to send acknowledgments back to RabbitMQ
         self._consume_acknowledgement_queue()
-        conn = self.rabbitmq.connection()
         queues = [
             self.rabbitmq.config.priority_queue_name,
             self.rabbitmq.config.standard_queue_name,
@@ -326,3 +326,31 @@ def reprocess(self, crash_ids):
             ):
                 success = False
         return success
+
+
+#==============================================================================
+class PriorityjobRabbitMQCrashStore(RabbitMQCrashStorage):
+    required_config = Namespace()
+    required_config.rabbitmq_class = change_default(
+        RabbitMQCrashStorage,
+        'rabbitmq_class',
+        ConnectionContext,
+    )
+    required_config.add_option(
+        'routing_key',
+        default='socorro.priority',
+        doc='the name of the queue to receive crashes',
+    )
+
+    def process(self, crash_ids):
+        if not isinstance(crash_ids, (list, tuple)):
+            crash_ids = [crash_ids]
+        success = bool(crash_ids)
+        for crash_id in crash_ids:
+            if not self.save_raw_crash(
+                DotDict({'legacy_processing': 0}),
+                [],
+                crash_id
+            ):
+                success = False
+        return success

socorro/external/rabbitmq/priorityjobs.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# XXX This file is probably not used anywhere and can be deleted.
+
 import pika
 from pika.exceptions import ChannelClosed
 

socorro/middleware/middleware_app.py

@@ -41,7 +41,6 @@
     (r'/backfill/(.*)', 'backfill.Backfill'),
     (r'/correlations/signatures/(.*)', 'correlations.CorrelationsSignatures'),
     (r'/correlations/(.*)', 'correlations.Correlations'),
-    (r'/crash_data/(.*)', 'crash_data.CrashData'),
     (
         r'/crashes/'
         r'(comments|count_by_day|daily|frequency|signatures|'
@@ -136,8 +135,7 @@ class MiddlewareApp(App):
     required_config.implementations.add_option(
         'service_overrides',
         doc='comma separated list of class overrides, e.g `Crashes: hbase`',
-        default='CrashData: fs, '
-                'Correlations: http, '
+        default='Correlations: http, '
                 'CorrelationsSignatures: http, '
                 'SuperSearch: es, '
                 'Priorityjobs: rabbitmq, '