fixes bug 876051 - all correlation signatures by query, r=rhelmer

docs/middleware.rst

@@ -42,6 +42,7 @@ New-style, documented services
     * `/util/versions_info/ <#versions-info>`_
 * `/crontabber_state/ <#crontabber-state>`_
 * `/correlations/ <#correlations>`_
+    * `/correlations/signatures/ <#correlation-signatures>`_
 
 Old-style, undocumented services
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1666,6 +1667,65 @@ keys but empty like this::
     }
 
 
+.. ############################################################################
+   Correlation Signatures API
+   ############################################################################
+
+Correlation Signatures
+----------------------
+
+Return all signatures that have correlations about specific search
+parameters
+
+API specifications
+^^^^^^^^^^^^^^^^^^
+
++----------------+--------------------------------------------------------------------------------------------------------------------------------------+
+| HTTP method    | GET                                                                                                                                  |
++----------------+--------------------------------------------------------------------------------------------------------------------------------------+
+| URL schema     | /correlations/signatures/(parameters)                                                                                                |
++----------------+--------------------------------------------------------------------------------------------------------------------------------------+
+| Full URL       | /correlations/signatures/report_type/(report_type)/product/(product)/version/(version)/platforms/(platforms)                         |
++----------------+--------------------------------------------------------------------------------------------------------------------------------------+
+| Example        | http://socorro-api/bpapi/correlations/signatures/report_type/core-counts/product/Firefox/version/24.0a1/platforms/Windows%20NT+Linux |
++----------------+--------------------------------------------------------------------------------------------------------------------------------------+
+
+
+Mandatory parameters
+^^^^^^^^^^^^^^^^^^^^
+
++----------------+------------------+-------------------+--------------------------------+
+| Name           | Type of value    | Default value     | Description                    |
++================+==================+===================+================================+
+| report\_type   | String           | None              | Eg. ``core-counts``            |
++----------------+------------------+-------------------+--------------------------------+
+| product        | String           | None              | Eg. ``Firefox``                |
++----------------+------------------+-------------------+--------------------------------+
+| version        | String           | None              | Eg. ``24.0a1``                 |
++----------------+------------------+-------------------+--------------------------------+
+| platforms      | List of strings  | None              | Eg. ``Mac%20OS%20X+Linux``     |
++----------------+------------------+-------------------+--------------------------------+
+
+
+Optional parameters
+^^^^^^^^^^^^^^^^^^^
+
+None
+
+Return value
+^^^^^^^^^^^^
+
+Returns a structure with the keys ``hits`` and ``total``::
+
+    {
+        "hits": [
+            "js::GCMarker::processMarkStackTop(js::SliceBudget&)",
+            "gfxSVGGlyphs::~gfxSVGGlyphs()",
+            "mozilla::layers::ImageContainer::GetCurrentSize()"
+        ],
+        "total": 3
+    }
+
 .. ############################################################################
    Report List API
    ############################################################################

scripts/config/webapiconfig.py.dist

@@ -200,6 +200,7 @@ servicesList.default = [
     crashes_daily.CrashesDaily,
     platforms.Platforms,
     crontabber_state.CrontabberState,
+    correlations.CorrelationsSignatures,
     correlations.Correlations,
 ]
 

socorro/external/http/correlations.py

@@ -23,6 +23,7 @@ def file_age(f):
 
 
 COUNT_REGEX = re.compile('\((\d+) crashes\)')
+SIGNATURE_LINE_START_REGEX = re.compile('\s{2}\w')
 
 
 class Correlations(object):
@@ -40,6 +41,21 @@ def get(self, **kwargs):
         ]
         params = external_common.parse_arguments(filters, kwargs)
 
+        content = self._get_content(params)
+
+        data = self._parse_content(
+            content,
+            params['platform'],
+            params['signature']
+        )
+        reason, count, load = data
+        return {
+            'reason': reason,
+            'count': count,
+            'load': '\n'.join(load),
+        }
+
+    def _get_content(self, params):
         if 'http' in self.config and 'correlations' in self.config.http:
             # new middleware!
             base_url = self.config.http.correlations.base_url
@@ -80,18 +96,7 @@ def get(self, **kwargs):
             if save_download:
                 with open(tmp_filepath, 'w') as f:
                     f.write(content)
-
-        data = self._parse_content(
-            content,
-            params['platform'],
-            params['signature']
-        )
-        reason, count, load = data
-        return {
-            'reason': reason,
-            'count': count,
-            'load': '\n'.join(load),
-        }
+        return content
 
     @staticmethod
     def _download(url_start):
@@ -142,3 +147,47 @@ def _parse_content(content, platform, signature):
                         load.append(line.strip())
 
         return reason, count, load
+
+
+class CorrelationsSignatures(Correlations):
+
+    def __init__(self, config):
+        self.config = config
+
+    def get(self, **kwargs):
+        filters = [
+            ('report_type', None, 'str'),
+            ('product', None, 'str'),
+            ('version', None, 'str'),
+            ('platforms', None, 'list'),
+        ]
+
+        params = external_common.parse_arguments(filters, kwargs)
+        content = self._get_content(params)
+        signatures = self._parse_signatures(content, params['platforms'])
+
+        return {
+            'hits': signatures,
+            'total': len(signatures),
+        }
+
+    @staticmethod
+    def _parse_signatures(content, platforms):
+        """return a list of signatures that these platforms mention"""
+        signatures = []
+
+        on = False
+        for line in content.splitlines():
+            if line and not line.startswith(' '):
+                # change of platform
+                if line in platforms:
+                    on = True
+                else:
+                    on = False
+            elif on:
+                # if starts with exactly two spaces it contains the signature
+                if SIGNATURE_LINE_START_REGEX.match(line):
+                    this_signature = line.split('|')[-2].strip()
+                    signatures.append(this_signature)
+
+        return signatures

socorro/middleware/correlations_service.py

@@ -23,3 +23,19 @@ def get(self, *args):
         module = self.get_module(params)
         impl = module.Correlations(config=self.context)
         return impl.get(**params)
+
+
+class CorrelationsSignatures(DataAPIService):
+    """return correlations for a specific search"""
+
+    service_name = "correlations"
+    uri = "/correlations/signatures/(.*)"
+
+    def __init__(self, config):
+        super(CorrelationsSignatures, self).__init__(config)
+
+    def get(self, *args):
+        params = self.parse_query_string(args[0])
+        module = self.get_module(params)
+        impl = module.CorrelationsSignatures(config=self.context)
+        return impl.get(**params)

socorro/middleware/middleware_app.py

@@ -48,6 +48,7 @@
     (r'/report/(list)/(.*)', 'report.Report'),
     (r'/util/(versions_info)/(.*)', 'util.Util'),
     (r'/crontabber_state/(.*)', 'crontabber_state.CrontabberState'),
+    (r'/correlations/signatures/(.*)', 'correlations.CorrelationsSignatures'),
     (r'/correlations/(.*)', 'correlations.Correlations'),
 )
 
@@ -125,7 +126,9 @@ class MiddlewareApp(App):
     required_config.implementations.add_option(
         'service_overrides',
         doc='comma separated list of class overrides, e.g `Crashes: hbase`',
-        default='CrashData: fs, Correlations: http',  # e.g. 'Crashes: es',
+        default='CrashData: fs, '
+                'Correlations: http, '
+                'CorrelationsSignatures: http',
         from_string_converter=items_list_converter
     )
 

socorro/unittest/external/http/test_correlations.py

@@ -246,3 +246,72 @@ def mocked_get(url, **kwargs):
 
         finally:
             shutil.rmtree(tmp_directory)
+
+
+class TestCorrelationsSignatures(unittest.TestCase):
+
+    @staticmethod
+    def _get_model(overrides=None):
+        config_values = {
+            'base_url': 'http://crashanalysis.com',
+            'save_root': '',
+            'save_download': False,
+            'save_seconds': 1000,
+        }
+        if overrides:
+            config_values.update(overrides)
+        cls = correlations.CorrelationsSignatures
+        config = DotDict()
+        config.http = DotDict()
+        config.http.correlations = DotDict(config_values)
+        return cls(config)
+
+    @mock.patch('requests.get')
+    def test_simple_download(self, rget):
+
+        def mocked_get(url, **kwargs):
+            if 'core-counts' in url:
+                return Response(SAMPLE_CORE_COUNTS)
+            raise NotImplementedError
+
+        rget.side_effect = mocked_get
+
+        model = self._get_model()
+
+        params = {
+            'product': 'Firefox',
+            'report_type': 'core-counts',
+            'version': '24.0a1',
+        }
+        result = model.get(**dict(params, platforms=['Mac OS X', 'Linux']))
+        assert result['total']
+        self.assertEqual(result['total'], 9)
+        # belongs to Mac OS X
+        self.assertTrue(
+            'JS_HasPropertyById(JSContext*, JSObject*, long, int*)'
+            in result['hits']
+        )
+        # belongs to Linux
+        self.assertTrue('js::types::IdToTypeId(long)' in result['hits'])
+        # belongs to Windows NT
+        self.assertTrue('js::types::IdToTypeId(int)' not in result['hits'])
+
+    @mock.patch('requests.get')
+    def test_no_signatures(self, rget):
+
+        def mocked_get(url, **kwargs):
+            if 'core-counts' in url:
+                return Response(SAMPLE_CORE_COUNTS)
+            raise NotImplementedError
+
+        rget.side_effect = mocked_get
+
+        model = self._get_model()
+
+        params = {
+            'product': 'Firefox',
+            'report_type': 'core-counts',
+            'version': '24.0a1',
+        }
+        result = model.get(**dict(params, platforms=['OS/2']))
+        self.assertEqual(result['total'], 0)