From 3ea7b4f240917754cefd57839b155a86acada8f4 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Wed, 5 Jun 2013 16:35:13 -0700 Subject: [PATCH] fixes bug 876051 - all correlation signatures by query, r=rhelmer --- docs/middleware.rst | 60 +++++++++++++++ scripts/config/webapiconfig.py.dist | 1 + socorro/external/http/correlations.py | 73 ++++++++++++++++--- socorro/middleware/correlations_service.py | 16 ++++ socorro/middleware/middleware_app.py | 5 +- .../external/http/test_correlations.py | 69 ++++++++++++++++++ 6 files changed, 211 insertions(+), 13 deletions(-) diff --git a/docs/middleware.rst b/docs/middleware.rst index 3068670e89..9ff3951166 100644 --- a/docs/middleware.rst +++ b/docs/middleware.rst @@ -42,6 +42,7 @@ New-style, documented services * `/util/versions_info/ <#versions-info>`_ * `/crontabber_state/ <#crontabber-state>`_ * `/correlations/ <#correlations>`_ + * `/correlations/signatures/ <#correlation-signatures>`_ Old-style, undocumented services ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1666,6 +1667,65 @@ keys but empty like this:: } +.. ############################################################################ + Correlation Signatures API + ############################################################################ + +Correlation Signatures +---------------------- + +Return all signatures that have correlations about specific search +parameters + +API specifications +^^^^^^^^^^^^^^^^^^ + ++----------------+--------------------------------------------------------------------------------------------------------------------------------------+ +| HTTP method | GET | ++----------------+--------------------------------------------------------------------------------------------------------------------------------------+ +| URL schema | /correlations/signatures/(parameters) | ++----------------+--------------------------------------------------------------------------------------------------------------------------------------+ +| Full URL | /correlations/signatures/report_type/(report_type)/product/(product)/version/(version)/platforms/(platforms) | ++----------------+--------------------------------------------------------------------------------------------------------------------------------------+ +| Example | http://socorro-api/bpapi/correlations/signatures/report_type/core-counts/product/Firefox/version/24.0a1/platforms/Windows%20NT+Linux | ++----------------+--------------------------------------------------------------------------------------------------------------------------------------+ + + +Mandatory parameters +^^^^^^^^^^^^^^^^^^^^ + ++----------------+------------------+-------------------+--------------------------------+ +| Name | Type of value | Default value | Description | ++================+==================+===================+================================+ +| report\_type | String | None | Eg. ``core-counts`` | ++----------------+------------------+-------------------+--------------------------------+ +| product | String | None | Eg. ``Firefox`` | ++----------------+------------------+-------------------+--------------------------------+ +| version | String | None | Eg. ``24.0a1`` | ++----------------+------------------+-------------------+--------------------------------+ +| platforms | List of strings | None | Eg. ``Mac%20OS%20X+Linux`` | ++----------------+------------------+-------------------+--------------------------------+ + + +Optional parameters +^^^^^^^^^^^^^^^^^^^ + +None + +Return value +^^^^^^^^^^^^ + +Returns a structure with the keys ``hits`` and ``total``:: + + { + "hits": [ + "js::GCMarker::processMarkStackTop(js::SliceBudget&)", + "gfxSVGGlyphs::~gfxSVGGlyphs()", + "mozilla::layers::ImageContainer::GetCurrentSize()" + ], + "total": 3 + } + .. ############################################################################ Report List API ############################################################################ diff --git a/scripts/config/webapiconfig.py.dist b/scripts/config/webapiconfig.py.dist index 4de0b9eb3b..50a9fa4eea 100644 --- a/scripts/config/webapiconfig.py.dist +++ b/scripts/config/webapiconfig.py.dist @@ -200,6 +200,7 @@ servicesList.default = [ crashes_daily.CrashesDaily, platforms.Platforms, crontabber_state.CrontabberState, + correlations.CorrelationsSignatures, correlations.Correlations, ] diff --git a/socorro/external/http/correlations.py b/socorro/external/http/correlations.py index e54fe8bcf4..4a62848946 100644 --- a/socorro/external/http/correlations.py +++ b/socorro/external/http/correlations.py @@ -23,6 +23,7 @@ def file_age(f): COUNT_REGEX = re.compile('\((\d+) crashes\)') +SIGNATURE_LINE_START_REGEX = re.compile('\s{2}\w') class Correlations(object): @@ -40,6 +41,21 @@ def get(self, **kwargs): ] params = external_common.parse_arguments(filters, kwargs) + content = self._get_content(params) + + data = self._parse_content( + content, + params['platform'], + params['signature'] + ) + reason, count, load = data + return { + 'reason': reason, + 'count': count, + 'load': '\n'.join(load), + } + + def _get_content(self, params): if 'http' in self.config and 'correlations' in self.config.http: # new middleware! base_url = self.config.http.correlations.base_url @@ -80,18 +96,7 @@ def get(self, **kwargs): if save_download: with open(tmp_filepath, 'w') as f: f.write(content) - - data = self._parse_content( - content, - params['platform'], - params['signature'] - ) - reason, count, load = data - return { - 'reason': reason, - 'count': count, - 'load': '\n'.join(load), - } + return content @staticmethod def _download(url_start): @@ -142,3 +147,47 @@ def _parse_content(content, platform, signature): load.append(line.strip()) return reason, count, load + + +class CorrelationsSignatures(Correlations): + + def __init__(self, config): + self.config = config + + def get(self, **kwargs): + filters = [ + ('report_type', None, 'str'), + ('product', None, 'str'), + ('version', None, 'str'), + ('platforms', None, 'list'), + ] + + params = external_common.parse_arguments(filters, kwargs) + content = self._get_content(params) + signatures = self._parse_signatures(content, params['platforms']) + + return { + 'hits': signatures, + 'total': len(signatures), + } + + @staticmethod + def _parse_signatures(content, platforms): + """return a list of signatures that these platforms mention""" + signatures = [] + + on = False + for line in content.splitlines(): + if line and not line.startswith(' '): + # change of platform + if line in platforms: + on = True + else: + on = False + elif on: + # if starts with exactly two spaces it contains the signature + if SIGNATURE_LINE_START_REGEX.match(line): + this_signature = line.split('|')[-2].strip() + signatures.append(this_signature) + + return signatures diff --git a/socorro/middleware/correlations_service.py b/socorro/middleware/correlations_service.py index a65f3cba19..ee8a62f6ad 100644 --- a/socorro/middleware/correlations_service.py +++ b/socorro/middleware/correlations_service.py @@ -23,3 +23,19 @@ def get(self, *args): module = self.get_module(params) impl = module.Correlations(config=self.context) return impl.get(**params) + + +class CorrelationsSignatures(DataAPIService): + """return correlations for a specific search""" + + service_name = "correlations" + uri = "/correlations/signatures/(.*)" + + def __init__(self, config): + super(CorrelationsSignatures, self).__init__(config) + + def get(self, *args): + params = self.parse_query_string(args[0]) + module = self.get_module(params) + impl = module.CorrelationsSignatures(config=self.context) + return impl.get(**params) diff --git a/socorro/middleware/middleware_app.py b/socorro/middleware/middleware_app.py index 33c31231df..a8f7653bee 100644 --- a/socorro/middleware/middleware_app.py +++ b/socorro/middleware/middleware_app.py @@ -48,6 +48,7 @@ (r'/report/(list)/(.*)', 'report.Report'), (r'/util/(versions_info)/(.*)', 'util.Util'), (r'/crontabber_state/(.*)', 'crontabber_state.CrontabberState'), + (r'/correlations/signatures/(.*)', 'correlations.CorrelationsSignatures'), (r'/correlations/(.*)', 'correlations.Correlations'), ) @@ -125,7 +126,9 @@ class MiddlewareApp(App): required_config.implementations.add_option( 'service_overrides', doc='comma separated list of class overrides, e.g `Crashes: hbase`', - default='CrashData: fs, Correlations: http', # e.g. 'Crashes: es', + default='CrashData: fs, ' + 'Correlations: http, ' + 'CorrelationsSignatures: http', from_string_converter=items_list_converter ) diff --git a/socorro/unittest/external/http/test_correlations.py b/socorro/unittest/external/http/test_correlations.py index 5c3164d460..00a63fa048 100644 --- a/socorro/unittest/external/http/test_correlations.py +++ b/socorro/unittest/external/http/test_correlations.py @@ -246,3 +246,72 @@ def mocked_get(url, **kwargs): finally: shutil.rmtree(tmp_directory) + + +class TestCorrelationsSignatures(unittest.TestCase): + + @staticmethod + def _get_model(overrides=None): + config_values = { + 'base_url': 'http://crashanalysis.com', + 'save_root': '', + 'save_download': False, + 'save_seconds': 1000, + } + if overrides: + config_values.update(overrides) + cls = correlations.CorrelationsSignatures + config = DotDict() + config.http = DotDict() + config.http.correlations = DotDict(config_values) + return cls(config) + + @mock.patch('requests.get') + def test_simple_download(self, rget): + + def mocked_get(url, **kwargs): + if 'core-counts' in url: + return Response(SAMPLE_CORE_COUNTS) + raise NotImplementedError + + rget.side_effect = mocked_get + + model = self._get_model() + + params = { + 'product': 'Firefox', + 'report_type': 'core-counts', + 'version': '24.0a1', + } + result = model.get(**dict(params, platforms=['Mac OS X', 'Linux'])) + assert result['total'] + self.assertEqual(result['total'], 9) + # belongs to Mac OS X + self.assertTrue( + 'JS_HasPropertyById(JSContext*, JSObject*, long, int*)' + in result['hits'] + ) + # belongs to Linux + self.assertTrue('js::types::IdToTypeId(long)' in result['hits']) + # belongs to Windows NT + self.assertTrue('js::types::IdToTypeId(int)' not in result['hits']) + + @mock.patch('requests.get') + def test_no_signatures(self, rget): + + def mocked_get(url, **kwargs): + if 'core-counts' in url: + return Response(SAMPLE_CORE_COUNTS) + raise NotImplementedError + + rget.side_effect = mocked_get + + model = self._get_model() + + params = { + 'product': 'Firefox', + 'report_type': 'core-counts', + 'version': '24.0a1', + } + result = model.get(**dict(params, platforms=['OS/2'])) + self.assertEqual(result['total'], 0)