From 2cbaeb132c48d800c32707dfe038132096c7ec4d Mon Sep 17 00:00:00 2001 From: Rob Smith Date: Wed, 31 Dec 2014 12:31:04 -0800 Subject: [PATCH] pep8 fixes --- src/collectors/amavis/test/testamavis.py | 1 + src/collectors/jolokia/cassandra_jolokia.py | 58 +++++++++++-------- .../jolokia/test/testcassandra_jolokia.py | 54 +++++++++++------ src/collectors/jolokia/test/testjolokia.py | 9 +-- src/collectors/pgbouncer/pgbouncer.py | 14 +++-- .../pgbouncer/test/testpgbouncer.py | 9 ++- src/collectors/postgres/postgres.py | 3 +- 7 files changed, 95 insertions(+), 53 deletions(-) diff --git a/src/collectors/amavis/test/testamavis.py b/src/collectors/amavis/test/testamavis.py index 34289cff3..b43dfb363 100644 --- a/src/collectors/amavis/test/testamavis.py +++ b/src/collectors/amavis/test/testamavis.py @@ -16,6 +16,7 @@ MOCK_PATH = os.path.join(os.path.dirname(__file__), 'mock-amavisd-agent') + class TestAmavisCollector(CollectorTestCase): def setUp(self): config = get_collector_config('AmavisCollector', { diff --git a/src/collectors/jolokia/cassandra_jolokia.py b/src/collectors/jolokia/cassandra_jolokia.py index 4a1c56036..6904ea123 100644 --- a/src/collectors/jolokia/cassandra_jolokia.py +++ b/src/collectors/jolokia/cassandra_jolokia.py @@ -1,14 +1,16 @@ # coding=utf-8 """ -Collects Cassandra JMX metrics from the Jolokia Agent. Extends the JolokiaCollector to -interpret Histogram beans with information about the distribution of request latencies. +Collects Cassandra JMX metrics from the Jolokia Agent. Extends the +JolokiaCollector to interpret Histogram beans with information about the +distribution of request latencies. #### Example Configuration -CassandraJolokiaCollector uses a regular expression to determine which attributes represent histograms. -This regex can be overridden by providing a `histogram_regex` in your configuration. You can also override -`percentiles` to collect specific percentiles from the histogram statistics. The format is shown below -with the default values. +CassandraJolokiaCollector uses a regular expression to determine which +attributes represent histograms. This regex can be overridden by providing a +`histogram_regex` in your configuration. You can also override `percentiles` to +collect specific percentiles from the histogram statistics. The format is shown +below with the default values. CassandraJolokiaCollector.conf @@ -27,10 +29,14 @@ class CassandraJolokiaCollector(JolokiaCollector): # override to allow setting which percentiles will be collected def get_default_config_help(self): - config_help = super(CassandraJolokiaCollector, self).get_default_config_help() + config_help = super(CassandraJolokiaCollector, + self).get_default_config_help() config_help.update({ - 'percentiles': 'Comma separated list of percentiles to be collected (e.g., "50,95,99").', - 'histogram_regex': 'Filter to only process attributes that match this regex' + 'percentiles': + 'Comma separated list of percentiles to be collected ' + '(e.g., "50,95,99").', + 'histogram_regex': + 'Filter to only process attributes that match this regex' }) return config_help @@ -49,33 +55,39 @@ def __init__(self, config, handlers): self.update_config(self.config) def update_config(self, config): - if config.has_key('percentiles'): - self.percentiles = map(int, string.split(config['percentiles'], ',')) - if config.has_key('histogram_regex'): + if 'percentiles' in config: + self.percentiles = map(int, string.split(config['percentiles'], + ',')) + if 'histogram_regex' in config: self.histogram_regex = re.compile(config['histogram_regex']) - # override: Interpret beans that match the `histogram_regex` as histograms, and collect - # percentiles from them. + # override: Interpret beans that match the `histogram_regex` as histograms, + # and collect percentiles from them. def interpret_bean_with_list(self, prefix, values): if not self.histogram_regex.match(prefix): return buckets = values for percentile in self.percentiles: - percentile_value = self.compute_percentile(self.offsets, buckets, percentile) + percentile_value = self.compute_percentile( + self.offsets, buckets, percentile) self.publish("%s.p%s" % (prefix, percentile), percentile_value) - # Adapted from Cassandra docs: http://www.datastax.com/documentation/cassandra/2.0/cassandra/tools/toolsCFhisto.html - # The index corresponds to the x-axis in a histogram. It represents buckets of values, which are - # a series of ranges. Each offset includes the range of values greater than the previous offset - # and less than or equal to the current offset. The offsets start at 1 and each subsequent offset - # is calculated by multiplying the previous offset by 1.2, rounding up, and removing duplicates. The - # offsets can range from 1 to approximately 25 million, with less precision as the offsets get larger. + # Adapted from Cassandra docs: + # http://www.datastax.com/documentation/cassandra/2.0/cassandra/tools/toolsCFhisto.html + # The index corresponds to the x-axis in a histogram. It represents buckets + # of values, which are a series of ranges. Each offset includes the range of + # values greater than the previous offset and less than or equal to the + # current offset. The offsets start at 1 and each subsequent offset is + # calculated by multiplying the previous offset by 1.2, rounding up, and + # removing duplicates. The offsets can range from 1 to approximately 25 + # million, with less precision as the offsets get larger. def compute_percentile(self, offsets, buckets, percentile_int): non_zero_points_sum = sum(buckets) if non_zero_points_sum is 0: return 0 - middle_point_index = math.floor(non_zero_points_sum * (percentile_int / float(100))) + middle_point_index = math.floor( + non_zero_points_sum * (percentile_int / float(100))) points_seen = 0 for index, bucket in enumerate(buckets): @@ -95,4 +107,4 @@ def create_offsets(self, bucket_count): offsets.append(next_num) last_num = next_num - return offsets \ No newline at end of file + return offsets diff --git a/src/collectors/jolokia/test/testcassandra_jolokia.py b/src/collectors/jolokia/test/testcassandra_jolokia.py index be25a1e20..9ddeddd59 100644 --- a/src/collectors/jolokia/test/testcassandra_jolokia.py +++ b/src/collectors/jolokia/test/testcassandra_jolokia.py @@ -23,10 +23,12 @@ def setUp(self): # Used for all the tests so the expected numbers are all the same. def fixture_values_a(self): - return [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,1,1,8,5,6,1,6,5,3,8,9,10,7,8,7,5,5,5,3,3,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + values = [0]*92 + values[30:56] = [3, 3, 1, 1, 8, 5, 6, 1, 6, 5, 3, 8, 9, 10, 7, 8, 7, 5, + 5, 5, 3, 3, 2, 2, 2] def empty_fixture_values(self): - return [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + return [0]*91 def expected_percentiles_for_fixture_a(self, percentile_key): return { @@ -42,27 +44,36 @@ def test_import(self): def test_should_compute_percentiles_accurately(self): ninety_offsets = self.collector.create_offsets(90) - percentile_value = self.collector.compute_percentile(ninety_offsets, self.fixture_values_a(), 50) + percentile_value = self.collector.compute_percentile( + ninety_offsets, self.fixture_values_a(), 50) self.assertEqual(percentile_value, 398.0) def test_should_compute_percentiles_accurately_when_empty(self): ninety_offsets = self.collector.create_offsets(90) - self.assertEqual(self.collector.compute_percentile(ninety_offsets, self.empty_fixture_values(), 50), 0.0) - self.assertEqual(self.collector.compute_percentile(ninety_offsets, self.empty_fixture_values(), 95), 0.0) - self.assertEqual(self.collector.compute_percentile(ninety_offsets, self.empty_fixture_values(), 99), 0.0) + self.assertEqual(self.collector.compute_percentile( + ninety_offsets, self.empty_fixture_values(), 50), 0.0) + self.assertEqual(self.collector.compute_percentile( + ninety_offsets, self.empty_fixture_values(), 95), 0.0) + self.assertEqual(self.collector.compute_percentile( + ninety_offsets, self.empty_fixture_values(), 99), 0.0) @patch.object(Collector, 'publish') def test_should_not_collect_non_histogram_attributes(self, publish_mock): - self.collector.interpret_bean_with_list('RecentReadLatencyMicros', self.fixture_values_a()) + self.collector.interpret_bean_with_list( + 'RecentReadLatencyMicros', self.fixture_values_a()) self.assertPublishedMany(publish_mock, {}) @patch.object(Collector, 'publish') def test_should_collect_metrics_histogram_attributes(self, publish_mock): - self.collector.interpret_bean_with_list('RecentReadLatencyHistogramMicros', self.fixture_values_a()) + self.collector.interpret_bean_with_list( + 'RecentReadLatencyHistogramMicros', self.fixture_values_a()) self.assertPublishedMany(publish_mock, { - 'RecentReadLatencyHistogramMicros.p50': self.expected_percentiles_for_fixture_a('p50'), - 'RecentReadLatencyHistogramMicros.p95': self.expected_percentiles_for_fixture_a('p95'), - 'RecentReadLatencyHistogramMicros.p99': self.expected_percentiles_for_fixture_a('p99') + 'RecentReadLatencyHistogramMicros.p50': + self.expected_percentiles_for_fixture_a('p50'), + 'RecentReadLatencyHistogramMicros.p95': + self.expected_percentiles_for_fixture_a('p95'), + 'RecentReadLatencyHistogramMicros.p99': + self.expected_percentiles_for_fixture_a('p99') }) @patch.object(Collector, 'publish') @@ -70,10 +81,13 @@ def test_should_respect_percentiles_config(self, publish_mock): self.collector.update_config({ 'percentiles': '25,75' }) - self.collector.interpret_bean_with_list('RecentReadLatencyHistogramMicros', self.fixture_values_a()) + self.collector.interpret_bean_with_list( + 'RecentReadLatencyHistogramMicros', self.fixture_values_a()) self.assertPublishedMany(publish_mock, { - 'RecentReadLatencyHistogramMicros.p25': self.expected_percentiles_for_fixture_a('p25'), - 'RecentReadLatencyHistogramMicros.p75': self.expected_percentiles_for_fixture_a('p75'), + 'RecentReadLatencyHistogramMicros.p25': + self.expected_percentiles_for_fixture_a('p25'), + 'RecentReadLatencyHistogramMicros.p75': + self.expected_percentiles_for_fixture_a('p75'), }) @patch.object(Collector, 'publish') @@ -81,11 +95,15 @@ def test_should_respect_histogram_regex_config(self, publish_mock): self.collector.update_config({ 'histogram_regex': '^WackyMetric' }) - self.collector.interpret_bean_with_list('WackyMetricSeventeen', self.fixture_values_a()) + self.collector.interpret_bean_with_list( + 'WackyMetricSeventeen', self.fixture_values_a()) self.assertPublishedMany(publish_mock, { - 'WackyMetricSeventeen.p50': self.expected_percentiles_for_fixture_a('p50'), - 'WackyMetricSeventeen.p95': self.expected_percentiles_for_fixture_a('p95'), - 'WackyMetricSeventeen.p99': self.expected_percentiles_for_fixture_a('p99') + 'WackyMetricSeventeen.p50': + self.expected_percentiles_for_fixture_a('p50'), + 'WackyMetricSeventeen.p95': + self.expected_percentiles_for_fixture_a('p95'), + 'WackyMetricSeventeen.p99': + self.expected_percentiles_for_fixture_a('p99') }) ################################################################################ diff --git a/src/collectors/jolokia/test/testjolokia.py b/src/collectors/jolokia/test/testjolokia.py index 4673fce65..2b52ace26 100644 --- a/src/collectors/jolokia/test/testjolokia.py +++ b/src/collectors/jolokia/test/testjolokia.py @@ -76,18 +76,19 @@ def se(url): defaultpath=self.collector.config['path']) self.assertPublishedMany(publish_mock, metrics) - @patch.object(Collector, 'publish') @patch.object(JolokiaCollector, 'interpret_bean_with_list') - def test_should_allow_interpretation_of_list_values(self, interpret_bean_with_list_mock, publish_mock): + def test_should_allow_interpretation_of_list_values( + self, interpret_bean_with_list_mock, publish_mock): self.collector.collect_bean('prefix', { 'RecentWriteLatencyMicros': 100, - 'RecentReadLatencyHistogramMicros': [1,2,3] + 'RecentReadLatencyHistogramMicros': [1, 2, 3] }) self.assertPublishedMany(publish_mock, { 'prefix.RecentWriteLatencyMicros': 100 }) - interpret_bean_with_list_mock.assert_called_with('prefix.RecentReadLatencyHistogramMicros', [1, 2, 3]) + interpret_bean_with_list_mock.assert_called_with( + 'prefix.RecentReadLatencyHistogramMicros', [1, 2, 3]) def get_metrics(self): prefix = 'java.lang.name_ParNew.type_GarbageCollector.LastGcInfo' diff --git a/src/collectors/pgbouncer/pgbouncer.py b/src/collectors/pgbouncer/pgbouncer.py index 7777ae8ca..66a85ce5e 100644 --- a/src/collectors/pgbouncer/pgbouncer.py +++ b/src/collectors/pgbouncer/pgbouncer.py @@ -90,10 +90,12 @@ def collect(self): user = instance.get('user') or self.config['user'] password = instance.get('password') or self.config['password'] - for database, stats in self._get_stats_by_database(host, port, user, password).iteritems(): + for database, stats in self._get_stats_by_database( + host, port, user, password).iteritems(): for stat_name, stat_value in stats.iteritems(): - self.publish(self._get_metric_name(name, database, stat_name), - stat_value) + self.publish( + self._get_metric_name(name, database, stat_name), + stat_value) def _get_metric_name(self, name, database, stat_name): name = name.replace('.', '_').replace(':', '_').strip() @@ -102,7 +104,11 @@ def _get_metric_name(self, name, database, stat_name): def _get_stats_by_database(self, host, port, user, password): # Mapping of database name -> stats. databases = defaultdict(dict) - conn = psycopg2.connect(database='pgbouncer', user=user, password=password, host=host, port=port) + conn = psycopg2.connect(database='pgbouncer', + user=user, + password=password, + host=host, + port=port) # Avoid using transactions, set isolation level to autocommit conn.set_isolation_level(0) diff --git a/src/collectors/pgbouncer/test/testpgbouncer.py b/src/collectors/pgbouncer/test/testpgbouncer.py index 4c34012fb..35a558808 100644 --- a/src/collectors/pgbouncer/test/testpgbouncer.py +++ b/src/collectors/pgbouncer/test/testpgbouncer.py @@ -39,7 +39,8 @@ def test_default(self, publish, _get_stats_by_database): self.collector.collect() - _get_stats_by_database.assert_called_with('localhost', '6432', 'postgres', '') + _get_stats_by_database.assert_called_with( + 'localhost', '6432', 'postgres', '') self.assertPublished(publish, 'default.foo.bar', 42) @@ -95,8 +96,10 @@ def test_override_user_password(self, _get_stats_by_database): collector = PgbouncerCollector(config, None) collector.collect() - _get_stats_by_database.assert_any_call('127.0.0.1', '6433', 'postgres', 'foobar') - _get_stats_by_database.assert_any_call('127.0.0.2', '6432', 'pgbouncer', '') + _get_stats_by_database.assert_any_call( + '127.0.0.1', '6433', 'postgres', 'foobar') + _get_stats_by_database.assert_any_call( + '127.0.0.2', '6432', 'pgbouncer', '') ################################################################################ diff --git a/src/collectors/postgres/postgres.py b/src/collectors/postgres/postgres.py index 3ccbdf8aa..4f32f06bb 100644 --- a/src/collectors/postgres/postgres.py +++ b/src/collectors/postgres/postgres.py @@ -363,7 +363,8 @@ class ConnectionStateStats(QueryStats): LEFT JOIN (SELECT CASE WHEN waiting THEN 'waiting' WHEN state = 'idle' THEN 'idle' - WHEN state LIKE 'idle in transaction%' THEN 'idletransaction' + WHEN state LIKE 'idle in transaction%' + THEN 'idletransaction' WHEN state = 'disabled' THEN 'unknown' WHEN query = ''