Skip to content

Commit

Permalink
pep8 fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
kormoc committed Dec 31, 2014
1 parent 3d786b0 commit 2cbaeb1
Show file tree
Hide file tree
Showing 7 changed files with 95 additions and 53 deletions.
1 change: 1 addition & 0 deletions src/collectors/amavis/test/testamavis.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

MOCK_PATH = os.path.join(os.path.dirname(__file__), 'mock-amavisd-agent')


class TestAmavisCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('AmavisCollector', {
Expand Down
58 changes: 35 additions & 23 deletions src/collectors/jolokia/cassandra_jolokia.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
# coding=utf-8

"""
Collects Cassandra JMX metrics from the Jolokia Agent. Extends the JolokiaCollector to
interpret Histogram beans with information about the distribution of request latencies.
Collects Cassandra JMX metrics from the Jolokia Agent. Extends the
JolokiaCollector to interpret Histogram beans with information about the
distribution of request latencies.
#### Example Configuration
CassandraJolokiaCollector uses a regular expression to determine which attributes represent histograms.
This regex can be overridden by providing a `histogram_regex` in your configuration. You can also override
`percentiles` to collect specific percentiles from the histogram statistics. The format is shown below
with the default values.
CassandraJolokiaCollector uses a regular expression to determine which
attributes represent histograms. This regex can be overridden by providing a
`histogram_regex` in your configuration. You can also override `percentiles` to
collect specific percentiles from the histogram statistics. The format is shown
below with the default values.
CassandraJolokiaCollector.conf
Expand All @@ -27,10 +29,14 @@
class CassandraJolokiaCollector(JolokiaCollector):
# override to allow setting which percentiles will be collected
def get_default_config_help(self):
config_help = super(CassandraJolokiaCollector, self).get_default_config_help()
config_help = super(CassandraJolokiaCollector,
self).get_default_config_help()
config_help.update({
'percentiles': 'Comma separated list of percentiles to be collected (e.g., "50,95,99").',
'histogram_regex': 'Filter to only process attributes that match this regex'
'percentiles':
'Comma separated list of percentiles to be collected '
'(e.g., "50,95,99").',
'histogram_regex':
'Filter to only process attributes that match this regex'
})
return config_help

Expand All @@ -49,33 +55,39 @@ def __init__(self, config, handlers):
self.update_config(self.config)

def update_config(self, config):
if config.has_key('percentiles'):
self.percentiles = map(int, string.split(config['percentiles'], ','))
if config.has_key('histogram_regex'):
if 'percentiles' in config:
self.percentiles = map(int, string.split(config['percentiles'],
','))
if 'histogram_regex' in config:
self.histogram_regex = re.compile(config['histogram_regex'])

# override: Interpret beans that match the `histogram_regex` as histograms, and collect
# percentiles from them.
# override: Interpret beans that match the `histogram_regex` as histograms,
# and collect percentiles from them.
def interpret_bean_with_list(self, prefix, values):
if not self.histogram_regex.match(prefix):
return

buckets = values
for percentile in self.percentiles:
percentile_value = self.compute_percentile(self.offsets, buckets, percentile)
percentile_value = self.compute_percentile(
self.offsets, buckets, percentile)
self.publish("%s.p%s" % (prefix, percentile), percentile_value)

# Adapted from Cassandra docs: http://www.datastax.com/documentation/cassandra/2.0/cassandra/tools/toolsCFhisto.html
# The index corresponds to the x-axis in a histogram. It represents buckets of values, which are
# a series of ranges. Each offset includes the range of values greater than the previous offset
# and less than or equal to the current offset. The offsets start at 1 and each subsequent offset
# is calculated by multiplying the previous offset by 1.2, rounding up, and removing duplicates. The
# offsets can range from 1 to approximately 25 million, with less precision as the offsets get larger.
# Adapted from Cassandra docs:
# http://www.datastax.com/documentation/cassandra/2.0/cassandra/tools/toolsCFhisto.html
# The index corresponds to the x-axis in a histogram. It represents buckets
# of values, which are a series of ranges. Each offset includes the range of
# values greater than the previous offset and less than or equal to the
# current offset. The offsets start at 1 and each subsequent offset is
# calculated by multiplying the previous offset by 1.2, rounding up, and
# removing duplicates. The offsets can range from 1 to approximately 25
# million, with less precision as the offsets get larger.
def compute_percentile(self, offsets, buckets, percentile_int):
non_zero_points_sum = sum(buckets)
if non_zero_points_sum is 0:
return 0
middle_point_index = math.floor(non_zero_points_sum * (percentile_int / float(100)))
middle_point_index = math.floor(
non_zero_points_sum * (percentile_int / float(100)))

points_seen = 0
for index, bucket in enumerate(buckets):
Expand All @@ -95,4 +107,4 @@ def create_offsets(self, bucket_count):
offsets.append(next_num)
last_num = next_num

return offsets
return offsets
54 changes: 36 additions & 18 deletions src/collectors/jolokia/test/testcassandra_jolokia.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ def setUp(self):

# Used for all the tests so the expected numbers are all the same.
def fixture_values_a(self):
return [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,1,1,8,5,6,1,6,5,3,8,9,10,7,8,7,5,5,5,3,3,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
values = [0]*92
values[30:56] = [3, 3, 1, 1, 8, 5, 6, 1, 6, 5, 3, 8, 9, 10, 7, 8, 7, 5,
5, 5, 3, 3, 2, 2, 2]

def empty_fixture_values(self):
return [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
return [0]*91

def expected_percentiles_for_fixture_a(self, percentile_key):
return {
Expand All @@ -42,50 +44,66 @@ def test_import(self):

def test_should_compute_percentiles_accurately(self):
ninety_offsets = self.collector.create_offsets(90)
percentile_value = self.collector.compute_percentile(ninety_offsets, self.fixture_values_a(), 50)
percentile_value = self.collector.compute_percentile(
ninety_offsets, self.fixture_values_a(), 50)
self.assertEqual(percentile_value, 398.0)

def test_should_compute_percentiles_accurately_when_empty(self):
ninety_offsets = self.collector.create_offsets(90)
self.assertEqual(self.collector.compute_percentile(ninety_offsets, self.empty_fixture_values(), 50), 0.0)
self.assertEqual(self.collector.compute_percentile(ninety_offsets, self.empty_fixture_values(), 95), 0.0)
self.assertEqual(self.collector.compute_percentile(ninety_offsets, self.empty_fixture_values(), 99), 0.0)
self.assertEqual(self.collector.compute_percentile(
ninety_offsets, self.empty_fixture_values(), 50), 0.0)
self.assertEqual(self.collector.compute_percentile(
ninety_offsets, self.empty_fixture_values(), 95), 0.0)
self.assertEqual(self.collector.compute_percentile(
ninety_offsets, self.empty_fixture_values(), 99), 0.0)

@patch.object(Collector, 'publish')
def test_should_not_collect_non_histogram_attributes(self, publish_mock):
self.collector.interpret_bean_with_list('RecentReadLatencyMicros', self.fixture_values_a())
self.collector.interpret_bean_with_list(
'RecentReadLatencyMicros', self.fixture_values_a())
self.assertPublishedMany(publish_mock, {})

@patch.object(Collector, 'publish')
def test_should_collect_metrics_histogram_attributes(self, publish_mock):
self.collector.interpret_bean_with_list('RecentReadLatencyHistogramMicros', self.fixture_values_a())
self.collector.interpret_bean_with_list(
'RecentReadLatencyHistogramMicros', self.fixture_values_a())
self.assertPublishedMany(publish_mock, {
'RecentReadLatencyHistogramMicros.p50': self.expected_percentiles_for_fixture_a('p50'),
'RecentReadLatencyHistogramMicros.p95': self.expected_percentiles_for_fixture_a('p95'),
'RecentReadLatencyHistogramMicros.p99': self.expected_percentiles_for_fixture_a('p99')
'RecentReadLatencyHistogramMicros.p50':
self.expected_percentiles_for_fixture_a('p50'),
'RecentReadLatencyHistogramMicros.p95':
self.expected_percentiles_for_fixture_a('p95'),
'RecentReadLatencyHistogramMicros.p99':
self.expected_percentiles_for_fixture_a('p99')
})

@patch.object(Collector, 'publish')
def test_should_respect_percentiles_config(self, publish_mock):
self.collector.update_config({
'percentiles': '25,75'
})
self.collector.interpret_bean_with_list('RecentReadLatencyHistogramMicros', self.fixture_values_a())
self.collector.interpret_bean_with_list(
'RecentReadLatencyHistogramMicros', self.fixture_values_a())
self.assertPublishedMany(publish_mock, {
'RecentReadLatencyHistogramMicros.p25': self.expected_percentiles_for_fixture_a('p25'),
'RecentReadLatencyHistogramMicros.p75': self.expected_percentiles_for_fixture_a('p75'),
'RecentReadLatencyHistogramMicros.p25':
self.expected_percentiles_for_fixture_a('p25'),
'RecentReadLatencyHistogramMicros.p75':
self.expected_percentiles_for_fixture_a('p75'),
})

@patch.object(Collector, 'publish')
def test_should_respect_histogram_regex_config(self, publish_mock):
self.collector.update_config({
'histogram_regex': '^WackyMetric'
})
self.collector.interpret_bean_with_list('WackyMetricSeventeen', self.fixture_values_a())
self.collector.interpret_bean_with_list(
'WackyMetricSeventeen', self.fixture_values_a())
self.assertPublishedMany(publish_mock, {
'WackyMetricSeventeen.p50': self.expected_percentiles_for_fixture_a('p50'),
'WackyMetricSeventeen.p95': self.expected_percentiles_for_fixture_a('p95'),
'WackyMetricSeventeen.p99': self.expected_percentiles_for_fixture_a('p99')
'WackyMetricSeventeen.p50':
self.expected_percentiles_for_fixture_a('p50'),
'WackyMetricSeventeen.p95':
self.expected_percentiles_for_fixture_a('p95'),
'WackyMetricSeventeen.p99':
self.expected_percentiles_for_fixture_a('p99')
})

################################################################################
Expand Down
9 changes: 5 additions & 4 deletions src/collectors/jolokia/test/testjolokia.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,18 +76,19 @@ def se(url):
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)


@patch.object(Collector, 'publish')
@patch.object(JolokiaCollector, 'interpret_bean_with_list')
def test_should_allow_interpretation_of_list_values(self, interpret_bean_with_list_mock, publish_mock):
def test_should_allow_interpretation_of_list_values(
self, interpret_bean_with_list_mock, publish_mock):
self.collector.collect_bean('prefix', {
'RecentWriteLatencyMicros': 100,
'RecentReadLatencyHistogramMicros': [1,2,3]
'RecentReadLatencyHistogramMicros': [1, 2, 3]
})
self.assertPublishedMany(publish_mock, {
'prefix.RecentWriteLatencyMicros': 100
})
interpret_bean_with_list_mock.assert_called_with('prefix.RecentReadLatencyHistogramMicros', [1, 2, 3])
interpret_bean_with_list_mock.assert_called_with(
'prefix.RecentReadLatencyHistogramMicros', [1, 2, 3])

def get_metrics(self):
prefix = 'java.lang.name_ParNew.type_GarbageCollector.LastGcInfo'
Expand Down
14 changes: 10 additions & 4 deletions src/collectors/pgbouncer/pgbouncer.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,12 @@ def collect(self):
user = instance.get('user') or self.config['user']
password = instance.get('password') or self.config['password']

for database, stats in self._get_stats_by_database(host, port, user, password).iteritems():
for database, stats in self._get_stats_by_database(
host, port, user, password).iteritems():
for stat_name, stat_value in stats.iteritems():
self.publish(self._get_metric_name(name, database, stat_name),
stat_value)
self.publish(
self._get_metric_name(name, database, stat_name),
stat_value)

def _get_metric_name(self, name, database, stat_name):
name = name.replace('.', '_').replace(':', '_').strip()
Expand All @@ -102,7 +104,11 @@ def _get_metric_name(self, name, database, stat_name):
def _get_stats_by_database(self, host, port, user, password):
# Mapping of database name -> stats.
databases = defaultdict(dict)
conn = psycopg2.connect(database='pgbouncer', user=user, password=password, host=host, port=port)
conn = psycopg2.connect(database='pgbouncer',
user=user,
password=password,
host=host,
port=port)

# Avoid using transactions, set isolation level to autocommit
conn.set_isolation_level(0)
Expand Down
9 changes: 6 additions & 3 deletions src/collectors/pgbouncer/test/testpgbouncer.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ def test_default(self, publish, _get_stats_by_database):

self.collector.collect()

_get_stats_by_database.assert_called_with('localhost', '6432', 'postgres', '')
_get_stats_by_database.assert_called_with(
'localhost', '6432', 'postgres', '')

self.assertPublished(publish, 'default.foo.bar', 42)

Expand Down Expand Up @@ -95,8 +96,10 @@ def test_override_user_password(self, _get_stats_by_database):
collector = PgbouncerCollector(config, None)
collector.collect()

_get_stats_by_database.assert_any_call('127.0.0.1', '6433', 'postgres', 'foobar')
_get_stats_by_database.assert_any_call('127.0.0.2', '6432', 'pgbouncer', '')
_get_stats_by_database.assert_any_call(
'127.0.0.1', '6433', 'postgres', 'foobar')
_get_stats_by_database.assert_any_call(
'127.0.0.2', '6432', 'pgbouncer', '')


################################################################################
Expand Down
3 changes: 2 additions & 1 deletion src/collectors/postgres/postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,8 @@ class ConnectionStateStats(QueryStats):
LEFT JOIN
(SELECT CASE WHEN waiting THEN 'waiting'
WHEN state = 'idle' THEN 'idle'
WHEN state LIKE 'idle in transaction%' THEN 'idletransaction'
WHEN state LIKE 'idle in transaction%'
THEN 'idletransaction'
WHEN state = 'disabled'
THEN 'unknown'
WHEN query = '<insufficient privilege>'
Expand Down

0 comments on commit 2cbaeb1

Please sign in to comment.