From 0464605aac3a4103bdc5858c7477633f9a1c116e Mon Sep 17 00:00:00 2001 From: Benjamin Fernandes Date: Mon, 20 Oct 2014 11:41:59 -0400 Subject: [PATCH 1/4] Minor cleanup and fix --- aggregator.py | 2 +- tests/test_dogstatsd.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/aggregator.py b/aggregator.py index 9570279d48..0af5ac7145 100644 --- a/aggregator.py +++ b/aggregator.py @@ -494,7 +494,7 @@ def parse_event_packet(self, packet): elif m[0] == u'#': event['tags'] = sorted(m[1:].split(u',')) return event - except IndexError, ValueError: + except (IndexError, ValueError): raise Exception(u'Unparseable event packet: %s' % packet) def submit_packets(self, packets): diff --git a/tests/test_dogstatsd.py b/tests/test_dogstatsd.py index be5129c255..ac60c2970b 100644 --- a/tests/test_dogstatsd.py +++ b/tests/test_dogstatsd.py @@ -95,7 +95,6 @@ def test_tags(self): nt.assert_equal(third['host'], 'myhost') def test_tags_gh442(self): - import util import dogstatsd from aggregator import api_formatter From a36a2930c8a9da2e3f271b77452a5c1ec8ce8b7c Mon Sep 17 00:00:00 2001 From: Benjamin Fernandes Date: Mon, 20 Oct 2014 12:47:41 -0400 Subject: [PATCH 2/4] Make dogstatsd support magic tags Tags host and device are converted into metric attributes --- aggregator.py | 26 +++++++++++++++++++++++++- tests/test_dogstatsd.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/aggregator.py b/aggregator.py index 0af5ac7145..f26c70ca34 100644 --- a/aggregator.py +++ b/aggregator.py @@ -511,7 +511,31 @@ def submit_packets(self, packets): self.count += 1 parsed_packets = self.parse_metric_packet(packet) for name, value, mtype, tags, sample_rate in parsed_packets: - self.submit_metric(name, value, mtype, tags=tags, sample_rate=sample_rate) + hostname, device_name, tags = self._extract_magic_tags(tags) + self.submit_metric(name, value, mtype, tags=tags, hostname=hostname, + device_name=device_name, sample_rate=sample_rate) + + def _extract_magic_tags(self, tags): + """Magic tags (host, device) override metric hostname and device_name attributes""" + hostname = None + device_name = None + # This implementation avoid list operations for the common case + if tags: + tags_to_remove = [] + for tag in tags: + if tag.startswith('host:'): + hostname = tag[5:] + tags_to_remove.append(tag) + elif tag.startswith('device:'): + device_name = tag[7:] + tags_to_remove.append(tag) + if tags_to_remove: + # tags is a tuple already sorted, we convert it into a list to pop elements + tags = list(tags) + for tag in tags_to_remove: + tags.remove(tag) + tags = tuple(tags) or None + return hostname, device_name, tags def submit_metric(self, name, value, mtype, tags=None, hostname=None, device_name=None, timestamp=None, sample_rate=1): diff --git a/tests/test_dogstatsd.py b/tests/test_dogstatsd.py index ac60c2970b..bd097934a4 100644 --- a/tests/test_dogstatsd.py +++ b/tests/test_dogstatsd.py @@ -94,6 +94,40 @@ def test_tags(self): nt.assert_equal(third['points'][0][1], 16) nt.assert_equal(third['host'], 'myhost') + def test_magic_tags(self): + stats = MetricsAggregator('myhost') + stats.submit_packets('my.gauge.a:1|c|#host:test-a') + stats.submit_packets('my.gauge.b:4|c|#tag1,tag2,host:test-b') + stats.submit_packets('my.gauge.b:8|c|#host:test-b,tag2,tag1') + stats.submit_packets('my.gauge.c:10|c|#tag3') + stats.submit_packets('my.gauge.c:16|c|#device:floppy,tag3') + + + metrics = self.sort_metrics(stats.flush()) + + nt.assert_equal(len(metrics), 4) + first, second, third, fourth = metrics + + nt.assert_equal(first['metric'], 'my.gauge.a') + nt.assert_equal(first['tags'], None) + nt.assert_equal(first['points'][0][1], 1) + nt.assert_equal(first['host'], 'test-a') + + nt.assert_equal(second['metric'], 'my.gauge.b') + nt.assert_equal(second['tags'], ('tag1', 'tag2')) + nt.assert_equal(second['points'][0][1], 12) + nt.assert_equal(second['host'], 'test-b') + + nt.assert_equal(third['metric'], 'my.gauge.c') + nt.assert_equal(third['tags'], ('tag3', )) + nt.assert_equal(third['points'][0][1], 10) + nt.assert_equal(third['device_name'], None) + + nt.assert_equal(fourth['metric'], 'my.gauge.c') + nt.assert_equal(fourth['tags'], ('tag3', )) + nt.assert_equal(fourth['points'][0][1], 16) + nt.assert_equal(fourth['device_name'], 'floppy') + def test_tags_gh442(self): import dogstatsd from aggregator import api_formatter From c5008544632795a49192234633cddb892c3a4681 Mon Sep 17 00:00:00 2001 From: Benjamin Fernandes Date: Mon, 20 Oct 2014 13:29:43 -0400 Subject: [PATCH 3/4] Make dogstatsd test fully deterministic --- tests/test_dogstatsd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dogstatsd.py b/tests/test_dogstatsd.py index bd097934a4..9c8c54ddae 100644 --- a/tests/test_dogstatsd.py +++ b/tests/test_dogstatsd.py @@ -13,7 +13,7 @@ class TestUnitDogStatsd(unittest.TestCase): @staticmethod def sort_metrics(metrics): def sort_by(m): - return (m['metric'], ','.join(m['tags'] or [])) + return (m['metric'], m['host'], m['device_name'], ','.join(m['tags'] or [])) return sorted(metrics, key=sort_by) @staticmethod @@ -102,7 +102,6 @@ def test_magic_tags(self): stats.submit_packets('my.gauge.c:10|c|#tag3') stats.submit_packets('my.gauge.c:16|c|#device:floppy,tag3') - metrics = self.sort_metrics(stats.flush()) nt.assert_equal(len(metrics), 4) From 954c4645cff4a1fd41924a1e4473577b55aec44e Mon Sep 17 00:00:00 2001 From: Benjamin Fernandes Date: Mon, 20 Oct 2014 15:11:06 -0400 Subject: [PATCH 4/4] Allow empty host in dogstatsd to unset the attribute --- aggregator.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aggregator.py b/aggregator.py index f26c70ca34..a4610aedbd 100644 --- a/aggregator.py +++ b/aggregator.py @@ -615,7 +615,8 @@ def submit_metric(self, name, value, mtype, tags=None, hostname=None, # Note: if you change the way that context is created, please also change create_empty_metrics, # which counts on this order - hostname = hostname or self.hostname + # Keep hostname with empty string to unset it + hostname = hostname if hostname is not None else self.hostname if tags is None: context = (name, tuple(), hostname, device_name) @@ -736,8 +737,9 @@ def submit_metric(self, name, value, mtype, tags=None, hostname=None, device_name=None, timestamp=None, sample_rate=1): # Avoid calling extra functions to dedupe tags if there are none - hostname = hostname or self.hostname - + # Keep hostname with empty string to unset it + hostname = hostname if hostname is not None else self.hostname + if tags is None: context = (name, tuple(), hostname, device_name) else: @@ -805,7 +807,7 @@ def flush(self): return metrics -def api_formatter(metric, value, timestamp, tags, hostname, device_name=None, +def api_formatter(metric, value, timestamp, tags, hostname=None, device_name=None, metric_type=None, interval=None): return { 'metric': metric,