Skip to content

Commit

Permalink
Implement AnyTagValue for easier testing with tags (#141)
Browse files Browse the repository at this point in the history 
This makes it easier to test with tags where the value is depenedent on
something like a host name which changes.
  • Loading branch information
@willkg
willkg committed Jun 24, 2024
1 parent 92907cd commit f72a814
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 68 deletions.
30 changes: 30 additions & 0 deletions src/markus/testing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from copy import copy
from functools import total_ordering

import functools

Expand All @@ -24,6 +25,35 @@ def _print_on_failure(metricsmock, *args, **kwargs):
return _print_on_failure


@total_ordering
class AnyTagValue:
"""Matches a markus metrics tag with any value"""

def __init__(self, key):
self.key = key

def __repr__(self):
return f"<AnyTagValue {self.key}>"

def get_other_key(self, other):
# This is comparing against a tag string
if ":" in other:
other_key, _ = other.split(":")
else:
other_key = other
return other_key

def __eq__(self, other):
if isinstance(other, AnyTagValue):
return self.key == other.key
return self.key == self.get_other_key(other)

def __lt__(self, other):
if isinstance(other, AnyTagValue):
return self.key < other.key
return self.key < self.get_other_key(other)


class MetricsMock:
"""Mock for recording metrics events and testing them.
Expand Down
155 changes: 87 additions & 68 deletions tests/test_mock.py → tests/test_testing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,42 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import markus
from markus.testing import MetricsMock
from markus.testing import AnyTagValue, MetricsMock

import pytest


class TestAnyTagValue:
def test_equality(self):
# values are equal even if instances aren't
assert AnyTagValue("host") == AnyTagValue("host")

# equal to "host:" with any value
assert AnyTagValue("host") == "host:1234"
assert AnyTagValue("host") == "host:5678"
assert AnyTagValue("host") == "host"

# not equal to a different tag
assert AnyTagValue("host") != "env:prod"

def test_sorting(self):
items = [AnyTagValue("host"), "env:prod", "color:blue"]
items.sort()
assert list(items) == ["color:blue", "env:prod", AnyTagValue("host")]

items = ["env:prod", "color:blue", AnyTagValue("host"), "host"]
items.sort()
assert list(items) == ["color:blue", "env:prod", AnyTagValue("host"), "host"]

def test_assertions(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1, tags=["host:12345", "env:prod"])

mm.assert_incr("test.key1", value=1, tags=[AnyTagValue("host"), "env:prod"])
mm.assert_incr("test.key1", value=1, tags=["env:prod", AnyTagValue("host")])


class TestMetricsMock:
"""Verify the MetricsMock works as advertised"""

Expand All @@ -16,14 +47,14 @@ def test_print_records(self):
# for debugging tests. So we're just going to run it and make sure it
# doesn't throw errors.
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1")

mm.print_records()

def test_clear_records(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1, tags=["env:stage"])

assert len(mm.get_records()) == 1
Expand All @@ -34,76 +65,64 @@ def test_clear_records(self):

def test_filter_records_fun_name(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1, tags=["env:stage"])

key1_metrics = mm.filter_records(stat="foobar.key1", value=1)
key1_metrics = mm.filter_records(stat="test.key1", value=1)
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=1
)
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key1", value=1)
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="timing", stat="foobar.key1", value=1
fun_name="timing", stat="test.key1", value=1
)
assert len(key1_metrics) == 0

def test_filter_records_key(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1, tags=["env:stage"])

key1_metrics = mm.filter_records(fun_name="incr", value=1)
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=1
)
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key1", value=1)
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=1
)
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key1", value=1)
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key2", value=1
)
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key2", value=1)
assert len(key1_metrics) == 0

def test_filter_records_value(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1, tags=["env:stage"])

key1_metrics = mm.filter_records(fun_name="incr", stat="foobar.key1")
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key1")
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=1
)
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key1", value=1)
assert len(key1_metrics) == 1

key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=5
)
key1_metrics = mm.filter_records(fun_name="incr", stat="test.key1", value=5)
assert len(key1_metrics) == 0

def test_filter_records_tags(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1, tags=["env:stage"])
mymetrics.incr("key2", value=3, tags=["env:prod"])

key1_metrics = mm.filter_records(tags=["env:stage"])
assert len(key1_metrics) == 1
assert key1_metrics[0].key == "foobar.key1"
assert key1_metrics[0].key == "test.key1"

key1_metrics = mm.filter_records(tags=["env:prod"])
assert len(key1_metrics) == 1
assert key1_metrics[0].key == "foobar.key2"
assert key1_metrics[0].key == "test.key2"

key1_metrics = mm.filter_records(tags=["env:dev"])
assert len(key1_metrics) == 0
Expand All @@ -115,114 +134,114 @@ def test_has_record(self):
#
# If that ever changes, we should update this test.
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1)

assert mm.has_record(fun_name="incr", stat="foobar.key1", value=1)
assert mm.has_record(fun_name="incr", stat="test.key1", value=1)

assert not mm.has_record(fun_name="incr", stat="foobar.key1", value=5)
assert not mm.has_record(fun_name="incr", stat="test.key1", value=5)

def test_configure_doesnt_affect_override(self):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1)

assert mm.has_record(fun_name="incr", stat="foobar.key1", value=1)
assert mm.has_record(fun_name="incr", stat="test.key1", value=1)

assert not mm.has_record(fun_name="incr", stat="foobar.key1", value=5)
assert not mm.has_record(fun_name="incr", stat="test.key1", value=5)

def test_incr_helpers(self):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.incr("key1", value=1)
mymetrics.incr("keymultiple", value=1)
mymetrics.incr("keymultiple", value=1)

mm.assert_incr(stat="foobar.key1")
mm.assert_incr(stat="test.key1")

mm.assert_incr_once(stat="foobar.key1")
mm.assert_incr_once(stat="test.key1")
with pytest.raises(AssertionError):
mm.assert_incr_once(stat="foobar.keymultiple")
mm.assert_incr_once(stat="test.keymultiple")

mm.assert_not_incr(stat="foobar.keynot")
mm.assert_not_incr(stat="foobar.key1", value=5)
mm.assert_not_incr(stat="test.keynot")
mm.assert_not_incr(stat="test.key1", value=5)
with pytest.raises(AssertionError):
mm.assert_not_incr(stat="foobar.key1")
mm.assert_not_incr(stat="test.key1")

def test_gauge_helpers(self):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.gauge("key1", value=5)
mymetrics.gauge("keymultiple", value=5)
mymetrics.gauge("keymultiple", value=5)

mm.assert_gauge(stat="foobar.key1")
mm.assert_gauge(stat="test.key1")

mm.assert_gauge_once(stat="foobar.key1")
mm.assert_gauge_once(stat="test.key1")
with pytest.raises(AssertionError):
mm.assert_gauge_once(stat="foobar.keymultiple")
mm.assert_gauge_once(stat="test.keymultiple")

mm.assert_not_gauge(stat="foobar.keynot")
mm.assert_not_gauge(stat="foobar.key1", value=10)
mm.assert_not_gauge(stat="test.keynot")
mm.assert_not_gauge(stat="test.key1", value=10)
with pytest.raises(AssertionError):
mm.assert_not_gauge(stat="foobar.key1")
mm.assert_not_gauge(stat="test.key1")

def test_timing_helpers(self):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.timing("key1", value=1)
mymetrics.timing("keymultiple", value=1)
mymetrics.timing("keymultiple", value=1)

mm.assert_timing(stat="foobar.key1")
mm.assert_timing(stat="test.key1")

mm.assert_timing_once(stat="foobar.key1")
mm.assert_timing_once(stat="test.key1")
with pytest.raises(AssertionError):
mm.assert_timing_once(stat="foobar.keymultiple")
mm.assert_timing_once(stat="test.keymultiple")

mm.assert_not_timing(stat="foobar.keynot")
mm.assert_not_timing(stat="foobar.key1", value=5)
mm.assert_not_timing(stat="test.keynot")
mm.assert_not_timing(stat="test.key1", value=5)
with pytest.raises(AssertionError):
mm.assert_not_timing(stat="foobar.key1")
mm.assert_not_timing(stat="test.key1")

def test_histogram_helpers(self):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.histogram("key1", value=1)
mymetrics.histogram("keymultiple", value=1)
mymetrics.histogram("keymultiple", value=1)

mm.assert_histogram(stat="foobar.key1")
mm.assert_histogram(stat="test.key1")

mm.assert_histogram_once(stat="foobar.key1")
mm.assert_histogram_once(stat="test.key1")
with pytest.raises(AssertionError):
mm.assert_histogram_once(stat="foobar.keymultiple")
mm.assert_histogram_once(stat="test.keymultiple")

mm.assert_not_histogram(stat="foobar.keynot")
mm.assert_not_histogram(stat="foobar.key1", value=5)
mm.assert_not_histogram(stat="test.keynot")
mm.assert_not_histogram(stat="test.key1", value=5)
with pytest.raises(AssertionError):
mm.assert_not_histogram(stat="foobar.key1")
mm.assert_not_histogram(stat="test.key1")

def test_print_on_failure(self, capsys):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics = markus.get_metrics("test")
mymetrics.histogram("keymultiple", value=1)
mymetrics.histogram("keymultiple", value=1)

with pytest.raises(AssertionError):
mm.assert_histogram_once(stat="foobar.keymultiple")
mm.assert_histogram_once(stat="test.keymultiple")

# On assertion error, the assert_* methods will print the metrics
# records to stdout.
captured = capsys.readouterr()
expected = (
"<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n"
"<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n"
"<MetricsRecord type=histogram key=test.keymultiple value=1 tags=[]>\n"
"<MetricsRecord type=histogram key=test.keymultiple value=1 tags=[]>\n"
)
assert captured.out == expected

0 comments on commit f72a814

Please sign in to comment.