From 023ee63ee986c87d6e9a00a493cd7b98101d4cca Mon Sep 17 00:00:00 2001 From: agspoon <39888940+agspoon@users.noreply.github.com> Date: Thu, 9 Mar 2023 06:17:40 -0800 Subject: [PATCH] feat: Add time parser (#17) * test: Add tests for various forms of rtl_433 timestamps test the following rtl_433 timestamp formats, report_meta time:utc:tz report_meta time:tz report_meta time:iso:tz report_meta time:unix report_meta time:utc:tz:usec report_meta time:tz:usec report_meta time:iso:tz:usec report_meta time:unix:usec Standard "report_meta time" is covered by the metrics tests. * feat: Add generic date/time string parsing from python-dateutil rtl_433 can output date/time in multiple formats. The dateutil module provides a way to parse these formats generically, and handles timezones well. * chore: fix code style. --------- Co-authored-by: Andrew Wilkinson --- prom433/prometheus.py | 15 ++++++++++---- requirements.txt | 1 + tests/test_prometheus.py | 40 ++++++++++++++++++++++++++++++++++++++ tests/timestamp_sample.txt | 8 ++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 tests/timestamp_sample.txt diff --git a/prom433/prometheus.py b/prom433/prometheus.py index a0f11d5..a95d3fe 100644 --- a/prom433/prometheus.py +++ b/prom433/prometheus.py @@ -15,6 +15,9 @@ # along with this program. If not, see . from datetime import datetime, timezone +import dateutil.parser +import dateutil.tz +import dateutil.utils import json import logging @@ -96,8 +99,9 @@ METRICS_CONVERT = { "prom433_radio_clock": - lambda x: datetime.strptime(x, "%Y-%m-%dT%H:%M:%S") - .replace(tzinfo=timezone.utc).timestamp() + lambda x: dateutil.utils.default_tzinfo(dateutil.parser.parse(x), + dateutil.tz.tzoffset("UTC", 0)) + .timestamp() } TAG_KEYS = {"id", "channel", "model"} @@ -146,8 +150,11 @@ def prometheus(message, drop_after): for key, value in payload.items(): if key == "time": - time_value = datetime.strptime(payload[key], "%Y-%m-%d %H:%M:%S") \ - .timestamp() + if ':' in payload[key]: + time_value = dateutil.parser.parse(payload[key]).timestamp() + else: + time_value = datetime.fromtimestamp(float(payload[key])) \ + .timestamp() elif key in TAG_KEYS: tags[key] = value elif key in METRIC_NAME: diff --git a/requirements.txt b/requirements.txt index 4122c26..b995864 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pycodestyle==2.10.0 coveralls==3.3.1 python-semantic-release==7.33.0 paho-mqtt==1.6.1 +python-dateutil==2.8.1 diff --git a/tests/test_prometheus.py b/tests/test_prometheus.py index 5c8777a..471d052 100644 --- a/tests/test_prometheus.py +++ b/tests/test_prometheus.py @@ -23,6 +23,7 @@ MESSAGE_TEXT = open("tests/output_sample.txt", "rb").read().decode("utf8") DROP_TEXT = open("tests/dropmetric_sample.txt", "rb").read().decode("utf8") +TIMESTAMP_TEXT = open("tests/timestamp_sample.txt", "rb").read().decode("utf8") def mock_popen(args, stdout): @@ -84,3 +85,42 @@ def test_drop_metric_disabled(self): # The Fineoffset hasn't been seen for an hour, # but dropping is disabled. self.assertIn("""Fineoffset-WHx080""", prom) + + def test_timestamp(self): + for line in TIMESTAMP_TEXT.split("\n"): + prometheus(line, 0) + + prom = get_metrics() + + # Test for time in format time:utc:tz + self.assertIn( + """prom433_last_message{id="1", """ + + """model="LaCrosse-TX"} 1677374905.000000""", prom) + # Test for time in format time:tz + self.assertIn( + """prom433_last_message{id="2", """ + + """model="LaCrosse-TX"} 1677374905.000000""", prom) + # Test for time in format time:iso:tz + self.assertIn( + """prom433_last_message{id="3", """ + + """model="LaCrosse-TX"} 1677374905.000000""", prom) + # Test for time in format time:unix + self.assertIn( + """prom433_last_message{id="4", """ + + """model="LaCrosse-TX"} 1677374905.000000""", prom) + # Test for time in format time:utc:tz:usec + self.assertIn( + """prom433_last_message{id="101", """ + + """model="LaCrosse-TX31UIT"} 1677374905.538138""", prom) + # Test for time in format time:tz:usec + self.assertIn( + """prom433_last_message{id="102", """ + + """model="LaCrosse-TX31UIT"} 1677374905.538138""", prom) + # Test for time in format time:iso:tz:usec + self.assertIn( + """prom433_last_message{id="103", """ + + """model="LaCrosse-TX31UIT"} 1677374905.538138""", prom) + # Test for time in format time:unix:usec + self.assertIn( + """prom433_last_message{id="104", """ + + """model="LaCrosse-TX31UIT"} 1677374905.538138""", prom) diff --git a/tests/timestamp_sample.txt b/tests/timestamp_sample.txt new file mode 100644 index 0000000..95a795e --- /dev/null +++ b/tests/timestamp_sample.txt @@ -0,0 +1,8 @@ +{"time" : "2023-02-26 01:28:25Z", "model" : "LaCrosse-TX", "id" : 1, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "2023-02-25 17:28:25-0800", "model" : "LaCrosse-TX", "id" : 2, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "2023-02-25T17:28:25-0800", "model" : "LaCrosse-TX", "id" : 3, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "1677374905", "model" : "LaCrosse-TX", "id" : 4, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "2023-02-26 01:28:25.538138Z", "model" : "LaCrosse-TX31UIT", "id" : 101, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "2023-02-25 17:28:25.538138-0800", "model" : "LaCrosse-TX31UIT", "id" : 102, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "2023-02-25T17:28:25.538138-0800", "model" : "LaCrosse-TX31UIT", "id" : 103, "temperature_C" : 4.000, "mic" : "PARITY"} +{"time" : "1677374905.538138", "model" : "LaCrosse-TX31UIT", "id" : 104, "temperature_C" : 4.000, "mic" : "PARITY"} \ No newline at end of file