feat: Rework metrics so we can handle more types of messages, and rec…

…ord the last time a message was seen for a device.

BREAKING CHANGE: All metrics have been renamed.

bin/prom433

@@ -20,18 +20,24 @@ import sys
 
 from prom433 import get_arguments, serve, prometheus
 
+def on_connect(client, userdata, flags, rc):
+    print("Connected with result code "+str(rc))
+
+    client.subscribe("rtl_433/+/events")
+
 def main():
     args = get_arguments(sys.argv[1:])
 
-    client = mqtt.Client("prom433")
-    client.connect(args.mqtt)
-
-    client.loop_start()
+    client = mqtt.Client()
 
-    client.subscribe("rtl_433/+/events")
+    client.on_connect = on_connect
     client.on_message=lambda client, userdata, message: \
                             prometheus(message.payload.decode("utf-8"))
 
+    client.connect(args.mqtt)
+
+    client.loop_start()
+
     serve(args)
 
 if __name__ == "__main__":

prom433/prometheus.py

@@ -14,54 +14,63 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from datetime import datetime
 import json
 import logging
 
-METRICS = {}
-
-WEATHER_METRIC = "%s{id=\"%i\"} %f"
+METRICS = {
+    "prom433_last_messsage": {}
+}
 
-WEATHER_TEMP_HELP = \
-    "# HELP weather_temperature The temperature in degrees celcius."
-WEATHER_TEMP_TYPE = "# TYPE weather_temperature gauge"
+HELP_FORMAT = "#HELP %s %s"
+TYPE_FORMAT = "#TYPE %s %s "
+METRIC_FORMAT = "%s{%s} %f"
 
-WEATHER_HUM_HELP = "# HELP weather_humidity The humidity in %."
-WEATHER_HUM_TYPE = "# TYPE weather_humidity gauge"
+TEMP_HELP = \
+    "The temperature in degrees celcius."
+TEMP_TYPE = "gauge"
 
-WEATHER_WIND_AVG_HELP = \
-    "# HELP weather_wind_avg The average windspeed in km/h."
-WEATHER_WIND_AVG_TYPE = "# TYPE weather_wind_avg gauge"
-WEATHER_WIND_MAX_HELP = \
-    "# HELP weather_wind_max The maximum windspeed in km/h."
-WEATHER_WIND_MAX_TYPE = "# TYPE weather_wind_max gauge"
-WEATHER_WIND_DIR_HELP = \
-    "# HELP weather_wind_dir The wind direction in degrees."
-WEATHER_WIND_DIR_TYPE = "# TYPE weather_wind_dir gauge"
+HUMIDITY_HELP = "The humidity in %."
+HUMIDITY_TYPE = "gauge"
 
-WEATHER_RAIN_HELP = "# HELP weather_rain The total rainfall in mm."
-WEATHER_RAIN_TYPE = "# TYPE weather_rain counter"
+WIND_AVG_HELP = "The average windspeed in km/h."
+WIND_AVG_TYPE = "gauge"
+WIND_MAX_HELP = "The maximum windspeed in km/h."
+WIND_MAX_TYPE = "gauge"
+WIND_DIR_HELP = "The wind direction in degrees."
+WIND_DIR_TYPE = "gauge"
 
-WEATHER_BATTERY_HELP = "# HELP weather_rain The battery status."
-WEATHER_BATTERY_TYPE = "# TYPE weather_battery gauge"
+RAIN_HELP = "The total rainfall in mm."
+RAIN_TYPE = "counter"
 
-NEXUS_METRIC = "%s{id=\"%i\", channel=\"%i\"} %f"
+BATTERY_HELP = "The battery status."
+BATTERY_TYPE = "gauge"
 
-NEXUS_TEMP_HELP = \
-    "# HELP nexus_temperature The temperature in degrees celcius."
-NEXUS_TEMP_TYPE = "# TYPE nexus_temperature gauge"
-NEXUS_HUM_HELP = "# HELP nexus_humidity The humidity in %."
-NEXUS_HUM_TYPE = "# TYPE nexus_humidity gauge"
-NEXUS_BATTERY_HELP = "# HELP nexus_battery The battery status."
-NEXUS_BATTERY_TYPE = "# TYPE nexus_battery gauge"
+LAST_MESSAGE_HELP = "The time the last message was received."
+LAST_MESSAGE_TYPE = "counter"
 
 METRICS_PREFIXES = {
-    "weather_temperature": [WEATHER_TEMP_HELP, WEATHER_TEMP_TYPE],
-    "nexus_temperature": [NEXUS_TEMP_HELP, NEXUS_TEMP_TYPE],
+    "prom433_battery_ok": [BATTERY_HELP, BATTERY_TYPE],
+    "prom433_temperature": [TEMP_HELP, TEMP_TYPE],
+    "prom433_humidity": [HUMIDITY_HELP, HUMIDITY_TYPE],
+    "prom433_wind_dir_deg": [WIND_DIR_HELP, WIND_DIR_TYPE],
+    "prom433_wind_avg": [WIND_AVG_HELP, WIND_AVG_TYPE],
+    "prom433_wind_max": [WIND_MAX_HELP, WIND_MAX_TYPE],
+    "prom433_rain": [RAIN_HELP, RAIN_TYPE],
+    "prom433_last_messsage": [LAST_MESSAGE_HELP, LAST_MESSAGE_TYPE]
 }
 
-METRIC_FORMATS = {
-    "weather_temperature": WEATHER_METRIC,
-    "nexus_temperature": NEXUS_METRIC
+TAG_KEYS = {"id", "channel", "model"}
+
+METRIC_NAME = {
+    "battery_ok": "prom433_battery_ok",
+    "temperature_C": "prom433_temperature",
+    "humidity": "prom433_humidity",
+    "wind_dir_deg": "prom433_wind_dir_deg",
+    "wind_avg_km_h": "prom433_wind_avg",
+    "wind_max_km_h": "prom433_wind_max",
+    "rain_mm": "prom433_rain",
+    "last_message": "prom433_last_message"
 }
 
 # {"time" : "2021-05-08 15:27:58", "model" : "Fineoffset-WHx080",
@@ -75,31 +84,41 @@
 def prometheus(message):
     payload = json.loads(message)
 
-    if payload["model"] == "Fineoffset-WHx080":
-        weather(payload)
-    elif payload["model"] == "Nexus-TH":
-        nexus(payload)
-    else:
-        model = payload["model"]
-        logging.warn(f"Unknown message model {model}")
+    tags, data, unknown = {}, {}, {}
 
+    for key, value in payload.items():
+        if key == "time":
+            time_value = datetime.strptime(payload[key], "%Y-%m-%d %H:%M:%S") \
+                        .timestamp()
+        elif key in TAG_KEYS:
+            tags[key] = value
+        elif key in METRIC_NAME:
+            data[key] = value
+        else:
+            unknown[key] = value
 
-def get_metrics():
-    lines = []
-    for metric_name in sorted(set([m[0] for m in METRICS])):
-        lines.extend(METRICS_PREFIXES[metric_name])
-        for metric_key, value in METRICS.items():
-            if metric_key[0] == metric_name:
-                lines.append(METRIC_FORMATS[metric_name]
-                             % (metric_key + (value, )))
+    tag_value = ", ".join(["%s=\"%s\"" % (k, payload[k])
+                           for k in sorted(tags)])
 
-    return "\n".join(lines)
+    METRICS["prom433_last_messsage"][tag_value] = time_value
+    for key in data:
+        metric = METRIC_NAME[key]
+        if metric not in METRICS:
+            METRICS[metric] = {}
+        METRICS[metric][tag_value] = payload[key]
 
+    if len(unknown) > 0:
+        logging.warn(f"Message has unknown tags ({unknown}): {message}")
 
-def weather(payload):
-    METRICS[("weather_temperature", payload["id"])] = payload["temperature_C"]
 
+def get_metrics():
+    lines = []
+    for metric_name in sorted(METRICS.keys()):
+        lines.append(HELP_FORMAT
+                     % (metric_name, METRICS_PREFIXES[metric_name][0]))
+        lines.append(TYPE_FORMAT
+                     % (metric_name, METRICS_PREFIXES[metric_name][1]))
+        for (tags, values) in METRICS[metric_name].items():
+            lines.append(METRIC_FORMAT % (metric_name, tags, values))
 
-def nexus(payload):
-    METRICS[("nexus_temperature", payload["id"], payload["channel"])] = \
-        payload["temperature_C"]
+    return "\n".join(lines)

tests/output_sample.txt

@@ -10,4 +10,6 @@
 {"time" : "2021-07-12 20:48:29", "model" : "Nexus-TH", "id" : 137, "channel" : 2, "battery_ok" : 0, "temperature_C" : 23.800, "humidity" : 66}
 {"time" : "2021-07-12 20:48:47", "model" : "Fineoffset-WHx080", "subtype" : 0, "id" : 250, "battery_ok" : 1, "temperature_C" : 16.000, "humidity" : 99, "wind_dir_deg" : 180, "wind_avg_km_h" : 0.000, "wind_max_km_h" : 0.000, "rain_mm" : 95.400, "mic" : "CRC"}
 {"time" : "2021-07-12 20:48:49", "model" : "Nexus-TH", "id" : 132, "channel" : 3, "battery_ok" : 0, "temperature_C" : 23.100, "humidity" : 65}
-{"time" : "2021-07-12 20:48:50", "model" : "Nexus-TH", "id" : 147, "channel" : 1, "battery_ok" : 0, "temperature_C" : 23.100, "humidity" : 62}
+{"time" : "2021-07-12 20:48:50", "model" : "Nexus-TH", "id" : 147, "channel" : 1, "battery_ok" : 0, "temperature_C" : 23.100, "humidity" : 62}
+{"time":"2022-04-27 20:40:05","model":"Eurochron-EFTH800","id":2820,"channel":4,"battery_ok":1,"temperature_C":21.0,"humidity":44,"mic":"CRC"}
+{"time":"2022-04-27 20:40:15","model":"Eurochron-EFTH800","id":1940,"channel":2,"battery_ok":1,"temperature_C":22.3,"humidity":41,"mic":"CRC"}

tests/test_prometheus.py

@@ -42,6 +42,12 @@ def test_prometheus(self):
         print(prom)
 
         self.assertIn(
-            """nexus_temperature{id="147", channel="1"} 23.100000""", prom)
+            """prom433_temperature{channel="1", id="147\"""" +
+            """, model="Nexus-TH"} 23.100000""", prom)
         self.assertIn(
-            """weather_temperature{id="250"} 16.000000""", prom)
+            """prom433_temperature{id="250", """ +
+            """model="Fineoffset-WHx080"} 16.000000""",
+            prom)
+        self.assertIn(
+            """prom433_temperature{channel="2", id="1940", """ +
+            """model="Eurochron-EFTH800"} 22.300000""", prom)

tests/test_server.py

@@ -69,5 +69,5 @@ def test_metrics(self):
         handler.do_GET()
 
         handler.wfile.seek(0)
-        self.assertTrue(
-            "weather_temperature" in handler.wfile.read().decode("utf8"))
+        self.assertIn(
+            "prom433_temperature", handler.wfile.read().decode("utf8"))