Merge pull request #10308 from goughes/alertmanager-api

Add AlertManager API

src/python/Utils/Timers.py

@@ -7,6 +7,7 @@
 
 from builtins import object
 import time
+from datetime import tzinfo, timedelta
 
 
 def timeFunction(func):
@@ -52,3 +53,47 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         runtime = end - self.start
         msg = '{label} took {time} seconds to complete'
         print(msg.format(label=self.label, time=runtime))
+
+
+class LocalTimezone(tzinfo):
+    """
+    A required python 2 class to determine current timezone for formatting rfc3339 timestamps
+    Required for sending alerts to the MONIT AlertManager
+    Can be removed once WMCore starts using python3
+
+    Details of class can be found at: https://docs.python.org/2/library/datetime.html#tzinfo-objects
+    """
+
+    def __init__(self):
+        super(LocalTimezone, self).__init__()
+        self.ZERO = timedelta(0)
+        self.STDOFFSET = timedelta(seconds=-time.timezone)
+        if time.daylight:
+            self.DSTOFFSET = timedelta(seconds=-time.altzone)
+        else:
+            self.DSTOFFSET = self.STDOFFSET
+
+        self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET
+
+    def utcoffset(self, dt):
+        if self._isdst(dt):
+            return self.DSTOFFSET
+        else:
+            return self.STDOFFSET
+
+    def dst(self, dt):
+        if self._isdst(dt):
+            return self.DSTDIFF
+        else:
+            return self.ZERO
+
+    def tzname(self, dt):
+        return time.tzname[self._isdst(dt)]
+
+    def _isdst(self, dt):
+        tt = (dt.year, dt.month, dt.day,
+              dt.hour, dt.minute, dt.second,
+              dt.weekday(), 0, 0)
+        stamp = time.mktime(tt)
+        tt = time.localtime(stamp)
+        return tt.tm_isdst > 0

src/python/WMCore/Services/AlertManager/AlertManagerAPI.py

@@ -0,0 +1,105 @@
+"""
+
+AlertManagerAPI - send alerts to MONIT AlertManager via API calls
+"""
+
+from __future__ import division
+from datetime import timedelta, datetime
+import socket
+import json
+import logging
+
+from WMCore.Services.pycurl_manager import RequestHandler
+from Utils.Timers import LocalTimezone
+
+
+class AlertManagerAPI(object):
+    """
+    A class used to send alerts via the MONIT AlertManager API
+    """
+
+    def __init__(self, alertManagerUrl, logger=None):
+        self.alertManagerUrl = alertManagerUrl
+        # sender's hostname is added as an annotation
+        self.hostname = socket.gethostname()
+        self.mgr = RequestHandler()
+        self.ltz = LocalTimezone()
+        self.headers = {"Content-Type": "application/json"}
+        self.validSeverity = ["high", "medium", "low"]
+        self.logger = logger if logger else logging.getLogger()
+
+    def sendAlert(self, alertName, severity, summary, description, service, tag="wmcore", endSecs=600, generatorURL=""):
+        """
+        :param alertName: a unique name for the alert
+        :param severity: low, medium, high
+        :param summary: a short description of the alert
+        :param description: a longer informational message with details about the alert
+        :param service: the name of the service firing an alert
+        :param tag: a unique tag used to help route the alert
+        :param endSecs: how many minutes until the alarm is silenced
+        :param generatorURL: this URL will be sent to AlertManager and configured as a clickable "Source" link in the web interface
+
+        AlertManager JSON format reference: https://www.prometheus.io/docs/alerting/latest/clients/
+        [
+          {
+            "labels": {
+            "alertname": "<requiredAlertName>",
+            "<labelname>": "<labelvalue>",
+            ...
+          },
+            "annotations": {
+              "<labelname>": "<labelvalue>",
+            ...
+          },
+            "startsAt": "<rfc3339>", # optional, will be current time if not present
+            "endsAt": "<rfc3339>",
+            "generatorURL": "<generator_url>" # optional
+          },
+        ]
+        """
+
+        if not self._isValidSeverity(severity):
+            return False
+
+        request = []
+        alert = {}
+        labels = {}
+        annotations = {}
+
+        # add labels
+        labels["alertname"] = alertName
+        labels["severity"] = severity
+        labels["tag"] = tag
+        labels["service"] = service
+        alert["labels"] = labels
+
+        # add annotations
+        annotations["hostname"] = self.hostname
+        annotations["summary"] = summary
+        annotations["description"] = description
+        alert["annotations"] = annotations
+
+        # In python3 we won't need the LocalTimezone class
+        # Will change to d = datetime.now().astimezone() + timedelta(seconds=endSecs)
+        d = datetime.now(self.ltz) + timedelta(seconds=endSecs)
+        alert["endsAt"] = d.isoformat("T")
+        alert["generatorURL"] = generatorURL
+
+        request.append(alert)
+        # need to do this because pycurl_manager only accepts dict and encoded strings type
+        params = json.dumps(request)
+
+        res = self.mgr.getdata(self.alertManagerUrl, params=params, headers=self.headers, verb='POST')
+
+        return res
+
+    def _isValidSeverity(self, severity):
+        """
+        Used to check if the severity of the alert matches the valid levels: low, medium, high
+        :param severity: severity of the alert
+        :return: True or False
+        """
+        if severity not in self.validSeverity:
+            logging.critical("Alert submitted to AlertManagerAPI with invalid severity: %s", severity)
+            return False
+        return True