diff --git a/src/python/Utils/Timers.py b/src/python/Utils/Timers.py index 7ada58245a..dbdd46583f 100644 --- a/src/python/Utils/Timers.py +++ b/src/python/Utils/Timers.py @@ -7,6 +7,7 @@ from builtins import object import time +from datetime import tzinfo, timedelta def timeFunction(func): @@ -52,3 +53,47 @@ def __exit__(self, exc_type, exc_val, exc_tb): runtime = end - self.start msg = '{label} took {time} seconds to complete' print(msg.format(label=self.label, time=runtime)) + + +class LocalTimezone(tzinfo): + + """ + A required python 2 class to determine current timezone for formatting rfc3339 timestamps + https://docs.python.org/2/library/datetime.html#tzinfo-objects + Required for sending alerts to the MONIT AlertManager + Can be removed once WMCore starts using python3 + """ + + def __init__(self): + super(LocalTimezone, self).__init__() + self.ZERO = timedelta(0) + self.STDOFFSET = timedelta(seconds=-time.timezone) + if time.daylight: + self.DSTOFFSET = timedelta(seconds=-time.altzone) + else: + self.DSTOFFSET = self.STDOFFSET + + self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET + + def utcoffset(self, dt): + if self._isdst(dt): + return self.DSTOFFSET + else: + return self.STDOFFSET + + def dst(self, dt): + if self._isdst(dt): + return self.DSTDIFF + else: + return self.ZERO + + def tzname(self, dt): + return time.tzname[self._isdst(dt)] + + def _isdst(self, dt): + tt = (dt.year, dt.month, dt.day, + dt.hour, dt.minute, dt.second, + dt.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) + return tt.tm_isdst > 0 diff --git a/src/python/WMCore/Services/AlertManager/AlertManagerAPI.py b/src/python/WMCore/Services/AlertManager/AlertManagerAPI.py new file mode 100644 index 0000000000..b63e1cb7fb --- /dev/null +++ b/src/python/WMCore/Services/AlertManager/AlertManagerAPI.py @@ -0,0 +1,94 @@ +""" + +AlertManagerAPI - send alerts to MONIT AlertManager via API calls +""" + +from __future__ import division +from datetime import timedelta, datetime +import socket +import json +import logging + +from WMCore.Services.pycurl_manager import RequestHandler +from Utils.Timers import LocalTimezone + + +class AlertManagerAPI(object): + """ + A class used to send alerts via the MONIT AlertManager API + """ + + def __init__(self, alertManagerUrl): + self.alertManagerUrl = alertManagerUrl + # sender's hostname is added as an annotation + self.hostname = socket.gethostname() + self.mgr = RequestHandler() + self.ltz = LocalTimezone() + + def send(self, alertName, severity, summary, description, tag="wmcore", service="", endSecs=600, generatorURL=""): + """ + :param alertName: a unique name for the alert + :param severity: low, medium, high + :param summary: a short description of the alert + :param description: a longer informational message with details about the alert + :param service: the name of the service firing an alert + :param endSecs: how many minutes until the alarm is silenced + :param generatorURL: this URL will be sent to AlertManager and configured as a clickable "Source" link in the web interface + + AlertManager JSON format reference: https://www.prometheus.io/docs/alerting/latest/clients/ + [ + { + "labels": { + "alertname": "", + "": "", + ... + }, + "annotations": { + "": "", + ... + }, + "startsAt": "", # optional, will be current time if not present + "endsAt": "", + "generatorURL": "" # optional + }, + ] + """ + + validSeverity = ["high", "medium", "low"] + if severity not in validSeverity: + logging.critical("Alert submitted to AlertManagerAPI with invalid severity: %s" % severity) + return + + headers = {"Content-Type": "application/json"} + request = [] + alert = {} + labels = {} + annotations = {} + + # add labels + labels["alertname"] = alertName + labels["severity"] = severity + labels["tag"] = tag + labels["service"] = service + alert["labels"] = labels + + # add annotations + annotations["hostname"] = self.hostname + annotations["summary"] = summary + annotations["description"] = description + alert["annotations"] = annotations + + # In python3 we won't need the LocalTimezone class + # Will change to d = datetime.now().astimezone() + timedelta(seconds = endSecs) + d = datetime.now(self.ltz) + timedelta(seconds = endSecs) + alert["endsAt"] = d.isoformat("T") + alert["generatorURL"] = generatorURL + + request.append(alert) + # need to do this because pycurl_manager only accepts dict and encoded strings type + params = json.dumps(request) + + res = self.mgr.getdata(self.alertManagerUrl, params=params, headers=headers, verb='POST') + + return res + \ No newline at end of file