diff --git a/src/python/Utils/Timers.py b/src/python/Utils/Timers.py index 7ada58245a..2206b67801 100644 --- a/src/python/Utils/Timers.py +++ b/src/python/Utils/Timers.py @@ -7,6 +7,7 @@ from builtins import object import time +from datetime import tzinfo, timedelta, datetime def timeFunction(func): @@ -52,3 +53,44 @@ def __exit__(self, exc_type, exc_val, exc_tb): runtime = end - self.start msg = '{label} took {time} seconds to complete' print(msg.format(label=self.label, time=runtime)) + +# Below is a required python 2 class to determine current timezone for formatting rfc3339 timestamps +# https://docs.python.org/2/library/datetime.html#tzinfo-objects +# Required for sending alerts to the MONIT AlertManager +# Can be removed once WMCore starts using python3 + +class LocalTimezone(tzinfo): + + def __init__(self): + + self.ZERO = timedelta(0) + self.STDOFFSET = timedelta(seconds=-time.timezone) + if time.daylight: + self.DSTOFFSET = timedelta(seconds=-time.altzone) + else: + self.DSTOFFSET = self.STDOFFSET + + self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET + + def utcoffset(self, dt): + if self._isdst(dt): + return self.DSTOFFSET + else: + return self.STDOFFSET + + def dst(self, dt): + if self._isdst(dt): + return self.DSTDIFF + else: + return self.ZERO + + def tzname(self, dt): + return time.tzname[self._isdst(dt)] + + def _isdst(self, dt): + tt = (dt.year, dt.month, dt.day, + dt.hour, dt.minute, dt.second, + dt.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) + return tt.tm_isdst > 0 diff --git a/src/python/WMCore/Services/AlertManager/AlertManagerAPI.py b/src/python/WMCore/Services/AlertManager/AlertManagerAPI.py new file mode 100644 index 0000000000..ccd190da1f --- /dev/null +++ b/src/python/WMCore/Services/AlertManager/AlertManagerAPI.py @@ -0,0 +1,84 @@ +""" + +AlertManagerAPI - send alerts to MONIT AlertManager via API calls +""" + +from __future__ import division +from datetime import tzinfo, timedelta, datetime +import socket +import time as _time +import json + +from WMCore.Services.pycurl_manager import RequestHandler +from Utils.Timers import LocalTimezone + +class AlertManagerAPI(object): + + + + def __init__(self, alertManagerUrl): + self.alertManagerUrl = alertManagerUrl + # sender's hostname is added as an annotation + self.hostname = socket.gethostname() + self.mgr = RequestHandler() + self.ltz = LocalTimezone() + + def send(self, alertName, severity, labels, end, generatorURL, annotations): + """ + configDict: the WMCore service configuration + alertName: a unique name for the alert + severity: low, medium, high + labels: a customizable dict of key:value pairs used for routing the alert, ex. "tag": "microservices" + must be configured in AlertManager by MONIT team + end: how many minutes until the alarm is silenced + generatorURL: this URL will be sent to AlertManager and configured as a clickable "Source" link in the web interface + annotations: a customizable dict of key:value pairs to be appended to the alert, typically: + "hostname": "the hostname where the alert generated" (appended automatically) + "summary": "a short description of the alert" + "description": "a longer informational message with details about the alert" + + AlertManager JSON format reference: https://www.prometheus.io/docs/alerting/latest/clients/ + [ + { + "labels": { + "alertname": "", + "": "", + ... + }, + "annotations": { + "": "", + ... + }, + "startsAt": "", # optional, will be current time if not present + "endsAt": "", + "generatorURL": "" + }, + ] + """ + + headers = {"Content-Type": "application/json"} + request = [] + alert = {} + + # add labels + labels["alertname"] = alertName + labels["severity"] = severity + alert["labels"] = labels + + # add annotations + annotations["hostname"] = self.hostname + alert["annotations"] = annotations + + # In python3 we won't need the LocalTimezone class + # Will change to d = datetime.now().astimezone() + timedelta(minutes = self.end) + d = datetime.now(self.ltz) + timedelta(minutes = end) + alert["endsAt"] = d.isoformat("T") + alert["generatorURL"] = generatorURL + + request.append(alert) + # need to do this because pycurl_manager only accepts dict and encoded strings type + params = json.dumps(request) + + res = self.mgr.getdata(self.alertManagerUrl, params=params, headers=headers, verb='POST') + + return res \ No newline at end of file