Skip to content

Commit

Permalink
Add AlertManager API
Browse files Browse the repository at this point in the history
  • Loading branch information
goughes committed Feb 24, 2021
1 parent e049d6c commit 18bc5f3
Showing 1 changed file with 123 additions and 0 deletions.
123 changes: 123 additions & 0 deletions src/python/WMCore/Services/AlertManager/AlertManagerAPI.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
"""
AlertManagerAPI - send alerts to MONIT AlertManager via API calls
"""

from __future__ import division
from datetime import tzinfo, timedelta, datetime
import socket
import time as _time
import json

from WMCore.Services.pycurl_manager import RequestHandler


class AlertManagerAPI(object):

"""
configDict: the WMCore service configuration
alertName: a unique name for the alert
severity: low, medium, high
labels: a customizable dict of key:value pairs used for routing the alert, ex. "tag": "microservices"
must be configured in AlertManager by MONIT team
end: how many minutes until the alarm is silenced
generatorURL: this URL will be sent to AlertManager and configured as a clickable "Source" link in the web interface
annotations: a customizable dict of key:value pairs to be appended to the alert, typically:
"hostname": "the hostname where the alert generated" (appended automatically)
"summary": "a short description of the alert"
"description": "a longer informational message with details about the alert"
AlertManager JSON format reference: https://www.prometheus.io/docs/alerting/latest/clients/
[
{
"labels": {
"alertname": "<requiredAlertName>",
"<labelname>": "<labelvalue>",
...
},
"annotations": {
"<labelname>": "<labelvalue>",
...
},
"startsAt": "<rfc3339>", # optional, will be current time if not present
"endsAt": "<rfc3339>",
"generatorURL": "<generator_url>"
},
]
"""

def __init__(self, configDict, alertName, severity, labels, end, generatorURL, annotations):
self.alertManagerUrl = configDict.get("alertManagerUrl", "http://cms-monitoring.cern.ch:30093/api/v1/alerts")
self.alertName = alertName
self.end = end
self.severity = severity
self.labels = labels
self.generatorURL = generatorURL
self.annotations = annotations

def send(self):
headers = {"Content-Type": "application/json"}
request = []
alert = {}

self.labels["alertname"] = self.alertName
alert["labels"] = self.labels
# add the sender's hostname as an annotation
self.annotations["hostname"] = socket.gethostname()
alert["annotations"] = self.annotations

# In python3 we won't need the LocalTimezone class
# Will change to d = datetime.now().astimezone() + timedelta(minutes = self.end)
ltz = LocalTimezone()
d = datetime.now(ltz) + timedelta(minutes = self.end)
alert["endsAt"] = d.isoformat("T")
alert["generatorURL"] = self.generatorURL

request.append(alert)
# need to do this because pycurl_manager only accepts dict and encoded strings type
params = json.dumps(request)

mgr = RequestHandler()
res = mgr.getdata(self.alertManagerUrl, params=params, headers=headers, verb='POST')

return res

# Below is a required python 2 class to determine current timezone
# https://docs.python.org/2/library/datetime.html#tzinfo-objects
# Everything below can be removed when we move to python3

ZERO = timedelta(0)
STDOFFSET = timedelta(seconds=-_time.timezone)
if _time.daylight:
DSTOFFSET = timedelta(seconds=-_time.altzone)
else:
DSTOFFSET = STDOFFSET

DSTDIFF = DSTOFFSET - STDOFFSET


class LocalTimezone(tzinfo):

def utcoffset(self, dt):
if self._isdst(dt):
return DSTOFFSET
else:
return STDOFFSET

def dst(self, dt):
if self._isdst(dt):
return DSTDIFF
else:
return ZERO

def tzname(self, dt):
return _time.tzname[self._isdst(dt)]

def _isdst(self, dt):
tt = (dt.year, dt.month, dt.day,
dt.hour, dt.minute, dt.second,
dt.weekday(), 0, 0)
stamp = _time.mktime(tt)
tt = _time.localtime(stamp)
return tt.tm_isdst > 0

0 comments on commit 18bc5f3

Please sign in to comment.