Add link to Prometheus UI in alert description
- extract from generatorURL field
- add ALERTS{alertname="<name>",alertstate="firing"} query
- unify alert subject format with email and slack
Change-Id: Ic16e09552dc447861dc552f60c9e313234dadd00
Related-PROD: PRODX-7212
diff --git a/entrypoint.sh b/entrypoint.sh
index 4e212e2..17540a2 100755
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -2,6 +2,7 @@
export SIMPLE_SETTINGS=${SIMPLE_SETTINGS:-sf_notifier.settings.production}
export SF_NOTIFIER_ALERT_ID_HASH_FUNC=${SF_NOTIFIER_ALERT_ID_HASH_FUNC:-sha256}
+export SF_NOTIFIER_ADD_LINKS=${SF_NOTIFIER_ADD_LINKS:-False}
WORKERS=${SF_NOTIFIER_WORKERS:-4}
BUFFER=${SF_NOTIFIER_BUFFER_SIZE:-32768}
diff --git a/sf_notifier/helpers.py b/sf_notifier/helpers.py
index e9c221b..1c99a0a 100644
--- a/sf_notifier/helpers.py
+++ b/sf_notifier/helpers.py
@@ -14,7 +14,7 @@
# under the License.
import os
-
+import urllib
RESOLVED_STATUSES = ('UP', 'OK', 'resolved')
@@ -25,7 +25,12 @@
return file_path
-def _format_subject(alert):
+def _format_subject(alert, add_links=False):
+ if add_links:
+ severity = alert['labels']['severity'].upper()
+ name = alert['labels']['alertname']
+ return '[{}] {}'.format(severity, name)
+
subject = alert['annotations']['summary']
host = alert['labels'].get('host')
if host is not None:
@@ -37,11 +42,35 @@
return description.split(' Modify the')[0]
-def _format_description(alert):
- return _remove_advice(alert['annotations']['description'])
+def _get_prometheus_link(alert):
+ # switch from Console to Graph tab to show trends
+ condition_url = alert['generatorURL'].replace('g0.tab=1', 'g0.tab=0')
+
+ # list alert instances in the cluster
+ # because of Python formatting - double braces were used
+ alert_query = ('g1.range_input=1h&'
+ 'g1.expr=ALERTS{{alertname="{alertname}",'
+ 'alertstate="firing"}}&'
+ 'g1.tab=0')
+ alert_query = alert_query.format(**alert['labels'])
+ alert_query = urllib.quote(alert_query)
+ return '{}&{}'.format(condition_url, alert_query)
-def alert_fields_and_action(alert):
+def _format_description(alert, add_links=False):
+ description_old = _remove_advice(alert['annotations']['description'])
+
+ if not add_links:
+ return description_old
+
+ msg = ('To check trends of the alert underlying condition metric '
+ ' and alert occurrence click on Prometheus UI link')
+
+ prometheus_url = _get_prometheus_link(alert)
+ return '{}\n\n{}:\n{}'.format(description_old, msg, prometheus_url)
+
+
+def alert_fields_and_action(alert, add_links=False):
fields = []
if alert['status'] in RESOLVED_STATUSES:
@@ -50,7 +79,7 @@
else:
action = 'create_case'
# Order of keys matters
- fields.append(_format_subject(alert))
- fields.append(_format_description(alert))
+ fields.append(_format_subject(alert, add_links))
+ fields.append(_format_description(alert, add_links))
fields.append(alert['labels'])
return fields, action
diff --git a/sf_notifier/server.py b/sf_notifier/server.py
index 4d3ddc0..92144b8 100644
--- a/sf_notifier/server.py
+++ b/sf_notifier/server.py
@@ -70,7 +70,8 @@
for alert in data['alerts']:
try:
alert['labels']['env_id'] = sf_cli.environment
- fields, action = alert_fields_and_action(alert)
+ fields, action = alert_fields_and_action(
+ alert, add_links=settings.ADD_LINKS)
except KeyError as key:
msg = 'Alert misses {} key.'.format(key)
app.logger.error(msg)
diff --git a/sf_notifier/settings/development.py b/sf_notifier/settings/development.py
index 08607ac..06d3065 100644
--- a/sf_notifier/settings/development.py
+++ b/sf_notifier/settings/development.py
@@ -1,3 +1,5 @@
+import os
+
VERSION = 'development'
LOGGING = {
@@ -27,4 +29,7 @@
'CONFIGURE_LOGGING': True,
}
+ADD_LINKS = os.environ.get('SF_NOTIFIER_ADD_LINKS',
+ False) in ['true', 'True', True]
+
SF_CONFIG = {}
diff --git a/sf_notifier/settings/production.py b/sf_notifier/settings/production.py
index 931c764..dfc9639 100644
--- a/sf_notifier/settings/production.py
+++ b/sf_notifier/settings/production.py
@@ -1,3 +1,5 @@
+import os
+
VERSION = 'production'
LOGGING = {
@@ -37,4 +39,7 @@
'CONFIGURE_LOGGING': True,
}
+ADD_LINKS = os.environ.get('SF_NOTIFIER_ADD_LINKS',
+ False) in ['true', 'True', True]
+
SF_CONFIG = {}
diff --git a/sf_notifier/vars/development b/sf_notifier/vars/development
index b6c27d6..3f9fb6d 100644
--- a/sf_notifier/vars/development
+++ b/sf_notifier/vars/development
@@ -1 +1,2 @@
export SIMPLE_SETTINGS=sf_notifier.settings.development
+export SF_NOTIFIER_ADD_LINKS=True
diff --git a/sf_notifier/vars/production b/sf_notifier/vars/production
index 7c6bbb8..9436c9b 100644
--- a/sf_notifier/vars/production
+++ b/sf_notifier/vars/production
@@ -1 +1,2 @@
export SIMPLE_SETTINGS=sf_notifier.settings.production
+export SF_NOTIFIER_ADD_LINKS=False