Initial webhook receiver
Change-Id: I4786b133f103a65a381e2b281224b0a9bc8d4182
Related-bug: PROD-24880 (PROD:24880)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..67385f1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+venv
+.tox
+*.egg-info
+**/*.pyc
+sf_notifier/vars/secret
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..04b5e7d
--- /dev/null
+++ b/README.md
@@ -0,0 +1,51 @@
+# sf-notifier
+
+Prometheus Alertmanager webhook receiver sending alert notification to Salesforce.
+
+## Development
+
+### Setup
+
+Install Python dependencies:
+
+```
+$ virtualenv venv
+$ venv/bin/pip install -e .
+$ source sf_notifier/vars/development
+```
+
+Add to `settings/development.py` credentials for your Salesforce customer (not engineering) account:
+
+```
+SF_CONFIG = {
+ 'AUTH_URL': 'xxx',
+ 'USERNAME': 'xxx',
+ 'PASSWORD': 'xxx',
+ 'ORGANIZATION_ID': 'xxx',
+ 'ENVIRONMENT_ID': 'xxx',
+ 'SANDBOX_ENABLED': True
+}
+```
+
+You may also specify environment variables to override Python settings:
+
+```
+export SFDC_AUTH_URL="xxx"
+export SFDC_USERNAME="xxx"
+export SFDC_PASSWORD="xxx"
+export SFDC_ORGANIZATION_ID="xxx"
+export SFDC_ENVIRONMENT_ID="xxx"
+export SFDC_SANDBOX_ENABLED=true
+```
+
+Run server:
+
+```
+$ venv/bin/flask run
+```
+
+Check in browser:
+
+```
+http://127.0.0.1:5000/health
+```
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..d2154e7
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,22 @@
+asn1crypto==0.24.0
+certifi==2018.10.15
+cffi==1.11.5
+chardet==3.0.4
+Click==7.0
+cryptography==2.4.1
+enum34==1.1.6
+Flask==1.0.2
+idna==2.7
+ipaddress==1.0.22
+itsdangerous==1.1.0
+Jinja2==2.10
+MarkupSafe==1.1.0
+pycparser==2.19
+pyOpenSSL==18.0.0
+PyYAML==3.13
+requests==2.20.1
+simple-salesforce==0.74.2
+simple-settings==0.13.0
+six==1.11.0
+urllib3==1.24.1
+Werkzeug==0.14.1
diff --git a/run-func-tests.sh b/run-func-tests.sh
new file mode 100755
index 0000000..ba1f8ce
--- /dev/null
+++ b/run-func-tests.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+set -e
+
+exit 0
diff --git a/run-tests.sh b/run-tests.sh
new file mode 100755
index 0000000..ba1f8ce
--- /dev/null
+++ b/run-tests.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+set -e
+
+exit 0
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..2161fae
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,7 @@
+from distutils.core import setup
+
+setup(
+ name='sf-notifier',
+ version='0.1',
+ packages=['sf_notifier'],
+)
diff --git a/sf_notifier/__init__.py b/sf_notifier/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/sf_notifier/__init__.py
diff --git a/sf_notifier/helpers.py b/sf_notifier/helpers.py
new file mode 100644
index 0000000..8d9d7d5
--- /dev/null
+++ b/sf_notifier/helpers.py
@@ -0,0 +1,35 @@
+# Copyright 2018: Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+RESOLVED_STATUSES = ('UP', 'OK', 'resolved')
+
+
+def alert_fields_and_action(alert):
+ fields = []
+ action = 'create_case'
+
+ if alert['status'] in RESOLVED_STATUSES:
+ fields.append(alert['labels'])
+ action = 'close_case'
+ else:
+ # Order matters
+ fields.append('[sf-notifier] {}'.format(
+ alert['annotations']['summary'])
+ )
+ fields.append(alert['annotations']['description'])
+ fields.append(alert['status'])
+ fields.append(alert['labels'])
+ return fields, action
diff --git a/sf_notifier/salesforce/__init__.py b/sf_notifier/salesforce/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/sf_notifier/salesforce/__init__.py
diff --git a/sf_notifier/salesforce/client.py b/sf_notifier/salesforce/client.py
new file mode 100644
index 0000000..84fc0b4
--- /dev/null
+++ b/sf_notifier/salesforce/client.py
@@ -0,0 +1,208 @@
+# Copyright 2018: Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import hashlib
+import logging
+import os
+import uuid
+
+import requests
+
+from simple_salesforce import Salesforce
+from simple_salesforce import exceptions as sf_exceptions
+
+
+STATE_MAP = {
+ 'OK': '060 Informational',
+ 'UP': '060 Informational',
+ 'UNKNOWN': '070 Unknown',
+ 'WARNING': '080 Warning',
+ 'MINOR': '080 Warning',
+ 'MAJOR': '090 Critical',
+ 'CRITICAL': '090 Critical',
+ 'DOWN': '090 Critical',
+ 'UNREACHABLE': '090 Critical',
+}
+
+CONFIG_FIELD_MAP = {
+ 'auth_url': 'instance',
+ 'username': 'username',
+ 'password': 'password',
+ 'organization_id': 'organizationId',
+ 'environment_id': 'environment_id',
+ 'sandbox_enabled': 'domain',
+}
+
+logger = logging.getLogger(__name__)
+
+
+def sf_auth_retry(method):
+ def wrapper(self, *args, **kwargs):
+ try:
+ return method(self, *args, **kwargs)
+ except sf_exceptions.SalesforceExpiredSession:
+ logger.warning('Salesforce session expired.')
+ self._auth()
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+class SfNotifierError(Exception):
+ def __init__(self, message, errors):
+ super(SfNotifierError, self).__init__(message)
+ self.errors = errors
+
+
+class SalesforceClient(object):
+
+ def __init__(self, config):
+ self.session = requests.Session()
+ self.config = self._validate_config(config)
+ self.environment = self.config.pop('environment_id')
+ self._auth()
+ self._registered_alerts = {}
+
+ def _validate_config(self, config):
+ kwargs = {'session': self.session}
+
+ for param, field in CONFIG_FIELD_MAP.iteritems():
+ setting_var = param.upper()
+ env_var = 'SFDC_{}'.format(setting_var)
+ kwargs[field] = os.environ.get(
+ env_var, getattr(config, setting_var, None))
+
+ if field == 'domain':
+ kwargs[field] = kwargs[field] and 'test'
+ continue
+
+ if kwargs[field] is None:
+ msg = ('Invalid config: missing "{}" field or "{}" environment'
+ ' variable.').format(param, env_var)
+ logger.error(msg)
+ raise SfNotifierError(msg)
+ return kwargs
+
+ def _auth(self):
+ self.sf = Salesforce(**self.config)
+ logger.info('Salesforce authentication successful.')
+
+ def _get_alert_id(self, labels):
+ alert_id_data = ''
+ for key in sorted(labels):
+ alert_id_data += labels[key].replace(".", "\\.")
+ return hashlib.sha256(alert_id_data).hexdigest()
+
+ @sf_auth_retry
+ def _create_case(self, subject, body, labels, alert_id):
+
+ if alert_id in self._registered_alerts:
+ logger.info('Duplicate case for alert: {}.'.format(alert_id))
+ return 1, self._registered_alerts[alert_id]['Id']
+
+ severity = labels.get('severity', 'unknown').upper()
+ payload = {
+ 'Subject': subject,
+ 'Description': body,
+ 'IsMosAlert__c': 'true',
+ 'Alert_Priority__c': STATE_MAP.get(severity, '070 Unknown'),
+ 'Alert_Host__c': labels.get('host') or labels.get(
+ 'instance', 'UNKNOWN'
+ ),
+ 'Alert_Service__c': labels.get('service', 'UNKNOWN'),
+ 'Environment2__c': self.environment,
+ 'Alert_ID__c': alert_id,
+ }
+ logger.info('Try to create case: {}'.format(payload))
+ try:
+ case = self.sf.Case.create(payload)
+ except sf_exceptions.SalesforceMalformedRequest as ex:
+ msg = ex.content[0]['message']
+ err_code = ex.content[0]['errorCode']
+
+ if err_code == 'DUPLICATE_VALUE':
+ logger.info('Duplicate case: {}.'.format(msg))
+ case_id = msg.split()[-1]
+ self._registered_alerts[alert_id] = {'Id': case_id}
+ return 1, case_id
+ else:
+ raise
+
+ self._registered_alerts[alert_id] = {'Id': case['id']}
+ return 0, case['id']
+
+ @sf_auth_retry
+ def _get_case(self, case_id):
+ return self.sf.Case.get(case_id)
+
+ @sf_auth_retry
+ def _update_case(self, case_id, data):
+ return self.sf.Case.update(case_id, data)
+
+ @sf_auth_retry
+ def _close_case(self, case_id):
+ logger.info('Try to close case: {}.'.format(case_id))
+ update = self.sf.Case.update(
+ case_id,
+ {'Status': 'Auto-solved', 'Alert_ID__c': uuid.uuid4().hex}
+ )
+ logger.info('Closed case: {}.'.format(case_id))
+ return update
+
+ @sf_auth_retry
+ def _create_feed_item(self, subject, body, case_id):
+ feed_item = {'Title': subject, 'ParentId': case_id, 'Body': body}
+ return self.sf.FeedItem.create(feed_item)
+
+ @sf_auth_retry
+ def _get_case_by_alert_id(self, alert_id):
+ logger.info('Try to get case by alert ID: {}.'.format(alert_id))
+
+ if alert_id in self._registered_alerts:
+ return self._registered_alerts[alert_id]
+ try:
+ return self.sf.Case.get_by_custom_id('Alert_ID__c', alert_id)
+ except sf_exceptions.SalesforceResourceNotFound:
+ logger.warning('Alert ID: {} was already solved.'.format(alert_id))
+
+ def create_case(self, subject, body, status, labels):
+ alert_id = self._get_alert_id(labels)
+
+ error_code, case_id = self._create_case(subject, body,
+ labels, alert_id)
+
+ response = {'case_id': case_id, 'alert_id': alert_id,
+ 'status': 'created'}
+
+ if error_code != 2:
+ self._create_feed_item(subject, body, case_id)
+ if error_code == 1:
+ response['status'] = 'duplicate'
+ return response
+
+ def close_case(self, labels):
+ alert_id = self._get_alert_id(labels)
+ case = self._get_case_by_alert_id(alert_id)
+
+ response = {'alert_id': alert_id, 'status': 'resolved'}
+
+ if case is None:
+ return response
+
+ if self._registered_alerts.get(alert_id):
+ del self._registered_alerts[alert_id]
+
+ response['case_id'] = case['Id']
+ response['closed'] = self._close_case(case['Id'])
+ return response
diff --git a/sf_notifier/server.py b/sf_notifier/server.py
new file mode 100644
index 0000000..182f55d
--- /dev/null
+++ b/sf_notifier/server.py
@@ -0,0 +1,78 @@
+# Copyright 2018: Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+from logging.config import dictConfig
+
+from flask import Flask, Response, jsonify, request
+
+from sf_notifier.helpers import alert_fields_and_action
+from sf_notifier.salesforce.client import SalesforceClient
+
+from simple_salesforce.exceptions import SalesforceMalformedRequest
+
+from simple_settings import settings
+
+
+dictConfig(settings.LOGGING)
+
+app = Flask('__name__')
+sf_cli = SalesforceClient(settings.SF_CONFIG)
+
+
+@app.route('/health', methods=['GET'])
+def health():
+ app.logger.info('Health: OK!')
+ return 'OK!'
+
+
+@app.route('/hook', methods=['POST'])
+def webhook_receiver():
+
+ try:
+ data = json.loads(request.data)
+ except ValueError:
+ return Response(json.dumps({'error': 'Invalid request data.'}),
+ status=400,
+ mimetype='application/json')
+
+ app.logger.info('Received requests: {}'.format(data))
+
+ cases = []
+ for alert in data['alerts']:
+ try:
+ fields, action = alert_fields_and_action(alert)
+ except KeyError as key:
+ msg = 'Alert misses {} key.'.format(key)
+ app.logger.error(msg)
+ return Response(json.dumps({'error': msg}),
+ status=400,
+ mimetype='application/json')
+
+ if fields:
+ try:
+ cases.append(getattr(sf_cli, action)(*fields))
+ except SalesforceMalformedRequest:
+ return Response(json.dumps({'error': 'Request failure.'}),
+ status=500,
+ mimetype='application/json')
+
+ if len(cases) == 1:
+ return jsonify(cases[0])
+ return jsonify(cases)
+
+
+if __name__ == '__main__':
+ app.run()
diff --git a/sf_notifier/settings/__init__.py b/sf_notifier/settings/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/sf_notifier/settings/__init__.py
diff --git a/sf_notifier/settings/development.py b/sf_notifier/settings/development.py
new file mode 100644
index 0000000..39e7006
--- /dev/null
+++ b/sf_notifier/settings/development.py
@@ -0,0 +1,28 @@
+LOGGING = {
+ 'version': 1,
+ 'formatters': {'default': {
+ 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+ }},
+ 'handlers': {'wsgi': {
+ 'class': 'logging.StreamHandler',
+ 'stream': 'ext://flask.logging.wsgi_errors_stream',
+ 'formatter': 'default'
+ }},
+ 'loggers': {
+ 'sf_notifier.server': {
+ 'level': 'INFO',
+ 'handlers': ['wsgi']
+ },
+ 'sf_notifier.salesforce.client': {
+ 'level': 'INFO',
+ 'handlers': ['wsgi']
+ }
+ }
+}
+
+SIMPLE_SETTINGS = {
+ 'OVERRIDE_BY_ENV': True,
+ 'CONFIGURE_LOGGING': True,
+}
+
+SF_CONFIG = {}
diff --git a/sf_notifier/vars/development b/sf_notifier/vars/development
new file mode 100644
index 0000000..0792fb3
--- /dev/null
+++ b/sf_notifier/vars/development
@@ -0,0 +1,4 @@
+export FLASK_APP=sf_notifier/server.py
+export FLASK_ENV=development
+
+export SIMPLE_SETTINGS=sf_notifier.settings.development
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 0000000..05c2b98
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1,9 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+flake8-docstrings==0.2.1.post1 # MIT
+flake8-import-order>=0.17.1 #LGPLv3
+bandit>=1.1.0 # Apache-2.0
+sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
+
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..9dccfbc
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,47 @@
+[tox]
+minversion = 2.0
+skipsdist = True
+envlist = py27,pycodestyle,flake8,bandit,releasenotes
+
+[testenv]
+usedevelop = True
+setenv = VIRTUAL_ENV={envdir}
+ OS_STDOUT_NOCAPTURE=False
+ OS_STDERR_NOCAPTURE=False
+deps =
+ -r{toxinidir}/test-requirements.txt
+ -r{toxinidir}/requirements.txt
+commands =
+ {toxinidir}/run-tests.sh {posargs}
+ {toxinidir}/run-func-tests.sh {posargs}
+
+[testenv:flake8]
+basepython = python2
+commands =
+ flake8 -v
+
+[testenv:pycodestyle]
+basepython = python2
+commands =
+ pycodestyle -v
+
+[testenv:bandit]
+basepython = python2
+commands = bandit -r -x tests -s B110,B410 sf_notifier
+
+[testenv:venv]
+basepython = python2
+commands = {posargs}
+
+[flake8]
+exclude = .tox,.eggs,doc,venv
+show-source = true
+enable-extensions = H904
+
+[testenv:docs]
+basepython = python2
+deps =
+ -e
+ .[test,doc]
+ doc
+commands = doc8 --ignore-path doc/source/rest.rst,doc/source/comparison-table.rst doc/source