Add monitoring for certificates expiration
Related-Prod: PROD-36870
Change-Id: I9da03d8f4a231835a947d8c5787b40010cb25361
diff --git a/dogtag/files/dogtag_cert_expiration_metrics.conf b/dogtag/files/dogtag_cert_expiration_metrics.conf
new file mode 100644
index 0000000..7beac6d
--- /dev/null
+++ b/dogtag/files/dogtag_cert_expiration_metrics.conf
@@ -0,0 +1,3 @@
+{%- from "dogtag/map.jinja" import server with context %}
+subsystems: {{ server.subsystems }}
+pushgateway_url: {{ server.pushgateway.host }}:{{ server.pushgateway.port }}
diff --git a/dogtag/files/dogtag_cert_expiration_metrics.py b/dogtag/files/dogtag_cert_expiration_metrics.py
new file mode 100644
index 0000000..b803a9a
--- /dev/null
+++ b/dogtag/files/dogtag_cert_expiration_metrics.py
@@ -0,0 +1,42 @@
+import base64
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from datetime import datetime
+import requests
+import socket
+import ssl
+import subprocess
+import yaml
+
+hostname = socket.gethostname()
+now = datetime.now()
+cmd_output_list = []
+
+with open('/etc/dogtag/dogtag_cert_expiration_metrics.conf') as f:
+ config = yaml.load(f)
+
+subsystems = [k.lower() for k, v in config['subsystems'].items() if v['enabled']]
+
+for subsystem in subsystems:
+ proc=subprocess.Popen('/usr/sbin/pki-server subsystem-cert-find '+subsystem, shell=True, stdout=subprocess.PIPE, )
+ cmd_output=proc.communicate()[0].rstrip().split("\n",3)[3].split("\n\n")
+ cmd_output_list.extend(cmd_output)
+
+for i in cmd_output_list:
+ cert_dict = dict(item.replace(' ','').split(":") for item in i.split("\n"))
+
+ # Get certificate expiration seconds
+ cert_raw = cert_dict["Certificate"]
+ cert_bytes = base64.b64decode(cert_raw)
+ cert_pem = ssl.DER_cert_to_PEM_cert(cert_bytes)
+ cert = x509.load_pem_x509_certificate(cert_pem.encode('ascii'), default_backend())
+ delta = cert.not_valid_after - now
+ delta_seconds = delta.days * 86400
+
+ # Send data to prometheus pushgateway
+ headers = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
+ url = "http://{0}/metrics/job/{1}".format(config['pushgateway_url'], cert_dict["Nickname"])
+ data = "x509_cert_expiry{{source=\"{0}\", host=\"{1}\"}} {2}\n".format(cert_dict["Nickname"], hostname, delta_seconds)
+ r = requests.post(url, headers=headers, data=data)
+ print(r.reason)
+ print(r.status_code)
diff --git a/dogtag/map.jinja b/dogtag/map.jinja
index 6667042..0ec3ebe 100644
--- a/dogtag/map.jinja
+++ b/dogtag/map.jinja
@@ -1,4 +1,6 @@
+{%- set pushgateway_host = salt['pillar.get'](key='stacklight_monitor_hostname', default='mon') %}
{%- load_yaml as server_defaults %}
+
default:
backup_dirs:
- /etc/pki/pki-tomcat
@@ -17,6 +19,9 @@
TPS:
pkgs: [pki-tps, pki-tps-client]
pki_authdb_basedn: 'o=pki-tomcat-TPS'
+ pushgateway:
+ host: {{ pushgateway_host }}
+ port: 15012
Debian:
pkgs:
- python-nss
diff --git a/dogtag/server/init.sls b/dogtag/server/init.sls
index 59d68c4..d775007 100644
--- a/dogtag/server/init.sls
+++ b/dogtag/server/init.sls
@@ -90,6 +90,31 @@
{%- endif %}
{%- endfor %}
+/opt/dogtag/dogtag_cert_expiration_metrics.py:
+ file.managed:
+ - source: salt://dogtag/files/dogtag_cert_expiration_metrics.py
+ - makedirs: True
+ - user: root
+ - group: root
+ - mode: 640
+
+/etc/dogtag/dogtag_cert_expiration_metrics.conf:
+ file.managed:
+ - source: salt://dogtag/files/dogtag_cert_expiration_metrics.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 640
+ - require:
+ - dogtag_server_packages
+
+python /opt/dogtag/dogtag_cert_expiration_metrics.py:
+ cron.present:
+ - user: root
+ - special: '@daily'
+ - require:
+ - file: /opt/dogtag/dogtag_cert_expiration_metrics.py
+
{%- if server.get('role', 'master') == 'master' %}
{%- if server.get('subsystems', {}).get('CA', {}).pki_client_pkcs12_password is defined %}
{%- set pki_client_pks12_password = server.subsystems.CA.pki_client_pkcs12_password %}