[Tooling update] health_checks
* Added:
- keystone fernet keys check
- keystone credential keys check
Prod-Related: PROD-31970
Change-Id: I5dd73e61fe8b44ecb725520874be06221b93a971
diff --git a/README.rst b/README.rst
index ba5e04b..84c1cf9 100644
--- a/README.rst
+++ b/README.rst
@@ -938,6 +938,19 @@
salt -C 'I@opencontrail:compute' health_checks.libvirt_capabilities
+Check keystone fernet keys are in sync
+
+.. code-block:: bash
+
+ salt-call health_checks.keystone_keys_check
+ salt-call health_checks.keystone_keys_check keys_dir='/var/lib/keystone/fernet-keys'
+
+Check keystone credential keys are in sync
+
+.. code-block:: bash
+
+ salt-call health_checks.keystone_keys_check keys_type='credential'
+
Encrypted pillars
~~~~~~~~~~~~~~~~~
diff --git a/_modules/health_checks.py b/_modules/health_checks.py
index 68a5426..7a11ab5 100644
--- a/_modules/health_checks.py
+++ b/_modules/health_checks.py
@@ -1,3 +1,4 @@
+import hashlib
import requests
import subprocess
import socket
@@ -1416,6 +1417,24 @@
return peers
+def contrail_control_peer_status(api_host='auto', api_port=8083, fields=default_peer_filter):
+
+ ''' Contrail control peer status '''
+
+ peer_status = {}
+
+ for peer_elem in contrail_control_peers_summary():
+ elem = {}
+ for attr in peer_elem:
+ if attr in fields:
+ elem[attr] = peer_elem[attr]
+
+ peer_name = peer_elem["peer"]
+ peer_status[peer_name] = elem
+
+ return peer_status
+
+
def _get_object(json_obj, obj_path):
''' Retrieve subelemet of an JSON object or value '''
@@ -1529,19 +1548,83 @@
return "Unsupported xml tree for this function %s" % str(stdout)
-def contrail_control_peer_status(api_host='auto', api_port=8083, fields=default_peer_filter):
+def keystone_keys_attractor(keys_dir='/var/lib/keystone/fernet-keys', keys_ids=range(0,-4,-1)):
- ''' Contrail control peer status '''
+ ''' JSON formatted dict of keystone keys sha256 sums '''
- peer_status = {}
+ keys = os.listdir(keys_dir)
+ keys.sort()
+ keys_dict = {}
+ try:
+ for i in keys_ids:
+ with open("%s/%s" % (keys_dir, str(keys[i])), 'r') as key_file:
+ _iter1 = hashlib.sha256(key_file.read()).hexdigest()
+ _iter2 = hashlib.sha256(_iter1).hexdigest()
+ _iter3 = hashlib.sha256(_iter2).hexdigest()
+ keys_dict[str(keys[i])] = _iter3
+ except:
+ pass
- for peer_elem in contrail_control_peers_summary():
- elem = {}
- for attr in peer_elem:
- if attr in fields:
- elem[attr] = peer_elem[attr]
+ return keys_dict
- peer_name = peer_elem["peer"]
- peer_status[peer_name] = elem
- return peer_status
+def keystone_keys_check(target='I@keystone:server', target_type='compound', ignore_dead=False, **kwargs):
+
+ ''' Check cluster keystone keys are in sync '''
+
+ keys_type = kwargs.get("keys_type", 'fernet')
+
+ supported_key_types = ['fernet', 'credential']
+ if keys_type not in supported_key_types:
+ logger.error("Unsupported keys type: %s" % str(keys_type))
+ logger.error("Supported keys type are: %s" % str(supported_key_types))
+ __context__['retcode'] = 2
+ return False
+
+ agent = "keystone %s keys sync" % keys_type
+ keys_dir_default = '/var/lib/keystone/%s-keys' % keys_type
+ keys_dir = kwargs.get("keys_dir", keys_dir_default)
+
+ out = __salt__['saltutil.cmd']( tgt=target,
+ tgt_type=target_type,
+ fun='health_checks.keystone_keys_attractor',
+ arg=["keys_dir='%s'" % keys_dir],
+ timeout=3
+ ) or None
+
+ if not _minions_output(out, agent, ignore_dead):
+ __context__['retcode'] = 2
+ return False
+
+ cluster_attractors = []
+ failed_minions = []
+ verified_minions = []
+ attractor = {}
+
+ for minion in out:
+ verified_minions.append(minion)
+ attractor = out[minion]['ret']
+ if attractor == {}:
+ failed_minions.append(minion)
+ if attractor not in cluster_attractors:
+ cluster_attractors.append(attractor)
+
+ if not _failed_minions(out, agent, failed_minions):
+ __context__['retcode'] = 2
+ return False
+
+ if len(cluster_attractors) > 1:
+ failed_minions = []
+ for minion in out:
+ failed_minions.append(minion)
+
+ if not _failed_minions(out, agent, failed_minions):
+ __context__['retcode'] = 2
+ return False
+
+ if kwargs.get("debug", False):
+ logger.info("%s check done." % agent)
+ logger.info(verified_minions)
+
+ return True
+