[Tooling update] health_checks
* Added:
- contrail process list
- extended contrail_check to check nodetool status
- extended contrail_check to check processes uptime
* Code shugar fix: sorting import modules
Prod-Related: PROD-31970
Change-Id: I878f2307ce13800eec1558579b20fcefa49e2b25
diff --git a/README.rst b/README.rst
index 84c1cf9..bad75e7 100644
--- a/README.rst
+++ b/README.rst
@@ -932,6 +932,13 @@
salt -C s-ntw03* health_checks.contrail_control_peer_status
+Fetch contrail nodes contrail processes uptime:
+
+.. code-block:: bash
+
+ salt -C 'I@opencontrail:compute' health_checks.contrail_process_list debug=True
+ salt -C 'I@opencontrail:control' health_checks.contrail_process_list role=controller debug=True
+
Fetch libvirt supported machine types for compute node
.. code-block:: bash
diff --git a/_modules/health_checks.py b/_modules/health_checks.py
index 7a11ab5..9428fd5 100644
--- a/_modules/health_checks.py
+++ b/_modules/health_checks.py
@@ -1,12 +1,13 @@
+import datetime
import hashlib
-import requests
-import subprocess
-import socket
-import salt.utils
+import json
import logging
import os
import re
-import json
+import requests
+import salt.utils
+import socket
+import subprocess
import yaml
__author__ = "Dzmitry Stremkouski"
@@ -195,11 +196,60 @@
return True
-def contrail_check(target='I@opencontrail:control or I@opencontrail:collector or I@opencontrail:compute', target_type='compound', ignore_dead=False, **kwargs):
+def contrail_process_list(**kwargs):
+
+ ''' Retrieve contrail process pids and start_time '''
+
+ cmd = ['contrail-status', '-d']
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+
+ procs = {}
+ for line in stdout.split('\n'):
+ if re.findall('^(\S+).*pid ([0-9]+),.*$', line):
+ stat = line.split()
+ procs[stat[0]] = int(stat[3][:-1])
+
+ if kwargs.get('role', 'compute') == 'controller':
+
+ for service in ['zookeeper', 'ifmap-server']:
+ cmd = ['service', service, 'status']
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+
+ for line in stdout.split('\n'):
+ if re.findall('^(\S+).*process ([0-9]+)$', line):
+ stat = line.split()
+ procs[stat[0]] = int(stat[3])
+
+ ctime = int(datetime.datetime.now().strftime("%s"))
+ btime_re = re.compile(r"^btime (\d+)$", re.MULTILINE)
+ btime_groups = btime_re.search(open("/proc/stat").read())
+ btime = int(btime_groups.groups()[0])
+ clk_tck = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
+ for proc in procs:
+ pid = procs[proc]
+ try:
+ with open('/proc/%s/stat' % str(pid), 'r') as f:
+ stat = f.read()
+ jitty_time = int(stat.split(') ')[1].split()[19]) / clk_tck
+ proc_uptime = ctime - btime - int(jitty_time)
+ except:
+ proc_uptime = 0
+ procs[proc] = { 'pid': pid, 'uptime': proc_uptime }
+
+ return procs
+
+
+def contrail_check(target='I@opencontrail:control or I@opencontrail:collector or I@opencontrail:compute', nodetool_target='I@opencontrail:control or I@opencontrail:collector', compute_target='I@opencontrail:compute', target_type='compound', nodetool_target_type='compound', compute_target_type='compound', nodetool_expected_size=3, proc_min_uptime=30, ignore_dead=False, **kwargs):
''' Verify contrail status returns nothing critical '''
agent = "Contrail status"
+
+ # Check #1 contrail-status
out = __salt__['saltutil.cmd']( tgt=target,
tgt_type=target_type,
fun='cmd.run',
@@ -225,9 +275,100 @@
if not _failed_minions(out, agent, failed_minions):
__context__['retcode'] = 2
return False
-
if kwargs.get("debug", False):
logger.info(validated)
+
+ # Check #2 nodetool
+ out = __salt__['saltutil.cmd']( tgt=nodetool_target,
+ tgt_type=nodetool_target_type,
+ fun='cmd.run',
+ arg=['nodetool status'],
+ timeout=5
+ ) or None
+
+ if not _minions_output(out, agent, ignore_dead):
+ __context__['retcode'] = 2
+ return False
+
+ failed_minions = []
+ pattern = '^UN'
+ prog = re.compile(pattern)
+
+ validated = []
+ for minion in out:
+ size = 0
+ for line in out[minion]['ret'].split('\n'):
+ if prog.match(line):
+ size += 1
+ if not size == nodetool_expected_size and minion not in failed_minions:
+ failed_minions.append(minion)
+ validated.append(minion)
+
+ if not _failed_minions(out, agent, failed_minions):
+ __context__['retcode'] = 2
+ return False
+ if kwargs.get("debug", False):
+ logger.info(validated)
+
+ # Check #3 process status control
+ out = __salt__['saltutil.cmd']( tgt=nodetool_target,
+ tgt_type=nodetool_target_type,
+ fun='health_checks.contrail_process_list',
+ arg=['role=controller'],
+ timeout=5
+ ) or None
+
+ if not _minions_output(out, agent, ignore_dead):
+ __context__['retcode'] = 2
+ return False
+
+ failed_minions = []
+ validated = []
+ for minion in out:
+ procs = out[minion]['ret']
+ for proc in procs:
+ proc_uptime = procs[proc]['uptime']
+ if proc_uptime < proc_min_uptime:
+ if minion not in failed_minions:
+ failed_minions.append(minion)
+ logger.error({'minion': minion, 'name': proc, 'uptime': proc_uptime})
+ validated.append(minion)
+
+ if not _failed_minions(out, agent, failed_minions):
+ __context__['retcode'] = 2
+ return False
+ if kwargs.get("debug", False):
+ logger.info(validated)
+
+ # Check #4 process status computes
+ out = __salt__['saltutil.cmd']( tgt=compute_target,
+ tgt_type=compute_target_type,
+ fun='health_checks.contrail_process_list',
+ timeout=5
+ ) or None
+
+ if not _minions_output(out, agent, ignore_dead):
+ __context__['retcode'] = 2
+ return False
+
+ failed_minions = []
+ validated = []
+ for minion in out:
+ procs = out[minion]['ret']
+ for proc in procs:
+ proc_uptime = procs[proc]['uptime']
+ if proc_uptime < proc_min_uptime:
+ if minion not in failed_minions:
+ failed_minions.append(minion)
+ logger.error({'minion': minion, 'name': proc, 'uptime': proc_uptime})
+ validated.append(minion)
+
+ if not _failed_minions(out, agent, failed_minions):
+ __context__['retcode'] = 2
+ return False
+ if kwargs.get("debug", False):
+ logger.info(validated)
+
return True