Port all changes from github
Change-Id: Ie78388290ad2780074327c26508cdce73805f0da
diff --git a/cvp_checks/fixtures/base.py b/cvp_checks/fixtures/base.py
index 85313d0..96abad9 100644
--- a/cvp_checks/fixtures/base.py
+++ b/cvp_checks/fixtures/base.py
@@ -8,6 +8,7 @@
nodes = utils.calculate_groups()
+
@pytest.fixture(scope='session', params=nodes.values(), ids=nodes.keys())
def nodes_in_group(request):
return request.param
diff --git a/cvp_checks/global_config.yaml b/cvp_checks/global_config.yaml
index 4faa46d..1a182ef 100644
--- a/cvp_checks/global_config.yaml
+++ b/cvp_checks/global_config.yaml
@@ -6,10 +6,12 @@
# Can be found on cfg* node using
# "salt-call pillar.get _param:salt_master_host"
# and "salt-call pillar.get _param:salt_master_port"
+# or "salt-call pillar.get _param:jenkins_salt_api_url"
# SALT_USERNAME by default: salt
# It can be verified with "salt-call shadow.info salt"
# SALT_PASSWORD you can find on cfg* node using
-# "grep -r salt_api_password /srv/salt/reclass/classes"
+# "salt-call pillar.get _param:salt_api_password"
+# or "grep -r salt_api_password /srv/salt/reclass/classes"
SALT_URL: <salt_url>
SALT_USERNAME: <salt_usr>
SALT_PASSWORD: <salt_pwd>
@@ -32,17 +34,39 @@
# Groups can be defined using pillars.
# Uncomment this section to enable this.
# Otherwise groups will be discovered automaticaly
-#groups: {cmp: 'nova:compute'}
+# Tips:
+# 1) you don't need to separate kvm and kvm_glusterfs nodes
+# 2) Use I@pillar or mask like ctl* for targetting nodes
+
+groups: {
+ cmp: 'I@nova:compute',
+ ctl: 'I@keystone:server',
+ msg: 'I@rabbitmq:server',
+ dbs: 'I@galera:*',
+ prx: 'I@nginx:server',
+ mon: 'I@prometheus:server and not I@influxdb:server',
+ log: 'I@kibana:server',
+ mtr: 'I@influxdb:server',
+ kvm: 'I@salt:control',
+ cid: 'I@docker:host and not I@prometheus:server and not I@kubernetes:*',
+ ntw: 'I@opencontrail:database',
+ ceph_mon: 'I@ceph:mon',
+ ceph_osd: 'I@ceph:osd',
+ k8-ctl: 'I@etcd:server',
+ k8-cmp: 'I@kubernetes:* and not I@etcd:*',
+ cfg: 'I@salt:master',
+ gtw: 'I@neutron:gateway'
+}
# mtu test setting
# this test may skip groups (see example)
test_mtu:
{ #"skipped_groups": ["dbs"]
- "skipped_ifaces": ["bonding_masters", "lo", "veth", "tap", "cali"]}
+ "skipped_ifaces": ["bonding_masters", "lo", "veth", "tap", "cali", "qv", "qb"]}
# mask for interfaces to skip
# ntp test setting
# this test may skip specific node (use fqdn)
test_ntp_sync:
{ #"skipped_nodes": [""],
- "time_deviation": 5}
+ "time_deviation": 1}
diff --git a/cvp_checks/tests/ceph/test_ceph.py b/cvp_checks/tests/ceph/test_ceph_osd.py
similarity index 100%
rename from cvp_checks/tests/ceph/test_ceph.py
rename to cvp_checks/tests/ceph/test_ceph_osd.py
diff --git a/cvp_checks/tests/ceph/test_ceph_pg_count.py b/cvp_checks/tests/ceph/test_ceph_pg_count.py
new file mode 100644
index 0000000..46e50c4
--- /dev/null
+++ b/cvp_checks/tests/ceph/test_ceph_pg_count.py
@@ -0,0 +1,94 @@
+import pytest
+import math
+
+def __next_power_of2(total_pg):
+ count = 0
+ if (total_pg and not(total_pg & (total_pg - 1))):
+ return total_pg
+ while( total_pg != 0):
+ total_pg >>= 1
+ count += 1
+
+ return 1 << count
+
+
+def test_ceph_pg_count(local_salt_client):
+ """
+ Test aimed to calculate placement groups for Ceph cluster
+ according formula below.
+ Formula to calculate PG num:
+ Total PGs =
+ (Total_number_of_OSD * 100) / max_replication_count / pool count
+ pg_num and pgp_num should be the same and
+ set according formula to higher value of powered 2
+ """
+
+ ceph_monitors = local_salt_client.cmd(
+ 'ceph:mon',
+ 'test.ping',
+ expr_form='pillar')
+
+ if not ceph_monitors:
+ pytest.skip("Ceph is not found on this environment")
+
+ monitor = ceph_monitors.keys()[0]
+ pools = local_salt_client.cmd(
+ monitor, 'cmd.run',
+ ["rados lspools"],
+ expr_form='glob').get(
+ ceph_monitors.keys()[0]).split('\n')
+
+ total_osds = int(local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ['ceph osd tree | grep osd | grep "up\|down" | wc -l'],
+ expr_form='glob').get(ceph_monitors.keys()[0]))
+
+ raw_pool_replications = local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd dump | grep size | awk '{print $3, $6}'"],
+ expr_form='glob').get(ceph_monitors.keys()[0]).split('\n')
+
+ pool_replications = {}
+ for replication in raw_pool_replications:
+ pool_replications[replication.split()[0]] = int(replication.split()[1])
+
+ max_replication_value = 0
+ for repl_value in pool_replications.values():
+ if repl_value > max_replication_value:
+ max_replication_value = repl_value
+
+ total_pg = (total_osds * 100) / max_replication_value / len(pools)
+ correct_pg_num = __next_power_of2(total_pg)
+
+ pools_pg_num = {}
+ pools_pgp_num = {}
+ for pool in pools:
+ pg_num = int(local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd pool get {} pg_num".format(pool)],
+ expr_form='glob').get(ceph_monitors.keys()[0]).split()[1])
+ pools_pg_num[pool] = pg_num
+ pgp_num = int(local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd pool get {} pgp_num".format(pool)],
+ expr_form='glob').get(ceph_monitors.keys()[0]).split()[1])
+ pools_pgp_num[pool] = pgp_num
+
+ wrong_pg_num_pools = []
+ pg_pgp_not_equal_pools = []
+ for pool in pools:
+ if pools_pg_num[pool] != pools_pgp_num[pool]:
+ pg_pgp_not_equal_pools.append(pool)
+ if pools_pg_num[pool] < correct_pg_num:
+ wrong_pg_num_pools.append(pool)
+
+ assert not pg_pgp_not_equal_pools, \
+ "For pools {} PG and PGP are not equal " \
+ "but should be".format(pg_pgp_not_equal_pools)
+ assert not wrong_pg_num_pools, "For pools {} " \
+ "PG number lower than Correct PG number, " \
+ "but should be equal or higher".format(wrong_pg_num_pools)
diff --git a/cvp_checks/tests/ceph/test_ceph_replicas.py b/cvp_checks/tests/ceph/test_ceph_replicas.py
new file mode 100644
index 0000000..62af49d
--- /dev/null
+++ b/cvp_checks/tests/ceph/test_ceph_replicas.py
@@ -0,0 +1,49 @@
+import pytest
+
+
+def test_ceph_replicas(local_salt_client):
+ """
+ Test aimed to check number of replicas
+ for most of deployments if there is no
+ special requirement for that.
+ """
+
+ ceph_monitors = local_salt_client.cmd(
+ 'ceph:mon',
+ 'test.ping',
+ expr_form='pillar')
+
+ if not ceph_monitors:
+ pytest.skip("Ceph is not found on this environment")
+
+ monitor = ceph_monitors.keys()[0]
+
+ raw_pool_replicas = local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd dump | grep size | " \
+ "awk '{print $3, $5, $6, $7, $8}'"],
+ expr_form='glob').get(
+ ceph_monitors.keys()[0]).split('\n')
+
+ pools_replicas = {}
+ for pool in raw_pool_replicas:
+ pool_name = pool.split(" ", 1)[0]
+ pool_replicas = {}
+ raw_replicas = pool.split(" ", 1)[1].split()
+ for elem in raw_replicas:
+ pool_replicas[raw_replicas[0]] = int(raw_replicas[1])
+ pool_replicas[raw_replicas[2]] = int(raw_replicas[3])
+ pools_replicas[pool_name] = pool_replicas
+
+ error = []
+ for pool, replicas in pools_replicas.items():
+ for replica, value in replicas.items():
+ if replica == 'min_size' and value < 2:
+ error.append(pool + " " + replica + " "
+ + str(value) + " must be 2")
+ if replica == 'size' and value < 3:
+ error.append(pool + " " + replica + " "
+ + str(value) + " must be 3")
+
+ assert not error, "Wrong pool replicas found\n{}".format(error)
diff --git a/cvp_checks/tests/ceph/test_ceph_tell_bench.py b/cvp_checks/tests/ceph/test_ceph_tell_bench.py
new file mode 100644
index 0000000..db45435
--- /dev/null
+++ b/cvp_checks/tests/ceph/test_ceph_tell_bench.py
@@ -0,0 +1,55 @@
+import pytest
+import json
+import math
+
+
+def test_ceph_tell_bench(local_salt_client):
+ """
+ Test checks that each OSD MB per second speed
+ is not lower than 10 MB comparing with AVG.
+ Bench command by default writes 1Gb on each OSD
+ with the default values of 4M
+ and gives the "bytes_per_sec" speed for each OSD.
+
+ """
+ ceph_monitors = local_salt_client.cmd(
+ 'ceph:mon',
+ 'test.ping',
+ expr_form='pillar')
+
+ if not ceph_monitors:
+ pytest.skip("Ceph is not found on this environment")
+
+ cmd_result = local_salt_client.cmd(
+ ceph_monitors.keys()[0],
+ 'cmd.run', ["ceph tell osd.* bench -f json"],
+ expr_form='glob').get(
+ ceph_monitors.keys()[0]).split('\n')
+
+ cmd_result = filter(None, cmd_result)
+
+ osd_pool = {}
+ for osd in cmd_result:
+ osd_ = osd.split(" ")
+ osd_pool[osd_[0]] = osd_[1]
+
+ mbps_sum = 0
+ osd_count = 0
+ for osd in osd_pool:
+ osd_count += 1
+ mbps_sum += json.loads(
+ osd_pool[osd])['bytes_per_sec'] / 1000000
+
+ mbps_avg = mbps_sum / osd_count
+ result = {}
+ for osd in osd_pool:
+ mbps = json.loads(
+ osd_pool[osd])['bytes_per_sec'] / 1000000
+ if math.fabs(mbps_avg - mbps) > 10:
+ result[osd] = osd_pool[osd]
+
+ assert len(result) == 0, \
+ "Performance of {0} OSD(s) lower " \
+ "than AVG performance ({1} mbps), " \
+ "please check Ceph for possible problems".format(
+ json.dumps(result, indent=4), mbps_avg)
diff --git a/cvp_checks/tests/test_cinder_services.py b/cvp_checks/tests/test_cinder_services.py
index 7e4c294..2117f6f 100644
--- a/cvp_checks/tests/test_cinder_services.py
+++ b/cvp_checks/tests/test_cinder_services.py
@@ -1,9 +1,14 @@
+import pytest
+
+
def test_cinder_services(local_salt_client):
service_down = local_salt_client.cmd(
- 'keystone:server',
+ 'cinder:controller',
'cmd.run',
['. /root/keystonerc; cinder service-list | grep "down\|disabled"'],
expr_form='pillar')
+ if not service_down:
+ pytest.skip("Cinder is not found on this environment")
cinder_volume = local_salt_client.cmd(
'keystone:server',
'cmd.run',
@@ -12,4 +17,4 @@
assert service_down[service_down.keys()[0]] == '', \
'''Some cinder services are in wrong state'''
assert cinder_volume[cinder_volume.keys()[0]] == '1', \
- '''Some nova services are in wrong state'''
+ '''There are more than 1 host/backend for cinder'''
diff --git a/cvp_checks/tests/test_k8s.py b/cvp_checks/tests/test_k8s.py
index f0cfd44..c3a5ff9 100644
--- a/cvp_checks/tests/test_k8s.py
+++ b/cvp_checks/tests/test_k8s.py
@@ -41,7 +41,7 @@
if 'STATUS' in line or 'proto' in line:
continue
else:
- if 'Ready' not in line:
+ if 'Ready' != line.split()[1]:
errors.append(line)
break
assert not errors, 'k8s is not healthy: {}'.format(json.dumps(
diff --git a/cvp_checks/tests/test_oss.py b/cvp_checks/tests/test_oss.py
index 4b42f15..58a4151 100644
--- a/cvp_checks/tests/test_oss.py
+++ b/cvp_checks/tests/test_oss.py
@@ -10,8 +10,10 @@
['haproxy:proxy:listen:stats:binds:address'],
expr_form='pillar')
HAPROXY_STATS_IP = [node for node in result if result[node]]
+ proxies = {"http": None, "https": None}
csv_result = requests.get('http://{}:9600/haproxy?stats;csv"'.format(
- result[HAPROXY_STATS_IP[0]])).content
+ result[HAPROXY_STATS_IP[0]]),
+ proxies=proxies).content
data = csv_result.lstrip('# ')
wrong_data = []
list_of_services = ['aptly', 'openldap', 'gerrit', 'jenkins', 'postgresql',
diff --git a/cvp_checks/tests/test_rabbit_cluster.py b/cvp_checks/tests/test_rabbit_cluster.py
index eb2666f..daae7ce 100644
--- a/cvp_checks/tests/test_rabbit_cluster.py
+++ b/cvp_checks/tests/test_rabbit_cluster.py
@@ -1,7 +1,10 @@
+from cvp_checks import utils
+
+
def test_checking_rabbitmq_cluster(local_salt_client):
# disable config for this test
# it may be reintroduced in future
- # config = utils.get_configuration(__file__)
+ config = utils.get_configuration()
# request pillar data from rmq nodes
rabbitmq_pillar_data = local_salt_client.cmd(
'rabbitmq:server', 'pillar.data',
@@ -10,15 +13,17 @@
# with required cluster size for each node
control_dict = {}
required_cluster_size_dict = {}
- for node in rabbitmq_pillar_data:
- cluster_size_from_the_node = len(
- rabbitmq_pillar_data[node]['rabbitmq:cluster']['members'])
- required_cluster_size_dict.update({node: cluster_size_from_the_node})
-
# request actual data from rmq nodes
rabbit_actual_data = local_salt_client.cmd(
'rabbitmq:server', 'cmd.run',
['rabbitmqctl cluster_status'], expr_form='pillar')
+ for node in rabbitmq_pillar_data:
+ if node in config.get('skipped_nodes'):
+ del rabbit_actual_data[node]
+ continue
+ cluster_size_from_the_node = len(
+ rabbitmq_pillar_data[node]['rabbitmq:cluster']['members'])
+ required_cluster_size_dict.update({node: cluster_size_from_the_node})
# find actual cluster size for each node
for node in rabbit_actual_data:
diff --git a/cvp_checks/tests/test_salt_master.py b/cvp_checks/tests/test_salt_master.py
index 2e282d6..7649767 100644
--- a/cvp_checks/tests/test_salt_master.py
+++ b/cvp_checks/tests/test_salt_master.py
@@ -6,3 +6,15 @@
expr_form='pillar')
assert 'nothing to commit' in git_status.values()[0], 'Git status showed' \
' some unmerged changes {}'''.format(git_status.values()[0])
+
+
+def test_reclass_smoke(local_salt_client):
+ reclass = local_salt_client.cmd(
+ 'salt:master',
+ 'cmd.run',
+ ['reclass-salt --top; echo $?'],
+ expr_form='pillar')
+ result = reclass[reclass.keys()[0]][-1]
+
+ assert result == '0', 'Reclass is broken' \
+ '\n {}'.format(reclass)
diff --git a/cvp_checks/tests/test_stacklight.py b/cvp_checks/tests/test_stacklight.py
index d554a91..f63b233 100644
--- a/cvp_checks/tests/test_stacklight.py
+++ b/cvp_checks/tests/test_stacklight.py
@@ -10,11 +10,14 @@
'pillar.get',
['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
+ proxies = {"http": None, "https": None}
for node in salt_output.keys():
IP = salt_output[node]
- assert requests.get('http://{}:9200/'.format(IP)).status_code == 200, \
+ assert requests.get('http://{}:9200/'.format(IP),
+ proxies=proxies).status_code == 200, \
'Cannot check elasticsearch url on {}.'.format(IP)
- resp = requests.get('http://{}:9200/_cat/health'.format(IP)).content
+ resp = requests.get('http://{}:9200/_cat/health'.format(IP),
+ proxies=proxies).content
assert resp.split()[3] == 'green', \
'elasticsearch status is not good {}'.format(
json.dumps(resp, indent=4))
@@ -42,8 +45,10 @@
['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
IP = salt_output.values()[0]
+ proxies = {"http": None, "https": None}
resp = json.loads(requests.post('http://{0}:9200/log-{1}/_search?pretty'.
format(IP, today),
+ proxies=proxies,
data='{"size": 0, "aggs": '
'{"uniq_hostname": '
'{"terms": {"size": 500, '
@@ -124,3 +129,15 @@
assert 'NOT OK' not in result.values(), \
'''Some containers are in incorrect state:
{}'''.format(json.dumps(result, indent=4))
+
+
+def test_running_telegraf_services(local_salt_client):
+ salt_output = local_salt_client.cmd('telegraf:agent',
+ 'service.status',
+ 'telegraf',
+ expr_form='pillar')
+ result = [{node: status} for node, status
+ in salt_output.items()
+ if status is False]
+ assert result == [], 'Telegraf service is not running ' \
+ 'on following nodes:'.format(result)
diff --git a/cvp_checks/tests/test_ui_addresses.py b/cvp_checks/tests/test_ui_addresses.py
index 22e8319..69c0f6a 100644
--- a/cvp_checks/tests/test_ui_addresses.py
+++ b/cvp_checks/tests/test_ui_addresses.py
@@ -34,20 +34,19 @@
def test_ui_prometheus(local_salt_client):
- pytest.skip("This test doesn't work. Skipped")
- IP = utils.get_monitoring_ip('keepalived_prometheus_vip_address')
+ IP = utils.get_monitoring_ip('stacklight_monitor_address')
result = local_salt_client.cmd(
'keystone:server',
'cmd.run',
- ['curl http://{}:15010/ 2>&1 | \
- grep loading'.format(IP)],
+ ['curl http://{}:15010/graph 2>&1 | \
+ grep Prometheus'.format(IP)],
expr_form='pillar')
assert len(result[result.keys()[0]]) != 0, \
'Prometheus page is not reachable on {} from ctl nodes'.format(IP)
def test_ui_alert_manager(local_salt_client):
- IP = utils.get_monitoring_ip('cluster_public_host')
+ IP = utils.get_monitoring_ip('stacklight_monitor_address')
result = local_salt_client.cmd(
'keystone:server',
'cmd.run',
diff --git a/cvp_checks/utils/__init__.py b/cvp_checks/utils/__init__.py
index 840b4ea..74af967 100644
--- a/cvp_checks/utils/__init__.py
+++ b/cvp_checks/utils/__init__.py
@@ -2,11 +2,14 @@
import yaml
import requests
import re
+import sys, traceback
class salt_remote:
def cmd(self, tgt, fun, param=None, expr_form=None, tgt_type=None):
config = get_configuration()
+ url = config['SALT_URL']
+ proxies = {"http": None, "https": None}
headers = {'Accept': 'application/json'}
login_payload = {'username': config['SALT_USERNAME'],
'password': config['SALT_PASSWORD'], 'eauth': 'pam'}
@@ -16,16 +19,23 @@
if param:
accept_key_payload['arg'] = param
- login_request = requests.post(os.path.join(config['SALT_URL'],
- 'login'),
- headers=headers, data=login_payload)
- if login_request.ok:
- request = requests.post(config['SALT_URL'], headers=headers,
- data=accept_key_payload,
- cookies=login_request.cookies)
- return request.json()['return'][0]
- else:
- raise EnvironmentError("401 Not authorized.")
+ try:
+ login_request = requests.post(os.path.join(url, 'login'),
+ headers=headers, data=login_payload,
+ proxies=proxies)
+ if login_request.ok:
+ request = requests.post(url, headers=headers,
+ data=accept_key_payload,
+ cookies=login_request.cookies,
+ proxies=proxies)
+ return request.json()['return'][0]
+ except Exception:
+ print "\033[91m\nConnection to SaltMaster " \
+ "was not established.\n" \
+ "Please make sure that you " \
+ "provided correct credentials.\033[0m\n"
+ traceback.print_exc(file=sys.stdout)
+ sys.exit()
def init_salt_client():
@@ -76,12 +86,13 @@
node_groups = {}
nodes_names = set ()
expr_form = ''
- if 'groups' in config.keys():
+ all_nodes = set(local_salt_client.cmd('*', 'test.ping'))
+ if 'groups' in config.keys() and 'PB_GROUPS' in os.environ.keys() and \
+ os.environ['PB_GROUPS'].lower() != 'false':
nodes_names.update(config['groups'].keys())
- expr_form = 'pillar'
+ expr_form = 'compound'
else:
- nodes = local_salt_client.cmd('*', 'test.ping')
- for node in nodes:
+ for node in all_nodes:
index = re.search('[0-9]{1,3}$', node.split('.')[0])
if index:
nodes_names.add(node.split('.')[0][:-len(index.group(0))])
@@ -89,19 +100,42 @@
nodes_names.add(node)
expr_form = 'pcre'
+ gluster_nodes = local_salt_client.cmd('I@salt:control and '
+ 'I@glusterfs:server',
+ 'test.ping', expr_form='compound')
+ kvm_nodes = local_salt_client.cmd('I@salt:control and not '
+ 'I@glusterfs:server',
+ 'test.ping', expr_form='compound')
+
for node_name in nodes_names:
skipped_groups = config.get('skipped_groups') or []
if node_name in skipped_groups:
continue
if expr_form == 'pcre':
- nodes = local_salt_client.cmd(node_name,
+ nodes = local_salt_client.cmd('{}[0-9]{{1,3}}'.format(node_name),
'test.ping',
expr_form=expr_form)
else:
nodes = local_salt_client.cmd(config['groups'][node_name],
'test.ping',
expr_form=expr_form)
- node_groups[node_name]=[x for x in nodes if x not in config['skipped_nodes']]
+ if nodes == {}:
+ continue
+
+ node_groups[node_name]=[x for x in nodes
+ if x not in config['skipped_nodes']
+ if x not in gluster_nodes.keys()
+ if x not in kvm_nodes.keys()]
+ all_nodes = set(all_nodes - set(node_groups[node_name]))
+ if node_groups[node_name] == []:
+ del node_groups[node_name]
+ if kvm_nodes:
+ node_groups['kvm'] = kvm_nodes.keys()
+ node_groups['kvm_gluster'] = gluster_nodes.keys()
+ all_nodes = set(all_nodes - set(kvm_nodes.keys()))
+ all_nodes = set(all_nodes - set(gluster_nodes.keys()))
+ if all_nodes:
+ print ("These nodes were not collected {0}. Check config (groups section)".format(all_nodes))
return node_groups
diff --git a/setup.py b/setup.py
index 8d8f66d..f13131c 100755
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,14 @@
# -*- coding: utf-8 -*-
from setuptools import setup, find_packages
+import os
+def read(fname):
+ return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+def get_requirements_list(requirements):
+ all_requirements = read(requirements)
+ return all_requirements
with open('README.md') as f:
readme = f.read()
@@ -16,5 +23,6 @@
long_description=readme,
author='Mirantis',
license=license,
+ install_requires=get_requirements_list('./requirements.txt'),
packages=find_packages(exclude=('tests', 'docs'))
)