Port all changes from github
Change-Id: Ie78388290ad2780074327c26508cdce73805f0da
diff --git a/cvp_checks/tests/ceph/test_ceph.py b/cvp_checks/tests/ceph/test_ceph_osd.py
similarity index 100%
rename from cvp_checks/tests/ceph/test_ceph.py
rename to cvp_checks/tests/ceph/test_ceph_osd.py
diff --git a/cvp_checks/tests/ceph/test_ceph_pg_count.py b/cvp_checks/tests/ceph/test_ceph_pg_count.py
new file mode 100644
index 0000000..46e50c4
--- /dev/null
+++ b/cvp_checks/tests/ceph/test_ceph_pg_count.py
@@ -0,0 +1,94 @@
+import pytest
+import math
+
+def __next_power_of2(total_pg):
+ count = 0
+ if (total_pg and not(total_pg & (total_pg - 1))):
+ return total_pg
+ while( total_pg != 0):
+ total_pg >>= 1
+ count += 1
+
+ return 1 << count
+
+
+def test_ceph_pg_count(local_salt_client):
+ """
+ Test aimed to calculate placement groups for Ceph cluster
+ according formula below.
+ Formula to calculate PG num:
+ Total PGs =
+ (Total_number_of_OSD * 100) / max_replication_count / pool count
+ pg_num and pgp_num should be the same and
+ set according formula to higher value of powered 2
+ """
+
+ ceph_monitors = local_salt_client.cmd(
+ 'ceph:mon',
+ 'test.ping',
+ expr_form='pillar')
+
+ if not ceph_monitors:
+ pytest.skip("Ceph is not found on this environment")
+
+ monitor = ceph_monitors.keys()[0]
+ pools = local_salt_client.cmd(
+ monitor, 'cmd.run',
+ ["rados lspools"],
+ expr_form='glob').get(
+ ceph_monitors.keys()[0]).split('\n')
+
+ total_osds = int(local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ['ceph osd tree | grep osd | grep "up\|down" | wc -l'],
+ expr_form='glob').get(ceph_monitors.keys()[0]))
+
+ raw_pool_replications = local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd dump | grep size | awk '{print $3, $6}'"],
+ expr_form='glob').get(ceph_monitors.keys()[0]).split('\n')
+
+ pool_replications = {}
+ for replication in raw_pool_replications:
+ pool_replications[replication.split()[0]] = int(replication.split()[1])
+
+ max_replication_value = 0
+ for repl_value in pool_replications.values():
+ if repl_value > max_replication_value:
+ max_replication_value = repl_value
+
+ total_pg = (total_osds * 100) / max_replication_value / len(pools)
+ correct_pg_num = __next_power_of2(total_pg)
+
+ pools_pg_num = {}
+ pools_pgp_num = {}
+ for pool in pools:
+ pg_num = int(local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd pool get {} pg_num".format(pool)],
+ expr_form='glob').get(ceph_monitors.keys()[0]).split()[1])
+ pools_pg_num[pool] = pg_num
+ pgp_num = int(local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd pool get {} pgp_num".format(pool)],
+ expr_form='glob').get(ceph_monitors.keys()[0]).split()[1])
+ pools_pgp_num[pool] = pgp_num
+
+ wrong_pg_num_pools = []
+ pg_pgp_not_equal_pools = []
+ for pool in pools:
+ if pools_pg_num[pool] != pools_pgp_num[pool]:
+ pg_pgp_not_equal_pools.append(pool)
+ if pools_pg_num[pool] < correct_pg_num:
+ wrong_pg_num_pools.append(pool)
+
+ assert not pg_pgp_not_equal_pools, \
+ "For pools {} PG and PGP are not equal " \
+ "but should be".format(pg_pgp_not_equal_pools)
+ assert not wrong_pg_num_pools, "For pools {} " \
+ "PG number lower than Correct PG number, " \
+ "but should be equal or higher".format(wrong_pg_num_pools)
diff --git a/cvp_checks/tests/ceph/test_ceph_replicas.py b/cvp_checks/tests/ceph/test_ceph_replicas.py
new file mode 100644
index 0000000..62af49d
--- /dev/null
+++ b/cvp_checks/tests/ceph/test_ceph_replicas.py
@@ -0,0 +1,49 @@
+import pytest
+
+
+def test_ceph_replicas(local_salt_client):
+ """
+ Test aimed to check number of replicas
+ for most of deployments if there is no
+ special requirement for that.
+ """
+
+ ceph_monitors = local_salt_client.cmd(
+ 'ceph:mon',
+ 'test.ping',
+ expr_form='pillar')
+
+ if not ceph_monitors:
+ pytest.skip("Ceph is not found on this environment")
+
+ monitor = ceph_monitors.keys()[0]
+
+ raw_pool_replicas = local_salt_client.cmd(
+ monitor,
+ 'cmd.run',
+ ["ceph osd dump | grep size | " \
+ "awk '{print $3, $5, $6, $7, $8}'"],
+ expr_form='glob').get(
+ ceph_monitors.keys()[0]).split('\n')
+
+ pools_replicas = {}
+ for pool in raw_pool_replicas:
+ pool_name = pool.split(" ", 1)[0]
+ pool_replicas = {}
+ raw_replicas = pool.split(" ", 1)[1].split()
+ for elem in raw_replicas:
+ pool_replicas[raw_replicas[0]] = int(raw_replicas[1])
+ pool_replicas[raw_replicas[2]] = int(raw_replicas[3])
+ pools_replicas[pool_name] = pool_replicas
+
+ error = []
+ for pool, replicas in pools_replicas.items():
+ for replica, value in replicas.items():
+ if replica == 'min_size' and value < 2:
+ error.append(pool + " " + replica + " "
+ + str(value) + " must be 2")
+ if replica == 'size' and value < 3:
+ error.append(pool + " " + replica + " "
+ + str(value) + " must be 3")
+
+ assert not error, "Wrong pool replicas found\n{}".format(error)
diff --git a/cvp_checks/tests/ceph/test_ceph_tell_bench.py b/cvp_checks/tests/ceph/test_ceph_tell_bench.py
new file mode 100644
index 0000000..db45435
--- /dev/null
+++ b/cvp_checks/tests/ceph/test_ceph_tell_bench.py
@@ -0,0 +1,55 @@
+import pytest
+import json
+import math
+
+
+def test_ceph_tell_bench(local_salt_client):
+ """
+ Test checks that each OSD MB per second speed
+ is not lower than 10 MB comparing with AVG.
+ Bench command by default writes 1Gb on each OSD
+ with the default values of 4M
+ and gives the "bytes_per_sec" speed for each OSD.
+
+ """
+ ceph_monitors = local_salt_client.cmd(
+ 'ceph:mon',
+ 'test.ping',
+ expr_form='pillar')
+
+ if not ceph_monitors:
+ pytest.skip("Ceph is not found on this environment")
+
+ cmd_result = local_salt_client.cmd(
+ ceph_monitors.keys()[0],
+ 'cmd.run', ["ceph tell osd.* bench -f json"],
+ expr_form='glob').get(
+ ceph_monitors.keys()[0]).split('\n')
+
+ cmd_result = filter(None, cmd_result)
+
+ osd_pool = {}
+ for osd in cmd_result:
+ osd_ = osd.split(" ")
+ osd_pool[osd_[0]] = osd_[1]
+
+ mbps_sum = 0
+ osd_count = 0
+ for osd in osd_pool:
+ osd_count += 1
+ mbps_sum += json.loads(
+ osd_pool[osd])['bytes_per_sec'] / 1000000
+
+ mbps_avg = mbps_sum / osd_count
+ result = {}
+ for osd in osd_pool:
+ mbps = json.loads(
+ osd_pool[osd])['bytes_per_sec'] / 1000000
+ if math.fabs(mbps_avg - mbps) > 10:
+ result[osd] = osd_pool[osd]
+
+ assert len(result) == 0, \
+ "Performance of {0} OSD(s) lower " \
+ "than AVG performance ({1} mbps), " \
+ "please check Ceph for possible problems".format(
+ json.dumps(result, indent=4), mbps_avg)
diff --git a/cvp_checks/tests/test_cinder_services.py b/cvp_checks/tests/test_cinder_services.py
index 7e4c294..2117f6f 100644
--- a/cvp_checks/tests/test_cinder_services.py
+++ b/cvp_checks/tests/test_cinder_services.py
@@ -1,9 +1,14 @@
+import pytest
+
+
def test_cinder_services(local_salt_client):
service_down = local_salt_client.cmd(
- 'keystone:server',
+ 'cinder:controller',
'cmd.run',
['. /root/keystonerc; cinder service-list | grep "down\|disabled"'],
expr_form='pillar')
+ if not service_down:
+ pytest.skip("Cinder is not found on this environment")
cinder_volume = local_salt_client.cmd(
'keystone:server',
'cmd.run',
@@ -12,4 +17,4 @@
assert service_down[service_down.keys()[0]] == '', \
'''Some cinder services are in wrong state'''
assert cinder_volume[cinder_volume.keys()[0]] == '1', \
- '''Some nova services are in wrong state'''
+ '''There are more than 1 host/backend for cinder'''
diff --git a/cvp_checks/tests/test_k8s.py b/cvp_checks/tests/test_k8s.py
index f0cfd44..c3a5ff9 100644
--- a/cvp_checks/tests/test_k8s.py
+++ b/cvp_checks/tests/test_k8s.py
@@ -41,7 +41,7 @@
if 'STATUS' in line or 'proto' in line:
continue
else:
- if 'Ready' not in line:
+ if 'Ready' != line.split()[1]:
errors.append(line)
break
assert not errors, 'k8s is not healthy: {}'.format(json.dumps(
diff --git a/cvp_checks/tests/test_oss.py b/cvp_checks/tests/test_oss.py
index 4b42f15..58a4151 100644
--- a/cvp_checks/tests/test_oss.py
+++ b/cvp_checks/tests/test_oss.py
@@ -10,8 +10,10 @@
['haproxy:proxy:listen:stats:binds:address'],
expr_form='pillar')
HAPROXY_STATS_IP = [node for node in result if result[node]]
+ proxies = {"http": None, "https": None}
csv_result = requests.get('http://{}:9600/haproxy?stats;csv"'.format(
- result[HAPROXY_STATS_IP[0]])).content
+ result[HAPROXY_STATS_IP[0]]),
+ proxies=proxies).content
data = csv_result.lstrip('# ')
wrong_data = []
list_of_services = ['aptly', 'openldap', 'gerrit', 'jenkins', 'postgresql',
diff --git a/cvp_checks/tests/test_rabbit_cluster.py b/cvp_checks/tests/test_rabbit_cluster.py
index eb2666f..daae7ce 100644
--- a/cvp_checks/tests/test_rabbit_cluster.py
+++ b/cvp_checks/tests/test_rabbit_cluster.py
@@ -1,7 +1,10 @@
+from cvp_checks import utils
+
+
def test_checking_rabbitmq_cluster(local_salt_client):
# disable config for this test
# it may be reintroduced in future
- # config = utils.get_configuration(__file__)
+ config = utils.get_configuration()
# request pillar data from rmq nodes
rabbitmq_pillar_data = local_salt_client.cmd(
'rabbitmq:server', 'pillar.data',
@@ -10,15 +13,17 @@
# with required cluster size for each node
control_dict = {}
required_cluster_size_dict = {}
- for node in rabbitmq_pillar_data:
- cluster_size_from_the_node = len(
- rabbitmq_pillar_data[node]['rabbitmq:cluster']['members'])
- required_cluster_size_dict.update({node: cluster_size_from_the_node})
-
# request actual data from rmq nodes
rabbit_actual_data = local_salt_client.cmd(
'rabbitmq:server', 'cmd.run',
['rabbitmqctl cluster_status'], expr_form='pillar')
+ for node in rabbitmq_pillar_data:
+ if node in config.get('skipped_nodes'):
+ del rabbit_actual_data[node]
+ continue
+ cluster_size_from_the_node = len(
+ rabbitmq_pillar_data[node]['rabbitmq:cluster']['members'])
+ required_cluster_size_dict.update({node: cluster_size_from_the_node})
# find actual cluster size for each node
for node in rabbit_actual_data:
diff --git a/cvp_checks/tests/test_salt_master.py b/cvp_checks/tests/test_salt_master.py
index 2e282d6..7649767 100644
--- a/cvp_checks/tests/test_salt_master.py
+++ b/cvp_checks/tests/test_salt_master.py
@@ -6,3 +6,15 @@
expr_form='pillar')
assert 'nothing to commit' in git_status.values()[0], 'Git status showed' \
' some unmerged changes {}'''.format(git_status.values()[0])
+
+
+def test_reclass_smoke(local_salt_client):
+ reclass = local_salt_client.cmd(
+ 'salt:master',
+ 'cmd.run',
+ ['reclass-salt --top; echo $?'],
+ expr_form='pillar')
+ result = reclass[reclass.keys()[0]][-1]
+
+ assert result == '0', 'Reclass is broken' \
+ '\n {}'.format(reclass)
diff --git a/cvp_checks/tests/test_stacklight.py b/cvp_checks/tests/test_stacklight.py
index d554a91..f63b233 100644
--- a/cvp_checks/tests/test_stacklight.py
+++ b/cvp_checks/tests/test_stacklight.py
@@ -10,11 +10,14 @@
'pillar.get',
['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
+ proxies = {"http": None, "https": None}
for node in salt_output.keys():
IP = salt_output[node]
- assert requests.get('http://{}:9200/'.format(IP)).status_code == 200, \
+ assert requests.get('http://{}:9200/'.format(IP),
+ proxies=proxies).status_code == 200, \
'Cannot check elasticsearch url on {}.'.format(IP)
- resp = requests.get('http://{}:9200/_cat/health'.format(IP)).content
+ resp = requests.get('http://{}:9200/_cat/health'.format(IP),
+ proxies=proxies).content
assert resp.split()[3] == 'green', \
'elasticsearch status is not good {}'.format(
json.dumps(resp, indent=4))
@@ -42,8 +45,10 @@
['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
IP = salt_output.values()[0]
+ proxies = {"http": None, "https": None}
resp = json.loads(requests.post('http://{0}:9200/log-{1}/_search?pretty'.
format(IP, today),
+ proxies=proxies,
data='{"size": 0, "aggs": '
'{"uniq_hostname": '
'{"terms": {"size": 500, '
@@ -124,3 +129,15 @@
assert 'NOT OK' not in result.values(), \
'''Some containers are in incorrect state:
{}'''.format(json.dumps(result, indent=4))
+
+
+def test_running_telegraf_services(local_salt_client):
+ salt_output = local_salt_client.cmd('telegraf:agent',
+ 'service.status',
+ 'telegraf',
+ expr_form='pillar')
+ result = [{node: status} for node, status
+ in salt_output.items()
+ if status is False]
+ assert result == [], 'Telegraf service is not running ' \
+ 'on following nodes:'.format(result)
diff --git a/cvp_checks/tests/test_ui_addresses.py b/cvp_checks/tests/test_ui_addresses.py
index 22e8319..69c0f6a 100644
--- a/cvp_checks/tests/test_ui_addresses.py
+++ b/cvp_checks/tests/test_ui_addresses.py
@@ -34,20 +34,19 @@
def test_ui_prometheus(local_salt_client):
- pytest.skip("This test doesn't work. Skipped")
- IP = utils.get_monitoring_ip('keepalived_prometheus_vip_address')
+ IP = utils.get_monitoring_ip('stacklight_monitor_address')
result = local_salt_client.cmd(
'keystone:server',
'cmd.run',
- ['curl http://{}:15010/ 2>&1 | \
- grep loading'.format(IP)],
+ ['curl http://{}:15010/graph 2>&1 | \
+ grep Prometheus'.format(IP)],
expr_form='pillar')
assert len(result[result.keys()[0]]) != 0, \
'Prometheus page is not reachable on {} from ctl nodes'.format(IP)
def test_ui_alert_manager(local_salt_client):
- IP = utils.get_monitoring_ip('cluster_public_host')
+ IP = utils.get_monitoring_ip('stacklight_monitor_address')
result = local_salt_client.cmd(
'keystone:server',
'cmd.run',