Merge "Add Dockerfile for cvp-sanity-checks framework"
diff --git a/cvp_checks/global_config.yaml b/cvp_checks/global_config.yaml
index 40303d6..72e6980 100644
--- a/cvp_checks/global_config.yaml
+++ b/cvp_checks/global_config.yaml
@@ -66,6 +66,7 @@
# mask for interfaces to skip
drivetrain_version: ''
+jenkins_test_job: ''
# ntp test setting
# this test may skip specific node (use fqdn)
diff --git a/cvp_checks/tests/test_cinder_services.py b/cvp_checks/tests/test_cinder_services.py
index 20bd17d..f6aaf4a 100644
--- a/cvp_checks/tests/test_cinder_services.py
+++ b/cvp_checks/tests/test_cinder_services.py
@@ -8,7 +8,8 @@
['cinder:controller:backend'],
expr_form='pillar')
if not cinder_backends_info:
- pytest.skip("Cinder is not found on this environment")
+ pytest.skip("Cinder service or cinder:controller pillar \
+ are not found on this environment.")
service_down = local_salt_client.cmd(
'keystone:server',
'cmd.run',
diff --git a/cvp_checks/tests/test_contrail.py b/cvp_checks/tests/test_contrail.py
index 2d358e9..5e7e108 100644
--- a/cvp_checks/tests/test_contrail.py
+++ b/cvp_checks/tests/test_contrail.py
@@ -4,7 +4,7 @@
pytestmark = pytest.mark.usefixtures("contrail")
STATUS_FILTER = r'grep -Pv "(==|^$|Disk|unix|support|boot|\*\*|FOR NODE)"'
-STATUS_COMMAND = "contrail-status"
+STATUS_COMMAND = "contrail-status -t 10"
def get_contrail_status(salt_client, pillar, command, processor):
return salt_client.cmd(
diff --git a/cvp_checks/tests/test_drivetrain.py b/cvp_checks/tests/test_drivetrain.py
index 70b61ec..ae83d57 100644
--- a/cvp_checks/tests/test_drivetrain.py
+++ b/cvp_checks/tests/test_drivetrain.py
@@ -1,9 +1,148 @@
-from jenkinsapi.jenkins import Jenkins
+import jenkins
from xml.dom import minidom
from cvp_checks import utils
import json
import pytest
+import time
+import os
+from pygerrit2 import GerritRestAPI, HTTPBasicAuth
+from requests import HTTPError
+import git
+def join_to_gerrit(local_salt_client, gerrit_user, gerrit_password):
+ gerrit_port = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_port'],
+ expr_form='compound').values()[0]
+ gerrit_address = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_host'],
+ expr_form='compound').values()[0]
+ url = 'http://{0}:{1}'.format(gerrit_address,gerrit_port)
+ auth = HTTPBasicAuth(gerrit_user, gerrit_password)
+ rest = GerritRestAPI(url=url, auth=auth)
+ return rest
+
+def join_to_jenkins(local_salt_client, jenkins_user, jenkins_password):
+ jenkins_port = local_salt_client.cmd(
+ 'I@jenkins:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_jenkins_bind_port'],
+ expr_form='compound').values()[0]
+ jenkins_address = local_salt_client.cmd(
+ 'I@jenkins:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_jenkins_bind_host'],
+ expr_form='compound').values()[0]
+ jenkins_url = 'http://{0}:{1}'.format(jenkins_address,jenkins_port)
+ server = jenkins.Jenkins(jenkins_url, username=jenkins_user, password=jenkins_password)
+ return server
+
+def get_password(local_salt_client,service):
+ password = local_salt_client.cmd(
+ service,
+ 'pillar.get',
+ ['_param:openldap_admin_password'],
+ expr_form='pillar').values()[0]
+ return password
+
+def test_drivetrain_gerrit(local_salt_client):
+ gerrit_password = get_password(local_salt_client,'gerrit:client')
+ gerrit_error = ''
+ current_date = time.strftime("%Y%m%d-%H.%M.%S", time.localtime())
+ test_proj_name = "test-dt-{0}".format(current_date)
+ gerrit_port = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_port'],
+ expr_form='compound').values()[0]
+ gerrit_address = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_host'],
+ expr_form='compound').values()[0]
+ try:
+ #Connecting to gerrit and check connection
+ server = join_to_gerrit(local_salt_client,'admin',gerrit_password)
+ gerrit_check = server.get("/changes/?q=owner:self%20status:open")
+ #Check deleteproject plugin and skip test if the plugin is not installed
+ gerrit_plugins = server.get("/plugins/?all")
+ if 'deleteproject' not in gerrit_plugins:
+ pytest.skip("Delete-project plugin is not installed")
+ #Create test project and add description
+ server.put("/projects/"+test_proj_name)
+ server.put("/projects/"+test_proj_name+"/description",json={"description":"Test DriveTrain project","commit_message": "Update the project description"})
+ except HTTPError, e:
+ gerrit_error = e
+ try:
+ #Create test folder and init git
+ repo_dir = os.path.join(os.getcwd(),test_proj_name)
+ file_name = os.path.join(repo_dir, current_date)
+ repo = git.Repo.init(repo_dir)
+ #Add remote url for this git repo
+ origin = repo.create_remote('origin', 'http://admin:{1}@{2}:{3}/{0}.git'.format(test_proj_name,gerrit_password,gerrit_address,gerrit_port))
+ #Add commit-msg hook to automatically add Change-Id to our commit
+ os.system("curl -Lo {0}/.git/hooks/commit-msg 'http://admin:{1}@{2}:{3}/tools/hooks/commit-msg' > /dev/null 2>&1".format(repo_dir,gerrit_password,gerrit_address,gerrit_port))
+ os.system("chmod u+x {0}/.git/hooks/commit-msg".format(repo_dir))
+ #Create a test file
+ f = open(file_name, 'w+')
+ f.write("This is a test file for DriveTrain test")
+ f.close()
+ #Add file to git and commit it to Gerrit for review
+ repo.index.add([file_name])
+ repo.index.commit("This is a test commit for DriveTrain test")
+ repo.git.push("origin", "HEAD:refs/for/master")
+ #Get change id from Gerrit. Set Code-Review +2 and submit this change
+ changes = server.get("/changes/?q=project:{0}".format(test_proj_name))
+ last_change = changes[0].get('change_id')
+ server.post("/changes/{0}/revisions/1/review".format(last_change),json={"message":"All is good","labels":{"Code-Review":"+2"}})
+ server.post("/changes/{0}/submit".format(last_change))
+ except HTTPError, e:
+ gerrit_error = e
+ finally:
+ #Delete test project
+ server.post("/projects/"+test_proj_name+"/deleteproject~delete")
+ assert gerrit_error == '',\
+ 'Something is wrong with Gerrit'.format(gerrit_error)
+
+
+def test_drivetrain_jenkins_job(local_salt_client):
+ jenkins_password = get_password(local_salt_client,'jenkins:client')
+ server = join_to_jenkins(local_salt_client,'admin',jenkins_password)
+ #Getting Jenkins test job name from configuration
+ config = utils.get_configuration()
+ jenkins_test_job = config['jenkins_test_job']
+ if not jenkins_test_job or jenkins_test_job == '':
+ jenkins_test_job = 'git-mirror-downstream-mk-pipelines'
+ if server.get_job_name(jenkins_test_job):
+ next_build_num = server.get_job_info(jenkins_test_job)['nextBuildNumber']
+ #If this is first build number skip building check
+ if next_build_num != 1:
+ #Check that test job is not running at this moment,
+ #Otherwise skip the test
+ last_build_num = server.get_job_info(jenkins_test_job)['lastBuild'].get('number')
+ last_build_status = server.get_build_info(jenkins_test_job,last_build_num)['building']
+ if last_build_status:
+ pytest.skip("Test job {0} is already running").format(jenkins_test_job)
+ #This jenkins module doesn't work with build_job function without parameters
+ #Just send some fake parameters. All others will be used from default values
+ param_dict = {'foo':'bar'}
+ server.build_job(jenkins_test_job, param_dict)
+ timeout = 0
+ #Use job status True by default to exclude timeout between build job and start job.
+ job_status = True
+ while job_status and ( timeout < 180 ):
+ time.sleep(10)
+ timeout += 10
+ job_status = server.get_build_info(jenkins_test_job,next_build_num)['building']
+ job_result = server.get_build_info(jenkins_test_job,next_build_num)['result']
+ else:
+ pytest.skip("The job {0} was not found").format(test_job_name)
+ assert job_result == 'SUCCESS', \
+ '''Test job '{0}' build was not successfull or timeout is too small
+ '''.format(jenkins_test_job)
def test_drivetrain_services_replicas(local_salt_client):
salt_output = local_salt_client.cmd(
@@ -57,33 +196,22 @@
expected_version = config['drivetrain_version'] or []
if not expected_version or expected_version == '':
pytest.skip("drivetrain_version is not defined. Skipping")
- jenkins_password = local_salt_client.cmd(
- 'jenkins:client',
- 'pillar.get',
- ['_param:openldap_admin_password'],
- expr_form='pillar').values()[0]
- jenkins_port = local_salt_client.cmd(
- 'I@jenkins:client and not I@salt:master',
- 'pillar.get',
- ['_param:haproxy_jenkins_bind_port'],
- expr_form='compound').values()[0]
- jenkins_address = local_salt_client.cmd(
- 'I@jenkins:client and not I@salt:master',
- 'pillar.get',
- ['_param:haproxy_jenkins_bind_host'],
- expr_form='compound').values()[0]
+ jenkins_password = get_password(local_salt_client,'jenkins:client')
version_mismatch = []
- jenkins_url = 'http://{0}:{1}'.format(jenkins_address,jenkins_port)
- server = Jenkins(jenkins_url, username='admin', password=jenkins_password)
- for job_name, job_instance in server.get_jobs():
- job_config = job_instance.get_config()
+ server = join_to_jenkins(local_salt_client,'admin',jenkins_password)
+ for job_instance in server.get_jobs():
+ job_name = job_instance.get('name')
+ job_config = server.get_job_config(job_name)
xml_data = minidom.parseString(job_config)
BranchSpec = xml_data.getElementsByTagName('hudson.plugins.git.BranchSpec')
+ #We use master branch for pipeline-library in case of 'testing,stable,nighlty' versions
+ if expected_version in ['testing','nightly','stable']:
+ expected_version = 'master'
if BranchSpec:
actual_version = BranchSpec[0].getElementsByTagName('name')[0].childNodes[0].data
- if actual_version != expected_version and 'master' not in actual_version:
+ if ( actual_version != expected_version ) and ( job_name not in ['cvp-func','cvp-ha','cvp-perf'] ) :
version_mismatch.append("Job {0} has {1} branch."
- "Expected {2}".format(job_instance.name,
+ "Expected {2}".format(job_name,
actual_version,
expected_version))
assert len(version_mismatch) == 0, \
diff --git a/cvp_checks/tests/test_galera_cluster.py b/cvp_checks/tests/test_galera_cluster.py
index 5a74bdd..676f09b 100644
--- a/cvp_checks/tests/test_galera_cluster.py
+++ b/cvp_checks/tests/test_galera_cluster.py
@@ -9,7 +9,8 @@
expr_form='pillar')
if not gs:
- pytest.skip("Galera is not found on this environment")
+ pytest.skip("Galera service or galera:* pillar \
+ are not found on this environment.")
size_cluster = []
amount = len(gs)
diff --git a/cvp_checks/tests/test_nova_services.py b/cvp_checks/tests/test_nova_services.py
index 7332a49..84ca19b 100644
--- a/cvp_checks/tests/test_nova_services.py
+++ b/cvp_checks/tests/test_nova_services.py
@@ -1,8 +1,16 @@
+import pytest
+
+
def test_nova_services_status(local_salt_client):
result = local_salt_client.cmd(
'keystone:server',
'cmd.run',
['. /root/keystonerc; nova service-list | grep "down\|disabled" | grep -v "Forced down"'],
expr_form='pillar')
+
+ if not result:
+ pytest.skip("Nova service or keystone:server pillar \
+ are not found on this environment.")
+
assert result[result.keys()[0]] == '', \
'''Some nova services are in wrong state'''
diff --git a/cvp_checks/tests/test_stacklight.py b/cvp_checks/tests/test_stacklight.py
index f63b233..80917f1 100644
--- a/cvp_checks/tests/test_stacklight.py
+++ b/cvp_checks/tests/test_stacklight.py
@@ -1,6 +1,7 @@
import json
import requests
import datetime
+import pytest
from cvp_checks import utils
@@ -10,6 +11,11 @@
'pillar.get',
['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
+
+ if not salt_output:
+ pytest.skip("Kibana service or kibana:server pillar \
+ are not found on this environment.")
+
proxies = {"http": None, "https": None}
for node in salt_output.keys():
IP = salt_output[node]
@@ -44,6 +50,11 @@
'pillar.get',
['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
+
+ if not salt_output:
+ pytest.skip("Kibana service or kibana:server pillar \
+ are not found on this environment.")
+
IP = salt_output.values()[0]
proxies = {"http": None, "https": None}
resp = json.loads(requests.post('http://{0}:9200/log-{1}/_search?pretty'.
@@ -77,6 +88,12 @@
'cmd.run',
['docker service ls'],
expr_form='pillar')
+
+ if not salt_output:
+ pytest.skip("Docker replicas or \
+ docker:client:stack:monitoring pillar \
+ are not found on this environment.")
+
wrong_items = []
for line in salt_output[salt_output.keys()[0]].split('\n'):
if line[line.find('/') - 1] != line[line.find('/') + 1] \
@@ -98,6 +115,11 @@
['curl -s http://{}:15010/alerts | grep icon-chevron-down | '
'grep -v "0 active"'.format(IP)],
expr_form='pillar')
+
+ if not nodes_info:
+ pytest.skip("Prometheus server url or keystone:server \
+ pillar are not found on this environment.")
+
result = nodes_info[nodes_info.keys()[0]].replace('</td>', '').replace(
'<td><i class="icon-chevron-down"></i> <b>', '').replace('</b>', '')
assert result == '', 'AlertManager page has some alerts! {}'.format(
@@ -110,6 +132,11 @@
'cmd.run',
['docker service ps $(docker stack services -q monitoring)'],
expr_form='pillar')
+
+ if not salt_output:
+ pytest.skip("Docker or docker:swarm:role:master \
+ pillar are not found on this environment.")
+
result = {}
# for old reclass models, docker:swarm:role:master can return
# 2 nodes instead of one. Here is temporary fix.
@@ -136,8 +163,13 @@
'service.status',
'telegraf',
expr_form='pillar')
+
+ if not salt_output:
+ pytest.skip("Telegraf or telegraf:agent \
+ pillar are not found on this environment.")
+
result = [{node: status} for node, status
in salt_output.items()
if status is False]
assert result == [], 'Telegraf service is not running ' \
- 'on following nodes:'.format(result)
+ 'on following nodes: {}'.format(result)
diff --git a/cvp_checks/tests/test_ui_addresses.py b/cvp_checks/tests/test_ui_addresses.py
index 69c0f6a..2bf75d4 100644
--- a/cvp_checks/tests/test_ui_addresses.py
+++ b/cvp_checks/tests/test_ui_addresses.py
@@ -57,7 +57,7 @@
def test_ui_grafana(local_salt_client):
- IP = utils.get_monitoring_ip('cluster_public_host')
+ IP = utils.get_monitoring_ip('stacklight_monitor_address')
result = local_salt_client.cmd(
'keystone:server',
'cmd.run',
diff --git a/cvp_checks/utils/__init__.py b/cvp_checks/utils/__init__.py
index 6ad3a6d..91501a8 100644
--- a/cvp_checks/utils/__init__.py
+++ b/cvp_checks/utils/__init__.py
@@ -12,7 +12,10 @@
class salt_remote:
def cmd(self, tgt, fun, param=None, expr_form=None, tgt_type=None):
config = get_configuration()
- url = config['SALT_URL']
+ url = config['SALT_URL'].strip()
+ if not re.match("^(http|https)://", url):
+ raise AuthenticationError("Salt URL should start \
+ with http or https, given - {}".format(url))
proxies = {"http": None, "https": None}
headers = {'Accept': 'application/json'}
login_payload = {'username': config['SALT_USERNAME'],
diff --git a/requirements.txt b/requirements.txt
index 3c11ee2..38b0da6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
pytest==3.0.6
-requests
+requests==2.10.0
flake8
PyYAML
-jenkinsapi
+python-jenkins==0.4.11
+pygerrit2==2.0.6
+gitpython