Renamed folder with tests to make them consistent with cvp-runner.groovy
and CVP jobs in cluster Jenkins
Return rsync service into inconsistency_rule
Related-Task: #PROD-23604(PROD:23604)
Change-Id: I94afe350bd1d9c184bafe8e9e270aeb4c6c24c50
diff --git a/test_set/cvp-sanity/tests/test_drivetrain.py b/test_set/cvp-sanity/tests/test_drivetrain.py
new file mode 100644
index 0000000..4628e70
--- /dev/null
+++ b/test_set/cvp-sanity/tests/test_drivetrain.py
@@ -0,0 +1,330 @@
+import jenkins
+from xml.dom import minidom
+import utils
+import json
+import pytest
+import time
+import os
+from pygerrit2 import GerritRestAPI, HTTPBasicAuth
+from requests import HTTPError
+import git
+import ldap
+import ldap.modlist as modlist
+
+def join_to_gerrit(local_salt_client, gerrit_user, gerrit_password):
+ gerrit_port = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_port'],
+ expr_form='compound').values()[0]
+ gerrit_address = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_host'],
+ expr_form='compound').values()[0]
+ url = 'http://{0}:{1}'.format(gerrit_address,gerrit_port)
+ auth = HTTPBasicAuth(gerrit_user, gerrit_password)
+ rest = GerritRestAPI(url=url, auth=auth)
+ return rest
+
+def join_to_jenkins(local_salt_client, jenkins_user, jenkins_password):
+ jenkins_port = local_salt_client.cmd(
+ 'I@jenkins:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_jenkins_bind_port'],
+ expr_form='compound').values()[0]
+ jenkins_address = local_salt_client.cmd(
+ 'I@jenkins:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_jenkins_bind_host'],
+ expr_form='compound').values()[0]
+ jenkins_url = 'http://{0}:{1}'.format(jenkins_address,jenkins_port)
+ server = jenkins.Jenkins(jenkins_url, username=jenkins_user, password=jenkins_password)
+ return server
+
+def get_password(local_salt_client,service):
+ password = local_salt_client.cmd(
+ service,
+ 'pillar.get',
+ ['_param:openldap_admin_password'],
+ expr_form='pillar').values()[0]
+ return password
+
+def test_drivetrain_gerrit(local_salt_client):
+ gerrit_password = get_password(local_salt_client,'gerrit:client')
+ gerrit_error = ''
+ current_date = time.strftime("%Y%m%d-%H.%M.%S", time.localtime())
+ test_proj_name = "test-dt-{0}".format(current_date)
+ gerrit_port = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_port'],
+ expr_form='compound').values()[0]
+ gerrit_address = local_salt_client.cmd(
+ 'I@gerrit:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_gerrit_bind_host'],
+ expr_form='compound').values()[0]
+ try:
+ #Connecting to gerrit and check connection
+ server = join_to_gerrit(local_salt_client,'admin',gerrit_password)
+ gerrit_check = server.get("/changes/?q=owner:self%20status:open")
+ #Check deleteproject plugin and skip test if the plugin is not installed
+ gerrit_plugins = server.get("/plugins/?all")
+ if 'deleteproject' not in gerrit_plugins:
+ pytest.skip("Delete-project plugin is not installed")
+ #Create test project and add description
+ server.put("/projects/"+test_proj_name)
+ server.put("/projects/"+test_proj_name+"/description",json={"description":"Test DriveTrain project","commit_message": "Update the project description"})
+ except HTTPError, e:
+ gerrit_error = e
+ try:
+ #Create test folder and init git
+ repo_dir = os.path.join(os.getcwd(),test_proj_name)
+ file_name = os.path.join(repo_dir, current_date)
+ repo = git.Repo.init(repo_dir)
+ #Add remote url for this git repo
+ origin = repo.create_remote('origin', 'http://admin:{1}@{2}:{3}/{0}.git'.format(test_proj_name,gerrit_password,gerrit_address,gerrit_port))
+ #Add commit-msg hook to automatically add Change-Id to our commit
+ os.system("curl -Lo {0}/.git/hooks/commit-msg 'http://admin:{1}@{2}:{3}/tools/hooks/commit-msg' > /dev/null 2>&1".format(repo_dir,gerrit_password,gerrit_address,gerrit_port))
+ os.system("chmod u+x {0}/.git/hooks/commit-msg".format(repo_dir))
+ #Create a test file
+ f = open(file_name, 'w+')
+ f.write("This is a test file for DriveTrain test")
+ f.close()
+ #Add file to git and commit it to Gerrit for review
+ repo.index.add([file_name])
+ repo.index.commit("This is a test commit for DriveTrain test")
+ repo.git.push("origin", "HEAD:refs/for/master")
+ #Get change id from Gerrit. Set Code-Review +2 and submit this change
+ changes = server.get("/changes/?q=project:{0}".format(test_proj_name))
+ last_change = changes[0].get('change_id')
+ server.post("/changes/{0}/revisions/1/review".format(last_change),json={"message":"All is good","labels":{"Code-Review":"+2"}})
+ server.post("/changes/{0}/submit".format(last_change))
+ except HTTPError, e:
+ gerrit_error = e
+ finally:
+ #Delete test project
+ server.post("/projects/"+test_proj_name+"/deleteproject~delete")
+ assert gerrit_error == '',\
+ 'Something is wrong with Gerrit'.format(gerrit_error)
+
+def test_drivetrain_openldap(local_salt_client):
+ '''Create a test user 'DT_test_user' in openldap,
+ add the user to admin group, login using the user to Jenkins.
+ Add the user to devops group in Gerrit and then login to Gerrit,
+ using test_user credentials. Finally, delete the user from admin
+ group and openldap
+ '''
+ ldap_password = get_password(local_salt_client,'openldap:client')
+ #Check that ldap_password is exists, otherwise skip test
+ if not ldap_password:
+ pytest.skip("Openldap service or openldap:client pillar \
+ are not found on this environment.")
+ ldap_port = local_salt_client.cmd(
+ 'I@openldap:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_openldap_bind_port'],
+ expr_form='compound').values()[0]
+ ldap_address = local_salt_client.cmd(
+ 'I@openldap:client and not I@salt:master',
+ 'pillar.get',
+ ['_param:haproxy_openldap_bind_host'],
+ expr_form='compound').values()[0]
+ ldap_dc = local_salt_client.cmd(
+ 'openldap:client',
+ 'pillar.get',
+ ['_param:openldap_dn'],
+ expr_form='pillar').values()[0]
+ ldap_con_admin = local_salt_client.cmd(
+ 'openldap:client',
+ 'pillar.get',
+ ['openldap:client:server:auth:user'],
+ expr_form='pillar').values()[0]
+ ldap_url = 'ldap://{0}:{1}'.format(ldap_address,ldap_port)
+ ldap_error = ''
+ ldap_result = ''
+ gerrit_result = ''
+ gerrit_error = ''
+ jenkins_error = ''
+ #Test user's CN
+ test_user_name = 'DT_test_user'
+ test_user = 'cn={0},ou=people,{1}'.format(test_user_name,ldap_dc)
+ #Admins group CN
+ admin_gr_dn = 'cn=admins,ou=groups,{0}'.format(ldap_dc)
+ #List of attributes for test user
+ attrs = {}
+ attrs['objectclass'] = ['organizationalRole','simpleSecurityObject','shadowAccount']
+ attrs['cn'] = test_user_name
+ attrs['uid'] = test_user_name
+ attrs['userPassword'] = 'aSecretPassw'
+ attrs['description'] = 'Test user for CVP DT test'
+ searchFilter = 'cn={0}'.format(test_user_name)
+ #Get a test job name from config
+ config = utils.get_configuration()
+ jenkins_cvp_job = config['jenkins_cvp_job']
+ #Open connection to ldap and creating test user in admins group
+ try:
+ ldap_server = ldap.initialize(ldap_url)
+ ldap_server.simple_bind_s(ldap_con_admin,ldap_password)
+ ldif = modlist.addModlist(attrs)
+ ldap_server.add_s(test_user,ldif)
+ ldap_server.modify_s(admin_gr_dn,[(ldap.MOD_ADD, 'memberUid', [test_user_name],)],)
+ #Check search test user in LDAP
+ searchScope = ldap.SCOPE_SUBTREE
+ ldap_result = ldap_server.search_s(ldap_dc, searchScope, searchFilter)
+ except ldap.LDAPError, e:
+ ldap_error = e
+ try:
+ #Check connection between Jenkins and LDAP
+ jenkins_server = join_to_jenkins(local_salt_client,test_user_name,'aSecretPassw')
+ jenkins_version = jenkins_server.get_job_name(jenkins_cvp_job)
+ #Check connection between Gerrit and LDAP
+ gerrit_server = join_to_gerrit(local_salt_client,'admin',ldap_password)
+ gerrit_check = gerrit_server.get("/changes/?q=owner:self%20status:open")
+ #Add test user to devops-contrib group in Gerrit and check login
+ _link = "/groups/devops-contrib/members/{0}".format(test_user_name)
+ gerrit_add_user = gerrit_server.put(_link)
+ gerrit_server = join_to_gerrit(local_salt_client,test_user_name,'aSecretPassw')
+ gerrit_result = gerrit_server.get("/changes/?q=owner:self%20status:open")
+ except HTTPError, e:
+ gerrit_error = e
+ except jenkins.JenkinsException, e:
+ jenkins_error = e
+ finally:
+ ldap_server.modify_s(admin_gr_dn,[(ldap.MOD_DELETE, 'memberUid', [test_user_name],)],)
+ ldap_server.delete_s(test_user)
+ ldap_server.unbind_s()
+ assert ldap_error == '', \
+ '''Something is wrong with connection to LDAP:
+ {0}'''.format(e)
+ assert jenkins_error == '', \
+ '''Connection to Jenkins was not established:
+ {0}'''.format(e)
+ assert gerrit_error == '', \
+ '''Connection to Gerrit was not established:
+ {0}'''.format(e)
+ assert ldap_result !=[], \
+ '''Test user was not found'''
+
+def test_drivetrain_jenkins_job(local_salt_client):
+ jenkins_password = get_password(local_salt_client,'jenkins:client')
+ server = join_to_jenkins(local_salt_client,'admin',jenkins_password)
+ #Getting Jenkins test job name from configuration
+ config = utils.get_configuration()
+ jenkins_test_job = config['jenkins_test_job']
+ if not server.get_job_name(jenkins_test_job):
+ server.create_job(jenkins_test_job, jenkins.EMPTY_CONFIG_XML)
+ if server.get_job_name(jenkins_test_job):
+ next_build_num = server.get_job_info(jenkins_test_job)['nextBuildNumber']
+ #If this is first build number skip building check
+ if next_build_num != 1:
+ #Check that test job is not running at this moment,
+ #Otherwise skip the test
+ last_build_num = server.get_job_info(jenkins_test_job)['lastBuild'].get('number')
+ last_build_status = server.get_build_info(jenkins_test_job,last_build_num)['building']
+ if last_build_status:
+ pytest.skip("Test job {0} is already running").format(jenkins_test_job)
+ server.build_job(jenkins_test_job)
+ timeout = 0
+ #Use job status True by default to exclude timeout between build job and start job.
+ job_status = True
+ while job_status and ( timeout < 180 ):
+ time.sleep(10)
+ timeout += 10
+ job_status = server.get_build_info(jenkins_test_job,next_build_num)['building']
+ job_result = server.get_build_info(jenkins_test_job,next_build_num)['result']
+ else:
+ pytest.skip("The job {0} was not found").format(test_job_name)
+ assert job_result == 'SUCCESS', \
+ '''Test job '{0}' build was not successfull or timeout is too small
+ '''.format(jenkins_test_job)
+
+def test_drivetrain_services_replicas(local_salt_client):
+ salt_output = local_salt_client.cmd(
+ 'I@gerrit:client',
+ 'cmd.run',
+ ['docker service ls'],
+ expr_form='compound')
+ wrong_items = []
+ for line in salt_output[salt_output.keys()[0]].split('\n'):
+ if line[line.find('/') - 1] != line[line.find('/') + 1] \
+ and 'replicated' in line:
+ wrong_items.append(line)
+ assert len(wrong_items) == 0, \
+ '''Some DriveTrain services doesn't have expected number of replicas:
+ {}'''.format(json.dumps(wrong_items, indent=4))
+
+
+def test_drivetrain_components_and_versions(local_salt_client):
+ config = utils.get_configuration()
+ if not config['drivetrain_version']:
+ version = \
+ local_salt_client.cmd(
+ 'I@salt:master',
+ 'pillar.get',
+ ['_param:mcp_version'],
+ expr_form='compound').values()[0] or \
+ local_salt_client.cmd(
+ 'I@salt:master',
+ 'pillar.get',
+ ['_param:apt_mk_version'],
+ expr_form='compound').values()[0]
+ if not version:
+ pytest.skip("drivetrain_version is not defined. Skipping")
+ else:
+ version = config['drivetrain_version']
+ salt_output = local_salt_client.cmd(
+ 'I@gerrit:client',
+ 'cmd.run',
+ ['docker service ls'],
+ expr_form='compound')
+ # 'ldap_server' removed because it is an external component now v 1.1.8
+ not_found_services = ['gerrit_db', 'gerrit_server', 'jenkins_master',
+ 'jenkins_slave01', 'jenkins_slave02',
+ 'jenkins_slave03', 'ldap_admin', 'docker_registry',
+ 'docker_visualizer']
+ version_mismatch = []
+ for line in salt_output[salt_output.keys()[0]].split('\n'):
+ for service in not_found_services:
+ if service in line:
+ not_found_services.remove(service)
+ if version != line.split()[4].split(':')[1]:
+ version_mismatch.append("{0}: expected "
+ "version is {1}, actual - {2}".format(service,version,
+ line.split()[4].split(':')[1]))
+ continue
+ assert len(not_found_services) == 0, \
+ '''Some DriveTrain components are not found:
+ {}'''.format(json.dumps(not_found_services, indent=4))
+ assert len(version_mismatch) == 0, \
+ '''Version mismatch found:
+ {}'''.format(json.dumps(version_mismatch, indent=4))
+
+
+def test_jenkins_jobs_branch(local_salt_client):
+ config = utils.get_configuration()
+ expected_version = config['drivetrain_version'] or []
+ if not expected_version or expected_version == '':
+ pytest.skip("drivetrain_version is not defined. Skipping")
+ jenkins_password = get_password(local_salt_client,'jenkins:client')
+ version_mismatch = []
+ server = join_to_jenkins(local_salt_client,'admin',jenkins_password)
+ for job_instance in server.get_jobs():
+ job_name = job_instance.get('name')
+ job_config = server.get_job_config(job_name)
+ xml_data = minidom.parseString(job_config)
+ BranchSpec = xml_data.getElementsByTagName('hudson.plugins.git.BranchSpec')
+ #We use master branch for pipeline-library in case of 'testing,stable,nighlty' versions
+ if expected_version in ['testing','nightly','stable']:
+ expected_version = 'master'
+ if BranchSpec:
+ actual_version = BranchSpec[0].getElementsByTagName('name')[0].childNodes[0].data
+ if ( actual_version != expected_version ) and ( job_name not in ['upgrade-mcp-release'] ) :
+ version_mismatch.append("Job {0} has {1} branch."
+ "Expected {2}".format(job_name,
+ actual_version,
+ expected_version))
+ assert len(version_mismatch) == 0, \
+ '''Some DriveTrain jobs have version/branch mismatch:
+ {}'''.format(json.dumps(version_mismatch, indent=4))