Mark testresults automatically for each deployment
PROD-35718
Change-Id: I0289d9db8316d4ffcfd984b42be22e02b1abab1d
diff --git a/checklist.yaml b/checklist.yaml
index 570b16f..d962ec4 100644
--- a/checklist.yaml
+++ b/checklist.yaml
@@ -1,13 +1,4 @@
tests:
-# - title: 'tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin[id-51717b38-bdc1-458b-b636-1cf82d99f62f]'
-# status: 'ProdFailed'
-# errors:
-# - 'tempest.exceptions.BuildErrorException: Server'
-# - 'Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance'
-# defects: 'PROD-25148'
-#
-# - title: 'tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_create_server_with_scheduling_hint[id-fdcd9b33-0903-4e00-a1f7-b5f6543068d6]'
-# status: 'InfraFailed'
- title: 'tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway[id-6cc285d8-46bf-4f36-9b1a-783e3008ba79]'
errors:
@@ -21,17 +12,6 @@
status: "ProdFailed"
defects: 'PROD-25134'
- - title: 'test_prometheus_alert_count[ReclassRemoteDesync]'
- errors:
- - "local files under /srv/salt/reclass have diverged from expected remote state"
- status: "Won't Fix"
-
- - title: "test_prometheus_alert_count[ContrailBGPSessionsDown]"
- errors:
- - "1 OpenContrail BGP sessions on the ntw01 node are down for 2 minutes."
- status: "ProdFailed"
- defects: 'PROD-34379'
-
## ==================================================
- title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_delete_image_metadata_item"
errors:
@@ -39,25 +19,25 @@
status: "ProdFailed"
defects: "PROD-35212"
- - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_paused_server"
+ - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_paused_server[id-71bcb732-0261-11e7-9086-fa163e4fa634]"
errors:
- "502 GET https://10.6.0.80:8774/v2.1/images/"
status: "ProdFailed"
defects: "PROD-35212"
- - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_stopped_server"
+ - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_stopped_server[id-aaacd1d0-55a2-4ce8-818a-b5439df8adc9]"
errors:
- "502 GET https://10.6.0.80:8774/v2.1/images/"
status: "ProdFailed"
defects: "PROD-35212"
- - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_suspended_server"
+ - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_suspended_server[id-8ca07fec-0262-11e7-907e-fa163e4fa634]"
errors:
- "502 GET https://10.6.0.80:8774/v2.1/images/"
status: "ProdFailed"
defects: "PROD-35212"
- - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image"
+ - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314]"
errors:
- "502 GET https://10.6.0.80:8774/v2.1/images/"
status: "ProdFailed"
@@ -66,19 +46,26 @@
- title: ".setUpClass (tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON)"
errors:
- - "cls.server2['id'], wait_until='ACTIVE')"
+ - 'resp, body = self.get("images/%s" % image_id)'
- "tempest.lib.exceptions.UnexpectedResponseCode: Unexpected response code received"
- "Details: 502"
status: "ProdFailed"
defects: "PROD-35212"
- - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image"
+ - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314]"
errors:
- "message': u\"Cannot 'createImage' instance"
- "while it is in task_state image_uploading\", u'code': 409"
status: "ProdFailed"
defects: "PROD-35223"
+ - title: ".setUpClass (tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON)"
+ errors:
+ - 'resp, body = self.get("images/%s" % image_id)'
+ - "Details: 502"
+ status: "ProdFailed"
+ defects: "PROD-35212"
+
# ---------------------------------------------------
- title: "tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output_server_id_in_shutoff_status"
@@ -117,13 +104,19 @@
- "local files under /srv/salt/reclass have diverged from expected remote state"
status: "WontFix"
+ - title: "test_prometheus_alert_count[ContrailBGPSessionsDown]"
+ errors:
+ - "1 OpenContrail BGP sessions on the ntw01 node are down for 2 minutes."
+ status: "ProdFailed"
+ defects: 'PROD-34379'
## ==================================================
# LMA2.0_Automated
## ==================================================
- - title: "test_openstack_api_check_status_metrics "
+ - title: "test_openstack_api_check_status_metrics"
errors:
- - "Incorrect value in metric {u'value': [1600846840.578, u'0'], u'metric': {u'service_name': u'opencontrail', u'interface': u'public', u'service_id': u'35662704fa664daf95de90abc5180139', u'region': u'RegionOne', u'name': u'opencontrail', u'job': u'remote_agent_openstack_api', u'environment': u'heat-cicd-queens-contrail41-sl.local', u'instance': u'monitoring_remote_agent:9128', u'endpoint_id': u'721e2ad9e82148c9b5dbde40db83e47e', u'__name__': u'openstack_api_check_status'}}"
+ - "Incorrect value in metric"
+ - "u'metric': {u'service_name': u'opencontrail', u'interface': u'public'"
status: "WontFix"
defects: "PROD-35759"
diff --git a/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy b/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy
index 10ea257..dd866a3 100644
--- a/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy
+++ b/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy
@@ -263,18 +263,27 @@
stage("Clean the environment and clone tcp-qa") {
deleteDir()
+ // Install TestRail reporter to upload test results to TestRail
shared.verbose_sh("""\
[ -d /home/jenkins/venv_testrail_reporter ] || virtualenv /home/jenkins/venv_testrail_reporter
""", true, false, true)
shared.run_cmd("""\
. /home/jenkins/venv_testrail_reporter/bin/activate; pip install git+https://github.com/dis-xcom/testrail_reporter -U
""")
+ // Install DEVOPS tools
shared.verbose_sh("""\
[ -d /home/jenkins/fuel-devops30 ] || virtualenv /home/jenkins/fuel-devops30
""", true, false, true)
shared.run_cmd("""\
git clone https://gerrit.mcp.mirantis.com/mcp/tcp-qa ${PARENT_WORKSPACE}
""")
+ // Install TestRail reporter to mark known failures test results to TestRail
+ shared.verbose_sh("""\
+ [ -d /home/jenkins/venv_testrail_analyzer ] || virtualenv --python=python3 /home/jenkins/venv_testrail_analyzer
+ """, true, false, true)
+ shared.run_cmd("""\
+ . /home/jenkins/venv_testrail_analyzer/bin/activate; pip install git+https://github.com/ibumarskov/testrail-reporter
+ """)
shared.update_working_dir()
}
@@ -289,7 +298,7 @@
stage("Run the 'underlay' and 'salt-deployed' fixtures to bootstrap salt cluster") {
def xml_report_name = "deploy_salt.xml"
try {
- // deploy_salt.xml
+ // deploy_salt.xmli
shared.run_sh("""\
export ENV_NAME=${ENV_NAME}
export LAB_CONFIG_NAME=${LAB_CONFIG_NAME}
diff --git a/jobs/pipelines/swarm-testrail-report.groovy b/jobs/pipelines/swarm-testrail-report.groovy
index 17b770f..4af60af 100644
--- a/jobs/pipelines/swarm-testrail-report.groovy
+++ b/jobs/pipelines/swarm-testrail-report.groovy
@@ -19,6 +19,8 @@
def common = new com.mirantis.mk.Common()
def shared = new com.mirantis.system_qa.SharedPipeline()
def stacks = shared.get_steps_list(PASSED_STEPS)
+def testPlanNamePrefix = env.TEST_PLAN_NAME_PREFIX ?: "[2019.2.0-update]System"
+def testPlanName = env.TEST_PLAN_NAME ?: "${testPlanNamePrefix}-${MCP_VERSION}-${new Date().format('yyyy-MM-dd')}"
if (! env.PARENT_NODE_NAME) {
error "'PARENT_NODE_NAME' must be set from the parent deployment job!"
@@ -85,7 +87,7 @@
reporter_extra_options = [
"--testrail-add-missing-cases",
]
- ret = shared.upload_results_to_testrail(deployment_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+ ret = shared.upload_results_to_testrail(deployment_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
common.printMsg(ret.stdout, "blue")
report_url = ret.stdout.split("\n").each {
if (it.contains("[TestRun URL]")) {
@@ -106,7 +108,7 @@
"--testrail-add-missing-cases",
// "--testrail_configuration_name tcp-qa",
]
- ret = shared.upload_results_to_testrail(tcpqa_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+ ret = shared.upload_results_to_testrail(tcpqa_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
common.printMsg(ret.stdout, "blue")
report_url = ret.stdout.split("\n").each {
if (it.contains("[TestRun URL]")) {
@@ -129,11 +131,15 @@
// "--testrail_configuration_name tcp-qa",
]
ret = shared.upload_results_to_testrail(tempest_report_name,
+ testPlanName,
testSuiteName,
methodname,
testrail_name_template,
reporter_extra_options)
common.printMsg(ret.stdout, "blue")
+ marked = shared.mark_test_results(testPlanName, testSuiteName)
+ common.printMsg(marked.stdout, "blue")
+
report_url = ret.stdout.split("\n").each {
if (it.contains("[TestRun URL]")) {
common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
@@ -153,8 +159,11 @@
"--testrail-add-missing-cases",
"--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
]
- ret = shared.upload_results_to_testrail(stacklight_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+ ret = shared.upload_results_to_testrail(stacklight_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
common.printMsg(ret.stdout, "blue")
+ marked = shared.mark_test_results(testPlanName, testSuiteName)
+ common.printMsg(marked.stdout, "blue")
+
report_url = ret.stdout.split("\n").each {
if (it.contains("[TestRun URL]")) {
common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
@@ -175,9 +184,12 @@
"--testrail-add-missing-cases",
// "--testrail_configuration_name tcp-qa",
]
- ret = shared.upload_results_to_testrail(cvp_sanity_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
- common.printMsg(ret.stdout, "blue")
- report_url = ret.stdout.split("\n").each {
+ uploaded = shared.upload_results_to_testrail(cvp_sanity_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+ common.printMsg(uploaded.stdout, "blue")
+ marked = shared.mark_test_results(testPlanName, testSuiteName)
+ common.printMsg(marked.stdout, "blue")
+
+ report_url = uploaded.stdout.split("\n").each {
if (it.contains("[TestRun URL]")) {
common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
@@ -187,6 +199,7 @@
}
}
+
// Check if there were any exceptions during reporting
if (exception_message) {
throw new Exception(exception_message)
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index 6b5f82c..ccb0163 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -680,13 +680,11 @@
writeFile(file: filename, text: script, encoding: "UTF-8")
}
-def upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
+def upload_results_to_testrail(report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
def venvPath = '/home/jenkins/venv_testrail_reporter'
def testPlanDesc = env.LAB_CONFIG_NAME
def testrailURL = "https://mirantis.testrail.com"
def testrailProject = "Mirantis Cloud Platform"
- def testPlanNamePrefix = env.TEST_PLAN_NAME_PREFIX ?: "[2019.2.0-update]System"
- def testPlanName = env.TEST_PLAN_NAME ?: "${testPlanNamePrefix}-${MCP_VERSION}-${new Date().format('yyyy-MM-dd')}"
def testrailMilestone = "MCP1.1"
def testrailCaseMaxNameLenght = 250
def jobURL = env.BUILD_URL
@@ -744,3 +742,38 @@
def status = STATUS_MAP[result ?: 'FAILURE'] // currentBuild.result *must* be set at the finish of the try/catch
create_xml_report(filename, classname, name, status, "Deploy components: ${deploy_expected_stacks}", text, '', '')
}
+
+def mark_test_results(testPlanName, testSuiteName) {
+ def venvPath = '/home/jenkins/venv_testrail_analyzer'
+ def testrailURL = "https://mirantis.testrail.com"
+ def testrailProject = "Mirantis Cloud Platform"
+ def configName = env.LAB_CONFIG_NAME
+ def testRunName = "${configName} <${testSuiteName}>"
+
+ def script = """
+ . ${venvPath}/bin/activate
+ set -ex
+ export TESTRAIL_URL=${testrailURL}
+ testrail-reporter analyze -p "${testrailProject}" -t "${testPlanName}" -r "${testRunName}" checklist.yaml
+ """
+
+ def testrail_cred_id = params.TESTRAIL_CRED ?: 'testrail_system_tests'
+
+ withCredentials([
+ [$class : 'UsernamePasswordMultiBinding',
+ credentialsId : testrail_cred_id,
+ passwordVariable: 'TESTRAIL_PASSWORD',
+ usernameVariable: 'TESTRAIL_USER']
+ ]) {
+ def ret = [:]
+ ret.stdout = ''
+ ret.exception = ''
+ try {
+ ret.stdout = run_cmd_stdout(script)
+ } catch (Exception ex) {
+ ret.exception = ("""\
+##### Report to '${testRunName}' failed: #####\n""" + ex.message + "\n\n")
+ }
+ return ret
+ }
+}
\ No newline at end of file
diff --git a/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh b/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
index eb83ab4..cbf25d9 100644
--- a/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
+++ b/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
@@ -1,7 +1,9 @@
#!/bin/bash -xe
+LC_ALL=en_US.UTF-8
DEVOPS_VENV_PATH=/home/jenkins/fuel-devops30
REPORT_VENV_PATH=/home/jenkins/venv_testrail_reporter
+TESTMARKER_VENV_PATH=/home/jenkins/venv_testrail_analyzer
if [ ! -d ${DEVOPS_VENV_PATH} ]; then
virtualenv ${DEVOPS_VENV_PATH}
@@ -9,15 +11,25 @@
if [ ! -d ${REPORT_VENV_PATH} ]; then
virtualenv ${REPORT_VENV_PATH}
fi
+if [ ! -d ${TESTMARKER_VENV_PATH} ]; then
+ virtualenv --python=python3 ${TESTMARKER_VENV_PATH}
+fi
# Install tcp-qa requirements
. ${DEVOPS_VENV_PATH}/bin/activate
pip install -r https://raw.githubusercontent.com/Mirantis/tcp-qa/master/tcp_tests/requirements.txt
pip install psycopg2 # workaround for setup with PostgreSQL , to keep requirements.txt for Sqlite3 only
+deactivate
# Install xunit2testrail
. ${REPORT_VENV_PATH}/bin/activate
#pip install xunit2testrail -U
pip install git+https://github.com/dis-xcom/testrail_reporter -U # Removed accessing to an unexisting pastebin on srv62
+deactivate
+
+# Install testRail analyzer
+. ${TESTMARKER_VENV_PATH}/bin/activate
+pip install git+https://github.com/ibumarskov/testrail-reporter -U
+deactivate
chown -R jenkins:jenkins /home/jenkins/