Add testrail reporter for cicd deployments
Related task: #PROD-21660
Change-Id: I7e1406b82697895db630ec0d470c05d93ac8c81f
diff --git a/jobs/pipelines/deploy-cicd-and-run-tests.groovy b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
index 7566b34..27dbfd7 100644
--- a/jobs/pipelines/deploy-cicd-and-run-tests.groovy
+++ b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
@@ -6,8 +6,8 @@
currentBuild.description = "${NODE_NAME}:${ENV_NAME}"
-throttle(['fuel_devops_environment']) {
- node ("${NODE_NAME}") {
+def deploy(shared, common, steps) {
+ def report_text = ''
try {
stage("Clean the environment and clone tcp-qa") {
@@ -29,20 +29,14 @@
shared.swarm_deploy_platform(env.PLATFORM_STACK_INSTALL)
}
- stage("Run tests") {
- shared.swarm_run_pytest(steps)
- }
+ currentBuild.result = 'SUCCESS'
} catch (e) {
- common.printMsg("Job failed", "red")
+ common.printMsg("Deploy is failed: " + e.message , "red")
shared.run_cmd("""\
dos.py suspend ${ENV_NAME} || true
- dos.py snapshot ${ENV_NAME} test_failed || true
+ dos.py snapshot ${ENV_NAME} deploy_failed || true
""")
- throw e
- } finally {
- // TODO(ddmitriev): analyze the "def currentResult = currentBuild.result ?: 'SUCCESS'"
- // and report appropriate data to TestRail
if ("${env.SHUTDOWN_ENV_ON_TEARDOWN}" == "false") {
shared.run_cmd("""\
dos.py resume ${ENV_NAME} || true
@@ -54,8 +48,52 @@
dos.py destroy ${ENV_NAME} || true
""")
}
- shared.report_deploy_result(steps)
- shared.report_test_result()
+ report_text = e.message
+ throw e
+ } finally {
+ shared.create_deploy_result_report(steps, currentBuild.result, report_text)
+ }
+}
+
+def test(shared, common, steps) {
+ try {
+ stage("Run tests") {
+ shared.swarm_run_pytest(steps)
+ }
+
+ } catch (e) {
+ common.printMsg("Tests are failed: " + e.message, "red")
+ shared.run_cmd("""\
+ dos.py suspend ${ENV_NAME} || true
+ dos.py snapshot ${ENV_NAME} tests_failed || true
+ """)
+ throw e
+ } finally {
+ if ("${env.SHUTDOWN_ENV_ON_TEARDOWN}" == "false") {
+ shared.run_cmd("""\
+ dos.py resume ${ENV_NAME} || true
+ sleep 20 # Wait for I/O on the host calms down
+ dos.py time-sync ${ENV_NAME} || true
+ """)
+ } else {
+ shared.run_cmd("""\
+ dos.py destroy ${ENV_NAME} || true
+ """)
+ }
+ }
+}
+
+// main
+throttle(['fuel_devops_environment']) {
+ node ("${NODE_NAME}") {
+ try {
+ deploy(shared, common, steps)
+ test(shared, common, steps)
+ } catch (e) {
+ common.printMsg("Job is failed: " + e.message, "red")
+ throw e
+ } finally {
+ shared.swarm_testrail_report(steps)
}
}
}
\ No newline at end of file
diff --git a/jobs/pipelines/swarm-bootstrap-salt-cluster-devops.groovy b/jobs/pipelines/swarm-bootstrap-salt-cluster-devops.groovy
index 5a25b38..f4c8765 100644
--- a/jobs/pipelines/swarm-bootstrap-salt-cluster-devops.groovy
+++ b/jobs/pipelines/swarm-bootstrap-salt-cluster-devops.groovy
@@ -97,7 +97,7 @@
}
} catch (e) {
- common.printMsg("Job failed", "red")
+ common.printMsg("Job is failed: " + e.message, "red")
throw e
} finally {
// TODO(ddmitriev): analyze the "def currentResult = currentBuild.result ?: 'SUCCESS'"
diff --git a/jobs/pipelines/swarm-deploy-cicd.groovy b/jobs/pipelines/swarm-deploy-cicd.groovy
index 1dafaad..d067e07 100644
--- a/jobs/pipelines/swarm-deploy-cicd.groovy
+++ b/jobs/pipelines/swarm-deploy-cicd.groovy
@@ -60,7 +60,7 @@
}
} catch (e) {
- common.printMsg("Job failed", "red")
+ common.printMsg("Job is failed: " + e.message, "red")
throw e
} finally {
// TODO(ddmitriev): analyze the "def currentResult = currentBuild.result ?: 'SUCCESS'"
diff --git a/jobs/pipelines/swarm-deploy-platform.groovy b/jobs/pipelines/swarm-deploy-platform.groovy
index 2396656..54bc43d 100644
--- a/jobs/pipelines/swarm-deploy-platform.groovy
+++ b/jobs/pipelines/swarm-deploy-platform.groovy
@@ -60,7 +60,7 @@
}
} catch (e) {
- common.printMsg("Job failed", "red")
+ common.printMsg("Job is failed:" + e.message, "red")
throw e
} finally {
// TODO(ddmitriev): analyze the "def currentResult = currentBuild.result ?: 'SUCCESS'"
diff --git a/jobs/pipelines/swarm-run-pytest.groovy b/jobs/pipelines/swarm-run-pytest.groovy
index d7d9e32..52dd25d 100644
--- a/jobs/pipelines/swarm-run-pytest.groovy
+++ b/jobs/pipelines/swarm-run-pytest.groovy
@@ -67,7 +67,7 @@
}
} catch (e) {
- common.printMsg("Job failed", "red")
+ common.printMsg("Job is failed" + e.message, "red")
throw e
} finally {
// TODO(ddmitriev): analyze the "def currentResult = currentBuild.result ?: 'SUCCESS'"
diff --git a/jobs/pipelines/swarm-testrail-report.groovy b/jobs/pipelines/swarm-testrail-report.groovy
new file mode 100644
index 0000000..df437b3
--- /dev/null
+++ b/jobs/pipelines/swarm-testrail-report.groovy
@@ -0,0 +1,95 @@
+/**
+ *
+ * Deploy the product cluster using Jenkins master on CICD cluster
+ *
+ * Expected parameters:
+
+ * ENV_NAME Fuel-devops environment name
+ * MCP_VERSION MCP version, like 2018.4 or proposed
+ * PASSED_STEPS Steps passed to install components using Jenkins on CICD cluster: "salt,core,cicd,openstack:3200,stacklight:2400",
+ where 3200 and 2400 might be timeouts (not used in the testing pipeline)
+ * PARENT_NODE_NAME Name of the jenkins slave to create the environment
+ * PARENT_WORKSPACE Path to the workspace of the parent job to use tcp-qa repo
+ * TCP_QA_REFS Reference to the tcp-qa change on review.gerrithub.io, like refs/changes/46/418546/41
+ */
+
+@Library('tcp-qa')_
+
+def common = new com.mirantis.mk.Common()
+def shared = new com.mirantis.system_qa.SharedPipeline()
+def stacks = shared.get_steps_list(PASSED_STEPS)
+
+if (! env.PARENT_NODE_NAME) {
+ error "'PARENT_NODE_NAME' must be set from the parent deployment job!"
+}
+
+currentBuild.description = "${PARENT_NODE_NAME}:${ENV_NAME}"
+
+node ("${PARENT_NODE_NAME}") {
+ if (! fileExists("${PARENT_WORKSPACE}")) {
+ error "'PARENT_WORKSPACE' contains path to non-existing directory ${PARENT_WORKSPACE} on the node '${PARENT_NODE_NAME}'."
+ }
+ dir("${PARENT_WORKSPACE}") {
+ try {
+ def report_name = ''
+ def testSuiteName = ''
+ def methodname = ''
+ def testrail_name_template = ''
+ def reporter_extra_options = []
+
+ stage("Deployment report") {
+ report_name = "deployment_${ENV_NAME}.xml"
+ testSuiteName = "[MCP] Integration automation"
+ methodname = '{methodname}'
+ testrail_name_template = '{title}'
+ reporter_extra_options = [
+ "--testrail-add-missing-cases",
+ "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
+ "--testrail-case-section-name \'All\'",
+ ]
+ shared.upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+ }
+
+ stage("tcp-qa cases report") {
+ report_name = "nosetests.xml"
+ testSuiteName = "[MCP_X] integration cases"
+ methodname = "{methodname}"
+ testrail_name_template = "{title}"
+ shared.upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template)
+ }
+
+ if ('openstack' in stacks) {
+ stage("Tempest report") {
+ report_name = "report_*.xml"
+ testSuiteName = "[MCP1.1_PIKE]Tempest"
+ methodname = "{classname}.{methodname}"
+ testrail_name_template = "{title}"
+ shared.upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template)
+ }
+ }
+
+ if ('k8s' in stacks) {
+ stage("Tempest report") {
+ println "TBD"
+ // K8s conformance report
+ }
+ }
+
+ if ('stacklight' in stacks) {
+ stage("stacklight-pytest report") {
+ report_name = "report.xml"
+ testSuiteName = "LMA2.0_Automated"
+ methodname = "{methodname}"
+ testrail_name_template = "{title}"
+ shared.upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template)
+ }
+ }
+
+ } catch (e) {
+ common.printMsg("Job is failed: " + e.message, "red")
+ throw e
+ } finally {
+ // reporting is failed for some reason
+ }
+ }
+}
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index 1a6d22e..7aee6fd 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -185,6 +185,23 @@
parameters: parameters
}
+def swarm_testrail_report(String passed_steps) {
+ // Run pytest tests
+ def common = new com.mirantis.mk.Common()
+ def parameters = [
+ string(name: 'ENV_NAME', value: "${ENV_NAME}"),
+ string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
+ string(name: 'PASSED_STEPS', value: passed_steps),
+ string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
+ string(name: 'PARENT_WORKSPACE', value: pwd()),
+ string(name: 'TCP_QA_REFS', value: "${TCP_QA_REFS}"),
+ ]
+ common.printMsg("Start building job 'swarm-testrail-report' with parameters:", "purple")
+ common.prettyPrint(parameters)
+ build job: 'swarm-testrail-report',
+ parameters: parameters
+}
+
def generate_cookied_model() {
def common = new com.mirantis.mk.Common()
// do not fail if environment doesn't exists
@@ -344,8 +361,76 @@
return steps.split(',').collect { it.split(':')[0] }
}
-def report_deploy_result(deploy_expected_stacks) {
+def create_xml_report(String filename, String classname, String name, String status='success', String status_message='', String text='', String stdout='', String stderr='') {
+ // <filename> is name of the XML report file that will be created
+ // <status> is one of the 'success', 'skipped', 'failure' or 'error'
+ // 'error' status is assumed as 'Blocker' in TestRail reporter
+ run_cmd("""\
+cat << \'EOF\' > ${filename}
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+ <testsuite>
+ <testcase classname=\"${classname}\" name=\"${name}\" time=\"0\">
+ <${status} message=\"${status_message}\">${text}</${status}>
+ <system-out>${stdout}</system-out>
+ <system-err>${stderr}</system-err>
+ </testcase>
+ </testsuite>
+EOF
+""")
}
-def report_test_result() {
+def upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
+ def venvPath = '/home/jenkins/venv_testrail_reporter'
+ def testPlanDesc = env.ENV_NAME
+ def testrailURL = "https://mirantis.testrail.com"
+ def testrailProject = "Mirantis Cloud Platform"
+ def testPlanName = "[MCP-Q2]System-${MCP_VERSION}-${new Date().format('yyyy-MM-dd')}"
+ def testrailMilestone = "MCP1.1"
+ def jobURL = env.BUILD_URL
+
+ def reporterOptions = [
+ "--verbose",
+ "--env-description \"${testPlanDesc}\"",
+ "--testrail-run-update",
+ "--testrail-url \"${testrailURL}\"",
+ "--testrail-user \"\${TESTRAIL_USER}\"",
+ "--testrail-password \"\${TESTRAIL_PASSWORD}\"",
+ "--testrail-project \"${testrailProject}\"",
+ "--testrail-plan-name \"${testPlanName}\"",
+ "--testrail-milestone \"${testrailMilestone}\"",
+ "--testrail-suite \"${testSuiteName}\"",
+ "--xunit-name-template \"${methodname}\"",
+ "--testrail-name-template \"${testrail_name_template}\"",
+ "--test-results-link \"${jobURL}\"",
+ ] + reporter_extra_options
+
+ def script = """
+ . ${venvPath}/bin/activate
+ set -ex
+ report_xml=\$(find \$(pwd) -name "${report_name}")
+ if [ -n "\${report_xml}" ]; then
+ report ${reporterOptions.join(' ')} \${report_xml}
+ fi
+ """
+
+ def testrail_cred_id = params.TESTRAIL_CRED ?: 'testrail_system_tests'
+
+ withCredentials([
+ [$class : 'UsernamePasswordMultiBinding',
+ credentialsId : testrail_cred_id,
+ passwordVariable: 'TESTRAIL_PASSWORD',
+ usernameVariable: 'TESTRAIL_USER']
+ ]) {
+ return run_cmd(script)
+ }
+}
+
+
+def create_deploy_result_report(deploy_expected_stacks, result, text) {
+ def STATUS_MAP = ['SUCCESS': 'success', 'FAILURE': 'failure', 'UNSTABLE': 'failure', 'ABORTED': 'error']
+ def classname = "Deploy"
+ def name = "deployment_${ENV_NAME}"
+ def filename = "\$(pwd)/${name}.xml"
+ def status = STATUS_MAP[result ?: 'FAILURE'] // currentBuild.result *must* be set at the finish of the try/catch
+ create_xml_report(filename, classname, name, status, "Deploy components: ${deploy_expected_stacks}", text, '', '')
}