Add reporting of [CVP Sanity] results
- add a new method get_artifact() to JenkinsClient
- add a CLI tool get_jenkins_job_artifact.py , example usage:
export ENV_NAME=cookied-cicd-queens-dvr-sl
. tcp_tests/utils/env_salt
. tcp_tests/utils/env_jenkins_cicd
tcp_tests/utils/get_jenkins_job_artifact.py \
--job-name cvp-sanity \
--build-number lastBuild \
--artifact-path validation_artifacts/cvp-sanity_report.xml \
--destination-name ./cvp-sanity_report.xml
- add the XML report downloader to the test "test_run_cvp_func_sanity"
- add new report type "CVP Sanity" to the testrail reporter
swarm-testrail-report.groovy
Closes-Bug: #PROD-25356
Change-Id: Ic34d76c62c7f70ada5b941e3ffc5b22e1be769d0
diff --git a/jobs/pipelines/swarm-testrail-report.groovy b/jobs/pipelines/swarm-testrail-report.groovy
index 92adfce..b9cbb16 100644
--- a/jobs/pipelines/swarm-testrail-report.groovy
+++ b/jobs/pipelines/swarm-testrail-report.groovy
@@ -61,12 +61,15 @@
def k8s_conformance_virtlet_report_name = sh(script: "find ${PARENT_WORKSPACE} -name \"conformance_virtlet_result.xml\" -printf \"'%p'\" ", returnStdout: true)
// stacklight_report_name =~ "stacklight_report.xml" or "report.xml"
def stacklight_report_name = sh(script: "find ${PARENT_WORKSPACE} -name \"*report.xml\"", returnStdout: true)
+ // cvp_sanity_report_name =~ cvp_sanity_report.xml
+ def cvp_sanity_report_name = sh(script: "find ${PARENT_WORKSPACE} -name \"cvp_sanity_results.xml\" -printf \"'%p'\" ", returnStdout: true)
common.printMsg(deployment_report_name ? "Found deployment report: ${deployment_report_name}" : "Deployment report not found", deployment_report_name ? "blue" : "red")
common.printMsg(tcpqa_report_name ? "Found tcp-qa report: ${tcpqa_report_name}" : "tcp-qa report not found", tcpqa_report_name ? "blue" : "red")
common.printMsg(tempest_report_name ? "Found tempest report: ${tempest_report_name}" : "tempest report not found", tempest_report_name ? "blue" : "red")
common.printMsg(k8s_conformance_report_name ? "Found k8s conformance report: ${k8s_conformance_report_name}" : "k8s conformance report not found", k8s_conformance_report_name ? "blue" : "red")
common.printMsg(k8s_conformance_virtlet_report_name ? "Found k8s conformance virtlet report: ${k8s_conformance_virtlet_report_name}" : "k8s conformance virtlet report not found", k8s_conformance_virtlet_report_name ? "blue" : "red")
common.printMsg(stacklight_report_name ? "Found stacklight-pytest report: ${stacklight_report_name}" : "stacklight-pytest report not found", stacklight_report_name ? "blue" : "red")
+ common.printMsg(cvp_sanity_report_name ? "Found CVP Sanity report: ${cvp_sanity_report_name}" : "CVP Sanity report not found", cvp_sanity_report_name ? "blue" : "red")
if (deployment_report_name) {
@@ -191,6 +194,28 @@
}
}
+ if ('cicd' in stacks && cvp_sanity_report_name) {
+ stage("CVP Sanity report") {
+ testSuiteName = "[MCP] cvp sanity"
+ methodname = '{methodname}'
+ testrail_name_template = '{title}'
+ reporter_extra_options = [
+ "--send-duplicates",
+ "--testrail-add-missing-cases",
+ "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
+ "--testrail-case-section-name \'All\'",
+ ]
+ report_result = shared.upload_results_to_testrail(cvp_sanity_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+ common.printMsg(report_result, "blue")
+ report_url = report_result.split("\n").each {
+ if (it.contains("[TestRun URL]")) {
+ common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
+ description += "\n<a href=" + it.trim().split().last() + ">${testSuiteName}</a>"
+ }
+ }
+ }
+ }
+
} catch (e) {
common.printMsg("Job is failed", "purple")
throw e
diff --git a/tcp_tests/managers/jenkins/client.py b/tcp_tests/managers/jenkins/client.py
index e713e71..6404432 100644
--- a/tcp_tests/managers/jenkins/client.py
+++ b/tcp_tests/managers/jenkins/client.py
@@ -196,3 +196,26 @@
self.__client._build_url(WORKFLOW_DESCRIPTION, locals()))
response = self.__client.jenkins_open(req)
return json.loads(response)
+
+ def get_artifact(self, name, build_id, artifact_path, destination_name):
+ '''Wait until the specified build is finished
+
+ :param name: ``str``, job name
+ :param build_id: ``str``, build id or "lastBuild"
+ :param artifact_path: ``str``, path and filename of the artifact
+ relative to the job URL
+ :param artifact_path: ``str``, destination path and filename
+ on the local filesystem where to save
+ the artifact content
+ :returns: requests object with headers and console output, ``obj``
+ '''
+ folder_url, short_name = self.__client._get_job_folder(name)
+
+ DOWNLOAD_URL = ('%(folder_url)sjob/%(short_name)s/%(build_id)s/'
+ 'artifact/%(artifact_path)s')
+ req = requests.Request(
+ 'GET',
+ self.__client._build_url(DOWNLOAD_URL, locals()))
+
+ response = self.__client.jenkins_request(req)
+ return response.content
diff --git a/tcp_tests/tests/system/test_cvp_pipelines.py b/tcp_tests/tests/system/test_cvp_pipelines.py
index 80154a1..cb6a5bb 100644
--- a/tcp_tests/tests/system/test_cvp_pipelines.py
+++ b/tcp_tests/tests/system/test_cvp_pipelines.py
@@ -12,12 +12,15 @@
# License for the specific language governing permissions and limitations
# under the License.
+import jenkins
import pytest
+import os
from tcp_tests import logger
from tcp_tests import settings
from tcp_tests.utils import run_jenkins_job
from tcp_tests.utils import get_jenkins_job_stages
+from tcp_tests.utils import get_jenkins_job_artifact
LOG = logger.logger
@@ -100,6 +103,7 @@
1. Get CICD Jenkins access credentials from salt
2. Run job cvp-sanity
3. Get passed stages from cvp-sanity
+ 4. Download XML report from the job
"""
salt = salt_actions
show_step(1)
@@ -145,9 +149,26 @@
LOG.info(description)
LOG.info('\n'.join(stages))
-
- assert cvp_func_sanity_result == 'SUCCESS', "{0}\n{1}".format(
- description, '\n'.join(stages))
+ LOG.info('Job {0} result: {1}'.format(job_name,
+ cvp_func_sanity_result))
+ # Download XML report
+ show_step(4)
+ destination_name = os.path.join(settings.LOGS_DIR,
+ "cvp_sanity_results.xml")
+ # Do not fail the test case when the job is failed, but
+ # artifact with the XML report is present in the job.
+ try:
+ get_jenkins_job_artifact.download_artifact(
+ host=jenkins_url,
+ username=jenkins_user,
+ password=jenkins_pass,
+ job_name=job_name,
+ build_number='lastBuild',
+ artifact_path='validation_artifacts/cvp-sanity_report.xml',
+ destination_name=destination_name)
+ except jenkins.NotFoundException:
+ raise jenkins.NotFoundException("{0}\n{1}".format(
+ description, '\n'.join(stages)))
@pytest.mark.grab_versions
@pytest.mark.parametrize("_", [settings.ENV_NAME])
diff --git a/tcp_tests/utils/get_jenkins_job_artifact.py b/tcp_tests/utils/get_jenkins_job_artifact.py
new file mode 100755
index 0000000..9e2295c
--- /dev/null
+++ b/tcp_tests/utils/get_jenkins_job_artifact.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Copyright 2019 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import os
+import sys
+
+sys.path.append(os.getcwd())
+try:
+ from tcp_tests.managers.jenkins.client import JenkinsClient
+except ImportError:
+ print("ImportError: Run the application from the tcp-qa directory or "
+ "set the PYTHONPATH environment variable to directory which contains"
+ " ./tcp_tests")
+ sys.exit(1)
+
+
+def load_params():
+ """
+ Parse CLI arguments and environment variables
+
+ Returns: ArgumentParser instance
+ """
+ env_host = os.environ.get('JENKINS_URL', None)
+ env_username = os.environ.get('JENKINS_USER', None)
+ env_password = os.environ.get('JENKINS_PASS', None)
+ env_job_name = os.environ.get('JOB_NAME', None)
+ env_build_number = os.environ.get('BUILD_NUMBER', 'lastBuild')
+
+ parser = argparse.ArgumentParser(description=(
+ 'Host, username and password may be specified either by the command '
+ 'line arguments or using environment variables: JENKINS_URL, '
+ 'JENKINS_USER, JENKINS_PASS. \nCommand line arguments have the highest'
+ ' priority, after that the environment variables are used as defaults.'
+ ))
+ parser.add_argument('--host',
+ metavar='JENKINS_URL',
+ help='Jenkins Host',
+ default=env_host)
+ parser.add_argument('--username',
+ metavar='JENKINS_USER',
+ help='Jenkins Username',
+ default=env_username)
+ parser.add_argument('--password',
+ metavar='JENKINS_PASS',
+ help='Jenkins Password or API token',
+ default=env_password)
+ parser.add_argument('--job-name',
+ metavar='JOB_NAME',
+ help='Jenkins job name',
+ default=env_job_name)
+ parser.add_argument('--build-number',
+ metavar='BUILD_NUMBER',
+ help='Jenkins job build number',
+ default=env_build_number)
+ parser.add_argument('--artifact-path',
+ help='Relative path of the artifact in Jenkins',
+ default=None,
+ type=str)
+ parser.add_argument('--destination-name',
+ help='Local filename for the saving artifact',
+ default=None,
+ type=str)
+ return parser
+
+
+def download_artifact(host, username, password,
+ job_name, build_number,
+ artifact_path, destination_name):
+
+ jenkins = JenkinsClient(
+ host=host,
+ username=username,
+ password=password)
+
+ content = jenkins.get_artifact(job_name, build_number,
+ artifact_path, destination_name)
+
+ with open(destination_name, 'wb') as f:
+ f.write(content)
+
+
+def main(args=None):
+ parser = load_params()
+ opts = parser.parse_args()
+
+ if (opts.host is None or opts.job_name is None
+ or opts.artifact_path is None or opts.destination_name is None):
+ print("JENKINS_URL, job_name and destination_name are required!")
+ parser.print_help()
+ return 10
+ else:
+ download_artifact(
+ opts.host,
+ opts.username,
+ opts.password,
+ opts.job_name,
+ opts.build_number,
+ opts.artifact_path,
+ opts.destination_name)
+
+
+if __name__ == "__main__":
+ sys.exit(main())