Extend conformance runs and get all results
Change-Id: I87acc1a26ab245031565d4dd195e4f4e35cb66d1
diff --git a/tcp_tests/fixtures/k8s_fixtures.py b/tcp_tests/fixtures/k8s_fixtures.py
index c6fa204..fbfe644 100644
--- a/tcp_tests/fixtures/k8s_fixtures.py
+++ b/tcp_tests/fixtures/k8s_fixtures.py
@@ -84,18 +84,64 @@
@pytest.fixture(scope='function')
-def virtlet_logs(request, func_name, underlay, k8s_deployed):
- """Finalizer to extract virtlet conformance logs"""
+def k8s_logs(request, func_name, underlay, k8s_deployed):
+ """Finalizer to extract conformance logs
- grab_virtlet_result = request.keywords.get('grab_virtlet_results', None)
+ Usage:
+ @pytest.mark.grab_k8s_result(name=['file1', 'file2'])
+ ^^^^^
+ This mark says tcp-qa to download files that counted in array as
+ parameter 'name'. Files should be located at ctl01. Files will be
+ downloaded to the host, where your test runs.
+
+ @pytest.mark.extract(container_system='docker', extract_from='conformance',
+ files_to_extract=['report'])
+ ^^^^^
+ This mark says tcp-qa to copy files from container. Docker or k8s system
+ supported.
+ container_system param says function what strategy should be
+ used.
+ extract_from param says what container should be used to copy. Note
+ that we are using grep to determine container ID, so if you have multiple
+ container with same substring to copy you may encounter unexpected issues.
+ files_to_extract param - this is array with paths of files/dirs to copy.
+
+ @pytest.mark.merge_xunit(path='/root/report',
+ output='/root/conformance_result.xml')
+ ^^^^^
+ This mark will help you to merge xunit results in case if you have
+ multiple reports because of multiple threads.
+ path param says where xml results stored
+ output param says where result will be saved
+ """
+
+ grab_k8s_result = request.keywords.get('grab_k8s_results', None)
+ extract = request.keywords.get('extract', None)
+ merge_xunit = request.keywords.get('merge_xunit', None)
def test_fin():
if hasattr(request.node, 'rep_call') and \
(request.node.rep_call.passed or request.node.rep_call.failed)\
- and grab_virtlet_result:
- files = utils.extract_name_from_mark(grab_virtlet_result) \
+ and grab_k8s_result:
+ files = utils.extract_name_from_mark(grab_k8s_result) \
or "{}".format(func_name)
- k8s_deployed.extract_file_to_node()
+ if extract:
+ container_system = utils.extract_name_from_mark(
+ extract, 'container_system')
+ extract_from = utils.extract_name_from_mark(
+ extract, 'extract_from')
+ files_to_extract = utils.extract_name_from_mark(
+ extract, 'files_to_extract')
+ for path in files_to_extract:
+ k8s_deployed.extract_file_to_node(
+ system=container_system, container=extract_from,
+ file_path=path)
+ else:
+ k8s_deployed.extract_file_to_node()
+ if merge_xunit:
+ path = utils.extract_name_from_mark(merge_xunit, 'path')
+ output = utils.extract_name_from_mark(merge_xunit, 'output')
+ k8s_deployed.combine_xunit(path, output)
k8s_deployed.download_k8s_logs(files)
request.addfinalizer(test_fin)
diff --git a/tcp_tests/helpers/utils.py b/tcp_tests/helpers/utils.py
index 8cfa67a..15f9c8f 100644
--- a/tcp_tests/helpers/utils.py
+++ b/tcp_tests/helpers/utils.py
@@ -373,17 +373,18 @@
return template
-def extract_name_from_mark(mark):
+def extract_name_from_mark(mark, info='name'):
"""Simple function to extract name from pytest mark
:param mark: pytest.mark.MarkInfo
+ :param info: Kwarg with information
:rtype: string or None
"""
if mark:
if len(mark.args) > 0:
return mark.args[0]
- elif 'name' in mark.kwargs:
- return mark.kwargs['name']
+ elif info in mark.kwargs:
+ return mark.kwargs[info]
return None
diff --git a/tcp_tests/managers/k8smanager.py b/tcp_tests/managers/k8smanager.py
index e37ccaa..fd7952c 100644
--- a/tcp_tests/managers/k8smanager.py
+++ b/tcp_tests/managers/k8smanager.py
@@ -346,8 +346,8 @@
with self.__underlay.remote(
node_name=self.ctl_host) as remote:
result = remote.check_call(
- "docker run --rm --net=host -e API_SERVER="
- "'http://127.0.0.1:8080' {}".format(
+ "set -o pipefail; docker run --net=host -e API_SERVER="
+ "'http://127.0.0.1:8080' {} | tee k8s_conformance.log".format(
self.__config.k8s.k8s_conformance_image),
timeout=timeout)['stdout']
return result
@@ -591,12 +591,32 @@
master_host = self.__config.salt.salt_master_host
with self.__underlay.remote(host=master_host) as r:
for log_file in files:
- cmd = "rsync {0}:/root/{1} /root/".format(self.ctl_host,
- log_file)
+ cmd = "rsync -r {0}:/root/{1} /root/".format(self.ctl_host,
+ log_file)
r.check_call(cmd, raise_on_err=False)
LOG.info("Downloading the artifact {0}".format(log_file))
r.download(destination=log_file, target=os.getcwd())
+ def combine_xunit(self, path, output):
+ """
+ Function to combine multiple xmls with test results to
+ one.
+
+ :param path: Path where xmls to combine located
+ :param output: Path to xml file where output will stored
+ :return:
+ """
+ with self.__underlay.remote(node_name=self.ctl_host) as r:
+ cmd = "pip install xunitmerge"
+ LOG.debug('Installing xunitmerge')
+ r.check_call(cmd)
+ LOG.debug('Merging xunit')
+ cmd = ("cd {0}; arg = ''; "
+ "for i in $(ls | grep xml); "
+ "do arg=\"$arg $i\"; done && "
+ "xunitmerge $arg {1}".format(path, output))
+ r.check_call(cmd)
+
def manage_cncf_archive(self):
"""
Function to untar archive, move files, that we are needs to the
diff --git a/tcp_tests/tests/system/test_install_k8s.py b/tcp_tests/tests/system/test_install_k8s.py
index 018f561..e0e88cf 100644
--- a/tcp_tests/tests/system/test_install_k8s.py
+++ b/tcp_tests/tests/system/test_install_k8s.py
@@ -184,10 +184,17 @@
k8s_actions.run_conformance()
LOG.info("*************** DONE **************")
+ @pytest.mark.extract(container_system='docker', extract_from='conformance',
+ files_to_extract=['report'])
+ @pytest.mark.merge_xunit(path='/root/report',
+ output='/root/conformance_result.xml')
+ @pytest.mark.grab_k8s_results(name=['k8s_conformance.log',
+ 'conformance_result.xml'])
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.cz8116
- def test_only_k8s_install(self, config, k8s_deployed, k8s_actions):
+ def test_only_k8s_install(self, config, show_step,
+ k8s_deployed, k8s_actions, k8s_logs):
"""Test for deploying MCP environment with k8s and check it
Scenario:
@@ -199,5 +206,6 @@
"""
if config.k8s.k8s_conformance_run:
+ show_step(5)
k8s_actions.run_conformance()
LOG.info("*************** DONE **************")
diff --git a/tcp_tests/tests/system/test_virtlet_actions.py b/tcp_tests/tests/system/test_virtlet_actions.py
index 7fd593e..1cf4bee 100644
--- a/tcp_tests/tests/system/test_virtlet_actions.py
+++ b/tcp_tests/tests/system/test_virtlet_actions.py
@@ -90,11 +90,11 @@
k8s_deployed.delete_vm(target_yaml)
@pytest.mark.grab_versions
- @pytest.mark.grab_virtlet_results(name=['virtlet_conformance.log',
- 'report.xml'])
+ @pytest.mark.grab_k8s_results(name=['virtlet_conformance.log',
+ 'report.xml'])
@pytest.mark.fail_snapshot
def test_virtlet_conformance(self, show_step, config, k8s_deployed,
- virtlet_logs):
+ k8s_logs):
"""Test run of virtlet conformance tests
Scenario: