Fix runtestmanager for CICD deployments
In CICD deployments, there are no workarounds that install
docker.io before using 'runtest' formula.
Add preparation of necessary packages to runtestmanager.
- Refactor RuntestManager to use salt CLI instead of salt API
because salt API dones't allow to catch errors from modules
- Add preparations for docker packages, Neutron private and
public networks based on underlay.yaml address pools, etc.
Change-Id: Iaff8046b9cde6185cd6718a5c2064998ea7dc507
diff --git a/tcp_tests/managers/runtestmanager.py b/tcp_tests/managers/runtestmanager.py
index c400556..036f415 100644
--- a/tcp_tests/managers/runtestmanager.py
+++ b/tcp_tests/managers/runtestmanager.py
@@ -14,9 +14,6 @@
import json
import os
-import time
-
-from devops.helpers import helpers
from tcp_tests import logger
from tcp_tests import settings
@@ -25,71 +22,6 @@
TEMPEST_CFG_DIR = '/tmp/test'
-CONFIG = {
- 'classes': ['service.runtest.tempest',
- 'service.runtest.tempest.services.manila.glance'],
- 'parameters': {
- '_param': {
- 'runtest_tempest_cfg_dir': TEMPEST_CFG_DIR,
- 'runtest_tempest_cfg_name': 'tempest.conf',
- 'runtest_tempest_public_net': 'net04_ext',
- 'tempest_test_target': 'gtw01*'
- },
- 'neutron': {
- 'client': {
- 'enabled': True
- }
- },
- 'runtest': {
- 'enabled': True,
- 'keystonerc_node': 'ctl01*',
- 'tempest': {
- 'enabled': True,
- 'cfg_dir': '${_param:runtest_tempest_cfg_dir}',
- 'cfg_name': '${_param:runtest_tempest_cfg_name}',
- 'DEFAULT': {
- 'log_file': 'tempest.log'
- },
- 'compute': {
- 'build_timeout': 600,
- 'max_microversion': 2.53,
- 'min_compute_nodes': 2,
- 'min_microversion': 2.1,
- 'volume_device_name': 'vdc'
- },
- 'convert_to_uuid': {
- 'network': {
- 'public_network_id':
- '${_param:runtest_tempest_public_net}'
- }
- },
- 'dns_feature_enabled': {
- 'api_admin': False,
- 'api_v1': False,
- 'api_v2': True,
- 'api_v2_quotas': True,
- 'api_v2_root_recordsets': True,
- 'bug_1573141_fixed': True
- },
- 'heat_plugin': {
- 'floating_network_name':
- '${_param:runtest_tempest_public_net}'
- },
- 'network': {
- 'floating_network_name':
- '${_param:runtest_tempest_public_net}'
- },
- 'share': {
- 'capability_snapshot_support': True,
- 'run_driver_assisted_migration_tests': False,
- 'run_manage_unmanage_snapshot_tests': False,
- 'run_manage_unmanage_tests': False,
- 'run_migration_with_preserve_snapshots_tests': False,
- 'run_quota_tests': True,
- 'run_replication_tests': False,
- 'run_snapshot_tests': True,
- }}}}}
-
class RuntestManager(object):
"""Helper manager for execution tempest via runtest-formula"""
@@ -98,60 +30,117 @@
image_version = settings.TEMPEST_IMAGE_VERSION
container_name = 'run-tempest-ci'
master_host = "cfg01"
- master_tgt = "{}*".format(master_host)
+ control_host = "ctl01"
class_name = "runtest"
run_cmd = '/bin/bash -c "run-tempest"'
- def __init__(self, underlay, salt_api, cluster_name,
+ def __init__(self, config, underlay, salt_api, cluster_name,
domain_name, tempest_threads,
tempest_pattern=settings.TEMPEST_PATTERN,
run_cmd=None, target='gtw01'):
+ self.__config = config
self.underlay = underlay
self.__salt_api = salt_api
- self.target = target
self.cluster_name = cluster_name
self.domain_name = domain_name
self.tempest_threads = tempest_threads
self.tempest_pattern = tempest_pattern
self.run_cmd = run_cmd or self.run_cmd
+ self.target_name = self.underlay.get_target_node_names(target)[0]
+ self.master_name = self.underlay.get_target_node_names(
+ self.master_host)[0]
+ self.control_name = self.underlay.get_target_node_names(
+ self.control_host)[0]
@property
def salt_api(self):
return self.__salt_api
- def install_python_lib(self):
- return self.salt_api.local(
- "{}*".format(self.target),
- 'pip.install', 'docker'), None
+ @property
+ def runtest_pillar(self):
+ public_net = self.__config.underlay.dhcp_ranges[
+ settings.EXTERNAL_ADDRESS_POOL_NAME]
+ public_gateway = public_net["gateway"].encode("ascii")
+ public_cidr = public_net["cidr"].encode("ascii")
+ public_allocation_start = public_net["start"].encode("ascii")
+ public_allocation_end = public_net["end"].encode("ascii")
- def run_salt_minion_state(self):
- return self.salt_api.local('cfg01*', 'state.sls', 'salt.minion')
-
- def create_networks(self):
- return self.salt_api.local('cfg01*', 'state.sls', 'neutron.client')
-
- def create_flavors(self):
- return self.salt_api.local('cfg01*', 'state.sls', 'nova.client')
-
- def set_property(self):
- return self.salt_api.local(
- tgt='ctl01*',
- fun='cmd.run',
- args='. /root/keystonercv3; openstack '
- 'flavor set m1.tiny_test '
- '--property hw:mem_page_size=small')
-
- def create_cirros(self):
- return self.salt_api.local('cfg01*', 'state.sls', 'glance.client')
-
- def generate_config(self):
- return self.salt_api.local('cfg01*', 'state.sls', 'runtest')
+ return {
+ 'classes': ['service.runtest.tempest',
+ 'service.runtest.tempest.public_net',
+ 'service.runtest.tempest.services.manila.glance'],
+ 'parameters': {
+ '_param': {
+ 'runtest_tempest_cfg_dir': TEMPEST_CFG_DIR,
+ 'runtest_tempest_cfg_name': 'tempest.conf',
+ 'runtest_tempest_public_net': 'public',
+ 'openstack_public_neutron_subnet_gateway': public_gateway,
+ 'openstack_public_neutron_subnet_cidr': public_cidr,
+ 'openstack_public_neutron_subnet_allocation_start':
+ public_allocation_start,
+ 'openstack_public_neutron_subnet_allocation_end':
+ public_allocation_end,
+ 'tempest_test_target': self.target_name.encode("ascii"),
+ },
+ 'neutron': {
+ 'client': {
+ 'enabled': True
+ }
+ },
+ 'runtest': {
+ 'enabled': True,
+ 'keystonerc_node': 'ctl01*',
+ 'tempest': {
+ 'enabled': True,
+ 'cfg_dir': '${_param:runtest_tempest_cfg_dir}',
+ 'cfg_name': '${_param:runtest_tempest_cfg_name}',
+ 'DEFAULT': {
+ 'log_file': 'tempest.log'
+ },
+ 'compute': {
+ 'build_timeout': 600,
+ 'max_microversion': 2.53,
+ 'min_compute_nodes': 2,
+ 'min_microversion': 2.1,
+ 'volume_device_name': 'vdc'
+ },
+ 'convert_to_uuid': {
+ 'network': {
+ 'public_network_id':
+ '${_param:runtest_tempest_public_net}'
+ }
+ },
+ 'dns_feature_enabled': {
+ 'api_admin': False,
+ 'api_v1': False,
+ 'api_v2': True,
+ 'api_v2_quotas': True,
+ 'api_v2_root_recordsets': True,
+ 'bug_1573141_fixed': True
+ },
+ 'heat_plugin': {
+ 'floating_network_name':
+ '${_param:runtest_tempest_public_net}'
+ },
+ 'network': {
+ 'floating_network_name':
+ '${_param:runtest_tempest_public_net}'
+ },
+ 'share': {
+ 'capability_snapshot_support': True,
+ 'run_driver_assisted_migration_tests': False,
+ 'run_manage_unmanage_snapshot_tests': False,
+ 'run_manage_unmanage_tests': False,
+ 'run_migration_with_preserve_snapshots_tests':
+ False,
+ 'run_quota_tests': True,
+ 'run_replication_tests': False,
+ 'run_snapshot_tests': True,
+ }}}}}
def fetch_arficats(self, username=None, file_format='xml'):
- target_name = next(node_name for node_name
- in self.underlay.node_names() if
- self.target in node_name)
- with self.underlay.remote(node_name=target_name, username=None) as tgt:
+ with self.underlay.remote(node_name=self.target_name,
+ username=None) as tgt:
result = tgt.execute('find {} -name "report_*.{}"'.format(
TEMPEST_CFG_DIR, file_format))
LOG.debug("Find result {0}".format(result))
@@ -163,42 +152,38 @@
destination=report, # noqa
target=os.getcwd())
- def store_runtest_model(self, config=CONFIG):
- master_name = next(node_name for node_name
- in self.underlay.node_names() if
- self.master_host in node_name)
+ def store_runtest_model(self, runtest_pillar=None):
with self.underlay.yaml_editor(
file_path="/srv/salt/reclass/classes/cluster/"
"{cluster_name}/infra/"
"{class_name}.yml".format(
cluster_name=self.cluster_name,
class_name=self.class_name),
- node_name=master_name) as editor:
- editor.content = config
+ node_name=self.master_name) as editor:
+ editor.content = runtest_pillar or self.runtest_pillar
with self.underlay.yaml_editor(
file_path="/srv/salt/reclass/nodes/_generated/"
"cfg01.{domain_name}.yml".format(
domain_name=self.domain_name),
- node_name=master_name) as editor:
+ node_name=self.master_name) as editor:
editor.content['classes'].append(
'cluster.{cluster_name}.infra.{class_name}'.format(
cluster_name=self.cluster_name,
class_name=self.class_name))
- self.salt_api.local('*', 'saltutil.refresh_pillar')
- self.salt_api.local('*', 'saltutil.sync_all')
-
def save_runtime_logs(self, logs=None, inspect=None):
if logs:
with open("{path}/{target}_tempest_run.log".format(
- path=settings.LOGS_DIR, target=self.target), 'w') as f:
+ path=settings.LOGS_DIR,
+ target=self.target_name), 'w') as f:
LOG.info("Save tempest console log")
container_log = logs
f.write(container_log.encode('ascii', 'ignore'))
if inspect:
with open("{path}/{target}_tempest_container_info.json.log".format(
- path=settings.LOGS_DIR, target=self.target), 'w') as f:
+ path=settings.LOGS_DIR,
+ target=self.target_name), 'w') as f:
LOG.info("Save tempest container inspect data")
container_inspect = json.dumps(inspect,
@@ -208,104 +193,123 @@
def prepare(self, dpdk=None):
self.store_runtest_model()
- res = self.install_python_lib()
- LOG.info(json.dumps(res, indent=4))
+ salt_cmd = "salt -l info --hard-crash --state-output=mixed "
+ salt_call_cmd = "salt-call -l info --hard-crash --state-output=mixed "
+ commands = [
+ {
+ 'description': "Sync salt objects for runtest model",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_cmd + "'*' saltutil.refresh_pillar && " +
+ salt_cmd + "'*' saltutil.sync_all")},
+ {
+ 'description': ("Install docker.io package and "
+ "enable packets forwarding"),
+ 'node_name': self.target_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " pkg.install docker.io && " +
+ " iptables --policy FORWARD ACCEPT")},
+ {
+ 'description': "Install PyPI docker package",
+ 'node_name': self.target_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " pip.install setuptools && " +
+ salt_call_cmd + " pip.install docker")},
+ {
+ 'description': "Run salt.minion state for runtest formula",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " state.sls salt.minion && "
+ " sleep 20")},
+ {
+ 'description': "Enforce keystone state for neutronv2",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " state.sls keystone.client")},
+ {
+ 'description': "Create networks for Tempest tests",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " state.sls neutron.client")},
+ {
+ 'description': "Create flavors for Tempest tests",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " state.sls nova.client")},
+ {
+ 'description': "Create cirros image for Tempest",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " state.sls glance.client")},
+ {
+ 'description': "Generate config for Tempest",
+ 'node_name': self.master_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " state.sls runtest")},
+ ]
- res = self.run_salt_minion_state()
- LOG.info(json.dumps(res, indent=4))
- time.sleep(20)
-
- res = self.create_networks()
- LOG.info(json.dumps(res, indent=4))
- time.sleep(20)
-
- res = self.create_flavors()
- LOG.info(json.dumps(res, indent=4))
- time.sleep(20)
if dpdk:
- res = self.set_property()
- LOG.info('Update flavor property')
- LOG.info(json.dumps(res, indent=4))
- time.sleep(20)
+ commands.append({
+ 'description': "Configure flavor for DPDK",
+ 'node_name': self.control_name,
+ 'cmd': ("set -ex;" +
+ salt_call_cmd + " cmd.run "
+ " '. /root/keystonercv3;"
+ " openstack flavor set m1.tiny_test"
+ " --property hw:mem_page_size=small'")},
+ )
- res = self.create_cirros()
- LOG.info(json.dumps(res, indent=4))
- time.sleep(20)
-
- res = self.generate_config()
- LOG.info(json.dumps(res, indent=4))
- time.sleep(20)
+ self.__salt_api.execute_commands(commands=commands,
+ label="Prepare for Tempest")
def run_tempest(self, timeout=600):
- tgt = "{}*".format(self.target)
- params = {
- "name": self.container_name,
- "image": "{}:{}".format(self.image_name, self.image_version),
- "environment": {
- "ARGS": "-r {tempest_pattern} -w "
- "{tempest_threads} ".format(
- tempest_pattern=self.tempest_pattern,
- tempest_threads=self.tempest_threads) # noqa
- },
- "binds": [
- "{cfg_dir}/tempest.conf:/etc/tempest/tempest.conf".format(cfg_dir=TEMPEST_CFG_DIR), # noqa
- "/tmp/:/tmp/",
- "{cfg_dir}:/root/tempest".format(cfg_dir=TEMPEST_CFG_DIR),
- "/etc/ssl/certs/:/etc/ssl/certs/"
- ],
- "auto_remove": False,
- "cmd": self.run_cmd
- }
+ tgt = self.target_name
+ image_nameversion = "{}:{}".format(self.image_name, self.image_version)
- res = self.salt_api.local(tgt, 'dockerng.pull', "{}:{}".format(
- self.image_name, self.image_version))
- LOG.info("Tempest image has beed pulled- \n{}".format(
- json.dumps(res, indent=4)))
+ docker_args = (
+ " --name {container_name} "
+ " -e ARGS=\"-r {tempest_pattern} -w {tempest_threads}\""
+ " -v {cfg_dir}/tempest.conf:/etc/tempest/tempest.conf"
+ " -v /tmp/:/tmp/"
+ " -v {cfg_dir}:/root/tempest"
+ " -v /etc/ssl/certs/:/etc/ssl/certs/"
+ " --rm"
+ " {image_nameversion} {run_cmd}"
+ .format(
+ container_name=self.container_name,
+ image_nameversion=image_nameversion,
+ tempest_pattern=self.tempest_pattern,
+ tempest_threads=self.tempest_threads,
+ cfg_dir=TEMPEST_CFG_DIR,
+ run_cmd=self.run_cmd,
+ ))
- res = self.salt_api.local(tgt, 'dockerng.create', kwargs=params)
- LOG.info("Tempest container has been created - \n{}".format(
- json.dumps(res, indent=4)))
+ commands = [
+ {
+ 'description': "Run Tempest tests {0}".format(
+ image_nameversion),
+ 'node_name': self.target_name,
+ 'cmd': ("set -ex;" +
+ " docker rm --force {container_name} || true;"
+ " docker run {docker_args}"
+ .format(container_name=self.container_name,
+ docker_args=docker_args)),
+ 'timeout': timeout},
+ ]
- res = self.salt_api.local(tgt, 'dockerng.start', self.container_name)
- LOG.info("Tempest container has been started - \n{}".format(
- json.dumps(res, indent=4)))
-
- def wait_status(s):
- inspect_res = self.salt_api.local(tgt,
- 'dockerng.inspect',
- self.container_name)
- if 'return' in inspect_res:
- inspect = inspect_res['return']
- inspect = inspect[0]
- inspect = next(inspect.iteritems())[1]
- status = inspect['State']['Status']
-
- return status.lower() == s.lower()
-
- return False
-
- helpers.wait(lambda: wait_status('exited'),
- timeout=timeout,
- timeout_msg=('Tempest run didnt finished '
- 'in {}'.format(timeout)))
+ self.__salt_api.execute_commands(commands=commands,
+ label="Run Tempest tests")
inspect_res = self.salt_api.local(tgt,
'dockerng.inspect',
self.container_name)
inspect = inspect_res['return'][0]
inspect = next(inspect.iteritems())[1]
- if inspect['State']['ExitCode'] != 0:
- LOG.error("Tempest running failed")
- LOG.info("Tempest tests have been finished - \n{}".format(
- json.dumps(res, indent=4)))
-
logs_res = self.salt_api.local(tgt,
'dockerng.logs',
self.container_name)
logs = logs_res['return'][0]
logs = next(logs.iteritems())[1]
- LOG.info("Tempest result - \n{}".format(
- logs.encode('ascii', 'ignore')))
res = self.salt_api.local(tgt, 'dockerng.rm', self.container_name)
LOG.info("Tempest container was removed".format(