Merge "Set keepalive interval for paramiko SSHClient"
diff --git a/tcp_tests/helpers/utils.py b/tcp_tests/helpers/utils.py
index 46bf9c8..f2311d4 100644
--- a/tcp_tests/helpers/utils.py
+++ b/tcp_tests/helpers/utils.py
@@ -281,8 +281,13 @@
return {node.tag: loader.construct_scalar(node)}
yaml.add_multi_constructor("!", multi_constructor)
- with self.__get_file() as file_obj:
- self.__documents = [x for x in yaml.load_all(file_obj)]
+ with self.__get_file(mode="a+") as file_obj:
+ file_obj.seek(0)
+ self.__documents = [x for x in yaml.load_all(file_obj)] or [{}, ]
+ # try:
+ # self.__documents = [x for x in yaml.load_all(file_obj)]
+ # except IOError:
+ # self.__documents[self.__document_id] = {}
return self.__documents[self.__document_id]
def write_content(self, content=None):
diff --git a/tcp_tests/managers/runtestmanager.py b/tcp_tests/managers/runtestmanager.py
new file mode 100644
index 0000000..9ed4a9c
--- /dev/null
+++ b/tcp_tests/managers/runtestmanager.py
@@ -0,0 +1,301 @@
+# Copyright 2018 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import json
+
+from devops.helpers import helpers
+
+from tcp_tests import logger
+from tcp_tests import settings
+
+
+LOG = logger.logger
+
+TEMPEST_CFG_DIR = '/tmp/test'
+
+CONFIG = {
+ 'classes': ['service.runtest.tempest'],
+ 'parameters': {
+ '_param': {
+ 'runtest_tempest_cfg_dir': TEMPEST_CFG_DIR,
+ 'runtest_tempest_cfg_name': 'tempest.conf',
+ 'runtest_tempest_public_net': 'net04_ext',
+ 'tempest_test_target': 'gtw01*'
+ },
+ 'neutron': {
+ 'client': {
+ 'enabled': True
+ }
+ },
+ 'runtest': {
+ 'enabled': True,
+ 'keystonerc_node': 'ctl01*',
+ 'tempest': {
+ 'enabled': True,
+ 'cfg_dir': '${_param:runtest_tempest_cfg_dir}',
+ 'cfg_name': '${_param:runtest_tempest_cfg_name}',
+ 'DEFAULT': {
+ 'log_file': 'tempest.log'
+ },
+ 'compute': {
+ 'build_timeout': 600,
+ 'max_microversion': 2.53,
+ 'min_compute_nodes': 2,
+ 'min_microversion': 2.1,
+ 'volume_device_name': 'vdc'
+ },
+ 'convert_to_uuid': {
+ 'network': {
+ 'public_network_id':
+ '${_param:runtest_tempest_public_net}'
+ }
+ },
+ 'dns_feature_enabled': {
+ 'api_admin': False,
+ 'api_v1': False,
+ 'api_v2': True,
+ 'api_v2_quotas': True,
+ 'api_v2_root_recordsets': True,
+ 'bug_1573141_fixed': True
+ },
+ 'heat_plugin': {
+ 'floating_network_name':
+ '${_param:runtest_tempest_public_net}'
+ },
+ 'network': {
+ 'floating_network_name':
+ '${_param:runtest_tempest_public_net}'
+ },
+ 'share': {
+ 'backend_names': 'lvm',
+ 'capability_create_share_from_snapshot_support': True,
+ 'capability_snapshot_support': True,
+ 'default_share_type_name': 'default',
+ 'enable_ip_rules_for_protocols': 'nfs',
+ 'enable_user_rules_for_protocols': 'cifs',
+ 'max_api_microversion': 2.4,
+ 'min_api_microversion': 2.0,
+ 'run_driver_assisted_migration_tests': False,
+ 'run_host_assisted_migration_tests': True,
+ 'run_manage_unmanage_snapshot_tests': False,
+ 'run_manage_unmanage_tests': False,
+ 'run_migration_with_preserve_snapshots_tests': False,
+ 'run_mount_snapshot_tests': True,
+ 'run_quota_tests': True,
+ 'run_replication_tests': False,
+ 'run_revert_to_snapshot_tests': True,
+ 'run_share_group_tests': False,
+ 'run_shrink_tests': False,
+ 'run_snapshot_tests': True,
+ 'share_creation_retry_number': 2,
+ 'suppress_errors_in_cleanup': True
+ }}}}}
+
+
+class RuntestManager(object):
+ """Helper manager for execution tempest via runtest-formula"""
+
+ image_name = settings.TEMPEST_IMAGE
+ image_version = settings.TEMPEST_IMAGE_VERSION
+ container_name = 'run-tempest-ci'
+ master_host = "cfg01"
+ master_tgt = "{}*".format(master_host)
+ class_name = "runtest"
+ run_cmd = '/bin/bash -c "run-tempest"'
+
+ def __init__(self, underlay, salt_api, cluster_name, domain_name,
+ tempest_threads, tempest_exclude_test_args,
+ tempest_pattern=settings.TEMPEST_PATTERN,
+ run_cmd=None, target='gtw01'):
+ self.underlay = underlay
+ self.__salt_api = salt_api
+ self.target = target
+ self.cluster_name = cluster_name
+ self.domain_name = domain_name
+ self.tempest_threads = tempest_threads
+ self.tempest_exclude_test_args = tempest_exclude_test_args
+ self.tempest_pattern = tempest_pattern
+ self.run_cmd = run_cmd or self.run_cmd
+
+ @property
+ def salt_api(self):
+ return self.__salt_api
+
+ def install_python_lib(self):
+ return self.salt_api.local(
+ "{}*".format(self.target),
+ 'pip.install', 'docker'), None
+
+ def install_formula(self):
+ return self.salt_api.local(
+ self.master_tgt,
+ 'pkg.install', 'salt-formula-runtest'), None
+
+ def create_networks(self):
+ return self.salt_api.enforce_state(self.master_tgt, 'neutron.client')
+
+ def create_flavors(self):
+ return self.salt_api.enforce_state(self.master_tgt, 'nova.client')
+
+ def create_cirros(self):
+ return self.salt_api.enforce_state(self.master_tgt, 'glance.client')
+
+ def generate_config(self):
+ return self.salt_api.enforce_state(self.master_tgt, 'runtest')
+
+ def fetch_arficats(self, username=None):
+ target_name = next(node_name for node_name
+ in self.underlay.node_names() if
+ self.target in node_name)
+ with self.underlay.remote(node_name=target_name, username=None) as tgt:
+ tgt.download(
+ destination="{cfg_dir}/report_*.xml".format(cfg_dir=TEMPEST_CFG_DIR), # noqa
+ target="{}".format(os.environ.get("PWD")))
+
+ def store_runtest_model(self, config=CONFIG):
+ master_name = next(node_name for node_name
+ in self.underlay.node_names() if
+ self.master_host in node_name)
+ with self.underlay.yaml_editor(
+ file_path="/srv/salt/reclass/classes/cluster/"
+ "{cluster_name}/infra/"
+ "{class_name}.yml".format(
+ cluster_name=self.cluster_name,
+ class_name=self.class_name),
+ node_name=master_name) as editor:
+ editor.content = config
+
+ with self.underlay.yaml_editor(
+ file_path="/srv/salt/reclass/nodes/_generated/"
+ "cfg01.{domain_name}.yml".format(
+ domain_name=self.domain_name),
+ node_name=master_name) as editor:
+ editor.content['classes'].append(
+ 'cluster.{cluster_name}.infra.{class_name}'.format(
+ cluster_name=self.cluster_name,
+ class_name=self.class_name))
+
+ self.salt_api.local('*', 'saltutil.refresh_pillar')
+ self.salt_api.local('*', 'saltutil.sync_all')
+
+ def save_runtime_logs(self, logs=None, inspect=None):
+ if logs:
+ with open("{path}/{target}_tempest_run.log".format(
+ path=settings.LOGS_DIR, target=self.target), 'w') as f:
+ LOG.info("Save tempest console log")
+ container_log = logs
+ f.write(container_log)
+
+ if inspect:
+ with open("{path}/{target}_tempest_container_info.json".format(
+ path=settings.LOGS_DIR, target=self.target), 'w') as f:
+ LOG.info("Save tempest containes inspect data")
+
+ container_inspect = json.dumps(inspect,
+ indent=4, sort_keys=True)
+ f.write(container_inspect)
+
+ def prepare(self):
+ self.store_runtest_model()
+ res = self.install_formula()
+ LOG.info(json.dumps(res, indent=4))
+ res = self.install_python_lib()
+ LOG.info(json.dumps(res, indent=4))
+ res = self.create_networks()
+ LOG.info(json.dumps(res, indent=4))
+ res = self.create_flavors()
+ LOG.info(json.dumps(res, indent=4))
+ res = self.create_cirros()
+ LOG.info(json.dumps(res, indent=4))
+ res = self.generate_config()
+ LOG.info(json.dumps(res, indent=4))
+
+ def run_tempest(self, timeout=600):
+ tgt = "{}*".format(self.target)
+ params = {
+ "name": self.container_name,
+ "image": self.image_name,
+ "environment": {
+ "ARGS": "-r {tempest_pattern} -w "
+ "{tempest_threads} "
+ "{tempest_exclude_test_args}".format(
+ tempest_pattern=self.tempest_pattern,
+ tempest_threads=self.tempest_threads,
+ tempest_exclude_test_args=self.tempest_exclude_test_args) # noqa
+ },
+ "binds": [
+ "{cfg_dir}/tempest.conf:/etc/tempest/tempest.conf".format(cfg_dir=TEMPEST_CFG_DIR), # noqa
+ "/tmp/:/tmp/",
+ "{cfg_dir}:/root/tempest".format(cfg_dir=TEMPEST_CFG_DIR),
+ "/etc/ssl/certs/:/etc/ssl/certs/"
+ ],
+ "auto_remove": False,
+ "cmd": self.run_cmd
+ }
+
+ res = self.salt_api.local(tgt, 'dockerng.pull', self.image_name)
+ LOG.info("Tempest image has beed pulled- \n{}".format(
+ json.dumps(res, indent=4)))
+
+ res = self.salt_api.local(tgt, 'dockerng.create', kwargs=params)
+ LOG.info("Tempest container has been created - \n{}".format(
+ json.dumps(res, indent=4)))
+
+ res = self.salt_api.local(tgt, 'dockerng.start', self.container_name)
+ LOG.info("Tempest container has been started - \n{}".format(
+ json.dumps(res, indent=4)))
+
+ def wait_status(s):
+ inspect_res = self.salt_api.local(tgt,
+ 'dockerng.inspect',
+ self.container_name)
+ if 'return' in inspect_res:
+ inspect = inspect_res['return']
+ inspect = inspect[0]
+ inspect = next(inspect.iteritems())[1]
+ status = inspect['State']['Status']
+
+ return status.lower() == s.lower()
+
+ return False
+
+ helpers.wait(lambda: wait_status('exited'),
+ timeout=timeout,
+ timeout_msg=('Tempest run didnt finished '
+ 'in {}'.format(timeout)))
+
+ inspect_res = self.salt_api.local(tgt,
+ 'dockerng.inspect',
+ self.container_name)
+ inspect = inspect_res['return'][0]
+ inspect = next(inspect.iteritems())[1]
+ if inspect['State']['ExitCode'] != 0:
+ LOG.error("Tempest running failed")
+ LOG.info("Tempest tests have been finished - \n{}".format(
+ json.dumps(res, indent=4)))
+
+ logs_res = self.salt_api.local(tgt,
+ 'dockerng.logs',
+ self.container_name)
+ logs = logs_res['return'][0]
+ logs = next(logs.iteritems())[1]
+ LOG.info("Tempest result - \n{}".format(logs))
+
+ res = self.salt_api.local(tgt, 'dockerng.rm', self.container_name)
+ LOG.info("Tempest container was removed".format(
+ json.dumps(res, indent=4)))
+
+ return {'inspect': inspect,
+ 'logs': logs}
diff --git a/tcp_tests/settings.py b/tcp_tests/settings.py
index 33ab68c..6fc6504 100644
--- a/tcp_tests/settings.py
+++ b/tcp_tests/settings.py
@@ -69,3 +69,10 @@
PATTERN = os.environ.get('PATTERN', None)
RUN_TEMPEST = get_var_as_bool('RUN_TEMPEST', False)
RUN_SL_TESTS = get_var_as_bool('RUN_SL_TESTS', False)
+
+TEMPEST_IMAGE = os.environ.get(
+ 'TEMPEST_IMAGE',
+ 'docker-prod-virtual.docker.mirantis.net/mirantis/cicd/ci-tempest') # noqa
+TEMPEST_IMAGE_VERSION = os.environ.get('TEMPEST_IMAGE_VERSION', 'latest')
+TEMPEST_PATTERN = os.environ.get('TEMPEST_PATTERN', 'tempest')
+TEMPEST_TIMEOUT = int(os.environ.get('TEMPEST_TIMEOUT', 1800))
diff --git a/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml b/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml
index 4748f8a..38a8dcd 100644
--- a/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml
@@ -68,6 +68,24 @@
# Install slv2 infra
#Launch containers
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
- description: launch prometheus containers
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml b/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml
index 18521fd..3435917 100644
--- a/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml
@@ -68,6 +68,24 @@
# Install slv2 infra
#Launch containers
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
- description: launch prometheus containers
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml b/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml
index 69306b9..f4b05d0 100644
--- a/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml
@@ -66,6 +66,24 @@
skip_fail: false
# Install slv2 infra
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
- description: Install telegraf
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@telegraf:agent or I@telegraf:remote_agent' state.sls telegraf
node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml
index 87eb34c..37082f6 100644
--- a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml
@@ -67,6 +67,24 @@
# Install slv2 infra
#Launch containers
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
- description: launch prometheus containers
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml
index aaaa296..4045fe8 100644
--- a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml
@@ -67,6 +67,24 @@
# Install slv2 infra
#Launch containers
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
- description: launch prometheus containers
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
index bb65f49..5a62809 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
@@ -67,10 +67,22 @@
# Install slv2 infra
# Install MongoDB for alerta
-- description: Install MongoDB
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 10}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
skip_fail: false
- description: Install telegraf
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
index 9f02e20..d8ad417 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
@@ -67,6 +67,24 @@
# Install slv2 infra
#Launch containers
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
- description: launch prometheus containers
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
index e7c1175..bf71b59 100644
--- a/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
@@ -69,10 +69,22 @@
# Install slv2 infra
# Install MongoDB for alerta
-- description: Install MongoDB
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 10}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
skip_fail: false
#Launch containers
diff --git a/tcp_tests/templates/shared-salt.yaml b/tcp_tests/templates/shared-salt.yaml
index dedc832..b042a01 100644
--- a/tcp_tests/templates/shared-salt.yaml
+++ b/tcp_tests/templates/shared-salt.yaml
@@ -8,7 +8,9 @@
{% set SALT_MODELS_SYSTEM_REPOSITORY = os_env('SALT_MODELS_SYSTEM_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/reclass-system') %}
{% set SALT_MODELS_SYSTEM_COMMIT = os_env('SALT_MODELS_SYSTEM_COMMIT','') %}
{% set SALT_MODELS_SYSTEM_REF_CHANGE = os_env('SALT_MODELS_SYSTEM_REF_CHANGE','') %}
+{% set SALT_MODELS_SYSTEM_TAG = os_env('SALT_MODELS_SYSTEM_TAG','') %}
{% set COOKIECUTTER_REF_CHANGE = os_env('COOKIECUTTER_REF_CHANGE','') %}
+{% set COOKIECUTTER_TAG = os_env('COOKIECUTTER_TAG','') %}
{% set COOKIECUTTER_TEMPLATE_COMMIT = os_env('COOKIECUTTER_TEMPLATE_COMMIT','') %}
{% set ENVIRONMENT_TEMPLATE_REF_CHANGE = os_env('ENVIRONMENT_TEMPLATE_REF_CHANGE','') %}
@@ -239,6 +241,12 @@
pushd classes/system/;
git checkout {{ SALT_MODELS_SYSTEM_COMMIT }};
popd;
+ {%- if SALT_MODELS_SYSTEM_TAG != '' %}
+ pushd classes/system/;
+ git fetch --all --tags --prune
+ git checkout tags/{{ SALT_MODELS_SYSTEM_TAG }};
+ popd;
+ {%- endif %}
{%- elif SALT_MODELS_SYSTEM_REF_CHANGE != '' %}
pushd classes/system/ && \
{%- for item in SALT_MODELS_SYSTEM_REF_CHANGE.split(" ") %}
@@ -340,6 +348,13 @@
popd
{%- endif %}
+ {%- if COOKIECUTTER_TAG != '' %}
+ pushd /tmp/cookiecutter-templates
+ git fetch --all --tags --prune
+ git checkout tags/{{ COOKIECUTTER_TAG }}
+ popd
+ {%- endif %}
+
mkdir -p /srv/salt/reclass/classes/cluster/
mkdir -p /srv/salt/reclass/classes/system/
mkdir -p /srv/salt/reclass/classes/service/
diff --git a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
index 0f3abf6..80e29a1 100644
--- a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
@@ -11,7 +11,7 @@
{{ SHARED.MACRO_CLONE_RECLASS_MODELS() }}
-{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "fluentd" "runtest"') }}
+{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "fluentd" "runtest" "backupninja"') }}
{{ SHARED.MACRO_INSTALL_SALT_MINIONS() }}
diff --git a/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml b/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml
index 3f15080..187ea59 100644
--- a/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml
@@ -357,4 +357,3 @@
retry: {count: 1, delay: 5}
skip_fail: false
-{{ SHARED.RUN_NEW_TEMPEST() }}
diff --git a/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml b/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml
index 9084d80..a516c29 100644
--- a/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml
@@ -5,6 +5,7 @@
{% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
# Other salt model repository parameters see in shared-salt.yaml
{% set OVERRIDES = os_env('OVERRIDES', 'override_example: true') %}
+{% set OVERRIDES_FILENAME = os_env('OVERRIDES_FILENAME', '/srv/salt/reclass/classes/cluster/overrides.yml') %}
{% import 'shared-salt.yaml' as SHARED with context %}
@@ -19,16 +20,26 @@
{{ SHARED.MACRO_RUN_SALT_MASTER_UNDERLAY_STATES() }}
{%- if OVERRIDES != '' %}
+- description: Clear {{OVERRIDES_FILENAME}}
+ cmd: touch {{OVERRIDES_FILENAME}}
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
+
{%- for param in OVERRIDES.splitlines() %}
-{%- set key, value = param.replace(' ','').split(':') %}
- description: Override cluster parameters
cmd: |
- salt-call reclass.cluster_meta_set name='{{ key }}' value='{{ value }}' file_name=/srv/salt/reclass/classes/cluster/overrides.yml
- salt '*' saltutil.refresh_pillar
+ echo -e "{{ param }}" >> {{OVERRIDES_FILENAME}}
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
{%- endfor %}
+
+- description: Refresh pillar
+ cmd: salt '*' saltutil.refresh_pillar
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
{%- endif %}
{{ SHARED.MACRO_GENERATE_INVENTORY() }}
diff --git a/tcp_tests/templates/virtual-mcp-trusty/salt.yaml b/tcp_tests/templates/virtual-mcp-trusty/salt.yaml
index db9ba1c..d042292 100644
--- a/tcp_tests/templates/virtual-mcp-trusty/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-trusty/salt.yaml
@@ -4,7 +4,8 @@
{% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
# Other salt model repository parameters see in shared-salt.yaml
-{% set OVERRIDES = os_env('OVERRIDES', 'override_example: true') %}
+{% set OVERRIDES = os_env('OVERRIDES', '') %}
+{% set OVERRIDES_FILENAME = os_env('OVERRIDES_FILENAME', '/srv/salt/reclass/classes/cluster/overrides.yml') %}
{% import 'shared-salt.yaml' as SHARED with context %}
@@ -19,16 +20,26 @@
{{ SHARED.MACRO_RUN_SALT_MASTER_UNDERLAY_STATES() }}
{%- if OVERRIDES != '' %}
+- description: Clear {{OVERRIDES_FILENAME}}
+ cmd: touch {{OVERRIDES_FILENAME}}
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
+
{%- for param in OVERRIDES.splitlines() %}
-{%- set key, value = param.replace(' ','').split(':') %}
- description: Override cluster parameters
cmd: |
- salt-call reclass.cluster_meta_set name='{{ key }}' value='{{ value }}' file_name=/srv/salt/reclass/classes/cluster/overrides.yml
- salt '*' saltutil.refresh_pillar
+ echo -e "{{ param }}" >> {{OVERRIDES_FILENAME}}
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
{%- endfor %}
+
+- description: Refresh pillar
+ cmd: salt '*' saltutil.refresh_pillar
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
{%- endif %}
{{ SHARED.MACRO_GENERATE_INVENTORY() }}
diff --git a/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml b/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
index 5f9f3e6..52bf890 100644
--- a/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
@@ -68,10 +68,22 @@
# Install slv2 infra
# Install MongoDB for alerta
-- description: Install MongoDB
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 10}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
skip_fail: false
#Launch containers
diff --git a/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml b/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
index 23df071..766a7c1 100644
--- a/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
@@ -68,10 +68,22 @@
# Install slv2 infra
# Install MongoDB for alerta
-- description: Install MongoDB
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+ fi
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 10}
+ retry: {count: 1, delay: 10}
+ skip_fail: false
+
+- description: Configure Alerta if it is exists
+ cmd: |
+ if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+ salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+ fi
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
skip_fail: false
#Launch containers
diff --git a/tcp_tests/tests/system/test_install_mcp_ovs_pike.py b/tcp_tests/tests/system/test_install_mcp_ovs_pike.py
index 53dc1c1..c5138f0 100644
--- a/tcp_tests/tests/system/test_install_mcp_ovs_pike.py
+++ b/tcp_tests/tests/system/test_install_mcp_ovs_pike.py
@@ -14,6 +14,8 @@
import pytest
+from tcp_tests.managers.runtestmanager import RuntestManager
+
from tcp_tests import logger
from tcp_tests import settings
@@ -29,7 +31,9 @@
@pytest.mark.pike_ovs
def test_mcp_pike_ovs_install(self, underlay,
openstack_deployed,
- openstack_actions):
+ openstack_actions,
+ salt_actions,
+ config):
"""Test for deploying an mcp environment and check it
Scenario:
1. Prepare salt on hosts
@@ -39,12 +43,30 @@
"""
openstack_actions._salt.local(
- tgt='*', fun='cmd.run',
- args='service ntp stop; ntpd -gq; service ntp start')
+ tgt='*', fun='cmd.run',
+ args='service ntp stop; ntpd -gq; service ntp start')
if settings.RUN_TEMPEST:
- openstack_actions.run_tempest(pattern=settings.PATTERN)
- openstack_actions.download_tempest_report()
+ tempest_threads = 2
+ tempest_exclude_test_args = ''
+ tempest_pattern = settings.TEMPEST_PATTERN
+ cluster_name = settings.LAB_CONFIG_NAME
+ tempest_timeout = settings.TEMPEST_TIMEOUT
+ domain_name = "{}.local".format(cluster_name)
+ target = 'gtw01'
+ runtest = RuntestManager(
+ underlay, salt_actions,
+ cluster_name=cluster_name,
+ domain_name=domain_name,
+ tempest_threads=tempest_threads,
+ tempest_exclude_test_args=tempest_exclude_test_args,
+ tempest_pattern=tempest_pattern,
+ target=target)
+ runtest.prepare()
+ test_res = runtest.run_tempest(tempest_timeout)
+ runtest.fetch_arficats(username='root')
+ runtest.save_runtime_logs(**test_res)
+
LOG.info("*************** DONE **************")
@pytest.mark.grab_versions
@@ -177,8 +199,8 @@
"""
openstack_actions._salt.local(
- tgt='*', fun='cmd.run',
- args='service ntp stop; ntpd -gq; service ntp start')
+ tgt='*', fun='cmd.run',
+ args='service ntp stop; ntpd -gq; service ntp start')
if settings.RUN_TEMPEST:
openstack_actions.run_tempest(pattern=settings.PATTERN)
@@ -246,8 +268,8 @@
"""
openstack_actions._salt.local(
- tgt='*', fun='cmd.run',
- args='service ntp stop; ntpd -gq; service ntp start')
+ tgt='*', fun='cmd.run',
+ args='service ntp stop; ntpd -gq; service ntp start')
registry = 'docker-dev-local.docker.mirantis.net/mirantis/networking'
name = 'rally-tempest-net-features:latest'