Merge "Set keepalive interval for paramiko SSHClient"
diff --git a/tcp_tests/helpers/utils.py b/tcp_tests/helpers/utils.py
index 46bf9c8..f2311d4 100644
--- a/tcp_tests/helpers/utils.py
+++ b/tcp_tests/helpers/utils.py
@@ -281,8 +281,13 @@
                 return {node.tag: loader.construct_scalar(node)}
 
         yaml.add_multi_constructor("!", multi_constructor)
-        with self.__get_file() as file_obj:
-            self.__documents = [x for x in yaml.load_all(file_obj)]
+        with self.__get_file(mode="a+") as file_obj:
+            file_obj.seek(0)
+            self.__documents = [x for x in yaml.load_all(file_obj)] or [{}, ]
+            # try:
+            #     self.__documents = [x for x in yaml.load_all(file_obj)]
+            # except IOError:
+            #     self.__documents[self.__document_id] = {}
             return self.__documents[self.__document_id]
 
     def write_content(self, content=None):
diff --git a/tcp_tests/managers/runtestmanager.py b/tcp_tests/managers/runtestmanager.py
new file mode 100644
index 0000000..9ed4a9c
--- /dev/null
+++ b/tcp_tests/managers/runtestmanager.py
@@ -0,0 +1,301 @@
+#    Copyright 2018 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import json
+
+from devops.helpers import helpers
+
+from tcp_tests import logger
+from tcp_tests import settings
+
+
+LOG = logger.logger
+
+TEMPEST_CFG_DIR = '/tmp/test'
+
+CONFIG = {
+    'classes': ['service.runtest.tempest'],
+    'parameters': {
+        '_param': {
+            'runtest_tempest_cfg_dir': TEMPEST_CFG_DIR,
+            'runtest_tempest_cfg_name': 'tempest.conf',
+            'runtest_tempest_public_net': 'net04_ext',
+            'tempest_test_target': 'gtw01*'
+        },
+        'neutron': {
+            'client': {
+                'enabled': True
+            }
+        },
+        'runtest': {
+            'enabled': True,
+            'keystonerc_node': 'ctl01*',
+            'tempest': {
+                'enabled': True,
+                'cfg_dir': '${_param:runtest_tempest_cfg_dir}',
+                'cfg_name': '${_param:runtest_tempest_cfg_name}',
+                'DEFAULT': {
+                    'log_file': 'tempest.log'
+                },
+                'compute': {
+                    'build_timeout': 600,
+                    'max_microversion': 2.53,
+                    'min_compute_nodes': 2,
+                    'min_microversion': 2.1,
+                    'volume_device_name': 'vdc'
+                },
+                'convert_to_uuid': {
+                    'network': {
+                        'public_network_id':
+                        '${_param:runtest_tempest_public_net}'
+                    }
+                },
+                'dns_feature_enabled': {
+                    'api_admin': False,
+                    'api_v1': False,
+                    'api_v2': True,
+                    'api_v2_quotas': True,
+                    'api_v2_root_recordsets': True,
+                    'bug_1573141_fixed': True
+                },
+                'heat_plugin': {
+                    'floating_network_name':
+                    '${_param:runtest_tempest_public_net}'
+                },
+                'network': {
+                    'floating_network_name':
+                    '${_param:runtest_tempest_public_net}'
+                },
+                'share': {
+                    'backend_names': 'lvm',
+                    'capability_create_share_from_snapshot_support': True,
+                    'capability_snapshot_support': True,
+                    'default_share_type_name': 'default',
+                    'enable_ip_rules_for_protocols': 'nfs',
+                    'enable_user_rules_for_protocols': 'cifs',
+                    'max_api_microversion': 2.4,
+                    'min_api_microversion': 2.0,
+                    'run_driver_assisted_migration_tests': False,
+                    'run_host_assisted_migration_tests': True,
+                    'run_manage_unmanage_snapshot_tests': False,
+                    'run_manage_unmanage_tests': False,
+                    'run_migration_with_preserve_snapshots_tests': False,
+                    'run_mount_snapshot_tests': True,
+                    'run_quota_tests': True,
+                    'run_replication_tests': False,
+                    'run_revert_to_snapshot_tests': True,
+                    'run_share_group_tests': False,
+                    'run_shrink_tests': False,
+                    'run_snapshot_tests': True,
+                    'share_creation_retry_number': 2,
+                    'suppress_errors_in_cleanup': True
+                }}}}}
+
+
+class RuntestManager(object):
+    """Helper manager for execution tempest via runtest-formula"""
+
+    image_name = settings.TEMPEST_IMAGE
+    image_version = settings.TEMPEST_IMAGE_VERSION
+    container_name = 'run-tempest-ci'
+    master_host = "cfg01"
+    master_tgt = "{}*".format(master_host)
+    class_name = "runtest"
+    run_cmd = '/bin/bash -c "run-tempest"'
+
+    def __init__(self, underlay, salt_api, cluster_name, domain_name,
+                 tempest_threads, tempest_exclude_test_args,
+                 tempest_pattern=settings.TEMPEST_PATTERN,
+                 run_cmd=None, target='gtw01'):
+        self.underlay = underlay
+        self.__salt_api = salt_api
+        self.target = target
+        self.cluster_name = cluster_name
+        self.domain_name = domain_name
+        self.tempest_threads = tempest_threads
+        self.tempest_exclude_test_args = tempest_exclude_test_args
+        self.tempest_pattern = tempest_pattern
+        self.run_cmd = run_cmd or self.run_cmd
+
+    @property
+    def salt_api(self):
+        return self.__salt_api
+
+    def install_python_lib(self):
+        return self.salt_api.local(
+            "{}*".format(self.target),
+            'pip.install', 'docker'), None
+
+    def install_formula(self):
+        return self.salt_api.local(
+            self.master_tgt,
+            'pkg.install', 'salt-formula-runtest'), None
+
+    def create_networks(self):
+        return self.salt_api.enforce_state(self.master_tgt, 'neutron.client')
+
+    def create_flavors(self):
+        return self.salt_api.enforce_state(self.master_tgt, 'nova.client')
+
+    def create_cirros(self):
+        return self.salt_api.enforce_state(self.master_tgt, 'glance.client')
+
+    def generate_config(self):
+        return self.salt_api.enforce_state(self.master_tgt, 'runtest')
+
+    def fetch_arficats(self, username=None):
+        target_name = next(node_name for node_name
+                           in self.underlay.node_names() if
+                           self.target in node_name)
+        with self.underlay.remote(node_name=target_name, username=None) as tgt:
+            tgt.download(
+                destination="{cfg_dir}/report_*.xml".format(cfg_dir=TEMPEST_CFG_DIR),  # noqa
+                target="{}".format(os.environ.get("PWD")))
+
+    def store_runtest_model(self, config=CONFIG):
+        master_name = next(node_name for node_name
+                           in self.underlay.node_names() if
+                           self.master_host in node_name)
+        with self.underlay.yaml_editor(
+                file_path="/srv/salt/reclass/classes/cluster/"
+                          "{cluster_name}/infra/"
+                          "{class_name}.yml".format(
+                              cluster_name=self.cluster_name,
+                              class_name=self.class_name),
+                node_name=master_name) as editor:
+            editor.content = config
+
+        with self.underlay.yaml_editor(
+                file_path="/srv/salt/reclass/nodes/_generated/"
+                          "cfg01.{domain_name}.yml".format(
+                              domain_name=self.domain_name),
+                node_name=master_name) as editor:
+            editor.content['classes'].append(
+                'cluster.{cluster_name}.infra.{class_name}'.format(
+                    cluster_name=self.cluster_name,
+                    class_name=self.class_name))
+
+        self.salt_api.local('*', 'saltutil.refresh_pillar')
+        self.salt_api.local('*', 'saltutil.sync_all')
+
+    def save_runtime_logs(self, logs=None, inspect=None):
+        if logs:
+            with open("{path}/{target}_tempest_run.log".format(
+                    path=settings.LOGS_DIR, target=self.target), 'w') as f:
+                LOG.info("Save tempest console log")
+                container_log = logs
+                f.write(container_log)
+
+        if inspect:
+            with open("{path}/{target}_tempest_container_info.json".format(
+                    path=settings.LOGS_DIR, target=self.target), 'w') as f:
+                LOG.info("Save tempest containes inspect data")
+
+                container_inspect = json.dumps(inspect,
+                                               indent=4, sort_keys=True)
+                f.write(container_inspect)
+
+    def prepare(self):
+        self.store_runtest_model()
+        res = self.install_formula()
+        LOG.info(json.dumps(res, indent=4))
+        res = self.install_python_lib()
+        LOG.info(json.dumps(res, indent=4))
+        res = self.create_networks()
+        LOG.info(json.dumps(res, indent=4))
+        res = self.create_flavors()
+        LOG.info(json.dumps(res, indent=4))
+        res = self.create_cirros()
+        LOG.info(json.dumps(res, indent=4))
+        res = self.generate_config()
+        LOG.info(json.dumps(res, indent=4))
+
+    def run_tempest(self, timeout=600):
+        tgt = "{}*".format(self.target)
+        params = {
+            "name": self.container_name,
+            "image": self.image_name,
+            "environment": {
+                "ARGS": "-r {tempest_pattern} -w "
+                        "{tempest_threads} "
+                        "{tempest_exclude_test_args}".format(
+                            tempest_pattern=self.tempest_pattern,
+                            tempest_threads=self.tempest_threads,
+                            tempest_exclude_test_args=self.tempest_exclude_test_args)  # noqa
+            },
+            "binds": [
+                "{cfg_dir}/tempest.conf:/etc/tempest/tempest.conf".format(cfg_dir=TEMPEST_CFG_DIR),  # noqa
+                "/tmp/:/tmp/",
+                "{cfg_dir}:/root/tempest".format(cfg_dir=TEMPEST_CFG_DIR),
+                "/etc/ssl/certs/:/etc/ssl/certs/"
+            ],
+            "auto_remove": False,
+            "cmd": self.run_cmd
+        }
+
+        res = self.salt_api.local(tgt, 'dockerng.pull', self.image_name)
+        LOG.info("Tempest image has beed pulled- \n{}".format(
+            json.dumps(res, indent=4)))
+
+        res = self.salt_api.local(tgt, 'dockerng.create', kwargs=params)
+        LOG.info("Tempest container has been created - \n{}".format(
+            json.dumps(res, indent=4)))
+
+        res = self.salt_api.local(tgt, 'dockerng.start', self.container_name)
+        LOG.info("Tempest container has been started - \n{}".format(
+            json.dumps(res, indent=4)))
+
+        def wait_status(s):
+            inspect_res = self.salt_api.local(tgt,
+                                              'dockerng.inspect',
+                                              self.container_name)
+            if 'return' in inspect_res:
+                inspect = inspect_res['return']
+                inspect = inspect[0]
+                inspect = next(inspect.iteritems())[1]
+                status = inspect['State']['Status']
+
+                return status.lower() == s.lower()
+
+            return False
+
+        helpers.wait(lambda: wait_status('exited'),
+                     timeout=timeout,
+                     timeout_msg=('Tempest run didnt finished '
+                                  'in {}'.format(timeout)))
+
+        inspect_res = self.salt_api.local(tgt,
+                                          'dockerng.inspect',
+                                          self.container_name)
+        inspect = inspect_res['return'][0]
+        inspect = next(inspect.iteritems())[1]
+        if inspect['State']['ExitCode'] != 0:
+            LOG.error("Tempest running failed")
+        LOG.info("Tempest tests have been finished - \n{}".format(
+            json.dumps(res, indent=4)))
+
+        logs_res = self.salt_api.local(tgt,
+                                       'dockerng.logs',
+                                       self.container_name)
+        logs = logs_res['return'][0]
+        logs = next(logs.iteritems())[1]
+        LOG.info("Tempest result - \n{}".format(logs))
+
+        res = self.salt_api.local(tgt, 'dockerng.rm', self.container_name)
+        LOG.info("Tempest container was removed".format(
+            json.dumps(res, indent=4)))
+
+        return {'inspect': inspect,
+                'logs': logs}
diff --git a/tcp_tests/settings.py b/tcp_tests/settings.py
index 33ab68c..6fc6504 100644
--- a/tcp_tests/settings.py
+++ b/tcp_tests/settings.py
@@ -69,3 +69,10 @@
 PATTERN = os.environ.get('PATTERN', None)
 RUN_TEMPEST = get_var_as_bool('RUN_TEMPEST', False)
 RUN_SL_TESTS = get_var_as_bool('RUN_SL_TESTS', False)
+
+TEMPEST_IMAGE = os.environ.get(
+    'TEMPEST_IMAGE',
+    'docker-prod-virtual.docker.mirantis.net/mirantis/cicd/ci-tempest')  # noqa
+TEMPEST_IMAGE_VERSION = os.environ.get('TEMPEST_IMAGE_VERSION', 'latest')
+TEMPEST_PATTERN = os.environ.get('TEMPEST_PATTERN', 'tempest')
+TEMPEST_TIMEOUT = int(os.environ.get('TEMPEST_TIMEOUT', 1800))
diff --git a/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml b/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml
index 4748f8a..38a8dcd 100644
--- a/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-contrail-maas/sl.yaml
@@ -68,6 +68,24 @@
 
 # Install slv2 infra
 #Launch containers
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
 - description: launch prometheus containers
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml b/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml
index 18521fd..3435917 100644
--- a/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-contrail-nfv-maas/sl.yaml
@@ -68,6 +68,24 @@
 
 # Install slv2 infra
 #Launch containers
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
 - description: launch prometheus containers
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml b/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml
index 69306b9..f4b05d0 100644
--- a/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-mcp-dvr-vxlan/sl.yaml
@@ -66,6 +66,24 @@
   skip_fail: false
 
 # Install slv2 infra
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
 - description: Install telegraf
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@telegraf:agent or I@telegraf:remote_agent' state.sls telegraf
   node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml
index 87eb34c..37082f6 100644
--- a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail-nfv/sl.yaml
@@ -67,6 +67,24 @@
 
 # Install slv2 infra
 #Launch containers
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
 - description: launch prometheus containers
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml
index aaaa296..4045fe8 100644
--- a/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml
+++ b/tcp_tests/templates/cookied-bm-mcp-ocata-contrail/sl.yaml
@@ -67,6 +67,24 @@
 
 # Install slv2 infra
 #Launch containers
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
 - description: launch prometheus containers
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
index bb65f49..5a62809 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
@@ -67,10 +67,22 @@
 
 # Install slv2 infra
 # Install MongoDB for alerta
-- description: Install MongoDB
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
   skip_fail: false
 
 - description: Install telegraf
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
index 9f02e20..d8ad417 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
@@ -67,6 +67,24 @@
 
 # Install slv2 infra
 #Launch containers
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
 - description: launch prometheus containers
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
diff --git a/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
index e7c1175..bf71b59 100644
--- a/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
@@ -69,10 +69,22 @@
 
 # Install slv2 infra
 # Install MongoDB for alerta
-- description: Install MongoDB
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
   skip_fail: false
 
 #Launch containers
diff --git a/tcp_tests/templates/shared-salt.yaml b/tcp_tests/templates/shared-salt.yaml
index dedc832..b042a01 100644
--- a/tcp_tests/templates/shared-salt.yaml
+++ b/tcp_tests/templates/shared-salt.yaml
@@ -8,7 +8,9 @@
 {% set SALT_MODELS_SYSTEM_REPOSITORY = os_env('SALT_MODELS_SYSTEM_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/reclass-system') %}
 {% set SALT_MODELS_SYSTEM_COMMIT = os_env('SALT_MODELS_SYSTEM_COMMIT','') %}
 {% set SALT_MODELS_SYSTEM_REF_CHANGE = os_env('SALT_MODELS_SYSTEM_REF_CHANGE','') %}
+{% set SALT_MODELS_SYSTEM_TAG = os_env('SALT_MODELS_SYSTEM_TAG','') %}
 {% set COOKIECUTTER_REF_CHANGE = os_env('COOKIECUTTER_REF_CHANGE','') %}
+{% set COOKIECUTTER_TAG = os_env('COOKIECUTTER_TAG','') %}
 {% set COOKIECUTTER_TEMPLATE_COMMIT = os_env('COOKIECUTTER_TEMPLATE_COMMIT','') %}
 {% set ENVIRONMENT_TEMPLATE_REF_CHANGE = os_env('ENVIRONMENT_TEMPLATE_REF_CHANGE','') %}
 
@@ -239,6 +241,12 @@
     pushd classes/system/;
     git checkout {{ SALT_MODELS_SYSTEM_COMMIT }};
     popd;
+    {%- if SALT_MODELS_SYSTEM_TAG != '' %}
+    pushd classes/system/;
+    git fetch --all --tags --prune
+    git checkout tags/{{ SALT_MODELS_SYSTEM_TAG }};
+    popd;
+    {%- endif %}
     {%- elif SALT_MODELS_SYSTEM_REF_CHANGE != '' %}
     pushd classes/system/ && \
     {%- for item in SALT_MODELS_SYSTEM_REF_CHANGE.split(" ") %}
@@ -340,6 +348,13 @@
     popd
     {%- endif %}
 
+    {%- if COOKIECUTTER_TAG != '' %}
+    pushd /tmp/cookiecutter-templates
+    git fetch --all --tags --prune
+    git checkout tags/{{ COOKIECUTTER_TAG }}
+    popd
+    {%- endif %}
+
     mkdir -p /srv/salt/reclass/classes/cluster/
     mkdir -p /srv/salt/reclass/classes/system/
     mkdir -p /srv/salt/reclass/classes/service/
diff --git a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
index 0f3abf6..80e29a1 100644
--- a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
@@ -11,7 +11,7 @@
 
 {{ SHARED.MACRO_CLONE_RECLASS_MODELS() }}
 
-{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "fluentd" "runtest"') }}
+{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "fluentd" "runtest" "backupninja"') }}
 
 {{ SHARED.MACRO_INSTALL_SALT_MINIONS() }}
 
diff --git a/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml b/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml
index 3f15080..187ea59 100644
--- a/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-ovs/openstack.yaml
@@ -357,4 +357,3 @@
   retry: {count: 1, delay: 5}
   skip_fail: false
 
-{{ SHARED.RUN_NEW_TEMPEST() }}
diff --git a/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml b/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml
index 9084d80..a516c29 100644
--- a/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-sl-os/salt.yaml
@@ -5,6 +5,7 @@
 {% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
 # Other salt model repository parameters see in shared-salt.yaml
 {% set OVERRIDES = os_env('OVERRIDES', 'override_example: true') %}
+{% set OVERRIDES_FILENAME = os_env('OVERRIDES_FILENAME', '/srv/salt/reclass/classes/cluster/overrides.yml') %}
 
 {% import 'shared-salt.yaml' as SHARED with context %}
 
@@ -19,16 +20,26 @@
 {{ SHARED.MACRO_RUN_SALT_MASTER_UNDERLAY_STATES() }}
 
 {%- if OVERRIDES != '' %}
+- description: Clear {{OVERRIDES_FILENAME}}
+  cmd: touch {{OVERRIDES_FILENAME}}
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
 {%- for param in OVERRIDES.splitlines() %}
-{%- set key, value = param.replace(' ','').split(':') %}
 - description: Override cluster parameters
   cmd: |
-    salt-call reclass.cluster_meta_set name='{{ key }}' value='{{ value }}' file_name=/srv/salt/reclass/classes/cluster/overrides.yml
-    salt '*' saltutil.refresh_pillar
+    echo -e "{{ param }}" >> {{OVERRIDES_FILENAME}}
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 1}
   skip_fail: false
 {%- endfor %}
+
+- description: Refresh pillar
+  cmd: salt '*' saltutil.refresh_pillar
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
 {%- endif %}
 
 {{ SHARED.MACRO_GENERATE_INVENTORY() }}
diff --git a/tcp_tests/templates/virtual-mcp-trusty/salt.yaml b/tcp_tests/templates/virtual-mcp-trusty/salt.yaml
index db9ba1c..d042292 100644
--- a/tcp_tests/templates/virtual-mcp-trusty/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-trusty/salt.yaml
@@ -4,7 +4,8 @@
 
 {% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
 # Other salt model repository parameters see in shared-salt.yaml
-{% set OVERRIDES = os_env('OVERRIDES', 'override_example: true') %}
+{% set OVERRIDES = os_env('OVERRIDES', '') %}
+{% set OVERRIDES_FILENAME = os_env('OVERRIDES_FILENAME', '/srv/salt/reclass/classes/cluster/overrides.yml') %}
 
 {% import 'shared-salt.yaml' as SHARED with context %}
 
@@ -19,16 +20,26 @@
 {{ SHARED.MACRO_RUN_SALT_MASTER_UNDERLAY_STATES() }}
 
 {%- if OVERRIDES != '' %}
+- description: Clear {{OVERRIDES_FILENAME}}
+  cmd: touch {{OVERRIDES_FILENAME}}
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
 {%- for param in OVERRIDES.splitlines() %}
-{%- set key, value = param.replace(' ','').split(':') %}
 - description: Override cluster parameters
   cmd: |
-    salt-call reclass.cluster_meta_set name='{{ key }}' value='{{ value }}' file_name=/srv/salt/reclass/classes/cluster/overrides.yml
-    salt '*' saltutil.refresh_pillar
+    echo -e "{{ param }}" >> {{OVERRIDES_FILENAME}}
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 1}
   skip_fail: false
 {%- endfor %}
+
+- description: Refresh pillar
+  cmd: salt '*' saltutil.refresh_pillar
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
 {%- endif %}
 
 {{ SHARED.MACRO_GENERATE_INVENTORY() }}
diff --git a/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml b/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
index 5f9f3e6..52bf890 100644
--- a/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
@@ -68,10 +68,22 @@
 
 # Install slv2 infra
 # Install MongoDB for alerta
-- description: Install MongoDB
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
   skip_fail: false
 
 #Launch containers
diff --git a/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml b/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
index 23df071..766a7c1 100644
--- a/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
@@ -68,10 +68,22 @@
 
 # Install slv2 infra
 # Install MongoDB for alerta
-- description: Install MongoDB
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+- description: Install Mongo if target matches
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+    fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 1, delay: 10}
+  skip_fail: false
+
+- description: Configure Alerta if it is exists
+  cmd: |
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    fi
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 10}
   skip_fail: false
 
 #Launch containers
diff --git a/tcp_tests/tests/system/test_install_mcp_ovs_pike.py b/tcp_tests/tests/system/test_install_mcp_ovs_pike.py
index 53dc1c1..c5138f0 100644
--- a/tcp_tests/tests/system/test_install_mcp_ovs_pike.py
+++ b/tcp_tests/tests/system/test_install_mcp_ovs_pike.py
@@ -14,6 +14,8 @@
 
 import pytest
 
+from tcp_tests.managers.runtestmanager import RuntestManager
+
 from tcp_tests import logger
 from tcp_tests import settings
 
@@ -29,7 +31,9 @@
     @pytest.mark.pike_ovs
     def test_mcp_pike_ovs_install(self, underlay,
                                   openstack_deployed,
-                                  openstack_actions):
+                                  openstack_actions,
+                                  salt_actions,
+                                  config):
         """Test for deploying an mcp environment and check it
         Scenario:
         1. Prepare salt on hosts
@@ -39,12 +43,30 @@
 
         """
         openstack_actions._salt.local(
-                tgt='*', fun='cmd.run',
-                args='service ntp stop; ntpd -gq; service ntp start')
+            tgt='*', fun='cmd.run',
+            args='service ntp stop; ntpd -gq; service ntp start')
 
         if settings.RUN_TEMPEST:
-            openstack_actions.run_tempest(pattern=settings.PATTERN)
-            openstack_actions.download_tempest_report()
+            tempest_threads = 2
+            tempest_exclude_test_args = ''
+            tempest_pattern = settings.TEMPEST_PATTERN
+            cluster_name = settings.LAB_CONFIG_NAME
+            tempest_timeout = settings.TEMPEST_TIMEOUT
+            domain_name = "{}.local".format(cluster_name)
+            target = 'gtw01'
+            runtest = RuntestManager(
+                underlay, salt_actions,
+                cluster_name=cluster_name,
+                domain_name=domain_name,
+                tempest_threads=tempest_threads,
+                tempest_exclude_test_args=tempest_exclude_test_args,
+                tempest_pattern=tempest_pattern,
+                target=target)
+            runtest.prepare()
+            test_res = runtest.run_tempest(tempest_timeout)
+            runtest.fetch_arficats(username='root')
+            runtest.save_runtime_logs(**test_res)
+
         LOG.info("*************** DONE **************")
 
     @pytest.mark.grab_versions
@@ -177,8 +199,8 @@
 
         """
         openstack_actions._salt.local(
-                tgt='*', fun='cmd.run',
-                args='service ntp stop; ntpd -gq; service ntp start')
+            tgt='*', fun='cmd.run',
+            args='service ntp stop; ntpd -gq; service ntp start')
 
         if settings.RUN_TEMPEST:
             openstack_actions.run_tempest(pattern=settings.PATTERN)
@@ -246,8 +268,8 @@
 
         """
         openstack_actions._salt.local(
-                tgt='*', fun='cmd.run',
-                args='service ntp stop; ntpd -gq; service ntp start')
+            tgt='*', fun='cmd.run',
+            args='service ntp stop; ntpd -gq; service ntp start')
 
         registry = 'docker-dev-local.docker.mirantis.net/mirantis/networking'
         name = 'rally-tempest-net-features:latest'