Merge "Minor fixes for test_k8s_chain_update"
diff --git a/tcp_tests/helpers/oslo_cfg_types.py b/tcp_tests/helpers/oslo_cfg_types.py
index 8b322b7..aff8db0 100644
--- a/tcp_tests/helpers/oslo_cfg_types.py
+++ b/tcp_tests/helpers/oslo_cfg_types.py
@@ -102,15 +102,36 @@
     """
     def __init__(self, *args, **kwargs):
 
+        # if 'default' in kwargs:
+        #    # Load a default environment variable with expected type
+        #    kwargs['default'] = args[1](
+        #        os.environ.get(env_var_name, kwargs.get('default', None))
+        #    )
+
         env_var_name = args[0].upper()
-        if 'default' in kwargs:
-            # Load a default environment variable with expected type
-            kwargs['default'] = args[1](
-                os.environ.get(env_var_name, kwargs.get('default', None))
-            )
+        if env_var_name not in os.environ:
+            env_var_name = args[0]
+        if env_var_name in os.environ:
+            # args[1] is 'type' class for the current value
+            self.environment_value = args[1](os.environ.get(env_var_name))
+            default = kwargs.get('default', '')
+            kwargs['default'] = args[1](self.environment_value)
+            print('{0}={1} (default = {2}) # {3}'
+                  .format(env_var_name,
+                          self.environment_value,
+                          default,
+                          kwargs.get('help', '')))
+
         super(Cfg, self).__init__(*args, **kwargs)
 
         # Print info about default environment variables to console
-        print('{}={}  # {}'.format(env_var_name,
-                                   kwargs.get('default', ''),
-                                   kwargs.get('help', '')))
+        # print('{}={} (default)  # {}'.format(env_var_name,
+        #                                     kwargs.get('default', ''),
+        #                                     kwargs.get('help', '')))
+
+    def _get_from_namespace(self, namespace, group_name):
+        res = super(Cfg, self)._get_from_namespace(namespace, group_name)
+        # Use the value from enviroment variable instead of config
+        if hasattr(self, 'environment_value'):
+            res = (self.environment_value, res[1])
+        return res
diff --git a/tcp_tests/managers/backup_restore_manager.py b/tcp_tests/managers/backup_restore_manager.py
new file mode 100644
index 0000000..e3b8c23
--- /dev/null
+++ b/tcp_tests/managers/backup_restore_manager.py
@@ -0,0 +1,148 @@
+#    Copyright 2018 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from tcp_tests import logger
+
+
+LOG = logger.logger
+
+
+class BackupRestoreManager(object):
+    """Helper manager for execution backup restore"""
+
+    backup_cmd = 'backupninja -n --run /etc/backup.d/200.backup.rsync'
+
+    def __init__(self, underlay, salt_api, backup_cmd=None):
+        self.underlay = underlay
+        self.__salt_api = salt_api
+        self.backup_cmd = backup_cmd or self.backup_cmd
+
+    @property
+    def salt_api(self):
+        return self.__salt_api
+
+    def create_backup(self, tgt, backup_cmd=backup_cmd):
+        return self.salt_api.enforce_state(tgt, 'cmd.run', backup_cmd)
+
+    def restore_salt_master(self, tgt):
+        return self.salt_api.local(tgt, 'salt.master.restore')
+
+    def restore_salt_minion(self, tgt):
+        return self.salt_api.local(tgt, 'salt.minion.restore')
+
+    def create_mysql_backup_backupninja(self, tgt, ):
+        rets = []
+        res = self.salt_api.enforce_state(
+            tgt, 'cmd.run',
+            'backupninja -n --run /etc/backup.d/101.mysql')
+        rets.append(res)
+        res_rsync = self.salt_api.enforce_state(tgt, 'cmd.run')
+        rets.append(res_rsync)
+        return rets
+
+    def restore_mysql_backupninja(self, tgt):
+        # Running this state restores the databases and creates a file
+        # for every restored database in /root/mysql/flags.
+        return self.salt_api.local(tgt, 'mysql.client')
+
+    def create_mysql_xtrabackup(self, tgt, backup_cmd=backup_cmd):
+        # Should be run on mysql master node
+        return self.salt_api.enforce_state(
+            tgt, 'cmd.run', '/usr/local/bin/innobackupex-runner.sh')
+
+    def check_mysql_xtrabackup_rsynced(self, tgt='I@xtrabackup:server'):
+        return self.salt_api.enforce_state(
+            tgt, 'cmd.run', 'ls /var/backups/mysql/xtrabackup/full')
+
+    def stop_mysql_slave(self, tgt='I@galera:slave'):
+        return self.salt_api.enforce_state(tgt, 'service.stop mysql')
+
+    def remove_mysql_logs(self, tgt='I@galera:slave'):
+        return self.salt_api.enforce_state(
+            tgt, 'cmd.run', 'rm /var/lib/mysql/ib_logfile*')
+
+    def stop_mysql_master(self, tgt='I@galera:master'):
+        return self.salt_api.enforce_state(tgt, 'service.stop mysql')
+
+    def disconnect_wresp_master(self, tgt='I@galera:master'):
+        # TODO fins the way updated wresp
+        return self.salt_api.enforce_state(
+            tgt, 'cmd.run', 'wsrep_cluster_address=gcomm://')
+
+    def move_dbs_files_to_new_location(self, tgt='I@galera:master'):
+        cmds = ['mkdir -p /root/mysql/mysql.bak/',
+                'mv /var/lib/mysql/* /root/mysql/mysql.bak',
+                'rm /var/lib/mysql/.galera_bootstrap']
+        rest = []
+        for cmd in cmds:
+            res = self.salt_api.enforce_state(tgt, 'cmd.run', cmd)
+            rest.append(res)
+        return rest
+
+    def check_dbs_files_removed(self, tgt='I@galera:master'):
+        cmds = ['ls /var/lib/mysql/',
+                'ls -ld /var/lib/mysql/.?*']
+        rest = []
+        for cmd in cmds:
+            res = self.salt_api.enforce_state(tgt, 'cmd.run', cmd)
+            rest.append(res)
+        return rest
+
+    # run xtrabackup state on node where bacakup
+    def run_xtrabackup(self, tgt):
+        return self.salt_api.local(tgt, 'xtrabackup')
+
+    def start_mysql(self):
+        tgts = ['I@galera:master', 'I@galera:slave']
+        ret = []
+        for tgt in tgts:
+            res = self.salt_api.enforce_state(tgt, 'service.start mysql')
+            ret.append(res)
+        return ret
+
+    def check_galera_cluster(self, tgt='I@galera:master'):
+        return self.salt_api.enforce_state(
+            tgt, 'mysql.status | grep -A1 wsrep_cluster_size')
+
+    # #################Backup_Restore_Glance###################
+
+    def copy_glance_images_to_backup(self, path_to_backup,
+                                     tgt="I@glance:server and *01*"):
+        cmd = 'cp -a /var/lib/glance/images/. {}'.format(path_to_backup)
+        return self.salt_api.enforce_state(
+            tgt, 'cmd.run', cmd)
+
+    def copy_glance_images_from_backup(self, path_to_backup,
+                                       tgt="I@glance:server and *01*"):
+        cmd = 'cp -a {}/. /var/lib/glance/images/'.format(path_to_backup)
+        return self.salt_api.enforce_state(
+            tgt, 'cmd.run', cmd)
+
+    def check_images_after_backup(self, tgt="I@keystone:client"):
+        # TODO If the context of the Glance
+        # images files is lost, run the following commands:
+        # salt -C 'I@glance:server' cmd.run
+        # "chown glance:glance <IMAGE_FILE_NAME>"
+        # salt -C 'I@glance:server' cmd.run "chmod 640 <IMAGE_FILE_NAME>"
+        cmd = '. /root/keystonercv3; openstack image list'
+        return self.salt_api.enforce_state(tgt, 'cmd.run', cmd)
+
+    # #################Backup_Restore_cinder_volumes_and_snapshots###
+    # TODO Verify that a complete backup was created on
+    # the MySQL Galera Database Master node
+    # ls /var/backups/mysql/xtrabackup/full
+    # TODO(tleontovich): add method to check needed configs
+    # TODO (tleontovich): check pillars
+    # TODO (tleontovich): check  backup is created, and
+    # restore restores
diff --git a/tcp_tests/managers/envmanager_devops.py b/tcp_tests/managers/envmanager_devops.py
index d17c6bd..d02cff5 100644
--- a/tcp_tests/managers/envmanager_devops.py
+++ b/tcp_tests/managers/envmanager_devops.py
@@ -307,29 +307,58 @@
             raise exceptions.EnvironmentIsNotSet()
         self.__env.start()
         LOG.info('Environment "{0}" started'.format(self.__env.name))
+        check_cloudinit_started = '[ -f /is_cloud_init_started ]'
+        check_cloudinit_finished = '[ -f /is_cloud_init_finished ]'
+        passed = {}
         for node in self.__env.get_nodes(role__in=underlay_node_roles):
             LOG.info("Waiting for SSH on node '{0}' / {1} ...".format(
                 node.name, self.node_ip(node)))
 
-            def _ssh_wait(host,
-                          port,
-                          username=settings.SSH_NODE_CREDENTIALS['login'],
-                          password=settings.SSH_NODE_CREDENTIALS['password'],
-                          timeout=0):
+            def _ssh_check(host,
+                           port,
+                           username=settings.SSH_NODE_CREDENTIALS['login'],
+                           password=settings.SSH_NODE_CREDENTIALS['password'],
+                           timeout=0):
                 try:
                     ssh = ssh_client.SSHClient(
                         host=host, port=port,
                         auth=ssh_client.SSHAuth(
                             username=username,
                             password=password))
-                except AuthenticationException:
-                    return True
-                except BadAuthenticationType:
+
+                    # If '/is_cloud_init_started' exists, then wait for
+                    # the flag /is_cloud_init_finished
+                    if ssh.execute(check_cloudinit_started)['exit_code'] == 0:
+                        status = ssh.execute(
+                            check_cloudinit_finished)['exit_code'] == 0
+                    # Else, just wait for SSH
+                    else:
+                        status = ssh.execute('echo ok')['exit_code'] == 0
+                    return status
+
+                except (AuthenticationException, BadAuthenticationType):
                     return True
                 except Exception:
                     return False
 
-                return ssh.execute('echo ok')['exit_code'] == 0
+            def _ssh_wait(host,
+                          port,
+                          username=settings.SSH_NODE_CREDENTIALS['login'],
+                          password=settings.SSH_NODE_CREDENTIALS['password'],
+                          timeout=0):
+
+                if host in passed and passed[host] >= 2:
+                    # host already passed the check
+                    return True
+
+                for node in self.__env.get_nodes(role__in=underlay_node_roles):
+                    ip = self.node_ip(node)
+                    if ip not in passed:
+                        passed[ip] = 0
+                    if _ssh_check(ip, port):
+                        passed[ip] += 1
+                    else:
+                        passed[ip] = 0
 
             helpers.wait(
                 lambda: _ssh_wait(self.node_ip(node), 22),
diff --git a/tcp_tests/managers/jenkins/client.py b/tcp_tests/managers/jenkins/client.py
index 0b0d7de..2628e7a 100644
--- a/tcp_tests/managers/jenkins/client.py
+++ b/tcp_tests/managers/jenkins/client.py
@@ -1,4 +1,5 @@
 from __future__ import print_function
+import time
 
 import jenkins
 import requests
@@ -6,13 +7,40 @@
 from devops.helpers import helpers
 
 
+class JenkinsWrapper(jenkins.Jenkins):
+    """Workaround for the bug:
+       https://bugs.launchpad.net/python-jenkins/+bug/1775047
+    """
+    def _response_handler(self, response):
+        '''Handle response objects'''
+
+        # raise exceptions if occurred
+        response.raise_for_status()
+
+        headers = response.headers
+        if (headers.get('content-length') is None and
+                headers.get('transfer-encoding') is None and
+                (response.status_code == 201 and
+                 headers.get('location') is None) and
+                (response.content is None or len(response.content) <= 0)):
+            # response body should only exist if one of these is provided
+            raise jenkins.EmptyResponseException(
+                "Error communicating with server[%s]: "
+                "empty response" % self.server)
+
+        # Response objects will automatically return unicode encoded
+        # when accessing .text property
+        return response
+
+
 class JenkinsClient(object):
 
     def __init__(self, host=None, username=None, password=None):
         host = host or 'http://172.16.44.33:8081'
         username = username or 'admin'
         password = password or 'r00tme'
-        self.__client = jenkins.Jenkins(
+        # self.__client = jenkins.Jenkins(
+        self.__client = JenkinsWrapper(
             host,
             username=username,
             password=password)
@@ -50,62 +78,92 @@
     def run_build(self, name, params=None, timeout=600, verbose=False):
         params = params or self.make_defults_params(name)
         num = self.__client.build_job(name, params)
+        time.sleep(2)  # wait while job is started
 
         def is_blocked():
             queued = self.__client.get_queue_item(num)
             status = not queued['blocked']
             if not status and verbose:
                 print("pending the job [{}] : {}".format(name, queued['why']))
-            return status
+            return (status and
+                    'executable' in (queued or {}) and
+                    'number' in (queued['executable'] or {}))
 
         helpers.wait(
             is_blocked,
             timeout=timeout,
             interval=30,
             timeout_msg='Timeout waiting to run the job [{}]'.format(name))
-
         build_id = self.__client.get_queue_item(num)['executable']['number']
         return name, build_id
 
-    def wait_end_of_build(self, name, build_id, timeout=600,
-                          verbose=False):
+    def wait_end_of_build(self, name, build_id, timeout=600, interval=5,
+                          verbose=False, job_output_prefix=''):
+        '''Wait until the specified build is finished
+
+        :param name: ``str``, job name
+        :param build_id: ``int``, build id
+        :param timeout: ``int``, timeout waiting the job, sec
+        :param interval: ``int``, interval of polling the job result, sec
+        :param verbose: ``bool``, print the job console updates during waiting
+        :param job_output_prefix: ``str``, print the prefix for each console
+                                  output line, with the pre-defined
+                                  substitution keys:
+                                  - '{name}' : the current job name
+                                  - '{build_id}' : the current build-id
+                                  - '{time}' : the current time
+        :returns: requests object with headers and console output,  ``obj``
+        '''
         start = [0]
+        time_str = time.strftime("%H:%M:%S")
+        prefix = "\n" + job_output_prefix.format(job_name=name,
+                                                 build_number=build_id,
+                                                 time=time_str)
+        if verbose:
+            print(prefix, end='')
 
         def building():
             status = not self.build_info(name, build_id)['building']
             if verbose:
+                time_str = time.strftime("%H:%M:%S")
+                prefix = "\n" + job_output_prefix.format(
+                    job_name=name, build_number=build_id, time=time_str)
                 res = self.get_progressive_build_output(name,
                                                         build_id,
                                                         start=start[0])
                 if 'X-Text-Size' in res.headers:
                     text_size = int(res.headers['X-Text-Size'])
                     if start[0] < text_size:
-                        print(res.content, end='')
+                        text = res.content.decode('utf-8',
+                                                  errors='backslashreplace')
+                        print(text.replace("\n", prefix), end='')
                         start[0] = text_size
             return status
 
         helpers.wait(
             building,
             timeout=timeout,
+            interval=interval,
             timeout_msg='Timeout waiting, job {0} are not finished "{1}" build'
                         ' still'.format(name, build_id))
 
     def get_build_output(self, name, build_id):
         return self.__client.get_build_console_output(name, build_id)
 
-    def get_progressive_build_output(self, name, build_id, start=0,
-                                     raise_on_err=False):
+    def get_progressive_build_output(self, name, build_id, start=0):
         '''Get build console text.
 
         :param name: Job name, ``str``
-        :param name: Build id, ``int``
-        :param name: Start offset, ``int``
+        :param build_id: Build id, ``int``
+        :param start: Start offset, ``int``
         :returns: requests object with headers and console output,  ``obj``
         '''
         folder_url, short_name = self.__client._get_job_folder(name)
 
         PROGRESSIVE_CONSOLE_OUTPUT = (
             '%(folder_url)sjob/%(short_name)s/%(build_id)d/'
-            'logText/progressiveHtml?start=%(start)d')
-        url = self.__client._build_url(PROGRESSIVE_CONSOLE_OUTPUT, locals())
-        return(requests.get(url))
+            'logText/progressiveText?start=%(start)d')
+        req = requests.Request(
+                'GET',
+                self.__client._build_url(PROGRESSIVE_CONSOLE_OUTPUT, locals()))
+        return(self.__client.jenkins_request(req))
diff --git a/tcp_tests/managers/rallymanager.py b/tcp_tests/managers/rallymanager.py
index d5b8782..36174db 100644
--- a/tcp_tests/managers/rallymanager.py
+++ b/tcp_tests/managers/rallymanager.py
@@ -104,8 +104,13 @@
         self._underlay.check_call(cmd, node_name=self._node_name)
 
         LOG.info("Copy keystonercv3")
-        cmd = "cp /root/keystonercv3 /root/rally/keystonercv3"
-        self._underlay.check_call(cmd, node_name=self._node_name)
+        tgt = self._node_name.split('.')[0]
+        cmd = "scp -3 ctl01:/root/keystonercv3 " \
+              "{tgt}:/root/rally/keystonercv3".format(
+                  tgt=tgt)
+        domain = '.'.join(self._node_name.split('.')[1:])
+        self._underlay.check_call(cmd, node_name="cfg01.{domain}".format(
+            domain=domain))
         self._run()
 
         LOG.info("Create rally deployment")
diff --git a/tcp_tests/managers/runtestmanager.py b/tcp_tests/managers/runtestmanager.py
index 9ed4a9c..229a3ff 100644
--- a/tcp_tests/managers/runtestmanager.py
+++ b/tcp_tests/managers/runtestmanager.py
@@ -12,15 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import os
 import json
+import os
 
 from devops.helpers import helpers
 
 from tcp_tests import logger
 from tcp_tests import settings
 
-
 LOG = logger.logger
 
 TEMPEST_CFG_DIR = '/tmp/test'
@@ -79,28 +78,14 @@
                     '${_param:runtest_tempest_public_net}'
                 },
                 'share': {
-                    'backend_names': 'lvm',
-                    'capability_create_share_from_snapshot_support': True,
                     'capability_snapshot_support': True,
-                    'default_share_type_name': 'default',
-                    'enable_ip_rules_for_protocols': 'nfs',
-                    'enable_user_rules_for_protocols': 'cifs',
-                    'max_api_microversion': 2.4,
-                    'min_api_microversion': 2.0,
                     'run_driver_assisted_migration_tests': False,
-                    'run_host_assisted_migration_tests': True,
                     'run_manage_unmanage_snapshot_tests': False,
                     'run_manage_unmanage_tests': False,
                     'run_migration_with_preserve_snapshots_tests': False,
-                    'run_mount_snapshot_tests': True,
                     'run_quota_tests': True,
                     'run_replication_tests': False,
-                    'run_revert_to_snapshot_tests': True,
-                    'run_share_group_tests': False,
-                    'run_shrink_tests': False,
                     'run_snapshot_tests': True,
-                    'share_creation_retry_number': 2,
-                    'suppress_errors_in_cleanup': True
                 }}}}}
 
 
@@ -155,14 +140,21 @@
     def generate_config(self):
         return self.salt_api.enforce_state(self.master_tgt, 'runtest')
 
-    def fetch_arficats(self, username=None):
+    def fetch_arficats(self, username=None, file_format='xml'):
         target_name = next(node_name for node_name
                            in self.underlay.node_names() if
                            self.target in node_name)
         with self.underlay.remote(node_name=target_name, username=None) as tgt:
+            result = tgt.execute('find {} -name "report_*.{}"'.format(
+                TEMPEST_CFG_DIR, file_format))
+            LOG.debug("Find result {0}".format(result))
+            assert len(result['stdout']) > 0, ('No report found, please check'
+                                               ' if test run was successful.')
+            report = result['stdout'][0].rstrip()
+            LOG.debug("Found files {0}".format(report))
             tgt.download(
-                destination="{cfg_dir}/report_*.xml".format(cfg_dir=TEMPEST_CFG_DIR),  # noqa
-                target="{}".format(os.environ.get("PWD")))
+                destination=report,  # noqa
+                target=os.getcwd())
 
     def store_runtest_model(self, config=CONFIG):
         master_name = next(node_name for node_name
@@ -196,12 +188,12 @@
                     path=settings.LOGS_DIR, target=self.target), 'w') as f:
                 LOG.info("Save tempest console log")
                 container_log = logs
-                f.write(container_log)
+                f.write(container_log.encode('ascii', 'ignore'))
 
         if inspect:
-            with open("{path}/{target}_tempest_container_info.json".format(
+            with open("{path}/{target}_tempest_container_info.json.log".format(
                     path=settings.LOGS_DIR, target=self.target), 'w') as f:
-                LOG.info("Save tempest containes inspect data")
+                LOG.info("Save tempest container inspect data")
 
                 container_inspect = json.dumps(inspect,
                                                indent=4, sort_keys=True)
@@ -226,7 +218,7 @@
         tgt = "{}*".format(self.target)
         params = {
             "name": self.container_name,
-            "image": self.image_name,
+            "image": "{}:{}".format(self.image_name, self.image_version),
             "environment": {
                 "ARGS": "-r {tempest_pattern} -w "
                         "{tempest_threads} "
@@ -245,7 +237,8 @@
             "cmd": self.run_cmd
         }
 
-        res = self.salt_api.local(tgt, 'dockerng.pull', self.image_name)
+        res = self.salt_api.local(tgt, 'dockerng.pull', "{}:{}".format(
+            self.image_name, self.image_version))
         LOG.info("Tempest image has beed pulled- \n{}".format(
             json.dumps(res, indent=4)))
 
@@ -291,7 +284,8 @@
                                        self.container_name)
         logs = logs_res['return'][0]
         logs = next(logs.iteritems())[1]
-        LOG.info("Tempest result - \n{}".format(logs))
+        LOG.info("Tempest result - \n{}".format(
+            logs.encode('ascii', 'ignore')))
 
         res = self.salt_api.local(tgt, 'dockerng.rm', self.container_name)
         LOG.info("Tempest container was removed".format(
diff --git a/tcp_tests/managers/underlay_ssh_manager.py b/tcp_tests/managers/underlay_ssh_manager.py
index 44c8830..495e51d 100644
--- a/tcp_tests/managers/underlay_ssh_manager.py
+++ b/tcp_tests/managers/underlay_ssh_manager.py
@@ -357,7 +357,7 @@
         """
         remote = self.remote(node_name=node_name, host=host,
                              address_pool=address_pool)
-        with remote.get_sudo(remote):
+        with remote.sudo(enforce=True):
             return remote.check_call(
                 command=cmd, verbose=verbose, timeout=timeout,
                 error_info=error_info, expected=expected,
diff --git a/tcp_tests/settings.py b/tcp_tests/settings.py
index 57c217f..169d089 100644
--- a/tcp_tests/settings.py
+++ b/tcp_tests/settings.py
@@ -73,6 +73,6 @@
 TEMPEST_IMAGE = os.environ.get(
     'TEMPEST_IMAGE',
     'docker-prod-virtual.docker.mirantis.net/mirantis/cicd/ci-tempest')  # noqa
-TEMPEST_IMAGE_VERSION = os.environ.get('TEMPEST_IMAGE_VERSION', 'latest')
+TEMPEST_IMAGE_VERSION = os.environ.get('TEMPEST_IMAGE_VERSION', 'pike')
 TEMPEST_PATTERN = os.environ.get('TEMPEST_PATTERN', 'tempest')
-TEMPEST_TIMEOUT = int(os.environ.get('TEMPEST_TIMEOUT', 18000))
+TEMPEST_TIMEOUT = int(os.environ.get('TEMPEST_TIMEOUT', 60 * 60 * 5))
diff --git a/tcp_tests/templates/cookied-mcp-pike-dpdk/common-services.yaml b/tcp_tests/templates/cookied-mcp-pike-dpdk/common-services.yaml
index 4588a4e..8057165 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dpdk/common-services.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dpdk/common-services.yaml
@@ -1,4 +1,5 @@
 {% from 'cookied-mcp-pike-dpdk/underlay.yaml' import HOSTNAME_CFG01 with context %}
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
 
 # Install support services
 - description: Install keepalived on ctl01
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
index 5a62809..4f3d9bc 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr-ssl/sl.yaml
@@ -70,19 +70,20 @@
 - description: Install Mongo if target matches
   cmd: |
     if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.server
     fi
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Alerta if it is exists
+# Create MongoDB cluster
+- description: Install Mongo if target matches
   cmd: |
-    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.cluster
     fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 5, delay: 20}
   skip_fail: false
 
 - description: Install telegraf
@@ -162,22 +163,23 @@
   skip_fail: false
 
 # Configure the services running in Docker Swarm
-- description: Install prometheus alertmanager
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm' state.sls prometheus,heka.remote_collector -b 1
+- description: Configure prometheus in docker swarm
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls prometheus,heka.remote_collector
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: run docker state
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master' state.sls docker
+#Launch containers
+- description: launch prometheus containers
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 2, delay: 10}
   skip_fail: false
 
-- description: docker ps
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm' dockerng.ps
+- description: Check docker ps
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 2, delay: 10}
   skip_fail: false
 
 - description: Configure Grafana dashboards and datasources
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr/common-services.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr/common-services.yaml
index b1b2088..6e69e30 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr/common-services.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'cookied-mcp-pike-dvr/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 # Install support services
 - description: Install keepalived on ctl01
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
diff --git a/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
index d8ad417..995d21f 100644
--- a/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-dvr/sl.yaml
@@ -66,35 +66,24 @@
   skip_fail: false
 
 # Install slv2 infra
-#Launch containers
+# Install MongoDB for alerta
 - description: Install Mongo if target matches
   cmd: |
     if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.server
     fi
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Alerta if it is exists
+# Create MongoDB cluster
+- description: Install Mongo if target matches
   cmd: |
-    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.cluster
     fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
-  skip_fail: false
-
-- description: launch prometheus containers
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
-  skip_fail: false
-
-- description: Check docker ps
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 5, delay: 20}
   skip_fail: false
 
 - description: Install telegraf
@@ -208,15 +197,22 @@
 
 # Configure the services running in Docker Swarm
 - description: Configure prometheus in docker swarm
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls prometheus
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls prometheus,heka.remote_collector
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Remote Collector in Docker Swarm for Openstack deployments
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls heka.remote_collector
+#Launch containers
+- description: launch prometheus containers
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 2, delay: 10}
+  skip_fail: false
+
+- description: Check docker ps
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 2, delay: 10}
   skip_fail: false
 
 - description: Install sphinx
diff --git a/tcp_tests/templates/cookied-mcp-pike-ovs/common-services.yaml b/tcp_tests/templates/cookied-mcp-pike-ovs/common-services.yaml
index 27bac1d..78acdf1 100644
--- a/tcp_tests/templates/cookied-mcp-pike-ovs/common-services.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-ovs/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'cookied-mcp-pike-ovs/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 # Install support services
 - description: Install keepalived on ctl01
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
diff --git a/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml b/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
index bf71b59..83f70e5 100644
--- a/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
+++ b/tcp_tests/templates/cookied-mcp-pike-ovs/sl.yaml
@@ -72,32 +72,20 @@
 - description: Install Mongo if target matches
   cmd: |
     if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.server
     fi
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Alerta if it is exists
+# Create MongoDB cluster
+- description: Install Mongo if target matches
   cmd: |
-    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.cluster
     fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
-  skip_fail: false
-
-#Launch containers
-- description: launch prometheus containers
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
-  skip_fail: false
-
-- description: Check docker ps
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 5, delay: 20}
   skip_fail: false
 
 - description: Install telegraf
@@ -211,15 +199,22 @@
 
 # Configure the services running in Docker Swarm
 - description: Configure prometheus in docker swarm
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls prometheus
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls prometheus,heka.remote_collector
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Remote Collector in Docker Swarm for Openstack deployments
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' state.sls heka.remote_collector
+#Launch containers
+- description: launch prometheus containers
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 2, delay: 10}
+  skip_fail: false
+
+- description: Check docker ps
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 2, delay: 10}
   skip_fail: false
 
 - description: Install sphinx
diff --git a/tcp_tests/templates/shared-backup-restore.yaml b/tcp_tests/templates/shared-backup-restore.yaml
new file mode 100644
index 0000000..c85e684
--- /dev/null
+++ b/tcp_tests/templates/shared-backup-restore.yaml
@@ -0,0 +1,137 @@
+{# Collection of common macroses shared across different deployments #}
+
+{%- macro MACRO_BACKUP_BACKUPNINJA() %}
+- description: Apply backup state on minions
+  cmd: salt -C 'I@backupninja:server or backupninja:client' state.sls salt.minion
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Refresh grains and mine for the backupninja client node
+  cmd: |
+    salt -C 'I@backupninja:client' state.sls salt.minion.grains
+    salt -C 'I@backupninja:client' mine.flush
+    salt -C 'I@backupninja:client' mine.update
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the backupninja state to the backupninja client node
+  cmd: |
+    salt -C 'I@backupninja:client' state.sls backupninja
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Refresh grains for the backupninja server node
+  cmd: |
+    salt -C 'I@backupninja:server' state.sls salt.minion.grains
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the backupninja state to the backupninja server node
+  cmd: |
+    salt -C 'I@backupninja:server' state.sls backupninja
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+{%- endmacro %}
+
+{%- macro MACRO_BACKUP_XTRABACKUP() %}
+
+- description: Refresh pillars
+  cmd: salt '*' saltutil.refresh_pillar
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the salt.minion state
+  cmd: |
+    salt -C 'I@xtrabackup:client or I@xtrabackup:server' state.sls salt.minion
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Refresh grains for the xtrabackup client node
+  cmd: salt -C 'I@xtrabackup:client' saltutil.sync_grains
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Update the mine for the xtrabackup client node
+  cmd: |
+    salt -C 'I@xtrabackup:client' mine.flush
+    salt -C 'I@xtrabackup:client' mine.update
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the xtrabackup client state
+  cmd: |
+    salt -C 'I@xtrabackup:client' state.sls openssh.client,xtrabackup
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the xtrabackup server state
+  cmd: |
+    salt -C 'I@xtrabackup:server' state.sls xtrabackup
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+{%- endmacro %}
+
+{%- macro MACRO_BACKUP_CEPH() %}
+
+- description: Refresh pillars
+  cmd: salt '*' saltutil.refresh_pillar
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the salt.minion state
+  cmd: |
+    salt -C 'I@ceph:backup:client or I@ceph:backup:server' state.sls salt.minion
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Refresh grains for the ceph client node
+  cmd: salt -C 'I@ceph:backup:client' saltutil.sync_grains
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Update the mine for the  client node
+  cmd: |
+    salt -C 'I@ceph:backup:client' mine.flush
+    salt -C 'I@ceph:backup:client' mine.update
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the backup client state
+  cmd: |
+    salt -C 'I@ceph:backup:client' state.sls openssh.client,ceph.backup
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+- description: Apply the backup server state
+  cmd: |
+    salt -C 'I@ceph:backup:server' state.sls ceph.backup
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 1, delay: 1}
+  skip_fail: false
+
+{%- endmacro %}
+
+
+
+
+
+
+
diff --git a/tcp_tests/templates/shared-salt.yaml b/tcp_tests/templates/shared-salt.yaml
index b042a01..adfe43f 100644
--- a/tcp_tests/templates/shared-salt.yaml
+++ b/tcp_tests/templates/shared-salt.yaml
@@ -269,6 +269,8 @@
     find ${REPLACE_DIRS} -type f -exec sed -i 's/172\.16\.10/==IPV4_NET_CONTROL_PREFIX==/g' {} +
     find ${REPLACE_DIRS} -type f -exec sed -i 's/10\.1\.0/==IPV4_NET_TENANT_PREFIX==/g' {} +
     find ${REPLACE_DIRS} -type f -exec sed -i 's/10\.16\.0/==IPV4_NET_EXTERNAL_PREFIX==/g' {} +
+    find ${REPLACE_DIRS} -type f -exec sed -i 's/10\.60\.0/==IPV4_NET_CONTROL_PREFIX==/g' {} +
+    find ${REPLACE_DIRS} -type f -exec sed -i 's/10\.70\.0/==IPV4_NET_ADMIN_PREFIX==/g' {} +
 
     find ${REPLACE_DIRS} -type f -exec sed -i 's/==IPV4_NET_ADMIN_PREFIX==/{{ IPV4_NET_ADMIN_PREFIX }}/g' {} +
     find ${REPLACE_DIRS} -type f -exec sed -i 's/==IPV4_NET_CONTROL_PREFIX==/{{ IPV4_NET_CONTROL_PREFIX }}/g' {} +
@@ -1235,7 +1237,7 @@
 
 - description: Run tempest from new docker image
   cmd: |
-    docker run -e ARGS="-r {{TEMPEST_PATTERN }} -w 2 {{ EXCLUDE_TEST_ARGS }}" -v /root/test/tempest.conf:/etc/tempest/tempest.conf -v /tmp/:/tmp/ -v /root/test:/root/tempest -v /etc/ssl/certs/:/etc/ssl/certs/ --rm docker-prod-virtual.docker.mirantis.net/mirantis/cicd/ci-tempest /bin/bash -c "run-tempest"
+    docker run -e ARGS="-r {{TEMPEST_PATTERN }} -w 2 {{ EXCLUDE_TEST_ARGS }}" -v /root/test/tempest.conf:/etc/tempest/tempest.conf -v /tmp/:/tmp/ -v /root/test:/root/tempest -v /etc/ssl/certs/:/etc/ssl/certs/ --rm docker-prod-virtual.docker.mirantis.net/mirantis/cicd/ci-tempest:pike /bin/bash -c "run-tempest"
   node_name: {{ HOSTNAME_GTW01 }}
   retry: {count: 1, delay: 30}
   skip_fail: true
diff --git a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/ceph.yaml b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/ceph.yaml
index 6ad7b6e..30b046a 100644
--- a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/ceph.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/ceph.yaml
@@ -4,6 +4,8 @@
 {% from 'virtual-mcp-pike-dvr-ceph-rgw/underlay.yaml' import DOMAIN_NAME with context %}
 {% import 'shared-salt.yaml' as SHARED with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 # Install ceph mons
 - description: Update grains
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
@@ -165,4 +167,5 @@
   retry: {count: 2, delay: 5}
   skip_fail: false
 
+{{ BACKUP.MACRO_BACKUP_CEPH() }}
 {{ SHARED.RUN_NEW_TEMPEST() }}
\ No newline at end of file
diff --git a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/common-services.yaml b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/common-services.yaml
index 121d4c4..f285934 100644
--- a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/common-services.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'virtual-mcp-pike-dvr-ceph-rgw/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 # Install support services
 - description: Install keepalived on ctl01
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
@@ -115,3 +117,7 @@
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
+
+{{ BACKUP.MACRO_BACKUP_BACKUPNINJA() }}
+{{ BACKUP.MACRO_BACKUP_XTRABACKUP() }}
+
diff --git a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
index 80e29a1..94e5308 100644
--- a/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-dvr-ceph-rgw/salt.yaml
@@ -11,7 +11,7 @@
 
 {{ SHARED.MACRO_CLONE_RECLASS_MODELS() }}
 
-{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "fluentd" "runtest" "backupninja"') }}
+{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "fluentd" "runtest" "backupninja" "glusterfs"') }}
 
 {{ SHARED.MACRO_INSTALL_SALT_MINIONS() }}
 
@@ -37,4 +37,4 @@
   cmd: salt 'cmp02*' cmd.run "ip addr del {{ SHARED.IPV4_NET_CONTROL_PREFIX }}.106/24 dev ens4; ip addr flush dev ens4";
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
-  skip_fail: false
\ No newline at end of file
+  skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp-pike-dvr/common-services.yaml b/tcp_tests/templates/virtual-mcp-pike-dvr/common-services.yaml
index cbde3f0..d96c951 100644
--- a/tcp_tests/templates/virtual-mcp-pike-dvr/common-services.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-dvr/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'virtual-mcp-pike-dvr/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 # Install support services
 - description: Install keepalived on ctl01
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
@@ -115,3 +117,6 @@
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
+
+{{ BACKUP.MACRO_BACKUP_BACKUPNINJA() }}
+{{ BACKUP.MACRO_BACKUP_XTRABACKUP() }}
\ No newline at end of file
diff --git a/tcp_tests/templates/virtual-mcp-pike-ovs/common-services.yaml b/tcp_tests/templates/virtual-mcp-pike-ovs/common-services.yaml
index abaa50d..b7cc62b 100644
--- a/tcp_tests/templates/virtual-mcp-pike-ovs/common-services.yaml
+++ b/tcp_tests/templates/virtual-mcp-pike-ovs/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'virtual-mcp-pike-ovs/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 # Install support services
 - description: Install keepalived on ctl01
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
@@ -115,3 +117,6 @@
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
+
+{{ BACKUP.MACRO_BACKUP_BACKUPNINJA() }}
+{{ BACKUP.MACRO_BACKUP_XTRABACKUP() }}
\ No newline at end of file
diff --git a/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml b/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
index 52bf890..01dcabf 100644
--- a/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp11-k8s-calico/sl.yaml
@@ -1,6 +1,9 @@
 {% from 'virtual-mcp11-k8s-calico/underlay.yaml' import HOSTNAME_CFG01 with context %}
 {% from 'virtual-mcp11-k8s-calico/salt.yaml' import ENVIRONMENT_MODEL_INVENTORY_NAME with context %}
 
+
+{% import 'shared-sl-tests.yaml' as SHARED_SL_TESTS with context %}
+
 # Install docker swarm
 - description: Configure docker service
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm' state.sls docker.host
@@ -71,32 +74,20 @@
 - description: Install Mongo if target matches
   cmd: |
     if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.server
     fi
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Alerta if it is exists
+# Create MongoDB cluster
+- description: Install Mongo if target matches
   cmd: |
-    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.cluster
     fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
-  skip_fail: false
-
-#Launch containers
-- description: launch prometheus containers
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
-  skip_fail: false
-
-- description: Check docker ps
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 5, delay: 20}
   skip_fail: false
 
 - description: Install telegraf
@@ -215,6 +206,19 @@
   retry: {count: 1, delay: 10}
   skip_fail: false
 
+#Launch containers
+- description: launch prometheus containers
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 2, delay: 10}
+  skip_fail: false
+
+- description: Check docker ps
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 2, delay: 10}
+  skip_fail: false
+
 ###
 # From pipeline-library:
 # if (!common.checkContains('STACK_INSTALL', 'k8s')) {
@@ -265,3 +269,6 @@
   retry: {count: 1, delay: 10}
   skip_fail: false
 
+
+{{  SHARED_SL_TESTS.MACRO_CLONE_SL_TESTS() }}
+{{  SHARED_SL_TESTS.MACRO_CONFIGURE_TESTS() }}
diff --git a/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml b/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
index 766a7c1..c6e2234 100644
--- a/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp11-k8s-contrail/sl.yaml
@@ -1,6 +1,8 @@
 {% from 'virtual-mcp11-k8s-contrail/underlay.yaml' import HOSTNAME_CFG01 with context %}
 {% from 'virtual-mcp11-k8s-contrail/salt.yaml' import ENVIRONMENT_MODEL_INVENTORY_NAME with context %}
 
+{% import 'shared-sl-tests.yaml' as SHARED_SL_TESTS with context %}
+
 # Install docker swarm
 - description: Configure docker service
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm' state.sls docker.host
@@ -71,32 +73,20 @@
 - description: Install Mongo if target matches
   cmd: |
     if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.server
     fi
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
-- description: Configure Alerta if it is exists
+# Create MongoDB cluster
+- description: Install Mongo if target matches
   cmd: |
-    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@prometheus:alerta' match.pillar 'prometheus:alerta' ; then
-      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:alerta' state.sls prometheus.alerta
+    if salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' match.pillar 'mongodb:server' ; then
+      salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@mongodb:server' state.sls mongodb.cluster
     fi
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
-  skip_fail: false
-
-#Launch containers
-- description: launch prometheus containers
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
-  skip_fail: false
-
-- description: Check docker ps
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 2, delay: 10}
+  retry: {count: 5, delay: 20}
   skip_fail: false
 
 - description: Install telegraf
@@ -215,6 +205,19 @@
   retry: {count: 1, delay: 10}
   skip_fail: false
 
+#Launch containers
+- description: launch prometheus containers
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and I@prometheus:server' state.sls docker.client
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 2, delay: 10}
+  skip_fail: false
+
+- description: Check docker ps
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and I@prometheus:server' cmd.run "docker ps"
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 2, delay: 10}
+  skip_fail: false
+
 ###
 # From pipeline-library:
 # if (!common.checkContains('STACK_INSTALL', 'k8s')) {
@@ -265,3 +268,6 @@
   retry: {count: 1, delay: 10}
   skip_fail: false
 
+
+{{  SHARED_SL_TESTS.MACRO_CLONE_SL_TESTS() }}
+{{  SHARED_SL_TESTS.MACRO_CONFIGURE_TESTS() }}
\ No newline at end of file
diff --git a/tcp_tests/templates/virtual-offline-pike-ovs-dpdk/common-services.yaml b/tcp_tests/templates/virtual-offline-pike-ovs-dpdk/common-services.yaml
index 0890cd3..d4af905 100644
--- a/tcp_tests/templates/virtual-offline-pike-ovs-dpdk/common-services.yaml
+++ b/tcp_tests/templates/virtual-offline-pike-ovs-dpdk/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'virtual-offline-pike-ovs-dpdk/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 - description: remove apparmor
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
     '*' cmd.run 'service apparmor stop; service apparmor teardown; update-rc.d -f apparmor remove; apt-get -y remove apparmor'
@@ -122,3 +124,6 @@
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
+
+{{ BACKUP.MACRO_BACKUP_BACKUPNINJA() }}
+{{ BACKUP.MACRO_BACKUP_XTRABACKUP() }}
\ No newline at end of file
diff --git a/tcp_tests/templates/virtual-offline-ssl/common-services.yaml b/tcp_tests/templates/virtual-offline-ssl/common-services.yaml
index 0c75bb4..1eb8540 100644
--- a/tcp_tests/templates/virtual-offline-ssl/common-services.yaml
+++ b/tcp_tests/templates/virtual-offline-ssl/common-services.yaml
@@ -1,5 +1,7 @@
 {% from 'virtual-offline-ssl/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+{% import 'shared-backup-restore.yaml' as BACKUP with context %}
+
 - description: remove apparmor
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
     '*' cmd.run 'service apparmor stop; service apparmor teardown; update-rc.d -f apparmor remove; apt-get -y remove apparmor'
@@ -122,3 +124,6 @@
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
+
+{{ BACKUP.MACRO_BACKUP_BACKUPNINJA() }}
+{{ BACKUP.MACRO_BACKUP_XTRABACKUP() }}
\ No newline at end of file
diff --git a/tcp_tests/utils/__init__.py b/tcp_tests/utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tcp_tests/utils/__init__.py
diff --git a/tcp_tests/utils/create_devops_env.py b/tcp_tests/utils/create_devops_env.py
new file mode 100755
index 0000000..d7c062c
--- /dev/null
+++ b/tcp_tests/utils/create_devops_env.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+sys.path.append(os.getcwd())
+try:
+    from tcp_tests.helpers import ext
+    from tcp_tests.fixtures import config_fixtures
+    from tcp_tests.managers import envmanager_devops
+except ImportError:
+    print("ImportError: Run the application from the tcp-qa directory or "
+          "set the PYTHONPATH environment variable to directory which contains"
+          " ./tcp_tests")
+    sys.exit(1)
+
+
+def main():
+    """Create fuel-devops environment from template"""
+    config = config_fixtures.config()
+    env = envmanager_devops.EnvironmentManager(config=config)
+    if not env.has_snapshot(ext.SNAPSHOT.hardware):
+        env.create_snapshot(ext.SNAPSHOT.hardware)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/tcp_tests/utils/env_jenkins_cicd b/tcp_tests/utils/env_jenkins_cicd
new file mode 100755
index 0000000..04cdd56
--- /dev/null
+++ b/tcp_tests/utils/env_jenkins_cicd
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Source file to set access credentials to Jenkins API on CICD cluster
+# Requires parameters to work with libpepper:
+#  - SALTAPI_URL
+#  - SALTAPI_USER
+#  - SALTAPI_PASS
+# Example usage:
+# $> export ENV_NAME=some-test-environment
+# $> . ./tcp_tests/utils/env_salt
+# $> . ./tcp_tests/utils/env_jenkins_cicd
+# $> ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="{... json ...}"
+
+CURRENT_DIR=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+export PYTHONPATH="${CURRENT_DIR}/../.."
+
+if [ -z "$SALTAPI_URL" ]; then
+    echo "$SALTAPI_URL not found in the environment variables, getting values from salt-master is impossible."
+    unset JENKINS_URL
+    unset JENKINS_USER
+    unset JENKINS_PASS
+    unset JENKINS_START_TIMEOUT
+    unset JENKINS_BUILD_TIMEOUT
+else
+    MASTER="get_param.py -C I@docker:client:stack:jenkins pillar.get jenkins:client:master"
+    export JENKINS_HOST=$(${CURRENT_DIR}/${MASTER}:host)
+    export JENKINS_PORT=$(${CURRENT_DIR}/${MASTER}:port)
+    export JENKINS_URL=http://${JENKINS_HOST}:${JENKINS_PORT}
+    export JENKINS_USER=$(${CURRENT_DIR}/${MASTER}:username)
+    export JENKINS_PASS=$(${CURRENT_DIR}/${MASTER}:password)
+    export JENKINS_START_TIMEOUT=60
+    export JENKINS_BUILD_TIMEOUT=1800
+fi
+
+echo "export JENKINS_URL='$JENKINS_URL'  # Jenkins API URL"
+echo "export JENKINS_USER='${JENKINS_USER}'  # Jenkins API username"
+echo "export JENKINS_PASS='${JENKINS_PASS}'  # Jenkins API password or token"
+echo "export JENKINS_START_TIMEOUT='${JENKINS_START_TIMEOUT}'  # Timeout waiting for job in queue to start building"
+echo "export JENKINS_BUILD_TIMEOUT='${JENKINS_BUILD_TIMEOUT}'  # Timeout waiting for building job to complete"
diff --git a/tcp_tests/utils/env_jenkins_day01 b/tcp_tests/utils/env_jenkins_day01
new file mode 100755
index 0000000..c084a52
--- /dev/null
+++ b/tcp_tests/utils/env_jenkins_day01
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Source file to set access credentials to Jenkins API on salt-master node (day01 deployment steps)
+# Requires:
+#  - ENV_NAME
+# Example usage:
+# $> . ./tcp_tests/utils/env_salt
+# $> . ./tcp_tests/utils/env_jenkins_day01
+# $> ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="{... json ...}"
+
+CURRENT_DIR=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+export PYTHONPATH=${CURRENT_DIR}/../..
+
+SALT_MASTER_IP=${SALT_MASTER_IP:-$(for node in $(dos.py slave-ip-list --address-pool-name admin-pool01 ${ENV_NAME}); do echo $node|grep cfg01|cut -d',' -f2; done)}
+
+if [ -z "$SALT_MASTER_IP" ]; then
+    echo "SALT_MASTER_IP not found in the environment '${ENV_NAME}'"
+    unset JENKINS_URL
+    unset JENKINS_USER
+    unset JENKINS_PASS
+    unset JENKINS_START_TIMEOUT
+    unset JENKINS_BUILD_TIMEOUT
+else
+    # For run_jenkins_job.py
+    export JENKINS_URL=http://${SALT_MASTER_IP}:8081
+    export JENKINS_USER=admin
+    export JENKINS_PASS=r00tme
+    export JENKINS_START_TIMEOUT=60
+    export JENKINS_BUILD_TIMEOUT=1800
+fi
+
+echo "export JENKINS_URL='$JENKINS_URL'  # Jenkins API URL"
+echo "export JENKINS_USER='${JENKINS_USER}'  # Jenkins API username"
+echo "export JENKINS_PASS='${JENKINS_PASS}'  # Jenkins API password or token"
+echo "export JENKINS_START_TIMEOUT='${JENKINS_START_TIMEOUT}'  # Timeout waiting for job in queue to start building"
+echo "export JENKINS_BUILD_TIMEOUT='${JENKINS_BUILD_TIMEOUT}'  # Timeout waiting for building job to complete"
diff --git a/tcp_tests/utils/env_k8s b/tcp_tests/utils/env_k8s
new file mode 100755
index 0000000..01e20fd
--- /dev/null
+++ b/tcp_tests/utils/env_k8s
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Source file to set access credentials to Kubernetes API
+# Requires parameters to work with libpepper:
+#  - SALTAPI_URL
+#  - SALTAPI_USER
+#  - SALTAPI_PASS
+# Example usage:
+# $> export ENV_NAME=some-test-environment
+# $> . ./tcp_tests/utils/env_salt
+# $> . ./tcp_tests/utils/env_k8s
+# # now you can run tcp-qa test cases for k8s
+
+CURRENT_DIR=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+export PYTHONPATH=${CURRENT_DIR}/../..
+
+if [ -z "$SALTAPI_URL" ]; then
+    echo "$SALTAPI_URL not found in the environment variables, getting values from salt-master is impossible."
+    unset kube_host
+    unset kube_apiserver_port
+    unset kubernetes_admin_user
+    unset kubernetes_admin_password
+else
+    KUBE_TARGET='I@haproxy:proxy:enabled:true and I@kubernetes:master and *01*'
+    export kube_host=$(${PYTHONPATH}/tcp_tests/utils/get_param.py -C "${KUBE_TARGET}" pillar.get haproxy:proxy:listen:k8s_secure:binds:address)
+    export kube_apiserver_port=$(${PYTHONPATH}/tcp_tests/utils/get_param.py -C "${KUBE_TARGET}" pillar.get haproxy:proxy:listen:k8s_secure:binds:port)
+    export kubernetes_admin_user=$(${PYTHONPATH}/tcp_tests/utils/get_param.py -C "${KUBE_TARGET}" pillar.get kubernetes:master:admin:username)
+    export kubernetes_admin_password=$(${PYTHONPATH}/tcp_tests/utils/get_param.py -C "${KUBE_TARGET}" pillar.get kubernetes:master:admin:password)
+fi
+
+echo "export kube_host='$kube_host'  # Kubernetes API host"
+echo "export kube_apiserver_port='${kube_apiserver_port}'  # Kubernetes API port"
+echo "export kubernetes_admin_user='${kubernetes_admin_user}'  # Kubernetes admin user"
+echo "export kubernetes_admin_password='${kubernetes_admin_password}'  # Kubernetes admin password"
diff --git a/tcp_tests/utils/env_salt b/tcp_tests/utils/env_salt
new file mode 100755
index 0000000..12c4575
--- /dev/null
+++ b/tcp_tests/utils/env_salt
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Source file to set access credentials to salt-api for using with libpepper
+# Requires:
+#  - ENV_NAME
+# Example usage:
+# $> . ./tcp_tests/utils/env_salt
+# $> pepper -C 'I@linux:system' test.ping
+
+CURRENT_DIR=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+export PYTHONPATH=${CURRENT_DIR}/../..
+
+export SALT_MASTER_IP=${SALT_MASTER_IP:-$(for node in $(dos.py slave-ip-list --address-pool-name admin-pool01 ${ENV_NAME}); do echo $node|grep cfg01|cut -d',' -f2; done)}
+
+if [ -z "$SALT_MASTER_IP" ]; then
+    echo "SALT_MASTER_IP not found in the environment '${ENV_NAME}'"
+    unset SALT_MASTER_IP
+    unset SALTAPI_URL
+    unset SALTAPI_USER
+    unset SALTAPI_PASS
+    unset SALTAPI_EAUTH
+else
+    # For pepper client
+    export SALTAPI_URL=http://${SALT_MASTER_IP}:6969/
+    export SALTAPI_USER='salt'
+    export SALTAPI_PASS='hovno12345!'
+    export SALTAPI_EAUTH='pam'
+fi
+
+echo "export SALT_MASTER_IP='${SALT_MASTER_IP}'"
+echo "export SALTAPI_URL='${SALTAPI_URL}'"
+echo "export SALTAPI_USER='${SALTAPI_USER}'"
+echo "export SALTAPI_PASS='${SALTAPI_PASS}'"
+echo "export SALTAPI_EAUTH='${SALTAPI_EAUTH}'"
diff --git a/tcp_tests/utils/get_param.py b/tcp_tests/utils/get_param.py
new file mode 100755
index 0000000..6179e70
--- /dev/null
+++ b/tcp_tests/utils/get_param.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+"""
+A wrapper to ``pepper``, a CLI interface to a remote salt-api instance.
+
+Return a single parameter from the salt model for specified
+target and pillar.
+
+Fails if the result contains more than one parameter.
+
+Use the pepper CLI parameters to set salt-api access parameters
+or set the environment variables:
+
+  export SALTAPI_URL=http://${SALT_MASTER_IP}:6969/;
+  export SALTAPI_USER='salt';
+  export SALTAPI_PASS='pass';
+  export SALTAPI_EAUTH='pam';
+"""
+from __future__ import print_function
+
+import sys
+import json
+
+from pepper import cli
+
+
+runner = cli.PepperCli()
+runner.parser.description = __doc__
+
+
+if len(sys.argv) <= 1:
+    sys.argv.append('--help')
+
+results = []
+for res in runner.run():
+    results.append(res)
+
+if not results:
+    print("Empty response", file=sys.stderr)
+    sys.exit(1)
+
+if len(results) > 1:
+    print("Too many results", file=sys.stderr)
+    sys.exit(1)
+
+if results[0][0] != 0:
+    print("Error code returned", file=sys.stderr)
+    sys.exit(results[0][0])
+
+data = json.loads(results[0][1])
+nodes = data['return'][0].keys()
+
+if not nodes:
+    print("Wrong target: no minions selected", file=sys.stderr)
+    sys.exit(1)
+
+if len(nodes) > 1:
+    print("Wrong target: too many minions selected: {0}"
+          .format(nodes), file=sys.stderr)
+    sys.exit(1)
+
+print(data['return'][0][nodes[0]])
diff --git a/tcp_tests/utils/run_jenkins_job.py b/tcp_tests/utils/run_jenkins_job.py
new file mode 100755
index 0000000..00ebec6
--- /dev/null
+++ b/tcp_tests/utils/run_jenkins_job.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+
+import argparse
+import os
+import sys
+
+import json
+
+sys.path.append(os.getcwd())
+try:
+    from tcp_tests.managers.jenkins.client import JenkinsClient
+except ImportError:
+    print("ImportError: Run the application from the tcp-qa directory or "
+          "set the PYTHONPATH environment variable to directory which contains"
+          " ./tcp_tests")
+    sys.exit(1)
+
+
+EXIT_CODES = {
+  "SUCCESS": 0,
+  # 1 - python runtime execution error
+  # 2 - job unknown status
+  "FAILURE": 3,
+  "UNSTABLE": 4,
+  "ABORTED": 5,
+  "DISABLED": 6
+  # 10 - invalid cli options
+}
+
+
+def load_params():
+    """
+    Parse CLI arguments and environment variables
+
+    Returns: ArgumentParser instance
+    """
+    env_host = os.environ.get('JENKINS_URL', None)
+    env_username = os.environ.get('JENKINS_USER', None)
+    env_password = os.environ.get('JENKINS_PASS', None)
+    env_start_timeout = os.environ.get('JENKINS_START_TIMEOUT', 1800)
+    env_build_timeout = os.environ.get('JENKINS_BUILD_TIMEOUT', 3600 * 4)
+
+    parser = argparse.ArgumentParser(description=(
+        'Host, username and password may be specified either by the command '
+        'line arguments or using environment variables: JENKINS_URL, '
+        'JENKINS_USER, JENKINS_PASS, JENKINS_START_TIMEOUT, '
+        'JENKINS_BUILD_TIMEOUT. \nCommand line arguments have the highest '
+        'priority, after that the environment variables are used as defaults.'
+    ))
+    parser.add_argument('--host',
+                        metavar='JENKINS_URL',
+                        help='Jenkins Host',
+                        default=env_host)
+    parser.add_argument('--username',
+                        metavar='JENKINS_USER',
+                        help='Jenkins Username', default=env_username)
+    parser.add_argument('--password',
+                        metavar='JENKINS_PASS',
+                        help='Jenkins Password or API token',
+                        default=env_password)
+    parser.add_argument('--start-timeout',
+                        metavar='JENKINS_START_TIMEOUT',
+                        help='Timeout waiting until build is started',
+                        default=env_start_timeout,
+                        type=int)
+    parser.add_argument('--build-timeout',
+                        metavar='JENKINS_BUILD_TIMEOUT',
+                        help='Timeout waiting until build is finished',
+                        default=env_build_timeout,
+                        type=int)
+    parser.add_argument('--job-name',
+                        help='Jenkins job name to run',
+                        default=None)
+    parser.add_argument('--job-parameters',
+                        metavar='json-dict',
+                        help=('Job parameters to use instead of default '
+                              'values, as a json string, for example: '
+                              '--job-parameters=\'{"SALT_MASTER_URL": '
+                              '"http://localhost:6969"}\''),
+                        default={}, type=json.loads)
+    parser.add_argument('--job-output-prefix',
+                        help=('Jenkins job output prefix for each line in the '
+                              'output, if --verbose is enabled. Useful for the'
+                              ' pipelines that use multiple different runs of '
+                              'jobs. The string is a template for python '
+                              'format() function where the following arguments'
+                              ' are allowed: job_name, build_number. '
+                              'Example: --job-output-prefix=\"[ {job_name} '
+                              '#{build_number}, core ]\"'),
+                        default='',
+                        type=str)
+    parser.add_argument('--verbose',
+                        action='store_const',
+                        const=True,
+                        help='Show build console output',
+                        default=False)
+    return parser
+
+
+def print_build_header(build, job_params, opts):
+    print('\n#############################################################')
+    print('##### Building job [{0}] #{1} (timeout={2}) with the following '
+          'parameters:'.format(build[0], build[1], opts.build_timeout))
+    print('##### ' + '\n##### '.join(
+        [str(key) + ": " + str(val) for key, val in job_params.iteritems()]
+    ))
+    print('#############################################################')
+
+
+def print_build_footer(build, result, url):
+    print('\n\n#############################################################')
+    print('##### Completed job [{0}] #{1} at {2}: {3}'
+          .format(build[0], build[1], url, result))
+    print('#############################################################\n')
+
+
+def run_job(opts):
+
+    jenkins = JenkinsClient(
+        host=opts.host,
+        username=opts.username,
+        password=opts.password)
+
+    job_params = jenkins.make_defults_params(opts.job_name)
+    job_params.update(opts.job_parameters)
+
+    build = jenkins.run_build(opts.job_name,
+                              job_params,
+                              verbose=opts.verbose,
+                              timeout=opts.start_timeout)
+    if opts.verbose:
+        print_build_header(build, job_params, opts)
+
+    jenkins.wait_end_of_build(
+        name=build[0],
+        build_id=build[1],
+        timeout=opts.build_timeout,
+        interval=1,
+        verbose=opts.verbose,
+        job_output_prefix=opts.job_output_prefix)
+    result = jenkins.build_info(name=build[0],
+                                build_id=build[1])['result']
+    if opts.verbose:
+        print_build_footer(build, result, opts.host)
+
+    return EXIT_CODES.get(result, 2)
+
+
+def main(args=None):
+    parser = load_params()
+    opts = parser.parse_args()
+
+    if opts.host is None or opts.job_name is None:
+        print("JENKINS_URL and a job name are required!")
+        parser.print_help()
+        return 10
+    else:
+        exit_code = run_job(opts)
+        return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tcp_tests/utils/run_template_commands.py b/tcp_tests/utils/run_template_commands.py
new file mode 100755
index 0000000..5d2c178
--- /dev/null
+++ b/tcp_tests/utils/run_template_commands.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+import argparse
+import os
+import sys
+
+sys.path.append(os.getcwd())
+try:
+    from tcp_tests.fixtures import config_fixtures
+    from tcp_tests.managers import underlay_ssh_manager
+    from tcp_tests.managers import execute_commands
+except ImportError:
+    print("ImportError: Run the application from the tcp-qa directory or "
+          "set the PYTHONPATH environment variable to directory which contains"
+          " ./tcp_tests")
+    sys.exit(1)
+
+
+def load_params():
+    """
+    Parse CLI arguments and environment variables
+
+    Returns: ArgumentParser instance
+    """
+    parser = argparse.ArgumentParser(description=(
+        'Run commands from yaml templates'
+    ))
+    parser.add_argument('path_to_template',
+                        help='Path to YAML template')
+    parser.add_argument('--template-steps-label',
+                        help=('Text that will be shown as steps label'),
+                        default='',
+                        type=str)
+
+    return parser
+
+
+def main():
+    """Create fuel-devops environment from template"""
+    parser = load_params()
+    opts = parser.parse_args()
+
+    if opts.path_to_template is None:
+        parser.print_help()
+        return 10
+
+    path = os.path.abspath(opts.path_to_template)
+    label = opts.template_steps_label
+    if not label:
+        label = os.path.basename(path).split('.')[0]
+
+    config = config_fixtures.config()
+    underlay = underlay_ssh_manager.UnderlaySSHManager(config)
+
+    commands = underlay.read_template(path)
+
+    commander = execute_commands.ExecuteCommandsMixin(config, underlay)
+    commander.execute_commands(commands, label=label)
+
+
+if __name__ == '__main__':
+    sys.exit(main())