Add SALT API client
diff --git a/tcp_tests/managers/common_services_manager.py b/tcp_tests/managers/common_services_manager.py
index a91eb46..dee792c 100644
--- a/tcp_tests/managers/common_services_manager.py
+++ b/tcp_tests/managers/common_services_manager.py
@@ -12,18 +12,22 @@
# License for the specific language governing permissions and limitations
# under the License.
-class CommonServicesManager(object):
+from tcp_tests.managers.execute_commands import ExecuteCommandsMixin
+
+
+class CommonServicesManager(ExecuteCommandsMixin):
"""docstring for CommonServicesManager"""
- __config = None
- __underlay = None
+ _config = None
+ _underlay = None
- def __init__(self, config, underlay):
- self.__config = config
- self.__underlay = underlay
+ def __init__(self, config, underlay, salt=None):
+ self._config = config
+ self._underlay = underlay
+ self._salt = salt
super(CommonServicesManager, self).__init__()
def install(self, commands):
- self.__underlay.execute_commands(commands,
- label='Install common services')
- self.__config.common_services.common_services_installed = True
+ self.execute_commands(commands,
+ label='Install common services')
+ self._config.common_services.common_services_installed = True
diff --git a/tcp_tests/managers/envmanager_devops.py b/tcp_tests/managers/envmanager_devops.py
index e25faef..0874b49 100644
--- a/tcp_tests/managers/envmanager_devops.py
+++ b/tcp_tests/managers/envmanager_devops.py
@@ -33,7 +33,7 @@
class EnvironmentManager(object):
"""Class-helper for creating VMs via devops environments"""
- __config = None
+ _config = None
def __init__(self, config=None):
"""Initializing class instance and create the environment
@@ -45,7 +45,7 @@
"""
self.__devops_config = env_config.EnvironmentConfig()
self._env = None
- self.__config = config
+ self._config = config
if config.hardware.conf_path is not None:
self._devops_config.load_template(config.hardware.conf_path)
@@ -173,9 +173,9 @@
msg = "[ Create snapshot '{0}' ] {1}".format(name, description or '')
LOG.info("\n\n{0}\n{1}".format(msg, '*' * len(msg)))
- self.__config.hardware.current_snapshot = name
+ self._config.hardware.current_snapshot = name
LOG.info("Set current snapshot in config to '{0}'".format(
- self.__config.hardware.current_snapshot))
+ self._config.hardware.current_snapshot))
if self._env is not None:
LOG.info('trying to suspend ....')
self._env.suspend()
@@ -185,10 +185,11 @@
self._env.resume()
else:
raise exceptions.EnvironmentIsNotSet()
- settings_oslo.save_config(self.__config, name, self._env.name)
+ settings_oslo.save_config(self._config, name, self._env.name)
if settings.VIRTUAL_ENV:
- venv_msg = "source {0}/bin/activate;\n".format(settings.VIRTUAL_ENV)
+ venv_msg = "source {0}/bin/activate;\n".format(
+ settings.VIRTUAL_ENV)
else:
venv_msg = ""
LOG.info("To revert the snapshot:\n\n"
@@ -204,7 +205,7 @@
snapshot_name=name,
login=settings.SSH_NODE_CREDENTIALS['login'],
password=settings.SSH_NODE_CREDENTIALS['password'],
- salt_master_host=self.__config.salt.salt_master_host))
+ salt_master_host=self._config.salt.salt_master_host))
def _get_snapshot_config_name(self, snapshot_name):
"""Get config name for the environment"""
@@ -235,13 +236,13 @@
try:
test_config_path = self._get_snapshot_config_name(name)
- settings_oslo.reload_snapshot_config(self.__config,
+ settings_oslo.reload_snapshot_config(self._config,
test_config_path)
except cfg.ConfigFilesNotFoundError as conf_err:
LOG.error("Config file(s) {0} not found!".format(
conf_err.config_files))
- self.__config.hardware.current_snapshot = name
+ self._config.hardware.current_snapshot = name
def _create_environment(self):
"""Create environment and start VMs.
@@ -373,12 +374,12 @@
def set_dns_config(self):
# Set local nameserver to use by default
- if not self.__config.underlay.nameservers:
- self.__config.underlay.nameservers = [self.nameserver]
- if not self.__config.underlay.upstream_dns_servers:
- self.__config.underlay.upstream_dns_servers = [self.nameserver]
+ if not self._config.underlay.nameservers:
+ self._config.underlay.nameservers = [self.nameserver]
+ if not self._config.underlay.upstream_dns_servers:
+ self._config.underlay.upstream_dns_servers = [self.nameserver]
def set_address_pools_config(self):
"""Store address pools CIDRs in config object"""
for ap in self._env.get_address_pools():
- self.__config.underlay.address_pools[ap.name] = ap.net
+ self._config.underlay.address_pools[ap.name] = ap.net
diff --git a/tcp_tests/managers/envmanager_empty.py b/tcp_tests/managers/envmanager_empty.py
index b9ab8e1..702d723 100644
--- a/tcp_tests/managers/envmanager_empty.py
+++ b/tcp_tests/managers/envmanager_empty.py
@@ -18,7 +18,7 @@
class EnvironmentManagerEmpty(object):
"""Class-helper for creating VMs via devops environments"""
- __config = None
+ _config = None
def __init__(self, config=None):
"""Initializing class instance and create the environment
@@ -28,12 +28,12 @@
:param config.hardware.current_snapshot: name of the snapshot that
descriebe environment status.
"""
- self.__config = config
+ self._config = config
def lvm_storages(self):
"""Returns data of lvm_storages on nodes in environment
- It's expected that data of self.__config.lvm_storages will be
+ It's expected that data of self._config.lvm_storages will be
like this:
{
"node1": {
@@ -48,7 +48,7 @@
}
:rtype: dict
"""
- return self.__config.underlay.lvm
+ return self._config.underlay.lvm
def get_ssh_data(self, roles=None):
raise Exception("EnvironmentManagerEmpty doesn't have SSH details. "
@@ -60,24 +60,24 @@
- Store the state of the environment <name> to the 'config' object
- Save 'config' object to a file 'config_<name>.ini'
"""
- self.__config.hardware.current_snapshot = name
- settings_oslo.save_config(self.__config, name)
+ self._config.hardware.current_snapshot = name
+ settings_oslo.save_config(self._config, name)
def revert_snapshot(self, name):
"""Check the current state <name> of the environment
- Check that the <name> matches the current state of the environment
- that is stored in the 'self.__config.hardware.current_snapshot'
+ that is stored in the 'self._config.hardware.current_snapshot'
- Try to reload 'config' object from a file 'config_<name>.ini'
If the file not found, then pass with defaults.
- Set <name> as the current state of the environment after reload
:param name: string
"""
- if self.__config.hardware.current_snapshot != name:
+ if self._config.hardware.current_snapshot != name:
raise Exception(
"EnvironmentManagerEmpty cannot revert nodes from {} to {}"
- .format(self.__config.hardware.current_snapshot, name))
+ .format(self._config.hardware.current_snapshot, name))
def start(self):
"""Start environment"""
@@ -96,10 +96,10 @@
pass
def has_snapshot(self, name):
- return self.__config.hardware.current_snapshot == name
+ return self._config.hardware.current_snapshot == name
def has_snapshot_config(self, name):
- return self.__config.hardware.current_snapshot == name
+ return self._config.hardware.current_snapshot == name
def delete_environment(self):
"""Delete environment"""
diff --git a/tcp_tests/managers/execute_commands.py b/tcp_tests/managers/execute_commands.py
new file mode 100644
index 0000000..76f4bc9
--- /dev/null
+++ b/tcp_tests/managers/execute_commands.py
@@ -0,0 +1,191 @@
+
+import time
+
+from tcp_tests import logger
+from tcp_tests.helpers.log_helpers import pretty_repr
+
+LOG = logger.logger
+
+
+class ExecuteCommandsMixin(object):
+ """docstring for ExecuteCommands"""
+
+ def ensure_running_service(self, service_name, host, check_cmd,
+ state_running='start/running'):
+ """Check if the service_name running or try to restart it
+
+ :param service_name: name of the service that will be checked
+ :param node_name: node on which the service will be checked
+ :param check_cmd: shell command to ensure that the service is running
+ :param state_running: string for check the service state
+ """
+ cmd = "service {0} status | grep -q '{1}'".format(
+ service_name, state_running)
+ with self._underlay.remote(host=host) as remote:
+ result = remote.execute(cmd)
+ if result.exit_code != 0:
+ LOG.info("{0} is not in running state on the node {1},"
+ " trying to start".format(service_name, host))
+ cmd = ("service {0} stop;"
+ " sleep 3; killall -9 {0};"
+ "service {0} start; sleep 5;"
+ .format(service_name))
+ remote.execute(cmd)
+
+ remote.execute(check_cmd)
+ remote.execute(check_cmd)
+
+ def execute_commands(self, commands, label="Command"):
+ """Execute a sequence of commands
+
+ Main propose is to implement workarounds for salt formulas like:
+ - exit_code == 0 when there are actual failures
+ - salt_master and/or salt_minion stop working after executing a formula
+ - a formula fails at first run, but completes at next runs
+
+ :param label: label of the current sequence of the commands, for log
+ :param commands: list of dicts with the following data:
+ commands = [
+ ...
+ {
+ # Required:
+ 'cmd': 'shell command(s) to run',
+ 'node_name': 'name of the node to run the command(s)',
+ # Optional:
+ 'description': 'string with a readable command description',
+ 'retry': {
+ 'count': int, # How many times should be run the command
+ # until success
+ 'delay': int, # Delay between tries in seconds
+ },
+ 'skip_fail': bool # If True - continue with the next step
+ # without failure even if count number
+ # is reached.
+ # If False - rise an exception (default)
+ },
+ ...
+ ]
+ """
+ for n, step in enumerate(commands):
+ # Required fields
+ cmd = step.get('cmd')
+ do = step.get('do')
+ # node_name = step.get('node_name')
+ # Optional fields
+ description = step.get('description', cmd)
+ # retry = step.get('retry', {'count': 1, 'delay': 1})
+ # retry_count = retry.get('count', 1)
+ # retry_delay = retry.get('delay', 1)
+ # skip_fail = step.get('skip_fail', False)
+
+ msg = "[ {0} #{1} ] {2}".format(label, n + 1, description)
+ LOG.info("\n\n{0}\n{1}".format(msg, '=' * len(msg)))
+
+ if cmd:
+ self.execute_command(step)
+ elif do:
+ self.command2(step)
+
+ def execute_command(self, step):
+ # Required fields
+ cmd = step.get('cmd')
+ node_name = step.get('node_name')
+ # Optional fields
+ description = step.get('description', cmd)
+ retry = step.get('retry', {'count': 1, 'delay': 1})
+ retry_count = retry.get('count', 1)
+ retry_delay = retry.get('delay', 1)
+ skip_fail = step.get('skip_fail', False)
+
+ with self._underlay.remote(node_name=node_name) as remote:
+
+ for x in range(retry_count, 0, -1):
+ time.sleep(3)
+ result = remote.execute(cmd, verbose=True)
+
+ # Workaround of exit code 0 from salt in case of failures
+ failed = 0
+ for s in result['stdout']:
+ if s.startswith("Failed:"):
+ failed += int(s.split("Failed:")[1])
+
+ if result.exit_code != 0:
+ time.sleep(retry_delay)
+ LOG.info(
+ " === RETRY ({0}/{1}) ========================="
+ .format(x - 1, retry_count))
+ elif failed != 0:
+ LOG.error(
+ " === SALT returned exit code = 0 while "
+ "there are failed modules! ===")
+ LOG.info(
+ " === RETRY ({0}/{1}) ======================="
+ .format(x - 1, retry_count))
+ else:
+ if self._config.salt.salt_master_host != '0.0.0.0':
+ # Workarounds for crashed services
+ self.ensure_running_service(
+ "salt-master",
+ self._config.salt.salt_master_host,
+ "salt-call pillar.items",
+ 'active (running)') # Hardcoded for now
+ self.ensure_running_service(
+ "salt-minion",
+ self._config.salt.salt_master_host,
+ "salt 'cfg01*' pillar.items",
+ "active (running)") # Hardcoded for now
+ break
+
+ if x == 1 and skip_fail is False:
+ # In the last retry iteration, raise an exception
+ raise Exception("Step '{0}' failed"
+ .format(description))
+
+ def command2(self, step):
+ # Required fields
+ do = step['do']
+ target = step['target']
+ state = step.get('state')
+ states = step.get('states')
+ # Optional fields
+ args = step.get('args')
+ kwargs = step.get('kwargs')
+ description = step.get('description', do)
+ retry = step.get('retry', {'count': 1, 'delay': 1})
+ retry_count = retry.get('count', 1)
+ retry_delay = retry.get('delay', 1)
+ skip_fail = step.get('skip_fail', False)
+
+ if not bool(state) ^ bool(states):
+ raise ValueError("You should use state or states in step")
+
+ for x in range(retry_count, 0, -1):
+ time.sleep(3)
+
+ method = getattr(self._salt, self._salt._map[do])
+ command_ret = method(tgt=target, state=state or states,
+ args=args, kwargs=kwargs)
+ command_ret = command_ret if \
+ isinstance(command_ret, list) else [command_ret]
+ results = [(r['return'][0], f) for r, f in command_ret]
+
+ # FIMME: Change to debug level
+ LOG.info(" === States output =======================\n"
+ "{}\n"
+ " =========================================".format(
+ pretty_repr([r for r, f in results])))
+
+ all_fails = [f for r, f in results if f]
+ if all_fails:
+ LOG.error("States finished with failures.\n{}".format(
+ all_fails))
+ time.sleep(retry_delay)
+ LOG.info(" === RETRY ({0}/{1}) ========================="
+ .format(x - 1, retry_count))
+ else:
+ break
+
+ if x == 1 and skip_fail is False:
+ # In the last retry iteration, raise an exception
+ raise Exception("Step '{0}' failed"
+ .format(description))
diff --git a/tcp_tests/managers/opencontrail_manager.py b/tcp_tests/managers/opencontrail_manager.py
index 3c6621c..108794d 100644
--- a/tcp_tests/managers/opencontrail_manager.py
+++ b/tcp_tests/managers/opencontrail_manager.py
@@ -12,31 +12,33 @@
# License for the specific language governing permissions and limitations
# under the License.
-class OpenContrailManager(object):
+from tcp_tests.managers.execute_commands import ExecuteCommandsMixin
+
+
+class OpenContrailManager(ExecuteCommandsMixin):
"""docstring for OpenstackManager"""
- __config = None
- __underlay = None
- __openstack_actions = None
+ _config = None
+ _underlay = None
+ _openstack_actions = None
def __init__(self, config, underlay, openstack_deployed):
- self.__config = config
- self.__underlay = underlay
- self.__openstack_actions = openstack_deployed
+ self._config = config
+ self._underlay = underlay
+ self._openstack_actions = openstack_deployed
super(OpenContrailManager, self).__init__()
- def prepare_tests(commands):
- self.__underlay.execute_commands(commands=commands,
- label="Prepare Juniper contrail-test")
+ def prepare_tests(self, commands):
+ self.execute_commands(commands=commands,
+ label="Prepare Juniper contrail-test")
- def run_tests(tags='', features=''):
+ def run_tests(self, tags='', features=''):
cmd = "salt 'ctl01*' grains.get fqdn|tail -n1"
- result = self.__underlay.check_call(
- cmd, host=self.__config.salt.salt_master_host)
+ result = self._underlay.check_call(
+ cmd, host=self._config.salt.salt_master_host)
ctl01_name = result['stdout'].strip()
-
cmd = '. /etc/contrail/openstackrc; cd /opt/contrail-test; ./run_tests.sh'
if tags != '':
cmd += ' --tags ' + tags
@@ -44,6 +46,6 @@
if features != '':
cmd += ' --features ' + features
- self.__underlay.check_call(
+ self._underlay.check_call(
cmd,
node_name=ctl01_name)
diff --git a/tcp_tests/managers/openstack_manager.py b/tcp_tests/managers/openstack_manager.py
index b8725bd..55af1e2 100644
--- a/tcp_tests/managers/openstack_manager.py
+++ b/tcp_tests/managers/openstack_manager.py
@@ -12,18 +12,22 @@
# License for the specific language governing permissions and limitations
# under the License.
-class OpenstackManager(object):
+from tcp_tests.managers.execute_commands import ExecuteCommandsMixin
+
+
+class OpenstackManager(ExecuteCommandsMixin):
"""docstring for OpenstackManager"""
- __config = None
- __underlay = None
+ _config = None
+ _underlay = None
- def __init__(self, config, underlay):
- self.__config = config
- self.__underlay = underlay
+ def __init__(self, config, underlay, salt):
+ self._config = config
+ self._underlay = underlay
+ self._salt = salt
super(OpenstackManager, self).__init__()
def install(self, commands):
- self.__underlay.execute_commands(commands=commands,
- label="Install OpenStack services")
- self.__config.openstack.openstack_installed = True
+ self.execute_commands(commands,
+ label='Install OpenStack services')
+ self._config.common_services.common_services_installed = True
diff --git a/tcp_tests/managers/saltmanager.py b/tcp_tests/managers/saltmanager.py
index 1a89616..b65c74f 100644
--- a/tcp_tests/managers/saltmanager.py
+++ b/tcp_tests/managers/saltmanager.py
@@ -11,26 +11,165 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
+# import time
-class SaltManager(object):
+from collections import defaultdict
+
+from datetime import datetime
+from pepper.libpepper import Pepper
+from tcp_tests import settings
+from tcp_tests import logger
+from tcp_tests.managers.execute_commands import ExecuteCommandsMixin
+
+LOG = logger.logger
+
+
+class SaltManager(ExecuteCommandsMixin):
"""docstring for SaltManager"""
- __config = None
- __underlay = None
+ _config = None
+ _underlay = None
+ _map = {
+ 'enforceState': 'enforce_state',
+ 'enforceStates': 'enforce_states',
+ 'runState': 'run_state',
+ 'runStates': 'run_states',
+ }
- def __init__(self, config, underlay):
- self.__config = config
- self.__underlay = underlay
-
+ def __init__(self, config, underlay, host=None, port='6969'):
+ self._config = config
+ self._underlay = underlay
+ self._port = port
+ self._host = host
+ self._api = None
+ self._user = settings.SALT_USER
+ self._password = settings.SALT_PASSWORD
+ self._salt = self
super(SaltManager, self).__init__()
def install(self, commands):
- if self.__config.salt.salt_master_host == '0.0.0.0':
- # Temporary workaround. Underlay should be extended with roles
- salt_nodes = self.__underlay.node_names()
- self.__config.salt.salt_master_host = \
- self.__underlay.host_by_node_name(salt_nodes[0])
+ if commands[0].get('do'):
+ self.install2(commands)
+ else:
+ self.install1(commands)
- self.__underlay.execute_commands(commands=commands,
- label="Install and configure salt")
+ def install1(self, commands):
+ if self._config.salt.salt_master_host == '0.0.0.0':
+ # Temporary workaround. Underlay should be extended with roles
+ salt_nodes = self._underlay.node_names()
+ self._config.salt.salt_master_host = \
+ self._underlay.host_by_node_name(salt_nodes[0])
+
+ # self._underlay.execute_commands(commands=commands,
+ # label="Install and configure salt")
+ self.execute_commands(commands=commands,
+ label="Install and configure salt")
+
+ def install2(self, commands):
+ if self._config.salt.salt_master_host == '0.0.0.0':
+ # Temporary workaround. Underlay should be extended with roles
+ salt_nodes = self._underlay.node_names()
+ self._config.salt.salt_master_host = \
+ self._underlay.host_by_node_name(salt_nodes[0])
+
+ # self.run_commands(commands=commands,
+ # label="Install and configure salt")
+ self.execute_commands(commands=commands,
+ label="Install and configure salt")
+
+ @property
+ def port(self):
+ return self._port
+
+ @property
+ def host(self):
+ if self._host:
+ return self._host
+ elif self._config.salt.salt_master_host == '0.0.0.0':
+ # Temporary workaround. Underlay should be extended with roles
+ salt_nodes = self._underlay.node_names()
+ self._config.salt.salt_master_host = \
+ self._underlay.host_by_node_name(salt_nodes[0])
+
+ return self._config.salt.salt_master_host
+
+ @property
+ def api(self):
+ def login():
+ LOG.info("Authentication in Salt API")
+ self._api.login(
+ username=self._user,
+ password=self._password,
+ eauth='pam')
+ return datetime.now()
+
+ if self._api:
+ if (datetime.now() - self.__session_start).seconds < 5 * 60:
+ return self._api
+ else:
+ # FIXXME: Change to debug
+ LOG.info("Session's expired")
+ self.__session_start = login()
+ return self._api
+
+ LOG.info("Connect to Salt API")
+ url = "http://{host}:{port}".format(
+ host=self.host, port=self.port)
+ self._api = Pepper(url)
+ self.__session_start = login()
+ return self._api
+
+ def local(self, tgt, fun, args=None, kwargs=None):
+ return self.api.local(tgt, fun, args, kwargs, expr_form='compound')
+
+ def local_async(self, tgt, fun, args=None, kwargs=None):
+ return self.api.local_async(tgt, fun, args, kwargs)
+
+ def lookup_result(self, jid):
+ return self.api.lookup_jid(jid)
+
+ def check_result(self, r):
+ if len(r.get('return', [])) == 0:
+ raise LookupError("Result is empty or absent")
+
+ result = r['return'][0]
+ LOG.info("Job has result for %s nodes", result.keys())
+ fails = defaultdict(list)
+ for h in result:
+ host_result = result[h]
+ LOG.info("On %s executed:", h)
+ if isinstance(host_result, list):
+ fails[h].append(host_result)
+ continue
+ for t in host_result:
+ task = host_result[t]
+ if task['result'] is False:
+ fails[h].append(task)
+ LOG.error("%s - %s", t, task['result'])
+ else:
+ LOG.info("%s - %s", t, task['result'])
+
+ return fails if fails else None
+
+ def enforce_state(self, tgt, state, args=None, kwargs=None):
+ r = self.local(tgt=tgt, fun='state.sls', args=state)
+ f = self.check_result(r)
+ return r, f
+
+ def enforce_states(self, tgt, state, args=None, kwargs=None):
+ rets = []
+ for s in state:
+ r = self.enforce_state(tgt=tgt, state=s)
+ rets.append(r)
+ return rets
+
+ def run_state(self, tgt, state, args=None, kwargs=None):
+ return self.local(tgt=tgt, fun=state, args=args, kwargs=kwargs), None
+
+ def run_states(self, tgt, state, args=None, kwargs=None):
+ rets = []
+ for s in state:
+ r = self.run_state(tgt=tgt, state=s, args=args, kwargs=kwargs)
+ rets.append(r)
+ return rets
diff --git a/tcp_tests/managers/underlay_ssh_manager.py b/tcp_tests/managers/underlay_ssh_manager.py
index f4c5df3..18095c6 100644
--- a/tcp_tests/managers/underlay_ssh_manager.py
+++ b/tcp_tests/managers/underlay_ssh_manager.py
@@ -13,7 +13,6 @@
# under the License.
import random
-import time
from devops.helpers import helpers
from devops.helpers import ssh_client
@@ -375,116 +374,3 @@
}
template = utils.render_template(file_path, options=options)
return yaml.load(template)
-
- def ensure_running_service(self, service_name, host, check_cmd,
- state_running='start/running'):
- """Check if the service_name running or try to restart it
-
- :param service_name: name of the service that will be checked
- :param node_name: node on which the service will be checked
- :param check_cmd: shell command to ensure that the service is running
- :param state_running: string for check the service state
- """
- cmd = "service {0} status | grep -q '{1}'".format(
- service_name, state_running)
- with self.remote(host=host) as remote:
- result = remote.execute(cmd)
- if result.exit_code != 0:
- LOG.info("{0} is not in running state on the node {1},"
- " trying to start".format(service_name, host))
- cmd = ("service {0} stop;"
- " sleep 3; killall -9 {0};"
- "service {0} start; sleep 5;"
- .format(service_name))
- remote.execute(cmd)
-
- remote.execute(check_cmd)
- remote.execute(check_cmd)
-
- def execute_commands(self, commands, label="Command"):
- """Execute a sequence of commands
-
- Main propose is to implement workarounds for salt formulas like:
- - exit_code == 0 when there are actual failures
- - salt_master and/or salt_minion stop working after executing a formula
- - a formula fails at first run, but completes at next runs
-
- :param label: label of the current sequence of the commands, for log
- :param commands: list of dicts with the following data:
- commands = [
- ...
- {
- # Required:
- 'cmd': 'shell command(s) to run',
- 'node_name': 'name of the node to run the command(s)',
- # Optional:
- 'description': 'string with a readable command description',
- 'retry': {
- 'count': int, # How many times should be run the command
- # until success
- 'delay': int, # Delay between tries in seconds
- },
- 'skip_fail': bool # If True - continue with the next step
- # without failure even if count number
- # is reached.
- # If False - rise an exception (default)
- },
- ...
- ]
- """
- for n, step in enumerate(commands):
- # Required fields
- cmd = step.get('cmd')
- node_name = step.get('node_name')
- # Optional fields
- description = step.get('description', cmd)
- retry = step.get('retry', {'count': 1, 'delay': 1})
- retry_count = retry.get('count', 1)
- retry_delay = retry.get('delay', 1)
- skip_fail = step.get('skip_fail', False)
-
- msg = "[ {0} #{1} ] {2}".format(label, n+1, description)
- LOG.info("\n\n{0}\n{1}".format(msg, '=' * len(msg)))
-
- with self.remote(node_name=node_name) as remote:
-
- for x in range(retry_count, 0, -1):
- time.sleep(3)
- result = remote.execute(cmd, verbose=True)
-
- # Workaround of exit code 0 from salt in case of failures
- failed = 0
- for s in result['stdout']:
- if s.startswith("Failed:"):
- failed += int(s.split("Failed:")[1])
- if 'Minion did not return. [No response]' in s:
- failed += 1
-
- if result.exit_code != 0:
- time.sleep(retry_delay)
- LOG.info(" === RETRY ({0}/{1}) ========================="
- .format(x-1, retry_count))
- elif failed != 0:
- LOG.error(" === SALT returned exit code = 0 while "
- "there are failed modules! ===")
- LOG.info(" === RETRY ({0}/{1}) ======================="
- .format(x-1, retry_count))
- else:
- if self.__config.salt.salt_master_host != '0.0.0.0':
- # Workarounds for crashed services
- self.ensure_running_service(
- "salt-master",
- self.__config.salt.salt_master_host,
- "salt-call pillar.items",
- 'active (running)') # Hardcoded for now
- self.ensure_running_service(
- "salt-minion",
- self.__config.salt.salt_master_host,
- "salt 'cfg01*' pillar.items",
- "active (running)") # Hardcoded for now
- break
-
- if x == 1 and skip_fail == False:
- # In the last retry iteration, raise an exception
- raise Exception("Step '{0}' failed"
- .format(description))