Fix retry counter delay #2
Change-Id: Iab3025ca9c6cac7a67197933cdd672185083b1b0
diff --git a/tcp_tests/managers/execute_commands.py b/tcp_tests/managers/execute_commands.py
index 3c01867..af30bec 100644
--- a/tcp_tests/managers/execute_commands.py
+++ b/tcp_tests/managers/execute_commands.py
@@ -64,18 +64,20 @@
# skip_fail = step.get('skip_fail', False)
msg = "[ {0} #{1} ] {2}".format(label, n + 1, description)
- LOG.info("\n\n{0}\n{1}".format(msg, '=' * len(msg)))
+ log_msg = "\n\n{0}\n{1}".format(msg, '=' * len(msg))
if action_cmd:
- self.execute_command(step)
+ self.execute_command(step, msg)
elif action_do:
- self.command2(step)
+ self.command2(step, msg)
elif action_upload:
+ LOG.info(log_msg)
self.action_upload(step)
elif action_download:
+ LOG.info(log_msg)
self.action_download(step)
- def execute_command(self, step):
+ def execute_command(self, step, msg):
# Required fields
cmd = step.get('cmd')
node_name = step.get('node_name')
@@ -90,7 +92,13 @@
for x in range(retry_count, 0, -1):
time.sleep(3)
- result = remote.execute(cmd, verbose=True)
+
+ retry_msg = ' (try {0} of {1}, skip_fail={2})'.format(
+ retry_count - x + 1, retry_count, skip_fail)
+ LOG.info("\n\n{0}\n{1}".format(
+ msg + retry_msg, '=' * len(msg + retry_msg)))
+
+ result = remote.execute('set -ex; ' + cmd, verbose=True)
# Workaround of exit code 0 from salt in case of failures
failed = 0
@@ -106,16 +114,11 @@
if result.exit_code != 0:
time.sleep(retry_delay)
- LOG.info(
- " === RETRY ({0}/{1}) ========================="
- .format(x - 1, retry_count))
elif failed != 0:
LOG.error(
" === SALT returned exit code = 0 while "
"there are failed modules! ===")
- LOG.info(
- " === RETRY ({0}/{1}) ======================="
- .format(x - 1, retry_count))
+ time.sleep(retry_delay)
else:
break
@@ -124,7 +127,7 @@
raise Exception("Step '{0}' failed"
.format(description))
- def command2(self, step):
+ def command2(self, step, msg):
# Required fields
do = step['do']
target = step['target']
@@ -145,6 +148,11 @@
for x in range(retry_count, 0, -1):
time.sleep(3)
+ retry_msg = ' (try {0} of {1}, skip_fail={2})'.format(
+ retry_count - x + 1, retry_count, skip_fail)
+ LOG.info("\n\n{0}\n{1}".format(
+ msg + retry_msg, '=' * len(msg + retry_msg)))
+
method = getattr(self._salt, self._salt._map[do])
command_ret = method(tgt=target, state=state or states,
args=args, kwargs=kwargs)
@@ -163,8 +171,6 @@
LOG.error("States finished with failures.\n{}".format(
all_fails))
time.sleep(retry_delay)
- LOG.info(" === RETRY ({0}/{1}) ========================="
- .format(x - 1, retry_count))
else:
break
diff --git a/tcp_tests/managers/underlay_ssh_manager.py b/tcp_tests/managers/underlay_ssh_manager.py
index 3b64300..54984ae 100644
--- a/tcp_tests/managers/underlay_ssh_manager.py
+++ b/tcp_tests/managers/underlay_ssh_manager.py
@@ -478,7 +478,7 @@
with self.remote(host=master_host) as master:
# dump files
LOG.info("Archive artifacts on all nodes")
- master.check_call("salt '*' cmd.run '{0}'".format(dump_commands),
+ master.check_call('salt "*" cmd.run "{0}"'.format(dump_commands),
raise_on_err=False)
# create target dir for archives
diff --git a/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/oss.yaml b/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/oss.yaml
index 11155ed..f64b373 100644
--- a/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/oss.yaml
+++ b/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/oss.yaml
@@ -182,7 +182,7 @@
'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cluster_vip_address);
while true; do curl -sf ldap://${CICD_CONTROL_ADDRESS} && break; sleep 2; done'
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 6, delay: 20}
+ retry: {count: 10, delay: 30}
skip_fail: false
- description: "Setup OpenLDAP"
@@ -200,7 +200,7 @@
'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cluster_vip_address);
while true; do curl -sf http://${CICD_CONTROL_ADDRESS}:8080/config/server/version && break; sleep 2; done'
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 10}
+ retry: {count: 10, delay: 30}
skip_fail: false
- description: "Setup Gerrit"
@@ -222,7 +222,7 @@
sleep 2;
done'
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 10}
+ retry: {count: 10, delay: 30}
skip_fail: false
- description: "Setup Jenkins"