Working workaround for PROD-10894
diff --git a/tcp_tests/managers/execute_commands.py b/tcp_tests/managers/execute_commands.py
index de9ab3b..6b52953 100644
--- a/tcp_tests/managers/execute_commands.py
+++ b/tcp_tests/managers/execute_commands.py
@@ -113,7 +113,7 @@
# Workaround of exit code 0 from salt in case of failures
failed = 0
- for s in result['stdout']:
+ for s in result['stdout'] + result['stderr']:
if s.startswith("Failed:"):
failed += int(s.split("Failed:")[1])
if 'Minion did not return. [No response]' in s:
diff --git a/tcp_tests/templates/virtual-mcp-ocata-dvr/salt.yaml b/tcp_tests/templates/virtual-mcp-ocata-dvr/salt.yaml
index c952197..8df7f05 100644
--- a/tcp_tests/templates/virtual-mcp-ocata-dvr/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-ocata-dvr/salt.yaml
@@ -233,14 +233,16 @@
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' state.sls salt.master.service;
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 2, delay: 5}
skip_fail: false
- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
- cmd: systemctl restart salt-minion;
- salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt;
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' state.sls salt;
+ salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' saltutil.sync_all
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 5}
+ retry: {count: 5, delay: 5}
skip_fail: false
- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
@@ -263,7 +265,13 @@
skip_fail: false
- description: Show reclass-salt --top
- cmd: reclass-salt --top; salt-call --no-color state.sls salt.minion.cert -l info;
+ cmd: reclass-salt --top
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Execute salt.minion.cert
+ cmd: salt-call --no-color state.sls salt.minion.cert -l info;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp-ocata-ovs/salt.yaml b/tcp_tests/templates/virtual-mcp-ocata-ovs/salt.yaml
index 6c58458..c370134 100644
--- a/tcp_tests/templates/virtual-mcp-ocata-ovs/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-ocata-ovs/salt.yaml
@@ -233,14 +233,16 @@
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' state.sls salt.master.service;
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 2, delay: 5}
skip_fail: false
- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
- cmd: systemctl restart salt-minion;
- salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt;
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' state.sls salt;
+ salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' saltutil.sync_all
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 5}
+ retry: {count: 5, delay: 5}
skip_fail: false
- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
@@ -263,7 +265,13 @@
skip_fail: false
- description: Show reclass-salt --top
- cmd: reclass-salt --top; salt-call --no-color state.sls salt.minion.cert -l info;
+ cmd: reclass-salt --top
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Execute salt.minion.cert
+ cmd: salt-call --no-color state.sls salt.minion.cert -l info;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp11-dvr/salt.yaml b/tcp_tests/templates/virtual-mcp11-dvr/salt.yaml
index 0351c4d..b9bb5b9 100644
--- a/tcp_tests/templates/virtual-mcp11-dvr/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp11-dvr/salt.yaml
@@ -233,14 +233,16 @@
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' state.sls salt.master.service;
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 2, delay: 5}
skip_fail: false
- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
- cmd: systemctl restart salt-minion;
- salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt;
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' state.sls salt;
+ salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' saltutil.sync_all
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 5}
+ retry: {count: 5, delay: 5}
skip_fail: false
- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
@@ -263,7 +265,13 @@
skip_fail: false
- description: Show reclass-salt --top
- cmd: reclass-salt --top; salt-call --no-color state.sls salt.minion.cert -l info;
+ cmd: reclass-salt --top
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Execute salt.minion.cert
+ cmd: salt-call --no-color state.sls salt.minion.cert -l info;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp11-ovs-dpdk/salt.yaml b/tcp_tests/templates/virtual-mcp11-ovs-dpdk/salt.yaml
index 603cd5f..f39a27e 100644
--- a/tcp_tests/templates/virtual-mcp11-ovs-dpdk/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp11-ovs-dpdk/salt.yaml
@@ -233,14 +233,16 @@
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' state.sls salt.master.service;
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 2, delay: 5}
skip_fail: false
- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
- cmd: systemctl restart salt-minion;
- salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt;
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' state.sls salt;
+ salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' saltutil.sync_all
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 5}
+ retry: {count: 5, delay: 5}
skip_fail: false
- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
@@ -263,7 +265,13 @@
skip_fail: false
- description: Show reclass-salt --top
- cmd: reclass-salt --top; salt-call --no-color state.sls salt.minion.cert -l info;
+ cmd: reclass-salt --top
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Execute salt.minion.cert
+ cmd: salt-call --no-color state.sls salt.minion.cert -l info;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp11-ovs.new/salt.yaml b/tcp_tests/templates/virtual-mcp11-ovs.new/salt.yaml
index f85affe..113e064 100644
--- a/tcp_tests/templates/virtual-mcp11-ovs.new/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp11-ovs.new/salt.yaml
@@ -233,14 +233,16 @@
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' state.sls salt.master.service;
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 2, delay: 5}
skip_fail: false
- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
- cmd: systemctl restart salt-minion;
- salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt;
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' state.sls salt;
+ salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' saltutil.sync_all
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 5}
+ retry: {count: 5, delay: 5}
skip_fail: false
- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
@@ -263,7 +265,13 @@
skip_fail: false
- description: Show reclass-salt --top
- cmd: reclass-salt --top; salt-call --no-color state.sls salt.minion.cert -l info;
+ cmd: reclass-salt --top
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Execute salt.minion.cert
+ cmd: salt-call --no-color state.sls salt.minion.cert -l info;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp11-ovs/salt.yaml b/tcp_tests/templates/virtual-mcp11-ovs/salt.yaml
index f85affe..113e064 100644
--- a/tcp_tests/templates/virtual-mcp11-ovs/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp11-ovs/salt.yaml
@@ -233,14 +233,16 @@
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' state.sls salt.master.service;
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 2, delay: 5}
skip_fail: false
- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
- cmd: systemctl restart salt-minion;
- salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt;
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' state.sls salt;
+ salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@salt:master' saltutil.sync_all
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 2, delay: 5}
+ retry: {count: 5, delay: 5}
skip_fail: false
- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
@@ -263,7 +265,13 @@
skip_fail: false
- description: Show reclass-salt --top
- cmd: reclass-salt --top; salt-call --no-color state.sls salt.minion.cert -l info;
+ cmd: reclass-salt --top
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Execute salt.minion.cert
+ cmd: salt-call --no-color state.sls salt.minion.cert -l info;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false