Catch 'no responce' from minions error
When salt.minion formula is executed, 'salt-minion' service restarts
on the nodes and doesn't allow to complete all necessary states
(for example, salt.minion.cert on prx* nodes was never executed).
Also, join executing 'linux' and 'salt.minion' states for other nodes
into single steps instead of per-role execution.
diff --git a/tcp_tests/managers/underlay_ssh_manager.py b/tcp_tests/managers/underlay_ssh_manager.py
index b22566d..f4c5df3 100644
--- a/tcp_tests/managers/underlay_ssh_manager.py
+++ b/tcp_tests/managers/underlay_ssh_manager.py
@@ -457,6 +457,8 @@
for s in result['stdout']:
if s.startswith("Failed:"):
failed += int(s.split("Failed:")[1])
+ if 'Minion did not return. [No response]' in s:
+ failed += 1
if result.exit_code != 0:
time.sleep(retry_delay)
diff --git a/tcp_tests/templates/salt/virtual-mcp11-dvr-salt.yaml b/tcp_tests/templates/salt/virtual-mcp11-dvr-salt.yaml
index 4490973..4f1381a 100644
--- a/tcp_tests/templates/salt/virtual-mcp11-dvr-salt.yaml
+++ b/tcp_tests/templates/salt/virtual-mcp11-dvr-salt.yaml
@@ -174,9 +174,9 @@
master: {{ config.salt.salt_master_host }}
EOF
eatmydata apt-get install -y salt-minion;
- echo "Showing system info and metadata ...";
+ echo "Check for system info and metadata availability ...";
salt-call --no-color grains.items;
- salt-call --no-color pillar.data;
+ salt-call --no-color pillar.items;
node_name: {{ ssh['node_name'] }}
retry: {count: 1, delay: 1}
skip_fail: false
@@ -220,7 +220,7 @@
- description: Run 'reclass' formula on cfg01
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' reclass;
- salt-call --no-color state.sls salt.master;
+ salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt.master;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: true
@@ -269,64 +269,25 @@
# Bootstrap all nodes
-- description: Configure linux on controllers
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'ctl*' state.sls
- linux
+- description: Configure linux on other nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '* and not
+ cfg01*' state.sls linux
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
-- description: Configure linux on proxy
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'prx*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
- skip_fail: false
-
-- description: Configure linux on gtw
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'gtw*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 20}
- skip_fail: false
-
-- description: Configure linux on cmp
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'cmp*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 20}
- skip_fail: false
-
- description: Configure openssh on all nodes
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '* and not
- cfg*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
+ cfg01*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
-C '* and not cfg*' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
yes/' /etc/ssh/sshd_config && service ssh restart"
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
-- description: Configure salt.minion on ctl
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'ctl*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-- description: Configure salt.minion on prx
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'prx*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-
-- description: Configure salt.minion on gtw
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'gtw*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-- description: Configure salt.minion on cmp
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'cmp*' state.sls salt.minion
+- description: Configure salt.minion on other nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '* and not
+ cfg01*' state.sls salt.minion
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 3, delay: 5}
skip_fail: false
diff --git a/tcp_tests/templates/salt/virtual-mcp11-ovs-dpdk-salt.yaml b/tcp_tests/templates/salt/virtual-mcp11-ovs-dpdk-salt.yaml
index 6d1f7c4..aff7c32 100644
--- a/tcp_tests/templates/salt/virtual-mcp11-ovs-dpdk-salt.yaml
+++ b/tcp_tests/templates/salt/virtual-mcp11-ovs-dpdk-salt.yaml
@@ -174,9 +174,9 @@
master: {{ config.salt.salt_master_host }}
EOF
eatmydata apt-get install -y salt-minion;
- echo "Showing system info and metadata ...";
+ echo "Check for system info and metadata availability ...";
salt-call --no-color grains.items;
- salt-call --no-color pillar.data;
+ salt-call --no-color pillar.items;
node_name: {{ ssh['node_name'] }}
retry: {count: 1, delay: 1}
skip_fail: false
@@ -220,7 +220,7 @@
- description: Run 'reclass' formula on cfg01
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' reclass;
- salt-call --no-color state.sls salt.master;
+ salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt.master;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: true
@@ -269,64 +269,25 @@
# Bootstrap all nodes
-- description: Configure linux on controllers
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'ctl*' state.sls
- linux
+- description: Configure linux on other nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '* and not
+ cfg01*' state.sls linux
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
-- description: Configure linux on proxy
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'prx*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
- skip_fail: false
-
-- description: Configure linux on gtw
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'gtw*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 20}
- skip_fail: false
-
-- description: Configure linux on cmp
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'cmp*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 20}
- skip_fail: false
-
- description: Configure openssh on all nodes
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '* and not
- cfg*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
+ cfg01*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
-C '* and not cfg*' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
yes/' /etc/ssh/sshd_config && service ssh restart"
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
-- description: Configure salt.minion on ctl
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'ctl*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-- description: Configure salt.minion on prx
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'prx*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-
-- description: Configure salt.minion on gtw
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'gtw*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-- description: Configure salt.minion on cmp
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'cmp*' state.sls salt.minion
+- description: Configure salt.minion on other nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '* and not
+ cfg01*' state.sls salt.minion
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 3, delay: 5}
skip_fail: false
@@ -366,4 +327,3 @@
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 10}
skip_fail: false
-
diff --git a/tcp_tests/templates/salt/virtual-mcp11-ovs-salt.yaml b/tcp_tests/templates/salt/virtual-mcp11-ovs-salt.yaml
index 5d47b9e..483d37f 100644
--- a/tcp_tests/templates/salt/virtual-mcp11-ovs-salt.yaml
+++ b/tcp_tests/templates/salt/virtual-mcp11-ovs-salt.yaml
@@ -174,9 +174,9 @@
master: {{ config.salt.salt_master_host }}
EOF
eatmydata apt-get install -y salt-minion;
- echo "Showing system info and metadata ...";
+ echo "Check for system info and metadata availability ...";
salt-call --no-color grains.items;
- salt-call --no-color pillar.data;
+ salt-call --no-color pillar.items;
node_name: {{ ssh['node_name'] }}
retry: {count: 1, delay: 1}
skip_fail: false
@@ -220,7 +220,7 @@
- description: Run 'reclass' formula on cfg01
cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
-C 'I@salt:master' reclass;
- salt-call --no-color state.sls salt.master;
+ salt-call --hard-crash --state-output=mixed --state-verbose=False state.sls salt.master;
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: true
@@ -269,64 +269,25 @@
# Bootstrap all nodes
-- description: Configure linux on controllers
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'ctl*' state.sls
- linux
+- description: Configure linux on other nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '* and not
+ cfg01*' state.sls linux
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
-- description: Configure linux on proxy
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'prx*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
- skip_fail: false
-
-- description: Configure linux on gtw
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'gtw*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 20}
- skip_fail: false
-
-- description: Configure linux on cmp
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'cmp*' state.sls
- linux
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 20}
- skip_fail: false
-
- description: Configure openssh on all nodes
cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '* and not
- cfg*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
+ cfg01*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
-C '* and not cfg*' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
yes/' /etc/ssh/sshd_config && service ssh restart"
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 5}
skip_fail: false
-- description: Configure salt.minion on ctl
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'ctl*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-- description: Configure salt.minion on prx
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'prx*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-
-- description: Configure salt.minion on gtw
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'gtw*' state.sls salt.minion
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 3, delay: 5}
- skip_fail: false
-
-- description: Configure salt.minion on cmp
- cmd: salt --hard-crash --state-output=mixed --state-verbose=False 'cmp*' state.sls salt.minion
+- description: Configure salt.minion on other nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '* and not
+ cfg01*' state.sls salt.minion
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 3, delay: 5}
skip_fail: false
@@ -366,4 +327,3 @@
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 10}
skip_fail: false
-