add workarounds for ssh access and salt failed status checking
diff --git a/tcp_tests/templates/tcpcloud-default.yaml b/tcp_tests/templates/tcpcloud-default.yaml
index 43dcf4a..2523790 100644
--- a/tcp_tests/templates/tcpcloud-default.yaml
+++ b/tcp_tests/templates/tcpcloud-default.yaml
@@ -8,7 +8,7 @@
template:
devops_settings:
- env_name: !os_env ENV_NAME, tcpcloud-mk20
+ env_name: !os_env ENV_NAME, tcpcloud-mk22
address_pools:
public-pool01:
@@ -65,11 +65,11 @@
format: qcow2
nodes:
- - name: cfg01.mk20-lab-advanced.local
+ - name: cfg01.mk22-lab-advanced.local
role: salt_master
params:
- vcpu: !os_env SLAVE_NODE_CPU, 2
- memory: !os_env SLAVE_NODE_MEMORY, 4096
+ vcpu: !os_env SLAVE_NODE_CPU, 4
+ memory: !os_env SLAVE_NODE_MEMORY, 8192
boot:
- hd
cloud_init_volume_name: iso
@@ -103,7 +103,7 @@
networks:
- private
- - name: ctl01.mk20-lab-advanced.local
+ - name: ctl01.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 4
@@ -141,7 +141,7 @@
networks:
- private
- - name: ctl02.mk20-lab-advanced.local
+ - name: ctl02.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 4
@@ -179,7 +179,7 @@
networks:
- private
- - name: ctl03.mk20-lab-advanced.local
+ - name: ctl03.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 4
@@ -217,7 +217,7 @@
networks:
- private
- - name: web01.mk20-lab-advanced.local
+ - name: web01.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 1
@@ -255,7 +255,7 @@
networks:
- private
- - name: cmp01.mk20-lab-advanced.local
+ - name: cmp01.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 4
@@ -293,7 +293,7 @@
networks:
- private
- - name: mtr01.mk20-lab-advanced.local
+ - name: mtr01.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 1
@@ -331,7 +331,7 @@
networks:
- private
- - name: mon01.mk20-lab-advanced.local
+ - name: mon01.mk22-lab-advanced.local
role: salt_minion
params:
vcpu: !os_env SLAVE_NODE_CPU, 1
diff --git a/tcp_tests/tests/system/test_tcp_install.py b/tcp_tests/tests/system/test_tcp_install.py
index 75addde..130b2b6 100644
--- a/tcp_tests/tests/system/test_tcp_install.py
+++ b/tcp_tests/tests/system/test_tcp_install.py
@@ -36,263 +36,276 @@
@pytest.mark.steps({
1: {
'cmd': salt_cmd + "'cfg01*' state.sls linux",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
2: {
- 'cmd': salt_cmd + "'cfg01*' state.sls openssh",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': (salt_cmd + "'cfg01*' state.sls openssh;"
+ "sed -i 's/PasswordAuthentication no/"
+ "PasswordAuthentication yes/' "
+ "/etc/ssh/sshd_config && service ssh restart"),
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
3: {
'cmd': "echo ' StrictHostKeyChecking no' >> /root/.ssh/config",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 1, 'delay': 1},
'skip_fail': False,
},
4: {
'cmd': salt_cmd + "'cfg01*' state.sls salt",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
5: {
'cmd': salt_cmd + "'cfg01*' state.sls reclass.storage",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
6: {
'cmd': salt_cmd + "'*' saltutil.refresh_pillar",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
7: {
'cmd': salt_cmd + "'ctl*' state.sls ntp",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
8: {
- 'cmd': salt_cmd + "'ctl*' state.sls linux,salt.minion,openssh",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': (salt_cmd + "'ctl*' state.sls linux,salt.minion,openssh;"
+ + salt_cmd + "'ctl*' cmd.run "
+ "\"sed -i 's/PasswordAuthentication no/"
+ "PasswordAuthentication yes/' /etc/ssh/sshd_config && "
+ "service ssh restart\""),
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
9: {
'cmd': salt_cmd + "'ctl01*' state.sls keepalived",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
10: {
'cmd': salt_cmd + "'ctl01*' cmd.run 'ip a'",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
11: {
'cmd': salt_cmd + "'ctl0[23].*' state.sls keepalived",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
12: {
'cmd': salt_cmd + "'ctl*' state.sls glusterfs.server.service",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
13: {
'cmd': salt_call_cmd + "state.sls glusterfs.server.setup",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
14: {
'cmd': "gluster peer status",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
15: {
'cmd': "gluster volume status",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
16: {
'cmd': salt_cmd + "'ctl*' state.sls rabbitmq",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
17: {
- 'cmd': salt_call_cmd + "state.sls galera",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "apt-get -y --force-yes install salt-formula-galera",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
18: {
- 'cmd': salt_cmd + "'ctl0[23]*' state.sls galera",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_call_cmd + "state.sls galera",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
19: {
- 'cmd': salt_cmd + "'ctl01*' mysql.status | grep -A1 'wsrep_incoming_addresses:'",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl0[23]*' state.sls galera",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
20: {
- 'cmd': salt_cmd + "'ctl*' state.sls haproxy",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl01*' mysql.status | grep -A1 'wsrep_incoming_addresses:'",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
21: {
- 'cmd': salt_cmd + "'ctl*' cmd.run 'netstat -tulnp | grep 3306'",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' state.sls haproxy",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
22: {
- 'cmd': salt_call_cmd + "state.sls memcached,keystone",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' cmd.run 'netstat -tulnp | grep 3306'",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
23: {
'cmd': salt_call_cmd + "state.sls memcached,keystone",
- 'node_name': 'ctl02.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
24: {
'cmd': salt_call_cmd + "state.sls memcached,keystone",
- 'node_name': 'ctl03.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl02.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
25: {
- 'cmd': "source ~/keystonerc; keystone user-list",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_call_cmd + "state.sls memcached,keystone",
+ 'node_name': 'ctl03.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
26: {
- 'cmd': "source ~/keystonerc; keystone tenant-list",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "source ~/keystonerc; keystone user-list",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
27: {
- 'cmd': "source ~/keystonerc; keystone endpoint-list",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "source ~/keystonerc; keystone tenant-list",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
28: {
- 'cmd': salt_cmd + "'ctl*' state.sls glance",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "source ~/keystonerc; keystone endpoint-list",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
29: {
- 'cmd': salt_cmd + "'ctl*' state.sls glusterfs.client",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' state.sls glance",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
30: {
- 'cmd': salt_cmd + "'ctl*' cmd.run 'df -h'",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' state.sls glusterfs.client",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
31: {
- 'cmd': salt_call_cmd + "state.sls keystone",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' cmd.run 'df -h'",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
32: {
+ 'cmd': salt_call_cmd + "state.sls keystone",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
+ 'retry': {'count': 3, 'delay': 5},
+ 'skip_fail': False,
+ },
+ 33: {
'cmd': ("source ~/keystonerc;"
"wget http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-i386-disk.img;"
"glance image-create --name 'cirros-0.3.4'"
" --disk-format qcow2 --container-format bare"
" --progress --file /root/cirros-0.3.4-i386-disk.img;"
"glance image-list;"),
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
- 'retry': {'count': 3, 'delay': 5},
- 'skip_fail': False,
- },
- 33: {
- 'cmd': salt_cmd + "'ctl*' cmd.run 'ls -al /var/lib/keystone/fernet-keys' ",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
34: {
- 'cmd': salt_cmd + "'ctl*' cinder",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' cmd.run 'ls -al /var/lib/keystone/fernet-keys' ",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
35: {
- 'cmd': salt_cmd + "'ctl*' nova",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' cinder",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
36: {
- 'cmd': "source ~/keystonerc; cinder list",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' nova",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
37: {
- 'cmd': "source ~/keystonerc; nova-manage service list",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "source ~/keystonerc; cinder list",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
38: {
- 'cmd': "source ~/keystonerc; nova list",
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "source ~/keystonerc; nova-manage service list",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
39: {
- 'cmd': salt_cmd + "'ctl*' state.sls neutron",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': "source ~/keystonerc; nova list",
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
40: {
- 'cmd': ("source ~/keystonerc;"
- "neutron net-create --router:external=true --shared external;"
- "neutron subnet-create external 10.177.0.0/24;"
- "neutron floatingip-create;"),
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': salt_cmd + "'ctl*' state.sls neutron",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
41: {
- 'cmd': salt_cmd + "'ctl*' state.sls opencontrail.database",
- 'node_name': 'cfg01.mk20-lab-advanced.local', # hardcoded for now
+ 'cmd': ("source ~/keystonerc;"
+ "neutron net-create --router:external=true --shared external;"
+ "neutron subnet-create external 10.177.0.0/24;"
+ "neutron floatingip-create;"),
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
42: {
+ 'cmd': salt_cmd + "'ctl*' state.sls opencontrail.database",
+ 'node_name': 'cfg01.mk22-lab-advanced.local', # hardcoded for now
+ 'retry': {'count': 3, 'delay': 5},
+ 'skip_fail': False,
+ },
+ 43: {
'cmd': ("nodetool status;"
"nodetool compactionstats;"
"nodetool describecluster;"),
- 'node_name': 'ctl01.mk20-lab-advanced.local', # hardcoded for now
+ 'node_name': 'ctl01.mk22-lab-advanced.local', # hardcoded for now
'retry': {'count': 3, 'delay': 5},
'skip_fail': False,
},
@@ -321,33 +334,33 @@
14. Show glusterfs peer status
15. Show glusterfs volume status
16. Install RabbitMQ on all controllers
- 17. Install Galera on primary controller
- 18. Install Galera on other controllers
- 19. Check Galera addresses
- 20. Install haproxy on all controllers
- 21. Check haproxy on all controllers with Galera port
- 22. Install memcached and keystone on ctl01
- 23. Install memcached and keystone on ctl02
- 24. Install memcached and keystone on ctl03
- 25. Check keystone user-list
- 26. Check keystone tenant-list
- 27. Check keystone endpoint-list
- 28. Install glance on controllers
- 29. Install glusterfs on controllers
- 30. Check that glusterfs was added on controllers
- 31. *Workaround* Re-run keystone formula on ctl01 to create fernet keys
- 32. Check glance on ctl01
- 33. Check keystone fernet keys on controllers
- 34. Install cinder on controllers
- 35. Install nova on controllers
- 36. Check cinder status
- 37. Check nova services status
- 38. Check nova status
- 39. Install neutron on controllers
- 40. Create a neutron subnet
- 41. Install contrail database on controllers
- 42. Check cassandra status on ctl01
-
+ 17. *Workaround* Update salt-formula-galera on config node to the latest version
+ 18. Install Galera on primary controller
+ 19. Install Galera on other controllers
+ 20. Check Galera addresses
+ 21. Install haproxy on all controllers
+ 22. Check haproxy on all controllers with Galera port
+ 23. Install memcached and keystone on ctl01
+ 24. Install memcached and keystone on ctl02
+ 25. Install memcached and keystone on ctl03
+ 26. Check keystone user-list
+ 27. Check keystone tenant-list
+ 28. Check keystone endpoint-list
+ 29. Install glance on controllers
+ 30. Install glusterfs on controllers
+ 31. Check that glusterfs was added on controllers
+ 32. *Workaround* Re-run keystone formula on ctl01 to create fernet keys
+ 33. Check glance on ctl01
+ 34. Check keystone fernet keys on controllers
+ 35. Install cinder on controllers
+ 36. Install nova on controllers
+ 37. Check cinder status
+ 38. Check nova services status
+ 39. Check nova status
+ 40. Install neutron on controllers
+ 41. Create a neutron subnet
+ 42. Install contrail database on controllers
+ 43. Check cassandra status on ctl01
"""
for step in sorted(steps):
@@ -355,14 +368,27 @@
show_step(int(step))
with underlay.remote(node_name=steps[step]['node_name']) as remote:
for x in range(steps[step]['retry']['count']):
+
+ time.sleep(5)
+
result = remote.execute(steps[step]['cmd'], verbose=True)
+
+ # Workaround of exit code 0 from salt in case of failures
+ failed = 0
+ for s in result['stdout']:
+ if s.startswith("Failed:"):
+ failed += int(s.split("Failed:")[1])
+
if result.exit_code != 0:
time.sleep(steps[step]['retry']['delay'])
LOG.info(" ========================= retry...")
+ elif failed != 0:
+ LOG.error(" ================= SALT returned exit code = 0 while there are failed modules!")
+ LOG.info(" ========================= retry...")
else:
# Workarounds for crashed services
- tcp_actions.check_salt_service("salt-master", "cfg01.mk20-lab-advanced.local", "salt-call pillar.items") # Hardcoded for now
- tcp_actions.check_salt_service("salt-minion", "cfg01.mk20-lab-advanced.local", "salt 'cfg01*' pillar.items") # Hardcoded for now
+ tcp_actions.check_salt_service("salt-master", "cfg01.mk22-lab-advanced.local", "salt-call pillar.items") # Hardcoded for now
+ tcp_actions.check_salt_service("salt-minion", "cfg01.mk22-lab-advanced.local", "salt 'cfg01*' pillar.items") # Hardcoded for now
break
if x == 1 and steps[step]['skip_fail'] == False: