Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mcp / tcp-qa / 850e607f8de055e6ce72ea2cf8c9d52611825095 / . / tcp_tests / tests / system / test_failover_nodes.py
blob: 0abada90d6146b322efe3adcdde3ae3db4379fee [file] [log] [blame]
Dennis Dmitriev759baa12018-03-14 01:50:53 +0200[diff] [blame]1# Copyright 2017 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14import pytest
15
16from tcp_tests import logger
17from tcp_tests.helpers import ext
18
19LOG = logger.logger
20
21
22class TestFailoverNodes(object):
23 """Test class for testing OpenStack nodes failover"""
24
25 @pytest.mark.grab_versions
26 @pytest.mark.fail_snapshot
27 def test_warm_shutdown_ctl01_node(self, underlay, openstack_deployed,
28 openstack_actions, show_step):
29 """Test warm shutdown ctl01
30
31 Scenario:
32 1. Prepare salt on hosts
33 2. Setup controller nodes
34 3. Setup compute nodes
35 4. Shutdown ctl01
36 5. Run tempest smoke after failover
37
38
39 """
40 # STEP #1,2,3
41 show_step(1)
42 show_step(2)
43 show_step(3)
44 # STEP #4
45 show_step(4)
46 openstack_actions.warm_shutdown_openstack_nodes('ctl01')
47 # STEP #5
48 show_step(5)
49 openstack_actions.run_tempest(pattern='smoke')
50
51 LOG.info("*************** DONE **************")
52
53 @pytest.mark.grab_versions
54 @pytest.mark.fail_snapshot
55 def test_restart_ctl01_node(self, underlay, openstack_deployed,
56 openstack_actions, show_step):
57 """Test restart ctl01
58
59 Scenario:
60 1. Prepare salt on hosts
61 2. Setup controller nodes
62 3. Setup compute nodes
63 4. Restart ctl01
64 5. Run tempest smoke after failover
65
66
67 """
68 # STEP #1,2,3
69 show_step(1)
70 show_step(2)
71 show_step(3)
72
73 # STEP #4
74 show_step(4)
75 openstack_actions.warm_restart_nodes('ctl01')
76 # STEP #5
77 show_step(5)
78 openstack_actions.run_tempest(pattern='smoke')
79
80 LOG.info("*************** DONE **************")
81
82 @pytest.mark.grab_versions
83 @pytest.mark.fail_snapshot
84 def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
85 openstack_actions, show_step):
86 """Test warm shutdown cmp01
87
88 Scenario:
89 1. Prepare salt on hosts
90 2. Setup controller nodes
91 3. Setup compute nodes
92 4. Shutdown cmp01
93 5. Run tempest smoke after failover
94
95
96 """
97 # STEP #1,2,3
98 show_step(1)
99 show_step(2)
100 show_step(3)
101
102 # STEP #4
103 show_step(4)
104 openstack_actions.warm_shutdown_openstack_nodes('cmp01')
105 # STEP #5
106 show_step(5)
107 openstack_actions.run_tempest(pattern='smoke')
108
109 LOG.info("*************** DONE **************")
110
111 @pytest.mark.grab_versions
112 @pytest.mark.fail_snapshot
113 def test_restart_cmp01_node(self, underlay, openstack_deployed,
114 openstack_actions, show_step):
115 """Test restart cmp01
116
117 Scenario:
118 1. Prepare salt on hosts
119 2. Setup controller nodes
120 3. Setup compute nodes
121 4. Restart cmp01
122 5. Run tempest smoke after failover
123
124
125 """
126 # STEP #1,2,3
127 show_step(1)
128 show_step(2)
129 show_step(3)
130
131 # STEP #4
132 show_step(4)
133 openstack_actions.warm_restart_nodes('cmp01')
134 # STEP #5
135 show_step(5)
136 openstack_actions.run_tempest(pattern='smoke')
137
138 LOG.info("*************** DONE **************")
139
140 @pytest.mark.grab_versions
141 @pytest.mark.fail_snapshot
142 @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
143 def test_restart_mon01_node(self, openstack_actions,
144 sl_os_deployed, show_step):
145 """Test restart mon01
146
147 Scenario:
148 1. Prepare salt on hosts
149 2. Setup controller nodes
150 3. Setup compute, monitoring nodes
151 4. Check targets before restart
152 5. Restart mon01
153 6. Check targets after restart
154 7. Run LMA smoke after failover
155
156
157 """
158 # STEP #1,2,3
159 show_step(1)
160 show_step(2)
161 show_step(3)
162
163 # STEP #4
164 show_step(4)
165 mon_nodes = sl_os_deployed.get_monitoring_nodes()
166 LOG.debug('Mon nodes list {0}'.format(mon_nodes))
167 sl_os_deployed.check_prometheus_targets(mon_nodes)
168 before_result = sl_os_deployed.run_sl_tests_json(
169 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
170 'tests/prometheus/', 'test_alerts.py')
171 failed_tests = [test['name'] for test in
172 before_result if 'passed' not in test['outcome']]
173 # STEP #5
174 show_step(5)
175 openstack_actions.warm_restart_nodes('mon01')
176 # STEP #6
177 show_step(6)
178 sl_os_deployed.check_prometheus_targets(mon_nodes)
179 # STEP #7
180 show_step(7)
181 # Run SL component tetsts
182 after_result = sl_os_deployed.run_sl_tests_json(
183 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
184 'tests/prometheus/', 'test_alerts.py')
185 for test in after_result:
186 if test['name'] not in failed_tests:
187 assert 'passed' in test['outcome'], \
188 'Failed test {}'.format(test)
189 LOG.info("*************** DONE **************")
190
191 @pytest.mark.grab_versions
192 @pytest.mark.fail_snapshot
193 @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
194 def test_warm_shutdown_mon01_node(self, openstack_actions,
195 sl_os_deployed,
196 show_step):
197 """Test warm shutdown mon01
198
199 Scenario:
200 1. Prepare salt on hosts
201 2. Setup controller nodes
202 3. Setup compute, monitoring nodes
203 4. Check LMA before mon node shutdown
204 5. Shutdown mon01 node
205 6. Run LMA tests after failover
206
207
208 """
209 # STEP #1,2,3
210 show_step(1)
211 show_step(2)
212 show_step(3)
213
214 # STEP #4
215 show_step(4)
216 mon_nodes = sl_os_deployed.get_monitoring_nodes()
217 LOG.debug('Mon nodes list {0}'.format(mon_nodes))
218 sl_os_deployed.check_prometheus_targets(mon_nodes)
219 before_result = sl_os_deployed.run_sl_tests_json(
220 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
221 'tests/prometheus/', 'test_alerts.py')
222 failed_tests = [test['name'] for test in
223 before_result if 'passed' not in test['outcome']]
224 # STEP #5
225 show_step(5)
226 openstack_actions.warm_shutdown_openstack_nodes('mon01')
227 # STEP #6
228 show_step(6)
229 # Run SL component tetsts
230 after_result = sl_os_deployed.run_sl_tests_json(
231 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
232 'tests/prometheus/', 'test_alerts.py')
233 for test in after_result:
234 if test['name'] not in failed_tests:
235 assert 'passed' in test['outcome'], \
236 'Failed test {}'.format(test)
237 LOG.info("*************** DONE **************")
238
239 @pytest.mark.grab_versions
240 @pytest.mark.fail_snapshot
241 @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
242 def test_restart_mon_with_vip(self, sl_os_deployed,
243 openstack_actions, salt_actions,
244 common_services_actions, show_step):
245 """Test restart mon with VIP
246
247 Scenario:
248 1. Prepare salt on hosts
249 2. Setup controller nodes
250 3. Setup compute and monitoring nodes
251 4. Check LMA before mon node restart
252 5. Find mon minion id with VIP
253 6. Restart mon minion id with VIP
254 7. Check that VIP was actually migrated on a new node
255 8. Run tempest smoke after failover
256
257
258 """
259 # TR case #4753939
260 common_services_actions.check_keepalived_pillar()
261 salt = salt_actions
262
263 # STEP #1,2,3
264 show_step(1)
265 show_step(2)
266 show_step(3)
267
268 # STEP #4
269 show_step(4)
270 mon_nodes = sl_os_deployed.get_monitoring_nodes()
271 LOG.debug('Mon nodes list {0}'.format(mon_nodes))
272 before_result = sl_os_deployed.run_sl_tests_json(
273 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
274 'tests/prometheus/', 'test_alerts.py')
275 failed_tests = [test['name'] for test in
276 before_result if 'passed' not in test['outcome']]
277
278 # STEP #5
279 show_step(5)
280 mon_vip_pillar = salt.get_pillar(
281 tgt="mon0*",
282 pillar="_param:cluster_vip_address")[0]
283 vip = [vip for minion_id, vip in mon_vip_pillar.items()][0]
284 minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
285 LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
286
287 # STEP #6
288 show_step(6)
289 openstack_actions.warm_restart_nodes(minion_vip)
290
291 # STEP #7
292 show_step(7)
293 # Check that VIP has been actually migrated to a new node
294 new_minion_vip = common_services_actions.get_keepalived_vip_minion_id(
295 vip)
296 LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
297 assert new_minion_vip != minion_vip, (
298 "VIP {0} wasn't migrated from {1} after node reboot!"
299 .format(vip, new_minion_vip))
300 common_services_actions.check_keepalived_pillar()
301
302 # STEP #8
303 show_step(8)
304 # Run SL component tetsts
305 after_result = sl_os_deployed.run_sl_tests_json(
306 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
307 'tests/prometheus/', 'test_alerts.py')
308 for test in after_result:
309 if test['name'] not in failed_tests:
310 assert 'passed' in test['outcome'], \
311 'Failed test {}'.format(test)
312 LOG.info("*************** DONE **************")
313
314 @pytest.mark.grab_versions
315 @pytest.mark.fail_snapshot
316 @pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
317 def test_restart_ctl_with_vip(self, underlay, openstack_deployed,
318 openstack_actions, salt_actions,
319 common_services_actions, show_step):
320 """Test restart clt with VIP
321
322 Scenario:
323 1. Prepare salt on hosts
324 2. Setup controller nodes
325 3. Setup compute nodes
326 4. Find controller minion id with VIP
327 5. Restart controller minion id with VIP
328 6. Check that VIP was actually migrated on a new node
329 7. Run tempest smoke after failover
330
331
332 """
333 # TR case #3385671
334 common_services_actions.check_keepalived_pillar()
335 salt = salt_actions
336
337 # STEP #1,2,3
338 show_step(1)
339 show_step(2)
340 show_step(3)
341
342 # STEP #4
343 show_step(4)
344 ctl_vip_pillar = salt.get_pillar(
345 tgt="I@nova:controller:enabled:True",
346 pillar="_param:cluster_vip_address")[0]
347 vip = [vip for minion_id, vip in ctl_vip_pillar.items()][0]
348 minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
349 LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
350
351 # STEP #5
352 show_step(5)
353 openstack_actions.warm_restart_nodes(minion_vip)
354
355 # STEP #6
356 show_step(6)
357 # Check that VIP has been actually migrated to a new node
358 new_minion_vip = common_services_actions.get_keepalived_vip_minion_id(
359 vip)
360 LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
361 assert new_minion_vip != minion_vip, (
362 "VIP {0} wasn't migrated from {1} after node reboot!"
363 .format(vip, new_minion_vip))
364 common_services_actions.check_keepalived_pillar()
365
366 # STEP #7
367 show_step(7)
368 openstack_actions.run_tempest(pattern='smoke')
369
370 LOG.info("*************** DONE **************")