Redistribute failover tests on the modules
- test_failover.py renamed to test_failover_nodes.py
where are tests for different nodes restart/shutdown
- tests for influxdb moved into separated module
test_failover_stacklight_services.py
- test 'test_restart_ctl_with_vip' moved from
test_failover_openstack_services.py to test_failover_nodes.py
This change is required to run failover tests in the separated
jobs.
Change-Id: I4f19a31f4a53107920353e72c2c78f93c9151e10
diff --git a/tcp_tests/tests/system/test_failover.py b/tcp_tests/tests/system/test_failover.py
deleted file mode 100644
index 615e26f..0000000
--- a/tcp_tests/tests/system/test_failover.py
+++ /dev/null
@@ -1,603 +0,0 @@
-# Copyright 2017 Mirantis, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from devops.helpers import helpers
-import pytest
-
-from tcp_tests import logger
-from tcp_tests.helpers import ext
-
-LOG = logger.logger
-
-
-class TestFailover(object):
- """Test class for testing OpenStack nodes failover"""
-
- @staticmethod
- def check_influxdb_xfail(sl_deployed, node_name, value):
-
- def check_influxdb_data():
- return value in sl_deployed.check_data_in_influxdb(node_name)
-
- try:
- helpers.wait(
- check_influxdb_data,
- timeout=10, interval=2,
- timeout_msg=('Influxdb data {0} was not replicated to {1} '
- '[https://mirantis.jira.com/browse/PROD-16272]'
- .format(value, node_name)))
- except Exception:
- pytest.xfail('Influxdb data {0} was not replicated to {1} '
- '[https://mirantis.jira.com/browse/PROD-16272]'
- .format(value, node_name))
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- def test_warm_shutdown_ctl01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
- """Test warm shutdown ctl01
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute nodes
- 4. Shutdown ctl01
- 5. Run tempest smoke after failover
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
- # STEP #4
- show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('ctl01')
- # STEP #5
- show_step(5)
- openstack_actions.run_tempest(pattern='smoke')
-
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- def test_restart_ctl01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
- """Test restart ctl01
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute nodes
- 4. Restart ctl01
- 5. Run tempest smoke after failover
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- openstack_actions.warm_restart_nodes('ctl01')
- # STEP #5
- show_step(5)
- openstack_actions.run_tempest(pattern='smoke')
-
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
- """Test warm shutdown cmp01
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute nodes
- 4. Shutdown cmp01
- 5. Run tempest smoke after failover
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('cmp01')
- # STEP #5
- show_step(5)
- openstack_actions.run_tempest(pattern='smoke')
-
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- def test_restart_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
- """Test restart cmp01
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute nodes
- 4. Restart cmp01
- 5. Run tempest smoke after failover
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- openstack_actions.warm_restart_nodes('cmp01')
- # STEP #5
- show_step(5)
- openstack_actions.run_tempest(pattern='smoke')
-
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon01_node(self, openstack_actions,
- sl_os_deployed, show_step):
- """Test restart mon01
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute, monitoring nodes
- 4. Check targets before restart
- 5. Restart mon01
- 6. Check targets after restart
- 7. Run LMA smoke after failover
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- sl_os_deployed.check_prometheus_targets(mon_nodes)
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
- # STEP #5
- show_step(5)
- openstack_actions.warm_restart_nodes('mon01')
- # STEP #6
- show_step(6)
- sl_os_deployed.check_prometheus_targets(mon_nodes)
- # STEP #7
- show_step(7)
- # Run SL component tetsts
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_warm_shutdown_mon01_node(self, openstack_actions,
- sl_os_deployed,
- show_step):
- """Test warm shutdown mon01
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute, monitoring nodes
- 4. Check LMA before mon node shutdown
- 5. Shutdown mon01 node
- 6. Run LMA tests after failover
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- sl_os_deployed.check_prometheus_targets(mon_nodes)
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
- # STEP #5
- show_step(5)
- openstack_actions.warm_shutdown_openstack_nodes('mon01')
- # STEP #6
- show_step(6)
- # Run SL component tetsts
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_kill_influxdb_relay_mon01_node(self, sl_os_deployed,
- show_step):
- """Test kill influxdb relay on mon01 node
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute, monitoring nodes
- 4. Check LMA before mon node shutdown
- 5. Kill influxdb relay on mon01 node
- 6. Post data into influx
- 7. Get data from all healthy nodes
- 8. Start influx db
- 9. Request data on mon01
- 10. Run LMA tests after fail and compare with result before fail
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
-
- # STEP #5
- show_step(5)
- sl_os_deployed.kill_sl_service_on_node('mon01', 'influxdb-relay')
-
- # STEP #6
- show_step(6)
- sl_os_deployed.post_data_into_influx('mon02')
-
- # STEP #7
- show_step(7)
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon02')
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
-
- # STEP #8
- show_step(8)
- sl_os_deployed.start_service('mon01', 'influxdb-relay')
-
- # STEP #9
- show_step(9)
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon01')
-
- # STEP #10
- show_step(10)
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_kill_influxdb_mon01_node(self, sl_os_deployed, show_step):
- """Test kill influxdb on mon01 node
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute, monitoring nodes
- 4. Check LMA before mon node shutdown
- 5. Kill influxdb on mon01 node
- 6. Post data into influx
- 7. Get data from all healthy nodes
- 8. Start influx db
- 9. Request data on mon01
- 10. Run LMA tests after fail and compare with result before fail
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
- # STEP #5
- show_step(5)
- sl_os_deployed.kill_sl_service_on_node('mon01', 'influxd')
-
- # STEP #6
- show_step(6)
- sl_os_deployed.post_data_into_influx('mon02')
-
- # STEP #7
- show_step(7)
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon02')
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
-
- # STEP #8
- show_step(8)
- sl_os_deployed.start_service('mon01', 'influxd')
-
- # STEP #9
- show_step(9)
- self.check_influxdb_xfail(sl_os_deployed, 'mon01', 'mymeas')
-
- # STEP #10
- show_step(10)
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_stop_influxdb_relay_mon_nodes(self, sl_os_deployed,
- show_step):
- """Test stop influxdb relay on mon01 node
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute, monitoring nodes
- 4. Check LMA before mon node shutdown
- 5. Stop influxdb relay on mon01 and mon02 nodes
- 6. Post data into influx
- 7. Get data from all healthy nodes
- 8. Start influx db
- 9. Request data on mon01, 02
- 10. Run LMA tests after fail and compare with result before fail
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
- # STEP #5
- show_step(5)
- sl_os_deployed.stop_sl_service_on_node('mon01', 'influxdb-relay')
- sl_os_deployed.stop_sl_service_on_node('mon02', 'influxdb-relay')
-
- # STEP #6
- show_step(6)
- sl_os_deployed.post_data_into_influx('mon03')
-
- # STEP #7
- show_step(7)
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
-
- # STEP #8
- show_step(8)
- sl_os_deployed.start_service('mon01', 'influxdb-relay')
- sl_os_deployed.start_service('mon02', 'influxdb-relay')
-
- # STEP #9
- show_step(9)
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon01')
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon02')
-
- # STEP #10
- show_step(10)
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_stop_influxdb_mon_nodes(self, sl_os_deployed, show_step):
- """Test stop influxdb on mon01 node
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute, monitoring nodes
- 4. Check LMA before mon node shutdown
- 5. Stop influxdb on mon01 and mon02 node
- 6. Post data into influx
- 7. Get data from all healthy nodes
- 8. Start influx db
- 9. Request data on mon01
- 10. Run LMA tests after fail and compare with result before fail
-
-
- """
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
-
- # STEP #5
- show_step(5)
- sl_os_deployed.stop_sl_service_on_node('mon01', 'influxdb')
- sl_os_deployed.stop_sl_service_on_node('mon02', 'influxdb')
-
- # STEP #6
- show_step(6)
- sl_os_deployed.post_data_into_influx('mon03')
-
- # STEP #7
- show_step(7)
- assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
-
- # STEP #8
- show_step(8)
- sl_os_deployed.start_service('mon01', 'influxdb')
- sl_os_deployed.start_service('mon02', 'influxdb')
-
- # STEP #9
- show_step(9)
- self.check_influxdb_xfail(sl_os_deployed, 'mon01', 'mymeas')
- self.check_influxdb_xfail(sl_os_deployed, 'mon02', 'mymeas')
-
- # STEP #10
- show_step(10)
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon_with_vip(self, sl_os_deployed,
- openstack_actions, salt_actions,
- common_services_actions, show_step):
- """Test restart mon with VIP
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute and monitoring nodes
- 4. Check LMA before mon node restart
- 5. Find mon minion id with VIP
- 6. Restart mon minion id with VIP
- 7. Check that VIP was actually migrated on a new node
- 8. Run tempest smoke after failover
-
-
- """
- # TR case #4753939
- common_services_actions.check_keepalived_pillar()
- salt = salt_actions
-
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- mon_nodes = sl_os_deployed.get_monitoring_nodes()
- LOG.debug('Mon nodes list {0}'.format(mon_nodes))
- before_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- failed_tests = [test['name'] for test in
- before_result if 'passed' not in test['outcome']]
-
- # STEP #5
- show_step(5)
- mon_vip_pillar = salt.get_pillar(
- tgt="mon0*",
- pillar="_param:cluster_vip_address")[0]
- vip = [vip for minion_id, vip in mon_vip_pillar.items()][0]
- minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
- LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
-
- # STEP #6
- show_step(6)
- openstack_actions.warm_restart_nodes(minion_vip)
-
- # STEP #7
- show_step(7)
- # Check that VIP has been actually migrated to a new node
- new_minion_vip = common_services_actions.get_keepalived_vip_minion_id(
- vip)
- LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
- assert new_minion_vip != minion_vip, (
- "VIP {0} wasn't migrated from {1} after node reboot!"
- .format(vip, new_minion_vip))
- common_services_actions.check_keepalived_pillar()
-
- # STEP #8
- show_step(8)
- # Run SL component tetsts
- after_result = sl_os_deployed.run_sl_tests_json(
- 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
- 'tests/prometheus/', 'test_alerts.py')
- for test in after_result:
- if test['name'] not in failed_tests:
- assert 'passed' in test['outcome'], \
- 'Failed test {}'.format(test)
- LOG.info("*************** DONE **************")
diff --git a/tcp_tests/tests/system/test_failover_nodes.py b/tcp_tests/tests/system/test_failover_nodes.py
new file mode 100644
index 0000000..0abada9
--- /dev/null
+++ b/tcp_tests/tests/system/test_failover_nodes.py
@@ -0,0 +1,370 @@
+# Copyright 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import pytest
+
+from tcp_tests import logger
+from tcp_tests.helpers import ext
+
+LOG = logger.logger
+
+
+class TestFailoverNodes(object):
+ """Test class for testing OpenStack nodes failover"""
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ def test_warm_shutdown_ctl01_node(self, underlay, openstack_deployed,
+ openstack_actions, show_step):
+ """Test warm shutdown ctl01
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute nodes
+ 4. Shutdown ctl01
+ 5. Run tempest smoke after failover
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+ # STEP #4
+ show_step(4)
+ openstack_actions.warm_shutdown_openstack_nodes('ctl01')
+ # STEP #5
+ show_step(5)
+ openstack_actions.run_tempest(pattern='smoke')
+
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ def test_restart_ctl01_node(self, underlay, openstack_deployed,
+ openstack_actions, show_step):
+ """Test restart ctl01
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute nodes
+ 4. Restart ctl01
+ 5. Run tempest smoke after failover
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ openstack_actions.warm_restart_nodes('ctl01')
+ # STEP #5
+ show_step(5)
+ openstack_actions.run_tempest(pattern='smoke')
+
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
+ openstack_actions, show_step):
+ """Test warm shutdown cmp01
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute nodes
+ 4. Shutdown cmp01
+ 5. Run tempest smoke after failover
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ openstack_actions.warm_shutdown_openstack_nodes('cmp01')
+ # STEP #5
+ show_step(5)
+ openstack_actions.run_tempest(pattern='smoke')
+
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ def test_restart_cmp01_node(self, underlay, openstack_deployed,
+ openstack_actions, show_step):
+ """Test restart cmp01
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute nodes
+ 4. Restart cmp01
+ 5. Run tempest smoke after failover
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ openstack_actions.warm_restart_nodes('cmp01')
+ # STEP #5
+ show_step(5)
+ openstack_actions.run_tempest(pattern='smoke')
+
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_restart_mon01_node(self, openstack_actions,
+ sl_os_deployed, show_step):
+ """Test restart mon01
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute, monitoring nodes
+ 4. Check targets before restart
+ 5. Restart mon01
+ 6. Check targets after restart
+ 7. Run LMA smoke after failover
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ sl_os_deployed.check_prometheus_targets(mon_nodes)
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+ # STEP #5
+ show_step(5)
+ openstack_actions.warm_restart_nodes('mon01')
+ # STEP #6
+ show_step(6)
+ sl_os_deployed.check_prometheus_targets(mon_nodes)
+ # STEP #7
+ show_step(7)
+ # Run SL component tetsts
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_warm_shutdown_mon01_node(self, openstack_actions,
+ sl_os_deployed,
+ show_step):
+ """Test warm shutdown mon01
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute, monitoring nodes
+ 4. Check LMA before mon node shutdown
+ 5. Shutdown mon01 node
+ 6. Run LMA tests after failover
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ sl_os_deployed.check_prometheus_targets(mon_nodes)
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+ # STEP #5
+ show_step(5)
+ openstack_actions.warm_shutdown_openstack_nodes('mon01')
+ # STEP #6
+ show_step(6)
+ # Run SL component tetsts
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_restart_mon_with_vip(self, sl_os_deployed,
+ openstack_actions, salt_actions,
+ common_services_actions, show_step):
+ """Test restart mon with VIP
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute and monitoring nodes
+ 4. Check LMA before mon node restart
+ 5. Find mon minion id with VIP
+ 6. Restart mon minion id with VIP
+ 7. Check that VIP was actually migrated on a new node
+ 8. Run tempest smoke after failover
+
+
+ """
+ # TR case #4753939
+ common_services_actions.check_keepalived_pillar()
+ salt = salt_actions
+
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+
+ # STEP #5
+ show_step(5)
+ mon_vip_pillar = salt.get_pillar(
+ tgt="mon0*",
+ pillar="_param:cluster_vip_address")[0]
+ vip = [vip for minion_id, vip in mon_vip_pillar.items()][0]
+ minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
+
+ # STEP #6
+ show_step(6)
+ openstack_actions.warm_restart_nodes(minion_vip)
+
+ # STEP #7
+ show_step(7)
+ # Check that VIP has been actually migrated to a new node
+ new_minion_vip = common_services_actions.get_keepalived_vip_minion_id(
+ vip)
+ LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
+ assert new_minion_vip != minion_vip, (
+ "VIP {0} wasn't migrated from {1} after node reboot!"
+ .format(vip, new_minion_vip))
+ common_services_actions.check_keepalived_pillar()
+
+ # STEP #8
+ show_step(8)
+ # Run SL component tetsts
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
+ def test_restart_ctl_with_vip(self, underlay, openstack_deployed,
+ openstack_actions, salt_actions,
+ common_services_actions, show_step):
+ """Test restart clt with VIP
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute nodes
+ 4. Find controller minion id with VIP
+ 5. Restart controller minion id with VIP
+ 6. Check that VIP was actually migrated on a new node
+ 7. Run tempest smoke after failover
+
+
+ """
+ # TR case #3385671
+ common_services_actions.check_keepalived_pillar()
+ salt = salt_actions
+
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ ctl_vip_pillar = salt.get_pillar(
+ tgt="I@nova:controller:enabled:True",
+ pillar="_param:cluster_vip_address")[0]
+ vip = [vip for minion_id, vip in ctl_vip_pillar.items()][0]
+ minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
+
+ # STEP #5
+ show_step(5)
+ openstack_actions.warm_restart_nodes(minion_vip)
+
+ # STEP #6
+ show_step(6)
+ # Check that VIP has been actually migrated to a new node
+ new_minion_vip = common_services_actions.get_keepalived_vip_minion_id(
+ vip)
+ LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
+ assert new_minion_vip != minion_vip, (
+ "VIP {0} wasn't migrated from {1} after node reboot!"
+ .format(vip, new_minion_vip))
+ common_services_actions.check_keepalived_pillar()
+
+ # STEP #7
+ show_step(7)
+ openstack_actions.run_tempest(pattern='smoke')
+
+ LOG.info("*************** DONE **************")
diff --git a/tcp_tests/tests/system/test_failover_openstack_services.py b/tcp_tests/tests/system/test_failover_openstack_services.py
index efbba58..37cff72 100644
--- a/tcp_tests/tests/system/test_failover_openstack_services.py
+++ b/tcp_tests/tests/system/test_failover_openstack_services.py
@@ -600,60 +600,3 @@
assert not results['fail'], self.show_failed_msg(results['fail'])
LOG.info("*************** DONE **************")
-
- @pytest.mark.grab_versions
- @pytest.mark.fail_snapshot
- def test_restart_ctl_with_vip(self, underlay, openstack_deployed,
- openstack_actions, salt_actions,
- common_services_actions, show_step):
- """Test restart clt with VIP
-
- Scenario:
- 1. Prepare salt on hosts
- 2. Setup controller nodes
- 3. Setup compute nodes
- 4. Find controller minion id with VIP
- 5. Restart controller minion id with VIP
- 6. Check that VIP was actually migrated on a new node
- 7. Run tempest smoke after failover
-
-
- """
- # TR case #3385671
- common_services_actions.check_keepalived_pillar()
- salt = salt_actions
-
- # STEP #1,2,3
- show_step(1)
- show_step(2)
- show_step(3)
-
- # STEP #4
- show_step(4)
- ctl_vip_pillar = salt.get_pillar(
- tgt="I@nova:controller:enabled:True",
- pillar="_param:cluster_vip_address")[0]
- vip = [vip for minion_id, vip in ctl_vip_pillar.items()][0]
- minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
- LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
-
- # STEP #5
- show_step(5)
- openstack_actions.warm_restart_nodes(minion_vip)
-
- # STEP #6
- show_step(6)
- # Check that VIP has been actually migrated to a new node
- new_minion_vip = common_services_actions.get_keepalived_vip_minion_id(
- vip)
- LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
- assert new_minion_vip != minion_vip, (
- "VIP {0} wasn't migrated from {1} after node reboot!"
- .format(vip, new_minion_vip))
- common_services_actions.check_keepalived_pillar()
-
- # STEP #7
- show_step(7)
- openstack_actions.run_tempest(pattern='smoke')
-
- LOG.info("*************** DONE **************")
diff --git a/tcp_tests/tests/system/test_failover_stacklight_services.py b/tcp_tests/tests/system/test_failover_stacklight_services.py
new file mode 100644
index 0000000..3bb47eb
--- /dev/null
+++ b/tcp_tests/tests/system/test_failover_stacklight_services.py
@@ -0,0 +1,314 @@
+# Copyright 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from devops.helpers import helpers
+import pytest
+
+from tcp_tests import logger
+from tcp_tests.helpers import ext
+
+LOG = logger.logger
+
+
+class TestFailoverStacklightServices(object):
+ """Test class for testing OpenStack nodes failover"""
+
+ @staticmethod
+ def check_influxdb_xfail(sl_deployed, node_name, value):
+
+ def check_influxdb_data():
+ return value in sl_deployed.check_data_in_influxdb(node_name)
+
+ try:
+ helpers.wait(
+ check_influxdb_data,
+ timeout=10, interval=2,
+ timeout_msg=('Influxdb data {0} was not replicated to {1} '
+ '[https://mirantis.jira.com/browse/PROD-16272]'
+ .format(value, node_name)))
+ except Exception:
+ pytest.xfail('Influxdb data {0} was not replicated to {1} '
+ '[https://mirantis.jira.com/browse/PROD-16272]'
+ .format(value, node_name))
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_kill_influxdb_relay_mon01_node(self, sl_os_deployed,
+ show_step):
+ """Test kill influxdb relay on mon01 node
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute, monitoring nodes
+ 4. Check LMA before mon node shutdown
+ 5. Kill influxdb relay on mon01 node
+ 6. Post data into influx
+ 7. Get data from all healthy nodes
+ 8. Start influx db
+ 9. Request data on mon01
+ 10. Run LMA tests after fail and compare with result before fail
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+
+ # STEP #5
+ show_step(5)
+ sl_os_deployed.kill_sl_service_on_node('mon01', 'influxdb-relay')
+
+ # STEP #6
+ show_step(6)
+ sl_os_deployed.post_data_into_influx('mon02')
+
+ # STEP #7
+ show_step(7)
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon02')
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
+
+ # STEP #8
+ show_step(8)
+ sl_os_deployed.start_service('mon01', 'influxdb-relay')
+
+ # STEP #9
+ show_step(9)
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon01')
+
+ # STEP #10
+ show_step(10)
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_kill_influxdb_mon01_node(self, sl_os_deployed, show_step):
+ """Test kill influxdb on mon01 node
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute, monitoring nodes
+ 4. Check LMA before mon node shutdown
+ 5. Kill influxdb on mon01 node
+ 6. Post data into influx
+ 7. Get data from all healthy nodes
+ 8. Start influx db
+ 9. Request data on mon01
+ 10. Run LMA tests after fail and compare with result before fail
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+ # STEP #5
+ show_step(5)
+ sl_os_deployed.kill_sl_service_on_node('mon01', 'influxd')
+
+ # STEP #6
+ show_step(6)
+ sl_os_deployed.post_data_into_influx('mon02')
+
+ # STEP #7
+ show_step(7)
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon02')
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
+
+ # STEP #8
+ show_step(8)
+ sl_os_deployed.start_service('mon01', 'influxd')
+
+ # STEP #9
+ show_step(9)
+ self.check_influxdb_xfail(sl_os_deployed, 'mon01', 'mymeas')
+
+ # STEP #10
+ show_step(10)
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_stop_influxdb_relay_mon_nodes(self, sl_os_deployed,
+ show_step):
+ """Test stop influxdb relay on mon01 node
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute, monitoring nodes
+ 4. Check LMA before mon node shutdown
+ 5. Stop influxdb relay on mon01 and mon02 nodes
+ 6. Post data into influx
+ 7. Get data from all healthy nodes
+ 8. Start influx db
+ 9. Request data on mon01, 02
+ 10. Run LMA tests after fail and compare with result before fail
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+ # STEP #5
+ show_step(5)
+ sl_os_deployed.stop_sl_service_on_node('mon01', 'influxdb-relay')
+ sl_os_deployed.stop_sl_service_on_node('mon02', 'influxdb-relay')
+
+ # STEP #6
+ show_step(6)
+ sl_os_deployed.post_data_into_influx('mon03')
+
+ # STEP #7
+ show_step(7)
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
+
+ # STEP #8
+ show_step(8)
+ sl_os_deployed.start_service('mon01', 'influxdb-relay')
+ sl_os_deployed.start_service('mon02', 'influxdb-relay')
+
+ # STEP #9
+ show_step(9)
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon01')
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon02')
+
+ # STEP #10
+ show_step(10)
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")
+
+ @pytest.mark.grab_versions
+ @pytest.mark.fail_snapshot
+ @pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
+ def test_stop_influxdb_mon_nodes(self, sl_os_deployed, show_step):
+ """Test stop influxdb on mon01 node
+
+ Scenario:
+ 1. Prepare salt on hosts
+ 2. Setup controller nodes
+ 3. Setup compute, monitoring nodes
+ 4. Check LMA before mon node shutdown
+ 5. Stop influxdb on mon01 and mon02 node
+ 6. Post data into influx
+ 7. Get data from all healthy nodes
+ 8. Start influx db
+ 9. Request data on mon01
+ 10. Run LMA tests after fail and compare with result before fail
+
+
+ """
+ # STEP #1,2,3
+ show_step(1)
+ show_step(2)
+ show_step(3)
+
+ # STEP #4
+ show_step(4)
+ mon_nodes = sl_os_deployed.get_monitoring_nodes()
+ LOG.debug('Mon nodes list {0}'.format(mon_nodes))
+ before_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ failed_tests = [test['name'] for test in
+ before_result if 'passed' not in test['outcome']]
+
+ # STEP #5
+ show_step(5)
+ sl_os_deployed.stop_sl_service_on_node('mon01', 'influxdb')
+ sl_os_deployed.stop_sl_service_on_node('mon02', 'influxdb')
+
+ # STEP #6
+ show_step(6)
+ sl_os_deployed.post_data_into_influx('mon03')
+
+ # STEP #7
+ show_step(7)
+ assert 'mymeas' in sl_os_deployed.check_data_in_influxdb('mon03')
+
+ # STEP #8
+ show_step(8)
+ sl_os_deployed.start_service('mon01', 'influxdb')
+ sl_os_deployed.start_service('mon02', 'influxdb')
+
+ # STEP #9
+ show_step(9)
+ self.check_influxdb_xfail(sl_os_deployed, 'mon01', 'mymeas')
+ self.check_influxdb_xfail(sl_os_deployed, 'mon02', 'mymeas')
+
+ # STEP #10
+ show_step(10)
+ after_result = sl_os_deployed.run_sl_tests_json(
+ 'cfg01', '/root/stacklight-pytest/stacklight_tests/',
+ 'tests/prometheus/', 'test_alerts.py')
+ for test in after_result:
+ if test['name'] not in failed_tests:
+ assert 'passed' in test['outcome'], \
+ 'Failed test {}'.format(test)
+ LOG.info("*************** DONE **************")