tcp_tests/tests/system/test_failover_nodes.py

#    Copyright 2017 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import pytest

from tcp_tests import logger
from tcp_tests.helpers import ext

LOG = logger.logger


class TestFailoverNodes(object):
    """Test class for testing OpenStack nodes failover"""

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    def test_warm_shutdown_ctl01_node(self, underlay, hardware, show_step,
                                      openstack_deployed, openstack_actions):
        """Test warm shutdown ctl01

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute nodes
            4. Shutdown ctl01
            5. Run tempest smoke after failover


        """
        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)
        # STEP #4
        show_step(4)
        hardware.warm_shutdown_nodes(underlay, 'ctl01')
        # STEP #5
        show_step(5)
        openstack_actions.run_tempest(pattern='smoke')

        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    def test_restart_ctl01_node(self, underlay, hardware, openstack_deployed,
                                openstack_actions, show_step):
        """Test restart ctl01

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute nodes
            4. Restart ctl01
            5. Run tempest smoke after failover


        """
        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        hardware.warm_restart_nodes(underlay, 'ctl01')
        # STEP #5
        show_step(5)
        openstack_actions.run_tempest(pattern='smoke')

        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
                                      hardware, openstack_actions, show_step):
        """Test warm shutdown cmp01

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute nodes
            4. Shutdown cmp01
            5. Run tempest smoke after failover


        """
        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        hardware.warm_shutdown_nodes(underlay, 'cmp01')
        # STEP #5
        show_step(5)
        openstack_actions.run_tempest(pattern='smoke')

        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    def test_restart_cmp01_node(self, underlay, openstack_deployed,
                                openstack_actions, show_step, hardware):
        """Test restart cmp01

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute nodes
            4. Restart cmp01
            5. Run tempest smoke after failover


        """
        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        hardware.warm_restart_nodes(underlay, 'cmp01')
        # STEP #5
        show_step(5)
        openstack_actions.run_tempest(pattern='smoke')

        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    @pytest.mark.revert_snapshot(ext.SNAPSHOT.stacklight_deployed)
    def test_restart_mon01_node(self, openstack_actions, hardware, underlay,
                                sl_os_deployed, show_step):
        """Test restart mon01

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute, monitoring nodes
            4. Check targets before restart
            5. Restart mon01
            6. Check targets after restart
            7. Run LMA smoke after failover


        """
        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        mon_nodes = sl_os_deployed.get_monitoring_nodes()
        LOG.debug('Mon nodes list {0}'.format(mon_nodes))
        sl_os_deployed.check_prometheus_targets(mon_nodes)
        before_result = sl_os_deployed.run_sl_tests_json(
            'cfg01', '/root/stacklight-pytest/stacklight_tests/',
            'tests/', 'test_alerts.py')
        failed_tests = [test['name'] for test in
                        before_result if 'passed' not in test['outcome']]
        # STEP #5
        show_step(5)
        hardware.warm_restart_nodes(underlay, 'mon01')
        # STEP #6
        show_step(6)
        sl_os_deployed.check_prometheus_targets(mon_nodes)
        # STEP #7
        show_step(7)
        # Run SL component tetsts
        after_result = sl_os_deployed.run_sl_tests_json(
            'cfg01', '/root/stacklight-pytest/stacklight_tests/',
            'tests/', 'test_alerts.py')
        for test in after_result:
            if test['name'] not in failed_tests:
                assert 'passed' in test['outcome'], \
                    'Failed test {}'.format(test)
        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    @pytest.mark.revert_snapshot(ext.SNAPSHOT.stacklight_deployed)
    def test_warm_shutdown_mon01_node(self, underlay, hardware, sl_os_deployed,
                                      openstack_actions, show_step):
        """Test warm shutdown mon01

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute, monitoring nodes
            4. Check LMA before mon node shutdown
            5. Shutdown mon01 node
            6. Run LMA tests after failover


        """
        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        mon_nodes = sl_os_deployed.get_monitoring_nodes()
        LOG.debug('Mon nodes list {0}'.format(mon_nodes))
        sl_os_deployed.check_prometheus_targets(mon_nodes)
        before_result = sl_os_deployed.run_sl_tests_json(
            'cfg01', '/root/stacklight-pytest/stacklight_tests/',
            'tests/', 'test_alerts.py')
        failed_tests = [test['name'] for test in
                        before_result if 'passed' not in test['outcome']]
        # STEP #5
        show_step(5)
        hardware.warm_shutdown_nodes(underlay, 'mon01')
        # STEP #6
        show_step(6)
        # Run SL component tetsts
        after_result = sl_os_deployed.run_sl_tests_json(
            'cfg01', '/root/stacklight-pytest/stacklight_tests/',
            'tests/', 'test_alerts.py')
        for test in after_result:
            if test['name'] not in failed_tests:
                assert 'passed' in test['outcome'], \
                    'Failed test {}'.format(test)
        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    @pytest.mark.revert_snapshot(ext.SNAPSHOT.stacklight_deployed)
    def test_restart_mon_with_vip(self, underlay, hardware, sl_os_deployed,
                                  openstack_actions, salt_actions,
                                  core_actions, show_step):
        """Test restart mon with VIP

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute and monitoring nodes
            4. Check LMA before mon node restart
            5. Find mon minion id with VIP
            6. Restart mon minion id with VIP
            7. Check that VIP was actually migrated on a new node
            8. Run tempest smoke after failover


        """
        # TR case #4753939
        core_actions.check_keepalived_pillar()
        salt = salt_actions

        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        mon_nodes = sl_os_deployed.get_monitoring_nodes()
        LOG.debug('Mon nodes list {0}'.format(mon_nodes))
        before_result = sl_os_deployed.run_sl_tests_json(
            'cfg01', '/root/stacklight-pytest/stacklight_tests/',
            'tests/', 'test_alerts.py')
        failed_tests = [test['name'] for test in
                        before_result if 'passed' not in test['outcome']]

        # STEP #5
        show_step(5)
        mon_vip_pillar = salt.get_pillar(
            tgt="mon0*",
            pillar="_param:cluster_vip_address")[0]
        vip = [vip for minion_id, vip in mon_vip_pillar.items()][0]
        minion_vip = core_actions.get_keepalived_vip_minion_id(vip)
        LOG.info("VIP {0} is on {1}".format(vip, minion_vip))

        # STEP #6
        show_step(6)
        hardware.warm_restart_nodes(underlay, minion_vip)

        # STEP #7
        show_step(7)
        # Check that VIP has been actually migrated to a new node
        new_minion_vip = core_actions.get_keepalived_vip_minion_id(
            vip)
        LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
        assert new_minion_vip != minion_vip, (
            "VIP {0} wasn't migrated from {1} after node reboot!"
            .format(vip, new_minion_vip))
        core_actions.check_keepalived_pillar()

        # STEP #8
        show_step(8)
        # Run SL component tetsts
        after_result = sl_os_deployed.run_sl_tests_json(
            'cfg01', '/root/stacklight-pytest/stacklight_tests/',
            'tests/', 'test_alerts.py')
        for test in after_result:
            if test['name'] not in failed_tests:
                assert 'passed' in test['outcome'], \
                    'Failed test {}'.format(test)
        LOG.info("*************** DONE **************")

    @pytest.mark.grab_versions
    @pytest.mark.fail_snapshot
    @pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
    def test_restart_ctl_with_vip(self, underlay, hardware, openstack_deployed,
                                  openstack_actions, salt_actions,
                                  core_actions, show_step):
        """Test restart clt with VIP

        Scenario:
            1. Prepare salt on hosts
            2. Setup controller nodes
            3. Setup compute nodes
            4. Find controller minion id with VIP
            5. Restart controller minion id with VIP
            6. Check that VIP was actually migrated on a new node
            7. Run tempest smoke after failover


        """
        # TR case #3385671
        core_actions.check_keepalived_pillar()
        salt = salt_actions

        # STEP #1,2,3
        show_step(1)
        show_step(2)
        show_step(3)

        # STEP #4
        show_step(4)
        ctl_vip_pillar = salt.get_pillar(
            tgt="I@nova:controller:enabled:True",
            pillar="_param:cluster_vip_address")[0]
        vip = [vip for minion_id, vip in ctl_vip_pillar.items()][0]
        minion_vip = core_actions.get_keepalived_vip_minion_id(vip)
        LOG.info("VIP {0} is on {1}".format(vip, minion_vip))

        # STEP #5
        show_step(5)
        hardware.warm_restart_nodes(underlay, minion_vip)

        # STEP #6
        show_step(6)
        # Check that VIP has been actually migrated to a new node
        new_minion_vip = core_actions.get_keepalived_vip_minion_id(
            vip)
        LOG.info("Migrated VIP {0} is on {1}".format(vip, new_minion_vip))
        assert new_minion_vip != minion_vip, (
            "VIP {0} wasn't migrated from {1} after node reboot!"
            .format(vip, new_minion_vip))
        core_actions.check_keepalived_pillar()

        # STEP #7
        show_step(7)
        openstack_actions.run_tempest(pattern='smoke')

        LOG.info("*************** DONE **************")