Add k8s keepalived failover test
Rewrite openstack_actions to remove hardware dependency from it.
Rewrite conformance method to allow more flexibility.
Add timeout after restarting keepalived workaround because
sometimes it did not have time to go into MASTER state before
test starts.
Change-Id: I668844f91f4a8fb473e199977f1ebe6ca7ddc35a
Related-PROD: PROD-20878
diff --git a/tcp_tests/tests/system/test_failover_ceph.py b/tcp_tests/tests/system/test_failover_ceph.py
index 4a68705..2f1bc25 100644
--- a/tcp_tests/tests/system/test_failover_ceph.py
+++ b/tcp_tests/tests/system/test_failover_ceph.py
@@ -44,7 +44,7 @@
@pytest.mark.fail_snapshot
def test_restart_osd_node(self, func_name, underlay, config,
openstack_deployed, ceph_deployed,
- openstack_actions,
+ openstack_actions, hardware,
rally, show_step):
"""Test restart ceph osd node
@@ -78,7 +78,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('osd01')
+ hardware.warm_restart_nodes(underlay, 'osd01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
@@ -112,7 +112,7 @@
openstack_deployed, ceph_deployed,
common_services_actions,
salt_actions, openstack_actions,
- rally, show_step):
+ rally, show_step, hardware):
"""Test restart ceph cmn node
Scenario:
@@ -145,7 +145,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('cmn01')
+ hardware.warm_restart_nodes(underlay, 'cmn01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
@@ -177,7 +177,7 @@
@pytest.mark.fail_snapshot
def test_restart_rgw_node(self, func_name, underlay, config,
openstack_deployed, ceph_deployed,
- common_services_actions,
+ common_services_actions, hardware,
salt_actions, openstack_actions,
rally, show_step):
"""Test restart ceph rgw node
@@ -215,7 +215,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('rgw01')
+ hardware.warm_restart_nodes(underlay, 'rgw01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
diff --git a/tcp_tests/tests/system/test_failover_k8s.py b/tcp_tests/tests/system/test_failover_k8s.py
new file mode 100644
index 0000000..1ad43b9
--- /dev/null
+++ b/tcp_tests/tests/system/test_failover_k8s.py
@@ -0,0 +1,75 @@
+# Copyright 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import pytest
+import time
+
+from tcp_tests import logger
+
+LOG = logger.logger
+
+
+class TestFailoverK8s(object):
+
+ @pytest.mark.grap_versions
+ @pytest.mark.fail_snapshot
+ def test_k8s_master_vip_migration(self, show_step, k8s_deployed, underlay,
+ k8s_actions, common_services_actions,
+ config, hardware):
+ """Test restart and shutdown master with VIP
+
+ Scenario:
+ 1. Deploy mcp with k8s ha
+ 2. Check keepalived pillar configuration
+ 3. Find master node with assigned VIP
+ 4. Reboot server with VIP
+ 5. Check that VIP was migrated
+ 6. Check keepalived pillar configuration
+ 7. Check api server availability
+ 8. Run conformance on node with VIP
+ """
+ show_step(1)
+ show_step(2)
+ common_services_actions.check_keepalived_pillar()
+
+ show_step(3)
+ vip = k8s_actions.get_keepalived_vip()
+ LOG.info("VIP ip address: {}".format(vip))
+ minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
+
+ show_step(4)
+ hardware.warm_restart_nodes(underlay, minion_vip)
+
+ show_step(5)
+ try:
+ new_minion_vip =\
+ common_services_actions.get_keepalived_vip_minion_id(vip)
+ except Exception:
+ time.sleep(15)
+ new_minion_vip = \
+ common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} migrated to {1}".format(vip, new_minion_vip))
+ assert new_minion_vip != minion_vip
+
+ show_step(6)
+ common_services_actions.check_keepalived_pillar()
+
+ show_step(7)
+ curl_output = ''.join(underlay.check_call(
+ cmd="curl -k -s 'https://{}'".format(vip),
+ host=config.salt.salt_master_host, raise_on_err=False)['stdout'])
+ assert "apiVersion" in curl_output
+
+ show_step(8)
+ k8s_actions.run_conformance(node_name=new_minion_vip)
diff --git a/tcp_tests/tests/system/test_failover_nodes.py b/tcp_tests/tests/system/test_failover_nodes.py
index 0abada9..87a7de8 100644
--- a/tcp_tests/tests/system/test_failover_nodes.py
+++ b/tcp_tests/tests/system/test_failover_nodes.py
@@ -24,8 +24,8 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
- def test_warm_shutdown_ctl01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ def test_warm_shutdown_ctl01_node(self, underlay, hardware, show_step,
+ openstack_deployed, openstack_actions):
"""Test warm shutdown ctl01
Scenario:
@@ -43,7 +43,7 @@
show_step(3)
# STEP #4
show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('ctl01')
+ hardware.warm_shutdown_nodes(underlay, 'ctl01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -52,7 +52,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
- def test_restart_ctl01_node(self, underlay, openstack_deployed,
+ def test_restart_ctl01_node(self, underlay, hardware, openstack_deployed,
openstack_actions, show_step):
"""Test restart ctl01
@@ -72,7 +72,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_restart_nodes('ctl01')
+ hardware.warm_restart_nodes(underlay, 'ctl01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -82,7 +82,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ hardware, openstack_actions, show_step):
"""Test warm shutdown cmp01
Scenario:
@@ -101,7 +101,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('cmp01')
+ hardware.warm_shutdown_nodes(underlay, 'cmp01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -111,7 +111,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
def test_restart_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ openstack_actions, show_step, hardware):
"""Test restart cmp01
Scenario:
@@ -130,7 +130,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_restart_nodes('cmp01')
+ hardware.warm_restart_nodes(underlay, 'cmp01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -140,7 +140,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon01_node(self, openstack_actions,
+ def test_restart_mon01_node(self, openstack_actions, hardware, underlay,
sl_os_deployed, show_step):
"""Test restart mon01
@@ -172,7 +172,7 @@
before_result if 'passed' not in test['outcome']]
# STEP #5
show_step(5)
- openstack_actions.warm_restart_nodes('mon01')
+ hardware.warm_restart_nodes(underlay, 'mon01')
# STEP #6
show_step(6)
sl_os_deployed.check_prometheus_targets(mon_nodes)
@@ -191,9 +191,8 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_warm_shutdown_mon01_node(self, openstack_actions,
- sl_os_deployed,
- show_step):
+ def test_warm_shutdown_mon01_node(self, underlay, hardware, sl_os_deployed,
+ openstack_actions, show_step):
"""Test warm shutdown mon01
Scenario:
@@ -223,7 +222,7 @@
before_result if 'passed' not in test['outcome']]
# STEP #5
show_step(5)
- openstack_actions.warm_shutdown_openstack_nodes('mon01')
+ hardware.warm_shutdown_nodes(underlay, 'mon01')
# STEP #6
show_step(6)
# Run SL component tetsts
@@ -239,7 +238,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon_with_vip(self, sl_os_deployed,
+ def test_restart_mon_with_vip(self, underlay, hardware, sl_os_deployed,
openstack_actions, salt_actions,
common_services_actions, show_step):
"""Test restart mon with VIP
@@ -286,7 +285,7 @@
# STEP #6
show_step(6)
- openstack_actions.warm_restart_nodes(minion_vip)
+ hardware.warm_restart_nodes(underlay, minion_vip)
# STEP #7
show_step(7)
@@ -314,7 +313,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
- def test_restart_ctl_with_vip(self, underlay, openstack_deployed,
+ def test_restart_ctl_with_vip(self, underlay, hardware, openstack_deployed,
openstack_actions, salt_actions,
common_services_actions, show_step):
"""Test restart clt with VIP
@@ -350,7 +349,7 @@
# STEP #5
show_step(5)
- openstack_actions.warm_restart_nodes(minion_vip)
+ hardware.warm_restart_nodes(underlay, minion_vip)
# STEP #6
show_step(6)