Add k8s keepalived failover test
Rewrite openstack_actions to remove hardware dependency from it.
Rewrite conformance method to allow more flexibility.
Add timeout after restarting keepalived workaround because
sometimes it did not have time to go into MASTER state before
test starts.
Change-Id: I668844f91f4a8fb473e199977f1ebe6ca7ddc35a
Related-PROD: PROD-20878
diff --git a/tcp_tests/fixtures/ceph_fixtures.py b/tcp_tests/fixtures/ceph_fixtures.py
index 0b2ef50..94315c4 100644
--- a/tcp_tests/fixtures/ceph_fixtures.py
+++ b/tcp_tests/fixtures/ceph_fixtures.py
@@ -22,18 +22,17 @@
@pytest.fixture(scope='function')
-def ceph_actions(config, hardware, underlay, salt_deployed):
+def ceph_actions(config, underlay, salt_deployed):
"""Fixture that provides various actions for OpenStack
:param config: fixture provides oslo.config
- :param config: fixture provides oslo.config
:param underlay: fixture provides underlay manager
:param salt_deployed: fixture provides salt manager
:rtype: CephManager
For use in tests or fixtures to deploy a custom OpenStack
"""
- return ceph_manager.CephManager(config, underlay, hardware, salt_deployed)
+ return ceph_manager.CephManager(config, underlay, salt_deployed)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ceph_deployed)
diff --git a/tcp_tests/fixtures/k8s_fixtures.py b/tcp_tests/fixtures/k8s_fixtures.py
index 08176fc..f59ff61 100644
--- a/tcp_tests/fixtures/k8s_fixtures.py
+++ b/tcp_tests/fixtures/k8s_fixtures.py
@@ -13,6 +13,7 @@
# under the License.
import pytest
+import time
from tcp_tests.helpers import ext
from tcp_tests.helpers import utils
@@ -79,7 +80,8 @@
LOG.warning('Restarting keepalived service on controllers...')
k8s_actions._salt.local(tgt='ctl*', fun='cmd.run',
args='systemctl restart keepalived.service')
-
+ # give some time to keepalived to enter in MASTER state
+ time.sleep(5)
return k8s_actions
diff --git a/tcp_tests/fixtures/openstack_fixtures.py b/tcp_tests/fixtures/openstack_fixtures.py
index 8e92e77..8f997a3 100644
--- a/tcp_tests/fixtures/openstack_fixtures.py
+++ b/tcp_tests/fixtures/openstack_fixtures.py
@@ -22,7 +22,7 @@
@pytest.fixture(scope='function')
-def openstack_actions(config, hardware, underlay, salt_deployed):
+def openstack_actions(config, underlay, salt_deployed):
"""Fixture that provides various actions for OpenStack
:param config: fixture provides oslo.config
@@ -33,8 +33,7 @@
For use in tests or fixtures to deploy a custom OpenStack
"""
- return openstack_manager.OpenstackManager(config, underlay,
- hardware, salt_deployed)
+ return openstack_manager.OpenstackManager(config, underlay, salt_deployed)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
diff --git a/tcp_tests/managers/ceph_manager.py b/tcp_tests/managers/ceph_manager.py
index bd68496..b2111be 100644
--- a/tcp_tests/managers/ceph_manager.py
+++ b/tcp_tests/managers/ceph_manager.py
@@ -23,12 +23,10 @@
__config = None
__underlay = None
- __hardware = None
- def __init__(self, config, underlay, hardware, salt):
+ def __init__(self, config, underlay, salt):
self.__config = config
self.__underlay = underlay
- self.__hardware = hardware
self._salt = salt
super(CephManager, self).__init__(
config=config, underlay=underlay)
diff --git a/tcp_tests/managers/envmanager_devops.py b/tcp_tests/managers/envmanager_devops.py
index d02cff5..0e56756 100644
--- a/tcp_tests/managers/envmanager_devops.py
+++ b/tcp_tests/managers/envmanager_devops.py
@@ -420,6 +420,22 @@
timeout_msg=('Node {0} failed '
'to become active'.format(node)))
+ def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600):
+ node_names = underlay.get_target_node_names(nodes_prefix)
+ for node in node_names:
+ LOG.debug('Shutdown node {0}'.format(node))
+ underlay.check_call(cmd="shutdown +1", node_name=node)
+ for node in node_names:
+ self.wait_for_node_state(node, state='offline', timeout=timeout)
+
+ def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600):
+ self.warm_shutdown_nodes(underlay, nodes_prefix, timeout)
+ node_names = underlay.get_target_node_names(nodes_prefix)
+ for node in node_names:
+ LOG.debug('Starting node {0}'.format(node))
+ self.start_node(node)
+ self.wait_for_node_state(node, state='active', timeout=timeout)
+
def has_snapshot(self, name):
return self.__env.has_snapshot(name)
diff --git a/tcp_tests/managers/envmanager_empty.py b/tcp_tests/managers/envmanager_empty.py
index b543c87..c4bb57e 100644
--- a/tcp_tests/managers/envmanager_empty.py
+++ b/tcp_tests/managers/envmanager_empty.py
@@ -104,3 +104,11 @@
def delete_environment(self):
"""Delete environment"""
pass
+
+ def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600):
+ raise Exception(
+ "Node shutdown method unsupported on this environment manager")
+
+ def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600):
+ raise Exception(
+ "Node restart method unsupported on this environment manager")
diff --git a/tcp_tests/managers/k8smanager.py b/tcp_tests/managers/k8smanager.py
index 1a6144f..38521c7 100644
--- a/tcp_tests/managers/k8smanager.py
+++ b/tcp_tests/managers/k8smanager.py
@@ -343,15 +343,17 @@
return sum(pods)
def run_conformance(self, timeout=60 * 60, log_out='k8s_conformance.log',
- raise_on_err=True):
- with self.__underlay.remote(
- node_name=self.ctl_host) as remote:
- result = remote.check_call(
- "set -o pipefail; docker run --net=host -e API_SERVER="
- "'http://127.0.0.1:8080' {0} | tee {1}".format(
- self.__config.k8s.k8s_conformance_image, log_out),
- timeout=timeout, raise_on_err=raise_on_err)['stdout']
- return result
+ raise_on_err=True, node_name=None,
+ api_server='http://127.0.0.1:8080'):
+ if node_name is None:
+ node_name = self.ctl_host
+ cmd = "set -o pipefail; docker run --net=host -e API_SERVER="\
+ "'{api}' {image} | tee '{log}'".format(
+ api=api_server, image=self.__config.k8s.k8s_conformance_image,
+ log=log_out)
+ return self.__underlay.check_call(
+ cmd=cmd, node_name=node_name, timeout=timeout,
+ raise_on_err=raise_on_err)
def get_k8s_masters(self):
k8s_masters_fqdn = self._salt.get_pillar(tgt='I@kubernetes:master',
@@ -704,3 +706,14 @@
update_commands = self.__underlay.read_template(steps_path)
self.execute_commands(
update_commands, label="Updating kubernetes to '{}'".format(tag))
+
+ def get_keepalived_vip(self):
+ """
+ Return k8s VIP IP address
+
+ :return: str, IP address
+ """
+ ctl_vip_pillar = self._salt.get_pillar(
+ tgt="I@kubernetes:control:enabled:True",
+ pillar="_param:cluster_vip_address")[0]
+ return [vip for minion_id, vip in ctl_vip_pillar.items()][0]
diff --git a/tcp_tests/managers/openstack_manager.py b/tcp_tests/managers/openstack_manager.py
index 464dc56..d47aceb 100644
--- a/tcp_tests/managers/openstack_manager.py
+++ b/tcp_tests/managers/openstack_manager.py
@@ -26,12 +26,10 @@
__config = None
__underlay = None
- __hardware = None
- def __init__(self, config, underlay, hardware, salt):
+ def __init__(self, config, underlay, salt):
self.__config = config
self.__underlay = underlay
- self.__hardware = hardware
self._salt = salt
super(OpenstackManager, self).__init__(
config=config, underlay=underlay)
@@ -164,37 +162,6 @@
LOG.debug("Found files {0}".format(file_name))
r.download(destination=file_name, target=os.getcwd())
- def get_node_name_by_subname(self, node_sub_name):
- return [node_name for node_name
- in self.__underlay.node_names()
- if node_sub_name in node_name]
-
- def warm_shutdown_openstack_nodes(self, node_sub_name, timeout=10 * 60):
- """Gracefully shutting down the node """
- node_names = self.get_node_name_by_subname(node_sub_name)
- LOG.info('Shutting down nodes {}'.format(node_names))
- for node in node_names:
- LOG.debug('Shutdown node {0}'.format(node))
- self.__underlay.check_call(cmd="shutdown +1", node_name=node)
- for node in node_names:
- LOG.info('Destroy node {}'.format(node))
- self.__hardware.destroy_node(node)
- self.__hardware.wait_for_node_state(
- node, state='offline', timeout=timeout)
-
- def warm_start_nodes(self, node_sub_name, timeout=10 * 60):
- node_names = self.get_node_name_by_subname(node_sub_name)
- LOG.info('Starting nodes {}'.format(node_names))
- for node in node_names:
- self.__hardware.start_node(node)
- self.__hardware.wait_for_node_state(
- node, state='active', timeout=timeout)
-
- def warm_restart_nodes(self, node_names, timeout=10 * 60):
- LOG.info('Reboot (warm restart) nodes {0}'.format(node_names))
- self.warm_shutdown_openstack_nodes(node_names, timeout=timeout)
- self.warm_start_nodes(node_names)
-
def auth_in_horizon(self, host, port, user, password):
client = requests.session()
url = "http://{0}:{1}".format(
diff --git a/tcp_tests/tests/system/test_failover_ceph.py b/tcp_tests/tests/system/test_failover_ceph.py
index 4a68705..2f1bc25 100644
--- a/tcp_tests/tests/system/test_failover_ceph.py
+++ b/tcp_tests/tests/system/test_failover_ceph.py
@@ -44,7 +44,7 @@
@pytest.mark.fail_snapshot
def test_restart_osd_node(self, func_name, underlay, config,
openstack_deployed, ceph_deployed,
- openstack_actions,
+ openstack_actions, hardware,
rally, show_step):
"""Test restart ceph osd node
@@ -78,7 +78,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('osd01')
+ hardware.warm_restart_nodes(underlay, 'osd01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
@@ -112,7 +112,7 @@
openstack_deployed, ceph_deployed,
common_services_actions,
salt_actions, openstack_actions,
- rally, show_step):
+ rally, show_step, hardware):
"""Test restart ceph cmn node
Scenario:
@@ -145,7 +145,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('cmn01')
+ hardware.warm_restart_nodes(underlay, 'cmn01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
@@ -177,7 +177,7 @@
@pytest.mark.fail_snapshot
def test_restart_rgw_node(self, func_name, underlay, config,
openstack_deployed, ceph_deployed,
- common_services_actions,
+ common_services_actions, hardware,
salt_actions, openstack_actions,
rally, show_step):
"""Test restart ceph rgw node
@@ -215,7 +215,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('rgw01')
+ hardware.warm_restart_nodes(underlay, 'rgw01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
diff --git a/tcp_tests/tests/system/test_failover_k8s.py b/tcp_tests/tests/system/test_failover_k8s.py
new file mode 100644
index 0000000..1ad43b9
--- /dev/null
+++ b/tcp_tests/tests/system/test_failover_k8s.py
@@ -0,0 +1,75 @@
+# Copyright 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import pytest
+import time
+
+from tcp_tests import logger
+
+LOG = logger.logger
+
+
+class TestFailoverK8s(object):
+
+ @pytest.mark.grap_versions
+ @pytest.mark.fail_snapshot
+ def test_k8s_master_vip_migration(self, show_step, k8s_deployed, underlay,
+ k8s_actions, common_services_actions,
+ config, hardware):
+ """Test restart and shutdown master with VIP
+
+ Scenario:
+ 1. Deploy mcp with k8s ha
+ 2. Check keepalived pillar configuration
+ 3. Find master node with assigned VIP
+ 4. Reboot server with VIP
+ 5. Check that VIP was migrated
+ 6. Check keepalived pillar configuration
+ 7. Check api server availability
+ 8. Run conformance on node with VIP
+ """
+ show_step(1)
+ show_step(2)
+ common_services_actions.check_keepalived_pillar()
+
+ show_step(3)
+ vip = k8s_actions.get_keepalived_vip()
+ LOG.info("VIP ip address: {}".format(vip))
+ minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
+
+ show_step(4)
+ hardware.warm_restart_nodes(underlay, minion_vip)
+
+ show_step(5)
+ try:
+ new_minion_vip =\
+ common_services_actions.get_keepalived_vip_minion_id(vip)
+ except Exception:
+ time.sleep(15)
+ new_minion_vip = \
+ common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} migrated to {1}".format(vip, new_minion_vip))
+ assert new_minion_vip != minion_vip
+
+ show_step(6)
+ common_services_actions.check_keepalived_pillar()
+
+ show_step(7)
+ curl_output = ''.join(underlay.check_call(
+ cmd="curl -k -s 'https://{}'".format(vip),
+ host=config.salt.salt_master_host, raise_on_err=False)['stdout'])
+ assert "apiVersion" in curl_output
+
+ show_step(8)
+ k8s_actions.run_conformance(node_name=new_minion_vip)
diff --git a/tcp_tests/tests/system/test_failover_nodes.py b/tcp_tests/tests/system/test_failover_nodes.py
index 0abada9..87a7de8 100644
--- a/tcp_tests/tests/system/test_failover_nodes.py
+++ b/tcp_tests/tests/system/test_failover_nodes.py
@@ -24,8 +24,8 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
- def test_warm_shutdown_ctl01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ def test_warm_shutdown_ctl01_node(self, underlay, hardware, show_step,
+ openstack_deployed, openstack_actions):
"""Test warm shutdown ctl01
Scenario:
@@ -43,7 +43,7 @@
show_step(3)
# STEP #4
show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('ctl01')
+ hardware.warm_shutdown_nodes(underlay, 'ctl01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -52,7 +52,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
- def test_restart_ctl01_node(self, underlay, openstack_deployed,
+ def test_restart_ctl01_node(self, underlay, hardware, openstack_deployed,
openstack_actions, show_step):
"""Test restart ctl01
@@ -72,7 +72,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_restart_nodes('ctl01')
+ hardware.warm_restart_nodes(underlay, 'ctl01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -82,7 +82,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ hardware, openstack_actions, show_step):
"""Test warm shutdown cmp01
Scenario:
@@ -101,7 +101,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('cmp01')
+ hardware.warm_shutdown_nodes(underlay, 'cmp01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -111,7 +111,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
def test_restart_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ openstack_actions, show_step, hardware):
"""Test restart cmp01
Scenario:
@@ -130,7 +130,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_restart_nodes('cmp01')
+ hardware.warm_restart_nodes(underlay, 'cmp01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -140,7 +140,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon01_node(self, openstack_actions,
+ def test_restart_mon01_node(self, openstack_actions, hardware, underlay,
sl_os_deployed, show_step):
"""Test restart mon01
@@ -172,7 +172,7 @@
before_result if 'passed' not in test['outcome']]
# STEP #5
show_step(5)
- openstack_actions.warm_restart_nodes('mon01')
+ hardware.warm_restart_nodes(underlay, 'mon01')
# STEP #6
show_step(6)
sl_os_deployed.check_prometheus_targets(mon_nodes)
@@ -191,9 +191,8 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_warm_shutdown_mon01_node(self, openstack_actions,
- sl_os_deployed,
- show_step):
+ def test_warm_shutdown_mon01_node(self, underlay, hardware, sl_os_deployed,
+ openstack_actions, show_step):
"""Test warm shutdown mon01
Scenario:
@@ -223,7 +222,7 @@
before_result if 'passed' not in test['outcome']]
# STEP #5
show_step(5)
- openstack_actions.warm_shutdown_openstack_nodes('mon01')
+ hardware.warm_shutdown_nodes(underlay, 'mon01')
# STEP #6
show_step(6)
# Run SL component tetsts
@@ -239,7 +238,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon_with_vip(self, sl_os_deployed,
+ def test_restart_mon_with_vip(self, underlay, hardware, sl_os_deployed,
openstack_actions, salt_actions,
common_services_actions, show_step):
"""Test restart mon with VIP
@@ -286,7 +285,7 @@
# STEP #6
show_step(6)
- openstack_actions.warm_restart_nodes(minion_vip)
+ hardware.warm_restart_nodes(underlay, minion_vip)
# STEP #7
show_step(7)
@@ -314,7 +313,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
- def test_restart_ctl_with_vip(self, underlay, openstack_deployed,
+ def test_restart_ctl_with_vip(self, underlay, hardware, openstack_deployed,
openstack_actions, salt_actions,
common_services_actions, show_step):
"""Test restart clt with VIP
@@ -350,7 +349,7 @@
# STEP #5
show_step(5)
- openstack_actions.warm_restart_nodes(minion_vip)
+ hardware.warm_restart_nodes(underlay, minion_vip)
# STEP #6
show_step(6)