Merge "Add k8s keepalived failover test"
diff --git a/tcp_tests/fixtures/ceph_fixtures.py b/tcp_tests/fixtures/ceph_fixtures.py
index 0b2ef50..94315c4 100644
--- a/tcp_tests/fixtures/ceph_fixtures.py
+++ b/tcp_tests/fixtures/ceph_fixtures.py
@@ -22,18 +22,17 @@
@pytest.fixture(scope='function')
-def ceph_actions(config, hardware, underlay, salt_deployed):
+def ceph_actions(config, underlay, salt_deployed):
"""Fixture that provides various actions for OpenStack
:param config: fixture provides oslo.config
- :param config: fixture provides oslo.config
:param underlay: fixture provides underlay manager
:param salt_deployed: fixture provides salt manager
:rtype: CephManager
For use in tests or fixtures to deploy a custom OpenStack
"""
- return ceph_manager.CephManager(config, underlay, hardware, salt_deployed)
+ return ceph_manager.CephManager(config, underlay, salt_deployed)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ceph_deployed)
diff --git a/tcp_tests/fixtures/k8s_fixtures.py b/tcp_tests/fixtures/k8s_fixtures.py
index 08176fc..f59ff61 100644
--- a/tcp_tests/fixtures/k8s_fixtures.py
+++ b/tcp_tests/fixtures/k8s_fixtures.py
@@ -13,6 +13,7 @@
# under the License.
import pytest
+import time
from tcp_tests.helpers import ext
from tcp_tests.helpers import utils
@@ -79,7 +80,8 @@
LOG.warning('Restarting keepalived service on controllers...')
k8s_actions._salt.local(tgt='ctl*', fun='cmd.run',
args='systemctl restart keepalived.service')
-
+ # give some time to keepalived to enter in MASTER state
+ time.sleep(5)
return k8s_actions
diff --git a/tcp_tests/fixtures/openstack_fixtures.py b/tcp_tests/fixtures/openstack_fixtures.py
index 8e92e77..8f997a3 100644
--- a/tcp_tests/fixtures/openstack_fixtures.py
+++ b/tcp_tests/fixtures/openstack_fixtures.py
@@ -22,7 +22,7 @@
@pytest.fixture(scope='function')
-def openstack_actions(config, hardware, underlay, salt_deployed):
+def openstack_actions(config, underlay, salt_deployed):
"""Fixture that provides various actions for OpenStack
:param config: fixture provides oslo.config
@@ -33,8 +33,7 @@
For use in tests or fixtures to deploy a custom OpenStack
"""
- return openstack_manager.OpenstackManager(config, underlay,
- hardware, salt_deployed)
+ return openstack_manager.OpenstackManager(config, underlay, salt_deployed)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
diff --git a/tcp_tests/managers/ceph_manager.py b/tcp_tests/managers/ceph_manager.py
index bd68496..b2111be 100644
--- a/tcp_tests/managers/ceph_manager.py
+++ b/tcp_tests/managers/ceph_manager.py
@@ -23,12 +23,10 @@
__config = None
__underlay = None
- __hardware = None
- def __init__(self, config, underlay, hardware, salt):
+ def __init__(self, config, underlay, salt):
self.__config = config
self.__underlay = underlay
- self.__hardware = hardware
self._salt = salt
super(CephManager, self).__init__(
config=config, underlay=underlay)
diff --git a/tcp_tests/managers/envmanager_devops.py b/tcp_tests/managers/envmanager_devops.py
index 79bec94..7424a49 100644
--- a/tcp_tests/managers/envmanager_devops.py
+++ b/tcp_tests/managers/envmanager_devops.py
@@ -427,6 +427,22 @@
timeout_msg=('Node {0} failed '
'to become active'.format(node)))
+ def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600):
+ node_names = underlay.get_target_node_names(nodes_prefix)
+ for node in node_names:
+ LOG.debug('Shutdown node {0}'.format(node))
+ underlay.check_call(cmd="shutdown +1", node_name=node)
+ for node in node_names:
+ self.wait_for_node_state(node, state='offline', timeout=timeout)
+
+ def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600):
+ self.warm_shutdown_nodes(underlay, nodes_prefix, timeout)
+ node_names = underlay.get_target_node_names(nodes_prefix)
+ for node in node_names:
+ LOG.debug('Starting node {0}'.format(node))
+ self.start_node(node)
+ self.wait_for_node_state(node, state='active', timeout=timeout)
+
def has_snapshot(self, name):
return self.__env.has_snapshot(name)
diff --git a/tcp_tests/managers/envmanager_empty.py b/tcp_tests/managers/envmanager_empty.py
index b543c87..c4bb57e 100644
--- a/tcp_tests/managers/envmanager_empty.py
+++ b/tcp_tests/managers/envmanager_empty.py
@@ -104,3 +104,11 @@
def delete_environment(self):
"""Delete environment"""
pass
+
+ def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600):
+ raise Exception(
+ "Node shutdown method unsupported on this environment manager")
+
+ def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600):
+ raise Exception(
+ "Node restart method unsupported on this environment manager")
diff --git a/tcp_tests/managers/k8smanager.py b/tcp_tests/managers/k8smanager.py
index 1a6144f..38521c7 100644
--- a/tcp_tests/managers/k8smanager.py
+++ b/tcp_tests/managers/k8smanager.py
@@ -343,15 +343,17 @@
return sum(pods)
def run_conformance(self, timeout=60 * 60, log_out='k8s_conformance.log',
- raise_on_err=True):
- with self.__underlay.remote(
- node_name=self.ctl_host) as remote:
- result = remote.check_call(
- "set -o pipefail; docker run --net=host -e API_SERVER="
- "'http://127.0.0.1:8080' {0} | tee {1}".format(
- self.__config.k8s.k8s_conformance_image, log_out),
- timeout=timeout, raise_on_err=raise_on_err)['stdout']
- return result
+ raise_on_err=True, node_name=None,
+ api_server='http://127.0.0.1:8080'):
+ if node_name is None:
+ node_name = self.ctl_host
+ cmd = "set -o pipefail; docker run --net=host -e API_SERVER="\
+ "'{api}' {image} | tee '{log}'".format(
+ api=api_server, image=self.__config.k8s.k8s_conformance_image,
+ log=log_out)
+ return self.__underlay.check_call(
+ cmd=cmd, node_name=node_name, timeout=timeout,
+ raise_on_err=raise_on_err)
def get_k8s_masters(self):
k8s_masters_fqdn = self._salt.get_pillar(tgt='I@kubernetes:master',
@@ -704,3 +706,14 @@
update_commands = self.__underlay.read_template(steps_path)
self.execute_commands(
update_commands, label="Updating kubernetes to '{}'".format(tag))
+
+ def get_keepalived_vip(self):
+ """
+ Return k8s VIP IP address
+
+ :return: str, IP address
+ """
+ ctl_vip_pillar = self._salt.get_pillar(
+ tgt="I@kubernetes:control:enabled:True",
+ pillar="_param:cluster_vip_address")[0]
+ return [vip for minion_id, vip in ctl_vip_pillar.items()][0]
diff --git a/tcp_tests/managers/openstack_manager.py b/tcp_tests/managers/openstack_manager.py
index 464dc56..d47aceb 100644
--- a/tcp_tests/managers/openstack_manager.py
+++ b/tcp_tests/managers/openstack_manager.py
@@ -26,12 +26,10 @@
__config = None
__underlay = None
- __hardware = None
- def __init__(self, config, underlay, hardware, salt):
+ def __init__(self, config, underlay, salt):
self.__config = config
self.__underlay = underlay
- self.__hardware = hardware
self._salt = salt
super(OpenstackManager, self).__init__(
config=config, underlay=underlay)
@@ -164,37 +162,6 @@
LOG.debug("Found files {0}".format(file_name))
r.download(destination=file_name, target=os.getcwd())
- def get_node_name_by_subname(self, node_sub_name):
- return [node_name for node_name
- in self.__underlay.node_names()
- if node_sub_name in node_name]
-
- def warm_shutdown_openstack_nodes(self, node_sub_name, timeout=10 * 60):
- """Gracefully shutting down the node """
- node_names = self.get_node_name_by_subname(node_sub_name)
- LOG.info('Shutting down nodes {}'.format(node_names))
- for node in node_names:
- LOG.debug('Shutdown node {0}'.format(node))
- self.__underlay.check_call(cmd="shutdown +1", node_name=node)
- for node in node_names:
- LOG.info('Destroy node {}'.format(node))
- self.__hardware.destroy_node(node)
- self.__hardware.wait_for_node_state(
- node, state='offline', timeout=timeout)
-
- def warm_start_nodes(self, node_sub_name, timeout=10 * 60):
- node_names = self.get_node_name_by_subname(node_sub_name)
- LOG.info('Starting nodes {}'.format(node_names))
- for node in node_names:
- self.__hardware.start_node(node)
- self.__hardware.wait_for_node_state(
- node, state='active', timeout=timeout)
-
- def warm_restart_nodes(self, node_names, timeout=10 * 60):
- LOG.info('Reboot (warm restart) nodes {0}'.format(node_names))
- self.warm_shutdown_openstack_nodes(node_names, timeout=timeout)
- self.warm_start_nodes(node_names)
-
def auth_in_horizon(self, host, port, user, password):
client = requests.session()
url = "http://{0}:{1}".format(
diff --git a/tcp_tests/tests/system/test_failover_ceph.py b/tcp_tests/tests/system/test_failover_ceph.py
index 4a68705..2f1bc25 100644
--- a/tcp_tests/tests/system/test_failover_ceph.py
+++ b/tcp_tests/tests/system/test_failover_ceph.py
@@ -44,7 +44,7 @@
@pytest.mark.fail_snapshot
def test_restart_osd_node(self, func_name, underlay, config,
openstack_deployed, ceph_deployed,
- openstack_actions,
+ openstack_actions, hardware,
rally, show_step):
"""Test restart ceph osd node
@@ -78,7 +78,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('osd01')
+ hardware.warm_restart_nodes(underlay, 'osd01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
@@ -112,7 +112,7 @@
openstack_deployed, ceph_deployed,
common_services_actions,
salt_actions, openstack_actions,
- rally, show_step):
+ rally, show_step, hardware):
"""Test restart ceph cmn node
Scenario:
@@ -145,7 +145,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('cmn01')
+ hardware.warm_restart_nodes(underlay, 'cmn01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
@@ -177,7 +177,7 @@
@pytest.mark.fail_snapshot
def test_restart_rgw_node(self, func_name, underlay, config,
openstack_deployed, ceph_deployed,
- common_services_actions,
+ common_services_actions, hardware,
salt_actions, openstack_actions,
rally, show_step):
"""Test restart ceph rgw node
@@ -215,7 +215,7 @@
# STEP #3
show_step(3)
- openstack_actions.warm_restart_nodes('rgw01')
+ hardware.warm_restart_nodes(underlay, 'rgw01')
openstack_actions._salt.local(
tgt='*', fun='cmd.run',
diff --git a/tcp_tests/tests/system/test_failover_k8s.py b/tcp_tests/tests/system/test_failover_k8s.py
new file mode 100644
index 0000000..1ad43b9
--- /dev/null
+++ b/tcp_tests/tests/system/test_failover_k8s.py
@@ -0,0 +1,75 @@
+# Copyright 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import pytest
+import time
+
+from tcp_tests import logger
+
+LOG = logger.logger
+
+
+class TestFailoverK8s(object):
+
+ @pytest.mark.grap_versions
+ @pytest.mark.fail_snapshot
+ def test_k8s_master_vip_migration(self, show_step, k8s_deployed, underlay,
+ k8s_actions, common_services_actions,
+ config, hardware):
+ """Test restart and shutdown master with VIP
+
+ Scenario:
+ 1. Deploy mcp with k8s ha
+ 2. Check keepalived pillar configuration
+ 3. Find master node with assigned VIP
+ 4. Reboot server with VIP
+ 5. Check that VIP was migrated
+ 6. Check keepalived pillar configuration
+ 7. Check api server availability
+ 8. Run conformance on node with VIP
+ """
+ show_step(1)
+ show_step(2)
+ common_services_actions.check_keepalived_pillar()
+
+ show_step(3)
+ vip = k8s_actions.get_keepalived_vip()
+ LOG.info("VIP ip address: {}".format(vip))
+ minion_vip = common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} is on {1}".format(vip, minion_vip))
+
+ show_step(4)
+ hardware.warm_restart_nodes(underlay, minion_vip)
+
+ show_step(5)
+ try:
+ new_minion_vip =\
+ common_services_actions.get_keepalived_vip_minion_id(vip)
+ except Exception:
+ time.sleep(15)
+ new_minion_vip = \
+ common_services_actions.get_keepalived_vip_minion_id(vip)
+ LOG.info("VIP {0} migrated to {1}".format(vip, new_minion_vip))
+ assert new_minion_vip != minion_vip
+
+ show_step(6)
+ common_services_actions.check_keepalived_pillar()
+
+ show_step(7)
+ curl_output = ''.join(underlay.check_call(
+ cmd="curl -k -s 'https://{}'".format(vip),
+ host=config.salt.salt_master_host, raise_on_err=False)['stdout'])
+ assert "apiVersion" in curl_output
+
+ show_step(8)
+ k8s_actions.run_conformance(node_name=new_minion_vip)
diff --git a/tcp_tests/tests/system/test_failover_nodes.py b/tcp_tests/tests/system/test_failover_nodes.py
index 0abada9..87a7de8 100644
--- a/tcp_tests/tests/system/test_failover_nodes.py
+++ b/tcp_tests/tests/system/test_failover_nodes.py
@@ -24,8 +24,8 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
- def test_warm_shutdown_ctl01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ def test_warm_shutdown_ctl01_node(self, underlay, hardware, show_step,
+ openstack_deployed, openstack_actions):
"""Test warm shutdown ctl01
Scenario:
@@ -43,7 +43,7 @@
show_step(3)
# STEP #4
show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('ctl01')
+ hardware.warm_shutdown_nodes(underlay, 'ctl01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -52,7 +52,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
- def test_restart_ctl01_node(self, underlay, openstack_deployed,
+ def test_restart_ctl01_node(self, underlay, hardware, openstack_deployed,
openstack_actions, show_step):
"""Test restart ctl01
@@ -72,7 +72,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_restart_nodes('ctl01')
+ hardware.warm_restart_nodes(underlay, 'ctl01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -82,7 +82,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
def test_warm_shutdown_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ hardware, openstack_actions, show_step):
"""Test warm shutdown cmp01
Scenario:
@@ -101,7 +101,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_shutdown_openstack_nodes('cmp01')
+ hardware.warm_shutdown_nodes(underlay, 'cmp01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -111,7 +111,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
def test_restart_cmp01_node(self, underlay, openstack_deployed,
- openstack_actions, show_step):
+ openstack_actions, show_step, hardware):
"""Test restart cmp01
Scenario:
@@ -130,7 +130,7 @@
# STEP #4
show_step(4)
- openstack_actions.warm_restart_nodes('cmp01')
+ hardware.warm_restart_nodes(underlay, 'cmp01')
# STEP #5
show_step(5)
openstack_actions.run_tempest(pattern='smoke')
@@ -140,7 +140,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon01_node(self, openstack_actions,
+ def test_restart_mon01_node(self, openstack_actions, hardware, underlay,
sl_os_deployed, show_step):
"""Test restart mon01
@@ -172,7 +172,7 @@
before_result if 'passed' not in test['outcome']]
# STEP #5
show_step(5)
- openstack_actions.warm_restart_nodes('mon01')
+ hardware.warm_restart_nodes(underlay, 'mon01')
# STEP #6
show_step(6)
sl_os_deployed.check_prometheus_targets(mon_nodes)
@@ -191,9 +191,8 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_warm_shutdown_mon01_node(self, openstack_actions,
- sl_os_deployed,
- show_step):
+ def test_warm_shutdown_mon01_node(self, underlay, hardware, sl_os_deployed,
+ openstack_actions, show_step):
"""Test warm shutdown mon01
Scenario:
@@ -223,7 +222,7 @@
before_result if 'passed' not in test['outcome']]
# STEP #5
show_step(5)
- openstack_actions.warm_shutdown_openstack_nodes('mon01')
+ hardware.warm_shutdown_nodes(underlay, 'mon01')
# STEP #6
show_step(6)
# Run SL component tetsts
@@ -239,7 +238,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.sl_deployed)
- def test_restart_mon_with_vip(self, sl_os_deployed,
+ def test_restart_mon_with_vip(self, underlay, hardware, sl_os_deployed,
openstack_actions, salt_actions,
common_services_actions, show_step):
"""Test restart mon with VIP
@@ -286,7 +285,7 @@
# STEP #6
show_step(6)
- openstack_actions.warm_restart_nodes(minion_vip)
+ hardware.warm_restart_nodes(underlay, minion_vip)
# STEP #7
show_step(7)
@@ -314,7 +313,7 @@
@pytest.mark.grab_versions
@pytest.mark.fail_snapshot
@pytest.mark.revert_snapshot(ext.SNAPSHOT.openstack_deployed)
- def test_restart_ctl_with_vip(self, underlay, openstack_deployed,
+ def test_restart_ctl_with_vip(self, underlay, hardware, openstack_deployed,
openstack_actions, salt_actions,
common_services_actions, show_step):
"""Test restart clt with VIP
@@ -350,7 +349,7 @@
# STEP #5
show_step(5)
- openstack_actions.warm_restart_nodes(minion_vip)
+ hardware.warm_restart_nodes(underlay, minion_vip)
# STEP #6
show_step(6)