Add initial failover tests
* Warm restart of node by role
* Warm shutdown node by role
* Run func tests after fail
Need to do: assert test results for fails
Change-Id: I51934227340a60d5150e14f5004f2c3575264a36
diff --git a/tcp_tests/managers/openstack_manager.py b/tcp_tests/managers/openstack_manager.py
index 6454bdd..ebcc574 100644
--- a/tcp_tests/managers/openstack_manager.py
+++ b/tcp_tests/managers/openstack_manager.py
@@ -24,10 +24,12 @@
__config = None
__underlay = None
+ __hardware = None
- def __init__(self, config, underlay, salt):
+ def __init__(self, config, underlay, hardware, salt):
self.__config = config
self.__underlay = underlay
+ self.__hardware = hardware
self._salt = salt
super(OpenstackManager, self).__init__(
config=config, underlay=underlay)
@@ -88,3 +90,34 @@
file_name = result['stdout'][0].rstrip()
LOG.debug("Found files {0}".format(file_name))
r.download(destination=file_name, target=os.getcwd())
+
+ def get_node_name_by_subname(self, node_sub_name):
+ return [node_name for node_name
+ in self.__underlay.node_names()
+ if node_sub_name in node_name]
+
+ def warm_shutdown_openstack_nodes(self, node_sub_name, timeout=10 * 60):
+ """Gracefully shutting down the node """
+ node_names = self.get_node_name_by_subname(node_sub_name)
+ LOG.info('Shutting down nodes {}'.format(node_names))
+ for node in node_names:
+ LOG.debug('Shutdown node {0}'.format(node))
+ self.__underlay.check_call(cmd="shutdown +1", node_name=node)
+ for node in node_names:
+ LOG.info('Destroy node {}'.format(node))
+ self.__hardware.destroy_node(node)
+ self.__hardware.wait_for_node_state(
+ node, state='offline', timeout=timeout)
+
+ def warm_start_nodes(self, node_sub_name, timeout=10 * 60):
+ node_names = self.get_node_name_by_subname(node_sub_name)
+ LOG.info('Starting nodes {}'.format(node_names))
+ for node in node_names:
+ self.__hardware.start_node(node)
+ self.__hardware.wait_for_node_state(
+ node, state='active', timeout=timeout)
+
+ def warm_restart_nodes(self, node_names, timeout=10 * 60):
+ LOG.info('Reboot (warm restart) nodes {0}'.format(node_names))
+ self.warm_shutdown_openstack_nodes(node_names, timeout=timeout)
+ self.warm_start_nodes(node_names)