Workaround for uncomplete resources in complete stack
Change-Id: Ic35e8d83939fa65789931934963e8bc168726647
diff --git a/tcp_tests/managers/envmanager_heat.py b/tcp_tests/managers/envmanager_heat.py
index 77ba079..8d1e50e 100644
--- a/tcp_tests/managers/envmanager_heat.py
+++ b/tcp_tests/managers/envmanager_heat.py
@@ -44,7 +44,7 @@
LOG = logger.logger
EXPECTED_STACK_STATUS = "CREATE_COMPLETE"
-BAD_STACK_STATUSES = ["CREATE_FAILED"]
+BAD_STACK_STATUSES = ["CREATE_FAILED", "DELETE_FAILED"]
# Disable multiple notifications like:
# "InsecureRequestWarning: Unverified HTTPS request is being made."
@@ -331,11 +331,17 @@
])
return wrong_resources
- def wait_of_stack_status(self, status, delay=30, tries=60):
+ def wait_of_stack_status(self, status, delay=30, tries=60,
+ wait_for_delete=False):
@retry(exceptions.EnvironmentWrongStatus, delay=delay, tries=tries)
def wait():
- st = self._current_stack.stack_status
+ try:
+ st = self._current_stack.stack_status
+ except heat_exceptions.HTTPNotFound as ex:
+ if wait_for_delete is True:
+ return
+ raise ex
if st == status:
return
elif st in BAD_STACK_STATUSES:
@@ -508,6 +514,16 @@
LOG.info('Heat stack "{0}" ready'
.format(self.__config.hardware.heat_stack_name))
+ def _verify_resources_status(self, status):
+ """Check that all resources have verified `status`
+
+ In case when all resources have expected status return empty list,
+ otherwise return a list with resources with incorrect status.
+ """
+ ret = [r for r in self.__nested_resources if
+ r.resource_status != status]
+ return ret
+
def _create_environment(self):
tpl_files, template = template_utils.get_template_contents(
self.__config.hardware.heat_conf_path)
@@ -530,16 +546,60 @@
if env_files_list:
fields['environment_files'] = env_files_list
- self.__stacks.create(**fields)
- self.wait_of_stack_status(EXPECTED_STACK_STATUS, tries=140)
- LOG.info("Stack '{0}' created".format(
- self.__config.hardware.heat_stack_name))
+
+ @retry(heat_exceptions.HTTPBadGateway, delay=15, tries=20)
+ def safe_heat_stack_create():
+ self.__stacks.create(**fields)
+
+ @retry(exceptions.EnvironmentBadStatus, delay=60, tries=3)
+ def safe_create():
+ self.delete_environment()
+ safe_heat_stack_create()
+ self.wait_of_stack_status(EXPECTED_STACK_STATUS, tries=140)
+ LOG.info("Stack '%s' created",
+ self.__config.hardware.heat_stack_name)
+ incorrect_resources = self._verify_resources_status(
+ EXPECTED_STACK_STATUS)
+ if incorrect_resources:
+ LOG.info("Recreate the stack because some resources have "
+ "incorrect status")
+ for r in incorrect_resources:
+ LOG.error(
+ 'The resource %s has status %s. But it should be %s',
+ r.resource_name,
+ r.resource_status,
+ EXPECTED_STACK_STATUS)
+ st = self._current_stack.stack_status
+ raise exceptions.EnvironmentBadStatus(
+ self.__config.hardware.heat_stack_name,
+ EXPECTED_STACK_STATUS,
+ st,
+ incorrect_resources)
+ safe_create()
def stop(self):
"""Stop environment"""
LOG.warning("HEAT Manager doesn't support stop environment feature")
pass
+ def delete_environment(self):
+ if list(self.__stacks.list(
+ stack_name=self.__config.hardware.heat_stack_name)):
+ LOG.info("Delete stack '%s'",
+ self.__config.hardware.heat_stack_name)
+
+ @retry(heat_exceptions.HTTPBadGateway, delay=15, tries=20)
+ def safe_heat_stack_delete():
+ self.__stacks.delete(self._current_stack.id)
+
+ safe_heat_stack_delete()
+ self.wait_of_stack_status('DELETE_COMPLETE',
+ delay=30, tries=20,
+ wait_for_delete=True)
+ else:
+ LOG.warning("Can't delete stack '%s' due is absent",
+ self.__config.hardware.heat_stack_name)
+
# TODO(ddmitriev): add all Environment methods
@staticmethod
def node_ip(node, address_pool_name='admin-pool01'):