Add integration test for UPDATE_FAILED recovery
It's valid to attempt an update from UPDATE_FAILED state, so add
a functional test which proves this works.
This requires an adjustment to the common test.py _verify_status
because we now need to ensure we record the time of transition to
FAILED state as well as COMPLETE, or we'll mis-detect the failure
as a failure of the subsequent recovery update.
Co-Authored-By: Sergey Kraynev <skraynev@mirantis.com>
Change-Id: I5fc82b4c71a943ba15c04b08bcfd26fcb84dc7a4
diff --git a/common/test.py b/common/test.py
index 1039625..46911f0 100644
--- a/common/test.py
+++ b/common/test.py
@@ -250,9 +250,9 @@
def _verify_status(self, stack, stack_identifier, status, fail_regexp):
if stack.stack_status == status:
- # Handle UPDATE_COMPLETE case: Make sure we don't
- # wait for a stale UPDATE_COMPLETE status.
- if status == 'UPDATE_COMPLETE':
+ # Handle UPDATE_COMPLETE/FAILED case: Make sure we don't
+ # wait for a stale UPDATE_COMPLETE/FAILED status.
+ if status in ('UPDATE_FAILED', 'UPDATE_COMPLETE'):
if self.updated_time.get(
stack_identifier) != stack.updated_time:
self.updated_time[stack_identifier] = stack.updated_time
@@ -263,8 +263,8 @@
wait_for_action = status.split('_')[0]
if (stack.action == wait_for_action and
fail_regexp.search(stack.stack_status)):
- # Handle UPDATE_FAILED case.
- if status == 'UPDATE_FAILED':
+ # Handle UPDATE_COMPLETE/UPDATE_FAILED case.
+ if status in ('UPDATE_FAILED', 'UPDATE_COMPLETE'):
if self.updated_time.get(
stack_identifier) != stack.updated_time:
self.updated_time[stack_identifier] = stack.updated_time
@@ -279,7 +279,7 @@
stack_status_reason=stack.stack_status_reason)
def _wait_for_stack_status(self, stack_identifier, status,
- failure_pattern='^.*_FAILED$',
+ failure_pattern=None,
success_on_not_found=False):
"""
Waits for a Stack to reach a given status.
@@ -288,7 +288,13 @@
CREATE_COMPLETE, not just COMPLETE which is exposed
via the status property of Stack in heatclient
"""
- fail_regexp = re.compile(failure_pattern)
+ if failure_pattern:
+ fail_regexp = re.compile(failure_pattern)
+ elif 'FAILED' in status:
+ # If we're looking for e.g CREATE_FAILED, COMPLETE is unexpected.
+ fail_regexp = re.compile('^.*_COMPLETE$')
+ else:
+ fail_regexp = re.compile('^.*_FAILED$')
build_timeout = self.conf.build_timeout
build_interval = self.conf.build_interval
@@ -335,6 +341,8 @@
build_timeout = self.conf.build_timeout
build_interval = self.conf.build_interval
start = timeutils.utcnow()
+ self.updated_time[stack_identifier] = self.client.stacks.get(
+ stack_identifier).updated_time
while timeutils.delta_seconds(start,
timeutils.utcnow()) < build_timeout:
try:
diff --git a/functional/test_create_update.py b/functional/test_create_update.py
index b2aad34..1b273dd 100644
--- a/functional/test_create_update.py
+++ b/functional/test_create_update.py
@@ -243,6 +243,29 @@
self.assertEqual(updated_resources,
self.list_resources(stack_identifier))
+ def test_stack_update_from_failed(self):
+ # Prove it's possible to update from an UPDATE_FAILED state
+ template = _change_rsrc_properties(test_template_one_resource,
+ ['test1'],
+ {'value': 'test_update_failed'})
+ stack_identifier = self.stack_create(
+ template=template)
+ initial_resources = {'test1': 'OS::Heat::TestResource'}
+ self.assertEqual(initial_resources,
+ self.list_resources(stack_identifier))
+
+ tmpl_update = _change_rsrc_properties(
+ test_template_one_resource, ['test1'], {'fail': True})
+ # Update with bad template, we should fail
+ self.update_stack(stack_identifier, tmpl_update,
+ expected_status='UPDATE_FAILED')
+ # but then passing a good template should succeed
+ self.update_stack(stack_identifier, test_template_two_resource)
+ updated_resources = {'test1': 'OS::Heat::TestResource',
+ 'test2': 'OS::Heat::TestResource'}
+ self.assertEqual(updated_resources,
+ self.list_resources(stack_identifier))
+
def test_stack_update_provider(self):
template = _change_rsrc_properties(
test_template_one_resource, ['test1'],