Add integration test for UPDATE_FAILED recovery

It's valid to attempt an update from UPDATE_FAILED state, so add
a functional test which proves this works.

This requires an adjustment to the common test.py _verify_status
because we now need to ensure we record the time of transition to
FAILED state as well as COMPLETE, or we'll mis-detect the failure
as a failure of the subsequent recovery update.

Co-Authored-By: Sergey Kraynev <skraynev@mirantis.com>
Change-Id: I5fc82b4c71a943ba15c04b08bcfd26fcb84dc7a4
diff --git a/functional/test_create_update.py b/functional/test_create_update.py
index b2aad34..1b273dd 100644
--- a/functional/test_create_update.py
+++ b/functional/test_create_update.py
@@ -243,6 +243,29 @@
         self.assertEqual(updated_resources,
                          self.list_resources(stack_identifier))
 
+    def test_stack_update_from_failed(self):
+        # Prove it's possible to update from an UPDATE_FAILED state
+        template = _change_rsrc_properties(test_template_one_resource,
+                                           ['test1'],
+                                           {'value': 'test_update_failed'})
+        stack_identifier = self.stack_create(
+            template=template)
+        initial_resources = {'test1': 'OS::Heat::TestResource'}
+        self.assertEqual(initial_resources,
+                         self.list_resources(stack_identifier))
+
+        tmpl_update = _change_rsrc_properties(
+            test_template_one_resource, ['test1'], {'fail': True})
+        # Update with bad template, we should fail
+        self.update_stack(stack_identifier, tmpl_update,
+                          expected_status='UPDATE_FAILED')
+        # but then passing a good template should succeed
+        self.update_stack(stack_identifier, test_template_two_resource)
+        updated_resources = {'test1': 'OS::Heat::TestResource',
+                             'test2': 'OS::Heat::TestResource'}
+        self.assertEqual(updated_resources,
+                         self.list_resources(stack_identifier))
+
     def test_stack_update_provider(self):
         template = _change_rsrc_properties(
             test_template_one_resource, ['test1'],