Make test_server_cfn_init a scenario test
Converting test_server_cfn_init to a scenario test is long overdue
since it is more of a scenario than an exercising of the heat API.
The heat-slow job has gone non-voting because this test fails
~%50 of the time due to bug #1297560. The server boot log is now
logged regardless of success or failure to build up data to diagnose
the issue.
This also adds several convenience functions to the test base class.
Co-Authored-By: Steven Hardy <shardy@redhat.com>
Related-Bug: #1297560
Change-Id: I077aeaf2bf8b292699eb20c5a75c59df35645913
diff --git a/tempest/api/orchestration/stacks/test_server_cfn_init.py b/tempest/api/orchestration/stacks/test_server_cfn_init.py
deleted file mode 100644
index 4b845b1..0000000
--- a/tempest/api/orchestration/stacks/test_server_cfn_init.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import json
-import testtools
-
-from tempest.api.orchestration import base
-from tempest.common.utils import data_utils
-from tempest.common.utils.linux import remote_client
-from tempest import config
-from tempest import exceptions
-from tempest.openstack.common import log as logging
-from tempest import test
-
-CONF = config.CONF
-LOG = logging.getLogger(__name__)
-
-
-class ServerCfnInitTestJSON(base.BaseOrchestrationTest):
- existing_keypair = CONF.orchestration.keypair_name is not None
-
- @classmethod
- @test.safe_setup
- def setUpClass(cls):
- super(ServerCfnInitTestJSON, cls).setUpClass()
- if not CONF.orchestration.image_ref:
- raise cls.skipException("No image available to test")
- template = cls.load_template('cfn_init_signal')
- stack_name = data_utils.rand_name('heat')
- if CONF.orchestration.keypair_name:
- keypair_name = CONF.orchestration.keypair_name
- else:
- cls.keypair = cls._create_keypair()
- keypair_name = cls.keypair['name']
-
- # create the stack
- cls.stack_identifier = cls.create_stack(
- stack_name,
- template,
- parameters={
- 'key_name': keypair_name,
- 'flavor': CONF.orchestration.instance_type,
- 'image': CONF.orchestration.image_ref,
- 'network': cls._get_default_network()['id'],
- 'timeout': CONF.orchestration.build_timeout
- })
-
- @test.attr(type='slow')
- @testtools.skipIf(existing_keypair, 'Server ssh tests are disabled.')
- def test_can_log_into_created_server(self):
-
- sid = self.stack_identifier
- rid = 'SmokeServer'
-
- # wait for create to complete.
- self.client.wait_for_stack_status(sid, 'CREATE_COMPLETE')
-
- resp, body = self.client.get_resource(sid, rid)
- self.assertEqual('CREATE_COMPLETE', body['resource_status'])
-
- # fetch the IP address from servers client, since we can't get it
- # from the stack until stack create is complete
- resp, server = self.servers_client.get_server(
- body['physical_resource_id'])
-
- # Check that the user can authenticate with the generated password
- linux_client = remote_client.RemoteClient(server, 'ec2-user',
- pkey=self.keypair[
- 'private_key'])
- linux_client.validate_authentication()
-
- @test.attr(type='slow')
- def test_all_resources_created(self):
- sid = self.stack_identifier
- self.client.wait_for_resource_status(
- sid, 'WaitHandle', 'CREATE_COMPLETE')
- self.client.wait_for_resource_status(
- sid, 'SmokeSecurityGroup', 'CREATE_COMPLETE')
- self.client.wait_for_resource_status(
- sid, 'SmokeKeys', 'CREATE_COMPLETE')
- self.client.wait_for_resource_status(
- sid, 'CfnUser', 'CREATE_COMPLETE')
- self.client.wait_for_resource_status(
- sid, 'SmokeServer', 'CREATE_COMPLETE')
- try:
- self.client.wait_for_resource_status(
- sid, 'WaitCondition', 'CREATE_COMPLETE')
- except (exceptions.StackResourceBuildErrorException,
- exceptions.TimeoutException) as e:
- # attempt to log the server console to help with debugging
- # the cause of the server not signalling the waitcondition
- # to heat.
- resp, body = self.client.get_resource(sid, 'SmokeServer')
- server_id = body['physical_resource_id']
- LOG.debug('Console output for %s', server_id)
- resp, output = self.servers_client.get_console_output(
- server_id, None)
- LOG.debug(output)
- raise e
-
- # wait for create to complete.
- self.client.wait_for_stack_status(sid, 'CREATE_COMPLETE')
-
- # This is an assert of great significance, as it means the following
- # has happened:
- # - cfn-init read the provided metadata and wrote out a file
- # - a user was created and credentials written to the server
- # - a cfn-signal was built which was signed with provided credentials
- # - the wait condition was fulfilled and the stack has changed state
- wait_status = json.loads(
- self.get_stack_output(sid, 'WaitConditionStatus'))
- self.assertEqual('smoke test complete', wait_status['00000'])
diff --git a/tempest/scenario/manager.py b/tempest/scenario/manager.py
index e057c74..f83a4b8 100644
--- a/tempest/scenario/manager.py
+++ b/tempest/scenario/manager.py
@@ -16,9 +16,12 @@
import logging
import os
+import re
import six
import subprocess
+import time
+from heatclient import exc as heat_exceptions
import netaddr
from neutronclient.common import exceptions as exc
from novaclient import exceptions as nova_exceptions
@@ -32,6 +35,7 @@
from tempest import config
from tempest import exceptions
from tempest.openstack.common import log
+from tempest.openstack.common import timeutils
import tempest.test
CONF = config.CONF
@@ -1069,3 +1073,98 @@
for net in networks['networks']:
if net['name'] == CONF.compute.fixed_network_name:
return net
+
+ @staticmethod
+ def _stack_output(stack, output_key):
+ """Return a stack output value for a given key."""
+ return next((o['output_value'] for o in stack.outputs
+ if o['output_key'] == output_key), None)
+
+ def _ping_ip_address(self, ip_address, should_succeed=True):
+ cmd = ['ping', '-c1', '-w1', ip_address]
+
+ def ping():
+ proc = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ proc.wait()
+ return (proc.returncode == 0) == should_succeed
+
+ return tempest.test.call_until_true(
+ ping, CONF.orchestration.build_timeout, 1)
+
+ def _wait_for_resource_status(self, stack_identifier, resource_name,
+ status, failure_pattern='^.*_FAILED$'):
+ """Waits for a Resource to reach a given status."""
+ fail_regexp = re.compile(failure_pattern)
+ build_timeout = CONF.orchestration.build_timeout
+ build_interval = CONF.orchestration.build_interval
+
+ start = timeutils.utcnow()
+ while timeutils.delta_seconds(start,
+ timeutils.utcnow()) < build_timeout:
+ try:
+ res = self.client.resources.get(
+ stack_identifier, resource_name)
+ except heat_exceptions.HTTPNotFound:
+ # ignore this, as the resource may not have
+ # been created yet
+ pass
+ else:
+ if res.resource_status == status:
+ return
+ if fail_regexp.search(res.resource_status):
+ raise exceptions.StackResourceBuildErrorException(
+ resource_name=res.resource_name,
+ stack_identifier=stack_identifier,
+ resource_status=res.resource_status,
+ resource_status_reason=res.resource_status_reason)
+ time.sleep(build_interval)
+
+ message = ('Resource %s failed to reach %s status within '
+ 'the required time (%s s).' %
+ (res.resource_name, status, build_timeout))
+ raise exceptions.TimeoutException(message)
+
+ def _wait_for_stack_status(self, stack_identifier, status,
+ failure_pattern='^.*_FAILED$'):
+ """
+ Waits for a Stack to reach a given status.
+
+ Note this compares the full $action_$status, e.g
+ CREATE_COMPLETE, not just COMPLETE which is exposed
+ via the status property of Stack in heatclient
+ """
+ fail_regexp = re.compile(failure_pattern)
+ build_timeout = CONF.orchestration.build_timeout
+ build_interval = CONF.orchestration.build_interval
+
+ start = timeutils.utcnow()
+ while timeutils.delta_seconds(start,
+ timeutils.utcnow()) < build_timeout:
+ try:
+ stack = self.client.stacks.get(stack_identifier)
+ except heat_exceptions.HTTPNotFound:
+ # ignore this, as the stackource may not have
+ # been created yet
+ pass
+ else:
+ if stack.stack_status == status:
+ return
+ if fail_regexp.search(stack.stack_status):
+ raise exceptions.StackBuildErrorException(
+ stack_identifier=stack_identifier,
+ stack_status=stack.stack_status,
+ stack_status_reason=stack.stack_status_reason)
+ time.sleep(build_interval)
+
+ message = ('Stack %s failed to reach %s status within '
+ 'the required time (%s s).' %
+ (stack.stack_name, status, build_timeout))
+ raise exceptions.TimeoutException(message)
+
+ def _stack_delete(self, stack_identifier):
+ try:
+ self.client.stacks.delete(stack_identifier)
+ except heat_exceptions.HTTPNotFound:
+ pass
diff --git a/tempest/api/orchestration/stacks/templates/cfn_init_signal.yaml b/tempest/scenario/orchestration/cfn_init_signal.yaml
similarity index 97%
rename from tempest/api/orchestration/stacks/templates/cfn_init_signal.yaml
rename to tempest/scenario/orchestration/cfn_init_signal.yaml
index fa5345e..c95aabf 100644
--- a/tempest/api/orchestration/stacks/templates/cfn_init_signal.yaml
+++ b/tempest/scenario/orchestration/cfn_init_signal.yaml
@@ -62,7 +62,7 @@
#!/bin/bash -v
/opt/aws/bin/cfn-init
/opt/aws/bin/cfn-signal -e 0 --data "`cat /tmp/smoke-status`" \
- "WaitHandle"
+ --id smoke_status "WaitHandle"
WaitHandle:
Type: AWS::CloudFormation::WaitConditionHandle
WaitCondition:
diff --git a/tempest/scenario/orchestration/test_server_cfn_init.py b/tempest/scenario/orchestration/test_server_cfn_init.py
new file mode 100644
index 0000000..36e6126
--- /dev/null
+++ b/tempest/scenario/orchestration/test_server_cfn_init.py
@@ -0,0 +1,130 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from tempest import config
+from tempest import exceptions
+from tempest.openstack.common import log as logging
+from tempest.scenario import manager
+from tempest import test
+
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
+
+
+class CfnInitScenarioTest(manager.OrchestrationScenarioTest):
+
+ def setUp(self):
+ super(CfnInitScenarioTest, self).setUp()
+ if not CONF.orchestration.image_ref:
+ raise self.skipException("No image available to test")
+ self.client = self.orchestration_client
+ self.template_name = 'cfn_init_signal.yaml'
+
+ def assign_keypair(self):
+ self.stack_name = self._stack_rand_name()
+ if CONF.orchestration.keypair_name:
+ self.keypair = None
+ self.keypair_name = CONF.orchestration.keypair_name
+ else:
+ self.keypair = self.create_keypair()
+ self.keypair_name = self.keypair.id
+
+ def launch_stack(self):
+ net = self._get_default_network()
+ self.parameters = {
+ 'key_name': self.keypair_name,
+ 'flavor': CONF.orchestration.instance_type,
+ 'image': CONF.orchestration.image_ref,
+ 'timeout': CONF.orchestration.build_timeout,
+ 'network': net['id'],
+ }
+
+ # create the stack
+ self.template = self._load_template(__file__, self.template_name)
+ self.client.stacks.create(
+ stack_name=self.stack_name,
+ template=self.template,
+ parameters=self.parameters)
+
+ self.stack = self.client.stacks.get(self.stack_name)
+ self.stack_identifier = '%s/%s' % (self.stack_name, self.stack.id)
+ self.addCleanup(self._stack_delete, self.stack_identifier)
+
+ def check_stack(self):
+ sid = self.stack_identifier
+ self._wait_for_resource_status(
+ sid, 'WaitHandle', 'CREATE_COMPLETE')
+ self._wait_for_resource_status(
+ sid, 'SmokeSecurityGroup', 'CREATE_COMPLETE')
+ self._wait_for_resource_status(
+ sid, 'SmokeKeys', 'CREATE_COMPLETE')
+ self._wait_for_resource_status(
+ sid, 'CfnUser', 'CREATE_COMPLETE')
+ self._wait_for_resource_status(
+ sid, 'SmokeServer', 'CREATE_COMPLETE')
+
+ server_resource = self.client.resources.get(sid, 'SmokeServer')
+ server_id = server_resource.physical_resource_id
+ server = self.compute_client.servers.get(server_id)
+ server_ip = server.networks[CONF.compute.network_for_ssh][0]
+
+ if not self._ping_ip_address(server_ip):
+ self._log_console_output(servers=[server])
+ self.fail(
+ "Timed out waiting for %s to become reachable" % server_ip)
+
+ try:
+ self._wait_for_resource_status(
+ sid, 'WaitCondition', 'CREATE_COMPLETE')
+ except (exceptions.StackResourceBuildErrorException,
+ exceptions.TimeoutException) as e:
+ raise e
+ finally:
+ # attempt to log the server console regardless of WaitCondition
+ # going to complete. This allows successful and failed cloud-init
+ # logs to be compared
+ self._log_console_output(servers=[server])
+
+ self._wait_for_stack_status(sid, 'CREATE_COMPLETE')
+
+ stack = self.client.stacks.get(sid)
+
+ # This is an assert of great significance, as it means the following
+ # has happened:
+ # - cfn-init read the provided metadata and wrote out a file
+ # - a user was created and credentials written to the server
+ # - a cfn-signal was built which was signed with provided credentials
+ # - the wait condition was fulfilled and the stack has changed state
+ wait_status = json.loads(
+ self._stack_output(stack, 'WaitConditionStatus'))
+ self.assertEqual('smoke test complete', wait_status['smoke_status'])
+
+ if self.keypair:
+ # Check that the user can authenticate with the generated
+ # keypair
+ try:
+ linux_client = self.get_remote_client(
+ server_ip, username='ec2-user')
+ linux_client.validate_authentication()
+ except (exceptions.ServerUnreachable,
+ exceptions.SSHTimeout) as e:
+ self._log_console_output(servers=[server])
+ raise e
+
+ @test.attr(type='slow')
+ @test.services('orchestration', 'compute')
+ def test_server_cfn_init(self):
+ self.assign_keypair()
+ self.launch_stack()
+ self.check_stack()