Support classes for heat integration tests
These support classes started as a forklift of the classes needed
to run tempest scenario orchestration tests.
The original tempest code has been pared back to provide the
small subset required by heat integration tests. From this point on
these support classes can evolve to the specific needs of the
integration tests.
There is some unused code (especially in remote_client) which has
been left in as it may become useful in the future, and is already
extremely well reviewed and tested from being developed for tempest.
The script heat_integrationtests/generate_sample.sh will generate
an up-to-date heat_integrationtests/heat_integrationtests.conf.sample
file which can be copied to
heat_integrationtests/heat_integrationtests.conf
to override default configuration values. A local ConfigOpts is created
for each test to avoid any potential interaction with heat's
global CONF. Configuration options for credentials default to
being sourced from the environment.
The default tox testenv now excludes tests in heat_integrationtests.
A new testenv called "integration" will only run tests in
heat_integrationtests.
Integration tests will fail if preconditions are not met, including
a keystone endpoint, credentials and glance containing the expected
named image.
Devstack gate hooks have been moved to heat_integrationtests now that
the name of the package has been decided.
Change-Id: I174429c16bb606c5c325ee8b62c6e600ea77a6e6
Partial-Blueprint: functional-tests
diff --git a/common/test.py b/common/test.py
new file mode 100644
index 0000000..2973964
--- /dev/null
+++ b/common/test.py
@@ -0,0 +1,306 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+import random
+import re
+import six
+import subprocess
+import testtools
+import time
+
+from heatclient import exc as heat_exceptions
+
+from heat.openstack.common import timeutils
+from heat_integrationtests.common import clients
+from heat_integrationtests.common import config
+from heat_integrationtests.common import exceptions
+from heat_integrationtests.common import remote_client
+
+LOG = logging.getLogger(__name__)
+
+
+def call_until_true(func, duration, sleep_for):
+ """
+ Call the given function until it returns True (and return True) or
+ until the specified duration (in seconds) elapses (and return
+ False).
+
+ :param func: A zero argument callable that returns True on success.
+ :param duration: The number of seconds for which to attempt a
+ successful call of the function.
+ :param sleep_for: The number of seconds to sleep after an unsuccessful
+ invocation of the function.
+ """
+ now = time.time()
+ timeout = now + duration
+ while now < timeout:
+ if func():
+ return True
+ LOG.debug("Sleeping for %d seconds", sleep_for)
+ time.sleep(sleep_for)
+ now = time.time()
+ return False
+
+
+def rand_name(name=''):
+ randbits = str(random.randint(1, 0x7fffffff))
+ if name:
+ return name + '-' + randbits
+ else:
+ return randbits
+
+
+class HeatIntegrationTest(testtools.TestCase):
+
+ def setUp(self):
+ super(HeatIntegrationTest, self).setUp()
+
+ self.conf = config.init_conf()
+
+ self.assertIsNotNone(self.conf.auth_url,
+ 'No auth_url configured')
+ self.assertIsNotNone(self.conf.username,
+ 'No username configured')
+ self.assertIsNotNone(self.conf.password,
+ 'No password configured')
+
+ self.manager = clients.ClientManager(self.conf)
+ self.identity_client = self.manager.identity_client
+ self.orchestration_client = self.manager.orchestration_client
+ self.compute_client = self.manager.compute_client
+ self.network_client = self.manager.network_client
+ self.volume_client = self.manager.volume_client
+
+ def status_timeout(self, things, thing_id, expected_status,
+ error_status='ERROR',
+ not_found_exception=heat_exceptions.NotFound):
+ """
+ Given a thing and an expected status, do a loop, sleeping
+ for a configurable amount of time, checking for the
+ expected status to show. At any time, if the returned
+ status of the thing is ERROR, fail out.
+ """
+ self._status_timeout(things, thing_id,
+ expected_status=expected_status,
+ error_status=error_status,
+ not_found_exception=not_found_exception)
+
+ def _status_timeout(self,
+ things,
+ thing_id,
+ expected_status=None,
+ allow_notfound=False,
+ error_status='ERROR',
+ not_found_exception=heat_exceptions.NotFound):
+
+ log_status = expected_status if expected_status else ''
+ if allow_notfound:
+ log_status += ' or NotFound' if log_status != '' else 'NotFound'
+
+ def check_status():
+ # python-novaclient has resources available to its client
+ # that all implement a get() method taking an identifier
+ # for the singular resource to retrieve.
+ try:
+ thing = things.get(thing_id)
+ except not_found_exception:
+ if allow_notfound:
+ return True
+ raise
+ except Exception as e:
+ if allow_notfound and self.not_found_exception(e):
+ return True
+ raise
+
+ new_status = thing.status
+
+ # Some components are reporting error status in lower case
+ # so case sensitive comparisons can really mess things
+ # up.
+ if new_status.lower() == error_status.lower():
+ message = ("%s failed to get to expected status (%s). "
+ "In %s state.") % (thing, expected_status,
+ new_status)
+ raise exceptions.BuildErrorException(message,
+ server_id=thing_id)
+ elif new_status == expected_status and expected_status is not None:
+ return True # All good.
+ LOG.debug("Waiting for %s to get to %s status. "
+ "Currently in %s status",
+ thing, log_status, new_status)
+ if not call_until_true(
+ check_status,
+ self.conf.build_timeout,
+ self.conf.build_interval):
+ message = ("Timed out waiting for thing %s "
+ "to become %s") % (thing_id, log_status)
+ raise exceptions.TimeoutException(message)
+
+ def get_remote_client(self, server_or_ip, username, private_key=None):
+ if isinstance(server_or_ip, six.string_types):
+ ip = server_or_ip
+ else:
+ network_name_for_ssh = self.conf.network_for_ssh
+ ip = server_or_ip.networks[network_name_for_ssh][0]
+ if private_key is None:
+ private_key = self.keypair.private_key
+ linux_client = remote_client.RemoteClient(ip, username,
+ pkey=private_key,
+ conf=self.conf)
+ try:
+ linux_client.validate_authentication()
+ except exceptions.SSHTimeout:
+ LOG.exception('ssh connection to %s failed' % ip)
+ raise
+
+ return linux_client
+
+ def _log_console_output(self, servers=None):
+ if not servers:
+ servers = self.compute_client.servers.list()
+ for server in servers:
+ LOG.debug('Console output for %s', server.id)
+ LOG.debug(server.get_console_output())
+
+ def _load_template(self, base_file, file_name):
+ filepath = os.path.join(os.path.dirname(os.path.realpath(base_file)),
+ file_name)
+ with open(filepath) as f:
+ return f.read()
+
+ def create_keypair(self, client=None, name=None):
+ if client is None:
+ client = self.compute_client
+ if name is None:
+ name = rand_name('heat-keypair')
+ keypair = client.keypairs.create(name)
+ self.assertEqual(keypair.name, name)
+
+ def delete_keypair():
+ keypair.delete()
+
+ self.addCleanup(delete_keypair)
+ return keypair
+
+ @classmethod
+ def _stack_rand_name(cls):
+ return rand_name(cls.__name__)
+
+ def _get_default_network(self):
+ networks = self.network_client.list_networks()
+ for net in networks['networks']:
+ if net['name'] == self.conf.fixed_network_name:
+ return net
+
+ @staticmethod
+ def _stack_output(stack, output_key):
+ """Return a stack output value for a given key."""
+ return next((o['output_value'] for o in stack.outputs
+ if o['output_key'] == output_key), None)
+
+ def _ping_ip_address(self, ip_address, should_succeed=True):
+ cmd = ['ping', '-c1', '-w1', ip_address]
+
+ def ping():
+ proc = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ proc.wait()
+ return (proc.returncode == 0) == should_succeed
+
+ return call_until_true(
+ ping, self.conf.build_timeout, 1)
+
+ def _wait_for_resource_status(self, stack_identifier, resource_name,
+ status, failure_pattern='^.*_FAILED$',
+ success_on_not_found=False):
+ """Waits for a Resource to reach a given status."""
+ fail_regexp = re.compile(failure_pattern)
+ build_timeout = self.conf.build_timeout
+ build_interval = self.conf.build_interval
+
+ start = timeutils.utcnow()
+ while timeutils.delta_seconds(start,
+ timeutils.utcnow()) < build_timeout:
+ try:
+ res = self.client.resources.get(
+ stack_identifier, resource_name)
+ except heat_exceptions.HTTPNotFound:
+ if success_on_not_found:
+ return
+ # ignore this, as the resource may not have
+ # been created yet
+ else:
+ if res.resource_status == status:
+ return
+ if fail_regexp.search(res.resource_status):
+ raise exceptions.StackResourceBuildErrorException(
+ resource_name=res.resource_name,
+ stack_identifier=stack_identifier,
+ resource_status=res.resource_status,
+ resource_status_reason=res.resource_status_reason)
+ time.sleep(build_interval)
+
+ message = ('Resource %s failed to reach %s status within '
+ 'the required time (%s s).' %
+ (res.resource_name, status, build_timeout))
+ raise exceptions.TimeoutException(message)
+
+ def _wait_for_stack_status(self, stack_identifier, status,
+ failure_pattern='^.*_FAILED$',
+ success_on_not_found=False):
+ """
+ Waits for a Stack to reach a given status.
+
+ Note this compares the full $action_$status, e.g
+ CREATE_COMPLETE, not just COMPLETE which is exposed
+ via the status property of Stack in heatclient
+ """
+ fail_regexp = re.compile(failure_pattern)
+ build_timeout = self.conf.build_timeout
+ build_interval = self.conf.build_interval
+
+ start = timeutils.utcnow()
+ while timeutils.delta_seconds(start,
+ timeutils.utcnow()) < build_timeout:
+ try:
+ stack = self.client.stacks.get(stack_identifier)
+ except heat_exceptions.HTTPNotFound:
+ if success_on_not_found:
+ return
+ # ignore this, as the resource may not have
+ # been created yet
+ else:
+ if stack.stack_status == status:
+ return
+ if fail_regexp.search(stack.stack_status):
+ raise exceptions.StackBuildErrorException(
+ stack_identifier=stack_identifier,
+ stack_status=stack.stack_status,
+ stack_status_reason=stack.stack_status_reason)
+ time.sleep(build_interval)
+
+ message = ('Stack %s failed to reach %s status within '
+ 'the required time (%s s).' %
+ (stack.stack_name, status, build_timeout))
+ raise exceptions.TimeoutException(message)
+
+ def _stack_delete(self, stack_identifier):
+ try:
+ self.client.stacks.delete(stack_identifier)
+ except heat_exceptions.HTTPNotFound:
+ pass
+ self._wait_for_stack_status(
+ stack_identifier, 'DELETE_COMPLETE',
+ success_on_not_found=True)