| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| import fixtures |
| import logging |
| import os |
| import random |
| import re |
| import six |
| import subprocess |
| import testtools |
| import time |
| |
| from heatclient import exc as heat_exceptions |
| |
| from heat.openstack.common import timeutils |
| from heat_integrationtests.common import clients |
| from heat_integrationtests.common import config |
| from heat_integrationtests.common import exceptions |
| from heat_integrationtests.common import remote_client |
| |
| LOG = logging.getLogger(__name__) |
| _LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s" |
| |
| |
| def call_until_true(func, duration, sleep_for): |
| """ |
| Call the given function until it returns True (and return True) or |
| until the specified duration (in seconds) elapses (and return |
| False). |
| |
| :param func: A zero argument callable that returns True on success. |
| :param duration: The number of seconds for which to attempt a |
| successful call of the function. |
| :param sleep_for: The number of seconds to sleep after an unsuccessful |
| invocation of the function. |
| """ |
| now = time.time() |
| timeout = now + duration |
| while now < timeout: |
| if func(): |
| return True |
| LOG.debug("Sleeping for %d seconds", sleep_for) |
| time.sleep(sleep_for) |
| now = time.time() |
| return False |
| |
| |
| def rand_name(name=''): |
| randbits = str(random.randint(1, 0x7fffffff)) |
| if name: |
| return name + '-' + randbits |
| else: |
| return randbits |
| |
| |
| class HeatIntegrationTest(testtools.TestCase): |
| |
| def setUp(self): |
| super(HeatIntegrationTest, self).setUp() |
| |
| self.conf = config.init_conf() |
| |
| self.assertIsNotNone(self.conf.auth_url, |
| 'No auth_url configured') |
| self.assertIsNotNone(self.conf.username, |
| 'No username configured') |
| self.assertIsNotNone(self.conf.password, |
| 'No password configured') |
| |
| self.manager = clients.ClientManager(self.conf) |
| self.identity_client = self.manager.identity_client |
| self.orchestration_client = self.manager.orchestration_client |
| self.compute_client = self.manager.compute_client |
| self.network_client = self.manager.network_client |
| self.volume_client = self.manager.volume_client |
| self.useFixture(fixtures.FakeLogger(format=_LOG_FORMAT)) |
| |
| def status_timeout(self, things, thing_id, expected_status, |
| error_status='ERROR', |
| not_found_exception=heat_exceptions.NotFound): |
| """ |
| Given a thing and an expected status, do a loop, sleeping |
| for a configurable amount of time, checking for the |
| expected status to show. At any time, if the returned |
| status of the thing is ERROR, fail out. |
| """ |
| self._status_timeout(things, thing_id, |
| expected_status=expected_status, |
| error_status=error_status, |
| not_found_exception=not_found_exception) |
| |
| def _status_timeout(self, |
| things, |
| thing_id, |
| expected_status=None, |
| allow_notfound=False, |
| error_status='ERROR', |
| not_found_exception=heat_exceptions.NotFound): |
| |
| log_status = expected_status if expected_status else '' |
| if allow_notfound: |
| log_status += ' or NotFound' if log_status != '' else 'NotFound' |
| |
| def check_status(): |
| # python-novaclient has resources available to its client |
| # that all implement a get() method taking an identifier |
| # for the singular resource to retrieve. |
| try: |
| thing = things.get(thing_id) |
| except not_found_exception: |
| if allow_notfound: |
| return True |
| raise |
| except Exception as e: |
| if allow_notfound and self.not_found_exception(e): |
| return True |
| raise |
| |
| new_status = thing.status |
| |
| # Some components are reporting error status in lower case |
| # so case sensitive comparisons can really mess things |
| # up. |
| if new_status.lower() == error_status.lower(): |
| message = ("%s failed to get to expected status (%s). " |
| "In %s state.") % (thing, expected_status, |
| new_status) |
| raise exceptions.BuildErrorException(message, |
| server_id=thing_id) |
| elif new_status == expected_status and expected_status is not None: |
| return True # All good. |
| LOG.debug("Waiting for %s to get to %s status. " |
| "Currently in %s status", |
| thing, log_status, new_status) |
| if not call_until_true( |
| check_status, |
| self.conf.build_timeout, |
| self.conf.build_interval): |
| message = ("Timed out waiting for thing %s " |
| "to become %s") % (thing_id, log_status) |
| raise exceptions.TimeoutException(message) |
| |
| def get_remote_client(self, server_or_ip, username, private_key=None): |
| if isinstance(server_or_ip, six.string_types): |
| ip = server_or_ip |
| else: |
| network_name_for_ssh = self.conf.network_for_ssh |
| ip = server_or_ip.networks[network_name_for_ssh][0] |
| if private_key is None: |
| private_key = self.keypair.private_key |
| linux_client = remote_client.RemoteClient(ip, username, |
| pkey=private_key, |
| conf=self.conf) |
| try: |
| linux_client.validate_authentication() |
| except exceptions.SSHTimeout: |
| LOG.exception('ssh connection to %s failed' % ip) |
| raise |
| |
| return linux_client |
| |
| def _log_console_output(self, servers=None): |
| if not servers: |
| servers = self.compute_client.servers.list() |
| for server in servers: |
| LOG.debug('Console output for %s', server.id) |
| LOG.debug(server.get_console_output()) |
| |
| def _load_template(self, base_file, file_name): |
| filepath = os.path.join(os.path.dirname(os.path.realpath(base_file)), |
| file_name) |
| with open(filepath) as f: |
| return f.read() |
| |
| def create_keypair(self, client=None, name=None): |
| if client is None: |
| client = self.compute_client |
| if name is None: |
| name = rand_name('heat-keypair') |
| keypair = client.keypairs.create(name) |
| self.assertEqual(keypair.name, name) |
| |
| def delete_keypair(): |
| keypair.delete() |
| |
| self.addCleanup(delete_keypair) |
| return keypair |
| |
| @classmethod |
| def _stack_rand_name(cls): |
| return rand_name(cls.__name__) |
| |
| def _get_default_network(self): |
| networks = self.network_client.list_networks() |
| for net in networks['networks']: |
| if net['name'] == self.conf.fixed_network_name: |
| return net |
| |
| @staticmethod |
| def _stack_output(stack, output_key): |
| """Return a stack output value for a given key.""" |
| return next((o['output_value'] for o in stack.outputs |
| if o['output_key'] == output_key), None) |
| |
| def _ping_ip_address(self, ip_address, should_succeed=True): |
| cmd = ['ping', '-c1', '-w1', ip_address] |
| |
| def ping(): |
| proc = subprocess.Popen(cmd, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE) |
| proc.wait() |
| return (proc.returncode == 0) == should_succeed |
| |
| return call_until_true( |
| ping, self.conf.build_timeout, 1) |
| |
| def _wait_for_resource_status(self, stack_identifier, resource_name, |
| status, failure_pattern='^.*_FAILED$', |
| success_on_not_found=False): |
| """Waits for a Resource to reach a given status.""" |
| fail_regexp = re.compile(failure_pattern) |
| build_timeout = self.conf.build_timeout |
| build_interval = self.conf.build_interval |
| |
| start = timeutils.utcnow() |
| while timeutils.delta_seconds(start, |
| timeutils.utcnow()) < build_timeout: |
| try: |
| res = self.client.resources.get( |
| stack_identifier, resource_name) |
| except heat_exceptions.HTTPNotFound: |
| if success_on_not_found: |
| return |
| # ignore this, as the resource may not have |
| # been created yet |
| else: |
| if res.resource_status == status: |
| return |
| if fail_regexp.search(res.resource_status): |
| raise exceptions.StackResourceBuildErrorException( |
| resource_name=res.resource_name, |
| stack_identifier=stack_identifier, |
| resource_status=res.resource_status, |
| resource_status_reason=res.resource_status_reason) |
| time.sleep(build_interval) |
| |
| message = ('Resource %s failed to reach %s status within ' |
| 'the required time (%s s).' % |
| (res.resource_name, status, build_timeout)) |
| raise exceptions.TimeoutException(message) |
| |
| def _wait_for_stack_status(self, stack_identifier, status, |
| failure_pattern='^.*_FAILED$', |
| success_on_not_found=False): |
| """ |
| Waits for a Stack to reach a given status. |
| |
| Note this compares the full $action_$status, e.g |
| CREATE_COMPLETE, not just COMPLETE which is exposed |
| via the status property of Stack in heatclient |
| """ |
| fail_regexp = re.compile(failure_pattern) |
| build_timeout = self.conf.build_timeout |
| build_interval = self.conf.build_interval |
| |
| start = timeutils.utcnow() |
| while timeutils.delta_seconds(start, |
| timeutils.utcnow()) < build_timeout: |
| try: |
| stack = self.client.stacks.get(stack_identifier) |
| except heat_exceptions.HTTPNotFound: |
| if success_on_not_found: |
| return |
| # ignore this, as the resource may not have |
| # been created yet |
| else: |
| if stack.stack_status == status: |
| return |
| if fail_regexp.search(stack.stack_status): |
| raise exceptions.StackBuildErrorException( |
| stack_identifier=stack_identifier, |
| stack_status=stack.stack_status, |
| stack_status_reason=stack.stack_status_reason) |
| time.sleep(build_interval) |
| |
| message = ('Stack %s failed to reach %s status within ' |
| 'the required time (%s s).' % |
| (stack.stack_name, status, build_timeout)) |
| raise exceptions.TimeoutException(message) |
| |
| def _stack_delete(self, stack_identifier): |
| try: |
| self.client.stacks.delete(stack_identifier) |
| except heat_exceptions.HTTPNotFound: |
| pass |
| self._wait_for_stack_status( |
| stack_identifier, 'DELETE_COMPLETE', |
| success_on_not_found=True) |
| |
| def update_stack(self, stack_identifier, template, environment=None, |
| files=None): |
| env = environment or {} |
| env_files = files or {} |
| stack_name = stack_identifier.split('/')[0] |
| self.client.stacks.update( |
| stack_id=stack_identifier, |
| stack_name=stack_name, |
| template=template, |
| files=env_files, |
| disable_rollback=True, |
| parameters={}, |
| environment=env |
| ) |
| self._wait_for_stack_status(stack_identifier, 'UPDATE_COMPLETE') |
| |
| def list_resources(self, stack_identifier): |
| resources = self.client.resources.list(stack_identifier) |
| return dict((r.resource_name, r.resource_type) for r in resources) |
| |
| def stack_create(self, stack_name=None, template=None, files=None, |
| parameters=None, environment=None): |
| name = stack_name or self._stack_rand_name() |
| templ = template or self.template |
| templ_files = files or {} |
| params = parameters or {} |
| env = environment or {} |
| self.client.stacks.create( |
| stack_name=name, |
| template=templ, |
| files=templ_files, |
| disable_rollback=True, |
| parameters=params, |
| environment=env |
| ) |
| self.addCleanup(self.client.stacks.delete, name) |
| |
| stack = self.client.stacks.get(name) |
| stack_identifier = '%s/%s' % (name, stack.id) |
| self._wait_for_stack_status(stack_identifier, 'CREATE_COMPLETE') |
| return stack_identifier |