tcp_tests/managers/envmanager_heat.py

#    Copyright 2019 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import netaddr
import yaml
import pytest

from devops.helpers import helpers
from devops.helpers.helpers import ssh_client
from retry import retry

from cached_property import cached_property

from heatclient import client as heatclient
from heatclient import exc as heat_exceptions
from heatclient.common import template_utils
from keystoneauth1.identity import v3 as keystone_v3
from keystoneauth1 import session as keystone_session

import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning

from oslo_config import cfg
from paramiko.ssh_exception import (
    AuthenticationException,
    BadAuthenticationType)

from tcp_tests import settings
from tcp_tests import settings_oslo
from tcp_tests.helpers import exceptions
from tcp_tests import logger

LOG = logger.logger

EXPECTED_STACK_STATUS = "CREATE_COMPLETE"
BAD_STACK_STATUSES = ["CREATE_FAILED", "DELETE_FAILED"]

# Disable multiple notifications like:
# "InsecureRequestWarning: Unverified HTTPS request is being made."
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


class EnvironmentManagerHeat(object):
    """Class-helper for creating VMs via devops environments"""

    __config = None

    # Do not use self.__heatclient directly! Use properties
    # for necessary resources with catching HTTPUnauthorized exception
    __heatclient = None

    def __init__(self, config=None):
        """Create/connect to the Heat stack with test environment

        :param config: oslo.config object
        :param config.hardware.heat_version: Heat version
        :param config.hardware.os_auth_url: OS auth URL to access heat
        :param config.hardware.os_username: OS username
        :param config.hardware.os_password: OS password
        :param config.hardware.os_project_name: OS tenant name
        """
        self.__config = config

        if not self.__config.hardware.heat_stack_name:
            self.__config.hardware.heat_stack_name = settings.ENV_NAME

        self.__init_heatclient()

        try:
            stack_status = self._current_stack.stack_status
            if stack_status != EXPECTED_STACK_STATUS:
                raise exceptions.EnvironmentWrongStatus(
                    self.__config.hardware.heat_stack_name,
                    EXPECTED_STACK_STATUS,
                    stack_status
                )
            LOG.info("Heat stack '{0}' already exists".format(
                self.__config.hardware.heat_stack_name))
        except heat_exceptions.HTTPNotFound:
            self._create_environment()
            LOG.info("Heat stack '{0}' created".format(
                self.__config.hardware.heat_stack_name))

        self.set_address_pools_config()
        self.set_dhcp_ranges_config()

    @cached_property
    def _keystone_session(self):
        keystone_auth = keystone_v3.Password(
            auth_url=settings.OS_AUTH_URL,
            username=settings.OS_USERNAME,
            password=settings.OS_PASSWORD,
            project_name=settings.OS_PROJECT_NAME,
            user_domain_name=settings.OS_USER_DOMAIN_NAME,
            project_domain_name='Default')
        return keystone_session.Session(auth=keystone_auth, verify=False)

    def __init_heatclient(self):
        token = self._keystone_session.get_token()
        endpoint_url = self._keystone_session.get_endpoint(
            service_type='orchestration', endpoint_type='publicURL')
        self.__heatclient = heatclient.Client(
            version=settings.OS_HEAT_VERSION, endpoint=endpoint_url,
            token=token, insecure=True)

    @property
    def _current_stack(self):
        return self.__stacks.get(
            self.__config.hardware.heat_stack_name)

    @property
    def __stacks(self):
        try:
            return self.__heatclient.stacks
        except heat_exceptions.HTTPUnauthorized:
            LOG.warning("Authorization token outdated, refreshing")
            self.__init_heatclient()
            return self.__heatclient.stacks

    @property
    def __resources(self):
        try:
            return self.__heatclient.resources
        except heat_exceptions.HTTPUnauthorized:
            LOG.warning("Authorization token outdated, refreshing")
            self.__init_heatclient()
            return self.__heatclient.resources

    def __get_stack_parent(self, stack_id, stacks):
        """Find the parent ID of the specified stack_id in the 'stacks' list"""
        for stack in stacks:
            if stack_id == stack.id:
                if stack.parent:
                    try:
                        return self.__get_stack_parent(stack.parent, stacks)
                    except Exception:
                        return stack.id
                else:
                    return stack.id
        raise Exception("stack with ID {} not found!".format(stack_id))

    @property
    def __nested_resources(self):
        if hasattr(pytest, 'resources'):
            return pytest.resources
        pytest.resources = list()
        stacks = [s for s in self.__stacks.list(show_nested=True)]
        current_stack_id = self._current_stack.id
        for stack in stacks:
            parent_stack_id = self.__get_stack_parent(stack.id, stacks)
            if parent_stack_id == current_stack_id:
                # Add resources to list
                LOG.info("Get resources from stack {0}"
                         .format(stack.stack_name))
                pytest.resources.extend([
                    res for res in self.__resources.list(stack.id)
                ])
        LOG.info("Found {0} resources".format(len(pytest.resources)))
        return pytest.resources

    def _get_resources_by_type(self, resource_type):
        res = []
        for item in self.__nested_resources:
            if item.resource_type == resource_type:
                resource = self.__resources.get(
                    item.stack_name,
                    item.resource_name)
                res.append(resource)
        return res

    @cached_property
    def _nodes(self):
        """Get list of nodenames from heat

        Returns list of dicts.
        Example:
        - name: cfg01
          roles:
          - salt_master
          addresses:  # Optional. May be an empty dict
            admin-pool01: p.p.p.202
        - name: ctl01
          roles:
          - salt_minion
          - openstack_controller
          - openstack_messaging
          - openstack_database
          addresses: {} # Optional. May be an empty dict

        'name': taken from heat template resource's ['name'] parameter
        'roles': a list taken from resource's ['metadata']['roles'] parameter
        """
        address_pools = self._address_pools
        nodes = []
        for heat_node in self._get_resources_by_type("OS::Nova::Server"):
            # addresses will have the following dict structure:
            #   {'admin-pool01': <floating_ip1>,
            #    'private-pool01': <floating_ip2>,
            #    'external-pool01': <floating_ip3>
            #   }
            # , where key is one of roles from OS::Neutron::Subnet,
            # and value is a floating IP associated to the fixed IP
            # in this subnet (if exists).
            # If no floating IPs associated to the server,
            # then addresses will be an empty list.
            addresses = {}
            for network in heat_node.attributes['addresses']:
                fixed = None
                floating = None
                for address in heat_node.attributes['addresses'][network]:
                    addr_type = address['OS-EXT-IPS:type']
                    if addr_type == 'fixed':
                        fixed = address['addr']
                    elif addr_type == 'floating':
                        floating = address['addr']
                    else:
                        LOG.error("Unexpected OS-EXT-IPS:type={0} "
                                  "in node '{1}' for network '{2}'"
                                  .format(addr_type,
                                          heat_node.attributes['name'],
                                          network))
                if fixed is None and floating is None:
                    LOG.error("Unable to determine the correct IP address "
                              "in node '{0}' for network '{1}'"
                              .format(heat_node.attributes['name'], network))
                    continue
                # Check which address pool has the fixed address, and set
                # the floating address as the access to this address pool.
                for address_pool in address_pools:
                    pool_net = netaddr.IPNetwork(address_pool['cidr'])
                    if fixed in pool_net:
                        for role in address_pool['roles']:
                            # addresses[role] = floating
                            # Use fixed addresses for SSH access
                            addresses[role] = fixed
            if 'metadata' not in heat_node.attributes or \
                    'roles' not in heat_node.attributes['metadata']:
                raise Exception("Node {} doesn't have metadata:roles:[...,...]"
                                .format(heat_node.attributes['name']))

            nodes.append({
                'name': heat_node.attributes['name'],
                'roles': yaml.load(heat_node.attributes['metadata']['roles']),
                'addresses': addresses,
            })
        return nodes

    @cached_property
    def _address_pools(self):
        """Get address pools from subnets OS::Neutron::Subnet

        Returns list of dicts.
        Example:
        - roles:
          - admin-pool01
          cidr: x.x.x.x/y
          start: x.x.x.2
          end: x.x.x.254
          gateway: x.x.x.1 # or None
        """
        pools = []
        for heat_subnet in self._get_resources_by_type("OS::Neutron::Subnet"):
            pools.append({
                'roles': heat_subnet.attributes['tags'],
                'cidr': heat_subnet.attributes['cidr'],
                'gateway': heat_subnet.attributes['gateway_ip'],
                'start': heat_subnet.attributes[
                    'allocation_pools'][0]['start'],
                'end': heat_subnet.attributes['allocation_pools'][0]['end'],
            })
        return pools

    def _get_nodes_by_roles(self, roles=None):
        nodes = []
        if roles is None:
            return self._nodes

        for node in self._nodes:
            if set(node['roles']).intersection(set(roles)):
                nodes.append(node)
        return nodes

    def get_ssh_data(self, roles=None):
        """Generate ssh config for Underlay

        :param roles: list of strings
        """
        if roles is None:
            raise Exception("No roles specified for the environment!")

        config_ssh = []
        for d_node in self._get_nodes_by_roles(roles=roles):
            for pool_name in d_node['addresses']:
                ssh_data = {
                    'node_name': d_node['name'],
                    'minion_id': d_node['name'],
                    'roles': d_node['roles'],
                    'address_pool': pool_name,
                    'host': d_node['addresses'][pool_name],
                    'login': settings.SSH_NODE_CREDENTIALS['login'],
                    'password': settings.SSH_NODE_CREDENTIALS['password'],
                    'keys': [k['private']
                             for k in self.__config.underlay.ssh_keys]
                }
            config_ssh.append(ssh_data)
        return config_ssh

    def _get_resources_with_wrong_status(self):
        res = []
        for item in self.__nested_resources:
            if item.resource_status in BAD_STACK_STATUSES:
                res.append({
                    'resource_name': item.resource_name,
                    'resource_status': item.resource_status,
                    'resource_status_reason': item.resource_status_reason,
                    'resource_type': item.resource_type
                })
        wrong_resources = '\n'.join([
            "*** Heat stack resource '{0}' ({1}) has wrong status '{2}': {3}"
            .format(item['resource_name'],
                    item['resource_type'],
                    item['resource_status'],
                    item['resource_status_reason'])
            for item in res
        ])
        return wrong_resources

    def wait_of_stack_status(self, status, delay=30, tries=60,
                             wait_for_delete=False):

        @retry(exceptions.EnvironmentWrongStatus, delay=delay, tries=tries)
        def wait():
            try:
                st = self._current_stack.stack_status
            except heat_exceptions.HTTPNotFound as ex:
                if wait_for_delete is True:
                    return
                raise ex
            except heat_exceptions.HTTPException as ex:
                # tolerate HTTP timeouts from Heat
                if ex.code == 504:
                    raise exceptions.EnvironmentWrongStatus(
                        self.__config.hardware.heat_stack_name,
                        status,
                        "Heat API Temporary Unavailable"
                    )
                else:
                    raise ex
            if st == status:
                return
            elif st in BAD_STACK_STATUSES:
                wrong_resources = self._get_resources_with_wrong_status()
                raise exceptions.EnvironmentBadStatus(
                    self.__config.hardware.heat_stack_name,
                    status,
                    st,
                    wrong_resources
                )
            else:
                LOG.info("Stack {0} status: {1}".format(
                    self.__config.hardware.heat_stack_name, st))
                raise exceptions.EnvironmentWrongStatus(
                    self.__config.hardware.heat_stack_name,
                    status,
                    st
                )
        LOG.info("Waiting for stack '{0}' status <{1}>".format(
            self.__config.hardware.heat_stack_name, status))
        wait()

    def revert_snapshot(self, name):
        """Revert snapshot by name

        - Revert the heat snapshot in the environment
        - Try to reload 'config' object from a file 'config_<name>.ini'
          If the file not found, then pass with defaults.
        - Set <name> as the current state of the environment after reload

        :param name: string
        """
        LOG.info("Reading INI config (without reverting env to snapshot) "
                 "named '{0}'".format(name))

        try:
            test_config_path = self._get_snapshot_config_name(name)
            settings_oslo.reload_snapshot_config(self.__config,
                                                 test_config_path)
        except cfg.ConfigFilesNotFoundError as conf_err:
            LOG.error("Config file(s) {0} not found!".format(
                conf_err.config_files))

        self.__config.hardware.current_snapshot = name

    def create_snapshot(self, name, *args, **kwargs):
        """Create named snapshot of current env.

        - Create a snapshot for the environment
        - Save 'config' object to a file 'config_<name>.ini'

        :name: string
        """
        if not settings.MAKE_SNAPSHOT_STAGES:
            msg = ("[ SKIP snapshot '{0}' because MAKE_SNAPSHOT_STAGES=false ]"
                   .format(name))
            LOG.info("\n\n{0}\n{1}".format(msg, '*' * len(msg)))
            return

        LOG.info("Store INI config (without env snapshot) named '{0}'"
                 .format(name))
        self.__config.hardware.current_snapshot = name
        settings_oslo.save_config(self.__config,
                                  name,
                                  self.__config.hardware.heat_stack_name)

    def _get_snapshot_config_name(self, snapshot_name):
        """Get config name for the environment"""
        env_name = self.__config.hardware.heat_stack_name
        if env_name is None:
            env_name = 'config'
        test_config_path = os.path.join(
            settings.LOGS_DIR, '{0}_{1}.ini'.format(env_name, snapshot_name))
        return test_config_path

    def has_snapshot(self, name):
        # Heat doesn't support live snapshots, so just
        # check if an INI file was created for this environment,
        # assuming that the environment has the configuration
        # described in this INI.
        return self.has_snapshot_config(name)

    def has_snapshot_config(self, name):
        test_config_path = self._get_snapshot_config_name(name)
        return os.path.isfile(test_config_path)

    def start(self, underlay_node_roles, timeout=480):
        """Start environment"""
        LOG.warning("HEAT Manager doesn't support start environment feature. "
                    "Waiting for finish the bootstrap process on the nodes "
                    "with accessible SSH")

        check_cloudinit_finished = ('[ -f /is_cloud_init_finished ] || '
                                    '[ -f /var/log/mcp/.bootstrap_done ]')
        passed = {}
        nodes_by_roles = self._get_nodes_by_roles(roles=underlay_node_roles)
        for node in nodes_by_roles:
            try:
                node_ip = self.node_ip(node)
            except exceptions.EnvironmentNodeAccessError:
                LOG.warning("Node {0} doesn't have accessible IP address"
                            ", skipping".format(node['name']))
                continue

            LOG.info("Waiting for SSH on node '{0}' / {1} ...".format(
                node['name'], node_ip))

            def _readiness_check(
                    host,
                    port,
                    username=settings.SSH_NODE_CREDENTIALS['login'],
                    password=settings.SSH_NODE_CREDENTIALS['password'],
                    timeout=0):
                try:
                    ssh = ssh_client.SSHClient(
                        host=host, port=port,
                        auth=ssh_client.SSHAuth(
                            username=username,
                            password=password))

                    def bootstrap_is_successful():
                        is_cloudinit_completed = ssh.execute(
                            "tail -n1 /var/log/cloud-init.log |"
                            "grep -q 'finish: modules-final: SUCCESS'"
                                           )['exit_code'] == 0
                        # cfg node doesn't have
                        # 'finish: modules-final: SUCCESS' line
                        # in the logs because the cfg node is rebooted during
                        # bootstrap. Here is /var/log/mcp/.bootstrap_done
                        # file used as a flag
                        is_manually_set_flag = ssh.execute(
                            check_cloudinit_finished)['exit_code'] == 0

                        return is_cloudinit_completed or is_manually_set_flag

                    def no_cloudinit():
                        return ssh.execute("[ ! -d /var/lib/cloud/instance ]"
                                           )['exit_code'] == 0

                    if bootstrap_is_successful():
                        return True
                    elif no_cloudinit():
                        return ssh.execute('echo ok')['exit_code'] == 0
                    else:
                        return False

                except (AuthenticationException, BadAuthenticationType):
                    return True
                except Exception:
                    return False

            def _ssh_wait(host,
                          port,
                          username=settings.SSH_NODE_CREDENTIALS['login'],
                          password=settings.SSH_NODE_CREDENTIALS['password']):
                """
                Collects hosts have passed the SSH / cloud-init checks.
                And returns True if host has passed this check

                :param host: str, IP address to connect via SSH
                :param port: int, port to connect via SSH
                :param username: str, name of SSH user
                :param password: str, password for SSH user
                :return:  True if host has passed the probe test
                """
                if host in passed and passed[host] >= 1:
                    # host already passed the check
                    return True

                for _node in nodes_by_roles:
                    ip = self.node_ip(_node)
                    if ip not in passed:
                        passed[ip] = 0
                    if _readiness_check(ip, port):
                        passed[ip] += 1
                        LOG.info("{} is already ready".format(_node['name']))
                        nodes_by_roles.remove(_node)
                    else:
                        passed[ip] = 0

            helpers.wait(
                lambda: _ssh_wait(node_ip, 22),
                timeout=timeout,
                timeout_msg="Node '{}' didn't open SSH "
                            "(or didn't complete cloud-init) "
                            "in {} sec".format(
                    node['name'], timeout
                )
            )
        LOG.info('Heat stack "{0}" ready'
                 .format(self.__config.hardware.heat_stack_name))

    def _verify_resources_status(self, status):
        """Check that all resources have verified `status`

        In case when all resources have expected status return empty list,
            otherwise return a list with resources with incorrect status.
        """
        ret = [r for r in self.__nested_resources if
               r.resource_status != status]
        return ret

    def _create_environment(self):
        tpl_files, template = template_utils.get_template_contents(
            self.__config.hardware.heat_conf_path)
        env_files_list = []
        env_files, env = (
            template_utils.process_multiple_environments_and_files(
                env_paths=[self.__config.hardware.heat_env_path],
                env_list_tracker=env_files_list))

        mcp_version = settings.MCP_VERSION
        if "2019.2" in settings.MCP_VERSION:
            mcp_version = "2019.2.0"
        fields = {
            'stack_name': self.__config.hardware.heat_stack_name,
            'template': template,
            'files': dict(list(tpl_files.items()) + list(env_files.items())),
            'environment': env,
            'parameters': {
                'mcp_version': mcp_version,
                'env_name': settings.ENV_NAME,
                'deploy_empty_node': bool(settings.DEPLOY_EMPTY_NODE)
            }
        }

        if env_files_list:
            fields['environment_files'] = env_files_list

        @retry(heat_exceptions.HTTPBadGateway, delay=15, tries=20)
        def safe_heat_stack_create():
            self.__stacks.create(**fields)

        @retry(exceptions.EnvironmentBadStatus, delay=60, tries=1)
        def safe_create():
            self.delete_environment()
            safe_heat_stack_create()
            self.wait_of_stack_status(EXPECTED_STACK_STATUS, tries=140)
            LOG.info("Stack '%s' created",
                     self.__config.hardware.heat_stack_name)
            incorrect_resources = self._verify_resources_status(
                EXPECTED_STACK_STATUS)
            if incorrect_resources:
                LOG.info("Recreate the stack because some resources have "
                         "incorrect status")
                for r in incorrect_resources:
                    LOG.error(
                        'The resource %s has status %s. But it should be %s',
                        r.resource_name,
                        r.resource_status,
                        EXPECTED_STACK_STATUS)
                st = self._current_stack.stack_status
                raise exceptions.EnvironmentBadStatus(
                    self.__config.hardware.heat_stack_name,
                    EXPECTED_STACK_STATUS,
                    st,
                    incorrect_resources)
        safe_create()

    def stop(self):
        """Stop environment"""
        LOG.warning("HEAT Manager doesn't support stop environment feature")
        pass

    def delete_environment(self):
        if list(self.__stacks.list(
                    stack_name=self.__config.hardware.heat_stack_name)):
            LOG.info("Delete stack '%s'",
                     self.__config.hardware.heat_stack_name)

            @retry(heat_exceptions.HTTPBadGateway, delay=15, tries=20)
            def safe_heat_stack_delete():
                self.__stacks.delete(self._current_stack.id)

            safe_heat_stack_delete()
            self.wait_of_stack_status('DELETE_COMPLETE',
                                      delay=30, tries=20,
                                      wait_for_delete=True)
        else:
            LOG.warning("Can't delete stack '%s' due is absent",
                        self.__config.hardware.heat_stack_name)

# TODO(ddmitriev): add all Environment methods
    @staticmethod
    def node_ip(node, address_pool_name='admin-pool01'):
        """Determine node's IP

        :param node: a dict element from the self._nodes
        :return: string
        """
        if address_pool_name in node['addresses']:
            addr = node['addresses'][address_pool_name]
            LOG.debug('{0} IP= {1}'.format(node['name'], addr))
            return addr
        else:
            raise exceptions.EnvironmentNodeAccessError(
                node['name'],
                "No addresses available for the subnet {0}"
                .format(address_pool_name))

    def set_address_pools_config(self):
        """Store address pools CIDRs in config object"""
        for ap in self._address_pools:
            for role in ap['roles']:
                self.__config.underlay.address_pools[role] = ap['cidr']

    def set_dhcp_ranges_config(self):
        """Store DHCP ranges in config object"""
        for ap in self._address_pools:
            for role in ap['roles']:
                self.__config.underlay.dhcp_ranges[role] = {
                    "cidr": ap['cidr'],
                    "start": ap['start'],
                    "end": ap['end'],
                    "gateway": ap['gateway'],
                }

    def wait_for_node_state(self, node_name, state, timeout):
        raise NotImplementedError()

    def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600):
        raise NotImplementedError()

    def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600):
        raise NotImplementedError()

    @property
    def slave_nodes(self):
        raise NotImplementedError()