| # Copyright 2016 Mirantis, Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| import os |
| |
| from devops import error |
| from devops.helpers import helpers |
| from devops.helpers.helpers import ssh_client |
| from devops import models |
| from django import db |
| from oslo_config import cfg |
| |
| from paramiko.ssh_exception import ( |
| AuthenticationException, |
| BadAuthenticationType) |
| |
| from tcp_tests import settings |
| from tcp_tests import settings_oslo |
| from tcp_tests.helpers import env_config |
| from tcp_tests.helpers import ext |
| from tcp_tests.helpers import exceptions |
| from tcp_tests import logger |
| |
| LOG = logger.logger |
| |
| |
| class EnvironmentManager(object): |
| """Class-helper for creating VMs via devops environments""" |
| |
| __config = None |
| |
| def __init__(self, config=None): |
| """Initializing class instance and create the environment |
| |
| :param config: oslo.config object |
| :param config.hardware.conf_path: path to devops YAML template |
| :param config.hardware.current_snapshot: name of the snapshot that |
| descriebe environment status. |
| """ |
| self.__devops_config = env_config.EnvironmentConfig() |
| self.__env = None |
| self.__config = config |
| |
| if config.hardware.conf_path is not None: |
| options = { |
| 'config': self.__config, |
| } |
| self._devops_config.load_template(config.hardware.conf_path, |
| options=options) |
| else: |
| raise Exception("Devops YAML template is not set in config object") |
| |
| try: |
| self._get_env_by_name(self._d_env_name) |
| if not self.has_snapshot(config.hardware.current_snapshot): |
| raise exceptions.EnvironmentSnapshotMissing( |
| self._d_env_name, config.hardware.current_snapshot) |
| except error.DevopsObjNotFound: |
| LOG.info("Environment doesn't exist, creating a new one") |
| self._create_environment() |
| self.set_dns_config() |
| self.set_address_pools_config() |
| self.set_dhcp_ranges_config() |
| |
| @property |
| def _devops_config(self): |
| return self.__devops_config |
| |
| @_devops_config.setter |
| def _devops_config(self, conf): |
| """Setter for self.__devops_config |
| |
| :param conf: tcp_tests.helpers.env_config.EnvironmentConfig |
| """ |
| if not isinstance(conf, env_config.EnvironmentConfig): |
| msg = ("Unexpected type of devops config. Got '{0}' " + |
| "instead of '{1}'") |
| raise TypeError( |
| msg.format( |
| type(conf).__name__, |
| env_config.EnvironmentConfig.__name__ |
| ) |
| ) |
| self.__devops_config = conf |
| |
| @property |
| def _d_env_name(self): |
| """Get environment name from fuel devops config |
| |
| :rtype: string |
| """ |
| return self._devops_config['env_name'] |
| |
| def _get_env_by_name(self, name): |
| """Set existing environment by name |
| |
| :param name: string |
| """ |
| self.__env = models.Environment.get(name=name) |
| |
| def _get_default_node_group(self): |
| return self.__env.get_group(name='default') |
| |
| def _get_network_pool(self, net_pool_name): |
| default_node_group = self._get_default_node_group() |
| network_pool = default_node_group.get_network_pool(name=net_pool_name) |
| return network_pool |
| |
| def get_ssh_data(self, roles=None): |
| """Generate ssh config for Underlay |
| |
| :param roles: list of strings |
| """ |
| if roles is None: |
| raise Exception("No roles specified for the environment!") |
| |
| config_ssh = [] |
| for d_node in self.__env.get_nodes(role__in=roles): |
| ssh_data = { |
| 'node_name': d_node.name, |
| 'minion_id': d_node.name, |
| 'roles': [d_node.role], |
| 'address_pool': self._get_network_pool( |
| ext.NETWORK_TYPE.admin).address_pool.name, |
| 'host': self.node_ip(d_node), |
| 'login': settings.SSH_NODE_CREDENTIALS['login'], |
| 'password': settings.SSH_NODE_CREDENTIALS['password'], |
| 'keys': [k['private'] for k in self.__config.underlay.ssh_keys] |
| } |
| config_ssh.append(ssh_data) |
| return config_ssh |
| |
| def create_snapshot(self, name, description=None, force=False): |
| """Create named snapshot of current env. |
| |
| - Create a libvirt snapshots for all nodes in the environment |
| - Save 'config' object to a file 'config_<name>.ini' |
| |
| :name: string |
| """ |
| if not settings.MAKE_SNAPSHOT_STAGES and not force: |
| msg = ("[ SKIP snapshot '{0}' because MAKE_SNAPSHOT_STAGES=false ]" |
| " {1}".format(name, description or '')) |
| LOG.info("\n\n{0}\n{1}".format(msg, '*' * len(msg))) |
| return |
| msg = "[ Create snapshot '{0}' ] {1}".format(name, description or '') |
| LOG.info("\n\n{0}\n{1}".format(msg, '*' * len(msg))) |
| |
| self.__config.hardware.current_snapshot = name |
| LOG.info("Set current snapshot in config to '{0}'".format( |
| self.__config.hardware.current_snapshot)) |
| if self.__env is not None: |
| LOG.info('trying to suspend ....') |
| self.__env.suspend() |
| LOG.info('trying to snapshot ....') |
| self.__env.snapshot(name, description=description, force=True) |
| LOG.info('trying to resume ....') |
| self.__env.resume() |
| else: |
| raise exceptions.EnvironmentIsNotSet() |
| settings_oslo.save_config(self.__config, name, self.__env.name) |
| |
| if settings.VIRTUAL_ENV: |
| venv_msg = "source {0}/bin/activate;\n".format( |
| settings.VIRTUAL_ENV) |
| else: |
| venv_msg = "" |
| LOG.info("To revert the snapshot:\n\n" |
| "************************************\n" |
| "{venv_msg}" |
| "dos.py revert {env_name} {snapshot_name};\n" |
| "dos.py resume {env_name};\n" |
| "# dos.py time-sync {env_name}; # Optional\n" |
| "ssh -i {key_file} {login}@{salt_master_host} " |
| "# Optional password: {password}\n" |
| "************************************\n" |
| .format(venv_msg=venv_msg, |
| env_name=self._d_env_name, |
| snapshot_name=name, |
| login=settings.SSH_NODE_CREDENTIALS['login'], |
| password=settings.SSH_NODE_CREDENTIALS['password'], |
| salt_master_host=self.__config.salt.salt_master_host, |
| key_file=self.__config.underlay.ssh_key_file)) |
| |
| def _get_snapshot_config_name(self, snapshot_name): |
| """Get config name for the environment""" |
| env_name = self.__env.name |
| if env_name is None: |
| env_name = 'config' |
| test_config_path = os.path.join( |
| settings.LOGS_DIR, '{0}_{1}.ini'.format(env_name, snapshot_name)) |
| return test_config_path |
| |
| def revert_snapshot(self, name): |
| """Revert snapshot by name |
| |
| - Revert a libvirt snapshots for all nodes in the environment |
| - Try to reload 'config' object from a file 'config_<name>.ini' |
| If the file not found, then pass with defaults. |
| - Set <name> as the current state of the environment after reload |
| |
| :param name: string |
| """ |
| if not settings.MAKE_SNAPSHOT_STAGES: |
| LOG.info("SKIP reverting from snapshot '{0}' " |
| "because MAKE_SNAPSHOT_STAGES=false".format(name)) |
| return |
| |
| if self.__env is not None: |
| LOG.info("Suspending environment to stop IO") |
| self.__env.suspend() |
| LOG.info("Reverting from snapshot named '{0}'".format(name)) |
| self.__env.revert(name=name) |
| LOG.info("Resuming environment after revert") |
| self.__env.resume() |
| else: |
| raise exceptions.EnvironmentIsNotSet() |
| |
| try: |
| test_config_path = self._get_snapshot_config_name(name) |
| settings_oslo.reload_snapshot_config(self.__config, |
| test_config_path) |
| except cfg.ConfigFilesNotFoundError as conf_err: |
| LOG.error("Config file(s) {0} not found!".format( |
| conf_err.config_files)) |
| |
| self.__config.hardware.current_snapshot = name |
| |
| def _create_environment(self): |
| """Create environment and start VMs. |
| |
| If config was provided earlier, we simply create and start VMs, |
| otherwise we tries to generate config from self.config_file, |
| """ |
| if self._devops_config.config is None: |
| raise exceptions.DevopsConfigPathIsNotSet() |
| settings = self._devops_config |
| env_name = settings['env_name'] |
| LOG.debug( |
| 'Preparing to create environment named "{0}"'.format(env_name) |
| ) |
| if env_name is None: |
| LOG.error('Environment name is not set!') |
| raise exceptions.EnvironmentNameIsNotSet() |
| try: |
| self.__env = models.Environment.create_environment( |
| settings.config |
| ) |
| except db.IntegrityError: |
| LOG.error( |
| 'Seems like environment {0} already exists or contain errors' |
| ' in template.'.format(env_name) |
| ) |
| raise |
| self.__env.define() |
| LOG.info( |
| 'Environment "{0}" created'.format(env_name) |
| ) |
| |
| def start(self, underlay_node_roles, timeout=480): |
| """Method for start environment |
| |
| """ |
| if self.__env is None: |
| raise exceptions.EnvironmentIsNotSet() |
| self.__env.start() |
| LOG.info('Environment "{0}" started'.format(self.__env.name)) |
| check_cloudinit_started = '[ -f /is_cloud_init_started ]' |
| check_cloudinit_finished = ('[ -f /is_cloud_init_finished ] || ' |
| '[ -f /var/log/mcp/.bootstrap_done ]') |
| check_cloudinit_failed = 'cat /is_cloud_init_failed' |
| passed = {} |
| for node in self.__env.get_nodes(role__in=underlay_node_roles): |
| LOG.info("Waiting for SSH on node '{0}' / {1} ...".format( |
| node.name, self.node_ip(node))) |
| |
| def _ssh_check(host, |
| port, |
| username=settings.SSH_NODE_CREDENTIALS['login'], |
| password=settings.SSH_NODE_CREDENTIALS['password'], |
| timeout=0): |
| try: |
| ssh = ssh_client.SSHClient( |
| host=host, port=port, |
| auth=ssh_client.SSHAuth( |
| username=username, |
| password=password)) |
| |
| # If '/is_cloud_init_started' exists, then wait for |
| # the flag /is_cloud_init_finished |
| if ssh.execute(check_cloudinit_started)['exit_code'] == 0: |
| result = ssh.execute(check_cloudinit_failed) |
| if result['exit_code'] == 0: |
| raise exceptions.EnvironmentNodeIsNotStarted( |
| "{0}:{1}".format(host, port), |
| result.stdout_str) |
| |
| status = ssh.execute( |
| check_cloudinit_finished)['exit_code'] == 0 |
| # Else, just wait for SSH |
| else: |
| status = ssh.execute('echo ok')['exit_code'] == 0 |
| return status |
| |
| except (AuthenticationException, BadAuthenticationType): |
| return True |
| except Exception: |
| return False |
| |
| def _ssh_wait(host, |
| port, |
| username=settings.SSH_NODE_CREDENTIALS['login'], |
| password=settings.SSH_NODE_CREDENTIALS['password'], |
| timeout=0): |
| |
| if host in passed and passed[host] >= 2: |
| # host already passed the check |
| return True |
| |
| for node in self.__env.get_nodes(role__in=underlay_node_roles): |
| ip = self.node_ip(node) |
| if ip not in passed: |
| passed[ip] = 0 |
| if _ssh_check(ip, port): |
| passed[ip] += 1 |
| else: |
| passed[ip] = 0 |
| |
| helpers.wait( |
| lambda: _ssh_wait(self.node_ip(node), 22), |
| timeout=timeout, |
| timeout_msg="Node '{}' didn't open SSH in {} sec".format( |
| node.name, timeout |
| ) |
| ) |
| LOG.info('Environment "{0}" ready'.format(self.__env.name)) |
| |
| def resume(self): |
| """Resume environment""" |
| if self.__env is None: |
| raise exceptions.EnvironmentIsNotSet() |
| self.__env.resume() |
| |
| def suspend(self): |
| """Suspend environment""" |
| if self.__env is None: |
| raise exceptions.EnvironmentIsNotSet() |
| self.__env.suspend() |
| |
| def stop(self): |
| """Stop environment""" |
| if self.__env is None: |
| raise exceptions.EnvironmentIsNotSet() |
| self.__env.destroy() |
| |
| def destroy_node(self, node_name): |
| """Destroy node""" |
| node = self.__env.get_node(name=node_name) |
| node.destroy() |
| |
| def start_node(self, node_name): |
| """Start node""" |
| node = self.__env.get_node(name=node_name) |
| node.start() |
| |
| def reboot_node(self, node_name): |
| """Reboot node""" |
| node = self.__env.get_node(name=node_name) |
| node.reboot() |
| |
| def remove_node(self, node_name): |
| """Remove node""" |
| node = self.__env.get_node(name=node_name) |
| node.remove() |
| |
| def wait_for_node_state(self, node_name, state, timeout): |
| node = self.__env.get_node(name=node_name) |
| if 'active' in state: |
| helpers.wait(lambda: node.is_active(), |
| timeout=timeout, |
| timeout_msg=('Node {0} failed ' |
| 'to become active'.format(node))) |
| else: |
| helpers.wait(lambda: not node.is_active(), |
| timeout=timeout, |
| timeout_msg=('Node {0} failed ' |
| 'to become active'.format(node))) |
| |
| def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600): |
| node_names = underlay.get_target_node_names(nodes_prefix) |
| for node in node_names: |
| LOG.debug('Shutdown node {0}'.format(node)) |
| underlay.check_call(cmd="shutdown +1", node_name=node) |
| for node in node_names: |
| self.wait_for_node_state(node, state='offline', timeout=timeout) |
| |
| def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600): |
| self.warm_shutdown_nodes(underlay, nodes_prefix, timeout) |
| node_names = underlay.get_target_node_names(nodes_prefix) |
| for node in node_names: |
| LOG.debug('Starting node {0}'.format(node)) |
| self.start_node(node) |
| self.wait_for_node_state(node, state='active', timeout=timeout) |
| |
| def has_snapshot(self, name): |
| return self.__env.has_snapshot(name) |
| |
| def has_snapshot_config(self, name): |
| test_config_path = self._get_snapshot_config_name(name) |
| return os.path.isfile(test_config_path) |
| |
| def delete_environment(self): |
| """Delete environment |
| |
| """ |
| LOG.debug("Deleting environment") |
| self.__env.erase() |
| |
| def __get_nodes_by_role(self, node_role): |
| """Get node by given role name |
| |
| :param node_role: string |
| :rtype: devops.models.Node |
| """ |
| LOG.debug('Trying to get nodes by role {0}'.format(node_role)) |
| return self.__env.get_nodes(role=node_role) |
| |
| def __get_nodes_by_name(self, node_name): |
| """Get node by given role name |
| |
| :param node_name: string |
| :rtype: devops.models.Node |
| """ |
| LOG.debug('Trying to get nodes by role {0}'.format(node_name)) |
| return self.__env.get_nodes(name=node_name) |
| |
| @property |
| def master_nodes(self): |
| """Get all master nodes |
| |
| :rtype: list |
| """ |
| nodes = self.__get_nodes_by_role( |
| node_role=ext.UNDERLAY_NODE_ROLES.salt_master) |
| return nodes |
| |
| @property |
| def slave_nodes(self): |
| """Get all slave nodes |
| |
| :rtype: list |
| """ |
| nodes = self.__get_nodes_by_role( |
| node_role=ext.UNDERLAY_NODE_ROLES.salt_minion) |
| return nodes |
| |
| @staticmethod |
| def node_ip(node): |
| """Determine node's IP |
| |
| :param node: devops.models.Node |
| :return: string |
| """ |
| LOG.debug('Trying to determine {0} ip.'.format(node.name)) |
| return node.get_ip_address_by_network_name( |
| ext.NETWORK_TYPE.admin |
| ) |
| |
| @property |
| def nameserver(self): |
| return self.__env.router(ext.NETWORK_TYPE.admin) |
| |
| def set_dns_config(self): |
| # Set local nameserver to use by default |
| if not self.__config.underlay.nameservers: |
| self.__config.underlay.nameservers = [self.nameserver] |
| if not self.__config.underlay.upstream_dns_servers: |
| self.__config.underlay.upstream_dns_servers = [self.nameserver] |
| |
| def set_address_pools_config(self): |
| """Store address pools CIDRs in config object""" |
| for ap in self.__env.get_address_pools(): |
| self.__config.underlay.address_pools[ap.name] = ap.net |
| |
| def set_dhcp_ranges_config(self): |
| """Store DHCP ranges in config object""" |
| for ap in self.__env.get_address_pools(): |
| if "gateway" in ap.ip_reserved and "dhcp" in ap.ip_ranges: |
| self.__config.underlay.dhcp_ranges[ap.name] = { |
| "cidr": ap.net, |
| "start": ap.ip_range_start("dhcp"), |
| "end": ap.ip_range_end("dhcp"), |
| "gateway": ap.gateway, |
| } |