Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 1 | # Copyright 2016 Mirantis, Inc. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 | # not use this file except in compliance with the License. You may obtain |
| 5 | # a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations |
| 13 | # under the License. |
| 14 | |
| 15 | import os |
| 16 | |
| 17 | from devops import error |
| 18 | from devops.helpers import helpers |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 19 | from devops.helpers.helpers import ssh_client |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 20 | from devops import models |
| 21 | from django import db |
| 22 | from oslo_config import cfg |
| 23 | |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 24 | from paramiko.ssh_exception import ( |
| 25 | AuthenticationException, |
| 26 | BadAuthenticationType) |
| 27 | |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 28 | from tcp_tests import settings |
| 29 | from tcp_tests import settings_oslo |
Dmitry Tyzhnenko | 35413c0 | 2018-03-05 14:12:37 +0200 | [diff] [blame] | 30 | from tcp_tests.helpers import env_config |
| 31 | from tcp_tests.helpers import ext |
| 32 | from tcp_tests.helpers import exceptions |
| 33 | from tcp_tests import logger |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 34 | |
| 35 | LOG = logger.logger |
| 36 | |
| 37 | |
| 38 | class EnvironmentManager(object): |
| 39 | """Class-helper for creating VMs via devops environments""" |
| 40 | |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 41 | __config = None |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 42 | |
| 43 | def __init__(self, config=None): |
| 44 | """Initializing class instance and create the environment |
| 45 | |
| 46 | :param config: oslo.config object |
| 47 | :param config.hardware.conf_path: path to devops YAML template |
| 48 | :param config.hardware.current_snapshot: name of the snapshot that |
| 49 | descriebe environment status. |
| 50 | """ |
| 51 | self.__devops_config = env_config.EnvironmentConfig() |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 52 | self.__env = None |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 53 | self.__config = config |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 54 | |
| 55 | if config.hardware.conf_path is not None: |
Artem Panchenko | db0a97f | 2017-06-27 19:09:13 +0300 | [diff] [blame] | 56 | options = { |
| 57 | 'config': self.__config, |
| 58 | } |
| 59 | self._devops_config.load_template(config.hardware.conf_path, |
| 60 | options=options) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 61 | else: |
| 62 | raise Exception("Devops YAML template is not set in config object") |
| 63 | |
| 64 | try: |
| 65 | self._get_env_by_name(self._d_env_name) |
| 66 | if not self.has_snapshot(config.hardware.current_snapshot): |
| 67 | raise exceptions.EnvironmentSnapshotMissing( |
| 68 | self._d_env_name, config.hardware.current_snapshot) |
| 69 | except error.DevopsObjNotFound: |
| 70 | LOG.info("Environment doesn't exist, creating a new one") |
| 71 | self._create_environment() |
| 72 | self.set_dns_config() |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 73 | self.set_address_pools_config() |
Dennis Dmitriev | b6bcc5c | 2018-09-26 11:07:53 +0000 | [diff] [blame] | 74 | self.set_dhcp_ranges_config() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 75 | |
| 76 | @property |
| 77 | def _devops_config(self): |
| 78 | return self.__devops_config |
| 79 | |
| 80 | @_devops_config.setter |
| 81 | def _devops_config(self, conf): |
| 82 | """Setter for self.__devops_config |
| 83 | |
| 84 | :param conf: tcp_tests.helpers.env_config.EnvironmentConfig |
| 85 | """ |
| 86 | if not isinstance(conf, env_config.EnvironmentConfig): |
| 87 | msg = ("Unexpected type of devops config. Got '{0}' " + |
| 88 | "instead of '{1}'") |
| 89 | raise TypeError( |
| 90 | msg.format( |
| 91 | type(conf).__name__, |
| 92 | env_config.EnvironmentConfig.__name__ |
| 93 | ) |
| 94 | ) |
| 95 | self.__devops_config = conf |
| 96 | |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 97 | @property |
| 98 | def _d_env_name(self): |
| 99 | """Get environment name from fuel devops config |
| 100 | |
| 101 | :rtype: string |
| 102 | """ |
| 103 | return self._devops_config['env_name'] |
| 104 | |
| 105 | def _get_env_by_name(self, name): |
| 106 | """Set existing environment by name |
| 107 | |
| 108 | :param name: string |
| 109 | """ |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 110 | self.__env = models.Environment.get(name=name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 111 | |
| 112 | def _get_default_node_group(self): |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 113 | return self.__env.get_group(name='default') |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 114 | |
| 115 | def _get_network_pool(self, net_pool_name): |
| 116 | default_node_group = self._get_default_node_group() |
| 117 | network_pool = default_node_group.get_network_pool(name=net_pool_name) |
| 118 | return network_pool |
| 119 | |
| 120 | def get_ssh_data(self, roles=None): |
| 121 | """Generate ssh config for Underlay |
| 122 | |
| 123 | :param roles: list of strings |
| 124 | """ |
| 125 | if roles is None: |
| 126 | raise Exception("No roles specified for the environment!") |
| 127 | |
| 128 | config_ssh = [] |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 129 | for d_node in self.__env.get_nodes(role__in=roles): |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 130 | ssh_data = { |
| 131 | 'node_name': d_node.name, |
Dennis Dmitriev | 474e3f7 | 2016-10-21 16:46:09 +0300 | [diff] [blame] | 132 | 'roles': [d_node.role], |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 133 | 'address_pool': self._get_network_pool( |
dis | c529838 | 2016-11-23 16:03:33 +0200 | [diff] [blame] | 134 | ext.NETWORK_TYPE.admin).address_pool.name, |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 135 | 'host': self.node_ip(d_node), |
| 136 | 'login': settings.SSH_NODE_CREDENTIALS['login'], |
| 137 | 'password': settings.SSH_NODE_CREDENTIALS['password'], |
Artem Panchenko | db0a97f | 2017-06-27 19:09:13 +0300 | [diff] [blame] | 138 | 'keys': [k['private'] for k in self.__config.underlay.ssh_keys] |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 139 | } |
| 140 | config_ssh.append(ssh_data) |
| 141 | return config_ssh |
| 142 | |
Dennis Dmitriev | 411dd10 | 2017-09-15 16:04:47 +0300 | [diff] [blame] | 143 | def create_snapshot(self, name, description=None, force=False): |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 144 | """Create named snapshot of current env. |
| 145 | |
| 146 | - Create a libvirt snapshots for all nodes in the environment |
| 147 | - Save 'config' object to a file 'config_<name>.ini' |
| 148 | |
| 149 | :name: string |
| 150 | """ |
Dennis Dmitriev | 411dd10 | 2017-09-15 16:04:47 +0300 | [diff] [blame] | 151 | if not settings.MAKE_SNAPSHOT_STAGES and not force: |
| 152 | msg = ("[ SKIP snapshot '{0}' because MAKE_SNAPSHOT_STAGES=false ]" |
| 153 | " {1}".format(name, description or '')) |
| 154 | LOG.info("\n\n{0}\n{1}".format(msg, '*' * len(msg))) |
| 155 | return |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 156 | msg = "[ Create snapshot '{0}' ] {1}".format(name, description or '') |
| 157 | LOG.info("\n\n{0}\n{1}".format(msg, '*' * len(msg))) |
| 158 | |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 159 | self.__config.hardware.current_snapshot = name |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 160 | LOG.info("Set current snapshot in config to '{0}'".format( |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 161 | self.__config.hardware.current_snapshot)) |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 162 | if self.__env is not None: |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 163 | LOG.info('trying to suspend ....') |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 164 | self.__env.suspend() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 165 | LOG.info('trying to snapshot ....') |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 166 | self.__env.snapshot(name, description=description, force=True) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 167 | LOG.info('trying to resume ....') |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 168 | self.__env.resume() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 169 | else: |
| 170 | raise exceptions.EnvironmentIsNotSet() |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 171 | settings_oslo.save_config(self.__config, name, self.__env.name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 172 | |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 173 | if settings.VIRTUAL_ENV: |
Dmitry Tyzhnenko | 2b730a0 | 2017-04-07 19:31:32 +0300 | [diff] [blame] | 174 | venv_msg = "source {0}/bin/activate;\n".format( |
| 175 | settings.VIRTUAL_ENV) |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 176 | else: |
| 177 | venv_msg = "" |
| 178 | LOG.info("To revert the snapshot:\n\n" |
| 179 | "************************************\n" |
| 180 | "{venv_msg}" |
| 181 | "dos.py revert {env_name} {snapshot_name};\n" |
| 182 | "dos.py resume {env_name};\n" |
| 183 | "# dos.py time-sync {env_name}; # Optional\n" |
Artem Panchenko | db0a97f | 2017-06-27 19:09:13 +0300 | [diff] [blame] | 184 | "ssh -i {key_file} {login}@{salt_master_host} " |
| 185 | "# Optional password: {password}\n" |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 186 | "************************************\n" |
| 187 | .format(venv_msg=venv_msg, |
Dennis Dmitriev | 68671a6 | 2017-05-13 16:40:32 +0300 | [diff] [blame] | 188 | env_name=self._d_env_name, |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 189 | snapshot_name=name, |
| 190 | login=settings.SSH_NODE_CREDENTIALS['login'], |
| 191 | password=settings.SSH_NODE_CREDENTIALS['password'], |
Artem Panchenko | db0a97f | 2017-06-27 19:09:13 +0300 | [diff] [blame] | 192 | salt_master_host=self.__config.salt.salt_master_host, |
| 193 | key_file=self.__config.underlay.ssh_key_file)) |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 194 | |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 195 | def _get_snapshot_config_name(self, snapshot_name): |
| 196 | """Get config name for the environment""" |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 197 | env_name = self.__env.name |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 198 | if env_name is None: |
| 199 | env_name = 'config' |
| 200 | test_config_path = os.path.join( |
| 201 | settings.LOGS_DIR, '{0}_{1}.ini'.format(env_name, snapshot_name)) |
| 202 | return test_config_path |
| 203 | |
| 204 | def revert_snapshot(self, name): |
| 205 | """Revert snapshot by name |
| 206 | |
| 207 | - Revert a libvirt snapshots for all nodes in the environment |
| 208 | - Try to reload 'config' object from a file 'config_<name>.ini' |
| 209 | If the file not found, then pass with defaults. |
| 210 | - Set <name> as the current state of the environment after reload |
| 211 | |
| 212 | :param name: string |
| 213 | """ |
Dennis Dmitriev | 411dd10 | 2017-09-15 16:04:47 +0300 | [diff] [blame] | 214 | if not settings.MAKE_SNAPSHOT_STAGES: |
| 215 | LOG.info("SKIP reverting from snapshot '{0}' " |
| 216 | "because MAKE_SNAPSHOT_STAGES=false".format(name)) |
| 217 | return |
| 218 | |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 219 | if self.__env is not None: |
Dennis Dmitriev | a5978eb | 2018-02-21 10:12:33 +0200 | [diff] [blame] | 220 | LOG.info("Suspending environment to stop IO") |
| 221 | self.__env.suspend() |
| 222 | LOG.info("Reverting from snapshot named '{0}'".format(name)) |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 223 | self.__env.revert(name=name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 224 | LOG.info("Resuming environment after revert") |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 225 | self.__env.resume() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 226 | else: |
| 227 | raise exceptions.EnvironmentIsNotSet() |
| 228 | |
| 229 | try: |
| 230 | test_config_path = self._get_snapshot_config_name(name) |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 231 | settings_oslo.reload_snapshot_config(self.__config, |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 232 | test_config_path) |
| 233 | except cfg.ConfigFilesNotFoundError as conf_err: |
| 234 | LOG.error("Config file(s) {0} not found!".format( |
| 235 | conf_err.config_files)) |
| 236 | |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 237 | self.__config.hardware.current_snapshot = name |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 238 | |
| 239 | def _create_environment(self): |
| 240 | """Create environment and start VMs. |
| 241 | |
| 242 | If config was provided earlier, we simply create and start VMs, |
| 243 | otherwise we tries to generate config from self.config_file, |
| 244 | """ |
| 245 | if self._devops_config.config is None: |
| 246 | raise exceptions.DevopsConfigPathIsNotSet() |
| 247 | settings = self._devops_config |
| 248 | env_name = settings['env_name'] |
| 249 | LOG.debug( |
| 250 | 'Preparing to create environment named "{0}"'.format(env_name) |
| 251 | ) |
| 252 | if env_name is None: |
| 253 | LOG.error('Environment name is not set!') |
| 254 | raise exceptions.EnvironmentNameIsNotSet() |
| 255 | try: |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 256 | self.__env = models.Environment.create_environment( |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 257 | settings.config |
| 258 | ) |
| 259 | except db.IntegrityError: |
| 260 | LOG.error( |
dis | 2b2d863 | 2016-12-08 17:56:57 +0200 | [diff] [blame] | 261 | 'Seems like environment {0} already exists or contain errors' |
| 262 | ' in template.'.format(env_name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 263 | ) |
dis | 2b2d863 | 2016-12-08 17:56:57 +0200 | [diff] [blame] | 264 | raise |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 265 | self.__env.define() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 266 | LOG.info( |
Dennis Dmitriev | 53d3b77 | 2016-10-18 14:31:58 +0300 | [diff] [blame] | 267 | 'Environment "{0}" created'.format(env_name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 268 | ) |
| 269 | |
Dennis Dmitriev | 7b9538f | 2017-05-15 17:01:34 +0300 | [diff] [blame] | 270 | def start(self, underlay_node_roles, timeout=480): |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 271 | """Method for start environment |
| 272 | |
| 273 | """ |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 274 | if self.__env is None: |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 275 | raise exceptions.EnvironmentIsNotSet() |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 276 | self.__env.start() |
| 277 | LOG.info('Environment "{0}" started'.format(self.__env.name)) |
Dennis Dmitriev | b01b90e | 2018-06-07 14:57:53 +0300 | [diff] [blame] | 278 | check_cloudinit_started = '[ -f /is_cloud_init_started ]' |
Dennis Dmitriev | f220d97 | 2018-10-10 15:19:14 +0300 | [diff] [blame] | 279 | check_cloudinit_finished = ('[ -f /is_cloud_init_finished ] || ' |
| 280 | '[ -f /var/log/mcp/.bootstrap_done ]') |
Dennis Dmitriev | 86085b4 | 2018-07-02 14:14:25 +0300 | [diff] [blame] | 281 | check_cloudinit_failed = 'cat /is_cloud_init_failed' |
Dennis Dmitriev | b01b90e | 2018-06-07 14:57:53 +0300 | [diff] [blame] | 282 | passed = {} |
Dennis Dmitriev | 7b9538f | 2017-05-15 17:01:34 +0300 | [diff] [blame] | 283 | for node in self.__env.get_nodes(role__in=underlay_node_roles): |
Dennis Dmitriev | a63bac6 | 2017-05-15 18:36:26 +0300 | [diff] [blame] | 284 | LOG.info("Waiting for SSH on node '{0}' / {1} ...".format( |
| 285 | node.name, self.node_ip(node))) |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 286 | |
Dennis Dmitriev | b01b90e | 2018-06-07 14:57:53 +0300 | [diff] [blame] | 287 | def _ssh_check(host, |
| 288 | port, |
| 289 | username=settings.SSH_NODE_CREDENTIALS['login'], |
| 290 | password=settings.SSH_NODE_CREDENTIALS['password'], |
| 291 | timeout=0): |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 292 | try: |
| 293 | ssh = ssh_client.SSHClient( |
| 294 | host=host, port=port, |
| 295 | auth=ssh_client.SSHAuth( |
| 296 | username=username, |
| 297 | password=password)) |
Dennis Dmitriev | b01b90e | 2018-06-07 14:57:53 +0300 | [diff] [blame] | 298 | |
| 299 | # If '/is_cloud_init_started' exists, then wait for |
| 300 | # the flag /is_cloud_init_finished |
| 301 | if ssh.execute(check_cloudinit_started)['exit_code'] == 0: |
Dennis Dmitriev | 86085b4 | 2018-07-02 14:14:25 +0300 | [diff] [blame] | 302 | result = ssh.execute(check_cloudinit_failed) |
| 303 | if result['exit_code'] == 0: |
| 304 | raise exceptions.EnvironmentNodeIsNotStarted( |
| 305 | "{0}:{1}".format(host, port), |
| 306 | result.stdout_str) |
| 307 | |
Dennis Dmitriev | b01b90e | 2018-06-07 14:57:53 +0300 | [diff] [blame] | 308 | status = ssh.execute( |
| 309 | check_cloudinit_finished)['exit_code'] == 0 |
| 310 | # Else, just wait for SSH |
| 311 | else: |
| 312 | status = ssh.execute('echo ok')['exit_code'] == 0 |
| 313 | return status |
| 314 | |
| 315 | except (AuthenticationException, BadAuthenticationType): |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 316 | return True |
| 317 | except Exception: |
| 318 | return False |
| 319 | |
Dennis Dmitriev | b01b90e | 2018-06-07 14:57:53 +0300 | [diff] [blame] | 320 | def _ssh_wait(host, |
| 321 | port, |
| 322 | username=settings.SSH_NODE_CREDENTIALS['login'], |
| 323 | password=settings.SSH_NODE_CREDENTIALS['password'], |
| 324 | timeout=0): |
| 325 | |
| 326 | if host in passed and passed[host] >= 2: |
| 327 | # host already passed the check |
| 328 | return True |
| 329 | |
| 330 | for node in self.__env.get_nodes(role__in=underlay_node_roles): |
| 331 | ip = self.node_ip(node) |
| 332 | if ip not in passed: |
| 333 | passed[ip] = 0 |
| 334 | if _ssh_check(ip, port): |
| 335 | passed[ip] += 1 |
| 336 | else: |
| 337 | passed[ip] = 0 |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 338 | |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 339 | helpers.wait( |
Dmitry Tyzhnenko | b610afd | 2018-02-19 15:43:45 +0200 | [diff] [blame] | 340 | lambda: _ssh_wait(self.node_ip(node), 22), |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 341 | timeout=timeout, |
| 342 | timeout_msg="Node '{}' didn't open SSH in {} sec".format( |
| 343 | node.name, timeout |
| 344 | ) |
| 345 | ) |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 346 | LOG.info('Environment "{0}" ready'.format(self.__env.name)) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 347 | |
| 348 | def resume(self): |
| 349 | """Resume environment""" |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 350 | if self.__env is None: |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 351 | raise exceptions.EnvironmentIsNotSet() |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 352 | self.__env.resume() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 353 | |
| 354 | def suspend(self): |
| 355 | """Suspend environment""" |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 356 | if self.__env is None: |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 357 | raise exceptions.EnvironmentIsNotSet() |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 358 | self.__env.suspend() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 359 | |
| 360 | def stop(self): |
| 361 | """Stop environment""" |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 362 | if self.__env is None: |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 363 | raise exceptions.EnvironmentIsNotSet() |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 364 | self.__env.destroy() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 365 | |
Tatyana Leontovich | e5ccdb3 | 2017-10-09 20:10:43 +0300 | [diff] [blame] | 366 | def destroy_node(self, node_name): |
| 367 | """Destroy node""" |
| 368 | node = self.__env.get_node(name=node_name) |
| 369 | node.destroy() |
| 370 | |
| 371 | def start_node(self, node_name): |
| 372 | """Start node""" |
| 373 | node = self.__env.get_node(name=node_name) |
| 374 | node.start() |
| 375 | |
| 376 | def reboot_node(self, node_name): |
| 377 | """Reboot node""" |
| 378 | node = self.__env.get_node(name=node_name) |
| 379 | node.reboot() |
| 380 | |
| 381 | def remove_node(self, node_name): |
| 382 | """Remove node""" |
| 383 | node = self.__env.get_node(name=node_name) |
| 384 | node.remove() |
| 385 | |
| 386 | def wait_for_node_state(self, node_name, state, timeout): |
| 387 | node = self.__env.get_node(name=node_name) |
| 388 | if 'active' in state: |
| 389 | helpers.wait(lambda: node.is_active(), |
| 390 | timeout=timeout, |
| 391 | timeout_msg=('Node {0} failed ' |
| 392 | 'to become active'.format(node))) |
| 393 | else: |
| 394 | helpers.wait(lambda: not node.is_active(), |
| 395 | timeout=timeout, |
| 396 | timeout_msg=('Node {0} failed ' |
| 397 | 'to become active'.format(node))) |
| 398 | |
Vladimir Jigulin | ee1faa5 | 2018-06-25 13:00:51 +0400 | [diff] [blame] | 399 | def warm_shutdown_nodes(self, underlay, nodes_prefix, timeout=600): |
| 400 | node_names = underlay.get_target_node_names(nodes_prefix) |
| 401 | for node in node_names: |
| 402 | LOG.debug('Shutdown node {0}'.format(node)) |
| 403 | underlay.check_call(cmd="shutdown +1", node_name=node) |
| 404 | for node in node_names: |
| 405 | self.wait_for_node_state(node, state='offline', timeout=timeout) |
| 406 | |
| 407 | def warm_restart_nodes(self, underlay, nodes_prefix, timeout=600): |
| 408 | self.warm_shutdown_nodes(underlay, nodes_prefix, timeout) |
| 409 | node_names = underlay.get_target_node_names(nodes_prefix) |
| 410 | for node in node_names: |
| 411 | LOG.debug('Starting node {0}'.format(node)) |
| 412 | self.start_node(node) |
| 413 | self.wait_for_node_state(node, state='active', timeout=timeout) |
| 414 | |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 415 | def has_snapshot(self, name): |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 416 | return self.__env.has_snapshot(name) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 417 | |
| 418 | def has_snapshot_config(self, name): |
| 419 | test_config_path = self._get_snapshot_config_name(name) |
| 420 | return os.path.isfile(test_config_path) |
| 421 | |
| 422 | def delete_environment(self): |
| 423 | """Delete environment |
| 424 | |
| 425 | """ |
| 426 | LOG.debug("Deleting environment") |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 427 | self.__env.erase() |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 428 | |
| 429 | def __get_nodes_by_role(self, node_role): |
| 430 | """Get node by given role name |
| 431 | |
| 432 | :param node_role: string |
| 433 | :rtype: devops.models.Node |
| 434 | """ |
| 435 | LOG.debug('Trying to get nodes by role {0}'.format(node_role)) |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 436 | return self.__env.get_nodes(role=node_role) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 437 | |
Tatyana Leontovich | e5ccdb3 | 2017-10-09 20:10:43 +0300 | [diff] [blame] | 438 | def __get_nodes_by_name(self, node_name): |
| 439 | """Get node by given role name |
| 440 | |
| 441 | :param node_name: string |
| 442 | :rtype: devops.models.Node |
| 443 | """ |
| 444 | LOG.debug('Trying to get nodes by role {0}'.format(node_name)) |
| 445 | return self.__env.get_nodes(name=node_name) |
| 446 | |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 447 | @property |
| 448 | def master_nodes(self): |
| 449 | """Get all master nodes |
| 450 | |
| 451 | :rtype: list |
| 452 | """ |
| 453 | nodes = self.__get_nodes_by_role( |
Dennis Dmitriev | 53d3b77 | 2016-10-18 14:31:58 +0300 | [diff] [blame] | 454 | node_role=ext.UNDERLAY_NODE_ROLES.salt_master) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 455 | return nodes |
| 456 | |
| 457 | @property |
| 458 | def slave_nodes(self): |
| 459 | """Get all slave nodes |
| 460 | |
| 461 | :rtype: list |
| 462 | """ |
| 463 | nodes = self.__get_nodes_by_role( |
Dennis Dmitriev | 53d3b77 | 2016-10-18 14:31:58 +0300 | [diff] [blame] | 464 | node_role=ext.UNDERLAY_NODE_ROLES.salt_minion) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 465 | return nodes |
| 466 | |
Dennis Dmitriev | 53d3b77 | 2016-10-18 14:31:58 +0300 | [diff] [blame] | 467 | @staticmethod |
| 468 | def node_ip(node): |
| 469 | """Determine node's IP |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 470 | |
Dennis Dmitriev | 53d3b77 | 2016-10-18 14:31:58 +0300 | [diff] [blame] | 471 | :param node: devops.models.Node |
| 472 | :return: string |
| 473 | """ |
| 474 | LOG.debug('Trying to determine {0} ip.'.format(node.name)) |
| 475 | return node.get_ip_address_by_network_name( |
dis | c529838 | 2016-11-23 16:03:33 +0200 | [diff] [blame] | 476 | ext.NETWORK_TYPE.admin |
Dennis Dmitriev | 53d3b77 | 2016-10-18 14:31:58 +0300 | [diff] [blame] | 477 | ) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 478 | |
| 479 | @property |
| 480 | def nameserver(self): |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 481 | return self.__env.router(ext.NETWORK_TYPE.admin) |
Dennis Dmitriev | 6f59add | 2016-10-18 13:45:27 +0300 | [diff] [blame] | 482 | |
| 483 | def set_dns_config(self): |
| 484 | # Set local nameserver to use by default |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 485 | if not self.__config.underlay.nameservers: |
| 486 | self.__config.underlay.nameservers = [self.nameserver] |
| 487 | if not self.__config.underlay.upstream_dns_servers: |
| 488 | self.__config.underlay.upstream_dns_servers = [self.nameserver] |
Dennis Dmitriev | 99b26fe | 2017-04-26 12:34:44 +0300 | [diff] [blame] | 489 | |
| 490 | def set_address_pools_config(self): |
| 491 | """Store address pools CIDRs in config object""" |
Dennis Dmitriev | 2d60c8e | 2017-05-12 18:34:01 +0300 | [diff] [blame] | 492 | for ap in self.__env.get_address_pools(): |
Dmitry Tyzhnenko | 1fb041c | 2017-04-28 16:07:48 +0300 | [diff] [blame] | 493 | self.__config.underlay.address_pools[ap.name] = ap.net |
Dennis Dmitriev | b6bcc5c | 2018-09-26 11:07:53 +0000 | [diff] [blame] | 494 | |
| 495 | def set_dhcp_ranges_config(self): |
| 496 | """Store DHCP ranges in config object""" |
| 497 | for ap in self.__env.get_address_pools(): |
| 498 | if "gateway" in ap.ip_reserved and "dhcp" in ap.ip_ranges: |
| 499 | self.__config.underlay.dhcp_ranges[ap.name] = { |
| 500 | "cidr": ap.net, |
| 501 | "start": ap.ip_range_start("dhcp"), |
| 502 | "end": ap.ip_range_end("dhcp"), |
| 503 | "gateway": ap.gateway, |
| 504 | } |