koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 1 | import os.path |
| 2 | import socket |
| 3 | import logging |
kdanylov aka koder | b083333 | 2017-05-13 20:39:17 +0300 | [diff] [blame^] | 4 | from typing import Dict, Any, List, Tuple, cast |
| 5 | |
| 6 | from cephlib.common import to_ip |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 7 | |
| 8 | from .node_interfaces import NodeInfo |
| 9 | from .config import ConfigBlock, Config |
| 10 | from .ssh_utils import ConnCreds |
| 11 | from .openstack_api import (os_connect, find_vms, |
| 12 | OSCreds, get_openstack_credentials, prepare_os, launch_vms, clear_nodes) |
| 13 | from .test_run_class import TestRun |
| 14 | from .stage import Stage, StepOrder |
kdanylov aka koder | b083333 | 2017-05-13 20:39:17 +0300 | [diff] [blame^] | 15 | from .utils import LogError, StopTestError, get_creds_openrc |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 16 | |
| 17 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 18 | logger = logging.getLogger("wally") |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 19 | |
| 20 | |
| 21 | def get_floating_ip(vm: Any) -> str: |
| 22 | """Get VM floating IP address""" |
| 23 | |
| 24 | for net_name, ifaces in vm.addresses.items(): |
| 25 | for iface in ifaces: |
| 26 | if iface.get('OS-EXT-IPS:type') == "floating": |
| 27 | return iface['addr'] |
| 28 | |
| 29 | raise ValueError("VM {} has no floating ip".format(vm)) |
| 30 | |
| 31 | |
| 32 | def ensure_connected_to_openstack(ctx: TestRun) -> None: |
| 33 | if not ctx.os_connection is None: |
| 34 | if ctx.os_creds is None: |
| 35 | ctx.os_creds = get_OS_credentials(ctx) |
| 36 | ctx.os_connection = os_connect(ctx.os_creds) |
| 37 | |
| 38 | |
| 39 | def get_OS_credentials(ctx: TestRun) -> OSCreds: |
koder aka kdanilov | ffaf48d | 2016-12-27 02:25:29 +0200 | [diff] [blame] | 40 | stored = ctx.storage.get("openstack_openrc", None) |
| 41 | if stored is not None: |
| 42 | return OSCreds(*cast(List, stored)) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 43 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 44 | creds = None # type: OSCreds |
| 45 | os_creds = None # type: OSCreds |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 46 | force_insecure = False |
| 47 | cfg = ctx.config |
| 48 | |
| 49 | if 'openstack' in cfg.clouds: |
| 50 | os_cfg = cfg.clouds['openstack'] |
| 51 | if 'OPENRC' in os_cfg: |
| 52 | logger.info("Using OS credentials from " + os_cfg['OPENRC']) |
| 53 | creds_tuple = get_creds_openrc(os_cfg['OPENRC']) |
| 54 | os_creds = OSCreds(*creds_tuple) |
| 55 | elif 'ENV' in os_cfg: |
| 56 | logger.info("Using OS credentials from shell environment") |
| 57 | os_creds = get_openstack_credentials() |
| 58 | elif 'OS_TENANT_NAME' in os_cfg: |
| 59 | logger.info("Using predefined credentials") |
| 60 | os_creds = OSCreds(os_cfg['OS_USERNAME'].strip(), |
| 61 | os_cfg['OS_PASSWORD'].strip(), |
| 62 | os_cfg['OS_TENANT_NAME'].strip(), |
| 63 | os_cfg['OS_AUTH_URL'].strip(), |
| 64 | os_cfg.get('OS_INSECURE', False)) |
| 65 | |
| 66 | elif 'OS_INSECURE' in os_cfg: |
| 67 | force_insecure = os_cfg.get('OS_INSECURE', False) |
| 68 | |
| 69 | if os_creds is None and 'fuel' in cfg.clouds and 'openstack_env' in cfg.clouds['fuel'] and \ |
| 70 | ctx.fuel_openstack_creds is not None: |
| 71 | logger.info("Using fuel creds") |
| 72 | creds = ctx.fuel_openstack_creds |
| 73 | elif os_creds is None: |
| 74 | logger.error("Can't found OS credentials") |
| 75 | raise StopTestError("Can't found OS credentials", None) |
| 76 | |
| 77 | if creds is None: |
| 78 | creds = os_creds |
| 79 | |
| 80 | if force_insecure and not creds.insecure: |
| 81 | creds = OSCreds(creds.name, creds.passwd, creds.tenant, creds.auth_url, True) |
| 82 | |
| 83 | logger.debug(("OS_CREDS: user={0.name} tenant={0.tenant} " + |
| 84 | "auth_url={0.auth_url} insecure={0.insecure}").format(creds)) |
| 85 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 86 | ctx.storage.put(list(creds), "openstack_openrc") |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 87 | return creds |
| 88 | |
| 89 | |
| 90 | def get_vm_keypair_path(cfg: Config) -> Tuple[str, str]: |
| 91 | key_name = cfg.vm_configs['keypair_name'] |
| 92 | private_path = os.path.join(cfg.settings_dir, key_name + "_private.pem") |
| 93 | public_path = os.path.join(cfg.settings_dir, key_name + "_public.pub") |
| 94 | return (private_path, public_path) |
| 95 | |
| 96 | |
| 97 | class DiscoverOSStage(Stage): |
| 98 | """Discover openstack nodes and VMS""" |
| 99 | |
| 100 | config_block = 'openstack' |
| 101 | |
| 102 | # discover FUEL cluster first |
| 103 | priority = StepOrder.DISCOVER + 1 |
| 104 | |
| 105 | @classmethod |
| 106 | def validate(cls, conf: ConfigBlock) -> None: |
| 107 | pass |
| 108 | |
| 109 | def run(self, ctx: TestRun) -> None: |
kdanylov aka koder | cdfcdaf | 2017-04-29 10:03:39 +0300 | [diff] [blame] | 110 | if 'openstack' not in ctx.config.discover: |
kdanylov aka koder | 150b219 | 2017-04-01 16:53:01 +0300 | [diff] [blame] | 111 | logger.debug("Skip openstack discovery due to settings") |
| 112 | return |
| 113 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 114 | if 'all_nodes' in ctx.storage: |
| 115 | logger.debug("Skip openstack discovery, use previously discovered nodes") |
| 116 | return |
| 117 | |
| 118 | ensure_connected_to_openstack(ctx) |
| 119 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 120 | cfg = ctx.config.openstack |
| 121 | os_nodes_auth = cfg.auth # type: str |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 122 | if os_nodes_auth.count(":") == 2: |
| 123 | user, password, key_file = os_nodes_auth.split(":") # type: str, Optional[str], Optional[str] |
| 124 | if not password: |
| 125 | password = None |
| 126 | else: |
| 127 | user, password = os_nodes_auth.split(":") |
| 128 | key_file = None |
| 129 | |
kdanylov aka koder | cdfcdaf | 2017-04-29 10:03:39 +0300 | [diff] [blame] | 130 | if 'metadata' not in ctx.config.discover: |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 131 | services = ctx.os_connection.nova.services.list() # type: List[Any] |
| 132 | host_services_mapping = {} # type: Dict[str, List[str]] |
| 133 | |
| 134 | for service in services: |
| 135 | ip = cast(str, socket.gethostbyname(service.host)) |
| 136 | host_services_mapping.get(ip, []).append(service.binary) |
| 137 | |
| 138 | logger.debug("Found %s openstack service nodes" % len(host_services_mapping)) |
| 139 | |
| 140 | for host, services in host_services_mapping.items(): |
kdanylov aka koder | b083333 | 2017-05-13 20:39:17 +0300 | [diff] [blame^] | 141 | host_ip = to_ip(host) |
| 142 | if host != host_ip: |
| 143 | logger.info("Will use ip_addr %r instead of hostname %r", host_ip, host) |
| 144 | creds = ConnCreds(host=host_ip, user=user, passwd=password, key_file=key_file) |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 145 | ctx.merge_node(creds, set(services)) |
| 146 | # TODO: log OS nodes discovery results |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 147 | else: |
kdanylov aka koder | 150b219 | 2017-04-01 16:53:01 +0300 | [diff] [blame] | 148 | logger.info("Skip OS cluster discovery due to 'discovery' setting value") |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 149 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 150 | private_key_path = get_vm_keypair_path(ctx.config)[0] |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 151 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 152 | vm_creds = None # type: str |
| 153 | for vm_creds in cfg.get("vms", []): |
| 154 | user_name, vm_name_pattern = vm_creds.split("@", 1) |
| 155 | msg = "Vm like {} lookup failed".format(vm_name_pattern) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 156 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 157 | with LogError(msg): |
| 158 | msg = "Looking for vm with name like {0}".format(vm_name_pattern) |
| 159 | logger.debug(msg) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 160 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 161 | ensure_connected_to_openstack(ctx) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 162 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 163 | for ip, vm_id in find_vms(ctx.os_connection, vm_name_pattern): |
koder aka kdanilov | bbbe1dc | 2016-12-20 01:19:56 +0200 | [diff] [blame] | 164 | creds = ConnCreds(host=to_ip(ip), user=user_name, key_file=private_key_path) |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 165 | info = NodeInfo(creds, {'testnode'}) |
| 166 | info.os_vm_id = vm_id |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 167 | nid = info.node_id |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 168 | if nid in ctx.nodes_info: |
| 169 | logger.error("Test VM node has the same id(%s), as existing node %s", nid, ctx.nodes_info[nid]) |
| 170 | raise StopTestError() |
| 171 | ctx.nodes_info[nid] = info |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 172 | |
| 173 | |
| 174 | class CreateOSVMSStage(Stage): |
| 175 | "Spawn new VM's in Openstack cluster" |
| 176 | |
| 177 | priority = StepOrder.SPAWN # type: int |
| 178 | config_block = 'spawn_os_vms' # type: str |
| 179 | |
| 180 | def run(self, ctx: TestRun) -> None: |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 181 | if 'all_nodes' in ctx.storage: |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 182 | ctx.os_spawned_nodes_ids = ctx.storage.get('os_spawned_nodes_ids') |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 183 | logger.info("Skipping OS VMS discovery/spawn as all data found in storage") |
| 184 | return |
| 185 | |
| 186 | if 'os_spawned_nodes_ids' in ctx.storage: |
| 187 | logger.error("spawned_os_nodes_ids is found in storage, but no nodes_info is stored." + |
| 188 | "Fix this before continue") |
| 189 | raise StopTestError() |
| 190 | |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 191 | vm_spawn_config = ctx.config.spawn_os_vms |
| 192 | vm_image_config = ctx.config.vm_configs[vm_spawn_config.cfg_name] |
| 193 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 194 | ensure_connected_to_openstack(ctx) |
| 195 | params = vm_image_config.copy() |
| 196 | params.update(vm_spawn_config) |
| 197 | params.update(get_vm_keypair_path(ctx.config)) |
| 198 | params['group_name'] = ctx.config.run_uuid |
| 199 | params['keypair_name'] = ctx.config.vm_configs['keypair_name'] |
| 200 | |
| 201 | if not ctx.config.openstack.get("skip_preparation", False): |
| 202 | logger.info("Preparing openstack") |
| 203 | prepare_os(ctx.os_connection, params) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 204 | else: |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 205 | logger.info("Scip openstack preparation as 'skip_preparation' is set") |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 206 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 207 | ctx.os_spawned_nodes_ids = [] |
| 208 | with ctx.get_pool() as pool: |
| 209 | for info in launch_vms(ctx.os_connection, params, pool): |
| 210 | info.roles.add('testnode') |
koder aka kdanilov | 108ac36 | 2017-01-19 20:17:16 +0200 | [diff] [blame] | 211 | nid = info.node_id |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 212 | if nid in ctx.nodes_info: |
| 213 | logger.error("Test VM node has the same id(%s), as existing node %s", nid, ctx.nodes_info[nid]) |
| 214 | raise StopTestError() |
| 215 | ctx.nodes_info[nid] = info |
| 216 | ctx.os_spawned_nodes_ids.append(info.os_vm_id) |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 217 | |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 218 | ctx.storage.put(ctx.os_spawned_nodes_ids, 'os_spawned_nodes_ids') |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 219 | |
| 220 | def cleanup(self, ctx: TestRun) -> None: |
| 221 | # keep nodes in case of error for future test restart |
| 222 | if not ctx.config.keep_vm and ctx.os_spawned_nodes_ids: |
| 223 | logger.info("Removing nodes") |
| 224 | |
| 225 | clear_nodes(ctx.os_connection, ctx.os_spawned_nodes_ids) |
koder aka kdanilov | 7f59d56 | 2016-12-26 01:34:23 +0200 | [diff] [blame] | 226 | ctx.storage.rm('spawned_os_nodes') |
koder aka kdanilov | 39e449e | 2016-12-17 15:15:26 +0200 | [diff] [blame] | 227 | |
koder aka kdanilov | 962ee5f | 2016-12-19 02:40:08 +0200 | [diff] [blame] | 228 | logger.info("OS spawned nodes has been successfully removed") |