koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 1 | import re |
| 2 | import os |
| 3 | import time |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 4 | import os.path |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 5 | import logging |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 6 | import subprocess |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 7 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 8 | from concurrent.futures import ThreadPoolExecutor |
| 9 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 10 | from novaclient.exceptions import NotFound |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 11 | from novaclient.client import Client as n_client |
| 12 | from cinderclient.v1.client import Client as c_client |
| 13 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 14 | import wally |
| 15 | from wally.discover import Node |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 16 | |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 17 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 18 | logger = logging.getLogger("wally.vms") |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 19 | |
| 20 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 21 | STORED_OPENSTACK_CREDS = None |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 22 | NOVA_CONNECTION = None |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 23 | CINDER_CONNECTION = None |
| 24 | |
| 25 | |
| 26 | def ostack_get_creds(): |
| 27 | if STORED_OPENSTACK_CREDS is None: |
| 28 | env = os.environ.get |
| 29 | name = env('OS_USERNAME') |
| 30 | passwd = env('OS_PASSWORD') |
| 31 | tenant = env('OS_TENANT_NAME') |
| 32 | auth_url = env('OS_AUTH_URL') |
| 33 | return name, passwd, tenant, auth_url |
| 34 | else: |
| 35 | return STORED_OPENSTACK_CREDS |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 36 | |
| 37 | |
| 38 | def nova_connect(name=None, passwd=None, tenant=None, auth_url=None): |
| 39 | global NOVA_CONNECTION |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 40 | global STORED_OPENSTACK_CREDS |
| 41 | |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 42 | if NOVA_CONNECTION is None: |
| 43 | if name is None: |
| 44 | name, passwd, tenant, auth_url = ostack_get_creds() |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 45 | else: |
| 46 | STORED_OPENSTACK_CREDS = (name, passwd, tenant, auth_url) |
| 47 | |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 48 | NOVA_CONNECTION = n_client('1.1', name, passwd, tenant, auth_url) |
| 49 | return NOVA_CONNECTION |
| 50 | |
| 51 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 52 | def cinder_connect(name=None, passwd=None, tenant=None, auth_url=None): |
| 53 | global CINDER_CONNECTION |
| 54 | global STORED_OPENSTACK_CREDS |
| 55 | |
| 56 | if CINDER_CONNECTION is None: |
| 57 | if name is None: |
| 58 | name, passwd, tenant, auth_url = ostack_get_creds() |
| 59 | else: |
| 60 | STORED_OPENSTACK_CREDS = (name, passwd, tenant, auth_url) |
| 61 | CINDER_CONNECTION = c_client(name, passwd, tenant, auth_url) |
| 62 | return CINDER_CONNECTION |
| 63 | |
| 64 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 65 | def prepare_os_subpr(params, name=None, passwd=None, tenant=None, |
| 66 | auth_url=None): |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 67 | if name is None: |
| 68 | name, passwd, tenant, auth_url = ostack_get_creds() |
| 69 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 70 | MAX_VM_PER_NODE = 8 |
| 71 | serv_groups = " ".join(map(params['aa_group_name'].format, |
| 72 | range(MAX_VM_PER_NODE))) |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 73 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 74 | env = os.environ.copy() |
| 75 | env.update(dict( |
| 76 | OS_USERNAME=name, |
| 77 | OS_PASSWORD=passwd, |
| 78 | OS_TENANT_NAME=tenant, |
| 79 | OS_AUTH_URL=auth_url, |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 80 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 81 | FLAVOR_NAME=params['flavor']['name'], |
| 82 | FLAVOR_RAM=str(params['flavor']['ram_size']), |
| 83 | FLAVOR_HDD=str(params['flavor']['hdd_size']), |
| 84 | FLAVOR_CPU_COUNT=str(params['flavor']['cpu_count']), |
| 85 | |
| 86 | SERV_GROUPS=serv_groups, |
| 87 | KEYPAIR_NAME=params['keypair_name'], |
| 88 | |
| 89 | SECGROUP=params['security_group'], |
| 90 | |
| 91 | IMAGE_NAME=params['image']['name'], |
| 92 | KEY_FILE_NAME=params['keypair_file_private'], |
| 93 | IMAGE_URL=params['image']['url'], |
| 94 | )) |
| 95 | |
| 96 | spath = os.path.dirname(os.path.dirname(wally.__file__)) |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 97 | spath = os.path.join(spath, 'scripts/prepare.sh') |
| 98 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 99 | cmd = "bash {spath} >/dev/null".format(spath=spath) |
| 100 | subprocess.check_call(cmd, shell=True, env=env) |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 101 | |
koder aka kdanilov | 7e0f7cf | 2015-05-01 17:24:35 +0300 | [diff] [blame] | 102 | conn = nova_connect(name, passwd, tenant, auth_url) |
| 103 | while True: |
| 104 | status = conn.images.find(name='wally_ubuntu').status |
| 105 | if status == 'ACTIVE': |
| 106 | break |
| 107 | msg = "Image {0} is still in {1} state. Waiting 10 more seconds" |
| 108 | logger.info(msg.format('wally_ubuntu', status)) |
| 109 | time.sleep(10) |
| 110 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 111 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 112 | def find_vms(nova, name_prefix): |
| 113 | for srv in nova.servers.list(): |
| 114 | if srv.name.startswith(name_prefix): |
| 115 | for ips in srv.addresses.values(): |
| 116 | for ip in ips: |
| 117 | if ip.get("OS-EXT-IPS:type", None) == 'floating': |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 118 | yield ip['addr'], srv.id |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 119 | break |
| 120 | |
| 121 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 122 | def prepare_os(nova, params): |
| 123 | allow_ssh(nova, params['security_group']) |
| 124 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 125 | MAX_VM_PER_NODE = 8 |
| 126 | serv_groups = " ".join(map(params['aa_group_name'].format, |
| 127 | range(MAX_VM_PER_NODE))) |
| 128 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 129 | shed_ids = [] |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 130 | for shed_group in serv_groups: |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 131 | shed_ids.append(get_or_create_aa_group(nova, shed_group)) |
| 132 | |
| 133 | create_keypair(nova, |
| 134 | params['keypair_name'], |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 135 | params['keypair_name'] + ".pub", |
| 136 | params['keypair_name'] + ".pem") |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 137 | |
| 138 | create_image(nova, params['image']['name'], |
| 139 | params['image']['url']) |
| 140 | |
| 141 | create_flavor(nova, **params['flavor']) |
koder aka kdanilov | 4e9f3ed | 2015-04-14 11:26:12 +0300 | [diff] [blame] | 142 | |
| 143 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 144 | def get_or_create_aa_group(nova, name): |
| 145 | try: |
| 146 | group = nova.server_groups.find(name=name) |
| 147 | except NotFound: |
| 148 | group = nova.server_groups.create({'name': name, |
| 149 | 'policies': ['anti-affinity']}) |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 150 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 151 | return group.id |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 152 | |
| 153 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 154 | def allow_ssh(nova, group_name): |
| 155 | try: |
| 156 | secgroup = nova.security_groups.find(name=group_name) |
| 157 | except NotFound: |
| 158 | secgroup = nova.security_groups.create(group_name, |
| 159 | "allow ssh/ping to node") |
| 160 | |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 161 | nova.security_group_rules.create(secgroup.id, |
| 162 | ip_protocol="tcp", |
| 163 | from_port="22", |
| 164 | to_port="22", |
| 165 | cidr="0.0.0.0/0") |
| 166 | |
| 167 | nova.security_group_rules.create(secgroup.id, |
| 168 | ip_protocol="icmp", |
| 169 | from_port=-1, |
| 170 | cidr="0.0.0.0/0", |
| 171 | to_port=-1) |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 172 | return secgroup.id |
koder aka kdanilov | 652cd80 | 2015-04-13 12:21:07 +0300 | [diff] [blame] | 173 | |
| 174 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 175 | def create_image(nova, name, url): |
| 176 | pass |
| 177 | |
| 178 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 179 | def create_flavor(nova, name, ram_size, hdd_size, cpu_count): |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 180 | pass |
| 181 | |
| 182 | |
| 183 | def create_keypair(nova, name, pub_key_path, priv_key_path): |
| 184 | try: |
| 185 | nova.keypairs.find(name=name) |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 186 | # if file not found- delete and recreate |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 187 | except NotFound: |
| 188 | if os.path.exists(pub_key_path): |
| 189 | with open(pub_key_path) as pub_key_fd: |
| 190 | return nova.keypairs.create(name, pub_key_fd.read()) |
| 191 | else: |
| 192 | key = nova.keypairs.create(name) |
| 193 | |
| 194 | with open(priv_key_path, "w") as priv_key_fd: |
| 195 | priv_key_fd.write(key.private_key) |
| 196 | |
| 197 | with open(pub_key_path, "w") as pub_key_fd: |
| 198 | pub_key_fd.write(key.public_key) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 199 | |
| 200 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 201 | def create_volume(size, name): |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 202 | cinder = cinder_connect() |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 203 | vol = cinder.volumes.create(size=size, display_name=name) |
| 204 | err_count = 0 |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 205 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 206 | while vol.status != 'available': |
| 207 | if vol.status == 'error': |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 208 | if err_count == 3: |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 209 | logger.critical("Fail to create volume") |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 210 | raise RuntimeError("Fail to create volume") |
| 211 | else: |
| 212 | err_count += 1 |
| 213 | cinder.volumes.delete(vol) |
| 214 | time.sleep(1) |
| 215 | vol = cinder.volumes.create(size=size, display_name=name) |
| 216 | continue |
| 217 | time.sleep(1) |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 218 | vol = cinder.volumes.get(vol.id) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 219 | return vol |
| 220 | |
| 221 | |
| 222 | def wait_for_server_active(nova, server, timeout=240): |
| 223 | t = time.time() |
| 224 | while True: |
koder aka kdanilov | 3f35626 | 2015-02-13 08:06:14 -0800 | [diff] [blame] | 225 | time.sleep(1) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 226 | sstate = getattr(server, 'OS-EXT-STS:vm_state').lower() |
| 227 | |
| 228 | if sstate == 'active': |
| 229 | return True |
| 230 | |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 231 | if sstate == 'error': |
| 232 | return False |
| 233 | |
| 234 | if time.time() - t > timeout: |
| 235 | return False |
| 236 | |
| 237 | server = nova.servers.get(server) |
| 238 | |
| 239 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 240 | class Allocate(object): |
| 241 | pass |
| 242 | |
| 243 | |
| 244 | def get_floating_ips(nova, pool, amount): |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 245 | ip_list = nova.floating_ips.list() |
| 246 | |
| 247 | if pool is not None: |
| 248 | ip_list = [ip for ip in ip_list if ip.pool == pool] |
| 249 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 250 | return [ip for ip in ip_list if ip.instance_id is None][:amount] |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 251 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 252 | |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 253 | def launch_vms(params, already_has_count=0): |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 254 | logger.debug("Starting new nodes on openstack") |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 255 | count = params['count'] |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 256 | lst = NOVA_CONNECTION.services.list(binary='nova-compute') |
| 257 | srv_count = len([srv for srv in lst if srv.status == 'enabled']) |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 258 | |
| 259 | if isinstance(count, basestring): |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 260 | if count.startswith("x"): |
| 261 | count = srv_count * int(count[1:]) |
| 262 | else: |
| 263 | assert count.startswith('=') |
| 264 | count = int(count[1:]) - already_has_count |
| 265 | |
| 266 | if count <= 0: |
| 267 | return |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 268 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 269 | assert isinstance(count, (int, long)) |
| 270 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 271 | srv_params = "img: {image[name]}, flavor: {flavor[name]}".format(**params) |
koder aka kdanilov | 66839a9 | 2015-04-11 13:22:31 +0300 | [diff] [blame] | 272 | msg_templ = "Will start {0} servers with next params: {1}" |
koder aka kdanilov | cee4334 | 2015-04-14 22:52:53 +0300 | [diff] [blame] | 273 | logger.info(msg_templ.format(count, srv_params)) |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 274 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 275 | vm_params = dict( |
| 276 | img_name=params['image']['name'], |
| 277 | flavor_name=params['flavor']['name'], |
| 278 | group_name=params['group_name'], |
| 279 | keypair_name=params['keypair_name'], |
| 280 | vol_sz=params.get('vol_sz'), |
| 281 | network_zone_name=params.get("network_zone_name"), |
| 282 | flt_ip_pool=params.get('flt_ip_pool'), |
| 283 | name_templ=params.get('name_templ'), |
| 284 | scheduler_hints={"group": params['aa_group_name']}, |
| 285 | security_group=params['security_group'], |
| 286 | sec_group_size=srv_count |
| 287 | ) |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 288 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 289 | # precache all errors before start creating vms |
| 290 | private_key_path = params['keypair_file_private'] |
| 291 | creds = params['image']['creds'] |
| 292 | creds.format(ip="1.1.1.1", private_key_path="/some_path/xx") |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 293 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 294 | for ip, os_node in create_vms_mt(NOVA_CONNECTION, count, **vm_params): |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 295 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 296 | conn_uri = creds.format(ip=ip, private_key_path=private_key_path) |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 297 | yield Node(conn_uri, []), os_node.id |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 298 | |
| 299 | |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 300 | def get_free_server_grpoups(nova, template=None): |
| 301 | for g in nova.server_groups.list(): |
| 302 | if g.members == []: |
| 303 | if re.match(template, g.name): |
| 304 | yield str(g.name) |
| 305 | |
| 306 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 307 | def create_vms_mt(nova, amount, group_name, keypair_name, img_name, |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 308 | flavor_name, vol_sz=None, network_zone_name=None, |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 309 | flt_ip_pool=None, name_templ='wally-{id}', |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 310 | scheduler_hints=None, security_group=None, |
| 311 | sec_group_size=None): |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 312 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 313 | with ThreadPoolExecutor(max_workers=16) as executor: |
koder aka kdanilov | 97644f9 | 2015-02-13 11:11:08 -0800 | [diff] [blame] | 314 | if network_zone_name is not None: |
| 315 | network_future = executor.submit(nova.networks.find, |
| 316 | label=network_zone_name) |
| 317 | else: |
| 318 | network_future = None |
| 319 | |
| 320 | fl_future = executor.submit(nova.flavors.find, name=flavor_name) |
| 321 | img_future = executor.submit(nova.images.find, name=img_name) |
| 322 | |
| 323 | if flt_ip_pool is not None: |
| 324 | ips_future = executor.submit(get_floating_ips, |
| 325 | nova, flt_ip_pool, amount) |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 326 | logger.debug("Wait for floating ip") |
koder aka kdanilov | 97644f9 | 2015-02-13 11:11:08 -0800 | [diff] [blame] | 327 | ips = ips_future.result() |
| 328 | ips += [Allocate] * (amount - len(ips)) |
| 329 | else: |
| 330 | ips = [None] * amount |
| 331 | |
koder aka kdanilov | 7dec9df | 2015-02-15 21:35:19 -0800 | [diff] [blame] | 332 | logger.debug("Getting flavor object") |
koder aka kdanilov | 97644f9 | 2015-02-13 11:11:08 -0800 | [diff] [blame] | 333 | fl = fl_future.result() |
koder aka kdanilov | 7dec9df | 2015-02-15 21:35:19 -0800 | [diff] [blame] | 334 | logger.debug("Getting image object") |
koder aka kdanilov | 97644f9 | 2015-02-13 11:11:08 -0800 | [diff] [blame] | 335 | img = img_future.result() |
| 336 | |
| 337 | if network_future is not None: |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 338 | logger.debug("Waiting for network results") |
koder aka kdanilov | 97644f9 | 2015-02-13 11:11:08 -0800 | [diff] [blame] | 339 | nics = [{'net-id': network_future.result().id}] |
| 340 | else: |
| 341 | nics = None |
| 342 | |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 343 | names = [] |
| 344 | for i in range(amount): |
| 345 | names.append(name_templ.format(group=group_name, id=i)) |
koder aka kdanilov | 97644f9 | 2015-02-13 11:11:08 -0800 | [diff] [blame] | 346 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 347 | futures = [] |
koder aka kdanilov | 168f609 | 2015-04-19 02:33:38 +0300 | [diff] [blame] | 348 | logger.debug("Requesting new vm's") |
koder aka kdanilov | 6e2ae79 | 2015-03-04 18:02:24 -0800 | [diff] [blame] | 349 | |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 350 | orig_scheduler_hints = scheduler_hints.copy() |
| 351 | |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 352 | MAX_SHED_GROUPS = 32 |
| 353 | for start_idx in range(MAX_SHED_GROUPS): |
| 354 | pass |
| 355 | |
| 356 | group_name_template = scheduler_hints['group'].format("\\d+") |
| 357 | groups = list(get_free_server_grpoups(nova, group_name_template + "$")) |
| 358 | groups.sort() |
| 359 | |
| 360 | for idx, (name, flt_ip) in enumerate(zip(names, ips), 2): |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 361 | |
| 362 | scheduler_hints = None |
| 363 | if orig_scheduler_hints is not None and sec_group_size is not None: |
| 364 | if "group" in orig_scheduler_hints: |
| 365 | scheduler_hints = orig_scheduler_hints.copy() |
koder aka kdanilov | d5ed4da | 2015-05-07 23:33:23 +0300 | [diff] [blame^] | 366 | scheduler_hints['group'] = groups[idx // sec_group_size] |
koder aka kdanilov | c368eb6 | 2015-04-28 18:22:01 +0300 | [diff] [blame] | 367 | |
| 368 | if scheduler_hints is None: |
| 369 | scheduler_hints = orig_scheduler_hints.copy() |
| 370 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 371 | params = (nova, name, keypair_name, img, fl, |
| 372 | nics, vol_sz, flt_ip, scheduler_hints, |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 373 | flt_ip_pool, [security_group]) |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 374 | |
| 375 | futures.append(executor.submit(create_vm, *params)) |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 376 | res = [future.result() for future in futures] |
| 377 | logger.debug("Done spawning") |
| 378 | return res |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 379 | |
| 380 | |
| 381 | def create_vm(nova, name, keypair_name, img, |
| 382 | fl, nics, vol_sz=None, |
| 383 | flt_ip=False, |
| 384 | scheduler_hints=None, |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 385 | pool=None, |
| 386 | security_groups=None): |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 387 | for i in range(3): |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 388 | srv = nova.servers.create(name, |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 389 | flavor=fl, |
| 390 | image=img, |
| 391 | nics=nics, |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 392 | key_name=keypair_name, |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 393 | scheduler_hints=scheduler_hints, |
| 394 | security_groups=security_groups) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 395 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 396 | if not wait_for_server_active(nova, srv): |
| 397 | msg = "Server {0} fails to start. Kill it and try again" |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 398 | logger.debug(msg.format(srv)) |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 399 | nova.servers.delete(srv) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 400 | |
koder aka kdanilov | f86d7af | 2015-05-06 04:01:54 +0300 | [diff] [blame] | 401 | try: |
| 402 | for j in range(120): |
| 403 | srv = nova.servers.get(srv.id) |
| 404 | time.sleep(1) |
| 405 | else: |
| 406 | msg = "Server {0} delete timeout".format(srv.id) |
| 407 | raise RuntimeError(msg) |
| 408 | except NotFound: |
| 409 | pass |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 410 | else: |
| 411 | break |
koder aka kdanilov | cff7b2e | 2015-04-18 20:48:15 +0300 | [diff] [blame] | 412 | else: |
| 413 | raise RuntimeError("Failed to start server".format(srv.id)) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 414 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 415 | if vol_sz is not None: |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 416 | vol = create_volume(vol_sz, name) |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 417 | nova.volumes.create_server_volume(srv.id, vol.id, None) |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 418 | |
| 419 | if flt_ip is Allocate: |
| 420 | flt_ip = nova.floating_ips.create(pool) |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 421 | |
koder aka kdanilov | 7acd6bd | 2015-02-12 14:28:30 -0800 | [diff] [blame] | 422 | if flt_ip is not None: |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 423 | srv.add_floating_ip(flt_ip) |
Yulia Portnova | 0e64ea2 | 2015-03-20 17:27:22 +0200 | [diff] [blame] | 424 | |
koder aka kdanilov | da45e88 | 2015-04-06 02:24:42 +0300 | [diff] [blame] | 425 | return flt_ip.ip, nova.servers.get(srv.id) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 426 | |
| 427 | |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 428 | def clear_nodes(nodes_ids): |
| 429 | clear_all(NOVA_CONNECTION, nodes_ids, None) |
gstepanov | 023c1e4 | 2015-04-08 15:50:19 +0300 | [diff] [blame] | 430 | |
| 431 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 432 | def clear_all(nova, ids=None, name_templ=None): |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 433 | |
| 434 | def need_delete(srv): |
| 435 | if name_templ is not None: |
| 436 | return re.match(name_templ.format("\\d+"), srv.name) is not None |
| 437 | else: |
| 438 | return srv.id in ids |
| 439 | |
koder aka kdanilov | e87ae65 | 2015-04-20 02:14:35 +0300 | [diff] [blame] | 440 | volumes_to_delete = [] |
| 441 | cinder = cinder_connect() |
| 442 | for vol in cinder.volumes.list(): |
| 443 | for attachment in vol.attachments: |
| 444 | if attachment['server_id'] in ids: |
| 445 | volumes_to_delete.append(vol) |
| 446 | break |
| 447 | |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 448 | deleted_srvs = set() |
| 449 | for srv in nova.servers.list(): |
koder aka kdanilov | 1c2b511 | 2015-04-10 16:53:51 +0300 | [diff] [blame] | 450 | if need_delete(srv): |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 451 | logger.debug("Deleting server {0}".format(srv.name)) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 452 | nova.servers.delete(srv) |
| 453 | deleted_srvs.add(srv.id) |
| 454 | |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 455 | count = 0 |
| 456 | while True: |
| 457 | if count % 60 == 0: |
| 458 | logger.debug("Waiting till all servers are actually deleted") |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 459 | all_id = set(srv.id for srv in nova.servers.list()) |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 460 | if len(all_id.intersection(deleted_srvs)) == 0: |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 461 | break |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 462 | count += 1 |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 463 | time.sleep(1) |
koder aka kdanilov | 4500a5f | 2015-04-17 16:55:17 +0300 | [diff] [blame] | 464 | logger.debug("Done, deleting volumes") |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 465 | |
| 466 | # wait till vm actually deleted |
| 467 | |
koder aka kdanilov | 6b1341a | 2015-04-21 22:44:21 +0300 | [diff] [blame] | 468 | # logger.warning("Volume deletion commented out") |
| 469 | for vol in volumes_to_delete: |
| 470 | logger.debug("Deleting volume " + vol.display_name) |
| 471 | cinder.volumes.delete(vol) |
koder aka kdanilov | 4643fd6 | 2015-02-10 16:20:13 -0800 | [diff] [blame] | 472 | |
koder aka kdanilov | e21d747 | 2015-02-14 19:02:04 -0800 | [diff] [blame] | 473 | logger.debug("Clearing done (yet some volumes may still deleting)") |