blob: 0bd2b9af8feb40017e9b4357a59f0f1f52c3351c [file] [log] [blame]
koder aka kdanilov4643fd62015-02-10 16:20:13 -08001import re
2import os
3import time
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +03004import os.path
koder aka kdanilove21d7472015-02-14 19:02:04 -08005import logging
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +03006import subprocess
koder aka kdanilov4643fd62015-02-10 16:20:13 -08007
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -08008from concurrent.futures import ThreadPoolExecutor
9
koder aka kdanilov4500a5f2015-04-17 16:55:17 +030010from novaclient.exceptions import NotFound
koder aka kdanilov4643fd62015-02-10 16:20:13 -080011from novaclient.client import Client as n_client
12from cinderclient.v1.client import Client as c_client
13
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030014import wally
15from wally.discover import Node
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030016
koder aka kdanilov4643fd62015-02-10 16:20:13 -080017
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030018logger = logging.getLogger("wally.vms")
koder aka kdanilove21d7472015-02-14 19:02:04 -080019
20
koder aka kdanilove87ae652015-04-20 02:14:35 +030021STORED_OPENSTACK_CREDS = None
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030022NOVA_CONNECTION = None
koder aka kdanilove87ae652015-04-20 02:14:35 +030023CINDER_CONNECTION = None
24
25
26def ostack_get_creds():
27 if STORED_OPENSTACK_CREDS is None:
28 env = os.environ.get
29 name = env('OS_USERNAME')
30 passwd = env('OS_PASSWORD')
31 tenant = env('OS_TENANT_NAME')
32 auth_url = env('OS_AUTH_URL')
33 return name, passwd, tenant, auth_url
34 else:
35 return STORED_OPENSTACK_CREDS
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030036
37
38def nova_connect(name=None, passwd=None, tenant=None, auth_url=None):
39 global NOVA_CONNECTION
koder aka kdanilove87ae652015-04-20 02:14:35 +030040 global STORED_OPENSTACK_CREDS
41
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030042 if NOVA_CONNECTION is None:
43 if name is None:
44 name, passwd, tenant, auth_url = ostack_get_creds()
koder aka kdanilove87ae652015-04-20 02:14:35 +030045 else:
46 STORED_OPENSTACK_CREDS = (name, passwd, tenant, auth_url)
47
koder aka kdanilov1c2b5112015-04-10 16:53:51 +030048 NOVA_CONNECTION = n_client('1.1', name, passwd, tenant, auth_url)
49 return NOVA_CONNECTION
50
51
koder aka kdanilove87ae652015-04-20 02:14:35 +030052def cinder_connect(name=None, passwd=None, tenant=None, auth_url=None):
53 global CINDER_CONNECTION
54 global STORED_OPENSTACK_CREDS
55
56 if CINDER_CONNECTION is None:
57 if name is None:
58 name, passwd, tenant, auth_url = ostack_get_creds()
59 else:
60 STORED_OPENSTACK_CREDS = (name, passwd, tenant, auth_url)
61 CINDER_CONNECTION = c_client(name, passwd, tenant, auth_url)
62 return CINDER_CONNECTION
63
64
koder aka kdanilovc368eb62015-04-28 18:22:01 +030065def prepare_os_subpr(params, name=None, passwd=None, tenant=None,
66 auth_url=None):
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +030067 if name is None:
68 name, passwd, tenant, auth_url = ostack_get_creds()
69
koder aka kdanilovc368eb62015-04-28 18:22:01 +030070 MAX_VM_PER_NODE = 8
71 serv_groups = " ".join(map(params['aa_group_name'].format,
72 range(MAX_VM_PER_NODE)))
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +030073
koder aka kdanilovc368eb62015-04-28 18:22:01 +030074 env = os.environ.copy()
75 env.update(dict(
76 OS_USERNAME=name,
77 OS_PASSWORD=passwd,
78 OS_TENANT_NAME=tenant,
79 OS_AUTH_URL=auth_url,
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +030080
koder aka kdanilovc368eb62015-04-28 18:22:01 +030081 FLAVOR_NAME=params['flavor']['name'],
82 FLAVOR_RAM=str(params['flavor']['ram_size']),
83 FLAVOR_HDD=str(params['flavor']['hdd_size']),
84 FLAVOR_CPU_COUNT=str(params['flavor']['cpu_count']),
85
86 SERV_GROUPS=serv_groups,
87 KEYPAIR_NAME=params['keypair_name'],
88
89 SECGROUP=params['security_group'],
90
91 IMAGE_NAME=params['image']['name'],
92 KEY_FILE_NAME=params['keypair_file_private'],
93 IMAGE_URL=params['image']['url'],
94 ))
95
96 spath = os.path.dirname(os.path.dirname(wally.__file__))
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +030097 spath = os.path.join(spath, 'scripts/prepare.sh')
98
koder aka kdanilovc368eb62015-04-28 18:22:01 +030099 cmd = "bash {spath} >/dev/null".format(spath=spath)
100 subprocess.check_call(cmd, shell=True, env=env)
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +0300101
koder aka kdanilov7e0f7cf2015-05-01 17:24:35 +0300102 conn = nova_connect(name, passwd, tenant, auth_url)
103 while True:
104 status = conn.images.find(name='wally_ubuntu').status
105 if status == 'ACTIVE':
106 break
107 msg = "Image {0} is still in {1} state. Waiting 10 more seconds"
108 logger.info(msg.format('wally_ubuntu', status))
109 time.sleep(10)
110
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300111
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300112def find_vms(nova, name_prefix):
113 for srv in nova.servers.list():
114 if srv.name.startswith(name_prefix):
115 for ips in srv.addresses.values():
116 for ip in ips:
117 if ip.get("OS-EXT-IPS:type", None) == 'floating':
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300118 yield ip['addr'], srv.id
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300119 break
120
121
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300122def prepare_os(nova, params):
123 allow_ssh(nova, params['security_group'])
124
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300125 MAX_VM_PER_NODE = 8
126 serv_groups = " ".join(map(params['aa_group_name'].format,
127 range(MAX_VM_PER_NODE)))
128
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300129 shed_ids = []
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300130 for shed_group in serv_groups:
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300131 shed_ids.append(get_or_create_aa_group(nova, shed_group))
132
133 create_keypair(nova,
134 params['keypair_name'],
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300135 params['keypair_name'] + ".pub",
136 params['keypair_name'] + ".pem")
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300137
138 create_image(nova, params['image']['name'],
139 params['image']['url'])
140
141 create_flavor(nova, **params['flavor'])
koder aka kdanilov4e9f3ed2015-04-14 11:26:12 +0300142
143
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300144def get_or_create_aa_group(nova, name):
145 try:
146 group = nova.server_groups.find(name=name)
147 except NotFound:
148 group = nova.server_groups.create({'name': name,
149 'policies': ['anti-affinity']})
koder aka kdanilov652cd802015-04-13 12:21:07 +0300150
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300151 return group.id
koder aka kdanilov652cd802015-04-13 12:21:07 +0300152
153
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300154def allow_ssh(nova, group_name):
155 try:
156 secgroup = nova.security_groups.find(name=group_name)
157 except NotFound:
158 secgroup = nova.security_groups.create(group_name,
159 "allow ssh/ping to node")
160
koder aka kdanilov652cd802015-04-13 12:21:07 +0300161 nova.security_group_rules.create(secgroup.id,
162 ip_protocol="tcp",
163 from_port="22",
164 to_port="22",
165 cidr="0.0.0.0/0")
166
167 nova.security_group_rules.create(secgroup.id,
168 ip_protocol="icmp",
169 from_port=-1,
170 cidr="0.0.0.0/0",
171 to_port=-1)
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300172 return secgroup.id
koder aka kdanilov652cd802015-04-13 12:21:07 +0300173
174
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300175def create_image(nova, name, url):
176 pass
177
178
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300179def create_flavor(nova, name, ram_size, hdd_size, cpu_count):
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300180 pass
181
182
183def create_keypair(nova, name, pub_key_path, priv_key_path):
184 try:
185 nova.keypairs.find(name=name)
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300186 # if file not found- delete and recreate
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300187 except NotFound:
188 if os.path.exists(pub_key_path):
189 with open(pub_key_path) as pub_key_fd:
190 return nova.keypairs.create(name, pub_key_fd.read())
191 else:
192 key = nova.keypairs.create(name)
193
194 with open(priv_key_path, "w") as priv_key_fd:
195 priv_key_fd.write(key.private_key)
196
197 with open(pub_key_path, "w") as pub_key_fd:
198 pub_key_fd.write(key.public_key)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800199
200
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800201def create_volume(size, name):
koder aka kdanilove87ae652015-04-20 02:14:35 +0300202 cinder = cinder_connect()
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800203 vol = cinder.volumes.create(size=size, display_name=name)
204 err_count = 0
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300205
koder aka kdanilove87ae652015-04-20 02:14:35 +0300206 while vol.status != 'available':
207 if vol.status == 'error':
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800208 if err_count == 3:
koder aka kdanilove21d7472015-02-14 19:02:04 -0800209 logger.critical("Fail to create volume")
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800210 raise RuntimeError("Fail to create volume")
211 else:
212 err_count += 1
213 cinder.volumes.delete(vol)
214 time.sleep(1)
215 vol = cinder.volumes.create(size=size, display_name=name)
216 continue
217 time.sleep(1)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300218 vol = cinder.volumes.get(vol.id)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800219 return vol
220
221
222def wait_for_server_active(nova, server, timeout=240):
223 t = time.time()
224 while True:
koder aka kdanilov3f356262015-02-13 08:06:14 -0800225 time.sleep(1)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800226 sstate = getattr(server, 'OS-EXT-STS:vm_state').lower()
227
228 if sstate == 'active':
229 return True
230
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800231 if sstate == 'error':
232 return False
233
234 if time.time() - t > timeout:
235 return False
236
237 server = nova.servers.get(server)
238
239
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800240class Allocate(object):
241 pass
242
243
244def get_floating_ips(nova, pool, amount):
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800245 ip_list = nova.floating_ips.list()
246
247 if pool is not None:
248 ip_list = [ip for ip in ip_list if ip.pool == pool]
249
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800250 return [ip for ip in ip_list if ip.instance_id is None][:amount]
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800251
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800252
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300253def launch_vms(params, already_has_count=0):
koder aka kdanilovda45e882015-04-06 02:24:42 +0300254 logger.debug("Starting new nodes on openstack")
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300255 count = params['count']
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300256 lst = NOVA_CONNECTION.services.list(binary='nova-compute')
257 srv_count = len([srv for srv in lst if srv.status == 'enabled'])
koder aka kdanilovda45e882015-04-06 02:24:42 +0300258
259 if isinstance(count, basestring):
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300260 if count.startswith("x"):
261 count = srv_count * int(count[1:])
262 else:
263 assert count.startswith('=')
264 count = int(count[1:]) - already_has_count
265
266 if count <= 0:
267 return
koder aka kdanilovda45e882015-04-06 02:24:42 +0300268
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300269 assert isinstance(count, (int, long))
270
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300271 srv_params = "img: {image[name]}, flavor: {flavor[name]}".format(**params)
koder aka kdanilov66839a92015-04-11 13:22:31 +0300272 msg_templ = "Will start {0} servers with next params: {1}"
koder aka kdanilovcee43342015-04-14 22:52:53 +0300273 logger.info(msg_templ.format(count, srv_params))
koder aka kdanilovda45e882015-04-06 02:24:42 +0300274
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300275 vm_params = dict(
276 img_name=params['image']['name'],
277 flavor_name=params['flavor']['name'],
278 group_name=params['group_name'],
279 keypair_name=params['keypair_name'],
280 vol_sz=params.get('vol_sz'),
281 network_zone_name=params.get("network_zone_name"),
282 flt_ip_pool=params.get('flt_ip_pool'),
283 name_templ=params.get('name_templ'),
284 scheduler_hints={"group": params['aa_group_name']},
285 security_group=params['security_group'],
286 sec_group_size=srv_count
287 )
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300288
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300289 # precache all errors before start creating vms
290 private_key_path = params['keypair_file_private']
291 creds = params['image']['creds']
292 creds.format(ip="1.1.1.1", private_key_path="/some_path/xx")
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300293
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300294 for ip, os_node in create_vms_mt(NOVA_CONNECTION, count, **vm_params):
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300295
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300296 conn_uri = creds.format(ip=ip, private_key_path=private_key_path)
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300297 yield Node(conn_uri, []), os_node.id
koder aka kdanilovda45e882015-04-06 02:24:42 +0300298
299
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300300def get_free_server_grpoups(nova, template=None):
301 for g in nova.server_groups.list():
302 if g.members == []:
303 if re.match(template, g.name):
304 yield str(g.name)
305
306
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300307def create_vms_mt(nova, amount, group_name, keypair_name, img_name,
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800308 flavor_name, vol_sz=None, network_zone_name=None,
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300309 flt_ip_pool=None, name_templ='wally-{id}',
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300310 scheduler_hints=None, security_group=None,
311 sec_group_size=None):
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800312
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800313 with ThreadPoolExecutor(max_workers=16) as executor:
koder aka kdanilov97644f92015-02-13 11:11:08 -0800314 if network_zone_name is not None:
315 network_future = executor.submit(nova.networks.find,
316 label=network_zone_name)
317 else:
318 network_future = None
319
320 fl_future = executor.submit(nova.flavors.find, name=flavor_name)
321 img_future = executor.submit(nova.images.find, name=img_name)
322
323 if flt_ip_pool is not None:
324 ips_future = executor.submit(get_floating_ips,
325 nova, flt_ip_pool, amount)
koder aka kdanilove21d7472015-02-14 19:02:04 -0800326 logger.debug("Wait for floating ip")
koder aka kdanilov97644f92015-02-13 11:11:08 -0800327 ips = ips_future.result()
328 ips += [Allocate] * (amount - len(ips))
329 else:
330 ips = [None] * amount
331
koder aka kdanilov7dec9df2015-02-15 21:35:19 -0800332 logger.debug("Getting flavor object")
koder aka kdanilov97644f92015-02-13 11:11:08 -0800333 fl = fl_future.result()
koder aka kdanilov7dec9df2015-02-15 21:35:19 -0800334 logger.debug("Getting image object")
koder aka kdanilov97644f92015-02-13 11:11:08 -0800335 img = img_future.result()
336
337 if network_future is not None:
koder aka kdanilove21d7472015-02-14 19:02:04 -0800338 logger.debug("Waiting for network results")
koder aka kdanilov97644f92015-02-13 11:11:08 -0800339 nics = [{'net-id': network_future.result().id}]
340 else:
341 nics = None
342
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300343 names = []
344 for i in range(amount):
345 names.append(name_templ.format(group=group_name, id=i))
koder aka kdanilov97644f92015-02-13 11:11:08 -0800346
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800347 futures = []
koder aka kdanilov168f6092015-04-19 02:33:38 +0300348 logger.debug("Requesting new vm's")
koder aka kdanilov6e2ae792015-03-04 18:02:24 -0800349
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300350 orig_scheduler_hints = scheduler_hints.copy()
351
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300352 MAX_SHED_GROUPS = 32
353 for start_idx in range(MAX_SHED_GROUPS):
354 pass
355
356 group_name_template = scheduler_hints['group'].format("\\d+")
357 groups = list(get_free_server_grpoups(nova, group_name_template + "$"))
358 groups.sort()
359
360 for idx, (name, flt_ip) in enumerate(zip(names, ips), 2):
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300361
362 scheduler_hints = None
363 if orig_scheduler_hints is not None and sec_group_size is not None:
364 if "group" in orig_scheduler_hints:
365 scheduler_hints = orig_scheduler_hints.copy()
koder aka kdanilovd5ed4da2015-05-07 23:33:23 +0300366 scheduler_hints['group'] = groups[idx // sec_group_size]
koder aka kdanilovc368eb62015-04-28 18:22:01 +0300367
368 if scheduler_hints is None:
369 scheduler_hints = orig_scheduler_hints.copy()
370
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800371 params = (nova, name, keypair_name, img, fl,
372 nics, vol_sz, flt_ip, scheduler_hints,
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300373 flt_ip_pool, [security_group])
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800374
375 futures.append(executor.submit(create_vm, *params))
koder aka kdanilove21d7472015-02-14 19:02:04 -0800376 res = [future.result() for future in futures]
377 logger.debug("Done spawning")
378 return res
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800379
380
381def create_vm(nova, name, keypair_name, img,
382 fl, nics, vol_sz=None,
383 flt_ip=False,
384 scheduler_hints=None,
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300385 pool=None,
386 security_groups=None):
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800387 for i in range(3):
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800388 srv = nova.servers.create(name,
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300389 flavor=fl,
390 image=img,
391 nics=nics,
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800392 key_name=keypair_name,
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300393 scheduler_hints=scheduler_hints,
394 security_groups=security_groups)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800395
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800396 if not wait_for_server_active(nova, srv):
397 msg = "Server {0} fails to start. Kill it and try again"
koder aka kdanilove21d7472015-02-14 19:02:04 -0800398 logger.debug(msg.format(srv))
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800399 nova.servers.delete(srv)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800400
koder aka kdanilovf86d7af2015-05-06 04:01:54 +0300401 try:
402 for j in range(120):
403 srv = nova.servers.get(srv.id)
404 time.sleep(1)
405 else:
406 msg = "Server {0} delete timeout".format(srv.id)
407 raise RuntimeError(msg)
408 except NotFound:
409 pass
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800410 else:
411 break
koder aka kdanilovcff7b2e2015-04-18 20:48:15 +0300412 else:
413 raise RuntimeError("Failed to start server".format(srv.id))
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800414
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800415 if vol_sz is not None:
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800416 vol = create_volume(vol_sz, name)
koder aka kdanilove87ae652015-04-20 02:14:35 +0300417 nova.volumes.create_server_volume(srv.id, vol.id, None)
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800418
419 if flt_ip is Allocate:
420 flt_ip = nova.floating_ips.create(pool)
koder aka kdanilovda45e882015-04-06 02:24:42 +0300421
koder aka kdanilov7acd6bd2015-02-12 14:28:30 -0800422 if flt_ip is not None:
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800423 srv.add_floating_ip(flt_ip)
Yulia Portnova0e64ea22015-03-20 17:27:22 +0200424
koder aka kdanilovda45e882015-04-06 02:24:42 +0300425 return flt_ip.ip, nova.servers.get(srv.id)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800426
427
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300428def clear_nodes(nodes_ids):
429 clear_all(NOVA_CONNECTION, nodes_ids, None)
gstepanov023c1e42015-04-08 15:50:19 +0300430
431
koder aka kdanilove87ae652015-04-20 02:14:35 +0300432def clear_all(nova, ids=None, name_templ=None):
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300433
434 def need_delete(srv):
435 if name_templ is not None:
436 return re.match(name_templ.format("\\d+"), srv.name) is not None
437 else:
438 return srv.id in ids
439
koder aka kdanilove87ae652015-04-20 02:14:35 +0300440 volumes_to_delete = []
441 cinder = cinder_connect()
442 for vol in cinder.volumes.list():
443 for attachment in vol.attachments:
444 if attachment['server_id'] in ids:
445 volumes_to_delete.append(vol)
446 break
447
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800448 deleted_srvs = set()
449 for srv in nova.servers.list():
koder aka kdanilov1c2b5112015-04-10 16:53:51 +0300450 if need_delete(srv):
koder aka kdanilove21d7472015-02-14 19:02:04 -0800451 logger.debug("Deleting server {0}".format(srv.name))
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800452 nova.servers.delete(srv)
453 deleted_srvs.add(srv.id)
454
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300455 count = 0
456 while True:
457 if count % 60 == 0:
458 logger.debug("Waiting till all servers are actually deleted")
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800459 all_id = set(srv.id for srv in nova.servers.list())
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300460 if len(all_id.intersection(deleted_srvs)) == 0:
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800461 break
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300462 count += 1
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800463 time.sleep(1)
koder aka kdanilov4500a5f2015-04-17 16:55:17 +0300464 logger.debug("Done, deleting volumes")
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800465
466 # wait till vm actually deleted
467
koder aka kdanilov6b1341a2015-04-21 22:44:21 +0300468 # logger.warning("Volume deletion commented out")
469 for vol in volumes_to_delete:
470 logger.debug("Deleting volume " + vol.display_name)
471 cinder.volumes.delete(vol)
koder aka kdanilov4643fd62015-02-10 16:20:13 -0800472
koder aka kdanilove21d7472015-02-14 19:02:04 -0800473 logger.debug("Clearing done (yet some volumes may still deleting)")