mitigate issues during server remove, add server remove timeout
diff --git a/wally/report.py b/wally/report.py
index 424e1fa..f27491b 100644
--- a/wally/report.py
+++ b/wally/report.py
@@ -332,7 +332,10 @@
for name, val in data.items():
if not name.startswith('__'):
if val is None:
- data[name] = '-'
+ if name in ('direct_iops_w64_max', 'direct_iops_w_max'):
+ data[name] = ('-', '-', '-')
+ else:
+ data[name] = '-'
elif isinstance(val, (int, float, long)):
data[name] = round_3_digit(val)
@@ -344,10 +347,17 @@
data['bw_write_max'][1],
data['bw_write_max'][2])
+ # templ_name = 'report_ceph_1.html'
+
+ # import pprint
+ # pprint.pprint(data)
+ # pprint.pprint(info.__dict__)
+
images.update(data)
- return get_template(templ_name).format(lab_info=lab_description,
- comment=comment,
- **images)
+ templ = get_template(templ_name)
+ return templ.format(lab_info=lab_description,
+ comment=comment,
+ **images)
def io_chart(title, concurence,
diff --git a/wally/start_vms.py b/wally/start_vms.py
index 406b24d..0c4ccc7 100644
--- a/wally/start_vms.py
+++ b/wally/start_vms.py
@@ -706,45 +706,56 @@
clear_all(NOVA_CONNECTION, nodes_ids, None)
-def clear_all(nova, ids=None, name_templ=None):
+MAX_SERVER_DELETE_TIME = 120
- def need_delete(srv):
- if name_templ is not None:
- return re.match(name_templ.format("\\d+"), srv.name) is not None
- else:
- return srv.id in ids
- volumes_to_delete = []
- cinder = cinder_connect()
- for vol in cinder.volumes.list():
- for attachment in vol.attachments:
- if attachment['server_id'] in ids:
- volumes_to_delete.append(vol)
+def clear_all(nova, ids=None, name_templ=None,
+ max_server_delete_time=MAX_SERVER_DELETE_TIME):
+ try:
+ def need_delete(srv):
+ if name_templ is not None:
+ return re.match(name_templ.format("\\d+"), srv.name) is not None
+ else:
+ return srv.id in ids
+
+ volumes_to_delete = []
+ cinder = cinder_connect()
+ for vol in cinder.volumes.list():
+ for attachment in vol.attachments:
+ if attachment['server_id'] in ids:
+ volumes_to_delete.append(vol)
+ break
+
+ deleted_srvs = set()
+ for srv in nova.servers.list():
+ if need_delete(srv):
+ logger.debug("Deleting server {0}".format(srv.name))
+ nova.servers.delete(srv)
+ deleted_srvs.add(srv.id)
+
+ count = 0
+ while count < max_server_delete_time:
+ if count % 60 == 0:
+ logger.debug("Waiting till all servers are actually deleted")
+ all_id = set(srv.id for srv in nova.servers.list())
+ if len(all_id.intersection(deleted_srvs)) == 0:
break
+ count += 1
+ time.sleep(1)
+ else:
+ logger.warning("Failed to remove servers. " +
+ "You, probably, need to remove them manually")
+ return
+ logger.debug("Done, deleting volumes")
- deleted_srvs = set()
- for srv in nova.servers.list():
- if need_delete(srv):
- logger.debug("Deleting server {0}".format(srv.name))
- nova.servers.delete(srv)
- deleted_srvs.add(srv.id)
+ # wait till vm actually deleted
- count = 0
- while True:
- if count % 60 == 0:
- logger.debug("Waiting till all servers are actually deleted")
- all_id = set(srv.id for srv in nova.servers.list())
- if len(all_id.intersection(deleted_srvs)) == 0:
- break
- count += 1
- time.sleep(1)
- logger.debug("Done, deleting volumes")
+ # logger.warning("Volume deletion commented out")
+ for vol in volumes_to_delete:
+ logger.debug("Deleting volume " + vol.display_name)
+ cinder.volumes.delete(vol)
- # wait till vm actually deleted
-
- # logger.warning("Volume deletion commented out")
- for vol in volumes_to_delete:
- logger.debug("Deleting volume " + vol.display_name)
- cinder.volumes.delete(vol)
-
- logger.debug("Clearing done (yet some volumes may still deleting)")
+ logger.debug("Clearing done (yet some volumes may still deleting)")
+ except:
+ logger.exception("During removing servers. " +
+ "You, probably, need to remove them manually")
diff --git a/wally/suits/io/fio.py b/wally/suits/io/fio.py
index a3f0fe3..7b77461 100644
--- a/wally/suits/io/fio.py
+++ b/wally/suits/io/fio.py
@@ -140,9 +140,9 @@
for key, data in res.items():
conn_ids = sorted(conn_ids_set)
- matr = [data[conn_id] for conn_id in conn_ids]
-
- mm_res[key] = MeasurementMatrix(matr, conn_ids)
+ awail_ids = [conn_id for conn_id in conn_ids if conn_id in data]
+ matr = [data[conn_id] for conn_id in awail_ids]
+ mm_res[key] = MeasurementMatrix(matr, awail_ids)
raw_res = {}
for conn_id in conn_ids: