Merge pull request #6 from Mirantis/bt2
New cpu load option, bottlenecks
diff --git a/TODO b/TODO
index 6271e18..042c477 100644
--- a/TODO
+++ b/TODO
@@ -1,10 +1,23 @@
-Зарефакторить запуск/мониторинг/оставнов процесса по SSH
+посмотреть настройки qemu
-offline сенсоры
-dd запускать в фоне и чекать периодически
+рестарт fio при ошибке
+дамп промежуточных данных и воосстановление
+печатать fio параметры
+
+Зарефакторить запуск/мониторинг/оставнов процесса по SSH
+fio --client
+собрать новый fio под основные платформы и положить в git
+
+запуск в фоне с чеком - в отдельную ф-цию
+prefill запускать в фоне и чекать периодически
починить все подвисания во всех потоках - дампить стеки при подвисании
+
+v2 - Однопоточная версия, обработка и продолжение работы при большинстве ошибок
+
+
+
Finding bottlenecks (алена) - починить процессор
fadvise_hint=0
Изменить с репорте сенсоров все на %
diff --git a/report_templates/report_ceph.html b/report_templates/report_ceph.html
index 4ac903b..3b97b8e 100644
--- a/report_templates/report_ceph.html
+++ b/report_templates/report_ceph.html
@@ -9,6 +9,7 @@
<body>
<div class="page-header text-center">
<h2>Performance Report</h2>
+ <h3>{comment}</h3><br>
</div>
<!--
diff --git a/report_templates/report_cinder_iscsi.html b/report_templates/report_cinder_iscsi.html
new file mode 100644
index 0000000..364ea52
--- /dev/null
+++ b/report_templates/report_cinder_iscsi.html
@@ -0,0 +1,81 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Report</title>
+ <link rel="stylesheet"
+ href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+</head>
+
+<body>
+<div class="page-header text-center">
+ <h2>Performance Report</h2>
+ <h3>{comment}</h3><br>
+</div>
+<div class="container-fluid text-center">
+ <div class="row" style="margin-bottom: 40px">
+ <div class="col-md-12">
+ <center>
+ <table><tr><td>
+ <H4>Random direct performance,<br>blocks</H4>
+ <table style="width: auto;" class="table table-bordered table-striped">
+ <tr>
+ <td>Operation</td>
+ <td>IOPS</td>
+ </tr>
+ <tr>
+ <td>Read 4KiB</td>
+ <td><div align="right">{direct_iops_r_max[0]} ~ {direct_iops_r_max[1]}%</div></td>
+ </tr>
+ <tr>
+ <td>Write 64KiB</td>
+ <td><div align="right">{direct_iops_w64_max[0]} ~ {direct_iops_w64_max[1]}%</div></td>
+ </tr>
+ </table>
+ </td><td> </td><td>
+ <H4>Sequenced direct performance,<br>1MiB blocks</H4>
+ <table style="width: auto;" class="table table-bordered table-striped">
+ <tr>
+ <td>Operation</td>
+ <td>BW MiBps</td>
+ </tr>
+ <tr>
+ <td>Read</td>
+ <td><div align="right">{bw_read_max[0]} ~ {bw_read_max[1]}%</div></td>
+ </tr>
+ <tr>
+ <td>Write</td>
+ <td><div align="right">{bw_write_max[0]} ~ {bw_write_max[1]}%</div></td>
+ </tr>
+ </table>
+ </td><td> </td><td>
+ <H4>Maximal sync random write IOPS<br> for given latency, 4KiB</H4>
+ <table style="width: auto;" class="table table-bordered table-striped">
+ <tr>
+ <td>Latency ms</td>
+ <td>IOPS</td>
+ </tr>
+ <tr>
+ <td><div align="right">10</div></td>
+ <td><div align="right">{rws4k_10ms}</div></td>
+ </tr>
+ <tr>
+ <td><div align="right">30</div></td>
+ <td><div align="right">{rws4k_30ms}</div></td>
+ </tr>
+ <tr>
+ <td><div align="right">100</div></td>
+ <td><div align="right">{rws4k_100ms}</div></td>
+ </tr>
+ </table>
+ </td></tr></table>
+ </center>
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-md-6">{rand_read_4k}</div>
+ <div class="col-md-6">{rand_write_4k}</div>
+ </div>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git a/report_templates/report_hdd.html b/report_templates/report_hdd.html
index 5a540b8..43bf6c2 100644
--- a/report_templates/report_hdd.html
+++ b/report_templates/report_hdd.html
@@ -8,7 +8,8 @@
<body>
<div class="page-header text-center">
- <h2>Performance Report</h2>
+ <h2>Performance Report</h2><br>
+ <h3>{comment}</h3><br>
</div>
<div class="container-fluid text-center">
<div class="row" style="margin-bottom: 40px">
diff --git a/run.py b/run.py
new file mode 100644
index 0000000..0fe4373
--- /dev/null
+++ b/run.py
@@ -0,0 +1,4 @@
+import sys
+from wally.run_test import main
+
+exit(main(sys.argv))
diff --git a/scripts/hdd.fio b/scripts/hdd.fio
new file mode 100644
index 0000000..c682380
--- /dev/null
+++ b/scripts/hdd.fio
@@ -0,0 +1,17 @@
+[test]
+wait_for_previous=1
+group_reporting=1
+time_based=1
+buffered=0
+iodepth=1
+softrandommap=1
+filename=/var/lib/nova/instances/b9fa1a9f-7d43-4cc6-b6cc-133c2a84ab41/xxx.bin
+randrepeat=0
+size=10G
+ramp_time=5
+runtime=10
+blocksize=4k
+rw=randwrite
+sync=1
+numjobs=1
+
diff --git a/wally/config.py b/wally/config.py
index e587a8e..c760509 100644
--- a/wally/config.py
+++ b/wally/config.py
@@ -11,7 +11,7 @@
def pet_generate(x, y):
return str(uuid.uuid4())
-from pretty_yaml import dumps
+import pretty_yaml
cfg_dict = {}
@@ -50,6 +50,7 @@
saved_config_file='config.yaml',
vm_ids_fname='os_vm_ids',
html_report_file='{0}_report.html',
+ load_report_file='load_report.html',
text_report_file='report.txt',
log_file='log.txt',
sensor_storage='sensor_storage',
@@ -69,6 +70,7 @@
var_dir = cfg_dict.get('internal', {}).get('var_dir_root', '/tmp')
run_uuid = None
+
if explicit_folder is None:
for i in range(10):
run_uuid = pet_generate(2, "_")
@@ -80,18 +82,21 @@
results_dir = os.path.join(var_dir, run_uuid)
cfg_dict['run_uuid'] = run_uuid.replace('_', '-')
else:
+ if not os.path.isdir(explicit_folder):
+ ex2 = os.path.join(var_dir, explicit_folder)
+ if os.path.isdir(ex2):
+ explicit_folder = ex2
+ else:
+ raise RuntimeError("No such directory " + explicit_folder)
+
results_dir = explicit_folder
cfg_dict.update(get_test_files(results_dir))
mkdirs_if_unxists(cfg_dict['var_dir'])
if explicit_folder is not None:
- with open(cfg_dict['run_params_file']) as fd:
- cfg_dict['run_uuid'] = yaml.load(fd)['run_uuid']
+ cfg_dict.update(load_run_params(cfg_dict['run_params_file']))
run_uuid = cfg_dict['run_uuid']
- else:
- with open(cfg_dict['run_params_file'], 'w') as fd:
- fd.write(dumps({'run_uuid': cfg_dict['run_uuid']}))
mkdirs_if_unxists(cfg_dict['sensor_storage'])
@@ -106,6 +111,25 @@
mkdirs_if_unxists(cfg_dict['results'])
mkdirs_if_unxists(cfg_dict['hwinfo_directory'])
+ return results_dir
+
+
+def save_run_params():
+ params = {
+ 'comment': cfg_dict['comment'],
+ 'run_uuid': cfg_dict['run_uuid']
+ }
+ with open(cfg_dict['run_params_file'], 'w') as fd:
+ fd.write(pretty_yaml.dumps(params))
+
+
+def load_run_params(run_params_file):
+ with open(run_params_file) as fd:
+ dt = yaml.load(fd)
+
+ return dict(run_uuid=dt['run_uuid'],
+ comment=dt.get('comment'))
+
def color_me(color):
RESET_SEQ = "\033[0m"
diff --git a/wally/discover/discover.py b/wally/discover/discover.py
index 5802ac3..2f41562 100644
--- a/wally/discover/discover.py
+++ b/wally/discover/discover.py
@@ -30,6 +30,8 @@
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True):
nodes_to_run = []
clean_data = None
+ ctx.fuel_openstack_creds = None
+
for cluster in discover:
if cluster == "openstack" and not discover_nodes:
logger.warning("Skip openstack cluster discovery")
@@ -66,10 +68,14 @@
discover_nodes)
nodes, clean_data, openrc_dict = res
- ctx.fuel_openstack_creds = {'name': openrc_dict['username'],
- 'passwd': openrc_dict['password'],
- 'tenant': openrc_dict['tenant_name'],
- 'auth_url': openrc_dict['os_auth_url']}
+ if openrc_dict is None:
+ ctx.fuel_openstack_creds = None
+ else:
+ ctx.fuel_openstack_creds = {
+ 'name': openrc_dict['username'],
+ 'passwd': openrc_dict['password'],
+ 'tenant': openrc_dict['tenant_name'],
+ 'auth_url': openrc_dict['os_auth_url']}
env_name = clusters_info['fuel']['openstack_env']
env_f_name = env_name
@@ -79,11 +85,11 @@
fuel_openrc_fname = os.path.join(var_dir,
env_f_name + "_openrc")
- with open(fuel_openrc_fname, "w") as fd:
- fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
-
- msg = "Openrc for cluster {0} saves into {1}"
- logger.debug(msg.format(env_name, fuel_openrc_fname))
+ if ctx.fuel_openstack_creds is not None:
+ with open(fuel_openrc_fname, "w") as fd:
+ fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
+ msg = "Openrc for cluster {0} saves into {1}"
+ logger.debug(msg.format(env_name, fuel_openrc_fname))
nodes_to_run.extend(nodes)
elif cluster == "ceph":
diff --git a/wally/discover/fuel.py b/wally/discover/fuel.py
index 34adc07..665321e 100644
--- a/wally/discover/fuel.py
+++ b/wally/discover/fuel.py
@@ -89,9 +89,15 @@
logger.debug("Found %s fuel nodes for env %r" %
(len(nodes), fuel_data['openstack_env']))
+ if version > [6, 0]:
+ openrc = cluster.get_openrc()
+ else:
+ logger.warning("Getting openrc on fuel 6.0 is broken, skip")
+ openrc = None
+
return (nodes,
(ssh_conn, fuel_ext_iface, ips_ports),
- cluster.get_openrc())
+ openrc)
def download_master_key(conn):
diff --git a/wally/report.py b/wally/report.py
index 1b1dbf9..2aa5338 100644
--- a/wally/report.py
+++ b/wally/report.py
@@ -1,6 +1,8 @@
import os
+import csv
import bisect
import logging
+import itertools
import collections
from cStringIO import StringIO
@@ -14,7 +16,10 @@
import wally
from wally.utils import ssize2b
from wally.statistic import round_3_digit, data_property
-from wally.suits.io.fio_task_parser import get_test_sync_mode
+from wally.suits.io.fio_task_parser import (get_test_sync_mode,
+ get_test_summary,
+ parse_all_in_1,
+ abbv_name_to_full)
logger = logging.getLogger("wally.report")
@@ -24,6 +29,10 @@
def __init__(self):
self.direct_iops_r_max = 0
self.direct_iops_w_max = 0
+
+ # 64 used instead of 4k to faster feed caches
+ self.direct_iops_w64_max = 0
+
self.rws4k_10ms = 0
self.rws4k_30ms = 0
self.rws4k_100ms = 0
@@ -51,6 +60,8 @@
self.bw = None
self.iops = None
self.lat = None
+ self.lat_50 = None
+ self.lat_95 = None
self.raw_bw = []
self.raw_iops = []
@@ -80,10 +91,55 @@
return name_map
+def get_lat_perc_50_95(lat_mks):
+ curr_perc = 0
+ perc_50 = None
+ perc_95 = None
+ pkey = None
+ for key, val in sorted(lat_mks.items()):
+ if curr_perc + val >= 50 and perc_50 is None:
+ if pkey is None or val < 1.:
+ perc_50 = key
+ else:
+ perc_50 = (50. - curr_perc) / val * (key - pkey) + pkey
+
+ if curr_perc + val >= 95:
+ if pkey is None or val < 1.:
+ perc_95 = key
+ else:
+ perc_95 = (95. - curr_perc) / val * (key - pkey) + pkey
+ break
+
+ pkey = key
+ curr_perc += val
+
+ return perc_50 / 1000., perc_95 / 1000.
+
+
def process_disk_info(test_data):
+
name_map = group_by_name(test_data)
data = {}
for (name, summary), results in name_map.items():
+ lat_mks = collections.defaultdict(lambda: 0)
+ num_res = 0
+
+ for result in results:
+ num_res += len(result.raw_result['jobs'])
+ for job_info in result.raw_result['jobs']:
+ for k, v in job_info['latency_ms'].items():
+ if isinstance(k, str):
+ assert k[:2] == '>='
+ lat_mks[int(k[2:]) * 1000] += v
+ else:
+ lat_mks[k * 1000] += v
+
+ for k, v in job_info['latency_us'].items():
+ lat_mks[k] += v
+
+ for k, v in lat_mks.items():
+ lat_mks[k] = float(v) / num_res
+
testnodes_count_set = set(dt.vm_count for dt in results)
assert len(testnodes_count_set) == 1
@@ -102,6 +158,7 @@
pinfo.bw = data_property(map(sum, zip(*pinfo.raw_bw)))
pinfo.iops = data_property(map(sum, zip(*pinfo.raw_iops)))
pinfo.lat = data_property(sum(pinfo.raw_lat, []))
+ pinfo.lat_50, pinfo.lat_95 = get_lat_perc_50_95(lat_mks)
data[(p.name, summary)] = pinfo
return data
@@ -141,50 +198,94 @@
return open(templ_file, 'r').read()
+def group_by(data, func):
+ if len(data) < 2:
+ yield data
+ return
+
+ ndata = [(func(dt), dt) for dt in data]
+ ndata.sort(key=func)
+ pkey, dt = ndata[0]
+ curr_list = [dt]
+
+ for key, val in ndata[1:]:
+ if pkey != key:
+ yield curr_list
+ curr_list = [val]
+ else:
+ curr_list.append(val)
+ pkey = key
+
+ yield curr_list
+
+
@report('linearity', 'linearity_test')
-def linearity_report(processed_results, path, lab_info):
- labels_and_data = []
+def linearity_report(processed_results, lab_info, comment):
+ labels_and_data_mp = collections.defaultdict(lambda: [])
+ vls = {}
- vls = processed_results.values()[0].params.vals.copy()
- del vls['blocksize']
-
+ # plot io_time = func(bsize)
for res in processed_results.values():
if res.name.startswith('linearity_test'):
iotimes = [1000. / val for val in res.iops.raw]
- labels_and_data.append([res.p.blocksize, res.iops.raw, iotimes])
+
+ op_summ = get_test_summary(res.params)[:3]
+
+ labels_and_data_mp[op_summ].append(
+ [res.p.blocksize, res.iops.raw, iotimes])
+
cvls = res.params.vals.copy()
del cvls['blocksize']
- assert cvls == vls
+ del cvls['rw']
- labels_and_data.sort(key=lambda x: ssize2b(x[0]))
+ cvls.pop('sync', None)
+ cvls.pop('direct', None)
+ cvls.pop('buffered', None)
+
+ if op_summ not in vls:
+ vls[op_summ] = cvls
+ else:
+ assert cvls == vls[op_summ]
+
+ all_labels = None
_, ax1 = plt.subplots()
+ for name, labels_and_data in labels_and_data_mp.items():
+ labels_and_data.sort(key=lambda x: ssize2b(x[0]))
- labels, data, iotimes = zip(*labels_and_data)
- plt.boxplot(iotimes)
+ labels, _, iotimes = zip(*labels_and_data)
- if len(labels_and_data) > 2 and ssize2b(labels_and_data[-2][0]) >= 4096:
- xt = range(1, len(labels) + 1)
+ if all_labels is None:
+ all_labels = labels
+ else:
+ assert all_labels == labels
- def io_time(sz, bw, initial_lat):
- return sz / bw + initial_lat
+ plt.boxplot(iotimes)
+ if len(labels_and_data) > 2 and \
+ ssize2b(labels_and_data[-2][0]) >= 4096:
- x = numpy.array(map(ssize2b, labels))
- y = numpy.array([sum(dt) / len(dt) for dt in iotimes])
- popt, _ = scipy.optimize.curve_fit(io_time, x, y, p0=(100., 1.))
+ xt = range(1, len(labels) + 1)
- y1 = io_time(x, *popt)
- plt.plot(xt, y1, linestyle='--', label='LS linear approxomation')
+ def io_time(sz, bw, initial_lat):
+ return sz / bw + initial_lat
- for idx, (sz, _, _) in enumerate(labels_and_data):
- if ssize2b(sz) >= 4096:
- break
+ x = numpy.array(map(ssize2b, labels))
+ y = numpy.array([sum(dt) / len(dt) for dt in iotimes])
+ popt, _ = scipy.optimize.curve_fit(io_time, x, y, p0=(100., 1.))
- bw = (x[-1] - x[idx]) / (y[-1] - y[idx])
- lat = y[-1] - x[-1] / bw
- y2 = io_time(x, bw, lat)
+ y1 = io_time(x, *popt)
+ plt.plot(xt, y1, linestyle='--',
+ label=name + ' LS linear approx')
- plt.plot(xt, y2, linestyle='--',
- label='(4k & max) linear approxomation')
+ for idx, (sz, _, _) in enumerate(labels_and_data):
+ if ssize2b(sz) >= 4096:
+ break
+
+ bw = (x[-1] - x[idx]) / (y[-1] - y[idx])
+ lat = y[-1] - x[-1] / bw
+ y2 = io_time(x, bw, lat)
+ plt.plot(xt, y2, linestyle='--',
+ label=abbv_name_to_full(name) +
+ ' (4k & max) linear approx')
plt.setp(ax1, xticklabels=labels)
@@ -192,39 +293,58 @@
plt.ylabel("IO time, ms")
plt.subplots_adjust(top=0.85)
- plt.legend(bbox_to_anchor=(0.5, 1.2), loc='upper center')
+ plt.legend(bbox_to_anchor=(0.5, 1.15),
+ loc='upper center',
+ prop={'size': 10}, ncol=2)
plt.grid()
iotime_plot = get_emb_data_svg(plt)
plt.clf()
+ # plot IOPS = func(bsize)
_, ax1 = plt.subplots()
- plt.boxplot(data)
- plt.setp(ax1, xticklabels=labels)
+ for name, labels_and_data in labels_and_data_mp.items():
+ labels_and_data.sort(key=lambda x: ssize2b(x[0]))
+ _, data, _ = zip(*labels_and_data)
+ plt.boxplot(data)
+ avg = [float(sum(arr)) / len(arr) for arr in data]
+ xt = range(1, len(data) + 1)
+ plt.plot(xt, avg, linestyle='--',
+ label=abbv_name_to_full(name) + " avg")
+
+ plt.setp(ax1, xticklabels=labels)
plt.xlabel("Block size")
plt.ylabel("IOPS")
+ plt.legend(bbox_to_anchor=(0.5, 1.15),
+ loc='upper center',
+ prop={'size': 10}, ncol=2)
plt.grid()
plt.subplots_adjust(top=0.85)
iops_plot = get_emb_data_svg(plt)
- res1 = processed_results.values()[0]
+ res = set(get_test_lcheck_params(res) for res in processed_results.values())
+ ncount = list(set(res.testnodes_count for res in processed_results.values()))
+ conc = list(set(res.concurence for res in processed_results.values()))
+
+ assert len(conc) == 1
+ assert len(ncount) == 1
+
descr = {
- 'vm_count': res1.testnodes_count,
- 'concurence': res1.concurence,
- 'oper_descr': get_test_lcheck_params(res1).capitalize()
+ 'vm_count': ncount[0],
+ 'concurence': conc[0],
+ 'oper_descr': ", ".join(res).capitalize()
}
params_map = {'iotime_vs_size': iotime_plot,
'iops_vs_size': iops_plot,
'descr': descr}
- with open(path, 'w') as fd:
- fd.write(get_template('report_linearity.html').format(**params_map))
+ return get_template('report_linearity.html').format(**params_map)
@report('lat_vs_iops', 'lat_vs_iops')
-def lat_vs_iops(processed_results, path, lab_info):
+def lat_vs_iops(processed_results, lab_info, comment):
lat_iops = collections.defaultdict(lambda: [])
requsted_vs_real = collections.defaultdict(lambda: {})
@@ -271,11 +391,10 @@
'iops_vs_requested': plt_iops_vs_requested,
'oper_descr': get_test_lcheck_params(res1).capitalize()}
- with open(path, 'w') as fd:
- fd.write(get_template('report_iops_vs_lat.html').format(**params_map))
+ return get_template('report_iops_vs_lat.html').format(**params_map)
-def render_all_html(dest, info, lab_description, images, templ_name):
+def render_all_html(comment, info, lab_description, images, templ_name):
data = info.__dict__.copy()
for name, val in data.items():
if not name.startswith('__'):
@@ -290,18 +409,17 @@
data['bw_write_max'][1])
images.update(data)
- report = get_template(templ_name).format(lab_info=lab_description,
- **images)
-
- with open(dest, 'w') as fd:
- fd.write(report)
+ return get_template(templ_name).format(lab_info=lab_description,
+ comment=comment,
+ **images)
def io_chart(title, concurence,
latv, latv_min, latv_max,
iops_or_bw, iops_or_bw_err,
legend, log=False,
- boxplots=False):
+ boxplots=False,
+ latv_50=None, latv_95=None):
points = " MiBps" if legend == 'BW' else ""
lc = len(concurence)
width = 0.35
@@ -323,9 +441,14 @@
handles1, labels1 = p1.get_legend_handles_labels()
p2 = p1.twinx()
- p2.plot(xt, latv_max, label="lat max")
- p2.plot(xt, latv, label="lat avg")
- p2.plot(xt, latv_min, label="lat min")
+
+ if latv_50 is None:
+ p2.plot(xt, latv_max, label="lat max")
+ p2.plot(xt, latv, label="lat avg")
+ p2.plot(xt, latv_min, label="lat min")
+ else:
+ p2.plot(xt, latv_50, label="lat med")
+ p2.plot(xt, latv_95, label="lat 95%")
plt.xlim(0.5, lc + 0.5)
plt.xticks(xt, ["{0} * {1}".format(vm, th) for (vm, th) in concurence])
@@ -363,9 +486,14 @@
chart_data.sort(key=lambda x: x.concurence)
# if x.lat.average < max_lat]
- lat = [x.lat.average / 1000 for x in chart_data]
- lat_min = [x.lat.min / 1000 for x in chart_data]
- lat_max = [x.lat.max / 1000 for x in chart_data]
+ # lat = [x.lat.average / 1000 for x in chart_data]
+ # lat_min = [x.lat.min / 1000 for x in chart_data]
+ # lat_max = [x.lat.max / 1000 for x in chart_data]
+ lat = None
+ lat_min = None
+ lat_max = None
+ lat_50 = [x.lat_50 for x in chart_data]
+ lat_95 = [x.lat_95 for x in chart_data]
testnodes_count = x.testnodes_count
concurence = [(testnodes_count, x.concurence)
@@ -385,7 +513,7 @@
latv=lat, latv_min=lat_min, latv_max=lat_max,
iops_or_bw=data,
iops_or_bw_err=data_dev,
- legend=name)
+ legend=name, latv_50=lat_50, latv_95=lat_95)
files[fname] = fc
return files
@@ -412,27 +540,37 @@
def get_disk_info(processed_results):
di = DiskInfo()
- rws4k_iops_lat_th = []
-
di.direct_iops_w_max = find_max_where(processed_results,
'd', '4k', 'randwrite')
di.direct_iops_r_max = find_max_where(processed_results,
'd', '4k', 'randread')
- di.bw_write_max = find_max_where(processed_results,
- 'd', '16m', 'randwrite', False)
+ di.direct_iops_w64_max = find_max_where(processed_results,
+ 'd', '64k', 'randwrite')
+
+ for sz in ('16m', '64m'):
+ di.bw_write_max = find_max_where(processed_results,
+ 'd', sz, 'randwrite', False)
+ if di.bw_write_max is not None:
+ break
+
if di.bw_write_max is None:
di.bw_write_max = find_max_where(processed_results,
'd', '1m', 'write', False)
- di.bw_read_max = find_max_where(processed_results,
- 'd', '16m', 'randread', False)
+ for sz in ('16m', '64m'):
+ di.bw_read_max = find_max_where(processed_results,
+ 'd', sz, 'randread', False)
+ if di.bw_read_max is not None:
+ break
+
if di.bw_read_max is None:
di.bw_read_max = find_max_where(processed_results,
'd', '1m', 'read', False)
+ rws4k_iops_lat_th = []
for res in processed_results.values():
- if res.sync_mode == 's' and res.p.blocksize == '4k':
+ if res.sync_mode in 'xs' and res.p.blocksize == '4k':
if res.p.rw != 'randwrite':
continue
rws4k_iops_lat_th.append((res.iops.average,
@@ -473,7 +611,17 @@
return (med, conf_perc)
hdi.direct_iops_r_max = pp(di.direct_iops_r_max)
- hdi.direct_iops_w_max = pp(di.direct_iops_w_max)
+
+ if di.direct_iops_w_max is not None:
+ hdi.direct_iops_w_max = pp(di.direct_iops_w_max)
+ else:
+ hdi.direct_iops_w_max = None
+
+ if di.direct_iops_w64_max is not None:
+ hdi.direct_iops_w64_max = pp(di.direct_iops_w64_max)
+ else:
+ hdi.direct_iops_w64_max = None
+
hdi.bw_write_max = pp(di.bw_write_max)
hdi.bw_read_max = pp(di.bw_read_max)
@@ -483,33 +631,96 @@
return hdi
-@report('HDD', 'hdd_test')
-def make_hdd_report(processed_results, path, lab_info):
+@report('hdd', 'hdd')
+def make_hdd_report(processed_results, lab_info, comment):
plots = [
- ('hdd_test_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
- ('hdd_test_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS')
+ ('hdd_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+ ('hdd_rwx4k', 'rand_write_4k', 'Random write 4k sync IOPS')
]
images = make_plots(processed_results, plots)
di = get_disk_info(processed_results)
- render_all_html(path, di, lab_info, images, "report_hdd.html")
+ return render_all_html(comment, di, lab_info, images, "report_hdd.html")
-@report('Ceph', 'ceph_test')
-def make_ceph_report(processed_results, path, lab_info):
+@report('cinder_iscsi', 'cinder_iscsi')
+def make_cinder_iscsi_report(processed_results, lab_info, comment):
plots = [
- ('ceph_test_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
- ('ceph_test_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS'),
- ('ceph_test_rrd16m', 'rand_read_16m', 'Random read 16m direct MiBps'),
- ('ceph_test_rwd16m', 'rand_write_16m',
+ ('cinder_iscsi_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+ ('cinder_iscsi_rwx4k', 'rand_write_4k', 'Random write 4k sync IOPS')
+ ]
+ try:
+ images = make_plots(processed_results, plots)
+ except ValueError:
+ plots = [
+ ('cinder_iscsi_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+ ('cinder_iscsi_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS')
+ ]
+ images = make_plots(processed_results, plots)
+ di = get_disk_info(processed_results)
+ return render_all_html(comment, di, lab_info, images, "report_cinder_iscsi.html")
+
+
+@report('ceph', 'ceph')
+def make_ceph_report(processed_results, lab_info, comment):
+ plots = [
+ ('ceph_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+ ('ceph_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS'),
+ ('ceph_rrd16m', 'rand_read_16m', 'Random read 16m direct MiBps'),
+ ('ceph_rwd16m', 'rand_write_16m',
'Random write 16m direct MiBps'),
]
images = make_plots(processed_results, plots)
di = get_disk_info(processed_results)
- render_all_html(path, di, lab_info, images, "report_ceph.html")
+ return render_all_html(comment, di, lab_info, images, "report_ceph.html")
-def make_io_report(dinfo, results, path, lab_info=None):
+def make_load_report(idx, results_dir, fname):
+ dpath = os.path.join(results_dir, "io_" + str(idx))
+ files = sorted(os.listdir(dpath))
+ gf = lambda x: "_".join(x.rsplit(".", 1)[0].split('_')[:3])
+
+ for key, group in itertools.groupby(files, gf):
+ fname = os.path.join(dpath, key + ".fio")
+
+ cfgs = list(parse_all_in_1(open(fname).read(), fname))
+
+ fname = os.path.join(dpath, key + "_lat.log")
+
+ curr = []
+ arrays = []
+
+ with open(fname) as fd:
+ for offset, lat, _, _ in csv.reader(fd):
+ offset = int(offset)
+ lat = int(lat)
+ if len(curr) > 0 and curr[-1][0] > offset:
+ arrays.append(curr)
+ curr = []
+ curr.append((offset, lat))
+ arrays.append(curr)
+ conc = int(cfgs[0].vals.get('numjobs', 1))
+
+ if conc != 5:
+ continue
+
+ assert len(arrays) == len(cfgs) * conc
+
+ garrays = [[(0, 0)] for _ in range(conc)]
+
+ for offset in range(len(cfgs)):
+ for acc, new_arr in zip(garrays, arrays[offset * conc:(offset + 1) * conc]):
+ last = acc[-1][0]
+ for off, lat in new_arr:
+ acc.append((off / 1000. + last, lat / 1000.))
+
+ for cfg, arr in zip(cfgs, garrays):
+ plt.plot(*zip(*arr[1:]))
+ plt.show()
+ exit(1)
+
+
+def make_io_report(dinfo, comment, path, lab_info=None):
lab_info = {
"total_disk": "None",
"total_memory": "None",
@@ -520,6 +731,7 @@
try:
res_fields = sorted(v.name for v in dinfo.values())
+ found = False
for fields, name, func in report_funcs:
for field in fields:
pos = bisect.bisect_left(res_fields, field)
@@ -530,11 +742,24 @@
if not res_fields[pos].startswith(field):
break
else:
+ found = True
hpath = path.format(name)
- logger.debug("Generatins report " + name + " into " + hpath)
- func(dinfo, hpath, lab_info)
- break
- else:
+
+ try:
+ report = func(dinfo, lab_info, comment)
+ except:
+ logger.exception("Diring {0} report generation".format(name))
+ continue
+
+ try:
+ with open(hpath, "w") as fd:
+ fd.write(report)
+ except:
+ logger.exception("Diring saving {0} report".format(name))
+ continue
+ logger.info("Report {0} saved into {1}".format(name, hpath))
+
+ if not found:
logger.warning("No report generator found for this load")
except Exception as exc:
diff --git a/wally/run_test.py b/wally/run_test.py
index 9a3d04c..d4fb911 100755
--- a/wally/run_test.py
+++ b/wally/run_test.py
@@ -13,7 +13,15 @@
import contextlib
import collections
-import yaml
+from yaml import load as _yaml_load
+
+try:
+ from yaml import CLoader
+ yaml_load = functools.partial(_yaml_load, Loader=CLoader)
+except ImportError:
+ yaml_load = _yaml_load
+
+
import texttable
try:
@@ -30,7 +38,7 @@
from wally import utils, report, ssh_utils, start_vms
from wally.suits import IOPerfTest, PgBenchTest, MysqlTest
from wally.config import (cfg_dict, load_config, setup_loggers,
- get_test_files)
+ get_test_files, save_run_params, load_run_params)
from wally.sensors_utils import with_sensors_util, sensors_info_util
TOOL_TYPE_MAPPER = {
@@ -466,7 +474,8 @@
auth_url = os_cfg['OS_AUTH_URL'].strip()
if tenant is None and 'fuel' in cfg['clouds'] and \
- 'openstack_env' in cfg['clouds']['fuel']:
+ 'openstack_env' in cfg['clouds']['fuel'] and \
+ ctx.fuel_openstack_creds is not None:
logger.info("Using fuel creds")
creds = ctx.fuel_openstack_creds
elif tenant is None:
@@ -605,7 +614,7 @@
raw_results = cfg_dict['raw_results']
if os.path.exists(raw_results):
- cont = yaml.load(open(raw_results).read())
+ cont = yaml_load(open(raw_results).read())
else:
cont = []
@@ -640,6 +649,20 @@
print("\n" + rep + "\n")
+def test_load_report_stage(cfg, ctx):
+ load_rep_fname = cfg['load_report_file']
+ found = False
+ for idx, (tp, data) in enumerate(ctx.results.items()):
+ if 'io' == tp and data is not None:
+ if found:
+ logger.error("Making reports for more than one " +
+ "io block isn't supported! All " +
+ "report, except first are skipped")
+ continue
+ found = True
+ report.make_load_report(idx, cfg['results'], load_rep_fname)
+
+
def html_report_stage(cfg, ctx):
html_rep_fname = cfg['html_report_file']
found = False
@@ -652,7 +675,9 @@
continue
found = True
dinfo = report.process_disk_info(data)
- report.make_io_report(dinfo, data, html_rep_fname,
+ report.make_io_report(dinfo,
+ cfg.get('comment', ''),
+ html_rep_fname,
lab_info=ctx.hw_info)
@@ -662,15 +687,16 @@
logger.debug(str(node))
-def load_data_from(var_dir):
- def load_data_from_file(_, ctx):
- raw_results = os.path.join(var_dir, 'raw_results.yaml')
- ctx.results = {}
- for tp, results in yaml.load(open(raw_results).read()):
- cls = TOOL_TYPE_MAPPER[tp]
- ctx.results[tp] = map(cls.load, results)
+def load_data_from_file(var_dir, _, ctx):
+ raw_results = os.path.join(var_dir, 'raw_results.yaml')
+ ctx.results = {}
+ for tp, results in yaml_load(open(raw_results).read()):
+ cls = TOOL_TYPE_MAPPER[tp]
+ ctx.results[tp] = map(cls.load, results)
- return load_data_from_file
+
+def load_data_from(var_dir):
+ return functools.partial(load_data_from_file, var_dir)
def start_web_ui(cfg, ctx):
@@ -705,6 +731,7 @@
help="Don't run tests", default=False)
parser.add_argument("-p", '--post-process-only', metavar="VAR_DIR",
help="Only process data from previour run")
+ parser.add_argument("-x", '--xxx', action='store_true')
parser.add_argument("-k", '--keep-vm', action='store_true',
help="Don't remove test vm's", default=False)
parser.add_argument("-d", '--dont-discover-nodes', action='store_true',
@@ -715,6 +742,7 @@
parser.add_argument("--params", metavar="testname.paramname",
help="Test params", default=[])
parser.add_argument("--ls", action='store_true', default=False)
+ parser.add_argument("-c", "--comment", default="")
parser.add_argument("config_file")
return parser.parse_args(argv[1:])
@@ -727,15 +755,12 @@
return func.__name__ + " stage"
-def get_test_names(block):
- assert len(block.items()) == 1
- name, data = block.items()[0]
- if name == 'start_test_nodes':
- for in_blk in data['tests']:
- for i in get_test_names(in_blk):
- yield i
- else:
- yield name
+def get_test_names(raw_res):
+ res = set()
+ for tp, data in raw_res:
+ for block in data:
+ res.add("{0}({1})".format(tp, block.get('test_name', '-')))
+ return res
def list_results(path):
@@ -743,45 +768,51 @@
for dname in os.listdir(path):
- cfg = get_test_files(os.path.join(path, dname))
+ files_cfg = get_test_files(os.path.join(path, dname))
- if not os.path.isfile(cfg['raw_results']):
+ if not os.path.isfile(files_cfg['raw_results']):
continue
- res_mtime = time.ctime(os.path.getmtime(cfg['raw_results']))
- cfg = yaml.load(open(cfg['saved_config_file']).read())
+ mt = os.path.getmtime(files_cfg['raw_results'])
+ res_mtime = time.ctime(mt)
- test_names = []
+ raw_res = yaml_load(open(files_cfg['raw_results']).read())
+ test_names = ",".join(sorted(get_test_names(raw_res)))
- for block in cfg['tests']:
- test_names.extend(get_test_names(block))
+ params = load_run_params(files_cfg['run_params_file'])
- results.append((dname, test_names, res_mtime))
+ comm = params.get('comment')
+ results.append((mt, dname, test_names, res_mtime,
+ '-' if comm is None else comm))
- tab = texttable.Texttable(max_width=120)
+ tab = texttable.Texttable(max_width=200)
tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER)
- tab.set_cols_align(["l", "l", "l"])
- results.sort(key=lambda x: x[2])
+ tab.set_cols_align(["l", "l", "l", "l"])
+ results.sort()
- for data in results:
- dname, tests, mtime = data
- tab.add_row([dname, ', '.join(tests), mtime])
+ for data in results[::-1]:
+ tab.add_row(data[1:])
- tab.header(["Name", "Tests", "etime"])
+ tab.header(["Name", "Tests", "etime", "Comment"])
print(tab.draw())
def main(argv):
- if '--ls' in argv:
- list_results(argv[-1])
- exit(0)
-
if faulthandler is not None:
faulthandler.register(signal.SIGUSR1, all_threads=True)
opts = parse_args(argv)
- load_config(opts.config_file, opts.post_process_only)
+
+ if opts.ls:
+ list_results(opts.config_file)
+ exit(0)
+
+ data_dir = load_config(opts.config_file, opts.post_process_only)
+
+ if opts.post_process_only is None:
+ cfg_dict['comment'] = opts.comment
+ save_run_params()
if cfg_dict.get('logging', {}).get("extra_logs", False) or opts.extra_logs:
level = logging.DEBUG
@@ -796,7 +827,7 @@
if opts.post_process_only is not None:
stages = [
- load_data_from(opts.post_process_only)
+ load_data_from(data_dir)
]
else:
stages = [
@@ -823,7 +854,9 @@
console_report_stage,
]
- if not opts.no_html_report:
+ if opts.xxx:
+ report_stages.append(test_load_report_stage)
+ elif not opts.no_html_report:
report_stages.append(html_report_stage)
logger.info("All info would be stored into {0}".format(
diff --git a/wally/sensors_utils.py b/wally/sensors_utils.py
index e78bb5f..6b50311 100644
--- a/wally/sensors_utils.py
+++ b/wally/sensors_utils.py
@@ -81,5 +81,16 @@
cfg['sensors_remote_path'])
clear_old_sensors(sensors_configs)
- with sensors_info(sensors_configs, cfg['sensors_remote_path']) as res:
+ ctx = sensors_info(sensors_configs, cfg['sensors_remote_path'])
+ try:
+ res = ctx.__enter__()
yield res
+ except:
+ ctx.__exit__(None, None, None)
+ raise
+ finally:
+ try:
+ ctx.__exit__(None, None, None)
+ except:
+ logger.exception("During stop/collect sensors")
+ del res[:]
diff --git a/wally/suits/io/__init__.py b/wally/suits/io/__init__.py
index 978fa46..e548395 100644
--- a/wally/suits/io/__init__.py
+++ b/wally/suits/io/__init__.py
@@ -33,7 +33,8 @@
'raw_result': self.raw_result,
'run_interval': self.run_interval,
'vm_count': self.vm_count,
- 'test_name': self.test_name
+ 'test_name': self.test_name,
+ 'files': self.files
}
@classmethod
@@ -44,7 +45,8 @@
return cls(sec, data['params'], data['results'],
data['raw_result'], data['run_interval'],
- data['vm_count'], data['test_name'])
+ data['vm_count'], data['test_name'],
+ files=data.get('files', {}))
def get_slice_parts_offset(test_slice, real_inteval):
@@ -121,8 +123,8 @@
# take largest size
files[fname] = max(files.get(fname, 0), msz)
- cmd_templ = "dd oflag=direct " + \
- "if=/dev/zero of={0} bs={1} count={2}"
+ cmd_templ = "fio --name=xxx --filename={0} --direct=1" + \
+ " --bs=4m --size={1}m --rw=write"
if self.use_sudo:
cmd_templ = "sudo " + cmd_templ
@@ -131,10 +133,16 @@
stime = time.time()
for fname, curr_sz in files.items():
- cmd = cmd_templ.format(fname, 1024 ** 2, curr_sz)
+ cmd = cmd_templ.format(fname, curr_sz)
ssize += curr_sz
self.run_over_ssh(cmd, timeout=curr_sz)
+ # if self.use_sudo:
+ # self.run_over_ssh("sudo echo 3 > /proc/sys/vm/drop_caches",
+ # timeout=5)
+ # else:
+ # logging.warning("Can't flush caches as sudo us disabled")
+
ddtime = time.time() - stime
if ddtime > 1E-3:
fill_bw = int(ssize / ddtime)
@@ -225,10 +233,24 @@
logger.info("Will run tests: " + ", ".join(msgs))
nolog = (pos != 0) or not self.is_primary
- out_err, interval = self.do_run(barrier, fio_cfg_slice, pos,
- nolog=nolog)
+
+ max_retr = 3 if self.total_nodes_count == 1 else 1
+
+ for idx in range(max_retr):
+ try:
+ out_err, interval, files = self.do_run(barrier, fio_cfg_slice, pos,
+ nolog=nolog)
+ break
+ except Exception as exc:
+ logger.exception("During fio run")
+ if idx == max_retr - 1:
+ raise StopTestError("Fio failed", exc)
+ logger.info("Sleeping 30s and retrying")
+ time.sleep(30)
try:
+ # HACK
+ out_err = "{" + out_err.split("{", 1)[1]
full_raw_res = json.loads(out_err)
res = {"bw": [], "iops": [], "lat": [],
@@ -246,27 +268,32 @@
first = fio_cfg_slice[0]
p1 = first.vals.copy()
p1.pop('ramp_time', 0)
+ p1.pop('offset', 0)
for nxt in fio_cfg_slice[1:]:
assert nxt.name == first.name
p2 = nxt.vals
p2.pop('_ramp_time', 0)
-
+ p2.pop('offset', 0)
assert p1 == p2
+ tname = os.path.basename(self.config_fname)
+ if tname.endswith('.cfg'):
+ tname = tname[:-4]
+
tres = IOTestResults(first,
self.config_params, res,
full_raw_res, interval,
- vm_count=self.total_nodes_count)
- tres.test_name = os.path.basename(self.config_fname)
- if tres.test_name.endswith('.cfg'):
- tres.test_name = tres.test_name[:-4]
+ test_name=tname,
+ vm_count=self.total_nodes_count,
+ files=files)
self.on_result_cb(tres)
- except (OSError, StopTestError):
+ except StopTestError:
raise
except Exception as exc:
- msg_templ = "Error during postprocessing results: {0!s}"
- raise RuntimeError(msg_templ.format(exc))
+ msg_templ = "Error during postprocessing results"
+ logger.exception(msg_templ)
+ raise StopTestError(msg_templ.format(exc), exc)
finally:
barrier.exit()
@@ -379,20 +406,22 @@
with open(os.path.join(self.log_directory, fname), "w") as fd:
fd.write(result)
+ files = {}
+
for fname in log_files:
try:
fc = read_from_remote(sftp, fname)
except:
continue
sftp.remove(fname)
-
- loc_fname = "{0}_{1}_{2}".format(pos, fconn_id,
- fname.split('_')[-1])
+ ftype = fname.split('_')[-1].split(".")[0]
+ loc_fname = "{0}_{1}_{2}.log".format(pos, fconn_id, ftype)
+ files.setdefault(ftype, []).append(loc_fname)
loc_path = os.path.join(self.log_directory, loc_fname)
with open(loc_path, "w") as fd:
fd.write(fc)
- return result, (begin, end)
+ return result, (begin, end), files
@classmethod
def merge_results(cls, results):
diff --git a/wally/suits/io/ceph.cfg b/wally/suits/io/ceph.cfg
index 26aa65f..4a651e6 100644
--- a/wally/suits/io/ceph.cfg
+++ b/wally/suits/io/ceph.cfg
@@ -3,17 +3,14 @@
NUMJOBS={% 1, 5, 10, 15, 40 %}
NUMJOBS_SHORT={% 1, 2, 3, 10 %}
-TEST_FILE_SIZE=100G
-size={TEST_FILE_SIZE}
ramp_time=15
runtime=60
-NUM_ROUNDS=7
# ---------------------------------------------------------------------
# check different thread count, sync mode. (latency, iops) = func(th_count)
# ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
blocksize=4k
rw=randwrite
sync=1
@@ -22,7 +19,7 @@
# ---------------------------------------------------------------------
# direct write
# ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
blocksize=4k
rw=randwrite
direct=1
@@ -32,7 +29,7 @@
# check different thread count, direct read mode. (latency, iops) = func(th_count)
# also check iops for randread
# ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
blocksize=4k
rw=randread
direct=1
@@ -42,7 +39,7 @@
# this is essentially sequential write/read operations
# we can't use sequential with numjobs > 1 due to caching and block merging
# ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
blocksize=16m
rw={% randread, randwrite %}
direct=1
diff --git a/wally/suits/io/check_distribution.cfg b/wally/suits/io/check_distribution.cfg
index 9475cde..7c06813 100644
--- a/wally/suits/io/check_distribution.cfg
+++ b/wally/suits/io/check_distribution.cfg
@@ -1,13 +1,10 @@
[global]
include defaults.cfg
-NUM_ROUNDS=301
[distrubution_test_{TEST_SUMM}]
blocksize=4k
rw=randwrite
direct=1
-
+sync=1
ramp_time=5
runtime=30
-
-size=200G
diff --git a/wally/suits/io/check_linearity.cfg b/wally/suits/io/check_linearity.cfg
index 6af5fcc..42618d4 100644
--- a/wally/suits/io/check_linearity.cfg
+++ b/wally/suits/io/check_linearity.cfg
@@ -1,9 +1,7 @@
[global]
-
include defaults.cfg
-NUM_ROUNDS=7
-size={TEST_FILE_SIZE}
+direct=1
ramp_time=5
runtime=30
BLOCK_SIZES={% 512,1k,2k,4k,8k,16k,32k,128k,256k,512k,1m %}
@@ -11,10 +9,9 @@
# ---------------------------------------------------------------------
# check read and write linearity. oper_time = func(size)
# ---------------------------------------------------------------------
-# [linearity_test_{TEST_SUMM}]
-# blocksize={BLOCK_SIZES}
-# rw={% randwrite, randread %}
-# direct=1
+[linearity_test_{TEST_SUMM}]
+blocksize={BLOCK_SIZES}
+rw=randread
# ---------------------------------------------------------------------
# check sync write linearity. oper_time = func(size)
diff --git a/wally/suits/io/check_th_count.cfg b/wally/suits/io/check_th_count.cfg
index 1607634..745f189 100644
--- a/wally/suits/io/check_th_count.cfg
+++ b/wally/suits/io/check_th_count.cfg
@@ -1,18 +1,10 @@
-[defaults]
+[global]
+include defaults.cfg
-# this is critical for correct results in multy-node run
-randrepeat=0
-
-NUM_ROUNDS=7
ramp_time=5
-buffered=0
-wait_for_previous
-filename={FILENAME}
-iodepth=1
-size=10G
-time_based
runtime=30
-group_reporting
+direct=1
+
numjobs={% 1, 2, 5, 10, 15, 20, 25, 30, 35, 40 %}
# ---------------------------------------------------------------------
@@ -31,20 +23,15 @@
# 1m + read + direct
#
# ---------------------------------------------------------------------
-[concurrence_test_{TEST_SUMM} * {NUM_ROUNDS}]
-blocksize=4k
-rw={% randread %}
-direct=1
-sync=0
-
-[concurrence_test_{TEST_SUMM} * {NUM_ROUNDS}]
+[concurrence_{TEST_SUMM}]
blocksize=4k
rw=randwrite
-direct=0
+
+[concurrence_{TEST_SUMM}]
+blocksize=4k
+rw={% randread, randwrite %}
sync=1
-[concurrence_test_{TEST_SUMM} * {NUM_ROUNDS}]
+[concurrence_{TEST_SUMM}]
blocksize=1m
rw={% write, read %}
-direct=1
-sync=0
diff --git a/wally/suits/io/cinder_iscsi.cfg b/wally/suits/io/cinder_iscsi.cfg
new file mode 100644
index 0000000..4d19dd9
--- /dev/null
+++ b/wally/suits/io/cinder_iscsi.cfg
@@ -0,0 +1,56 @@
+[global]
+include defaults.cfg
+
+# NUMJOBS={% 1, 5, 10, 15, 20, 30, 40, 80 %}
+
+NUMJOBS={% 1, 3, 5, 10, 20, 40 %}
+
+direct=1
+ramp_time=5
+runtime=30
+
+# ---------------------------------------------------------------------
+# check different thread count, sync mode. (latency, iops) = func(th_count)
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=4k
+rw=randwrite
+sync=1
+numjobs={NUMJOBS}
+
+# ---------------------------------------------------------------------
+# check different thread count, direct read mode. (latency, iops) = func(th_count)
+# also check iops for randread
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=4k
+rw=randread
+numjobs={NUMJOBS}
+
+# ---------------------------------------------------------------------
+# check IOPS randwrite.
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=64k
+rw=randwrite
+ramp_time=180
+runtime=120
+
+# ---------------------------------------------------------------------
+# No reason for th count > 1 in case of sequantial operations
+# ot they became random
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=1m
+rw={% read,write %}
+offset={UNIQ_OFFSET}
+ramp_time=90
+runtime=30
+
+# [cinder_iscsi_{TEST_SUMM}]
+# blocksize=64m
+# rw={% randread,randwrite %}
+# direct=1
+# ramp_time=30
+# runtime=30
+#
diff --git a/wally/suits/io/defaults.cfg b/wally/suits/io/defaults.cfg
index 51a8145..9aff22c 100644
--- a/wally/suits/io/defaults.cfg
+++ b/wally/suits/io/defaults.cfg
@@ -1,7 +1,9 @@
buffered=0
group_reporting=1
iodepth=1
-softrandommap=1
+
+norandommap=1
+
thread=1
time_based=1
wait_for_previous=1
@@ -11,4 +13,11 @@
filename={FILENAME}
+size={TEST_FILE_SIZE}
+
+write_lat_log=fio_log
+write_iops_log=fio_log
+write_bw_log=fio_log
+log_avg_msec=500
+
diff --git a/wally/suits/io/fio_task_parser.py b/wally/suits/io/fio_task_parser.py
index 52c4bb3..aca0254 100644
--- a/wally/suits/io/fio_task_parser.py
+++ b/wally/suits/io/fio_task_parser.py
@@ -7,7 +7,7 @@
from collections import OrderedDict, namedtuple
-from wally.utils import sec_to_str
+from wally.utils import sec_to_str, ssize2b
SECTION = 0
@@ -50,20 +50,6 @@
return res
-def to_bytes(sz):
- sz = sz.lower()
- try:
- return int(sz)
- except ValueError:
- if sz[-1] == 'm':
- return (1024 ** 2) * int(sz[:-1])
- if sz[-1] == 'k':
- return 1024 * int(sz[:-1])
- if sz[-1] == 'g':
- return (1024 ** 3) * int(sz[:-1])
- raise
-
-
class ParseError(ValueError):
def __init__(self, msg, fname, lineno, line_cont=""):
ValueError.__init__(self, msg)
@@ -265,11 +251,30 @@
elif val.name in processed_vals:
val = processed_vals[val.name]
processed_vals[name] = val
+
sec = sec.copy()
sec.vals = processed_vals
return sec
+MAGIC_OFFSET = 0.1885
+
+
+def abbv_name_to_full(name):
+ assert len(name) == 3
+
+ smode = {
+ 'a': 'async',
+ 's': 'sync',
+ 'd': 'direct',
+ 'x': 'sync direct'
+ }
+ off_mode = {'s': 'sequential', 'r': 'random'}
+ oper = {'r': 'read', 'w': 'write'}
+ return smode[name[2]] + " " + \
+ off_mode[name[0]] + " " + oper[name[1]]
+
+
def finall_process(sec, counter=[0]):
sec = sec.copy()
@@ -279,6 +284,16 @@
sec.vals['unified_rw_reporting'] = '1'
+ sz = ssize2b(sec.vals['size'])
+ offset = sz * ((MAGIC_OFFSET * counter[0]) % 1.0)
+ offset = int(offset) // 1024 ** 2
+ new_vars = {'UNIQ_OFFSET': str(offset) + "m"}
+
+ for name, val in sec.vals.items():
+ if isinstance(val, Var):
+ if val.name in new_vars:
+ sec.vals[name] = new_vars[val.name]
+
params = sec.vals.copy()
params['UNIQ'] = 'UN{0}'.format(counter[0])
params['COUNTER'] = str(counter[0])
diff --git a/wally/suits/io/formatter.py b/wally/suits/io/formatter.py
index 84b0a13..59691b2 100644
--- a/wally/suits/io/formatter.py
+++ b/wally/suits/io/formatter.py
@@ -2,22 +2,27 @@
from wally.utils import ssize2b
from wally.statistic import round_3_digit
-from .fio_task_parser import get_test_summary, get_test_sync_mode
+from .fio_task_parser import get_test_sync_mode
+
+
+def getconc(data):
+ th_count = data.params.vals.get('numjobs')
+
+ if th_count is None:
+ th_count = data.params.vals.get('concurence', 1)
+ return th_count
def key_func(data):
p = data.params.vals
- th_count = data.params.vals.get('numjobs')
+ th_count = getconc(data)
- if th_count is None:
- th_count = data.params.vals.get('concurence', 1)
-
- return (p['rw'],
+ return (data.name.rsplit("_", 1)[0],
+ p['rw'],
get_test_sync_mode(data.params),
ssize2b(p['blocksize']),
- int(th_count) * data.testnodes_count,
- data.name)
+ int(th_count) * data.testnodes_count)
def format_results_for_console(dinfo):
@@ -36,8 +41,7 @@
"Cnf\n95%", "Dev%", "iops\nper vm", "KiBps\nper vm", "lat\nms"]
for data in items:
-
- curr_k = key_func(data)[:3]
+ curr_k = key_func(data)[:4]
if prev_k is not None:
if prev_k != curr_k:
diff --git a/wally/suits/io/hdd.cfg b/wally/suits/io/hdd.cfg
index 0eb85a6..6d3107a 100644
--- a/wally/suits/io/hdd.cfg
+++ b/wally/suits/io/hdd.cfg
@@ -5,18 +5,14 @@
NUMJOBS={% 1, 3, 5, 10, 20, 40 %}
-write_lat_log=fio_log
-write_iops_log=fio_log
-log_avg_msec=500
-
-size={TEST_FILE_SIZE}
ramp_time=5
runtime=30
+direct=1
# ---------------------------------------------------------------------
# check different thread count, sync mode. (latency, iops) = func(th_count)
# ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
blocksize=4k
rw=randwrite
sync=1
@@ -26,25 +22,22 @@
# check different thread count, direct read mode. (latency, iops) = func(th_count)
# also check iops for randread
# ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
blocksize=4k
rw=randread
-direct=1
numjobs={NUMJOBS}
# ---------------------------------------------------------------------
# check IOPS randwrite.
# ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
blocksize=4k
rw=randwrite
-direct=1
# ---------------------------------------------------------------------
# No reason for th count > 1 in case of sequantial operations
-# They became random
+# ot they became random
# ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
blocksize=1m
rw={% read, write %}
-direct=1
diff --git a/wally/suits/io/lat_vs_iops.cfg b/wally/suits/io/lat_vs_iops.cfg
index dbafcbb..16e73e2 100644
--- a/wally/suits/io/lat_vs_iops.cfg
+++ b/wally/suits/io/lat_vs_iops.cfg
@@ -1,15 +1,13 @@
[global]
include defaults.cfg
-TEST_FILE_SIZE=100G
-size={TEST_FILE_SIZE}
-
ramp_time=5
runtime=30
blocksize=4k
rw=randwrite
sync=1
+direct=1
# ---------------------------------------------------------------------
# latency as function from IOPS
diff --git a/wally/suits/io/long_test.cfg b/wally/suits/io/long_test.cfg
index fd420d8..a304b8b 100644
--- a/wally/suits/io/long_test.cfg
+++ b/wally/suits/io/long_test.cfg
@@ -1,31 +1,22 @@
-[defaults]
-
-# this is critical for correct results in multy-node run
-randrepeat=0
+[global]
+include defaults.cfg
# 24h test
NUM_ROUNDS1=270
NUM_ROUNDS2=261
-buffered=0
-wait_for_previous
-filename={FILENAME}
-iodepth=1
-size=50G
-time_based
-runtime=300
+direct=1
+blocksize=128k
+rw=randwrite
# ---------------------------------------------------------------------
# check read and write linearity. oper_time = func(size)
# ---------------------------------------------------------------------
-[24h_test * {NUM_ROUNDS1}]
-blocksize=128k
-rw=randwrite
-direct=1
+[24h_test]
runtime=30
+NUM_ROUND={NUM_ROUNDS1}
-[24h_test * {NUM_ROUNDS2}]
-blocksize=128k
-rw=randwrite
-direct=1
+[24h_test]
+runtime=300
+NUM_ROUND={NUM_ROUNDS2}
diff --git a/wally/suits/io/rrd.cfg b/wally/suits/io/rrd.cfg
index 8eaffea..78e1f0e 100644
--- a/wally/suits/io/rrd.cfg
+++ b/wally/suits/io/rrd.cfg
@@ -5,10 +5,6 @@
ramp_time=5
runtime=40
-write_lat_log=fio_log
-write_iops_log=fio_log
-log_avg_msec=500
-
# ---------------------------------------------------------------------
[rws_{TEST_SUMM}]
blocksize=4k
diff --git a/wally/suits/io/verify.cfg b/wally/suits/io/verify.cfg
index b3162eb..92c78e5 100644
--- a/wally/suits/io/verify.cfg
+++ b/wally/suits/io/verify.cfg
@@ -5,10 +5,6 @@
ramp_time=5
runtime=10
-write_lat_log=fio_log
-write_iops_log=fio_log
-log_avg_msec=500
-
# ---------------------------------------------------------------------
[verify_{TEST_SUMM}]
blocksize=4k
diff --git a/wally/suits/io/vm_count_ec2.cfg b/wally/suits/io/vm_count_ec2.cfg
index c6fc56c..3efcf00 100644
--- a/wally/suits/io/vm_count_ec2.cfg
+++ b/wally/suits/io/vm_count_ec2.cfg
@@ -1,37 +1,26 @@
-[defaults]
-buffered=0
-wait_for_previous=1
-filename={FILENAME}
-iodepth=1
-size=10G
+[global]
+include defaults.cfg
-# this is critical for correct results in multy-node run
-randrepeat=0
-
-time_based=1
ramp_time=5
runtime=30
-group_reporting=1
BW_LIMIT=60m
IOPS_LIMIT=100
+direct=1
NUMJOBS=1
-NUM_ROUNDS=7
# ---------------------------------------------------------------------
# check different thread count. (latency, bw) = func(th_count)
# ---------------------------------------------------------------------
-[vm_count_{TEST_SUMM} * {NUM_ROUNDS}]
-blocksize=16m
+[vmcount_{TEST_SUMM}]
+blocksize=4m
rw={% randwrite, randread %}
-direct=1
numjobs={NUMJOBS}
rate={BW_LIMIT}
-[vm_count_{TEST_SUMM} * {NUM_ROUNDS}]
+[vmcount_{TEST_SUMM}]
blocksize=4k
rw={% randwrite,randread %}
-direct=1
numjobs={NUMJOBS}
rate_iops={IOPS_LIMIT}
diff --git a/wally/suits/itest.py b/wally/suits/itest.py
index 4be7eed..1cbf88c 100644
--- a/wally/suits/itest.py
+++ b/wally/suits/itest.py
@@ -1,13 +1,96 @@
import abc
import os.path
+import functools
from wally.ssh_utils import run_over_ssh, copy_paths
+def cached_prop(func):
+ @property
+ @functools.wraps(func)
+ def closure(self):
+ val = getattr(self, "_" + func.__name__)
+ if val is NoData:
+ val = func(self)
+ setattr(self, "_" + func.__name__, val)
+ return val
+ return closure
+
+
+class NoData(object):
+ pass
+
+
+class VMThData(object):
+ "store set of values for VM_COUNT * TH_COUNT"
+
+
+class IOTestResult(object):
+ def __init__(self):
+ self.run_config = None
+ self.suite_config = None
+ self.run_interval = None
+
+ self.bw = None
+ self.lat = None
+ self.iops = None
+ self.slat = None
+ self.clat = None
+
+ self.fio_section = None
+
+ self._lat_log = NoData
+ self._iops_log = NoData
+ self._bw_log = NoData
+
+ self._sensors_data = NoData
+ self._raw_resuls = NoData
+
+ def to_jsonable(self):
+ pass
+
+ @property
+ def thread_count(self):
+ pass
+
+ @property
+ def sync_mode(self):
+ pass
+
+ @property
+ def abbrev_name(self):
+ pass
+
+ @property
+ def full_name(self):
+ pass
+
+ @cached_prop
+ def lat_log(self):
+ pass
+
+ @cached_prop
+ def iops_log(self):
+ pass
+
+ @cached_prop
+ def bw_log(self):
+ pass
+
+ @cached_prop
+ def sensors_data(self):
+ pass
+
+ @cached_prop
+ def raw_resuls(self):
+ pass
+
+
class TestResults(object):
def __init__(self, config, params, results,
- raw_result, run_interval, vm_count, test_name=None):
+ raw_result, run_interval, vm_count,
+ test_name, **attrs):
self.config = config
self.params = params
self.results = results
@@ -15,6 +98,7 @@
self.run_interval = run_interval
self.vm_count = vm_count
self.test_name = test_name
+ self.__dict__.update(attrs)
def __str__(self):
res = "{0}({1}):\n results:\n".format(