fixes in ceph tests, add linearity plot
diff --git a/configs/ceph_perf.yaml b/configs/ceph_perf.yaml
index c8466c1..17b95d3 100644
--- a/configs/ceph_perf.yaml
+++ b/configs/ceph_perf.yaml
@@ -36,7 +36,7 @@
- start_test_nodes:
openstack:
creds: /tmp/perf_tests/personalistic_sharron/test_openrc
- count: x4
+ count: x1
cfg_name: wally_1024
network_zone_name: net04
flt_ip_pool: net04_ext
diff --git a/configs/config_ada.yaml b/configs/config_ada.yaml
new file mode 100644
index 0000000..0dbfada
--- /dev/null
+++ b/configs/config_ada.yaml
@@ -0,0 +1,60 @@
+sensors:
+ roles_mapping:
+ testnode: system-cpu, block-io
+ compute: system-cpu
+ ceph-osd: system-cpu, block-io
+
+internal:
+ var_dir_root: /tmp/perf_tests
+
+explicit_nodes:
+ # "ssh://ubuntu@216.221.229.20::/root/disk_perf_test_tool/disk_io_perf.pem": testnode
+ # "ssh://ubuntu@216.221.229.21::/root/disk_perf_test_tool/disk_io_perf.pem": testnode
+ # "ssh://ubuntu@216.221.229.18::/root/disk_perf_test_tool/disk_io_perf.pem": testnode
+ # "ssh://ubuntu@216.221.229.17::/root/disk_perf_test_tool/disk_io_perf.pem": testnode
+ # "ssh://ubuntu@216.221.229.19::/root/disk_perf_test_tool/disk_io_perf.pem": testnode
+ # "ssh://ubuntu@216.221.229.22::/root/disk_perf_test_tool/disk_io_perf.pem": testnode
+ "ssh://node-8": compute
+ "ssh://node-9": compute
+ "ssh://node-10": compute
+ "ssh://node-5": ceph-osd
+ "ssh://node-7": ceph-osd
+ "ssh://node-13": ceph-osd
+
+vm_configs:
+ wally_1024:
+ image:
+ name: wally_ubuntu
+ url: https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
+ creds: "ssh://ubuntu@{ip}::{private_key_path}"
+
+ flavor:
+ name: wally_1024
+ hdd_size: 50
+ ram_size: 1024
+ cpu_count: 1
+
+ vol_sz: 30
+ keypair_name: wally_vm_key
+ name_templ: wally-{group}-{id}
+ aa_group_name: wally-aa-{0}
+ security_group: wally_ssh_to_everyone
+
+tests:
+ - start_test_nodes:
+ openstack:
+ creds: ENV
+ count: x1
+ cfg_name: wally_1024
+ network_zone_name: net04
+ flt_ip_pool: net04_ext
+
+ tests:
+ - io:
+ cfg: ceph
+ params:
+ FILENAME: /dev/vdb
+ NUM_ROUNDS: 7
+
+logging:
+ extra_logs: 1
diff --git a/wally/assumptions_check.py b/scripts/assumptions_check.py
similarity index 100%
rename from wally/assumptions_check.py
rename to scripts/assumptions_check.py
diff --git a/wally/report.py b/wally/report.py
index 2ab4664..ea8e943 100644
--- a/wally/report.py
+++ b/wally/report.py
@@ -1,4 +1,5 @@
import os
+import math
import bisect
import logging
@@ -39,58 +40,67 @@
return closure
-# def linearity_report(processed_results, path, lab_info):
-# names = {}
-# for tp1 in ('rand', 'seq'):
-# for oper in ('read', 'write'):
-# for sync in ('sync', 'direct', 'async'):
-# sq = (tp1, oper, sync)
-# name = "{0} {1} {2}".format(*sq)
-# names["".join(word[0] for word in sq)] = name
+def linearity_report(processed_results, path, lab_info):
+ names = {}
+ for tp1 in ('rand', 'seq'):
+ for oper in ('read', 'write'):
+ for sync in ('sync', 'direct', 'async'):
+ sq = (tp1, oper, sync)
+ name = "{0} {1} {2}".format(*sq)
+ names["".join(word[0] for word in sq)] = name
-# colors = ['red', 'green', 'blue', 'cyan',
-# 'magenta', 'black', 'yellow', 'burlywood']
-# markers = ['*', '^', 'x', 'o', '+', '.']
-# color = 0
-# marker = 0
+ colors = ['red', 'green', 'blue', 'cyan',
+ 'magenta', 'black', 'yellow', 'burlywood']
+ markers = ['*', '^', 'x', 'o', '+', '.']
+ color = 0
+ marker = 0
-# name_pref = 'linearity_test_'
-# plot_data = []
+ plot_data = {}
-# x = []
-# y = []
-# e = []
-# # values to make line
-# ax = []
-# ay = []
+ name_pref = 'linearity_test_rrd'
-# for res in processed_results.values():
-# if res.name.startswith(name_pref):
-# res
+ for res in processed_results.values():
+ if res.name.startswith(name_pref):
+ iotime = 1000000. / res.iops
+ iotime_max = iotime * (1 + res.dev * 3)
+ bsize = ssize_to_b(res.raw['blocksize'])
+ plot_data[bsize] = (iotime, iotime_max)
-# for sz, med, dev in sorted(filtered_data(data)):
-# iotime_ms = 1000. // med
-# iotime_max = 1000. // (med - dev * 3)
+ min_sz = min(plot_data)
+ min_iotime, _ = plot_data.pop(min_sz)
-# x.append(sz / 1024.0)
-# y.append(iotime_ms)
-# e.append(iotime_max - iotime_ms)
-# if vals is None or sz in vals:
-# ax.append(sz / 1024.0)
-# ay.append(iotime_ms)
+ x = []
+ y = []
+ e = []
-# plt.errorbar(x, y, e, linestyle='None', label=names[tp],
-# color=colors[color], ecolor="black",
-# marker=markers[marker])
-# ynew = approximate_line(ax, ay, ax, True)
-# plt.plot(ax, ynew, color=colors[color])
-# color += 1
-# marker += 1
-# plt.legend(loc=2)
-# plt.title("Linearity test by %i dots" % (len(vals)))
-# if plt:
-# linearity_report = report('linearity',
-# 'linearity_test')(linearity_report)
+ for k, (v, vmax) in sorted(plot_data.items()):
+ # y.append(math.log10(v - min_iotime))
+ # x.append(math.log10(k))
+ # e.append(y[-1] - math.log10(vmax - min_iotime))
+ y.append(v - min_iotime)
+ x.append(k)
+ e.append(y[-1] - (vmax - min_iotime))
+
+ print e
+
+ tp = 'rrd'
+ plt.errorbar(x, y, e, linestyle='None', label=names[tp],
+ color=colors[color], ecolor="black",
+ marker=markers[marker])
+ plt.yscale('log')
+ plt.xscale('log')
+ plt.show()
+
+ # ynew = approximate_line(ax, ay, ax, True)
+ # plt.plot(ax, ynew, color=colors[color])
+ # color += 1
+ # marker += 1
+ # plt.legend(loc=2)
+ # plt.title("Linearity test by %i dots" % (len(vals)))
+
+
+if plt:
+ linearity_report = report('linearity', 'linearity_test')(linearity_report)
def render_hdd_html(dest, info, lab_description):
@@ -288,14 +298,19 @@
def make_io_report(results, path, lab_url=None, creds=None):
- if lab_url is not None:
- username, password, tenant_name = parse_creds(creds)
- creds = {'username': username,
- 'password': password,
- "tenant_name": tenant_name}
- data = collect_lab_data(lab_url, creds)
- lab_info = total_lab_info(data)
- else:
+ lab_info = None
+ # if lab_url is not None:
+ # username, password, tenant_name = parse_creds(creds)
+ # creds = {'username': username,
+ # 'password': password,
+ # "tenant_name": tenant_name}
+ # try:
+ # data = collect_lab_data(lab_url, creds)
+ # lab_info = total_lab_info(data)
+ # except Exception as exc:
+ # logger.warning("Can't collect lab data: {0!s}".format(exc))
+
+ if lab_info is None:
lab_info = {
"total_disk": "None",
"total_memory": "None",
diff --git a/wally/sensors.html b/wally/sensors/webui/sensors.html
similarity index 100%
rename from wally/sensors.html
rename to wally/sensors/webui/sensors.html
diff --git a/wally/suits/io/ceph.cfg b/wally/suits/io/ceph.cfg
index 9e98dd6..c9b2f53 100644
--- a/wally/suits/io/ceph.cfg
+++ b/wally/suits/io/ceph.cfg
@@ -12,8 +12,8 @@
NUMJOBS_SHORT={% 1, 5, 10 %}
size=30G
-ramp_time=30
-runtime=60
+ramp_time=5
+runtime=30
# ---------------------------------------------------------------------
# check different thread count, sync mode. (latency, iops) = func(th_count)
diff --git a/wally/suits/io/results_loader.py b/wally/suits/io/results_loader.py
index 102fe4c..4dff186 100644
--- a/wally/suits/io/results_loader.py
+++ b/wally/suits/io/results_loader.py
@@ -34,14 +34,9 @@
assert len(results['bw']) % vm_count == 0
block_count = len(results['bw']) // vm_count
- # print
- # print name, block_count
- # print results['bw']
- # print split_and_add(results['bw'], block_count)
-
bw, bw_dev = med_dev(split_and_add(results['bw'], block_count))
- iops, iops_dev = med_dev(split_and_add(results['iops'],
- block_count))
+ iops, _ = med_dev(split_and_add(results['iops'],
+ block_count))
lat, lat_dev = med_dev(results['lat'])
dev = bw_dev / float(bw)
data[name] = PerfInfo(name, bw, iops, dev, lat, lat_dev, results,
diff --git a/wally/utils.py b/wally/utils.py
index 36601cd..8603f58 100644
--- a/wally/utils.py
+++ b/wally/utils.py
@@ -92,10 +92,10 @@
def ssize_to_b(ssize):
try:
- ssize = ssize.lower()
+ if isinstance(ssize, (int, long)):
+ return ssize
- if ssize.endswith("b"):
- ssize = ssize[:-1]
+ ssize = ssize.lower()
if ssize[-1] in SMAP:
return int(ssize[:-1]) * SMAP[ssize[-1]]
return int(ssize)
diff --git a/wally/rest_api.py b/web_app/rest_api.py
similarity index 100%
rename from wally/rest_api.py
rename to web_app/rest_api.py