postprocessing added (plot and dev)
diff --git a/scripts/postprocessing/stat.py b/scripts/postprocessing/stat.py
new file mode 100644
index 0000000..893c0fc
--- /dev/null
+++ b/scripts/postprocessing/stat.py
@@ -0,0 +1,199 @@
+import sys
+import time
+
+from copy import deepcopy
+
+import numpy
+import scipy.optimize as scp
+import matplotlib.pyplot as plt
+
+import io_py_result_processor as io_test
+
+key_pos = {'blocksize': 0, 'direct_io': 1, 'name': 2}
+actions = ['randwrite', 'randread', 'read', 'write']
+types = ['s', 'd']
+colors = ['red', 'green', 'blue', 'cyan',
+ 'magenta', 'black', 'yellow', 'burlywood']
+
+def get_key(x, no):
+ """ x = (), no = key_pos key """
+ keys = deepcopy(key_pos)
+ del keys[no]
+ key = [x[n] for n in keys.values()]
+ return tuple(key), x[key_pos[no]]
+
+
+def generate_groups(data, group_id):
+ """ select data for plot by group_id
+ data - processed_series"""
+ grouped = {}
+
+ for key, val in data.items():
+ new_key, group_val = get_key(key, group_id)
+ group = grouped.setdefault(new_key, {})
+ group[group_val] = val
+
+ return grouped
+
+
+def gen_dots(val):
+ """Generate dots from real data
+ val = dict (x:y)
+ return ox, oy lists """
+ oy = []
+ ox = []
+ for x in sorted(val.keys()):
+ ox.append(int(x[:-1]))
+ if val[x][0] != 0:
+ oy.append(1.0/val[x][0])
+ else:
+ oy.append(0)
+ return ox, oy
+
+
+def gen_line_numpy(x, y):
+ A = numpy.vstack([x, numpy.ones(len(x))]).T
+ coef = numpy.linalg.lstsq(A, y)[0]
+ funcLine = lambda tpl, x: tpl[0] * x + tpl[1]
+ print coef
+ return x, funcLine(coef, x)
+
+
+def gen_line_scipy(x, y):
+ funcLine = lambda tpl, x: tpl[0] * x + tpl[1]
+ ErrorFunc = lambda tpl, x, y: 1.0 - y/funcLine(tpl, x)
+ tplInitial = (1.0, 0.0)
+ # print x, y
+ tplFinal, success = scp.leastsq(ErrorFunc, tplInitial[:], args=(x, y),
+ diag=(1./x.mean(), 1./y.mean()))
+ if success not in range(1, 4):
+ raise ValueError("No line for this dots")
+ xx = numpy.linspace(x.min(), x.max(), 50)
+ print tplFinal
+ # print x, ErrorFunc(tplFinal, x, y)
+ return xx, funcLine(tplFinal, xx)
+
+
+def gen_app_plot(key, val, plot, color):
+ """ Plots with fake line and real dots around"""
+ ox, oy = gen_dots(val)
+ name = "_".join(str(k) for k in key)
+ if len(ox) < 2:
+ # skip single dots
+ return False
+ # create approximation
+ x = numpy.array(ox)#numpy.log(ox))
+ y = numpy.array(oy)#numpy.log(oy))
+ print x, y
+ try:
+ print name
+ x1, y1 = gen_line_scipy(x, y)
+ plot.plot(x1, y1, color=color)
+ #
+ #plot.loglog(x1, y1, color=color)
+ except ValueError:
+ # just don't draw it - it's ok
+ # we'll see no appr and bad dots
+ # not return False, because we need see dots
+ pass
+ plot.plot(x, y, '^', label=name, markersize=7, color=color)
+ #plot.loglog(x, y, '^', label=name, markersize=7, color=color)
+ return True
+
+
+def save_plot(key, val):
+ """ one plot from one dict item with value list"""
+ ox, oy = gen_dots(val)
+ name = "_".join(str(k) for k in key)
+ plt.plot(ox, oy, label=name)
+
+
+def plot_generation(fname, group_by):
+ """ plots for value group_by in imgs by actions"""
+ data = list(io_test.load_io_py_file(fname))
+ item = io_test.Data("hdr")
+ for key, vals in io_test.groupby_globally(data, io_test.key_func).items():
+ item.series[key] = [val['iops'] for val in vals]
+ io_test.process_inplace(item)
+
+ pr_data = generate_groups(item.processed_series, group_by)
+ print pr_data
+
+ #fig = plt.figure()
+ plot = plt.subplot(111)
+
+ for action in actions:
+ for tp in types:
+ color = 0
+ hasPlot = False
+ for key, val in pr_data.items():
+ if action in key and tp in key:
+ ok = gen_app_plot(key, val, plot, colors[color])
+ hasPlot = hasPlot or ok
+ color += 1
+ # use it for just connect dots
+ #save_plot(key, val)
+ if hasPlot:
+ # Shrink current axis by 10%
+ box = plot.get_position()
+ plot.set_position([box.x0, box.y0 + box.height * 0.1,
+ box.width, box.height * 0.9])
+
+ # Put a legend to the bottom
+ plot.legend(loc='lower center', bbox_to_anchor=(0.5, -0.25),
+ fancybox=True, shadow=True, ncol=4,
+ fontsize='xx-small')
+ plt.title("Plot for %s on %s" % (group_by, action))
+ plt.ylabel("time")
+ plt.xlabel(group_by)
+ plt.grid()
+ # use it if want scale plot somehow
+ # plt.axis([0.0, 5000.0, 0.0, 64.0])
+ name = "%s__%s_%s.png" % (group_by, action, tp)
+ plt.savefig(name, format='png', dpi=100)
+ plt.clf()
+ plot = plt.subplot(111)
+ color = 0
+
+
+def deviation_on_deviation(groups_list, data):
+ """ calc deviation of data all and by selection groups"""
+ total_dev = io_test.round_deviation(io_test.med_dev(data))
+ grouped_dev = [total_dev]
+ for group in groups_list:
+ beg = 0
+ end = group
+ local_dev = []
+ while end <= len(data):
+ local_dev.append(io_test.round_deviation(io_test.med_dev(data[beg:end]))[0])
+ beg += group
+ end += group
+ grouped_dev.append(io_test.round_deviation(io_test.med_dev(local_dev)))
+ return grouped_dev
+
+
+
+def deviation_generation(fname, groups_list):
+ """ Print deviation by groups for data from fname """
+ CONC_POS = key_pos['concurence']
+ int_list = [int(i) for i in groups_list]
+ data = list(io_test.load_io_py_file(fname))
+ item = io_test.Data("hdr")
+ for key, vals in io_test.groupby_globally(data, io_test.key_func).items():
+ item.series[key] = [val['iops'] * key[CONC_POS] for val in vals]
+ print deviation_on_deviation(int_list, item.series[key])
+
+
+def main(argv):
+ if argv[1] == "plot":
+ plot_generation(argv[2], argv[3])
+ elif argv[1] == "dev":
+ deviation_generation(argv[2], argv[3:])
+
+
+if __name__ == "__main__":
+ exit(main(sys.argv))
+
+
+
+