diff --git a/TODO b/TODO
index 6271e18..042c477 100644
--- a/TODO
+++ b/TODO
@@ -1,10 +1,23 @@
-Зарефакторить запуск/мониторинг/оставнов процесса по SSH
+посмотреть настройки qemu
 
-offline сенсоры
-dd запускать в фоне и чекать периодически
+рестарт fio при ошибке
+дамп промежуточных данных и воосстановление
+печатать fio параметры
+
+Зарефакторить запуск/мониторинг/оставнов процесса по SSH
+fio --client
+собрать новый fio под основные платформы и положить в git
+
+запуск в фоне с чеком - в отдельную ф-цию
+prefill запускать в фоне и чекать периодически
 починить все подвисания во всех потоках - дампить стеки при подвисании
 
 
+
+v2 - Однопоточная версия, обработка и продолжение работы при большинстве ошибок
+
+
+
 Finding bottlenecks (алена) - починить процессор
 fadvise_hint=0
 Изменить с репорте сенсоров все на %
diff --git a/report_templates/report_ceph.html b/report_templates/report_ceph.html
index 4ac903b..3b97b8e 100644
--- a/report_templates/report_ceph.html
+++ b/report_templates/report_ceph.html
@@ -9,6 +9,7 @@
 <body>
 <div class="page-header text-center">
   <h2>Performance Report</h2>
+  <h3>{comment}</h3><br>
 </div>
 
 <!--
diff --git a/report_templates/report_cinder_iscsi.html b/report_templates/report_cinder_iscsi.html
new file mode 100644
index 0000000..364ea52
--- /dev/null
+++ b/report_templates/report_cinder_iscsi.html
@@ -0,0 +1,81 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Report</title>
+    <link rel="stylesheet"
+          href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+</head>
+
+<body>
+<div class="page-header text-center">
+    <h2>Performance Report</h2>
+    <h3>{comment}</h3><br>
+</div>
+<div class="container-fluid text-center">
+    <div class="row" style="margin-bottom: 40px">
+        <div class="col-md-12">
+            <center>
+            <table><tr><td>
+                <H4>Random direct performance,<br>blocks</H4>
+                <table style="width: auto;" class="table table-bordered table-striped">
+                    <tr>
+                        <td>Operation</td>
+                        <td>IOPS</td>
+                    </tr>
+                    <tr>
+                        <td>Read 4KiB</td>
+                        <td><div align="right">{direct_iops_r_max[0]} ~ {direct_iops_r_max[1]}%</div></td>
+                    </tr>
+                    <tr>
+                        <td>Write 64KiB</td>
+                        <td><div align="right">{direct_iops_w64_max[0]} ~ {direct_iops_w64_max[1]}%</div></td>
+                    </tr>
+                </table>
+            </td><td>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td><td>
+                <H4>Sequenced direct performance,<br>1MiB blocks</H4>
+                <table style="width: auto;" class="table table-bordered table-striped">
+                    <tr>
+                        <td>Operation</td>
+                        <td>BW MiBps</td>
+                    </tr>
+                    <tr>
+                        <td>Read</td>
+                        <td><div align="right">{bw_read_max[0]} ~ {bw_read_max[1]}%</div></td>
+                    </tr>
+                    <tr>
+                        <td>Write</td>
+                        <td><div align="right">{bw_write_max[0]} ~ {bw_write_max[1]}%</div></td>
+                    </tr>
+                </table>
+            </td><td>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td><td>
+                <H4>Maximal sync random write IOPS<br> for given latency, 4KiB</H4>
+                <table style="width: auto;" class="table table-bordered table-striped">
+                    <tr>
+                        <td>Latency ms</td>
+                        <td>IOPS</td>
+                    </tr>
+                    <tr>
+                        <td><div align="right">10</div></td>
+                        <td><div align="right">{rws4k_10ms}</div></td>
+                    </tr>
+                    <tr>
+                        <td><div align="right">30</div></td>
+                        <td><div align="right">{rws4k_30ms}</div></td>
+                    </tr>
+                    <tr>
+                        <td><div align="right">100</div></td>
+                        <td><div align="right">{rws4k_100ms}</div></td>
+                    </tr>
+                </table>
+            </td></tr></table>
+            </center>
+        </div>
+    </div>
+    <div class="row">
+        <div class="col-md-6">{rand_read_4k}</div>
+        <div class="col-md-6">{rand_write_4k}</div>
+    </div>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git a/report_templates/report_hdd.html b/report_templates/report_hdd.html
index 5a540b8..43bf6c2 100644
--- a/report_templates/report_hdd.html
+++ b/report_templates/report_hdd.html
@@ -8,7 +8,8 @@
 
 <body>
 <div class="page-header text-center">
-    <h2>Performance Report</h2>
+    <h2>Performance Report</h2><br>
+    <h3>{comment}</h3><br>
 </div>
 <div class="container-fluid text-center">
     <div class="row" style="margin-bottom: 40px">
diff --git a/run.py b/run.py
new file mode 100644
index 0000000..0fe4373
--- /dev/null
+++ b/run.py
@@ -0,0 +1,4 @@
+import sys
+from wally.run_test import main
+
+exit(main(sys.argv))
diff --git a/scripts/hdd.fio b/scripts/hdd.fio
new file mode 100644
index 0000000..c682380
--- /dev/null
+++ b/scripts/hdd.fio
@@ -0,0 +1,17 @@
+[test]
+wait_for_previous=1
+group_reporting=1
+time_based=1
+buffered=0
+iodepth=1
+softrandommap=1
+filename=/var/lib/nova/instances/b9fa1a9f-7d43-4cc6-b6cc-133c2a84ab41/xxx.bin
+randrepeat=0
+size=10G
+ramp_time=5
+runtime=10
+blocksize=4k
+rw=randwrite
+sync=1
+numjobs=1
+
diff --git a/wally/config.py b/wally/config.py
index e587a8e..c760509 100644
--- a/wally/config.py
+++ b/wally/config.py
@@ -11,7 +11,7 @@
     def pet_generate(x, y):
         return str(uuid.uuid4())
 
-from pretty_yaml import dumps
+import pretty_yaml
 
 cfg_dict = {}
 
@@ -50,6 +50,7 @@
         saved_config_file='config.yaml',
         vm_ids_fname='os_vm_ids',
         html_report_file='{0}_report.html',
+        load_report_file='load_report.html',
         text_report_file='report.txt',
         log_file='log.txt',
         sensor_storage='sensor_storage',
@@ -69,6 +70,7 @@
 
     var_dir = cfg_dict.get('internal', {}).get('var_dir_root', '/tmp')
     run_uuid = None
+
     if explicit_folder is None:
         for i in range(10):
             run_uuid = pet_generate(2, "_")
@@ -80,18 +82,21 @@
             results_dir = os.path.join(var_dir, run_uuid)
         cfg_dict['run_uuid'] = run_uuid.replace('_', '-')
     else:
+        if not os.path.isdir(explicit_folder):
+            ex2 = os.path.join(var_dir, explicit_folder)
+            if os.path.isdir(ex2):
+                explicit_folder = ex2
+            else:
+                raise RuntimeError("No such directory " + explicit_folder)
+
         results_dir = explicit_folder
 
     cfg_dict.update(get_test_files(results_dir))
     mkdirs_if_unxists(cfg_dict['var_dir'])
 
     if explicit_folder is not None:
-        with open(cfg_dict['run_params_file']) as fd:
-            cfg_dict['run_uuid'] = yaml.load(fd)['run_uuid']
+        cfg_dict.update(load_run_params(cfg_dict['run_params_file']))
         run_uuid = cfg_dict['run_uuid']
-    else:
-        with open(cfg_dict['run_params_file'], 'w') as fd:
-            fd.write(dumps({'run_uuid': cfg_dict['run_uuid']}))
 
     mkdirs_if_unxists(cfg_dict['sensor_storage'])
 
@@ -106,6 +111,25 @@
     mkdirs_if_unxists(cfg_dict['results'])
     mkdirs_if_unxists(cfg_dict['hwinfo_directory'])
 
+    return results_dir
+
+
+def save_run_params():
+    params = {
+        'comment': cfg_dict['comment'],
+        'run_uuid': cfg_dict['run_uuid']
+    }
+    with open(cfg_dict['run_params_file'], 'w') as fd:
+        fd.write(pretty_yaml.dumps(params))
+
+
+def load_run_params(run_params_file):
+    with open(run_params_file) as fd:
+        dt = yaml.load(fd)
+
+    return dict(run_uuid=dt['run_uuid'],
+                comment=dt.get('comment'))
+
 
 def color_me(color):
     RESET_SEQ = "\033[0m"
diff --git a/wally/discover/discover.py b/wally/discover/discover.py
index 5802ac3..2f41562 100644
--- a/wally/discover/discover.py
+++ b/wally/discover/discover.py
@@ -30,6 +30,8 @@
 def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True):
     nodes_to_run = []
     clean_data = None
+    ctx.fuel_openstack_creds = None
+
     for cluster in discover:
         if cluster == "openstack" and not discover_nodes:
             logger.warning("Skip openstack cluster discovery")
@@ -66,10 +68,14 @@
                                            discover_nodes)
             nodes, clean_data, openrc_dict = res
 
-            ctx.fuel_openstack_creds = {'name': openrc_dict['username'],
-                                        'passwd': openrc_dict['password'],
-                                        'tenant': openrc_dict['tenant_name'],
-                                        'auth_url': openrc_dict['os_auth_url']}
+            if openrc_dict is None:
+                ctx.fuel_openstack_creds = None
+            else:
+                ctx.fuel_openstack_creds = {
+                    'name': openrc_dict['username'],
+                    'passwd': openrc_dict['password'],
+                    'tenant': openrc_dict['tenant_name'],
+                    'auth_url': openrc_dict['os_auth_url']}
 
             env_name = clusters_info['fuel']['openstack_env']
             env_f_name = env_name
@@ -79,11 +85,11 @@
             fuel_openrc_fname = os.path.join(var_dir,
                                              env_f_name + "_openrc")
 
-            with open(fuel_openrc_fname, "w") as fd:
-                fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
-
-            msg = "Openrc for cluster {0} saves into {1}"
-            logger.debug(msg.format(env_name, fuel_openrc_fname))
+            if ctx.fuel_openstack_creds is not None:
+                with open(fuel_openrc_fname, "w") as fd:
+                    fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
+                    msg = "Openrc for cluster {0} saves into {1}"
+                    logger.debug(msg.format(env_name, fuel_openrc_fname))
             nodes_to_run.extend(nodes)
 
         elif cluster == "ceph":
diff --git a/wally/discover/fuel.py b/wally/discover/fuel.py
index 34adc07..665321e 100644
--- a/wally/discover/fuel.py
+++ b/wally/discover/fuel.py
@@ -89,9 +89,15 @@
     logger.debug("Found %s fuel nodes for env %r" %
                  (len(nodes), fuel_data['openstack_env']))
 
+    if version > [6, 0]:
+        openrc = cluster.get_openrc()
+    else:
+        logger.warning("Getting openrc on fuel 6.0 is broken, skip")
+        openrc = None
+
     return (nodes,
             (ssh_conn, fuel_ext_iface, ips_ports),
-            cluster.get_openrc())
+            openrc)
 
 
 def download_master_key(conn):
diff --git a/wally/report.py b/wally/report.py
index 1b1dbf9..2aa5338 100644
--- a/wally/report.py
+++ b/wally/report.py
@@ -1,6 +1,8 @@
 import os
+import csv
 import bisect
 import logging
+import itertools
 import collections
 from cStringIO import StringIO
 
@@ -14,7 +16,10 @@
 import wally
 from wally.utils import ssize2b
 from wally.statistic import round_3_digit, data_property
-from wally.suits.io.fio_task_parser import get_test_sync_mode
+from wally.suits.io.fio_task_parser import (get_test_sync_mode,
+                                            get_test_summary,
+                                            parse_all_in_1,
+                                            abbv_name_to_full)
 
 
 logger = logging.getLogger("wally.report")
@@ -24,6 +29,10 @@
     def __init__(self):
         self.direct_iops_r_max = 0
         self.direct_iops_w_max = 0
+
+        # 64 used instead of 4k to faster feed caches
+        self.direct_iops_w64_max = 0
+
         self.rws4k_10ms = 0
         self.rws4k_30ms = 0
         self.rws4k_100ms = 0
@@ -51,6 +60,8 @@
         self.bw = None
         self.iops = None
         self.lat = None
+        self.lat_50 = None
+        self.lat_95 = None
 
         self.raw_bw = []
         self.raw_iops = []
@@ -80,10 +91,55 @@
     return name_map
 
 
+def get_lat_perc_50_95(lat_mks):
+    curr_perc = 0
+    perc_50 = None
+    perc_95 = None
+    pkey = None
+    for key, val in sorted(lat_mks.items()):
+        if curr_perc + val >= 50 and perc_50 is None:
+            if pkey is None or val < 1.:
+                perc_50 = key
+            else:
+                perc_50 = (50. - curr_perc) / val * (key - pkey) + pkey
+
+        if curr_perc + val >= 95:
+            if pkey is None or val < 1.:
+                perc_95 = key
+            else:
+                perc_95 = (95. - curr_perc) / val * (key - pkey) + pkey
+            break
+
+        pkey = key
+        curr_perc += val
+
+    return perc_50 / 1000., perc_95 / 1000.
+
+
 def process_disk_info(test_data):
+
     name_map = group_by_name(test_data)
     data = {}
     for (name, summary), results in name_map.items():
+        lat_mks = collections.defaultdict(lambda: 0)
+        num_res = 0
+
+        for result in results:
+            num_res += len(result.raw_result['jobs'])
+            for job_info in result.raw_result['jobs']:
+                for k, v in job_info['latency_ms'].items():
+                    if isinstance(k, str):
+                        assert k[:2] == '>='
+                        lat_mks[int(k[2:]) * 1000] += v
+                    else:
+                        lat_mks[k * 1000] += v
+
+                for k, v in job_info['latency_us'].items():
+                    lat_mks[k] += v
+
+        for k, v in lat_mks.items():
+            lat_mks[k] = float(v) / num_res
+
         testnodes_count_set = set(dt.vm_count for dt in results)
 
         assert len(testnodes_count_set) == 1
@@ -102,6 +158,7 @@
         pinfo.bw = data_property(map(sum, zip(*pinfo.raw_bw)))
         pinfo.iops = data_property(map(sum, zip(*pinfo.raw_iops)))
         pinfo.lat = data_property(sum(pinfo.raw_lat, []))
+        pinfo.lat_50, pinfo.lat_95 = get_lat_perc_50_95(lat_mks)
 
         data[(p.name, summary)] = pinfo
     return data
@@ -141,50 +198,94 @@
     return open(templ_file, 'r').read()
 
 
+def group_by(data, func):
+    if len(data) < 2:
+        yield data
+        return
+
+    ndata = [(func(dt), dt) for dt in data]
+    ndata.sort(key=func)
+    pkey, dt = ndata[0]
+    curr_list = [dt]
+
+    for key, val in ndata[1:]:
+        if pkey != key:
+            yield curr_list
+            curr_list = [val]
+        else:
+            curr_list.append(val)
+        pkey = key
+
+    yield curr_list
+
+
 @report('linearity', 'linearity_test')
-def linearity_report(processed_results, path, lab_info):
-    labels_and_data = []
+def linearity_report(processed_results, lab_info, comment):
+    labels_and_data_mp = collections.defaultdict(lambda: [])
+    vls = {}
 
-    vls = processed_results.values()[0].params.vals.copy()
-    del vls['blocksize']
-
+    # plot io_time = func(bsize)
     for res in processed_results.values():
         if res.name.startswith('linearity_test'):
             iotimes = [1000. / val for val in res.iops.raw]
-            labels_and_data.append([res.p.blocksize, res.iops.raw, iotimes])
+
+            op_summ = get_test_summary(res.params)[:3]
+
+            labels_and_data_mp[op_summ].append(
+                [res.p.blocksize, res.iops.raw, iotimes])
+
             cvls = res.params.vals.copy()
             del cvls['blocksize']
-            assert cvls == vls
+            del cvls['rw']
 
-    labels_and_data.sort(key=lambda x: ssize2b(x[0]))
+            cvls.pop('sync', None)
+            cvls.pop('direct', None)
+            cvls.pop('buffered', None)
+
+            if op_summ not in vls:
+                vls[op_summ] = cvls
+            else:
+                assert cvls == vls[op_summ]
+
+    all_labels = None
     _, ax1 = plt.subplots()
+    for name, labels_and_data in labels_and_data_mp.items():
+        labels_and_data.sort(key=lambda x: ssize2b(x[0]))
 
-    labels, data, iotimes = zip(*labels_and_data)
-    plt.boxplot(iotimes)
+        labels, _, iotimes = zip(*labels_and_data)
 
-    if len(labels_and_data) > 2 and ssize2b(labels_and_data[-2][0]) >= 4096:
-        xt = range(1, len(labels) + 1)
+        if all_labels is None:
+            all_labels = labels
+        else:
+            assert all_labels == labels
 
-        def io_time(sz, bw, initial_lat):
-            return sz / bw + initial_lat
+        plt.boxplot(iotimes)
+        if len(labels_and_data) > 2 and \
+           ssize2b(labels_and_data[-2][0]) >= 4096:
 
-        x = numpy.array(map(ssize2b, labels))
-        y = numpy.array([sum(dt) / len(dt) for dt in iotimes])
-        popt, _ = scipy.optimize.curve_fit(io_time, x, y, p0=(100., 1.))
+            xt = range(1, len(labels) + 1)
 
-        y1 = io_time(x, *popt)
-        plt.plot(xt, y1, linestyle='--', label='LS linear approxomation')
+            def io_time(sz, bw, initial_lat):
+                return sz / bw + initial_lat
 
-        for idx, (sz, _, _) in enumerate(labels_and_data):
-            if ssize2b(sz) >= 4096:
-                break
+            x = numpy.array(map(ssize2b, labels))
+            y = numpy.array([sum(dt) / len(dt) for dt in iotimes])
+            popt, _ = scipy.optimize.curve_fit(io_time, x, y, p0=(100., 1.))
 
-        bw = (x[-1] - x[idx]) / (y[-1] - y[idx])
-        lat = y[-1] - x[-1] / bw
-        y2 = io_time(x, bw, lat)
+            y1 = io_time(x, *popt)
+            plt.plot(xt, y1, linestyle='--',
+                     label=name + ' LS linear approx')
 
-        plt.plot(xt, y2, linestyle='--',
-                 label='(4k & max) linear approxomation')
+            for idx, (sz, _, _) in enumerate(labels_and_data):
+                if ssize2b(sz) >= 4096:
+                    break
+
+            bw = (x[-1] - x[idx]) / (y[-1] - y[idx])
+            lat = y[-1] - x[-1] / bw
+            y2 = io_time(x, bw, lat)
+            plt.plot(xt, y2, linestyle='--',
+                     label=abbv_name_to_full(name) +
+                     ' (4k & max) linear approx')
 
     plt.setp(ax1, xticklabels=labels)
 
@@ -192,39 +293,58 @@
     plt.ylabel("IO time, ms")
 
     plt.subplots_adjust(top=0.85)
-    plt.legend(bbox_to_anchor=(0.5, 1.2), loc='upper center')
+    plt.legend(bbox_to_anchor=(0.5, 1.15),
+               loc='upper center',
+               prop={'size': 10}, ncol=2)
     plt.grid()
     iotime_plot = get_emb_data_svg(plt)
     plt.clf()
 
+    # plot IOPS = func(bsize)
     _, ax1 = plt.subplots()
-    plt.boxplot(data)
-    plt.setp(ax1, xticklabels=labels)
 
+    for name, labels_and_data in labels_and_data_mp.items():
+        labels_and_data.sort(key=lambda x: ssize2b(x[0]))
+        _, data, _ = zip(*labels_and_data)
+        plt.boxplot(data)
+        avg = [float(sum(arr)) / len(arr) for arr in data]
+        xt = range(1, len(data) + 1)
+        plt.plot(xt, avg, linestyle='--',
+                 label=abbv_name_to_full(name) + " avg")
+
+    plt.setp(ax1, xticklabels=labels)
     plt.xlabel("Block size")
     plt.ylabel("IOPS")
+    plt.legend(bbox_to_anchor=(0.5, 1.15),
+               loc='upper center',
+               prop={'size': 10}, ncol=2)
     plt.grid()
     plt.subplots_adjust(top=0.85)
 
     iops_plot = get_emb_data_svg(plt)
 
-    res1 = processed_results.values()[0]
+    res = set(get_test_lcheck_params(res) for res in processed_results.values())
+    ncount = list(set(res.testnodes_count for res in processed_results.values()))
+    conc = list(set(res.concurence for res in processed_results.values()))
+
+    assert len(conc) == 1
+    assert len(ncount) == 1
+
     descr = {
-        'vm_count': res1.testnodes_count,
-        'concurence': res1.concurence,
-        'oper_descr': get_test_lcheck_params(res1).capitalize()
+        'vm_count': ncount[0],
+        'concurence': conc[0],
+        'oper_descr': ", ".join(res).capitalize()
     }
 
     params_map = {'iotime_vs_size': iotime_plot,
                   'iops_vs_size': iops_plot,
                   'descr': descr}
 
-    with open(path, 'w') as fd:
-        fd.write(get_template('report_linearity.html').format(**params_map))
+    return get_template('report_linearity.html').format(**params_map)
 
 
 @report('lat_vs_iops', 'lat_vs_iops')
-def lat_vs_iops(processed_results, path, lab_info):
+def lat_vs_iops(processed_results, lab_info, comment):
     lat_iops = collections.defaultdict(lambda: [])
     requsted_vs_real = collections.defaultdict(lambda: {})
 
@@ -271,11 +391,10 @@
                   'iops_vs_requested': plt_iops_vs_requested,
                   'oper_descr': get_test_lcheck_params(res1).capitalize()}
 
-    with open(path, 'w') as fd:
-        fd.write(get_template('report_iops_vs_lat.html').format(**params_map))
+    return get_template('report_iops_vs_lat.html').format(**params_map)
 
 
-def render_all_html(dest, info, lab_description, images, templ_name):
+def render_all_html(comment, info, lab_description, images, templ_name):
     data = info.__dict__.copy()
     for name, val in data.items():
         if not name.startswith('__'):
@@ -290,18 +409,17 @@
                             data['bw_write_max'][1])
 
     images.update(data)
-    report = get_template(templ_name).format(lab_info=lab_description,
-                                             **images)
-
-    with open(dest, 'w') as fd:
-        fd.write(report)
+    return get_template(templ_name).format(lab_info=lab_description,
+                                           comment=comment,
+                                           **images)
 
 
 def io_chart(title, concurence,
              latv, latv_min, latv_max,
              iops_or_bw, iops_or_bw_err,
              legend, log=False,
-             boxplots=False):
+             boxplots=False,
+             latv_50=None, latv_95=None):
     points = " MiBps" if legend == 'BW' else ""
     lc = len(concurence)
     width = 0.35
@@ -323,9 +441,14 @@
     handles1, labels1 = p1.get_legend_handles_labels()
 
     p2 = p1.twinx()
-    p2.plot(xt, latv_max, label="lat max")
-    p2.plot(xt, latv, label="lat avg")
-    p2.plot(xt, latv_min, label="lat min")
+
+    if latv_50 is None:
+        p2.plot(xt, latv_max, label="lat max")
+        p2.plot(xt, latv, label="lat avg")
+        p2.plot(xt, latv_min, label="lat min")
+    else:
+        p2.plot(xt, latv_50, label="lat med")
+        p2.plot(xt, latv_95, label="lat 95%")
 
     plt.xlim(0.5, lc + 0.5)
     plt.xticks(xt, ["{0} * {1}".format(vm, th) for (vm, th) in concurence])
@@ -363,9 +486,14 @@
         chart_data.sort(key=lambda x: x.concurence)
 
         #  if x.lat.average < max_lat]
-        lat = [x.lat.average / 1000 for x in chart_data]
-        lat_min = [x.lat.min / 1000 for x in chart_data]
-        lat_max = [x.lat.max / 1000 for x in chart_data]
+        # lat = [x.lat.average / 1000 for x in chart_data]
+        # lat_min = [x.lat.min / 1000 for x in chart_data]
+        # lat_max = [x.lat.max / 1000 for x in chart_data]
+        lat = None
+        lat_min = None
+        lat_max = None
+        lat_50 = [x.lat_50 for x in chart_data]
+        lat_95 = [x.lat_95 for x in chart_data]
 
         testnodes_count = x.testnodes_count
         concurence = [(testnodes_count, x.concurence)
@@ -385,7 +513,7 @@
                       latv=lat, latv_min=lat_min, latv_max=lat_max,
                       iops_or_bw=data,
                       iops_or_bw_err=data_dev,
-                      legend=name)
+                      legend=name, latv_50=lat_50, latv_95=lat_95)
         files[fname] = fc
 
     return files
@@ -412,27 +540,37 @@
 
 def get_disk_info(processed_results):
     di = DiskInfo()
-    rws4k_iops_lat_th = []
-
     di.direct_iops_w_max = find_max_where(processed_results,
                                           'd', '4k', 'randwrite')
     di.direct_iops_r_max = find_max_where(processed_results,
                                           'd', '4k', 'randread')
 
-    di.bw_write_max = find_max_where(processed_results,
-                                     'd', '16m', 'randwrite', False)
+    di.direct_iops_w64_max = find_max_where(processed_results,
+                                            'd', '64k', 'randwrite')
+
+    for sz in ('16m', '64m'):
+        di.bw_write_max = find_max_where(processed_results,
+                                         'd', sz, 'randwrite', False)
+        if di.bw_write_max is not None:
+            break
+
     if di.bw_write_max is None:
         di.bw_write_max = find_max_where(processed_results,
                                          'd', '1m', 'write', False)
 
-    di.bw_read_max = find_max_where(processed_results,
-                                    'd', '16m', 'randread', False)
+    for sz in ('16m', '64m'):
+        di.bw_read_max = find_max_where(processed_results,
+                                        'd', sz, 'randread', False)
+        if di.bw_read_max is not None:
+            break
+
     if di.bw_read_max is None:
         di.bw_read_max = find_max_where(processed_results,
                                         'd', '1m', 'read', False)
 
+    rws4k_iops_lat_th = []
     for res in processed_results.values():
-        if res.sync_mode == 's' and res.p.blocksize == '4k':
+        if res.sync_mode in 'xs' and res.p.blocksize == '4k':
             if res.p.rw != 'randwrite':
                 continue
             rws4k_iops_lat_th.append((res.iops.average,
@@ -473,7 +611,17 @@
         return (med, conf_perc)
 
     hdi.direct_iops_r_max = pp(di.direct_iops_r_max)
-    hdi.direct_iops_w_max = pp(di.direct_iops_w_max)
+
+    if di.direct_iops_w_max is not None:
+        hdi.direct_iops_w_max = pp(di.direct_iops_w_max)
+    else:
+        hdi.direct_iops_w_max = None
+
+    if di.direct_iops_w64_max is not None:
+        hdi.direct_iops_w64_max = pp(di.direct_iops_w64_max)
+    else:
+        hdi.direct_iops_w64_max = None
+
     hdi.bw_write_max = pp(di.bw_write_max)
     hdi.bw_read_max = pp(di.bw_read_max)
 
@@ -483,33 +631,96 @@
     return hdi
 
 
-@report('HDD', 'hdd_test')
-def make_hdd_report(processed_results, path, lab_info):
+@report('hdd', 'hdd')
+def make_hdd_report(processed_results, lab_info, comment):
     plots = [
-        ('hdd_test_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
-        ('hdd_test_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS')
+        ('hdd_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+        ('hdd_rwx4k', 'rand_write_4k', 'Random write 4k sync IOPS')
     ]
     images = make_plots(processed_results, plots)
     di = get_disk_info(processed_results)
-    render_all_html(path, di, lab_info, images, "report_hdd.html")
+    return render_all_html(comment, di, lab_info, images, "report_hdd.html")
 
 
-@report('Ceph', 'ceph_test')
-def make_ceph_report(processed_results, path, lab_info):
+@report('cinder_iscsi', 'cinder_iscsi')
+def make_cinder_iscsi_report(processed_results, lab_info, comment):
     plots = [
-        ('ceph_test_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
-        ('ceph_test_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS'),
-        ('ceph_test_rrd16m', 'rand_read_16m', 'Random read 16m direct MiBps'),
-        ('ceph_test_rwd16m', 'rand_write_16m',
+        ('cinder_iscsi_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+        ('cinder_iscsi_rwx4k', 'rand_write_4k', 'Random write 4k sync IOPS')
+    ]
+    try:
+        images = make_plots(processed_results, plots)
+    except ValueError:
+        plots = [
+            ('cinder_iscsi_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+            ('cinder_iscsi_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS')
+        ]
+        images = make_plots(processed_results, plots)
+    di = get_disk_info(processed_results)
+    return render_all_html(comment, di, lab_info, images, "report_cinder_iscsi.html")
+
+
+@report('ceph', 'ceph')
+def make_ceph_report(processed_results, lab_info, comment):
+    plots = [
+        ('ceph_rrd4k', 'rand_read_4k', 'Random read 4k direct IOPS'),
+        ('ceph_rws4k', 'rand_write_4k', 'Random write 4k sync IOPS'),
+        ('ceph_rrd16m', 'rand_read_16m', 'Random read 16m direct MiBps'),
+        ('ceph_rwd16m', 'rand_write_16m',
          'Random write 16m direct MiBps'),
     ]
 
     images = make_plots(processed_results, plots)
     di = get_disk_info(processed_results)
-    render_all_html(path, di, lab_info, images, "report_ceph.html")
+    return render_all_html(comment, di, lab_info, images, "report_ceph.html")
 
 
-def make_io_report(dinfo, results, path, lab_info=None):
+def make_load_report(idx, results_dir, fname):
+    dpath = os.path.join(results_dir, "io_" + str(idx))
+    files = sorted(os.listdir(dpath))
+    gf = lambda x: "_".join(x.rsplit(".", 1)[0].split('_')[:3])
+
+    for key, group in itertools.groupby(files, gf):
+        fname = os.path.join(dpath, key + ".fio")
+
+        cfgs = list(parse_all_in_1(open(fname).read(), fname))
+
+        fname = os.path.join(dpath, key + "_lat.log")
+
+        curr = []
+        arrays = []
+
+        with open(fname) as fd:
+            for offset, lat, _, _ in csv.reader(fd):
+                offset = int(offset)
+                lat = int(lat)
+                if len(curr) > 0 and curr[-1][0] > offset:
+                    arrays.append(curr)
+                    curr = []
+                curr.append((offset, lat))
+            arrays.append(curr)
+        conc = int(cfgs[0].vals.get('numjobs', 1))
+
+        if conc != 5:
+            continue
+
+        assert len(arrays) == len(cfgs) * conc
+
+        garrays = [[(0, 0)] for _ in range(conc)]
+
+        for offset in range(len(cfgs)):
+            for acc, new_arr in zip(garrays, arrays[offset * conc:(offset + 1) * conc]):
+                last = acc[-1][0]
+                for off, lat in new_arr:
+                    acc.append((off / 1000. + last, lat / 1000.))
+
+        for cfg, arr in zip(cfgs, garrays):
+            plt.plot(*zip(*arr[1:]))
+        plt.show()
+        exit(1)
+
+
+def make_io_report(dinfo, comment, path, lab_info=None):
     lab_info = {
         "total_disk": "None",
         "total_memory": "None",
@@ -520,6 +731,7 @@
     try:
         res_fields = sorted(v.name for v in dinfo.values())
 
+        found = False
         for fields, name, func in report_funcs:
             for field in fields:
                 pos = bisect.bisect_left(res_fields, field)
@@ -530,11 +742,24 @@
                 if not res_fields[pos].startswith(field):
                     break
             else:
+                found = True
                 hpath = path.format(name)
-                logger.debug("Generatins report " + name + " into " + hpath)
-                func(dinfo, hpath, lab_info)
-                break
-        else:
+
+                try:
+                    report = func(dinfo, lab_info, comment)
+                except:
+                    logger.exception("Diring {0} report generation".format(name))
+                    continue
+
+                try:
+                    with open(hpath, "w") as fd:
+                        fd.write(report)
+                except:
+                    logger.exception("Diring saving {0} report".format(name))
+                    continue
+                logger.info("Report {0} saved into {1}".format(name, hpath))
+
+        if not found:
             logger.warning("No report generator found for this load")
 
     except Exception as exc:
diff --git a/wally/run_test.py b/wally/run_test.py
index 9a3d04c..d4fb911 100755
--- a/wally/run_test.py
+++ b/wally/run_test.py
@@ -13,7 +13,15 @@
 import contextlib
 import collections
 
-import yaml
+from yaml import load as _yaml_load
+
+try:
+    from yaml import CLoader
+    yaml_load = functools.partial(_yaml_load, Loader=CLoader)
+except ImportError:
+    yaml_load = _yaml_load
+
+
 import texttable
 
 try:
@@ -30,7 +38,7 @@
 from wally import utils, report, ssh_utils, start_vms
 from wally.suits import IOPerfTest, PgBenchTest, MysqlTest
 from wally.config import (cfg_dict, load_config, setup_loggers,
-                          get_test_files)
+                          get_test_files, save_run_params, load_run_params)
 from wally.sensors_utils import with_sensors_util, sensors_info_util
 
 TOOL_TYPE_MAPPER = {
@@ -466,7 +474,8 @@
             auth_url = os_cfg['OS_AUTH_URL'].strip()
 
     if tenant is None and 'fuel' in cfg['clouds'] and \
-       'openstack_env' in cfg['clouds']['fuel']:
+       'openstack_env' in cfg['clouds']['fuel'] and \
+       ctx.fuel_openstack_creds is not None:
         logger.info("Using fuel creds")
         creds = ctx.fuel_openstack_creds
     elif tenant is None:
@@ -605,7 +614,7 @@
     raw_results = cfg_dict['raw_results']
 
     if os.path.exists(raw_results):
-        cont = yaml.load(open(raw_results).read())
+        cont = yaml_load(open(raw_results).read())
     else:
         cont = []
 
@@ -640,6 +649,20 @@
             print("\n" + rep + "\n")
 
 
+def test_load_report_stage(cfg, ctx):
+    load_rep_fname = cfg['load_report_file']
+    found = False
+    for idx, (tp, data) in enumerate(ctx.results.items()):
+        if 'io' == tp and data is not None:
+            if found:
+                logger.error("Making reports for more than one " +
+                             "io block isn't supported! All " +
+                             "report, except first are skipped")
+                continue
+            found = True
+            report.make_load_report(idx, cfg['results'], load_rep_fname)
+
+
 def html_report_stage(cfg, ctx):
     html_rep_fname = cfg['html_report_file']
     found = False
@@ -652,7 +675,9 @@
                 continue
             found = True
             dinfo = report.process_disk_info(data)
-            report.make_io_report(dinfo, data, html_rep_fname,
+            report.make_io_report(dinfo,
+                                  cfg.get('comment', ''),
+                                  html_rep_fname,
                                   lab_info=ctx.hw_info)
 
 
@@ -662,15 +687,16 @@
         logger.debug(str(node))
 
 
-def load_data_from(var_dir):
-    def load_data_from_file(_, ctx):
-        raw_results = os.path.join(var_dir, 'raw_results.yaml')
-        ctx.results = {}
-        for tp, results in yaml.load(open(raw_results).read()):
-            cls = TOOL_TYPE_MAPPER[tp]
-            ctx.results[tp] = map(cls.load, results)
+def load_data_from_file(var_dir, _, ctx):
+    raw_results = os.path.join(var_dir, 'raw_results.yaml')
+    ctx.results = {}
+    for tp, results in yaml_load(open(raw_results).read()):
+        cls = TOOL_TYPE_MAPPER[tp]
+        ctx.results[tp] = map(cls.load, results)
 
-    return load_data_from_file
+
+def load_data_from(var_dir):
+    return functools.partial(load_data_from_file, var_dir)
 
 
 def start_web_ui(cfg, ctx):
@@ -705,6 +731,7 @@
                         help="Don't run tests", default=False)
     parser.add_argument("-p", '--post-process-only', metavar="VAR_DIR",
                         help="Only process data from previour run")
+    parser.add_argument("-x", '--xxx',  action='store_true')
     parser.add_argument("-k", '--keep-vm', action='store_true',
                         help="Don't remove test vm's", default=False)
     parser.add_argument("-d", '--dont-discover-nodes', action='store_true',
@@ -715,6 +742,7 @@
     parser.add_argument("--params", metavar="testname.paramname",
                         help="Test params", default=[])
     parser.add_argument("--ls", action='store_true', default=False)
+    parser.add_argument("-c", "--comment", default="")
     parser.add_argument("config_file")
 
     return parser.parse_args(argv[1:])
@@ -727,15 +755,12 @@
         return func.__name__ + " stage"
 
 
-def get_test_names(block):
-    assert len(block.items()) == 1
-    name, data = block.items()[0]
-    if name == 'start_test_nodes':
-        for in_blk in data['tests']:
-            for i in get_test_names(in_blk):
-                yield i
-    else:
-        yield name
+def get_test_names(raw_res):
+    res = set()
+    for tp, data in raw_res:
+        for block in data:
+            res.add("{0}({1})".format(tp, block.get('test_name', '-')))
+    return res
 
 
 def list_results(path):
@@ -743,45 +768,51 @@
 
     for dname in os.listdir(path):
 
-        cfg = get_test_files(os.path.join(path, dname))
+        files_cfg = get_test_files(os.path.join(path, dname))
 
-        if not os.path.isfile(cfg['raw_results']):
+        if not os.path.isfile(files_cfg['raw_results']):
             continue
 
-        res_mtime = time.ctime(os.path.getmtime(cfg['raw_results']))
-        cfg = yaml.load(open(cfg['saved_config_file']).read())
+        mt = os.path.getmtime(files_cfg['raw_results'])
+        res_mtime = time.ctime(mt)
 
-        test_names = []
+        raw_res = yaml_load(open(files_cfg['raw_results']).read())
+        test_names = ",".join(sorted(get_test_names(raw_res)))
 
-        for block in cfg['tests']:
-            test_names.extend(get_test_names(block))
+        params = load_run_params(files_cfg['run_params_file'])
 
-        results.append((dname, test_names, res_mtime))
+        comm = params.get('comment')
+        results.append((mt, dname, test_names, res_mtime,
+                       '-' if comm is None else comm))
 
-    tab = texttable.Texttable(max_width=120)
+    tab = texttable.Texttable(max_width=200)
     tab.set_deco(tab.HEADER | tab.VLINES | tab.BORDER)
-    tab.set_cols_align(["l", "l", "l"])
-    results.sort(key=lambda x: x[2])
+    tab.set_cols_align(["l", "l", "l", "l"])
+    results.sort()
 
-    for data in results:
-        dname, tests, mtime = data
-        tab.add_row([dname, ', '.join(tests), mtime])
+    for data in results[::-1]:
+        tab.add_row(data[1:])
 
-    tab.header(["Name", "Tests", "etime"])
+    tab.header(["Name", "Tests", "etime", "Comment"])
 
     print(tab.draw())
 
 
 def main(argv):
-    if '--ls' in argv:
-        list_results(argv[-1])
-        exit(0)
-
     if faulthandler is not None:
         faulthandler.register(signal.SIGUSR1, all_threads=True)
 
     opts = parse_args(argv)
-    load_config(opts.config_file, opts.post_process_only)
+
+    if opts.ls:
+        list_results(opts.config_file)
+        exit(0)
+
+    data_dir = load_config(opts.config_file, opts.post_process_only)
+
+    if opts.post_process_only is None:
+        cfg_dict['comment'] = opts.comment
+        save_run_params()
 
     if cfg_dict.get('logging', {}).get("extra_logs", False) or opts.extra_logs:
         level = logging.DEBUG
@@ -796,7 +827,7 @@
 
     if opts.post_process_only is not None:
         stages = [
-            load_data_from(opts.post_process_only)
+            load_data_from(data_dir)
         ]
     else:
         stages = [
@@ -823,7 +854,9 @@
         console_report_stage,
     ]
 
-    if not opts.no_html_report:
+    if opts.xxx:
+        report_stages.append(test_load_report_stage)
+    elif not opts.no_html_report:
         report_stages.append(html_report_stage)
 
     logger.info("All info would be stored into {0}".format(
diff --git a/wally/sensors_utils.py b/wally/sensors_utils.py
index e78bb5f..6b50311 100644
--- a/wally/sensors_utils.py
+++ b/wally/sensors_utils.py
@@ -81,5 +81,16 @@
                                      cfg['sensors_remote_path'])
 
     clear_old_sensors(sensors_configs)
-    with sensors_info(sensors_configs, cfg['sensors_remote_path']) as res:
+    ctx = sensors_info(sensors_configs, cfg['sensors_remote_path'])
+    try:
+        res = ctx.__enter__()
         yield res
+    except:
+        ctx.__exit__(None, None, None)
+        raise
+    finally:
+        try:
+            ctx.__exit__(None, None, None)
+        except:
+            logger.exception("During stop/collect sensors")
+            del res[:]
diff --git a/wally/suits/io/__init__.py b/wally/suits/io/__init__.py
index 978fa46..e548395 100644
--- a/wally/suits/io/__init__.py
+++ b/wally/suits/io/__init__.py
@@ -33,7 +33,8 @@
             'raw_result': self.raw_result,
             'run_interval': self.run_interval,
             'vm_count': self.vm_count,
-            'test_name': self.test_name
+            'test_name': self.test_name,
+            'files': self.files
         }
 
     @classmethod
@@ -44,7 +45,8 @@
 
         return cls(sec, data['params'], data['results'],
                    data['raw_result'], data['run_interval'],
-                   data['vm_count'], data['test_name'])
+                   data['vm_count'], data['test_name'],
+                   files=data.get('files', {}))
 
 
 def get_slice_parts_offset(test_slice, real_inteval):
@@ -121,8 +123,8 @@
                 # take largest size
                 files[fname] = max(files.get(fname, 0), msz)
 
-        cmd_templ = "dd oflag=direct " + \
-                    "if=/dev/zero of={0} bs={1} count={2}"
+        cmd_templ = "fio --name=xxx --filename={0} --direct=1" + \
+                    " --bs=4m --size={1}m --rw=write"
 
         if self.use_sudo:
             cmd_templ = "sudo " + cmd_templ
@@ -131,10 +133,16 @@
         stime = time.time()
 
         for fname, curr_sz in files.items():
-            cmd = cmd_templ.format(fname, 1024 ** 2, curr_sz)
+            cmd = cmd_templ.format(fname, curr_sz)
             ssize += curr_sz
             self.run_over_ssh(cmd, timeout=curr_sz)
 
+        # if self.use_sudo:
+        #     self.run_over_ssh("sudo echo 3 > /proc/sys/vm/drop_caches",
+        #                       timeout=5)
+        # else:
+        #     logging.warning("Can't flush caches as sudo us disabled")
+
         ddtime = time.time() - stime
         if ddtime > 1E-3:
             fill_bw = int(ssize / ddtime)
@@ -225,10 +233,24 @@
                     logger.info("Will run tests: " + ", ".join(msgs))
 
                 nolog = (pos != 0) or not self.is_primary
-                out_err, interval = self.do_run(barrier, fio_cfg_slice, pos,
-                                                nolog=nolog)
+
+                max_retr = 3 if self.total_nodes_count == 1 else 1
+
+                for idx in range(max_retr):
+                    try:
+                        out_err, interval, files = self.do_run(barrier, fio_cfg_slice, pos,
+                                                               nolog=nolog)
+                        break
+                    except Exception as exc:
+                        logger.exception("During fio run")
+                        if idx == max_retr - 1:
+                            raise StopTestError("Fio failed", exc)
+                    logger.info("Sleeping 30s and retrying")
+                    time.sleep(30)
 
                 try:
+                    # HACK
+                    out_err = "{" + out_err.split("{", 1)[1]
                     full_raw_res = json.loads(out_err)
 
                     res = {"bw": [], "iops": [], "lat": [],
@@ -246,27 +268,32 @@
                     first = fio_cfg_slice[0]
                     p1 = first.vals.copy()
                     p1.pop('ramp_time', 0)
+                    p1.pop('offset', 0)
 
                     for nxt in fio_cfg_slice[1:]:
                         assert nxt.name == first.name
                         p2 = nxt.vals
                         p2.pop('_ramp_time', 0)
-
+                        p2.pop('offset', 0)
                         assert p1 == p2
 
+                    tname = os.path.basename(self.config_fname)
+                    if tname.endswith('.cfg'):
+                        tname = tname[:-4]
+
                     tres = IOTestResults(first,
                                          self.config_params, res,
                                          full_raw_res, interval,
-                                         vm_count=self.total_nodes_count)
-                    tres.test_name = os.path.basename(self.config_fname)
-                    if tres.test_name.endswith('.cfg'):
-                        tres.test_name = tres.test_name[:-4]
+                                         test_name=tname,
+                                         vm_count=self.total_nodes_count,
+                                         files=files)
                     self.on_result_cb(tres)
-                except (OSError, StopTestError):
+                except StopTestError:
                     raise
                 except Exception as exc:
-                    msg_templ = "Error during postprocessing results: {0!s}"
-                    raise RuntimeError(msg_templ.format(exc))
+                    msg_templ = "Error during postprocessing results"
+                    logger.exception(msg_templ)
+                    raise StopTestError(msg_templ.format(exc), exc)
 
         finally:
             barrier.exit()
@@ -379,20 +406,22 @@
             with open(os.path.join(self.log_directory, fname), "w") as fd:
                 fd.write(result)
 
+            files = {}
+
             for fname in log_files:
                 try:
                     fc = read_from_remote(sftp, fname)
                 except:
                     continue
                 sftp.remove(fname)
-
-                loc_fname = "{0}_{1}_{2}".format(pos, fconn_id,
-                                                 fname.split('_')[-1])
+                ftype = fname.split('_')[-1].split(".")[0]
+                loc_fname = "{0}_{1}_{2}.log".format(pos, fconn_id, ftype)
+                files.setdefault(ftype, []).append(loc_fname)
                 loc_path = os.path.join(self.log_directory, loc_fname)
                 with open(loc_path, "w") as fd:
                     fd.write(fc)
 
-        return result, (begin, end)
+        return result, (begin, end), files
 
     @classmethod
     def merge_results(cls, results):
diff --git a/wally/suits/io/ceph.cfg b/wally/suits/io/ceph.cfg
index 26aa65f..4a651e6 100644
--- a/wally/suits/io/ceph.cfg
+++ b/wally/suits/io/ceph.cfg
@@ -3,17 +3,14 @@
 
 NUMJOBS={% 1, 5, 10, 15, 40 %}
 NUMJOBS_SHORT={% 1, 2, 3, 10 %}
-TEST_FILE_SIZE=100G
 
-size={TEST_FILE_SIZE}
 ramp_time=15
 runtime=60
-NUM_ROUNDS=7
 
 # ---------------------------------------------------------------------
 # check different thread count, sync mode. (latency, iops) = func(th_count)
 # ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
 blocksize=4k
 rw=randwrite
 sync=1
@@ -22,7 +19,7 @@
 # ---------------------------------------------------------------------
 # direct write
 # ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
 blocksize=4k
 rw=randwrite
 direct=1
@@ -32,7 +29,7 @@
 # check different thread count, direct read mode. (latency, iops) = func(th_count)
 # also check iops for randread
 # ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
 blocksize=4k
 rw=randread
 direct=1
@@ -42,7 +39,7 @@
 # this is essentially sequential write/read operations
 # we can't use sequential with numjobs > 1 due to caching and block merging
 # ---------------------------------------------------------------------
-[ceph_test_{TEST_SUMM}]
+[ceph_{TEST_SUMM}]
 blocksize=16m
 rw={% randread, randwrite %}
 direct=1
diff --git a/wally/suits/io/check_distribution.cfg b/wally/suits/io/check_distribution.cfg
index 9475cde..7c06813 100644
--- a/wally/suits/io/check_distribution.cfg
+++ b/wally/suits/io/check_distribution.cfg
@@ -1,13 +1,10 @@
 [global]
 include defaults.cfg
-NUM_ROUNDS=301
 
 [distrubution_test_{TEST_SUMM}]
 blocksize=4k
 rw=randwrite
 direct=1
-
+sync=1
 ramp_time=5
 runtime=30
-
-size=200G
diff --git a/wally/suits/io/check_linearity.cfg b/wally/suits/io/check_linearity.cfg
index 6af5fcc..42618d4 100644
--- a/wally/suits/io/check_linearity.cfg
+++ b/wally/suits/io/check_linearity.cfg
@@ -1,9 +1,7 @@
 [global]
-
 include defaults.cfg
-NUM_ROUNDS=7
 
-size={TEST_FILE_SIZE}
+direct=1
 ramp_time=5
 runtime=30
 BLOCK_SIZES={% 512,1k,2k,4k,8k,16k,32k,128k,256k,512k,1m %}
@@ -11,10 +9,9 @@
 # ---------------------------------------------------------------------
 # check read and write linearity. oper_time = func(size)
 # ---------------------------------------------------------------------
-# [linearity_test_{TEST_SUMM}]
-# blocksize={BLOCK_SIZES}
-# rw={% randwrite, randread %}
-# direct=1
+[linearity_test_{TEST_SUMM}]
+blocksize={BLOCK_SIZES}
+rw=randread
 
 # ---------------------------------------------------------------------
 # check sync write linearity. oper_time = func(size)
diff --git a/wally/suits/io/check_th_count.cfg b/wally/suits/io/check_th_count.cfg
index 1607634..745f189 100644
--- a/wally/suits/io/check_th_count.cfg
+++ b/wally/suits/io/check_th_count.cfg
@@ -1,18 +1,10 @@
-[defaults]
+[global]
+include defaults.cfg
 
-# this is critical for correct results in multy-node run
-randrepeat=0
-
-NUM_ROUNDS=7
 ramp_time=5
-buffered=0
-wait_for_previous
-filename={FILENAME}
-iodepth=1
-size=10G
-time_based
 runtime=30
-group_reporting
+direct=1
+
 numjobs={% 1, 2, 5, 10, 15, 20, 25, 30, 35, 40 %}
 
 # ---------------------------------------------------------------------
@@ -31,20 +23,15 @@
 # 1m + read      + direct
 #
 # ---------------------------------------------------------------------
-[concurrence_test_{TEST_SUMM} * {NUM_ROUNDS}]
-blocksize=4k
-rw={% randread %}
-direct=1
-sync=0
-
-[concurrence_test_{TEST_SUMM} * {NUM_ROUNDS}]
+[concurrence_{TEST_SUMM}]
 blocksize=4k
 rw=randwrite
-direct=0
+
+[concurrence_{TEST_SUMM}]
+blocksize=4k
+rw={% randread, randwrite %}
 sync=1
 
-[concurrence_test_{TEST_SUMM} * {NUM_ROUNDS}]
+[concurrence_{TEST_SUMM}]
 blocksize=1m
 rw={% write, read %}
-direct=1
-sync=0
diff --git a/wally/suits/io/cinder_iscsi.cfg b/wally/suits/io/cinder_iscsi.cfg
new file mode 100644
index 0000000..4d19dd9
--- /dev/null
+++ b/wally/suits/io/cinder_iscsi.cfg
@@ -0,0 +1,56 @@
+[global]
+include defaults.cfg
+
+# NUMJOBS={% 1, 5, 10, 15, 20, 30, 40, 80 %}
+
+NUMJOBS={% 1, 3, 5, 10, 20, 40 %}
+
+direct=1
+ramp_time=5
+runtime=30
+
+# ---------------------------------------------------------------------
+# check different thread count, sync mode. (latency, iops) = func(th_count)
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=4k
+rw=randwrite
+sync=1
+numjobs={NUMJOBS}
+
+# ---------------------------------------------------------------------
+# check different thread count, direct read mode. (latency, iops) = func(th_count)
+# also check iops for randread
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=4k
+rw=randread
+numjobs={NUMJOBS}
+
+# ---------------------------------------------------------------------
+# check IOPS randwrite.
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=64k
+rw=randwrite
+ramp_time=180
+runtime=120
+
+# ---------------------------------------------------------------------
+# No reason for th count > 1 in case of sequantial operations
+# ot they became random
+# ---------------------------------------------------------------------
+[cinder_iscsi_{TEST_SUMM}]
+blocksize=1m
+rw={% read,write %}
+offset={UNIQ_OFFSET}
+ramp_time=90
+runtime=30
+
+# [cinder_iscsi_{TEST_SUMM}]
+# blocksize=64m
+# rw={% randread,randwrite %}
+# direct=1
+# ramp_time=30
+# runtime=30
+#
diff --git a/wally/suits/io/defaults.cfg b/wally/suits/io/defaults.cfg
index 51a8145..9aff22c 100644
--- a/wally/suits/io/defaults.cfg
+++ b/wally/suits/io/defaults.cfg
@@ -1,7 +1,9 @@
 buffered=0
 group_reporting=1
 iodepth=1
-softrandommap=1
+
+norandommap=1
+
 thread=1
 time_based=1
 wait_for_previous=1
@@ -11,4 +13,11 @@
 
 filename={FILENAME}
 
+size={TEST_FILE_SIZE}
+
+write_lat_log=fio_log
+write_iops_log=fio_log
+write_bw_log=fio_log
+log_avg_msec=500
+
 
diff --git a/wally/suits/io/fio_task_parser.py b/wally/suits/io/fio_task_parser.py
index 52c4bb3..aca0254 100644
--- a/wally/suits/io/fio_task_parser.py
+++ b/wally/suits/io/fio_task_parser.py
@@ -7,7 +7,7 @@
 from collections import OrderedDict, namedtuple
 
 
-from wally.utils import sec_to_str
+from wally.utils import sec_to_str, ssize2b
 
 
 SECTION = 0
@@ -50,20 +50,6 @@
         return res
 
 
-def to_bytes(sz):
-    sz = sz.lower()
-    try:
-        return int(sz)
-    except ValueError:
-        if sz[-1] == 'm':
-            return (1024 ** 2) * int(sz[:-1])
-        if sz[-1] == 'k':
-            return 1024 * int(sz[:-1])
-        if sz[-1] == 'g':
-            return (1024 ** 3) * int(sz[:-1])
-        raise
-
-
 class ParseError(ValueError):
     def __init__(self, msg, fname, lineno, line_cont=""):
         ValueError.__init__(self, msg)
@@ -265,11 +251,30 @@
             elif val.name in processed_vals:
                 val = processed_vals[val.name]
         processed_vals[name] = val
+
     sec = sec.copy()
     sec.vals = processed_vals
     return sec
 
 
+MAGIC_OFFSET = 0.1885
+
+
+def abbv_name_to_full(name):
+    assert len(name) == 3
+
+    smode = {
+        'a': 'async',
+        's': 'sync',
+        'd': 'direct',
+        'x': 'sync direct'
+    }
+    off_mode = {'s': 'sequential', 'r': 'random'}
+    oper = {'r': 'read', 'w': 'write'}
+    return smode[name[2]] + " " + \
+        off_mode[name[0]] + " " + oper[name[1]]
+
+
 def finall_process(sec, counter=[0]):
     sec = sec.copy()
 
@@ -279,6 +284,16 @@
 
     sec.vals['unified_rw_reporting'] = '1'
 
+    sz = ssize2b(sec.vals['size'])
+    offset = sz * ((MAGIC_OFFSET * counter[0]) % 1.0)
+    offset = int(offset) // 1024 ** 2
+    new_vars = {'UNIQ_OFFSET': str(offset) + "m"}
+
+    for name, val in sec.vals.items():
+        if isinstance(val, Var):
+            if val.name in new_vars:
+                sec.vals[name] = new_vars[val.name]
+
     params = sec.vals.copy()
     params['UNIQ'] = 'UN{0}'.format(counter[0])
     params['COUNTER'] = str(counter[0])
diff --git a/wally/suits/io/formatter.py b/wally/suits/io/formatter.py
index 84b0a13..59691b2 100644
--- a/wally/suits/io/formatter.py
+++ b/wally/suits/io/formatter.py
@@ -2,22 +2,27 @@
 
 from wally.utils import ssize2b
 from wally.statistic import round_3_digit
-from .fio_task_parser import get_test_summary, get_test_sync_mode
+from .fio_task_parser import get_test_sync_mode
+
+
+def getconc(data):
+    th_count = data.params.vals.get('numjobs')
+
+    if th_count is None:
+        th_count = data.params.vals.get('concurence', 1)
+    return th_count
 
 
 def key_func(data):
     p = data.params.vals
 
-    th_count = data.params.vals.get('numjobs')
+    th_count = getconc(data)
 
-    if th_count is None:
-        th_count = data.params.vals.get('concurence', 1)
-
-    return (p['rw'],
+    return (data.name.rsplit("_", 1)[0],
+            p['rw'],
             get_test_sync_mode(data.params),
             ssize2b(p['blocksize']),
-            int(th_count) * data.testnodes_count,
-            data.name)
+            int(th_count) * data.testnodes_count)
 
 
 def format_results_for_console(dinfo):
@@ -36,8 +41,7 @@
               "Cnf\n95%", "Dev%", "iops\nper vm", "KiBps\nper vm", "lat\nms"]
 
     for data in items:
-
-        curr_k = key_func(data)[:3]
+        curr_k = key_func(data)[:4]
 
         if prev_k is not None:
             if prev_k != curr_k:
diff --git a/wally/suits/io/hdd.cfg b/wally/suits/io/hdd.cfg
index 0eb85a6..6d3107a 100644
--- a/wally/suits/io/hdd.cfg
+++ b/wally/suits/io/hdd.cfg
@@ -5,18 +5,14 @@
 
 NUMJOBS={% 1, 3, 5, 10, 20, 40 %}
 
-write_lat_log=fio_log
-write_iops_log=fio_log
-log_avg_msec=500
-
-size={TEST_FILE_SIZE}
 ramp_time=5
 runtime=30
+direct=1
 
 # ---------------------------------------------------------------------
 # check different thread count, sync mode. (latency, iops) = func(th_count)
 # ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
 blocksize=4k
 rw=randwrite
 sync=1
@@ -26,25 +22,22 @@
 # check different thread count, direct read mode. (latency, iops) = func(th_count)
 # also check iops for randread
 # ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
 blocksize=4k
 rw=randread
-direct=1
 numjobs={NUMJOBS}
 
 # ---------------------------------------------------------------------
 # check IOPS randwrite.
 # ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
 blocksize=4k
 rw=randwrite
-direct=1
 
 # ---------------------------------------------------------------------
 # No reason for th count > 1 in case of sequantial operations
-# They became random
+# ot they became random
 # ---------------------------------------------------------------------
-[hdd_test_{TEST_SUMM}]
+[hdd_{TEST_SUMM}]
 blocksize=1m
 rw={% read, write %}
-direct=1
diff --git a/wally/suits/io/lat_vs_iops.cfg b/wally/suits/io/lat_vs_iops.cfg
index dbafcbb..16e73e2 100644
--- a/wally/suits/io/lat_vs_iops.cfg
+++ b/wally/suits/io/lat_vs_iops.cfg
@@ -1,15 +1,13 @@
 [global]
 include defaults.cfg
 
-TEST_FILE_SIZE=100G
-size={TEST_FILE_SIZE}
-
 ramp_time=5
 runtime=30
 
 blocksize=4k
 rw=randwrite
 sync=1
+direct=1
 
 # ---------------------------------------------------------------------
 # latency as function from IOPS
diff --git a/wally/suits/io/long_test.cfg b/wally/suits/io/long_test.cfg
index fd420d8..a304b8b 100644
--- a/wally/suits/io/long_test.cfg
+++ b/wally/suits/io/long_test.cfg
@@ -1,31 +1,22 @@
-[defaults]
-
-# this is critical for correct results in multy-node run
-randrepeat=0
+[global]
+include defaults.cfg
 
 # 24h test
 NUM_ROUNDS1=270
 NUM_ROUNDS2=261
 
-buffered=0
-wait_for_previous
-filename={FILENAME}
-iodepth=1
-size=50G
-time_based
-runtime=300
+direct=1
+blocksize=128k
+rw=randwrite
 
 # ---------------------------------------------------------------------
 # check read and write linearity. oper_time = func(size)
 # ---------------------------------------------------------------------
-[24h_test * {NUM_ROUNDS1}]
-blocksize=128k
-rw=randwrite
-direct=1
+[24h_test]
 runtime=30
+NUM_ROUND={NUM_ROUNDS1}
 
-[24h_test * {NUM_ROUNDS2}]
-blocksize=128k
-rw=randwrite
-direct=1
+[24h_test]
+runtime=300
+NUM_ROUND={NUM_ROUNDS2}
 
diff --git a/wally/suits/io/rrd.cfg b/wally/suits/io/rrd.cfg
index 8eaffea..78e1f0e 100644
--- a/wally/suits/io/rrd.cfg
+++ b/wally/suits/io/rrd.cfg
@@ -5,10 +5,6 @@
 ramp_time=5
 runtime=40
 
-write_lat_log=fio_log
-write_iops_log=fio_log
-log_avg_msec=500
-
 # ---------------------------------------------------------------------
 [rws_{TEST_SUMM}]
 blocksize=4k
diff --git a/wally/suits/io/verify.cfg b/wally/suits/io/verify.cfg
index b3162eb..92c78e5 100644
--- a/wally/suits/io/verify.cfg
+++ b/wally/suits/io/verify.cfg
@@ -5,10 +5,6 @@
 ramp_time=5
 runtime=10
 
-write_lat_log=fio_log
-write_iops_log=fio_log
-log_avg_msec=500
-
 # ---------------------------------------------------------------------
 [verify_{TEST_SUMM}]
 blocksize=4k
diff --git a/wally/suits/io/vm_count_ec2.cfg b/wally/suits/io/vm_count_ec2.cfg
index c6fc56c..3efcf00 100644
--- a/wally/suits/io/vm_count_ec2.cfg
+++ b/wally/suits/io/vm_count_ec2.cfg
@@ -1,37 +1,26 @@
-[defaults]
-buffered=0
-wait_for_previous=1
-filename={FILENAME}
-iodepth=1
-size=10G
+[global]
+include defaults.cfg
 
-# this is critical for correct results in multy-node run
-randrepeat=0
-
-time_based=1
 ramp_time=5
 runtime=30
 
-group_reporting=1
 BW_LIMIT=60m
 IOPS_LIMIT=100
 
+direct=1
 NUMJOBS=1
-NUM_ROUNDS=7
 
 # ---------------------------------------------------------------------
 # check different thread count. (latency, bw) = func(th_count)
 # ---------------------------------------------------------------------
-[vm_count_{TEST_SUMM} * {NUM_ROUNDS}]
-blocksize=16m
+[vmcount_{TEST_SUMM}]
+blocksize=4m
 rw={% randwrite, randread %}
-direct=1
 numjobs={NUMJOBS}
 rate={BW_LIMIT}
 
-[vm_count_{TEST_SUMM} * {NUM_ROUNDS}]
+[vmcount_{TEST_SUMM}]
 blocksize=4k
 rw={% randwrite,randread %}
-direct=1
 numjobs={NUMJOBS}
 rate_iops={IOPS_LIMIT}
diff --git a/wally/suits/itest.py b/wally/suits/itest.py
index 4be7eed..1cbf88c 100644
--- a/wally/suits/itest.py
+++ b/wally/suits/itest.py
@@ -1,13 +1,96 @@
 import abc
 import os.path
+import functools
 
 
 from wally.ssh_utils import run_over_ssh, copy_paths
 
 
+def cached_prop(func):
+    @property
+    @functools.wraps(func)
+    def closure(self):
+        val = getattr(self, "_" + func.__name__)
+        if val is NoData:
+            val = func(self)
+            setattr(self, "_" + func.__name__, val)
+        return val
+    return closure
+
+
+class NoData(object):
+    pass
+
+
+class VMThData(object):
+    "store set of values for VM_COUNT * TH_COUNT"
+
+
+class IOTestResult(object):
+    def __init__(self):
+        self.run_config = None
+        self.suite_config = None
+        self.run_interval = None
+
+        self.bw = None
+        self.lat = None
+        self.iops = None
+        self.slat = None
+        self.clat = None
+
+        self.fio_section = None
+
+        self._lat_log = NoData
+        self._iops_log = NoData
+        self._bw_log = NoData
+
+        self._sensors_data = NoData
+        self._raw_resuls = NoData
+
+    def to_jsonable(self):
+        pass
+
+    @property
+    def thread_count(self):
+        pass
+
+    @property
+    def sync_mode(self):
+        pass
+
+    @property
+    def abbrev_name(self):
+        pass
+
+    @property
+    def full_name(self):
+        pass
+
+    @cached_prop
+    def lat_log(self):
+        pass
+
+    @cached_prop
+    def iops_log(self):
+        pass
+
+    @cached_prop
+    def bw_log(self):
+        pass
+
+    @cached_prop
+    def sensors_data(self):
+        pass
+
+    @cached_prop
+    def raw_resuls(self):
+        pass
+
+
 class TestResults(object):
     def __init__(self, config, params, results,
-                 raw_result, run_interval, vm_count, test_name=None):
+                 raw_result, run_interval, vm_count,
+                 test_name, **attrs):
         self.config = config
         self.params = params
         self.results = results
@@ -15,6 +98,7 @@
         self.run_interval = run_interval
         self.vm_count = vm_count
         self.test_name = test_name
+        self.__dict__.update(attrs)
 
     def __str__(self):
         res = "{0}({1}):\n    results:\n".format(