large commit. refactoring, fio code totally reworker, huge improvenent in test time and results, etc
diff --git a/scripts/agent.py b/scripts/agent.py
index 56189f9..2873cb5 100644
--- a/scripts/agent.py
+++ b/scripts/agent.py
@@ -22,10 +22,10 @@
out, err = p.communicate()
- if not out is None:
+ if out is not None:
print out
- if not err is None:
+ if err is not None:
print err
node_port[ip] = base_port
@@ -35,18 +35,12 @@
def parse_command_line(argv):
- parser = argparse.ArgumentParser(description=
- "Connect to fuel master "
+ parser = argparse.ArgumentParser(description="Connect to fuel master " +
"and setup ssh agent")
- parser.add_argument(
- "--base_port", type=int, required=True)
-
+ parser.add_argument("--base_port", type=int, required=True)
# To do: fix clean to be False when string is False
- parser.add_argument(
- "--clean", type=bool, default=False)
-
- parser.add_argument(
- "--ports", type=str, nargs='+')
+ parser.add_argument("--clean", type=bool, default=False)
+ parser.add_argument("--ports", type=str, nargs='+')
return parser.parse_args(argv)
diff --git a/scripts/connector.py b/scripts/connector.py
index 4d9d32f..15bc538 100644
--- a/scripts/connector.py
+++ b/scripts/connector.py
@@ -6,7 +6,7 @@
from urlparse import urlparse
-from keystone import KeystoneAuth
+from disk_perf_test_tool.keystone import KeystoneAuth
def discover_fuel_nodes(fuel_url, creds, cluster_id):
@@ -92,17 +92,11 @@
def parse_command_line(argv):
parser = argparse.ArgumentParser(
description="Connect to fuel master and setup ssh agent")
- parser.add_argument(
- "--fuel_url", required=True)
- parser.add_argument(
- "--cluster_id", required=True)
- parser.add_argument(
- "--username", default="admin")
- parser.add_argument(
- "--tenantname", default="admin")
- parser.add_argument(
- "--password", default="admin")
-
+ parser.add_argument("--fuel-url", required=True)
+ parser.add_argument("--cluster-id", required=True)
+ parser.add_argument("--username", default="admin")
+ parser.add_argument("--tenantname", default="admin")
+ parser.add_argument("--password", default="admin")
return parser.parse_args(argv)
@@ -113,6 +107,8 @@
"password": args.password}
nodes = discover_fuel_nodes(args.fuel_url, creds, args.cluster_id)
+ print "Ready", nodes
+ sys.stdin.readline()
discover_fuel_nodes_clean(args.fuel_url, {"username": "root",
"password": "test37",
"port": 22}, nodes)
diff --git a/scripts/data2.py b/scripts/data2.py
index 08dbc77..4a8dad9 100644
--- a/scripts/data2.py
+++ b/scripts/data2.py
@@ -1,6 +1,6 @@
import sys
-import math
-import itertools
+from data_stat import med_dev, round_deviation, groupby_globally
+from data_stat import read_data_agent_result
def key(x):
@@ -10,36 +10,6 @@
x['__meta__']['concurence'])
-def med_dev(vals):
- med = sum(vals) / len(vals)
- dev = ((sum(abs(med - i) ** 2 for i in vals) / len(vals)) ** 0.5)
- return int(med), int(dev)
-
-
-def round_deviation(med_dev):
- med, dev = med_dev
-
- if dev < 1E-7:
- return med_dev
-
- dev_div = 10.0 ** (math.floor(math.log10(dev)) - 1)
- dev = int(dev / dev_div) * dev_div
- med = int(med / dev_div) * dev_div
- return (type(med_dev[0])(med),
- type(med_dev[1])(dev))
-
-
-def groupby_globally(data, key_func):
- grouped = {}
- grouped_iter = itertools.groupby(data, key_func)
-
- for (bs, cache_tp, act, conc), curr_data_it in grouped_iter:
- key = (bs, cache_tp, act, conc)
- grouped.setdefault(key, []).extend(curr_data_it)
-
- return grouped
-
-
template = "{bs:>4} {action:>12} {cache_tp:>3} {conc:>4}"
template += " | {iops[0]:>6} ~ {iops[1]:>5} | {bw[0]:>7} ~ {bw[1]:>6}"
template += " | {lat[0]:>6} ~ {lat[1]:>5} |"
@@ -54,21 +24,7 @@
def main(argv):
- data = []
-
- with open(argv[1]) as fc:
- block = None
- for line in fc:
- if line.startswith("{'__meta__':"):
- block = line
- elif block is not None:
- block += line
-
- if block is not None:
- if block.count('}') == block.count('{'):
- data.append(eval(block))
- block = None
-
+ data = read_data_agent_result(sys.argv[1])
grouped = groupby_globally(data, key)
print template.format(**headers)
diff --git a/scripts/data_stat.py b/scripts/data_stat.py
new file mode 100644
index 0000000..a4d46bf
--- /dev/null
+++ b/scripts/data_stat.py
@@ -0,0 +1,49 @@
+import math
+import itertools
+
+
+def med_dev(vals):
+ med = sum(vals) / len(vals)
+ dev = ((sum(abs(med - i) ** 2 for i in vals) / len(vals)) ** 0.5)
+ return int(med), int(dev)
+
+
+def round_deviation(med_dev):
+ med, dev = med_dev
+
+ if dev < 1E-7:
+ return med_dev
+
+ dev_div = 10.0 ** (math.floor(math.log10(dev)) - 1)
+ dev = int(dev / dev_div) * dev_div
+ med = int(med / dev_div) * dev_div
+ return (type(med_dev[0])(med),
+ type(med_dev[1])(dev))
+
+
+def groupby_globally(data, key_func):
+ grouped = {}
+ grouped_iter = itertools.groupby(data, key_func)
+
+ for (bs, cache_tp, act, conc), curr_data_it in grouped_iter:
+ key = (bs, cache_tp, act, conc)
+ grouped.setdefault(key, []).extend(curr_data_it)
+
+ return grouped
+
+
+def read_data_agent_result(fname):
+ data = []
+ with open(fname) as fc:
+ block = None
+ for line in fc:
+ if line.startswith("{'__meta__':"):
+ block = line
+ elif block is not None:
+ block += line
+
+ if block is not None:
+ if block.count('}') == block.count('{'):
+ data.append(eval(block))
+ block = None
+ return data
diff --git a/scripts/disk_io_pp.py b/scripts/disk_io_pp.py
new file mode 100644
index 0000000..bf901d3
--- /dev/null
+++ b/scripts/disk_io_pp.py
@@ -0,0 +1,60 @@
+import sys
+import collections
+
+import scipy.stats as stats
+import matplotlib.mlab as mlab
+import matplotlib.pyplot as plt
+
+from data_stat import med_dev, round_deviation
+from data_stat import read_data_agent_result
+
+data = read_data_agent_result(sys.argv[1])
+
+# for run in data:
+# for name, numbers in run['res'].items():
+# # med, dev = round_deviation(med_dev(numbers['iops']))
+# # print name, med, '~', dev
+# distr = collections.defaultdict(lambda: 0.0)
+# for i in numbers['iops']:
+# distr[i] += 1
+
+# print name
+# for key, val in sorted(distr.items()):
+# print " ", key, val
+# print
+
+
+
+# # example data
+# mu = 100 # mean of distribution
+# sigma = 15 # standard deviation of distribution
+# x = mu + sigma * np.random.randn(10000)
+
+x = data[0]['res'][sys.argv[2]]['iops']
+# mu, sigma = med_dev(x)
+# print mu, sigma
+
+# med_sz = 1
+# x2 = x[:len(x) // med_sz * med_sz]
+# x2 = [sum(vals) / len(vals) for vals in zip(*[x2[i::med_sz]
+# for i in range(med_sz)])]
+
+mu, sigma = med_dev(x)
+print mu, sigma
+print stats.normaltest(x)
+
+num_bins = 20
+# the histogram of the data
+n, bins, patches = plt.hist(x, num_bins, normed=1, facecolor='green', alpha=0.5)
+# add a 'best fit' line
+
+y = mlab.normpdf(bins, mu, sigma)
+plt.plot(bins, y, 'r--')
+
+plt.xlabel('Smarts')
+plt.ylabel('Probability')
+plt.title(r'Histogram of IQ: $\mu={}$, $\sigma={}$'.format(mu, sigma))
+
+# Tweak spacing to prevent clipping of ylabel
+plt.subplots_adjust(left=0.15)
+plt.show()