Implement MOL-168 - Allow to run set of tests on single vm
diff --git a/io_scenario/io.py b/io_scenario/io.py
index e1756f6..4f7cbcf 100644
--- a/io_scenario/io.py
+++ b/io_scenario/io.py
@@ -139,12 +139,12 @@
IOZoneParser.make_positions()
-def iozone_do_prepare(params, filename, pattern):
+def iozone_do_prepare(params, filename, pattern_base):
all_files = []
threads = int(params.concurence)
if 1 != threads:
- filename = filename + "_{}"
- all_files.extend(filename % i for i in range(threads))
+ filename = filename + "_{0}"
+ all_files.extend(filename .format(i) for i in range(threads))
else:
all_files.append(filename)
@@ -157,11 +157,11 @@
for fname in all_files:
with open(fname, "wb") as fd:
if fsz > 1024:
- pattern = pattern * 1024 * 1024
+ pattern = pattern_base * 1024 * 1024
for _ in range(int(fsz / 1024) + 1):
fd.write(pattern)
else:
- fd.write(pattern * 1024 * fsz)
+ fd.write(pattern_base * 1024 * fsz)
fd.flush()
return all_files
@@ -231,7 +231,10 @@
return res, " ".join(cmd)
-def run_iozone(benchmark, iozone_path, tmpname, timeout=None):
+def run_iozone(benchmark, iozone_path, tmpname,
+ prepare_only=False,
+ timeout=None):
+
if timeout is not None:
benchmark.size = benchmark.blocksize * 50
res_time = do_run_iozone(benchmark, tmpname, timeout,
@@ -244,7 +247,7 @@
benchmark.size = size
return do_run_iozone(benchmark, tmpname, timeout,
- iozone_path=iozone_path)
+ iozone_path=iozone_path, prepare_only=prepare_only)
def install_iozone_package():
@@ -299,6 +302,8 @@
def run_fio_once(benchmark, fio_path, tmpname, timeout=None):
+ if benchmark.size is None:
+ raise ValueError("You need to specify file size for fio")
cmd_line = [fio_path,
"--name=%s" % benchmark.action,
@@ -325,7 +330,7 @@
return json.loads(raw_out)["jobs"][0], " ".join(cmd_line)
-def run_fio(benchmark, fio_path, tmpname, timeout=None, prepare_only=False):
+def run_fio(benchmark, fio_path, tmpname, prepare_only=False, timeout=None):
if prepare_only:
return {}, ""
@@ -486,6 +491,7 @@
parser.add_argument(
"--prepare-only", default=False, dest='prepare_only',
action="store_true")
+ parser.add_argument("--concurrency", default=1, type=int)
return parser.parse_args(argv)
@@ -505,7 +511,7 @@
else:
argv_obj.iosize = ssize_to_kb(argv_obj.iosize)
- benchmark = BenchmarkOption(1,
+ benchmark = BenchmarkOption(argv_obj.concurrency,
argv_obj.iodepth,
argv_obj.action,
argv_obj.blocksize,
@@ -540,7 +546,9 @@
benchmark,
binary_path,
test_file_name,
- argv_obj.prepare_only)
+ argv_obj.prepare_only,
+ argv_obj.timeout)
+
if not argv_obj.prepare_only:
res['__meta__'] = benchmark.__dict__.copy()
res['__meta__']['cmdline'] = cmd
diff --git a/run_test.py b/run_test.py
index 04176b6..bc1f8fc 100644
--- a/run_test.py
+++ b/run_test.py
@@ -5,6 +5,7 @@
import pprint
import os.path
import argparse
+import traceback
import io_scenario
from itest import IOPerfTest
@@ -46,9 +47,12 @@
help="print some extra log info")
parser.add_argument("-o", "--io-opts", dest='io_opts',
- required=True,
help="cmd line options for io.py")
+ parser.add_argument("-f", "--io-opts-file", dest='io_opts_file',
+ type=argparse.FileType('r'), default=None,
+ help="file with cmd line options for io.py")
+
parser.add_argument("-t", "--test-directory", help="directory with test",
dest="test_directory", required=True)
@@ -69,65 +73,7 @@
return parser.parse_args(argv)
-def main(argv):
- opts = parse_args(argv)
-
- if not opts.extra_logs:
- def nolog(x):
- pass
-
- setlogger(nolog)
-
- script_args = [opt.strip()
- for opt in opts.io_opts.split(" ")
- if opt.strip() != ""]
-
- if opts.runner == "rally":
- runner = rally_runner.get_rally_runner(
- files_dir=os.path.dirname(io_scenario.__file__),
- rally_extra_opts=opts.runner_opts.split(" "),
- max_preparation_time=opts.max_preparation_time,
- keep_temp_files=opts.keep_temp_files)
- res = run_io_test(opts.tool_type,
- script_args,
- runner,
- opts.keep_temp_files)
- elif opts.runner == "ssh":
- user, key_file = opts.runner_opts.split(" ", 1)
-
- latest_start_time = opts.max_preparation_time + time.time()
-
- nova = nova_connect()
-
- # nova, amount, keypair_name, img_name,
- # flavor_name, vol_sz=None, network_zone_name=None,
- # flt_ip_pool=None, name_templ='ceph-test-{}',
- # scheduler_hints=None
-
- try:
- ips = [i[0] for i in create_vms_mt(nova, 3,
- keypair_name='ceph',
- img_name='ubuntu',
- flavor_name='ceph.512',
- network_zone_name='net04',
- flt_ip_pool='net04_ext')]
-
- uris = ["{0}@{1}::{2}".format(user, ip, key_file) for ip in ips]
-
- runner = ssh_runner.get_ssh_runner(uris,
- latest_start_time,
- opts.keep_temp_files)
- res = run_io_test(opts.tool_type,
- script_args,
- runner,
- opts.keep_temp_files)
- finally:
- clear_all(nova)
-
- print "=" * 80
- print pprint.pformat(res)
- print "=" * 80
-
+def print_measurements_stat(res):
if len(res) != 0:
bw_mean = 0.0
for measurement in res:
@@ -148,6 +94,126 @@
print
print "=" * 80
+
+def get_io_opts(io_opts_file, io_opts):
+ if io_opts_file is not None and io_opts is not None:
+ print "Options --io-opts-file and --io-opts can't be " + \
+ "provided same time"
+ exit(1)
+
+ if io_opts_file is None and io_opts is None:
+ print "Either --io-opts-file or --io-opts should " + \
+ "be provided"
+ exit(1)
+
+ if io_opts_file is not None:
+ io_opts = []
+
+ opt_lines = io_opts_file.readlines()
+ opt_lines = [i for i in opt_lines if i != "" and not i.startswith("#")]
+
+ for opt_line in opt_lines:
+ io_opts.append([opt.strip()
+ for opt in opt_line.split(" ")
+ if opt.strip() != ""])
+ else:
+ io_opts = [[opt.strip()
+ for opt in io_opts.split(" ")
+ if opt.strip() != ""]]
+
+ if len(io_opts) == 0:
+ print "Can't found parameters for io. Check" + \
+ "--io-opts-file or --io-opts options"
+ exit(1)
+
+ return io_opts
+
+
+def main(argv):
+ opts = parse_args(argv)
+
+ if not opts.extra_logs:
+ def nolog(x):
+ pass
+
+ setlogger(nolog)
+
+ io_opts = get_io_opts(opts.io_opts_file, opts.io_opts)
+
+ if opts.runner == "rally":
+ for script_args in io_opts:
+ runner = rally_runner.get_rally_runner(
+ files_dir=os.path.dirname(io_scenario.__file__),
+ rally_extra_opts=opts.runner_opts.split(" "),
+ max_preparation_time=opts.max_preparation_time,
+ keep_temp_files=opts.keep_temp_files)
+
+ res = run_io_test(opts.tool_type,
+ script_args,
+ runner,
+ opts.keep_temp_files)
+
+ print "=" * 80
+ print pprint.pformat(res)
+ print "=" * 80
+
+ print_measurements_stat(res)
+
+ elif opts.runner == "ssh":
+ create_vms_opts = {}
+ for opt in opts.runner_opts.split(","):
+ name, val = opt.split("=", 1)
+ create_vms_opts[name] = val
+
+ user = create_vms_opts.pop("user")
+ key_file = create_vms_opts.pop("key_file")
+ aff_group = create_vms_opts.pop("aff_group", None)
+ raw_count = create_vms_opts.pop("count", "x1")
+
+ if raw_count.startswith("x"):
+ raise NotImplementedError("xXXXX count not implemented yet")
+ else:
+ count = int(raw_count)
+
+ if aff_group is not None:
+ scheduler_hints = {'group': aff_group}
+ else:
+ scheduler_hints = None
+
+ create_vms_opts['scheduler_hints'] = scheduler_hints
+
+ latest_start_time = opts.max_preparation_time + time.time()
+
+ nova = nova_connect()
+
+ # nova, amount, keypair_name, img_name,
+ # flavor_name, vol_sz=None, network_zone_name=None,
+ # flt_ip_pool=None, name_templ='ceph-test-{}',
+ # scheduler_hints=None
+
+ try:
+ ips = [i[0] for i in create_vms_mt(nova, count, **create_vms_opts)]
+
+ uris = ["{0}@{1}::{2}".format(user, ip, key_file) for ip in ips]
+
+ for script_args in io_opts:
+ runner = ssh_runner.get_ssh_runner(uris,
+ latest_start_time,
+ opts.keep_temp_files)
+ res = run_io_test(opts.tool_type,
+ script_args,
+ runner,
+ opts.keep_temp_files)
+ print "=" * 80
+ print pprint.pformat(res)
+ print "=" * 80
+
+ print_measurements_stat(res)
+ except:
+ traceback.print_exc()
+ finally:
+ clear_all(nova)
+
return 0
diff --git a/scripts/data.py b/scripts/data.py
index dfe97a2..c53871d 100644
--- a/scripts/data.py
+++ b/scripts/data.py
@@ -16,7 +16,7 @@
for val in eval(block):
meta = val['__meta__']
meta['sync'] = 's' if meta['sync'] else 'a'
- key = "{action} {sync} {blocksize}k".format(**meta)
+ key = "{action} {sync} {blocksize}k {concurence}".format(**meta)
results.setdefault(key, []).append(val['bw_mean'])
processed_res = {}
@@ -31,8 +31,8 @@
def ksort(x):
- op, sync, sz = x.split(" ")
- return (op, sync, int(sz[:-1]))
+ op, sync, sz, conc = x.split(" ")
+ return (op, sync, int(sz[:-1]), int(conc))
def create_json_results(meta, file_data):
@@ -44,7 +44,7 @@
def show_data(*pathes):
- begin = "| {:>10} {:>5} {:>5}"
+ begin = "| {:>10} {:>5} {:>5} {:>3}"
first_file_templ = " | {:>6} ~ {:>5} {:>2}% {:>5}"
other_file_templ = " | {:>6} ~ {:>5} {:>2}% {:>5} ---- {:>6}%"
@@ -53,7 +53,7 @@
header_ln = line_templ.replace("<", "^").replace(">", "^")
- params = ["Oper", "Sync", "BSZ", "BW1", "DEV1", "%", "IOPS1"]
+ params = ["Oper", "Sync", "BSZ", "CC", "BW1", "DEV1", "%", "IOPS1"]
for pos in range(1, len(pathes)):
params += "BW{0}+DEV{0}+%+IOPS{0}+DIFF %".format(pos).split("+")
@@ -75,7 +75,7 @@
for k in sorted(common_keys, key=ksort):
tp = k.rsplit(" ", 1)[0]
- op, s, sz = k.split(" ")
+ op, s, sz, conc = k.split(" ")
s = 'sync' if s == 's' else 'async'
if tp != prev_tp and prev_tp is not None:
@@ -88,7 +88,7 @@
iops0 = m0 / int(sz[:-1])
perc0 = int(d0 * 100.0 / m0 + 0.5)
- data = [op, s, sz, m0, d0, perc0, iops0]
+ data = [op, s, sz, conc, m0, d0, perc0, iops0]
for result in results[1:]:
m, d = result[k]
diff --git a/scripts/run.sh b/scripts/run.sh
index b5bc41c..a5c8aaf 100644
--- a/scripts/run.sh
+++ b/scripts/run.sh
@@ -1,36 +1,48 @@
#!/bin/bash
-set -x
set -e
type="iozone"
-bsizes="1k 4k 64k 256k 1m"
-ops="write randwrite"
-osync="s a"
-num_times=3
+bsizes="1k" # 4k 64k 256k 1m
+ops="write" # randwrite"
+osync="s" # a
+num_times="1"
+concurrences="1 2 4 8 16 32 64 128"
+aff_group="0077d59c-bf5b-4326-8940-027e77d655ee"
-for bsize in $bsizes ; do
- for op in $ops ; do
- for sync in $osync ; do
- for counter in $(seq 1 $num_times) ; do
- if [[ "$ops" == "write" && "$osync" == "s" ]] ; then
- continue
- fi
+for concurrence in $concurrences; do
+ for bsize in $bsizes ; do
+ for op in $ops ; do
+ for sync in $osync ; do
+ for counter in $(seq 1 $num_times) ; do
+ # if [[ "$ops" == "write" && "$osync" == "s" ]] ; then
+ # continue
+ # fi
- if [[ "$sync" == "s" ]] ; then
- ssync="-s"
- factor="x500"
- else
- if [[ "$bsize" == "1k" || "$bsize" == "4k" ]] ; then
- continue
+ if [[ "$sync" == "s" ]] ; then
+ ssync="-s"
+ factor="x500"
+ else
+ if [[ "$bsize" == "1k" || "$bsize" == "4k" ]] ; then
+ continue
+ fi
+
+ ssync=
+ factor="r2"
fi
- ssync=
- factor="r2"
- fi
+ extra_opts="user=ubuntu,keypair_name=ceph,img_name=ubuntu,flavor_name=ceph.512"
+ extra_opts="${extra_opts},network_zone_name=net04,flt_ip_pool=net04_ext,key_file=ceph.pem"
+ extra_opts="${extra_opts},aff_group=${aff_group}"
- io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize $factor $ssync"
- python run_test.py --runner rally -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="--deployment $1"
+ io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize $factor $ssync --concurrency $concurrence"
+
+ echo $io_opts
+ # python run_test.py --runner ssh -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="$extra_opts"
+
+ # io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize $factor $ssync"
+ # python run_test.py --runner rally -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="--deployment $1"
+ done
done
done
done
diff --git a/scripts/run_2.sh b/scripts/run_2.sh
index 2265176..d343d69 100644
--- a/scripts/run_2.sh
+++ b/scripts/run_2.sh
@@ -24,10 +24,20 @@
# nova keypair-add ceph > ceph.pem
# fi
+# nova server-group-list | grep ' ceph ' | awk '{print $2}'
+aff_group="0077d59c-bf5b-4326-8940-027e77d655ee"
+
set -e
-io_opts="--type $type -a write --iodepth 16 --blocksize 1m --iosize x20"
-python run_test.py --runner ssh -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="ubuntu ceph.pem"
+iodepts="1"
+for iodepth in $iodepts; do
+ extra_opts="user=ubuntu,keypair_name=ceph,img_name=ubuntu,flavor_name=ceph.512"
+ extra_opts="${extra_opts},network_zone_name=net04,flt_ip_pool=net04_ext,key_file=ceph.pem"
+ extra_opts="${extra_opts},aff_group=${aff_group}"
+
+ io_opts="--type $type -a write --iodepth 16 --blocksize 1m --iosize x20"
+ python run_test.py --runner ssh -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="$extra_opts"
+done
# io_opts="--type $type -a write --iodepth 16 --blocksize 1m --iosize x20"
# python run_test.py --runner rally -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="--deployment perf-1"
diff --git a/ssh_runner.py b/ssh_runner.py
index 76fc6b5..3415382 100644
--- a/ssh_runner.py
+++ b/ssh_runner.py
@@ -6,7 +6,7 @@
from utils import ssh_connect
import itest
-from utils import get_barrier, log_error
+from utils import get_barrier, log_error, wait_on_barrier
conn_uri_attrs = ("user", "passwd", "host", "port", "path")
@@ -76,6 +76,8 @@
test_iter = itest.run_test_iter(obj, conn)
next(test_iter)
+ wait_on_barrier(barrier, latest_start_time)
+
with log_error("!Run test"):
return next(test_iter)
except:
@@ -111,6 +113,9 @@
while not result_queue.empty():
test_result.append(result_queue.get())
+ for conn in connections:
+ conn.close()
+
return test_result
return closure
diff --git a/starts_vms.py b/starts_vms.py
index 9a00b34..59ee694 100644
--- a/starts_vms.py
+++ b/starts_vms.py
@@ -49,14 +49,12 @@
def wait_for_server_active(nova, server, timeout=240):
t = time.time()
while True:
- time.sleep(5)
+ time.sleep(1)
sstate = getattr(server, 'OS-EXT-STS:vm_state').lower()
if sstate == 'active':
return True
- print "Curr state is", sstate, "waiting for active"
-
if sstate == 'error':
return False
@@ -130,7 +128,7 @@
nova.servers.delete(srv)
while True:
- print "wait till server deleted"
+ # print "wait till server deleted"
all_id = set(alive_srv.id for alive_srv in nova.servers.list())
if srv.id not in all_id:
break
@@ -139,16 +137,16 @@
break
if vol_sz is not None:
- print "creating volume"
+ # print "creating volume"
vol = create_volume(vol_sz, name)
- print "attach volume to server"
+ # print "attach volume to server"
nova.volumes.create_server_volume(srv.id, vol.id, None)
if flt_ip is Allocate:
flt_ip = nova.floating_ips.create(pool)
if flt_ip is not None:
- print "attaching ip to server"
+ # print "attaching ip to server"
srv.add_floating_ip(flt_ip)
return (flt_ip.ip, srv)
else:
diff --git a/utils.py b/utils.py
index 0a38ad5..5cc1ac2 100644
--- a/utils.py
+++ b/utils.py
@@ -29,7 +29,7 @@
return closure
-def ssh_connect(host, user, key_file, retry_count=10, timeout=5):
+def ssh_connect(host, user, key_file, retry_count=60, timeout=1):
ssh = paramiko.SSHClient()
ssh.load_host_keys('/dev/null')
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())