add directory reqursive copying
diff --git a/fix_rally.py b/fix_rally.py
index dc3a89a..e3b7bcf 100644
--- a/fix_rally.py
+++ b/fix_rally.py
@@ -5,6 +5,7 @@
import yaml
import json
import os.path
+import argparse
import datetime
import warnings
import functools
@@ -16,6 +17,8 @@
from rally.cmd.main import categories
from rally.benchmark.scenarios.vm.utils import VMScenario
+from ssh_copy_directory import put_dir_recursively, ssh_copy_file
+
def log(x):
dt_str = datetime.datetime.now().strftime("%H:%M:%S")
@@ -28,26 +31,25 @@
cond = multiprocessing.Condition()
def closure(timeout):
+ me_released = False
with cond:
val.value -= 1
-
- log("barrier value == {0}".format(val.value))
-
if val.value == 0:
+ me_released = True
cond.notify_all()
else:
cond.wait(timeout)
return val.value == 0
+ if me_released:
+ log("Test begins!")
+
return closure
-MAX_WAIT_TOUT = 60
-
-
@contextlib.contextmanager
def patch_VMScenario_run_command_over_ssh(paths,
- add_meta_cb,
+ on_result_cb,
barrier=None,
latest_start_time=None):
@@ -61,20 +63,30 @@
# rally code was changed
log("Prototype of VMScenario.run_command_over_ssh "
"was changed. Update patch code.")
- raise exceptions.ScriptError("monkeypath code fails on "
+ raise exceptions.ScriptError("monkeypatch code fails on "
"ssh._client.open_sftp()")
-
- for src, dst in paths.items():
- try:
- sftp.put(src, dst)
- except Exception as exc:
- tmpl = "Scp {0!r} => {1!r} failed - {2!r}"
- msg = tmpl.format(src, dst, exc)
- log(msg)
- raise exceptions.ScriptError(
- "monkeypath code fails on " + msg)
-
- sftp.close()
+ try:
+ for src, dst in paths.items():
+ try:
+ if os.path.isfile(src):
+ ssh_copy_file(sftp, src, dst)
+ elif os.path.isdir(src):
+ put_dir_recursively(sftp, src, dst)
+ else:
+ templ = "Can't copy {0!r} - " + \
+ "it neither file or directory"
+ msg = templ.format(src)
+ log(msg)
+ raise exceptions.ScriptError(msg)
+ except exceptions.ScriptError:
+ raise
+ except Exception as exc:
+ tmpl = "Scp {0!r} => {1!r} failed - {2!r}"
+ msg = tmpl.format(src, dst, exc)
+ log(msg)
+ raise exceptions.ScriptError(msg)
+ finally:
+ sftp.close()
log("Start io test")
@@ -94,7 +106,6 @@
try:
code, out, err = orig(self, ssh, *args, **kwargs)
- log("!!!!!!!!!!!!!!!!!!!! {} {} {}".format(code, out, err))
except Exception as exc:
log("Rally raises exception {0}".format(exc.message))
raise
@@ -111,7 +122,8 @@
pass
else:
if '__meta__' in result:
- add_meta_cb(result.pop('__meta__'))
+ on_result_cb(result)
+ result.pop('__meta__')
out = json.dumps(result)
except Exception as err:
log("Error during postprocessing results: {0!r}".format(err))
@@ -179,14 +191,14 @@
copy_files = {testtool_local: dst_testtool_path}
result_queue = multiprocessing.Queue()
- cb = result_queue.put
+ results_cb = result_queue.put
do_patch = patch_VMScenario_run_command_over_ssh
barrier = get_barrier(concurrency)
max_release_time = time.time() + max_preparation_time
- with do_patch(copy_files, cb, barrier, max_release_time):
+ with do_patch(copy_files, results_cb, barrier, max_release_time):
log("Start rally with 'task start {0}'".format(yaml_file))
rally_result = run_rally(['task', 'start', yaml_file])
@@ -198,23 +210,50 @@
finally:
os.unlink(yaml_file)
os.unlink(py_file)
- # store and process meta and results
+
+
+def parse_args(argv):
+ parser = argparse.ArgumentParser(
+ description="Run rally disk io performance test")
+ parser.add_argument("tool_type", help="test tool type",
+ choices=['iozone', 'fio'])
+ parser.add_argument("test_directory", help="directory with test")
+ parser.add_argument("-l", dest='extra_logs',
+ action='store_true', default=False,
+ help="print some extra log info")
+ parser.add_argument("-o", "--io-opts", dest='io_opts',
+ default=None, help="cmd line options for io.py")
+ return parser.parse_args(argv)
def main(argv):
- tool_type = argv[0]
- files_dir = argv[1]
-
+ opts = parse_args(argv)
dst_testtool_path = '/tmp/io_tool'
- testtool_py_argv = ['--type', tool_type,
- '-a', 'randwrite',
- '--iodepth', '2',
- '--blocksize', '4k',
- '--iosize', '20M',
- '--binary-path', dst_testtool_path,
- '-d']
- run_test(tool_type, testtool_py_argv, dst_testtool_path, files_dir)
+ if not opts.extra_logs:
+ global log
+
+ def nolog(x):
+ pass
+
+ log = nolog
+
+ if opts.io_opts is None:
+ testtool_py_argv = ['--type', opts.tool_type,
+ '-a', 'randwrite',
+ '--iodepth', '2',
+ '--blocksize', '4k',
+ '--iosize', '20M',
+ '--binary-path', dst_testtool_path,
+ '-d']
+ else:
+ testtool_py_argv = opts.io_opts.split(" ")
+
+ run_test(opts.tool_type,
+ testtool_py_argv,
+ dst_testtool_path,
+ opts.test_directory)
+ return 0
# ubuntu cloud image
# https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img