add directory reqursive copying
diff --git a/fix_rally.py b/fix_rally.py
index dc3a89a..e3b7bcf 100644
--- a/fix_rally.py
+++ b/fix_rally.py
@@ -5,6 +5,7 @@
import yaml
import json
import os.path
+import argparse
import datetime
import warnings
import functools
@@ -16,6 +17,8 @@
from rally.cmd.main import categories
from rally.benchmark.scenarios.vm.utils import VMScenario
+from ssh_copy_directory import put_dir_recursively, ssh_copy_file
+
def log(x):
dt_str = datetime.datetime.now().strftime("%H:%M:%S")
@@ -28,26 +31,25 @@
cond = multiprocessing.Condition()
def closure(timeout):
+ me_released = False
with cond:
val.value -= 1
-
- log("barrier value == {0}".format(val.value))
-
if val.value == 0:
+ me_released = True
cond.notify_all()
else:
cond.wait(timeout)
return val.value == 0
+ if me_released:
+ log("Test begins!")
+
return closure
-MAX_WAIT_TOUT = 60
-
-
@contextlib.contextmanager
def patch_VMScenario_run_command_over_ssh(paths,
- add_meta_cb,
+ on_result_cb,
barrier=None,
latest_start_time=None):
@@ -61,20 +63,30 @@
# rally code was changed
log("Prototype of VMScenario.run_command_over_ssh "
"was changed. Update patch code.")
- raise exceptions.ScriptError("monkeypath code fails on "
+ raise exceptions.ScriptError("monkeypatch code fails on "
"ssh._client.open_sftp()")
-
- for src, dst in paths.items():
- try:
- sftp.put(src, dst)
- except Exception as exc:
- tmpl = "Scp {0!r} => {1!r} failed - {2!r}"
- msg = tmpl.format(src, dst, exc)
- log(msg)
- raise exceptions.ScriptError(
- "monkeypath code fails on " + msg)
-
- sftp.close()
+ try:
+ for src, dst in paths.items():
+ try:
+ if os.path.isfile(src):
+ ssh_copy_file(sftp, src, dst)
+ elif os.path.isdir(src):
+ put_dir_recursively(sftp, src, dst)
+ else:
+ templ = "Can't copy {0!r} - " + \
+ "it neither file or directory"
+ msg = templ.format(src)
+ log(msg)
+ raise exceptions.ScriptError(msg)
+ except exceptions.ScriptError:
+ raise
+ except Exception as exc:
+ tmpl = "Scp {0!r} => {1!r} failed - {2!r}"
+ msg = tmpl.format(src, dst, exc)
+ log(msg)
+ raise exceptions.ScriptError(msg)
+ finally:
+ sftp.close()
log("Start io test")
@@ -94,7 +106,6 @@
try:
code, out, err = orig(self, ssh, *args, **kwargs)
- log("!!!!!!!!!!!!!!!!!!!! {} {} {}".format(code, out, err))
except Exception as exc:
log("Rally raises exception {0}".format(exc.message))
raise
@@ -111,7 +122,8 @@
pass
else:
if '__meta__' in result:
- add_meta_cb(result.pop('__meta__'))
+ on_result_cb(result)
+ result.pop('__meta__')
out = json.dumps(result)
except Exception as err:
log("Error during postprocessing results: {0!r}".format(err))
@@ -179,14 +191,14 @@
copy_files = {testtool_local: dst_testtool_path}
result_queue = multiprocessing.Queue()
- cb = result_queue.put
+ results_cb = result_queue.put
do_patch = patch_VMScenario_run_command_over_ssh
barrier = get_barrier(concurrency)
max_release_time = time.time() + max_preparation_time
- with do_patch(copy_files, cb, barrier, max_release_time):
+ with do_patch(copy_files, results_cb, barrier, max_release_time):
log("Start rally with 'task start {0}'".format(yaml_file))
rally_result = run_rally(['task', 'start', yaml_file])
@@ -198,23 +210,50 @@
finally:
os.unlink(yaml_file)
os.unlink(py_file)
- # store and process meta and results
+
+
+def parse_args(argv):
+ parser = argparse.ArgumentParser(
+ description="Run rally disk io performance test")
+ parser.add_argument("tool_type", help="test tool type",
+ choices=['iozone', 'fio'])
+ parser.add_argument("test_directory", help="directory with test")
+ parser.add_argument("-l", dest='extra_logs',
+ action='store_true', default=False,
+ help="print some extra log info")
+ parser.add_argument("-o", "--io-opts", dest='io_opts',
+ default=None, help="cmd line options for io.py")
+ return parser.parse_args(argv)
def main(argv):
- tool_type = argv[0]
- files_dir = argv[1]
-
+ opts = parse_args(argv)
dst_testtool_path = '/tmp/io_tool'
- testtool_py_argv = ['--type', tool_type,
- '-a', 'randwrite',
- '--iodepth', '2',
- '--blocksize', '4k',
- '--iosize', '20M',
- '--binary-path', dst_testtool_path,
- '-d']
- run_test(tool_type, testtool_py_argv, dst_testtool_path, files_dir)
+ if not opts.extra_logs:
+ global log
+
+ def nolog(x):
+ pass
+
+ log = nolog
+
+ if opts.io_opts is None:
+ testtool_py_argv = ['--type', opts.tool_type,
+ '-a', 'randwrite',
+ '--iodepth', '2',
+ '--blocksize', '4k',
+ '--iosize', '20M',
+ '--binary-path', dst_testtool_path,
+ '-d']
+ else:
+ testtool_py_argv = opts.io_opts.split(" ")
+
+ run_test(opts.tool_type,
+ testtool_py_argv,
+ dst_testtool_path,
+ opts.test_directory)
+ return 0
# ubuntu cloud image
# https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
diff --git a/ssh_copy_directory.py b/ssh_copy_directory.py
new file mode 100644
index 0000000..dca0843
--- /dev/null
+++ b/ssh_copy_directory.py
@@ -0,0 +1,68 @@
+import os.path
+
+
+def normalize_dirpath(dirpath):
+ while dirpath.endswith("/"):
+ dirpath = dirpath[:-1]
+ return dirpath
+
+
+def ssh_mkdir(sftp, remotepath, mode=0777, intermediate=False):
+ remotepath = normalize_dirpath(remotepath)
+ if intermediate:
+ try:
+ sftp.mkdir(remotepath, mode=mode)
+ except IOError:
+ ssh_mkdir(sftp, remotepath.rsplit("/", 1)[0], mode=mode,
+ intermediate=True)
+ return sftp.mkdir(remotepath, mode=mode)
+ else:
+ sftp.mkdir(remotepath, mode=mode)
+
+
+def ssh_copy_file(sftp, localfile, remfile, preserve_perm=True):
+ sftp.put(localfile, remfile)
+ if preserve_perm:
+ sftp.chmod(remfile, os.stat(localfile).st_mode & 0777)
+
+
+def put_dir_recursively(sftp, localpath, remotepath, preserve_perm=True):
+ "upload local directory to remote recursively"
+
+ assert remotepath.startswith("/"), "%s must be absolute path" % remotepath
+
+ # normalize
+ localpath = normalize_dirpath(localpath)
+ remotepath = normalize_dirpath(remotepath)
+
+ try:
+ sftp.chdir(remotepath)
+ localsuffix = localpath.rsplit("/", 1)[1]
+ remotesuffix = remotepath.rsplit("/", 1)[1]
+ if localsuffix != remotesuffix:
+ remotepath = os.path.join(remotepath, localsuffix)
+ except IOError:
+ pass
+
+ for root, dirs, fls in os.walk(localpath):
+ prefix = os.path.commonprefix([localpath, root])
+ suffix = root.split(prefix, 1)[1]
+ if suffix.startswith("/"):
+ suffix = suffix[1:]
+
+ remroot = os.path.join(remotepath, suffix)
+
+ try:
+ sftp.chdir(remroot)
+ except IOError:
+ if preserve_perm:
+ mode = os.stat(root).st_mode & 0777
+ else:
+ mode = 0777
+ ssh_mkdir(sftp, remroot, mode=mode, intermediate=True)
+ sftp.chdir(remroot)
+
+ for f in fls:
+ remfile = os.path.join(remroot, f)
+ localfile = os.path.join(root, f)
+ ssh_copy_file(sftp, localfile, remfile, preserve_perm)