Add stage base class, refactor discovery, etc
diff --git a/tests/test_agent.py b/tests/test_fio_parser.py
similarity index 83%
rename from tests/test_agent.py
rename to tests/test_fio_parser.py
index f9fa389..b496190 100644
--- a/tests/test_agent.py
+++ b/tests/test_fio_parser.py
@@ -5,7 +5,7 @@
from oktest import ok, main, test
-from wally.suits.io import agent
+from wally.suits.io import fio_task_parser
code_test_defaults = """
[defaults]
@@ -94,21 +94,18 @@
"""
-P = agent.parse_all_in_1
-
-
class AgentTest(unittest.TestCase):
@test("test_parse_value")
def test_parse_value(self):
x = "asdfasd adsd d"
- ok(agent.parse_value(x)) == x
- ok(agent.parse_value("10 2")) == "10 2"
- ok(agent.parse_value(None)).is_(None)
- ok(agent.parse_value("10")) == 10
- ok(agent.parse_value("20")) == 20
- ok(agent.parse_value("10.1") - 10.1) < 1E-7
- ok(agent.parse_value("{% 10, 20 %}")) == [10, 20]
- ok(agent.parse_value("{% 10,20 %}")) == [10, 20]
+ ok(fio_task_parser.parse_value(x)) == x
+ ok(fio_task_parser.parse_value("10 2")) == "10 2"
+ ok(fio_task_parser.parse_value("None")).is_(None)
+ ok(fio_task_parser.parse_value("10")) == 10
+ ok(fio_task_parser.parse_value("20")) == 20
+ ok(fio_task_parser.parse_value("10.1") - 10.1) < 1E-7
+ ok(fio_task_parser.parse_value("{% 10, 20 %}")) == [10, 20]
+ ok(fio_task_parser.parse_value("{% 10,20 %}")) == [10, 20]
code_test_compile_simplest = defaults + """
[sec1]
@@ -117,7 +114,7 @@
@test("test_compile_simplest")
def test_compile_simplest(self):
- sections = P(self.code_test_compile_simplest, {})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_compile_simplest, {})
sections = list(sections)
ok(len(sections)) == 1
@@ -138,7 +135,7 @@
@test("test_compile_defaults")
def test_compile_defaults(self):
- sections = P(self.code_test_params_in_defaults, {})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_params_in_defaults, {})
sections = list(sections)
ok(len(sections)) == 1
@@ -151,7 +148,7 @@
@test("test_defaults")
def test_defaults(self):
- sections = P(code_test_defaults, {})
+ sections = fio_task_parser.parse_all_in_1(code_test_defaults, {})
sections = list(sections)
ok(len(sections)) == 2
@@ -177,11 +174,10 @@
@test("test_external_params")
def test_external_params(self):
with self.assertRaises(KeyError):
- sections = P(self.code_test_ext_params, {})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_ext_params, {})
list(sections)
- sections = P(self.code_test_ext_params,
- {'RUNTIME': 20})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_ext_params, {'RUNTIME': 20})
sections = list(sections)
code_test_cycle = defaults + """
@@ -192,8 +188,7 @@
@test("test_cycle")
def test_cycle(self):
- sections = P(self.code_test_cycle,
- {'RUNTIME': 20})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_cycle, {'RUNTIME': 20})
sections = list(sections)
ok(len(sections)) == 2
ok(sections[0].vals['ramp_time']) == 20
@@ -208,8 +203,7 @@
@test("test_cycles")
def test_cycles(self):
- sections = P(self.code_test_cycles,
- {'RUNTIME': 20})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_cycles, {'RUNTIME': 20})
sections = list(sections)
ok(len(sections)) == 4
@@ -224,13 +218,12 @@
@test("test_time_estimate")
def test_time_estimate(self):
- sections = P(self.code_test_cycles,
- {'RUNTIME': 20})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_cycles, {'RUNTIME': 20})
sections = list(sections)
- etime = agent.calculate_execution_time(sections)
+ etime = fio_task_parser.calculate_execution_time(sections)
ok(etime) == 20 * 4 + 20 * 2 + 40 * 2
- ok(agent.sec_to_str(etime)) == "0:03:20"
+ ok(fio_task_parser.sec_to_str(etime)) == "0:03:20"
code_test_cycles2 = defaults + """
[sec1 * 7]
@@ -241,14 +234,13 @@
@test("test_time_estimate")
def test_time_estimate_large(self):
- sections = P(self.code_test_cycles2,
- {'RUNTIME': 30})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_cycles2, {'RUNTIME': 30})
sections = list(sections)
ok(sections[0].name) == 'sec1'
ok(len(sections)) == 7 * 4
- etime = agent.calculate_execution_time(sections)
+ etime = fio_task_parser.calculate_execution_time(sections)
# ramptime optimization
expected_time = (20 + 30 + 30 * 6) * 2
expected_time += (40 + 30 + 30 * 6) * 2
@@ -268,14 +260,14 @@
@test("test_time_estimate2")
def test_time_estimate_large2(self):
- sections = P(self.code_test_cycles3, {'RUNTIME': 30})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_cycles3, {'RUNTIME': 30})
sections = list(sections)
ok(sections[0].name) == 'sec1'
ok(sections[1].name) == 'sec1'
ok(len(sections)) == 7 * 4 * 2
- etime = agent.calculate_execution_time(sections)
+ etime = fio_task_parser.calculate_execution_time(sections)
# ramptime optimization
expected_time = (20 + 30 + 30 * 6) * 2
expected_time += (40 + 30 + 30 * 6) * 2
@@ -290,7 +282,7 @@
@test("test_repeat")
def test_repeat(self):
- sections = P(self.code_test_repeats, {})
+ sections = fio_task_parser.parse_all_in_1(self.code_test_repeats, {})
sections = list(sections)
ok(len(sections)) == 2 + 3
ok(sections[0].name) == 'sec1'
@@ -301,7 +293,7 @@
@test("test_real_tasks")
def test_real_tasks(self):
- tasks_dir = os.path.dirname(agent.__file__)
+ tasks_dir = os.path.dirname(fio_task_parser.__file__)
fname = os.path.join(tasks_dir, 'io_scenario_ceph.cfg')
fc = open(fname).read()
@@ -310,7 +302,7 @@
ok(len(sections)) == 7 * 9 * 4 + 7
- etime = agent.calculate_execution_time(sections)
+ etime = fio_task_parser.calculate_execution_time(sections)
# ramptime optimization
expected_time = (60 * 7 + 30) * 9 * 4 + (60 * 7 + 30)
ok(etime) == expected_time
diff --git a/tests/test_ssh.py b/tests/test_ssh.py
new file mode 100644
index 0000000..efc5f09
--- /dev/null
+++ b/tests/test_ssh.py
@@ -0,0 +1,33 @@
+import getpass
+
+from oktest import ok
+
+from wally import ssh_utils, ssh
+
+
+creds = "root@osd-0"
+
+
+def test_ssh_url_parser():
+ curr_user = getpass.getuser()
+ creds = {
+ "test": ssh_utils.ConnCreds("test", curr_user, port=23),
+ "test:13": ssh_utils.ConnCreds("test", curr_user, port=13),
+ "test::xxx.key": ssh_utils.ConnCreds("test", curr_user, port=23, key_file="xxx.key"),
+ "test:123:xxx.key": ssh_utils.ConnCreds("test", curr_user, port=123, key_file="xxx.key"),
+ "user@test": ssh_utils.ConnCreds("test", "user", port=23),
+ "user@test:13": ssh_utils.ConnCreds("test", "user", port=13),
+ "user@test::xxx.key": ssh_utils.ConnCreds("test", "user", port=23, key_file="xxx.key"),
+ "user@test:123:xxx.key": ssh_utils.ConnCreds("test", "user", port=123, key_file="xxx.key"),
+ "user:passwd:@test": ssh_utils.ConnCreds("test", curr_user, port=23, passwd="passwd:"),
+ "user:passwd:@test:123": ssh_utils.ConnCreds("test", curr_user, port=123, passwd="passwd:"),
+ }
+
+ for uri, expected in creds.items():
+ parsed = ssh_utils.parse_ssh_uri(uri)
+ ok(parsed.user) == expected.user
+ ok(parsed.addr.port) == expected.addr.port
+ ok(parsed.addr.host) == expected.addr.host
+ ok(parsed.key_file) == expected.key_file
+ ok(parsed.passwd) == expected.passwd
+
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 46f38e6..8fd8f4d 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -1,9 +1,11 @@
+import array
import shutil
import tempfile
import contextlib
-from oktest import ok, main, test
+import pytest
+from oktest import ok
from wally.storage import make_storage
@@ -35,19 +37,162 @@
for path, val in values.items():
storage[path] = val
+ with make_storage(root, existing=True) as storage:
+ for path, val in values.items():
+ ok(storage[path]) == val
+ ok(storage.get(path)) == val
+
+
+def test_path_list():
+ values = {
+ "int": 1,
+ "str/1": "test",
+ "bytes/2": b"test",
+ "none/s/1": None,
+ "bool/xx/1/2/1": None,
+ "float/s/1": 1.234,
+ "list": [1, 2, "3"],
+ "dict": {1: 3, "2": "4", "1.2": 1.3}
+ }
+
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ for path, val in values.items():
+ storage[path.split('/')] = val
with make_storage(root, existing=True) as storage:
for path, val in values.items():
- ok(storage[path]) == val
+ ok(storage[path.split('/')]) == val
+ ok(storage.get(path.split('/'))) == val
+
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ for path, val in values.items():
+ storage[path] = val
+
+ with make_storage(root, existing=True) as storage:
+ for path, val in values.items():
+ ok(storage[path.split('/')]) == val
+ ok(storage.get(path.split('/'))) == val
-def test_large_arrays():
- pass
+def test_list():
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ storage["x/some_path1"] = "1"
+ storage["x/some_path2"] = [1, 2, 3]
+ storage["x/some_path3"] = [1, 2, 3, 4]
+
+ storage["x/y/some_path11"] = "1"
+ storage["x/y/some_path22"] = [1, 2, 3]
+
+ with make_storage(root, existing=True) as storage:
+ assert 'x' in storage
+ assert 'x/y' in storage
+
+ assert {(False, 'x')} == set(storage.list())
+
+ assert {(True, 'some_path1'),
+ (True, 'some_path2'),
+ (True, 'some_path3'),
+ (False, "y")} == set(storage.list("x"))
+
+ assert {(True, 'some_path11'), (True, 'some_path22')} == set(storage.list("x/y"))
-def test_array_append():
- pass
+def test_overwrite():
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ storage["some_path"] = "1"
+ storage["some_path"] = [1, 2, 3]
+
+ with make_storage(root, existing=True) as storage:
+ assert storage["some_path"] == [1, 2, 3]
-def test_performance():
- pass
+def test_multy_level():
+ with in_temp_dir() as root:
+ values = {
+ "dict1": {1: {3: 4, 6: [12, {123, 3}, {4: 3}]}, "2": "4", "1.2": 1.3}
+ }
+
+ with make_storage(root, existing=False) as storage:
+ for path, val in values.items():
+ storage[path] = val
+
+ with make_storage(root, existing=True) as storage:
+ for path, val in values.items():
+ ok(storage[path]) == val
+
+
+def test_arrays():
+ with in_temp_dir() as root:
+ val_l = list(range(10000)) * 10
+ val_i = array.array("i", val_l)
+ val_f = array.array("f", map(float, val_l))
+ val_2f = val_f + val_f
+
+ with make_storage(root, existing=False) as storage:
+ storage.set_array(val_i, "array_i")
+ storage.set_array(val_f, "array_f")
+ storage.set_array(val_f, "array_x2")
+ storage.append(val_f, "array_x2")
+
+ with make_storage(root, existing=True) as storage:
+ ok(val_i) == storage.get_array("i", "array_i")
+ ok(val_f) == storage.get_array("f", "array_f")
+ ok(val_2f) == storage.get_array("f", "array_x2")
+
+
+class LoadMe:
+ def __init__(self, **vals):
+ self.__dict__.update(vals)
+
+
+def test_load_user_obj():
+ obj = LoadMe(x=1, y=12, z=[1,2,3], t="asdad", gg={"a": 1, "g": [["x"]]})
+
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ storage["obj"] = obj
+
+ with make_storage(root, existing=True) as storage:
+ obj2 = storage.load(LoadMe, "obj")
+ assert isinstance(obj2, LoadMe)
+ ok(obj2.__dict__) == obj.__dict__
+
+
+def test_path_not_exists():
+ with in_temp_dir() as root:
+ pass
+
+ with pytest.raises(IOError):
+ with make_storage(root, existing=True) as storage:
+ pass
+
+ with in_temp_dir() as root:
+ pass
+
+ with make_storage(root, existing=False) as storage:
+ with pytest.raises(IOError):
+ storage["x"]
+
+
+def test_incorrect_user_object():
+ obj = LoadMe(x=1, y=LoadMe(t=12))
+
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ with pytest.raises(ValueError):
+ storage["obj"] = obj
+
+
+def test_substorage():
+ with in_temp_dir() as root:
+ with make_storage(root, existing=False) as storage:
+ storage["x/y"] = "data"
+ storage.sub_storage("t")["r"] = "sub_data"
+
+ with make_storage(root, existing=True) as storage:
+ ok(storage["t/r"]) == "sub_data"
+ ok(storage.sub_storage("x")["y"]) == "data"