refactor result classes and code which stores/loads results from storage
diff --git a/wally/common_types.py b/wally/common_types.py
index ef809ee..ce497ad 100644
--- a/wally/common_types.py
+++ b/wally/common_types.py
@@ -1,5 +1,35 @@
-import abc
from typing import NamedTuple, Dict, Any
+from .istorable import IStorable
+
IP = str
IPAddr = NamedTuple("IPAddr", [("host", IP), ("port", int)])
+
+
+class ConnCreds(IStorable):
+ def __init__(self, host: str, user: str, passwd: str = None, port: str = '22',
+ key_file: str = None, key: bytes = None) -> None:
+ self.user = user
+ self.passwd = passwd
+ self.addr = IPAddr(host, int(port))
+ self.key_file = key_file
+ self.key = key
+
+ def __str__(self) -> str:
+ return "{}@{}:{}".format(self.user, self.addr.host, self.addr.port)
+
+ def __repr__(self) -> str:
+ return str(self)
+
+ def raw(self) -> Dict[str, Any]:
+ return {
+ 'user': self.user,
+ 'host': self.addr.host,
+ 'port': self.addr.port,
+ 'passwd': self.passwd,
+ 'key_file': self.key_file
+ }
+
+ @classmethod
+ def fromraw(cls, data: Dict[str, Any]) -> 'ConnCreds':
+ return cls(**data)
diff --git a/wally/istorable.py b/wally/istorable.py
new file mode 100644
index 0000000..467f901
--- /dev/null
+++ b/wally/istorable.py
@@ -0,0 +1,31 @@
+import abc
+from typing import Any, Union, List, Dict
+
+
+class IStorable(metaclass=abc.ABCMeta):
+ """Interface for type, which can be stored"""
+
+ @abc.abstractmethod
+ def raw(self) -> Dict[str, Any]:
+ pass
+
+ @abc.abstractclassmethod
+ def fromraw(cls, data: Dict[str, Any]) -> 'IStorable':
+ pass
+
+
+class Storable(IStorable):
+ """Default implementation"""
+
+ def raw(self) -> Dict[str, Any]:
+ return self.__dict__
+
+ @classmethod
+ def fromraw(cls, data: Dict[str, Any]) -> 'IStorable':
+ obj = cls.__new__(cls)
+ obj.__dict__.update(data)
+ return obj
+
+
+Basic = Union[int, str, bytes, bool, None]
+StorableType = Union[IStorable, Dict[str, Any], List[Any], int, str, bytes, bool, None]
diff --git a/wally/main.py b/wally/main.py
index 0de6791..0553b4e 100644
--- a/wally/main.py
+++ b/wally/main.py
@@ -2,6 +2,7 @@
import time
import signal
import pprint
+import getpass
import logging
import argparse
import functools
@@ -98,6 +99,7 @@
parser = argparse.ArgumentParser(prog='wally', description=descr)
parser.add_argument("-l", '--log-level', help="print some extra log info")
parser.add_argument("--ssh-key-passwd", default=None, help="Pass ssh key password")
+ parser.add_argument("--ssh-key-passwd-kbd", action="store_true", help="Enter ssh key password interactivelly")
parser.add_argument("-s", '--settings-dir', default=None,
help="Folder to store key/settings/history files")
@@ -320,6 +322,8 @@
if opts.ssh_key_passwd is not None:
set_ssh_key_passwd(opts.ssh_key_passwd)
+ elif opts.ssh_key_passwd_kbd:
+ set_ssh_key_passwd(getpass.getpass("Ssh key password: ").strip())
stages.sort(key=lambda x: x.priority)
diff --git a/wally/node_interfaces.py b/wally/node_interfaces.py
index bc07bb3..71efc54 100644
--- a/wally/node_interfaces.py
+++ b/wally/node_interfaces.py
@@ -2,7 +2,7 @@
from typing import Any, Set, Dict, NamedTuple, Optional
from .ssh_utils import ConnCreds
from .common_types import IPAddr
-from .result_classes import IStorable
+from .istorable import IStorable
RPCCreds = NamedTuple("RPCCreds", [("addr", IPAddr), ("key_file", str), ("cert_file", str)])
diff --git a/wally/process_results.py b/wally/process_results.py
index 0e8cb1c..5ca53af 100644
--- a/wally/process_results.py
+++ b/wally/process_results.py
@@ -1,53 +1,58 @@
# put all result preprocessing here
# selection, aggregation
+import logging
+
+
from .stage import Stage, StepOrder
from .test_run_class import TestRun
-from .statistic import calc_norm_stat_props, NormStatProps
-from .result_classes import NormStatProps
+from .statistic import calc_norm_stat_props, calc_histo_stat_props
+from .result_classes import TestJobConfig
+from .suits.itest import ResultStorage
+from .suits.io.fio_hist import get_lat_vals, expected_lat_bins
+from .utils import StopTestError
+
+logger = logging.getLogger("wally")
+
+import matplotlib
+
+# have to be before pyplot import to avoid tkinter(default graph frontend) import error
+matplotlib.use('svg')
+
+import matplotlib.pyplot as plt
+
class CalcStatisticStage(Stage):
priority = StepOrder.TEST + 1
def run(self, ctx: TestRun) -> None:
- results = {}
+ rstorage = ResultStorage(ctx.storage, TestJobConfig)
- for is_file, name in ctx.storage.list("result"):
- if is_file:
+ for suite_cfg, path in rstorage.list_suites():
+ if suite_cfg.test_type != 'fio':
continue
- path = "result/{}".format(name)
- info = ctx.storage.get("result/{}/info".format(name))
+ for job_cfg, path, _ in rstorage.list_jobs_in_suite(path):
+ results = {}
+ for node_id, dev, sensor_name in rstorage.list_ts_in_job(path):
+ ts = rstorage.load_ts(path, node_id, dev, sensor_name)
+ if dev == 'fio' and sensor_name == 'lat':
+ if ts.second_axis_size != expected_lat_bins:
+ logger.error("Sensor %s.%s on node %s has" +
+ "second_axis_size=%s. Can only process sensors with second_axis_size=%s.",
+ dev, sensor_name, node_id, ts.second_axis_size, expected_lat_bins)
+ continue
+ ts.bins_edges = get_lat_vals(ts.second_axis_size)
+ stat_prop = calc_histo_stat_props(ts)
- if info['test'] == 'fio':
- for node in info['nodes']:
- data_path = "{}/measurement/{}".format(path, node)
-
- iops = ctx.storage.get_array('Q', data_path, 'iops_data')
- iops_stat_path = "{}/iops_stat".format(data_path)
- if iops_stat_path in ctx.storage:
- iops_stat= ctx.storage.load(NormStatProps, iops_stat_path)
+ elif ts.second_axis_size != 1:
+ logger.warning("Sensor %s.%s on node %s provide 2D data with " +
+ "ts.second_axis_size=%s. Can't process it.",
+ dev, sensor_name, node_id, ts.second_axis_size)
+ continue
else:
- iops_stat = calc_norm_stat_props(iops)
- ctx.storage.put(iops_stat, iops_stat_path)
+ stat_prop = calc_norm_stat_props(ts)
- bw = ctx.storage.get_array('Q', data_path, 'bw_data')
- bw_stat_path = "{}/bw_stat".format(data_path)
- if bw_stat_path in ctx.storage:
- bw_stat = ctx.storage.load(NormStatProps, bw_stat_path)
- else:
- bw_stat = calc_norm_stat_props(bw)
- ctx.storage.put(bw_stat, bw_stat_path)
+ results[(node_id, dev, sensor_name)] = stat_prop
- lat = ctx.storage.get_array('L', data_path, 'lat_data')
- lat_stat = None
-
- results[name] = (iops, iops_stat, bw, bw_stat, lat, lat_stat)
-
- for name, (iops, iops_stat, bw, bw_stat, lat, lat_stat) in results.items():
- print(" ------------------- IOPS -------------------")
- print(iops_stat) # type: ignore
- print(" ------------------- BW -------------------")
- print(bw_stat) # type: ignore
- # print(" ------------------- LAT -------------------")
- # print(calc_stat_props(lat))
+ raise StopTestError()
diff --git a/wally/report.py b/wally/report.py
index 58dcf56..cf1289b 100644
--- a/wally/report.py
+++ b/wally/report.py
@@ -5,10 +5,13 @@
import numpy
import scipy
import matplotlib
+
+# have to be before pyplot import to avoid tkinter(default graph frontend) import error
+matplotlib.use('svg')
+
import matplotlib.pyplot as plt
-matplotlib.use('svg')
from .utils import ssize2b
diff --git a/wally/result_classes.py b/wally/result_classes.py
index e306525..3616f47 100644
--- a/wally/result_classes.py
+++ b/wally/result_classes.py
@@ -1,92 +1,102 @@
-import abc
import array
-from typing import Dict, List, Any, Optional
+from typing import Dict, List, Any, Optional, Tuple, cast
+
import numpy
-from scipy import stats
+from scipy.stats.mstats_basic import NormaltestResult
-from .utils import IStorable, Number, round_digits
+from .node_interfaces import IRPCNode
+from .istorable import IStorable, Storable
+from .utils import round_digits, Number
-class TimeSerie:
- name = None # type: str
- start_at = None # type: int
- step = None # type: int
- data = None # type: List[int]
- second_axis_size = None # type: int
- raw = None # type: Optional[bytes]
+class TestJobConfig(Storable):
+ def __init__(self) -> None:
+ self.summary = None # type: str
- def __init__(self, name: str, raw: Optional[bytes], second_axis_size: int,
- start_at: int, step: int, data: array.array) -> None:
+
+class TestSuiteConfig(IStorable):
+ """
+ Test suite input configuration.
+
+ test_type - test type name
+ params - parameters from yaml file for this test
+ run_uuid - UUID to be used to create file names & Co
+ nodes - nodes to run tests on
+ remote_dir - directory on nodes to be used for local files
+ """
+ def __init__(self,
+ test_type: str,
+ params: Dict[str, Any],
+ run_uuid: str,
+ nodes: List[IRPCNode],
+ remote_dir: str) -> None:
+ self.test_type = test_type
+ self.params = params
+ self.run_uuid = run_uuid
+ self.nodes = nodes
+ self.nodes_ids = [node.info.node_id() for node in nodes]
+ self.remote_dir = remote_dir
+
+ def __eq__(self, other: 'TestSuiteConfig') -> bool:
+ return (self.test_type == other.test_type and
+ self.params == other.params and
+ set(self.nodes_ids) == set(other.nodes_ids))
+
+ def raw(self) -> Dict[str, Any]:
+ res = self.__dict__.copy()
+ del res['nodes']
+ del res['run_uuid']
+ del res['remote_dir']
+ return res
+
+ @classmethod
+ def fromraw(cls, data: Dict[str, Any]) -> 'IStorable':
+ obj = cls.__new__(cls)
+ data = data.copy()
+ data['nodes'] = None
+ data['run_uuid'] = None
+ data['remote_dir'] = None
+ obj.__dict__.update(data)
+ return obj
+
+
+class TimeSeries:
+ """Data series from sensor - either system sensor or from load generator tool (e.g. fio)"""
+
+ def __init__(self,
+ name: str,
+ raw: Optional[bytes],
+ data: array.array,
+ times: array.array,
+ second_axis_size: int = 1,
+ bins_edges: List[float] = None) -> None:
+
+ # Sensor name. Typically DEV_NAME.METRIC
self.name = name
- self.start_at = start_at
- self.step = step
+
+ # Time series times and values. Time in ms from Unix epoch.
+ self.times = times # type: List[int]
+ self.data = data # type: List[int]
+
+ # Not equal to 1 in case of 2d sensors, like latency, when each measurement is a histogram.
self.second_axis_size = second_axis_size
- self.data = data # type: ignore
+
+ # Raw sensor data (is provided). Like log file for fio iops/bw/lat.
self.raw = raw
- def meta(self) -> Dict[str, Any]:
- return {
- "start_at": self.start_at,
- "step": self.step,
- "second_axis_size": self.second_axis_size
- }
+ # bin edges for historgam timeseries
+ self.bins_edges = bins_edges
-class SensorInfo:
- """Holds information from a single sensor from a single node"""
- node_id = None # type: str
- source_id = None # type: str
- sensor_name = None # type: str
- begin_time = None # type: int
- end_time = None # type: int
- data = None # type: List[int]
-
- def __init__(self, node_id: str, source_id: str, sensor_name: str) -> None:
- self.node_id = node_id
- self.source_id = source_id
- self.sensor_name = sensor_name
+# (node_name, source_dev, metric_name) => metric_results
+JobMetrics = Dict[Tuple[str, str, str], TimeSeries]
-class TestInfo:
- """Contains done test information"""
- name = None # type: str
- iteration_name = None # type: str
- nodes = None # type: List[str]
- start_time = None # type: int
- stop_time = None # type: int
- params = None # type: Dict[str, Any]
- config = None # type: str
- node_ids = None # type: List[str]
-
-
-class NodeTestResults:
- name = None # type: str
- node_id = None # type: str
- summary = None # type: str
-
- load_start_at = None # type: int
- load_stop_at = None # type: int
-
- series = None # type: Dict[str, TimeSerie]
-
- def __init__(self, name: str, node_id: str, summary: str) -> None:
- self.name = name
- self.node_id = node_id
- self.summary = summary
- self.series = {}
- self.extra_logs = {} # type: Dict[str, bytes]
-
-
-class NormStatProps(IStorable):
- "Statistic properties for timeserie"
+class StatProps(IStorable):
+ "Statistic properties for timeseries with unknown data distribution"
def __init__(self, data: List[Number]) -> None:
- self.average = None # type: float
- self.deviation = None # type: float
- self.confidence = None # type: float
- self.confidence_level = None # type: float
-
self.perc_99 = None # type: float
self.perc_95 = None # type: float
self.perc_90 = None # type: float
@@ -100,10 +110,52 @@
self.bins_edges = None # type: List[float]
self.data = data
- self.normtest = None # type: Any
+ def __str__(self) -> str:
+ res = ["{}(size = {}):".format(self.__class__.__name__, len(self.data))]
+ for name in ["perc_50", "perc_90", "perc_95", "perc_99"]:
+ res.append(" {} = {}".format(name, round_digits(getattr(self, name))))
+ res.append(" range {} {}".format(round_digits(self.min), round_digits(self.max)))
+ return "\n".join(res)
+
+ def __repr__(self) -> str:
+ return str(self)
+
+ def raw(self) -> Dict[str, Any]:
+ data = self.__dict__.copy()
+ data['bins_edges'] = list(self.bins_edges)
+ data['bins_populations'] = list(self.bins_populations)
+ return data
+
+ @classmethod
+ def fromraw(cls, data: Dict[str, Any]) -> 'StatProps':
+ data['bins_edges'] = numpy.array(data['bins_edges'])
+ data['bins_populations'] = numpy.array(data['bins_populations'])
+ res = cls.__new__(cls)
+ res.__dict__.update(data)
+ return res
+
+
+class HistoStatProps(StatProps):
+ """Statistic properties for 2D timeseries with unknown data distribution and histogram as input value.
+ Used for latency"""
+ def __init__(self, data: List[Number], second_axis_size: int) -> None:
+ self.second_axis_size = second_axis_size
+ StatProps.__init__(self, data)
+
+
+class NormStatProps(StatProps):
+ "Statistic properties for timeseries with normal data distribution. Used for iops/bw"
+ def __init__(self, data: List[Number]) -> None:
+ StatProps.__init__(self, data)
+
+ self.average = None # type: float
+ self.deviation = None # type: float
+ self.confidence = None # type: float
+ self.confidence_level = None # type: float
+ self.normtest = None # type: NormaltestResult
def __str__(self) -> str:
- res = ["StatProps(size = {}):".format(len(self.data)),
+ res = ["NormStatProps(size = {}):".format(len(self.data)),
" distr = {} ~ {}".format(round_digits(self.average), round_digits(self.deviation)),
" confidence({0.confidence_level}) = {1}".format(self, round_digits(self.confidence)),
" perc_50 = {}".format(round_digits(self.perc_50)),
@@ -114,43 +166,32 @@
" normtest = {0.normtest}".format(self)]
return "\n".join(res)
- def __repr__(self) -> str:
- return str(self)
-
def raw(self) -> Dict[str, Any]:
data = self.__dict__.copy()
- data['nortest'] = (data['nortest'].statistic, data['nortest'].pvalue)
+ data['normtest'] = (data['nortest'].statistic, data['nortest'].pvalue)
data['bins_edges'] = list(self.bins_edges)
return data
@classmethod
def fromraw(cls, data: Dict[str, Any]) -> 'NormStatProps':
- data['nortest'] = stats.mstats.NormaltestResult(data['nortest'].statistic, data['nortest'].pvalue)
- data['bins_edges'] = numpy.array(data['bins_edges'])
- res = cls.__new__(cls)
- res.__dict__.update(data)
- return res
+ data['normtest'] = NormaltestResult(*data['normtest'])
+ obj = StatProps.fromraw(data)
+ obj.__class__ = cls
+ return cast('NormStatProps', obj)
-class ProcessedTestResults:
- def __init__(self, info: Dict[str, Any],
- metrics: Dict[str, NormStatProps]) -> None:
- self.test = info['test']
- self.profile = info['profile']
- self.suite = info['suite']
- self.name = "{0.suite}.{0.test}.{0.profile}".format(self)
+JobStatMetrics = Dict[Tuple[str, str, str], StatProps]
+
+
+class TestJobResult:
+ """Contains done test job information"""
+
+ def __init__(self,
+ info: TestJobConfig,
+ begin_time: int,
+ end_time: int,
+ raw: JobMetrics) -> None:
self.info = info
- self.metrics = metrics # mapping {metrics_name: StatProps}
-
-
-# class FullTestResult:
-# test_info = None # type: TestInfo
-#
-# # TODO(koder): array.array or numpy.array?
-# # {(node_id, perf_metrics_name): values}
-# performance_data = None # type: Dict[Tuple[str, str], List[int]]
-#
-# # {(node_id, perf_metrics_name): values}
-# sensors_data = None # type: Dict[Tuple[str, str, str], SensorInfo]
-
-
+ self.run_interval = (begin_time, end_time)
+ self.raw = raw # type: JobMetrics
+ self.processed = None # type: JobStatMetrics
diff --git a/wally/run_test.py b/wally/run_test.py
index 2cdf0e8..a11b43d 100755
--- a/wally/run_test.py
+++ b/wally/run_test.py
@@ -11,7 +11,7 @@
from .stage import Stage, StepOrder
from .sensors import collect_sensors_data
from .suits.io.fio import IOPerfTest
-from .suits.itest import TestInputConfig
+from .suits.itest import TestSuiteConfig
from .suits.mysql import MysqlTest
from .suits.omgbench import OmgTest
from .suits.postgres import PgBenchTest
@@ -225,38 +225,44 @@
config_block = 'tests'
def run(self, ctx: TestRun) -> None:
- for test_group in ctx.config.get('tests', []):
- if not ctx.config.no_tests:
- test_nodes = [node for node in ctx.nodes if 'testnode' in node.info.roles]
+ if ctx.config.no_tests:
+ logger.info("Skiping tests, as 'no_tests' config settings is True")
+ return
- if not test_nodes:
- logger.error("No test nodes found")
- raise StopTestError()
+ for suite_idx, test_suite in enumerate(ctx.config.get('tests', [])):
+ test_nodes = [node for node in ctx.nodes if 'testnode' in node.info.roles]
- for name, params in test_group.items():
- vm_count = params.get('node_limit', None) # type: Optional[int]
+ if not test_nodes:
+ logger.error("No test nodes found")
+ raise StopTestError()
- # select test nodes
- if vm_count is None:
- curr_test_nodes = test_nodes
- else:
- curr_test_nodes = test_nodes[:vm_count]
+ if len(test_suite) != 1:
+ logger.error("Test suite %s contain more than one test. Put each test in separated group", suite_idx)
+ raise StopTestError()
- if not curr_test_nodes:
- logger.error("No nodes found for test, skipping it.")
- continue
+ name, params = list(test_suite.items())[0]
+ vm_count = params.get('node_limit', None) # type: Optional[int]
- test_cls = TOOL_TYPE_MAPPER[name]
- remote_dir = ctx.config.default_test_local_folder.format(name=name, uuid=ctx.config.run_uuid)
- test_cfg = TestInputConfig(test_cls.__name__,
- params=params,
- run_uuid=ctx.config.run_uuid,
- nodes=test_nodes,
- storage=ctx.storage,
- remote_dir=remote_dir)
+ # select test nodes
+ if vm_count is None:
+ curr_test_nodes = test_nodes
+ else:
+ curr_test_nodes = test_nodes[:vm_count]
- test_cls(test_cfg,
- on_idle=lambda: collect_sensors_data(ctx, False)).run()
+ if not curr_test_nodes:
+ logger.error("No nodes found for test, skipping it.")
+ continue
+
+ test_cls = TOOL_TYPE_MAPPER[name]
+ remote_dir = ctx.config.default_test_local_folder.format(name=name, uuid=ctx.config.run_uuid)
+ test_cfg = TestSuiteConfig(test_cls.name,
+ params=params,
+ run_uuid=ctx.config.run_uuid,
+ nodes=test_nodes,
+ remote_dir=remote_dir)
+
+ test_cls(storage=ctx.storage, config=test_cfg, idx=suite_idx,
+ on_idle=lambda: collect_sensors_data(ctx, False)).run()
@classmethod
def validate_config(cls, cfg: ConfigBlock) -> None:
diff --git a/wally/ssh.py b/wally/ssh.py
index 6bba020..fcb7fb3 100644
--- a/wally/ssh.py
+++ b/wally/ssh.py
@@ -10,7 +10,7 @@
import paramiko
from . import utils
-from .ssh_utils import ConnCreds, IPAddr
+from .common_types import ConnCreds, IPAddr
logger = logging.getLogger("wally")
NODE_KEYS = {} # type: Dict[IPAddr, paramiko.RSAKey]
diff --git a/wally/ssh_utils.py b/wally/ssh_utils.py
index e9e118d..ed857b6 100644
--- a/wally/ssh_utils.py
+++ b/wally/ssh_utils.py
@@ -1,12 +1,11 @@
import re
import getpass
import logging
-from typing import List, Dict, Any
+from typing import List, Dict
from . import utils
-from .common_types import IPAddr
-from .result_classes import IStorable
+from .common_types import ConnCreds
logger = logging.getLogger("wally")
@@ -47,35 +46,6 @@
uri_reg_exprs.append(templ.format(**re_dct))
-class ConnCreds(IStorable):
- def __init__(self, host: str, user: str, passwd: str = None, port: str = '22',
- key_file: str = None, key: bytes = None) -> None:
- self.user = user
- self.passwd = passwd
- self.addr = IPAddr(host, int(port))
- self.key_file = key_file
- self.key = key
-
- def __str__(self) -> str:
- return "{}@{}:{}".format(self.user, self.addr.host, self.addr.port)
-
- def __repr__(self) -> str:
- return str(self)
-
- def raw(self) -> Dict[str, Any]:
- return {
- 'user': self.user,
- 'host': self.addr.host,
- 'port': self.addr.port,
- 'passwd': self.passwd,
- 'key_file': self.key_file
- }
-
- @classmethod
- def fromraw(cls, data) -> 'ConnCreds':
- return cls(**data)
-
-
def parse_ssh_uri(uri: str) -> ConnCreds:
"""Parse ssh connection URL from one of following form
[ssh://]user:passwd@host[:port]
diff --git a/wally/statistic.py b/wally/statistic.py
index 2f68ca5..259ac69 100644
--- a/wally/statistic.py
+++ b/wally/statistic.py
@@ -1,8 +1,9 @@
import math
+import array
import logging
import itertools
import statistics
-from typing import Union, List, TypeVar, Callable, Iterable, Tuple, Any, cast, Dict
+from typing import List, Callable, Iterable, cast
import numpy
from scipy import stats, optimize
@@ -10,7 +11,7 @@
from numpy.polynomial.chebyshev import chebfit, chebval
-from .result_classes import NormStatProps
+from .result_classes import NormStatProps, HistoStatProps, TimeSeries
from .utils import Number
@@ -22,9 +23,10 @@
dev = lambda x: math.sqrt(statistics.variance(x))
-def calc_norm_stat_props(data: List[Number], confidence: float = 0.95) -> NormStatProps:
+def calc_norm_stat_props(ts: TimeSeries, confidence: float = 0.95) -> NormStatProps:
"Calculate statistical properties of array of numbers"
+ data = ts.data
res = NormStatProps(data)
if len(data) == 0:
@@ -58,6 +60,39 @@
return res
+def calc_histo_stat_props(ts: TimeSeries) -> HistoStatProps:
+ data = numpy.array(ts.data)
+ data.shape = [len(ts.data) // ts.second_axis_size, ts.second_axis_size]
+
+ res = HistoStatProps(ts.data, ts.second_axis_size)
+ aggregated = numpy.sum(data, axis=0)
+
+ full_sum = numpy.sum(aggregated)
+ expected = [full_sum * 0.5, full_sum * 0.9, full_sum * 0.95, full_sum * 0.99]
+ percentiles = []
+
+ val_min = None
+ val_max = None
+
+ for idx, val in enumerate(aggregated):
+ while expected and full_sum + val >= expected[0]:
+ percentiles.append(idx)
+ del expected[0]
+
+ full_sum += val
+
+ if val != 0:
+ if val_min is None:
+ val_min = idx
+ val_max = idx
+
+ res.perc_50, res.perc_90, res.perc_95, res.perc_99 = map(ts.bins_edges.__getitem__, percentiles)
+ res.min = ts.bins_edges[val_min]
+ res.max = ts.bins_edges[val_max]
+ res.bin_populations = aggregated
+ return res
+
+
def groupby_globally(data: Iterable, key_func: Callable):
grouped = {} # type: ignore
grouped_iter = itertools.groupby(data, key_func)
diff --git a/wally/storage.py b/wally/storage.py
index d33f8e5..e4e010c 100644
--- a/wally/storage.py
+++ b/wally/storage.py
@@ -7,6 +7,7 @@
import array
import shutil
import sqlite3
+import threading
from typing import Any, TypeVar, Type, IO, Tuple, cast, List, Dict, Iterable, Iterator
import yaml
@@ -16,7 +17,7 @@
from yaml import Loader, Dumper # type: ignore
-from .result_classes import Storable, IStorable
+from .result_classes import IStorable
class ISimpleStorage(metaclass=abc.ABCMeta):
@@ -59,7 +60,7 @@
class ISerializer(metaclass=abc.ABCMeta):
"""Interface for serialization class"""
@abc.abstractmethod
- def pack(self, value: Storable) -> bytes:
+ def pack(self, value: IStorable) -> bytes:
pass
@abc.abstractmethod
@@ -76,6 +77,7 @@
contains_sql = "SELECT 1 FROM wally_storage WHERE key=?"
rm_sql = "DELETE FROM wally_storage WHERE key LIKE '{}%'"
list2_sql = "SELECT key, length(data), type FROM wally_storage"
+ SQLITE3_THREADSAFE = 1
def __init__(self, db_path: str = None, existing: bool = False,
prefix: str = None, db: sqlite3.Connection = None) -> None:
@@ -89,8 +91,11 @@
raise IOError("No storage found at {!r}".format(db_path))
os.makedirs(os.path.dirname(db_path), exist_ok=True)
+ if sqlite3.threadsafety != self.SQLITE3_THREADSAFE:
+ raise RuntimeError("Sqlite3 compiled without threadsafe support, can't use DB storage on it")
+
try:
- self.db = sqlite3.connect(db_path)
+ self.db = sqlite3.connect(db_path, check_same_thread=False)
except sqlite3.OperationalError as exc:
raise IOError("Can't open database at {!r}".format(db_path)) from exc
@@ -224,7 +229,15 @@
pass
def list(self, path: str) -> Iterator[Tuple[bool, str]]:
- for fobj in os.scandir(self.j(path)):
+ path = self.j(path)
+
+ if not os.path.exists(path):
+ return
+
+ if not os.path.isdir(path):
+ raise OSError("{!r} is not a directory".format(path))
+
+ for fobj in os.scandir(path):
if fobj.path not in self.ignored:
if fobj.is_dir():
yield False, fobj.name
@@ -234,7 +247,7 @@
class YAMLSerializer(ISerializer):
"""Serialize data to yaml"""
- def pack(self, value: Storable) -> bytes:
+ def pack(self, value: IStorable) -> bytes:
try:
return yaml.dump(value, Dumper=Dumper, encoding="utf8")
except Exception as exc:
@@ -246,7 +259,7 @@
class SAFEYAMLSerializer(ISerializer):
"""Serialize data to yaml"""
- def pack(self, value: Storable) -> bytes:
+ def pack(self, value: IStorable) -> bytes:
try:
return yaml.safe_dump(value, encoding="utf8")
except Exception as exc:
@@ -274,7 +287,7 @@
fpath = "/".join(path)
return self.__class__(self.fs.sub_storage(fpath), self.db.sub_storage(fpath), self.serializer)
- def put(self, value: Storable, *path: str) -> None:
+ def put(self, value: IStorable, *path: str) -> None:
dct_value = value.raw() if isinstance(value, IStorable) else value
serialized = self.serializer.pack(dct_value)
fpath = "/".join(path)
@@ -313,7 +326,7 @@
def append_raw(self, value: bytes, *path: str) -> None:
with self.fs.get_fd("/".join(path), "rb+") as fd:
- fd.seek(offset=0, whence=os.SEEK_END)
+ fd.seek(0, os.SEEK_END)
fd.write(value)
def get_fd(self, path: str, mode: str = "r") -> IO:
diff --git a/wally/storage_structure.txt b/wally/storage_structure.txt
deleted file mode 100644
index bc4e77e..0000000
--- a/wally/storage_structure.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-config: Config - full configuration
-all_nodes: List[NodeInfo] - all nodes
-cli: List[str] - cli options
-spawned_nodes_ids: List[int] - list of openstack VM, spawned for test
-
-fuel_version: List[int] - FUEL master node version
-fuel_os_creds: OSCreds - openstack creds, discovered from fuel (or None)
-
-openstack_openrc: OSCreds - openrc used for openstack cluster
-
-info/comment : str - run comment
-info/run_uuid : str - run uuid
-info/run_time : float - run unix time
-
-# test results
-result/{descr}_{id}/info :
- test_config = {
- 'name': self.name,
- 'iteration_name': iter_name,
- 'iteration_config': iteration_config.raw(),
- 'params': self.config.params,
- 'nodes': self.sorted_nodes_ids,
- 'begin_time': min_start_time,
- 'end_time': max_stop_time
- }
-
-result/{descr}_{id}/measurement/{node}/{name}_raw : bytes - raw log, where name from {'bw', 'iops', 'lat'}
-result/{descr}_{id}/measurement/{node}/{name}_data - List[uint64] - measurements data.
-result/{descr}_{id}/measurement/{node}/{name}_meta - Dict[str, Any] - measurements metadata.
-
-metric/{node_name}/{dev}/{metric_name} : List[float] - node metrics data. E.g.:
- metric/node-22/cpu/load
- metric/node-22/sda/read_io
- metric/node-22/eth0/data_recv
-
-rpc_logs/{node_id} - str, rpc server log from node
diff --git a/wally/storage_structure.yaml b/wally/storage_structure.yaml
new file mode 100644
index 0000000..e748dd8
--- /dev/null
+++ b/wally/storage_structure.yaml
@@ -0,0 +1,36 @@
+# {node} - node id in format '\d+.\d+.\d+.\d+:\d+'
+# {descr} - test short description '[-a-zA-Z0-9]+'
+# {metric_name} - metrics name '[a-z_]+'
+# {id} - test/suite run id '\d+'
+# {dev} - device name '[^.]+'
+# {suite} - suite name '[a-z]+'
+# {profile} - profile name '[a-z_]+'
+
+
+config: Config # test input configuration
+all_nodes: List[NodeInfo] # all discovered nodes
+cli: List[str] # cli options
+spawned_nodes_ids: List[int] # list of openstack VM ids, spawned for test
+fuel_version: List[int] # FUEL master node version
+fuel_os_creds: OSCreds # openstack creds, discovered from fuel (or None)
+openstack_openrc: OSCreds # openrc used for openstack cluster
+info:
+ comment : str # run comment
+ run_uuid : str # run uuid
+ run_time : float # unix time when test first starts
+
+'results/{suite}_{profile}_{id}':
+ config: TestSuiteConfig # test job(iteration) input config, {id} is id of first job in suite
+ '{descr}_{id}':
+ config: TestJobConfig # test job(iteration) input config
+
+ # dev in next line is tool name - fio/vdbench/....
+ '{node}_{dev}.{metric_name}:raw' : bytes # raw log, where name from {'bw', 'iops', 'lat', ..}
+ '{node}_{dev}.{metric_name}': List[uint64] # measurements data concatenated with collect times in
+ # microseconds from unix epoch
+
+sensors:
+ '{node}_{dev}.{metric_name}:raw' : bytes # raw log, where name from {'bw', 'iops', 'lat', ..}
+ '{node}_{dev}.{metric_name}': List[uint64] # measurements data cotaneted with collect times in microseconds from unix epoch
+
+'rpc_logs/{node}' : bytes # rpc server log from node
diff --git a/wally/suits/io/fio.py b/wally/suits/io/fio.py
index c32dba2..7b2c3e3 100644
--- a/wally/suits/io/fio.py
+++ b/wally/suits/io/fio.py
@@ -7,9 +7,9 @@
from ...utils import StopTestError, get_os, ssize2b
from ...node_interfaces import IRPCNode
-from ..itest import ThreadedTest, IterationConfig, NodeTestResults
-from ...result_classes import TimeSerie
-from .fio_task_parser import execution_time, fio_cfg_compile, FioJobSection, FioParams, get_log_files, get_test_summary
+from ..itest import ThreadedTest
+from ...result_classes import TimeSeries, JobMetrics
+from .fio_task_parser import execution_time, fio_cfg_compile, FioJobConfig, FioParams, get_log_files
from . import rpc_plugin
from .fio_hist import expected_lat_bins
@@ -22,6 +22,7 @@
retry_time = 30
configs_dir = os.path.dirname(__file__) # type: str
name = 'fio'
+ job_config_cls = FioJobConfig
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
@@ -72,14 +73,13 @@
else:
self.file_size = ssize2b(self.load_params['FILESIZE'])
- self.fio_configs = list(fio_cfg_compile(self.load_profile, self.load_profile_path,
+ self.job_configs = list(fio_cfg_compile(self.load_profile, self.load_profile_path,
cast(FioParams, self.load_params)))
- if len(self.fio_configs) == 0:
+ if len(self.job_configs) == 0:
logger.error("Empty fio config provided")
raise StopTestError()
- self.iterations_configs = self.fio_configs # type: ignore
self.exec_folder = self.config.remote_dir
def config_node(self, node: IRPCNode) -> None:
@@ -129,15 +129,15 @@
node.copy_file(fio_path, bz_dest)
node.run("bzip2 --decompress {} ; chmod a+x {}".format(bz_dest, self.join_remote("fio")))
- def get_expected_runtime(self, iteration_info: IterationConfig) -> int:
- return execution_time(cast(FioJobSection, iteration_info))
+ def get_expected_runtime(self, job_config: FioJobConfig) -> int:
+ return execution_time(cast(FioJobConfig, job_config))
- def prepare_iteration(self, node: IRPCNode, iter_config: IterationConfig) -> None:
- node.put_to_file(self.remote_task_file, str(iter_config).encode("utf8"))
+ def prepare_iteration(self, node: IRPCNode, job_config: FioJobConfig) -> None:
+ node.put_to_file(self.remote_task_file, str(job_config).encode("utf8"))
# TODO: get a link to substorage as a parameter
- def run_iteration(self, node: IRPCNode, iter_config: IterationConfig, stor_prefix: str) -> NodeTestResults:
- f_iter_config = cast(FioJobSection, iter_config)
+ def run_iteration(self, node: IRPCNode, iter_config: FioJobConfig, job_root: str) -> JobMetrics:
+ f_iter_config = cast(FioJobConfig, iter_config)
exec_time = execution_time(f_iter_config)
fio_cmd_templ = "cd {exec_folder}; " + \
@@ -152,17 +152,15 @@
if must_be_empty:
logger.error("Unexpected fio output: %r", must_be_empty)
- res = NodeTestResults(self.__class__.__name__, node.info.node_id(), get_test_summary(f_iter_config))
+ res = {} # type: JobMetrics
- res.extra_logs['fio'] = node.get_file_content(self.remote_output_file)
- self.store_data(res.extra_logs['fio'], "raw", stor_prefix, "fio_raw")
+ # put fio output into storage
+ fio_out = node.get_file_content(self.remote_output_file)
+ self.rstorage.put_extra(job_root, node.info.node_id(), "fio_raw", fio_out)
node.conn.fs.unlink(self.remote_output_file)
files = [name for name in node.conn.fs.listdir(self.exec_folder)]
- expected_time_delta = 1000 # 1000ms == 1s
- max_time_diff = 50 # 50ms - 5%
-
for name, path in get_log_files(f_iter_config):
log_files = [fname for fname in files if fname.startswith(path)]
if len(log_files) != 1:
@@ -172,7 +170,6 @@
fname = os.path.join(self.exec_folder, log_files[0])
raw_result = node.get_file_content(fname) # type: bytes
- self.store_data(raw_result, "raw", stor_prefix, "{}_raw".format(name))
node.conn.fs.unlink(fname)
try:
@@ -182,23 +179,14 @@
raise StopTestError()
parsed = array.array('L' if name == 'lat' else 'Q')
- prev_ts = None
- load_start_at = None
+ times = array.array('Q')
- # TODO: need to adjust vals for timedelta
for idx, line in enumerate(log_data):
line = line.strip()
if line:
try:
time_ms_s, val_s, _, *rest = line.split(",")
- time_ms = int(time_ms_s.strip())
-
- if not prev_ts:
- prev_ts = time_ms - expected_time_delta
- load_start_at = time_ms
- elif abs(time_ms - prev_ts - expected_time_delta) > max_time_diff:
- logger.warning("Too large gap in {} log at {} - {}ms"
- .format(name, time_ms, time_ms - prev_ts))
+ times.append(int(time_ms_s.strip()))
if name == 'lat':
vals = [int(i.strip()) for i in rest]
@@ -215,17 +203,12 @@
logger.exception("Error during parse %s fio log file in line %s: %r", name, idx, line)
raise StopTestError()
- prev_ts += expected_time_delta
-
- res.series[name] = TimeSerie(name=name,
- raw=raw_result,
- second_axis_size=expected_lat_bins if name == 'lat' else 1,
- start_at=load_start_at,
- step=expected_time_delta,
- data=parsed)
-
- self.store_data(parsed, "array", stor_prefix, "{}_data".format(name))
- self.store_data(res.series[name].meta(), "yaml", stor_prefix, "{}_meta".format(name))
+ ts = TimeSeries(name=name,
+ raw=raw_result,
+ second_axis_size=expected_lat_bins if name == 'lat' else 1,
+ data=parsed,
+ times=times)
+ res[(node.info.node_id(), 'fio', name)] = ts
return res
diff --git a/wally/suits/io/fio_hist.py b/wally/suits/io/fio_hist.py
index eb5d9ee..a2ded70 100644
--- a/wally/suits/io/fio_hist.py
+++ b/wally/suits/io/fio_hist.py
@@ -52,6 +52,6 @@
return lower + (upper - lower) * edge
-def get_lat_vals(columns: int = 1216, coarseness: int = 0) -> List[float]:
+def get_lat_vals(columns: int = expected_lat_bins, coarseness: int = 0) -> List[float]:
return [plat_idx_to_val_coarse(val, coarseness) for val in range(columns)]
diff --git a/wally/suits/io/fio_task_parser.py b/wally/suits/io/fio_task_parser.py
index bdcf4a3..6940aaf 100644
--- a/wally/suits/io/fio_task_parser.py
+++ b/wally/suits/io/fio_task_parser.py
@@ -12,7 +12,7 @@
from ...result_classes import IStorable
-from ..itest import IterationConfig
+from ...result_classes import TestJobConfig
from ...utils import sec_to_str, ssize2b
@@ -38,13 +38,16 @@
("vm_count", int)])
-class FioJobSection(IterationConfig, IStorable):
+class FioJobConfig(TestJobConfig):
def __init__(self, name: str) -> None:
- self.name = name
+ TestJobConfig.__init__(self)
self.vals = OrderedDict() # type: Dict[str, Any]
- self.summary = None
+ self.name = name
- def copy(self) -> 'FioJobSection':
+ def __eq__(self, other: 'FioJobConfig') -> bool:
+ return self.vals == other.vals
+
+ def copy(self) -> 'FioJobConfig':
return copy.deepcopy(self)
def required_vars(self) -> Iterator[Tuple[str, Var]]:
@@ -56,7 +59,7 @@
return len(list(self.required_vars())) == 0
def __str__(self) -> str:
- res = "[{0}]\n".format(self.name)
+ res = "[{0}]\n".format(self.summary)
for name, val in self.vals.items():
if name.startswith('_') or name == name.upper():
@@ -68,15 +71,18 @@
return res
+ def __repr__(self) -> str:
+ return str(self)
+
def raw(self) -> Dict[str, Any]:
return {
- 'name': self.name,
- 'vals': list(map(list, self.vals.items())),
- 'summary': self.summary
+ 'vals': [[key, val] for key, val in self.vals.items()],
+ 'summary': self.summary,
+ 'name': self.name
}
@classmethod
- def fromraw(cls, data: Dict[str, Any]) -> 'FioJobSection':
+ def fromraw(cls, data: Dict[str, Any]) -> 'FioJobConfig':
obj = cls(data['name'])
obj.summary = data['summary']
obj.vals.update(data['vals'])
@@ -160,7 +166,7 @@
raise ParseError(str(exc), fname, lineno, oline)
-def fio_config_parse(lexer_iter: Iterable[CfgLine]) -> Iterator[FioJobSection]:
+def fio_config_parse(lexer_iter: Iterable[CfgLine]) -> Iterator[FioJobConfig]:
in_globals = False
curr_section = None
glob_vals = OrderedDict() # type: Dict[str, Any]
@@ -188,9 +194,7 @@
try:
cont = open(new_fname).read()
except IOError as err:
- msg = "Error while including file {0}: {1}"
- raise ParseError(msg.format(new_fname, err),
- fname, lineno, oline)
+ raise ParseError("Error while including file {}: {}".format(new_fname, err), fname, lineno, oline)
new_lines.extend(fio_config_lexer(cont, new_fname))
one_more = True
@@ -207,13 +211,11 @@
if name == 'global':
if sections_count != 0:
- raise ParseError("[global] section should" +
- " be only one and first",
- fname, lineno, oline)
+ raise ParseError("[global] section should be only one and first", fname, lineno, oline)
in_globals = True
else:
in_globals = False
- curr_section = FioJobSection(name)
+ curr_section = FioJobConfig(name)
curr_section.vals = glob_vals.copy()
sections_count += 1
else:
@@ -221,12 +223,9 @@
if in_globals:
glob_vals[name] = val
elif name == name.upper():
- raise ParseError("Param '" + name +
- "' not in [global] section",
- fname, lineno, oline)
+ raise ParseError("Param {!r} not in [global] section".format(name), fname, lineno, oline)
elif curr_section is None:
- raise ParseError("Data outside section",
- fname, lineno, oline)
+ raise ParseError("Data outside section", fname, lineno, oline)
else:
curr_section.vals[name] = val
@@ -234,7 +233,7 @@
yield curr_section
-def process_cycles(sec: FioJobSection) -> Iterator[FioJobSection]:
+def process_cycles(sec: FioJobConfig) -> Iterator[FioJobConfig]:
cycles = OrderedDict() # type: Dict[str, Any]
for name, val in sec.vals.items():
@@ -270,7 +269,7 @@
FioParams = Dict[str, FioParamsVal]
-def apply_params(sec: FioJobSection, params: FioParams) -> FioJobSection:
+def apply_params(sec: FioJobConfig, params: FioParams) -> FioJobConfig:
processed_vals = OrderedDict() # type: Dict[str, Any]
processed_vals.update(params)
for name, val in sec.vals.items():
@@ -307,7 +306,7 @@
MAGIC_OFFSET = 0.1885
-def final_process(sec: FioJobSection, counter: List[int] = [0]) -> FioJobSection:
+def final_process(sec: FioJobConfig, counter: List[int] = [0]) -> FioJobConfig:
sec = sec.copy()
sec.vals['unified_rw_reporting'] = '1'
@@ -340,7 +339,7 @@
return sec
-def get_test_sync_mode(sec: FioJobSection) -> str:
+def get_test_sync_mode(sec: FioJobConfig) -> str:
if isinstance(sec, dict):
vals = sec
else:
@@ -359,7 +358,7 @@
return 'a'
-def get_test_summary_tuple(sec: FioJobSection, vm_count: int = None) -> TestSumm:
+def get_test_summary_tuple(sec: FioJobConfig, vm_count: int = None) -> TestSumm:
if isinstance(sec, dict):
vals = sec
else:
@@ -382,7 +381,7 @@
vm_count)
-def get_test_summary(sec: FioJobSection, vm_count: int = None, noiodepth: bool = False) -> str:
+def get_test_summary(sec: FioJobConfig, vm_count: int = None, noiodepth: bool = False) -> str:
tpl = get_test_summary_tuple(sec, vm_count)
res = "{0.oper}{0.mode}{0.bsize}".format(tpl)
@@ -395,11 +394,11 @@
return res
-def execution_time(sec: FioJobSection) -> int:
+def execution_time(sec: FioJobConfig) -> int:
return sec.vals.get('ramp_time', 0) + sec.vals.get('runtime', 0)
-def parse_all_in_1(source:str, fname: str = None) -> Iterator[FioJobSection]:
+def parse_all_in_1(source:str, fname: str = None) -> Iterator[FioJobConfig]:
return fio_config_parse(fio_config_lexer(source, fname))
@@ -414,7 +413,7 @@
yield res
-def get_log_files(sec: FioJobSection) -> List[Tuple[str, str]]:
+def get_log_files(sec: FioJobConfig) -> List[Tuple[str, str]]:
res = [] # type: List[Tuple[str, str]]
for key, name in (('write_iops_log', 'iops'), ('write_bw_log', 'bw'), ('write_hist_log', 'lat')):
log = sec.vals.get(key)
@@ -423,7 +422,7 @@
return res
-def fio_cfg_compile(source: str, fname: str, test_params: FioParams) -> Iterator[FioJobSection]:
+def fio_cfg_compile(source: str, fname: str, test_params: FioParams) -> Iterator[FioJobConfig]:
it = parse_all_in_1(source, fname)
it = (apply_params(sec, test_params) for sec in it)
it = flatmap(process_cycles, it)
diff --git a/wally/suits/io/rrd.cfg b/wally/suits/io/rrd.cfg
index 850c9a3..ca3c613 100644
--- a/wally/suits/io/rrd.cfg
+++ b/wally/suits/io/rrd.cfg
@@ -1,7 +1,7 @@
[global]
include defaults_qd.cfg
ramp_time=0
-runtime=600
+runtime=10
[test_{TEST_SUMM}]
blocksize=60k
diff --git a/wally/suits/itest.py b/wally/suits/itest.py
index 446ad69..bc6b115 100644
--- a/wally/suits/itest.py
+++ b/wally/suits/itest.py
@@ -1,17 +1,19 @@
+import re
import abc
import time
+import array
+import struct
import logging
import os.path
import datetime
-from typing import Dict, Any, List, Optional, Callable, cast
+from typing import Any, List, Optional, Callable, cast, Iterator, Tuple, Iterable
-from concurrent.futures import ThreadPoolExecutor, wait
+from concurrent.futures import ThreadPoolExecutor, wait, Future
-from ..utils import Barrier, StopTestError, sec_to_str
+from ..utils import StopTestError, sec_to_str, get_time_interval_printable_info
from ..node_interfaces import IRPCNode
from ..storage import Storage
-from ..result_classes import NodeTestResults, IStorable
-from queue import Queue
+from ..result_classes import TestSuiteConfig, TestJobConfig, JobMetrics, TimeSeries
logger = logging.getLogger("wally")
@@ -20,35 +22,125 @@
__doc__ = "Contains base classes for performance tests"
-class TestInputConfig:
- """
- this class describe test input configuration
+class ResultStorage:
+ ts_header_format = "!IIIcc"
- test_type - test type name
- params - parameters from yaml file for this test
- test_uuid - UUID to be used to create file names & Co
- log_directory - local directory to store results
- nodes - nodes to run tests on
- remote_dir - directory on nodes to be used for local files
- """
- def __init__(self,
- test_type: str,
- params: Dict[str, Any],
- run_uuid: str,
- nodes: List[IRPCNode],
- storage: Storage,
- remote_dir: str) -> None:
- self.test_type = test_type
- self.params = params
- self.run_uuid = run_uuid
- self.nodes = nodes
+ def __init__(self, storage: Storage, job_config_cls: type) -> None:
self.storage = storage
- self.remote_dir = remote_dir
+ self.job_config_cls = job_config_cls
+ def get_suite_root(self, suite_type: str, idx: int) -> str:
+ return "results/{}_{}".format(suite_type, idx)
-class IterationConfig(IStorable):
- name = None # type: str
- summary = None # type: str
+ def get_job_root(self, suite_root: str, summary: str, run_id: int) -> str:
+ return "{}/{}_{}".format(suite_root, summary, run_id)
+
+ # store
+ def put_suite_config(self, config: TestSuiteConfig, root: str) -> None:
+ self.storage.put(config, root, "config.yml")
+
+ def put_job_config(self, config: TestJobConfig, root: str) -> None:
+ self.storage.put(config, root, "config.yml")
+
+ def get_suite_config(self, suite_root: str) -> TestSuiteConfig:
+ return self.storage.load(TestSuiteConfig, suite_root, "config.yml")
+
+ def get_job_node_prefix(self, job_root_path: str, node_id: str) -> str:
+ return "{}/{}".format(job_root_path, node_id)
+
+ def get_ts_path(self, job_root_path: str, node_id: str, dev: str, sensor_name: str) -> str:
+ return "{}_{}.{}".format(self.get_job_node_prefix(job_root_path, node_id), dev, sensor_name)
+
+ def put_ts(self, ts: TimeSeries, job_root_path: str, node_id: str, dev: str, sensor_name: str) -> None:
+ # TODO: check that 'metrics', 'dev' and 'node_id' match required patterns
+ root_path = self.get_ts_path(job_root_path, node_id, dev, sensor_name)
+
+ if len(ts.data) / ts.second_axis_size != len(ts.times):
+ logger.error("Unbalanced time series data. Array size has % elements, while time size has %",
+ len(ts.data) / ts.second_axis_size, len(ts.times))
+ raise StopTestError()
+
+ with self.storage.get_fd(root_path, "cb") as fd:
+ header = struct.pack(self.ts_header_format,
+ ts.second_axis_size,
+ len(ts.data),
+ len(ts.times),
+ cast(array.array, ts.data).typecode.encode("ascii"),
+ cast(array.array, ts.times).typecode.encode("ascii"))
+ fd.write(header)
+ cast(array.array, ts.data).tofile(fd)
+ cast(array.array, ts.times).tofile(fd)
+
+ if ts.raw is not None:
+ self.storage.put_raw(ts.raw, root_path + ":raw")
+
+ def put_extra(self, job_root: str, node_id: str, key: str, data: bytes) -> None:
+ self.storage.put_raw(data, job_root, node_id + "_" + key)
+
+ def list_suites(self) -> Iterator[Tuple[TestSuiteConfig, str]]:
+ """iterates over (suite_name, suite_id, suite_root_path)
+ primary this function output should be used as input into list_jobs_in_suite method
+ """
+ ts_re = re.compile(r"[a-zA-Z]+_\d+$")
+ for is_file, name in self.storage.list("results"):
+ if not is_file:
+ rr = ts_re.match(name)
+ if rr:
+ path = "results/" + name
+ yield self.get_suite_config(path), path
+
+ def list_jobs_in_suite(self, suite_root_path: str) -> Iterator[Tuple[TestJobConfig, str, int]]:
+ """iterates over (job_summary, job_root_path)
+ primary this function output should be used as input into list_ts_in_job method
+ """
+ ts_re = re.compile(r"(?P<job_summary>[a-zA-Z0-9]+)_(?P<id>\d+)$")
+ for is_file, name in self.storage.list(suite_root_path):
+ if is_file:
+ continue
+ rr = ts_re.match(name)
+ if rr:
+ config_path = "{}/{}/config.yml".format(suite_root_path, name)
+ if config_path in self.storage:
+ cfg = self.storage.load(self.job_config_cls, config_path)
+ yield cfg, "{}/{}".format(suite_root_path, name), int(rr.group("id"))
+
+ def list_ts_in_job(self, job_root_path: str) -> Iterator[Tuple[str, str, str]]:
+ """iterates over (node_id, device_name, sensor_name)
+ primary this function output should be used as input into load_ts method
+ """
+ # TODO: check that all TS files available
+ ts_re = re.compile(r"(?P<node_id>\d+\.\d+\.\d+\.\d+:\d+)_(?P<dev>[^.]+)\.(?P<sensor>[a-z_]+)$")
+ already_found = set()
+ for is_file, name in self.storage.list(job_root_path):
+ if not is_file:
+ continue
+ rr = ts_re.match(name)
+ if rr:
+ key = (rr.group("node_id"), rr.group("dev"), rr.group("sensor"))
+ if key not in already_found:
+ already_found.add(key)
+ yield key
+
+ def load_ts(self, root_path: str, node_id: str, dev: str, sensor_name: str) -> TimeSeries:
+ path = self.get_ts_path(root_path, node_id, dev, sensor_name)
+
+ with self.storage.get_fd(path, "rb") as fd:
+ header = fd.read(struct.calcsize(self.ts_header_format))
+ second_axis_size, data_sz, time_sz, data_typecode, time_typecode = \
+ struct.unpack(self.ts_header_format, header)
+
+ data = array.array(data_typecode.decode("ascii"))
+ times = array.array(time_typecode.decode("ascii"))
+
+ data.fromfile(fd, data_sz)
+ times.fromfile(fd, time_sz)
+
+ # calculate number of elements
+ return TimeSeries("{}.{}".format(dev, sensor_name),
+ raw=None,
+ data=data,
+ times=times,
+ second_axis_size=second_axis_size)
class PerfTest(metaclass=abc.ABCMeta):
@@ -56,13 +148,16 @@
name = None # type: str
max_retry = 3
retry_time = 30
+ job_config_cls = None # type: type
- def __init__(self, config: TestInputConfig, on_idle: Callable[[], None] = None) -> None:
+ def __init__(self, storage: Storage, config: TestSuiteConfig, idx: int, on_idle: Callable[[], None] = None) -> None:
self.config = config
self.stop_requested = False
- self.nodes = self.config.nodes # type: List[IRPCNode]
- self.sorted_nodes_ids = sorted(node.info.node_id() for node in self.nodes)
+ self.sorted_nodes_ids = sorted(node.info.node_id() for node in self.config.nodes)
self.on_idle = on_idle
+ self.storage = storage
+ self.rstorage = ResultStorage(self.storage, self.job_config_cls)
+ self.idx = idx
def request_stop(self) -> None:
self.stop_requested = True
@@ -90,174 +185,147 @@
def __init__(self, *args, **kwargs) -> None:
PerfTest.__init__(self, *args, **kwargs)
- self.iterations_configs = [None] # type: List[Optional[IterationConfig]]
- self.storage_q = Queue() # type: Any
+ self.job_configs = [None] # type: List[Optional[TestJobConfig]]
+ self.suite_root_path = self.rstorage.get_suite_root(self.config.test_type, self.idx)
@abc.abstractmethod
- def get_expected_runtime(self, iter_cfg: IterationConfig) -> Optional[int]:
+ def get_expected_runtime(self, iter_cfg: TestJobConfig) -> Optional[int]:
pass
- def get_not_done_stages(self, storage: Storage) -> Dict[int, IterationConfig]:
- not_done = {} # type: Dict[int, IterationConfig]
+ def get_not_done_stages(self) -> Iterable[Tuple[int, TestJobConfig]]:
+ all_jobs = dict(enumerate(self.job_configs))
+ for db_config, path, jid in self.rstorage.list_jobs_in_suite(self.suite_root_path):
+ if jid in all_jobs:
+ job_config = all_jobs[jid]
+ if job_config != db_config:
+ logger.error("Test info at path '%s/config' is not equal to expected config for iteration %s.%s." +
+ " Maybe configuration was changed before test was restarted. " +
+ "DB cfg is:\n %s\nExpected cfg is:\n %s\nFix DB or rerun test from beginning",
+ path, self.name, job_config.summary,
+ str(db_config).replace("\n", "\n "),
+ str(job_config).replace("\n", "\n "))
+ raise StopTestError()
- for run_id, iteration_config in enumerate(self.iterations_configs):
- info_path = "result/{}/info".format(run_id)
- if info_path in storage:
- info = cast(Dict[str, Any], storage.get(info_path)) # type: Dict[str, Any]
-
- assert isinstance(info, dict), \
- "Broken storage at path {}. Expect test info dict, obtain {!r}".format(info_path, info)
-
- info = info.copy()
- del info['begin_time']
- del info['end_time']
-
- iter_name = "Unnamed" if iteration_config is None else iteration_config.name
- expected_config = {
- 'name': self.name,
- 'iteration_name': iter_name,
- 'iteration_config': iteration_config.raw(),
- 'params': self.config.params,
- 'nodes': self.sorted_nodes_ids
- }
-
- assert info == expected_config, \
- ("Test info at path {} is not equal to expected config." +
- "Maybe configuration was changed before test was restarted. " +
- "Current cfg is {!r}, expected cfg is {!r}").format(info_path, info, expected_config)
-
- logger.info("Test iteration {} found in storage and will be skipped".format(iter_name))
- else:
- not_done[run_id] = iteration_config
-
- return not_done
+ logger.info("Test iteration %s.%s found in storage and will be skipped",
+ self.name, job_config.summary)
+ del all_jobs[jid]
+ return all_jobs.items()
def run(self) -> None:
- not_in_storage = self.get_not_done_stages(self.config.storage)
+ try:
+ cfg = self.rstorage.get_suite_config(self.suite_root_path)
+ except KeyError:
+ cfg = None
+
+ if cfg is not None and cfg != self.config:
+ logger.error("Current suite %s config is not equal to found in storage at %s",
+ self.config.test_type, self.suite_root_path)
+ raise StopTestError()
+
+ not_in_storage = list(self.get_not_done_stages())
if not not_in_storage:
logger.info("All test iteration in storage already. Skip test")
return
- logger.debug("Run test io.{} with profile {!r} on nodes {}.".format(self.name,
- self.load_profile_name,
- ",".join(self.sorted_nodes_ids)))
+ self.rstorage.put_suite_config(self.config, self.suite_root_path)
+
+ logger.debug("Run test %s with profile %r on nodes %s.", self.name,
+ self.load_profile_name,
+ ",".join(self.sorted_nodes_ids))
logger.debug("Prepare nodes")
- with ThreadPoolExecutor(len(self.nodes)) as pool:
- list(pool.map(self.config_node, self.nodes))
- # +5% - is a rough estimation for additional operations
- run_times = [self.get_expected_runtime(iteration_config) for iteration_config in not_in_storage.values()]
+ with ThreadPoolExecutor(len(self.config.nodes)) as pool:
+ # config nodes
+ list(pool.map(self.config_node, self.config.nodes))
+
+ run_times = [self.get_expected_runtime(job_config) for _, job_config in not_in_storage]
+
if None not in run_times:
+ # +5% - is a rough estimation for additional operations
expected_run_time = int(sum(run_times) * 1.05)
- exec_time_s = sec_to_str(expected_run_time)
- now_dt = datetime.datetime.now()
- end_dt = now_dt + datetime.timedelta(0, expected_run_time)
- logger.info("Entire test should takes aroud: {} and finished at {:%H:%M:%S}"
- .format(exec_time_s, end_dt))
- for run_id, iteration_config in sorted(not_in_storage.items()):
- iter_name = "Unnamed" if iteration_config is None else iteration_config.name
- logger.info("Run test iteration %s", iter_name)
+ exec_time_s, end_dt_s = get_time_interval_printable_info(expected_run_time)
+ logger.info("Entire test should takes around %s and finished at %s", exec_time_s, end_dt_s)
- current_result_path = "result/{}_{}".format(iteration_config.summary, run_id)
- results = [] # type: List[NodeTestResults]
+ for run_id, job_config in not_in_storage:
+ job_path = self.rstorage.get_job_root(self.suite_root_path, job_config.summary, run_id)
+
+ jfutures = [] # type: List[Future]
for idx in range(self.max_retry):
- logger.debug("Prepare iteration %s", iter_name)
+ logger.debug("Prepare job %s", job_config.summary)
# prepare nodes for new iterations
- futures = [pool.submit(self.prepare_iteration, node, iteration_config) for node in self.nodes]
- wait(futures)
+ wait([pool.submit(self.prepare_iteration, node, job_config) for node in self.config.nodes])
- # run iteration
- logger.debug("Run iteration %s", iter_name)
+ expected_job_time = self.get_expected_runtime(job_config)
+ exec_time_s, end_dt_s = get_time_interval_printable_info(expected_job_time)
+ logger.info("Job should takes around %s and finished at %s", exec_time_s, end_dt_s)
+
try:
- futures = []
- for node in self.nodes:
- path = "{}/measurement/{}".format(current_result_path, node.info.node_id())
- futures.append(pool.submit(self.run_iteration, node, iteration_config, path))
-
- results = [fut.result() for fut in futures]
+ jfutures = []
+ for node in self.config.nodes:
+ future = pool.submit(self.run_iteration, node, job_config, job_path)
+ jfutures.append(future)
+ # test completed successfully, stop retrying
break
except EnvironmentError:
if self.max_retry - 1 == idx:
logger.exception("Fio failed")
raise StopTestError()
logger.exception("During fio run")
- logger.info("Sleeping %ss and retrying", self.retry_time)
+ logger.info("Sleeping %ss and retrying job", self.retry_time)
time.sleep(self.retry_time)
start_times = [] # type: List[int]
stop_times = [] # type: List[int]
- # TODO: FIX result processing - NodeTestResults
- for result in results:
- for name, serie in result.series.items():
- start_times.append(serie.start_at)
- stop_times.append(serie.step * len(serie.data))
+ for future in jfutures:
+ for (node_id, dev, sensor_name), ts in future.result().items():
+ self.rstorage.put_ts(ts, job_path, node_id=node_id, dev=dev, sensor_name=sensor_name)
- min_start_time = min(start_times)
- max_start_time = max(start_times)
- min_stop_time = min(stop_times)
- max_stop_time = max(stop_times)
+ if len(ts.times) >= 2:
+ start_times.append(ts.times[0])
+ stop_times.append(ts.times[-1])
- max_allowed_time_diff = int((min_stop_time - max_start_time) * self.max_rel_time_diff)
- max_allowed_time_diff = max(max_allowed_time_diff, self.max_time_diff)
+ if len(start_times) > 0:
+ min_start_time = min(start_times)
+ max_start_time = max(start_times)
+ min_stop_time = min(stop_times)
+ max_stop_time = max(stop_times)
- if min_start_time + self.max_time_diff < max_allowed_time_diff:
- logger.warning("Too large difference in {}:{} start time - {}. Max recommended difference is {}"
- .format(self.name, iter_name, max_start_time - min_start_time, self.max_time_diff))
+ max_allowed_time_diff = int((min_stop_time - max_start_time) * self.max_rel_time_diff)
+ max_allowed_time_diff = max(max_allowed_time_diff, self.max_time_diff)
- if min_stop_time + self.max_time_diff < max_allowed_time_diff:
- logger.warning("Too large difference in {}:{} stop time - {}. Max recommended difference is {}"
- .format(self.name, iter_name, max_start_time - min_start_time, self.max_time_diff))
+ if min_start_time + self.max_time_diff < max_allowed_time_diff:
+ logger.warning("Too large difference in %s:%s start time - %s. " +
+ "Max recommended difference is %s",
+ self.name, job_config.summary,
+ max_start_time - min_start_time, self.max_time_diff)
- test_config = {
- 'suite': 'io',
- 'test': self.name,
- 'profile': self.load_profile_name,
- 'iteration_name': iter_name,
- 'iteration_config': iteration_config.raw(),
- 'params': self.config.params,
- 'nodes': self.sorted_nodes_ids,
- 'begin_time': min_start_time,
- 'end_time': max_stop_time
- }
+ if min_stop_time + self.max_time_diff < max_allowed_time_diff:
+ logger.warning("Too large difference in %s:%s stop time - %s. " +
+ "Max recommended difference is %s",
+ self.name, job_config.summary,
+ max_start_time - min_start_time, self.max_time_diff)
- self.process_storage_queue()
- self.config.storage.put(test_config, current_result_path, "info")
- self.config.storage.sync()
+ self.rstorage.put_job_config(job_config, job_path)
+ self.storage.sync()
if self.on_idle is not None:
self.on_idle()
- def store_data(self, val: Any, type: str, prefix: str, *path: str) -> None:
- self.storage_q.put((val, type, prefix, path))
-
- def process_storage_queue(self) -> None:
- while not self.storage_q.empty():
- value, val_type, subpath, val_path = self.storage_q.get()
- if val_type == 'raw':
- self.config.storage.put_raw(value, subpath, *val_path)
- elif val_type == 'yaml':
- self.config.storage.put(value, subpath, *val_path)
- elif val_type == 'array':
- self.config.storage.put_array(value, subpath, *val_path)
- else:
- logger.error("Internal logic error - unknown data stop type {!r}".format(val_path))
- raise StopTestError()
-
@abc.abstractmethod
def config_node(self, node: IRPCNode) -> None:
pass
@abc.abstractmethod
- def prepare_iteration(self, node: IRPCNode, iter_config: IterationConfig) -> None:
+ def prepare_iteration(self, node: IRPCNode, iter_config: TestJobConfig) -> None:
pass
@abc.abstractmethod
- def run_iteration(self, node: IRPCNode, iter_config: IterationConfig, stor_prefix: str) -> NodeTestResults:
+ def run_iteration(self, node: IRPCNode, iter_config: TestJobConfig, stor_prefix: str) -> JobMetrics:
pass
@@ -270,7 +338,7 @@
self.run_tout = self.config.params.get('run_tout', 3600)
self.iterations_configs = [None]
- def get_expected_runtime(self, iter_cfg: IterationConfig) -> Optional[int]:
+ def get_expected_runtime(self, iter_cfg: TestJobConfig) -> Optional[int]:
return None
def config_node(self, node: IRPCNode) -> None:
@@ -281,16 +349,16 @@
cmd += ' ' + self.config.params.get('prerun_opts', '')
node.run(cmd, timeout=self.prerun_tout)
- def prepare_iteration(self, node: IRPCNode, iter_config: IterationConfig) -> None:
+ def prepare_iteration(self, node: IRPCNode, iter_config: TestJobConfig) -> None:
pass
- def run_iteration(self, node: IRPCNode, iter_config: IterationConfig, stor_prefix: str) -> NodeTestResults:
+ def run_iteration(self, node: IRPCNode, iter_config: TestJobConfig, stor_prefix: str) -> JobMetrics:
# TODO: have to store logs
cmd = self.join_remote(self.run_script)
cmd += ' ' + self.config.params.get('run_opts', '')
return self.parse_results(node.run(cmd, timeout=self.run_tout))
@abc.abstractmethod
- def parse_results(self, data: str) -> NodeTestResults:
+ def parse_results(self, data: str) -> JobMetrics:
pass
diff --git a/wally/utils.py b/wally/utils.py
index 13cd675..078a019 100644
--- a/wally/utils.py
+++ b/wally/utils.py
@@ -1,12 +1,12 @@
import re
import os
-import abc
import sys
import math
import time
import uuid
import socket
import logging
+import datetime
import ipaddress
import threading
import contextlib
@@ -59,52 +59,10 @@
raise StopTestError(self.message) from value
-class IStorable(metaclass=abc.ABCMeta):
- """Interface for type, which can be stored"""
-
- @abc.abstractmethod
- def raw(self) -> Dict[str, Any]:
- pass
-
- @abc.abstractclassmethod
- def fromraw(cls, data: Dict[str, Any]) -> 'IStorable':
- pass
-
-
-Basic = Union[int, str, bytes, bool, None]
-Storable = Union[IStorable, Dict[str, Any], List[Any], int, str, bytes, bool, None]
-
-
class TaskFinished(Exception):
pass
-class Barrier:
- def __init__(self, count: int) -> None:
- self.count = count
- self.curr_count = 0
- self.cond = threading.Condition()
- self.exited = False
-
- def wait(self, timeout: int=None) -> bool:
- with self.cond:
- if self.exited:
- raise TaskFinished()
-
- self.curr_count += 1
- if self.curr_count == self.count:
- self.curr_count = 0
- self.cond.notify_all()
- return True
- else:
- self.cond.wait(timeout=timeout)
- return False
-
- def exit(self) -> None:
- with self.cond:
- self.exited = True
-
-
class Timeout(Iterable[float]):
def __init__(self, timeout: int, message: str = None, min_tick: int = 1, no_exc: bool = False) -> None:
self.end_time = time.time() + timeout
@@ -477,3 +435,10 @@
ip_addr = socket.gethostbyname(host_or_ip)
logger.info("Will use ip_addr %r instead of hostname %r", ip_addr, host_or_ip)
return ip_addr
+
+
+def get_time_interval_printable_info(seconds: int) -> Tuple[str, str]:
+ exec_time_s = sec_to_str(seconds)
+ now_dt = datetime.datetime.now()
+ end_dt = now_dt + datetime.timedelta(0, seconds)
+ return exec_time_s, "{:%H:%M:%S}".format(end_dt)