a lot of fixes, improve visualization speed, add c++ code
diff --git a/tests/test_hlstorage.py b/tests/test_hlstorage.py
new file mode 100644
index 0000000..dcc3f6e
--- /dev/null
+++ b/tests/test_hlstorage.py
@@ -0,0 +1,128 @@
+import shutil
+import tempfile
+import contextlib
+from typing import Tuple, Union, Dict, Any
+
+import numpy
+from oktest import ok
+
+
+from wally.result_classes import DataSource, TimeSeries, SuiteConfig
+from wally.suits.job import JobConfig, JobParams
+from wally.storage import make_storage
+from wally.hlstorage import ResultStorage
+
+
+@contextlib.contextmanager
+def in_temp_dir():
+ dname = tempfile.mkdtemp()
+ try:
+ yield dname
+ finally:
+ shutil.rmtree(dname)
+
+
+SUITE_ID = "suite1"
+JOB_ID = "job1"
+NODE_ID = "node1"
+SENSOR = "sensor"
+DEV = "dev"
+METRIC = "metric"
+TAG = "csv"
+DATA_UNITS = "x"
+TIME_UNITS = "us"
+
+
+class TestJobParams(JobParams):
+ def __init__(self) -> None:
+ JobParams.__init__(self)
+
+ @property
+ def summary(self) -> str:
+ return "UT_Job_CFG"
+
+ @property
+ def long_summary(self) -> str:
+ return "UT_Job_Config"
+
+ def copy(self, **updated) -> 'JobParams':
+ return self.__class__()
+
+ @property
+ def char_tpl(self) -> Tuple[Union[str, int, float, bool], ...]:
+ return (1, 2, 3)
+
+
+class TestJobConfig(JobConfig):
+ @property
+ def storage_id(self) -> str:
+ return JOB_ID
+
+ @property
+ def params(self) -> JobParams:
+ return TestJobParams()
+
+ def raw(self) -> Dict[str, Any]:
+ return {}
+
+ @classmethod
+ def fromraw(cls, data: Dict[str, Any]) -> 'TestJobConfig':
+ return cls()
+
+
+class TestSuiteConfig(SuiteConfig):
+ def __init__(self):
+ SuiteConfig.__init__(self, "UT", {}, "run_uuid", [], "/tmp", 0, False)
+ self.storage_id = SUITE_ID
+
+
+
+def test_sensor_ts():
+ with in_temp_dir() as root:
+ sensor_data = numpy.arange(5)
+ collected_at = numpy.arange(5) + 100
+
+ ds = DataSource(node_id=NODE_ID, sensor=SENSOR, dev=DEV, metric=METRIC)
+ cds = DataSource(node_id=NODE_ID, metric='collected_at')
+
+ with make_storage(root, existing=False) as storage:
+ rstorage = ResultStorage(storage)
+
+ rstorage.append_sensor(sensor_data, ds, units=DATA_UNITS, histo_bins=None)
+ rstorage.append_sensor(sensor_data, ds, units=DATA_UNITS, histo_bins=None)
+
+ rstorage.append_sensor(collected_at, cds, units=TIME_UNITS, histo_bins=None)
+ rstorage.append_sensor(collected_at + 5, cds, units=TIME_UNITS, histo_bins=None)
+
+ with make_storage(root, existing=True) as storage2:
+ rstorage2 = ResultStorage(storage2)
+ ts = rstorage2.load_sensor(ds)
+ assert (ts.data == numpy.array([0, 1, 2, 3, 4, 0, 1, 2, 3, 4])).all()
+ assert (ts.times == numpy.arange(10) + 100).all()
+
+
+def test_result_ts():
+ with in_temp_dir() as root:
+ sensor_data = numpy.arange(5, dtype=numpy.uint32)
+ collected_at = numpy.arange(5, dtype=numpy.uint32) + 100
+ ds = DataSource(suite_id=SUITE_ID, job_id=JOB_ID,
+ node_id=NODE_ID, sensor=SENSOR, dev=DEV, metric=METRIC, tag=TAG)
+
+ ts = TimeSeries("xxxx", None, sensor_data, collected_at, DATA_UNITS, ds, TIME_UNITS)
+
+ suite = TestSuiteConfig()
+ job = TestJobConfig(1)
+
+ with make_storage(root, existing=False) as storage:
+ rstorage = ResultStorage(storage)
+ rstorage.put_or_check_suite(suite)
+ rstorage.put_job(suite, job)
+ rstorage.put_ts(ts)
+
+ with make_storage(root, existing=True) as storage2:
+ rstorage2 = ResultStorage(storage2)
+ suits = list(rstorage2.iter_suite('UT'))
+ suits2 = list(rstorage2.iter_suite())
+ ok(len(suits)) == 1
+ ok(len(suits2)) == 1
+
diff --git a/tests/test_math.py b/tests/test_math.py
index 5a75858..d9c9688 100644
--- a/tests/test_math.py
+++ b/tests/test_math.py
@@ -1,5 +1,7 @@
import numpy
from wally.statistic import rebin_histogram
+from wally.result_classes import DataSource, TimeSeries
+from wally.data_selectors import interpolate_ts_on_seconds_border, c_interpolate_ts_on_seconds_border
def array_eq(x: numpy.array, y: numpy.array, max_diff: float = 1E-3) -> bool:
@@ -30,3 +32,63 @@
assert array_eq(new_edges, numpy.array([20, 30, 40]))
assert new_histo.sum() == curr_histo.sum()
assert list(new_histo) == [30, 10, 60]
+
+
+SUITE_ID = "suite1"
+JOB_ID = "job1"
+NODE_ID = "node1"
+SENSOR = "sensor"
+DEV = "dev"
+METRIC = "metric"
+TAG = "csv"
+DATA_UNITS = "x"
+TIME_UNITS = "ms"
+
+
+def test_interpolate():
+ ds = DataSource(node_id=NODE_ID, sensor=SENSOR, dev=DEV, metric=METRIC)
+ samples = 200
+ ms_coef = 1000
+ s_offset = 377 * ms_coef
+ ms_offset = 300 + s_offset
+
+ for i in range(16):
+ source_times = numpy.random.randint(100, size=samples, dtype='uint64') + \
+ ms_coef * numpy.arange(samples, dtype='uint64') + s_offset + ms_offset
+ source_values = numpy.random.randint(30, 60, size=samples, dtype='uint64')
+
+ ts = TimeSeries("test", raw=None, data=source_values, times=source_times, units=DATA_UNITS,
+ source=ds, time_units=TIME_UNITS)
+
+ # ts2 = interpolate_ts_on_seconds_border(ts)
+ ts2 = c_interpolate_ts_on_seconds_border(ts, nc=True)
+
+ # print()
+ # print(ts.times)
+ # print(ts.data, ts.data.sum())
+ # print(ts2.times)
+ # print(ts2.data, ts2.data.sum())
+
+ assert ts.time_units == 'ms'
+ assert ts2.time_units == 's'
+ assert ts2.times.dtype == ts.times.dtype
+ assert ts2.data.dtype == ts.data.dtype
+
+ assert ts.data.sum() == ts2.data.sum()
+
+ borders = 5
+ block_size = samples // 10
+ for begin_idx in numpy.random.randint(borders, samples - borders, size=20):
+ begin_idx = int(begin_idx)
+ end_idx = min(begin_idx + block_size, ts.times.size - 1)
+
+ first_cell_begin_time = ts.times[begin_idx - 1]
+ last_cell_end_time = ts.times[end_idx]
+ ts_sum = ts.data[begin_idx:end_idx].sum()
+
+ ts2_begin_idx = numpy.searchsorted(ts2.times, first_cell_begin_time // ms_coef)
+ ts2_end_idx = numpy.searchsorted(ts2.times, last_cell_end_time // ms_coef) + 1
+ ts2_max = ts.data[ts2_begin_idx: ts2_end_idx].sum()
+ ts2_min = ts.data[ts2_begin_idx + 1: ts2_end_idx - 1].sum()
+
+ assert ts2_min <= ts_sum <= ts2_max, "NOT {} <= {} <= {}".format(ts2_min, ts_sum, ts2_max)