| import os |
| import abc |
| import logging |
| import warnings |
| from io import BytesIO |
| from functools import wraps |
| from collections import defaultdict |
| from typing import Dict, Any, Iterator, Tuple, cast, List, Callable, Set |
| |
| import numpy |
| import scipy.stats |
| |
| # import matplotlib |
| # matplotlib.use('GTKAgg') |
| |
| from matplotlib.figure import Figure |
| import matplotlib.pyplot as plt |
| from matplotlib import gridspec |
| |
| from cephlib.common import float2str |
| from cephlib.plot import plot_hmap_with_y_histo, hmap_from_2d |
| |
| import wally |
| |
| from . import html |
| from .stage import Stage, StepOrder |
| from .test_run_class import TestRun |
| from .hlstorage import ResultStorage |
| from .node_interfaces import NodeInfo |
| from .utils import b2ssize, b2ssize_10, STORAGE_ROLES, unit_conversion_coef |
| from .statistic import (calc_norm_stat_props, calc_histo_stat_props, moving_average, moving_dev, |
| hist_outliers_perc, find_ouliers_ts, approximate_curve) |
| from .result_classes import (StatProps, DataSource, TimeSeries, NormStatProps, HistoStatProps, SuiteConfig) |
| from .suits.io.fio import FioTest, FioJobConfig |
| from .suits.io.fio_job import FioJobParams |
| from .suits.job import JobConfig |
| from .data_selectors import get_aggregated, AGG_TAG, summ_sensors, find_sensors_to_2d, find_nodes_by_roles |
| |
| |
| with warnings.catch_warnings(): |
| warnings.simplefilter("ignore") |
| import seaborn |
| |
| |
| logger = logging.getLogger("wally") |
| |
| |
| # ---------------- CONSTS --------------------------------------------------------------------------------------------- |
| |
| |
| DEBUG = False |
| LARGE_BLOCKS = 256 |
| |
| |
| # ---------------- PROFILES ------------------------------------------------------------------------------------------ |
| |
| |
| # this is default values, real values is loaded from config |
| |
| class ColorProfile: |
| primary_color = 'b' |
| suppl_color1 = 'teal' |
| suppl_color2 = 'magenta' |
| suppl_color3 = 'orange' |
| box_color = 'y' |
| err_color = 'red' |
| |
| noise_alpha = 0.3 |
| subinfo_alpha = 0.7 |
| |
| imshow_colormap = None # type: str |
| |
| |
| default_format = 'svg' |
| |
| |
| class StyleProfile: |
| dpi = 80 |
| grid = True |
| tide_layout = False |
| hist_boxes = 10 |
| hist_lat_boxes = 25 |
| hm_hist_bins_count = 25 |
| hm_x_slots = 25 |
| min_points_for_dev = 5 |
| |
| dev_range_x = 2.0 |
| dev_perc = 95 |
| |
| point_shape = 'o' |
| err_point_shape = '*' |
| |
| avg_range = 20 |
| approx_average = True |
| |
| curve_approx_level = 6 |
| curve_approx_points = 100 |
| assert avg_range >= min_points_for_dev |
| |
| # figure size in inches |
| figsize = (8, 4) |
| figsize_long = (5.5, 3) |
| |
| subplot_adjust_r = 0.75 |
| subplot_adjust_r_no_leg = 0.9 |
| title_font_size = 10 |
| |
| extra_io_spine = True |
| |
| legend_for_eng = True |
| # heatmap_interpolation = '1d' |
| heatmap_interpolation = None |
| heatmap_interpolation_points = 300 |
| outliers_q_nd = 3.0 |
| outliers_hide_q_nd = 4.0 |
| outliers_lat = (0.01, 0.9) |
| |
| violin_instead_of_box = True |
| violin_point_count = 30000 |
| |
| heatmap_colorbar = False |
| |
| min_iops_vs_qd_jobs = 3 |
| |
| qd_bins = [0, 1, 2, 4, 6, 8, 12, 16, 20, 26, 32, 40, 48, 56, 64, 96, 128] |
| iotime_bins = list(range(0, 1030, 50)) |
| block_size_bins = [0, 2, 4, 8, 16, 32, 48, 64, 96, 128, 192, 256, 384, 512, 1024, 2048] |
| |
| |
| DefColorProfile = ColorProfile() |
| DefStyleProfile = StyleProfile() |
| |
| |
| # ---------------- STRUCTS ------------------------------------------------------------------------------------------- |
| |
| |
| # TODO: need to be revised, have to user StatProps fields instead |
| class StoragePerfSummary: |
| def __init__(self, name: str) -> None: |
| self.direct_iops_r_max = 0 # type: int |
| self.direct_iops_w_max = 0 # type: int |
| |
| # 64 used instead of 4k to faster feed caches |
| self.direct_iops_w64_max = 0 # type: int |
| |
| self.rws4k_10ms = 0 # type: int |
| self.rws4k_30ms = 0 # type: int |
| self.rws4k_100ms = 0 # type: int |
| self.bw_write_max = 0 # type: int |
| self.bw_read_max = 0 # type: int |
| |
| self.bw = None # type: float |
| self.iops = None # type: float |
| self.lat = None # type: float |
| self.lat_50 = None # type: float |
| self.lat_95 = None # type: float |
| |
| |
| class IOSummary: |
| def __init__(self, |
| qd: int, |
| block_size: int, |
| nodes_count:int, |
| bw: NormStatProps, |
| lat: HistoStatProps) -> None: |
| |
| self.qd = qd |
| self.nodes_count = nodes_count |
| self.block_size = block_size |
| |
| self.bw = bw |
| self.lat = lat |
| |
| |
| # -------------- AGGREGATION AND STAT FUNCTIONS ---------------------------------------------------------------------- |
| |
| def make_iosum(rstorage: ResultStorage, suite: SuiteConfig, job: FioJobConfig) -> IOSummary: |
| lat = get_aggregated(rstorage, suite, job, "lat") |
| io = get_aggregated(rstorage, suite, job, "bw") |
| |
| return IOSummary(job.qd, |
| nodes_count=len(suite.nodes_ids), |
| block_size=job.bsize, |
| lat=calc_histo_stat_props(lat, rebins_count=StyleProfile.hist_boxes), |
| bw=calc_norm_stat_props(io, StyleProfile.hist_boxes)) |
| |
| |
| def is_sensor_numarray(sensor: str, metric: str) -> bool: |
| """Returns True if sensor provides one-dimension array of numeric values. One number per one measurement.""" |
| return True |
| |
| |
| LEVEL_SENSORS = {("block-io", "io_queue"), |
| ("system-cpu", "procs_blocked"), |
| ("system-cpu", "procs_queue")} |
| |
| |
| def is_level_sensor(sensor: str, metric: str) -> bool: |
| """Returns True if sensor measure level of any kind, E.g. queue depth.""" |
| return (sensor, metric) in LEVEL_SENSORS |
| |
| |
| def is_delta_sensor(sensor: str, metric: str) -> bool: |
| """Returns True if sensor provides deltas for cumulative value. E.g. io completed in given period""" |
| return not is_level_sensor(sensor, metric) |
| |
| # -------------- PLOT HELPERS FUNCTIONS ------------------------------------------------------------------------------ |
| |
| def get_emb_image(fig: Figure, format: str, **opts) -> bytes: |
| bio = BytesIO() |
| if format == 'svg': |
| fig.savefig(bio, format='svg', **opts) |
| img_start = "<!-- Created with matplotlib (http://matplotlib.org/) -->" |
| return bio.getvalue().decode("utf8").split(img_start, 1)[1].encode("utf8") |
| else: |
| fig.savefig(bio, format=format, **opts) |
| return bio.getvalue() |
| |
| |
| def provide_plot(func: Callable[..., None]) -> Callable[..., str]: |
| @wraps(func) |
| def closure1(storage: ResultStorage, |
| path: DataSource, |
| *args, **kwargs) -> str: |
| fpath = storage.check_plot_file(path) |
| if not fpath: |
| format = path.tag.split(".")[-1] |
| fig = plt.figure(figsize=StyleProfile.figsize) |
| func(fig, *args, **kwargs) |
| fpath = storage.put_plot_file(get_emb_image(fig, format=format, dpi=DefStyleProfile.dpi), path) |
| logger.debug("Plot %s saved to %r", path, fpath) |
| plt.close(fig) |
| return fpath |
| return closure1 |
| |
| |
| def apply_style(fig: Figure, title: str, style: StyleProfile, eng: bool = True, no_legend: bool = False) -> None: |
| |
| for ax in fig.axes: |
| ax.grid(style.grid) |
| |
| if (style.legend_for_eng or not eng) and not no_legend: |
| fig.subplots_adjust(right=StyleProfile.subplot_adjust_r) |
| legend_location = "center left" |
| legend_bbox_to_anchor = (1.03, 0.81) |
| for ax in fig.axes: |
| ax.legend(loc=legend_location, bbox_to_anchor=legend_bbox_to_anchor) |
| else: |
| fig.subplots_adjust(right=StyleProfile.subplot_adjust_r_no_leg) |
| |
| if style.tide_layout: |
| fig.set_tight_layout(True) |
| |
| fig.suptitle(title, fontsize=style.title_font_size) |
| |
| |
| # -------------- PLOT FUNCTIONS -------------------------------------------------------------------------------------- |
| |
| |
| @provide_plot |
| def plot_hist(fig: Figure, title: str, units: str, |
| prop: StatProps, |
| colors: ColorProfile = DefColorProfile, |
| style: StyleProfile = DefStyleProfile) -> None: |
| |
| ax = fig.add_subplot(111) |
| |
| # TODO: unit should came from ts |
| normed_bins = prop.bins_populations / prop.bins_populations.sum() |
| bar_width = prop.bins_edges[1] - prop.bins_edges[0] |
| ax.bar(prop.bins_edges, normed_bins, color=colors.box_color, width=bar_width, label="Real data") |
| |
| ax.set(xlabel=units, ylabel="Value probability") |
| |
| dist_plotted = False |
| if isinstance(prop, NormStatProps): |
| nprop = cast(NormStatProps, prop) |
| stats = scipy.stats.norm(nprop.average, nprop.deviation) |
| |
| new_edges, step = numpy.linspace(prop.bins_edges[0], prop.bins_edges[-1], |
| len(prop.bins_edges) * 10, retstep=True) |
| |
| ypoints = stats.cdf(new_edges) * 11 |
| ypoints = [next - prev for (next, prev) in zip(ypoints[1:], ypoints[:-1])] |
| xpoints = (new_edges[1:] + new_edges[:-1]) / 2 |
| |
| ax.plot(xpoints, ypoints, color=colors.primary_color, label="Expected from\nnormal\ndistribution") |
| dist_plotted = True |
| |
| ax.set_xlim(left=prop.bins_edges[0]) |
| if prop.log_bins: |
| ax.set_xscale('log') |
| |
| apply_style(fig, title, style, eng=True, no_legend=not dist_plotted) |
| |
| |
| @provide_plot |
| def plot_simple_over_time(fig: Figure, |
| tss: List[Tuple[str, numpy.ndarray]], |
| title: str, |
| ylabel: str, |
| xlabel: str = "time, s", |
| average: bool = False, |
| colors: ColorProfile = DefColorProfile, |
| style: StyleProfile = DefStyleProfile) -> None: |
| ax = fig.add_subplot(111) |
| for name, arr in tss: |
| if average: |
| avg_vals = moving_average(arr, style.avg_range) |
| if style.approx_average: |
| time_points = numpy.arange(len(avg_vals)) |
| avg_vals = approximate_curve(time_points, avg_vals, time_points, style.curve_approx_level) |
| arr = avg_vals |
| ax.plot(arr, label=name) |
| ax.set(xlabel=xlabel, ylabel=ylabel) |
| apply_style(fig, title, style, eng=True) |
| |
| |
| @provide_plot |
| def plot_hmap_from_2d(fig: Figure, |
| data2d: numpy.ndarray, |
| title: str, ylabel: str, xlabel: str = 'time, s', bins: numpy.ndarray = None, |
| colors: ColorProfile = DefColorProfile, style: StyleProfile = DefStyleProfile) -> None: |
| fig.set_size_inches(*style.figsize_long) |
| ioq1d, ranges = hmap_from_2d(data2d) |
| ax, _ = plot_hmap_with_y_histo(fig, ioq1d, ranges, bins=bins) |
| ax.set(ylabel=ylabel, xlabel=xlabel) |
| apply_style(fig, title, style, no_legend=True) |
| |
| |
| @provide_plot |
| def plot_v_over_time(fig: Figure, |
| title: str, |
| units: str, |
| ts: TimeSeries, |
| plot_avg_dev: bool = True, |
| plot_points: bool = True, |
| colors: ColorProfile = DefColorProfile, |
| style: StyleProfile = DefStyleProfile) -> None: |
| |
| min_time = min(ts.times) |
| |
| # convert time to ms |
| coef = float(unit_conversion_coef(ts.time_units, 's')) |
| time_points = numpy.array([(val_time - min_time) * coef for val_time in ts.times]) |
| |
| outliers_idxs = find_ouliers_ts(ts.data, cut_range=style.outliers_q_nd) |
| outliers_4q_idxs = find_ouliers_ts(ts.data, cut_range=style.outliers_hide_q_nd) |
| normal_idxs = numpy.logical_not(outliers_idxs) |
| outliers_idxs = outliers_idxs & numpy.logical_not(outliers_4q_idxs) |
| # hidden_outliers_count = numpy.count_nonzero(outliers_4q_idxs) |
| |
| data = ts.data[normal_idxs] |
| data_times = time_points[normal_idxs] |
| outliers = ts.data[outliers_idxs] |
| outliers_times = time_points[outliers_idxs] |
| |
| ax = fig.add_subplot(111) |
| |
| if plot_points: |
| alpha = colors.noise_alpha if plot_avg_dev else 1.0 |
| ax.plot(data_times, data, style.point_shape, |
| color=colors.primary_color, alpha=alpha, label="Data") |
| ax.plot(outliers_times, outliers, style.err_point_shape, |
| color=colors.err_color, label="Outliers") |
| |
| has_negative_dev = False |
| plus_minus = "\xb1" |
| |
| if plot_avg_dev and len(data) < style.avg_range * 2: |
| logger.warning("Array %r to small to plot average over %s points", title, style.avg_range) |
| elif plot_avg_dev: |
| avg_vals = moving_average(data, style.avg_range) |
| dev_vals = moving_dev(data, style.avg_range) |
| avg_times = moving_average(data_times, style.avg_range) |
| |
| if style.approx_average: |
| avg_vals = approximate_curve(avg_times, avg_vals, avg_times, style.curve_approx_level) |
| dev_vals = approximate_curve(avg_times, dev_vals, avg_times, style.curve_approx_level) |
| |
| ax.plot(avg_times, avg_vals, c=colors.suppl_color1, label="Average") |
| |
| low_vals_dev = avg_vals - dev_vals * style.dev_range_x |
| hight_vals_dev = avg_vals + dev_vals * style.dev_range_x |
| if style.dev_range_x - int(style.dev_range_x) < 0.01: |
| ax.plot(avg_times, low_vals_dev, c=colors.suppl_color2, |
| label="{}{}*stdev".format(plus_minus, int(style.dev_range_x))) |
| else: |
| ax.plot(avg_times, low_vals_dev, c=colors.suppl_color2, |
| label="{}{}*stdev".format(plus_minus, style.dev_range_x)) |
| ax.plot(avg_times, hight_vals_dev, c=colors.suppl_color2) |
| has_negative_dev = low_vals_dev.min() < 0 |
| |
| ax.set_xlim(-5, max(time_points) + 5) |
| ax.set_xlabel("Time, seconds from test begin") |
| |
| if plot_avg_dev: |
| ax.set_ylabel("{}. Average and {}stddev over {} points".format(units, plus_minus, style.avg_range)) |
| else: |
| ax.set_ylabel(units) |
| |
| if has_negative_dev: |
| ax.set_ylim(bottom=0) |
| |
| apply_style(fig, title, style, eng=True) |
| |
| |
| @provide_plot |
| def plot_lat_over_time(fig: Figure, |
| title: str, |
| ts: TimeSeries, |
| ylabel: str, |
| samples: int = 5, |
| colors: ColorProfile = DefColorProfile, style: StyleProfile = DefStyleProfile) -> None: |
| |
| min_time = min(ts.times) |
| times = [int(tm - min_time + 500) // 1000 for tm in ts.times] |
| ts_len = len(times) |
| step = ts_len / samples |
| points = [times[int(i * step + 0.5)] for i in range(samples)] |
| points.append(times[-1]) |
| bounds = list(zip(points[:-1], points[1:])) |
| agg_data = [] |
| positions = [] |
| labels = [] |
| |
| for begin, end in bounds: |
| agg_hist = ts.data[begin:end].sum(axis=0) |
| |
| if style.violin_instead_of_box: |
| # cut outliers |
| idx1, idx2 = hist_outliers_perc(agg_hist, style.outliers_lat) |
| agg_hist = agg_hist[idx1:idx2] |
| curr_bins_vals = ts.histo_bins[idx1:idx2] |
| |
| correct_coef = style.violin_point_count / sum(agg_hist) |
| if correct_coef > 1: |
| correct_coef = 1 |
| else: |
| curr_bins_vals = ts.histo_bins |
| correct_coef = 1 |
| |
| vals = numpy.empty(shape=[numpy.sum(agg_hist)], dtype='float32') |
| cidx = 0 |
| |
| non_zero, = agg_hist.nonzero() |
| for pos in non_zero: |
| count = int(agg_hist[pos] * correct_coef + 0.5) |
| |
| if count != 0: |
| vals[cidx: cidx + count] = curr_bins_vals[pos] |
| cidx += count |
| |
| agg_data.append(vals[:cidx]) |
| positions.append((end + begin) / 2) |
| labels.append(str((end + begin) // 2)) |
| |
| ax = fig.add_subplot(111) |
| if style.violin_instead_of_box: |
| patches = ax.violinplot(agg_data, |
| positions=positions, |
| showmeans=True, |
| showmedians=True, |
| widths=step / 2) |
| |
| patches['cmeans'].set_color("blue") |
| patches['cmedians'].set_color("green") |
| if style.legend_for_eng: |
| legend_location = "center left" |
| legend_bbox_to_anchor = (1.03, 0.81) |
| plt.legend([patches['cmeans'], patches['cmedians']], ["mean", "median"], |
| loc=legend_location, bbox_to_anchor=legend_bbox_to_anchor) |
| else: |
| ax.boxplot(agg_data, 0, '', positions=positions, labels=labels, widths=step / 4) |
| |
| ax.set_xlim(min(times), max(times)) |
| ax.set(ylabel=ylabel, xlabel="Time, seconds from test begin, sampled for ~{} seconds".format(int(step))) |
| |
| apply_style(fig, title, style, eng=True, no_legend=True) |
| |
| |
| @provide_plot |
| def plot_histo_heatmap(fig: Figure, |
| title: str, |
| ts: TimeSeries, |
| ylabel: str, |
| xlabel: str = "time, s", |
| colors: ColorProfile = DefColorProfile, style: StyleProfile = DefStyleProfile) -> None: |
| |
| fig.set_size_inches(*style.figsize_long) |
| |
| # only histogram-based ts can be plotted |
| assert len(ts.data.shape) == 2 |
| |
| # Find global outliers. As load is expected to be stable during one job |
| # outliers range can be detected globally |
| total_hist = ts.data.sum(axis=0) |
| idx1, idx2 = hist_outliers_perc(total_hist, |
| bounds_perc=style.outliers_lat, |
| min_bins_left=style.hm_hist_bins_count) |
| |
| # merge outliers with most close non-outliers cell |
| orig_data = ts.data[:, idx1:idx2].copy() |
| if idx1 > 0: |
| orig_data[:, 0] += ts.data[:, :idx1].sum(axis=1) |
| |
| if idx2 < ts.data.shape[1]: |
| orig_data[:, -1] += ts.data[:, idx2:].sum(axis=1) |
| |
| bins_vals = ts.histo_bins[idx1:idx2] |
| |
| # rebin over X axis |
| # aggregate some lines in ts.data to plot not more than style.hm_x_slots x bins |
| agg_idx = float(len(orig_data)) / style.hm_x_slots |
| if agg_idx >= 2: |
| data = numpy.zeros([style.hm_x_slots, orig_data.shape[1]], dtype=numpy.float32) # type: List[numpy.ndarray] |
| next = agg_idx |
| count = 0 |
| data_idx = 0 |
| for idx, arr in enumerate(orig_data): |
| if idx >= next: |
| data[data_idx] /= count |
| data_idx += 1 |
| next += agg_idx |
| count = 0 |
| data[data_idx] += arr |
| count += 1 |
| |
| if count > 1: |
| data[-1] /= count |
| else: |
| data = orig_data |
| |
| # rebin over Y axis |
| # ================= |
| |
| # don't using rebin_histogram here, as we need apply same bins for many arrays |
| step = (bins_vals[-1] - bins_vals[0]) / style.hm_hist_bins_count |
| new_bins_edges = numpy.arange(style.hm_hist_bins_count) * step + bins_vals[0] |
| bin_mapping = numpy.clip(numpy.searchsorted(new_bins_edges, bins_vals) - 1, 0, len(new_bins_edges) - 1) |
| |
| # map origin bins ranges to heatmap bins, iterate over rows |
| cmap = [] |
| for line in data: |
| curr_bins = [0] * style.hm_hist_bins_count |
| for idx, count in zip(bin_mapping, line): |
| curr_bins[idx] += count |
| cmap.append(curr_bins) |
| ncmap = numpy.array(cmap) |
| |
| # plot data |
| # ========= |
| |
| boxes = 3 |
| gs = gridspec.GridSpec(1, boxes) |
| ax = fig.add_subplot(gs[0, :boxes - 1]) |
| |
| labels = list(map(float2str, (new_bins_edges[:-1] + new_bins_edges[1:]) / 2)) + \ |
| [float2str(new_bins_edges[-1]) + "+"] |
| seaborn.heatmap(ncmap[:,::-1].T, xticklabels=False, cmap="Blues", ax=ax) |
| ax.set_yticklabels(labels, rotation='horizontal') |
| ax.set_xticklabels([]) |
| |
| # plot overall histogram |
| # ======================= |
| |
| ax2 = fig.add_subplot(gs[0, boxes - 1]) |
| ax2.set_yticklabels([]) |
| ax2.set_xticklabels([]) |
| |
| histo = ncmap.sum(axis=0).reshape((-1,)) |
| ax2.set_ylim(top=histo.size, bottom=0) |
| ax2.barh(numpy.arange(histo.size) + 0.5, width=histo) |
| |
| ax.set(ylabel=ylabel, xlabel=xlabel) |
| |
| apply_style(fig, title, style, eng=True, no_legend=True) |
| |
| |
| @provide_plot |
| def io_chart(fig: Figure, |
| title: str, |
| legend: str, |
| iosums: List[IOSummary], |
| iops_log_spine: bool = False, |
| lat_log_spine: bool = False, |
| colors: ColorProfile = DefColorProfile, style: StyleProfile = DefStyleProfile) -> None: |
| |
| # -------------- MAGIC VALUES --------------------- |
| # IOPS bar width |
| width = 0.35 |
| |
| # offset from center of bar to deviation/confidence range indicator |
| err_x_offset = 0.05 |
| |
| # extra space on top and bottom, comparing to maximal tight layout |
| extra_y_space = 0.05 |
| |
| # additional spine for BW/IOPS on left side of plot |
| extra_io_spine_x_offset = -0.1 |
| |
| # extra space on left and right sides |
| extra_x_space = 0.5 |
| |
| # legend location settings |
| legend_location = "center left" |
| legend_bbox_to_anchor = (1.1, 0.81) |
| |
| # plot box size adjust (only plot, not spines and legend) |
| plot_box_adjust = {'right': 0.66} |
| # -------------- END OF MAGIC VALUES --------------------- |
| |
| block_size = iosums[0].block_size |
| lc = len(iosums) |
| xt = list(range(1, lc + 1)) |
| |
| # x coordinate of middle of the bars |
| xpos = [i - width / 2 for i in xt] |
| |
| # import matplotlib.gridspec as gridspec |
| # gs = gridspec.GridSpec(1, 3, width_ratios=[1, 4, 1]) |
| # p1 = plt.subplot(gs[1]) |
| |
| logger.warning("Check coef usage!") |
| ax = fig.add_subplot(111) |
| |
| # plot IOPS/BW bars |
| if block_size >= LARGE_BLOCKS: |
| iops_primary = False |
| coef = float(unit_conversion_coef(iosums[0].bw.units, "MiBps")) |
| ax.set_ylabel("BW (MiBps)") |
| else: |
| iops_primary = True |
| coef = float(unit_conversion_coef(iosums[0].bw.units, "MiBps")) / block_size |
| ax.set_ylabel("IOPS") |
| |
| vals = [iosum.bw.average * coef for iosum in iosums] |
| |
| ax.bar(xpos, vals, width=width, color=colors.box_color, label=legend) |
| |
| # set correct x limits for primary IO spine |
| min_io = min(iosum.bw.average - iosum.bw.deviation * style.dev_range_x for iosum in iosums) |
| max_io = max(iosum.bw.average + iosum.bw.deviation * style.dev_range_x for iosum in iosums) |
| border = (max_io - min_io) * extra_y_space |
| io_lims = (min_io - border, max_io + border) |
| |
| ax.set_ylim(io_lims[0] * coef, io_lims[-1] * coef) |
| |
| # plot deviation and confidence error ranges |
| err1_legend = err2_legend = None |
| for pos, iosum in zip(xpos, iosums): |
| err1_legend = ax.errorbar(pos + width / 2 - err_x_offset, |
| iosum.bw.average * coef, |
| iosum.bw.deviation * style.dev_range_x * coef, |
| alpha=colors.subinfo_alpha, |
| color=colors.suppl_color1) # 'magenta' |
| err2_legend = ax.errorbar(pos + width / 2 + err_x_offset, |
| iosum.bw.average * coef, |
| iosum.bw.confidence * coef, |
| alpha=colors.subinfo_alpha, |
| color=colors.suppl_color2) # 'teal' |
| |
| if style.grid: |
| ax.grid(True) |
| |
| handles1, labels1 = ax.get_legend_handles_labels() |
| |
| handles1 += [err1_legend, err2_legend] |
| labels1 += ["{}% dev".format(style.dev_perc), |
| "{}% conf".format(int(100 * iosums[0].bw.confidence_level))] |
| |
| # extra y spine for latency on right side |
| ax2 = ax.twinx() |
| |
| # plot median and 95 perc latency |
| ax2.plot(xt, [iosum.lat.perc_50 for iosum in iosums], label="lat med") |
| ax2.plot(xt, [iosum.lat.perc_95 for iosum in iosums], label="lat 95%") |
| |
| # limit and label x spine |
| plt.xlim(extra_x_space, lc + extra_x_space) |
| plt.xticks(xt, ["{0} * {1}".format(iosum.qd, iosum.nodes_count) for iosum in iosums]) |
| ax.set_xlabel("QD * Test node count") |
| |
| # apply log scales for X spines, if set |
| if iops_log_spine: |
| ax.set_yscale('log') |
| |
| if lat_log_spine: |
| ax2.set_yscale('log') |
| |
| # extra y spine for BW/IOPS on left side |
| if style.extra_io_spine: |
| ax3 = ax.twinx() |
| if iops_log_spine: |
| ax3.set_yscale('log') |
| |
| if iops_primary: |
| ax3.set_ylabel("BW (MiBps)") |
| ax3.set_ylim(io_lims[0] * coef, io_lims[1] * coef) |
| else: |
| ax3.set_ylabel("IOPS") |
| ax3.set_ylim(io_lims[0] * coef, io_lims[1] * coef) |
| |
| ax3.spines["left"].set_position(("axes", extra_io_spine_x_offset)) |
| ax3.spines["left"].set_visible(True) |
| ax3.yaxis.set_label_position('left') |
| ax3.yaxis.set_ticks_position('left') |
| |
| ax2.set_ylabel("Latency (ms)") |
| |
| # legend box |
| handles2, labels2 = ax2.get_legend_handles_labels() |
| plt.legend(handles1 + handles2, labels1 + labels2, |
| loc=legend_location, |
| bbox_to_anchor=legend_bbox_to_anchor) |
| |
| # adjust central box size to fit legend |
| # plt.subplots_adjust(**plot_box_adjust) |
| apply_style(fig, title, style, eng=False, no_legend=True) |
| |
| |
| # -------------------- REPORT HELPERS -------------------------------------------------------------------------------- |
| |
| |
| class HTMLBlock: |
| data = None # type: str |
| js_links = [] # type: List[str] |
| css_links = [] # type: List[str] |
| order_attr = None # type: Any |
| |
| def __init__(self, data: str, order_attr: Any = None) -> None: |
| self.data = data |
| self.order_attr = order_attr |
| |
| def __eq__(self, o: Any) -> bool: |
| return o.order_attr == self.order_attr # type: ignore |
| |
| def __lt__(self, o: Any) -> bool: |
| return o.order_attr > self.order_attr # type: ignore |
| |
| |
| class Table: |
| def __init__(self, header: List[str]) -> None: |
| self.header = header |
| self.data = [] |
| |
| def add_line(self, values: List[str]) -> None: |
| self.data.append(values) |
| |
| def html(self): |
| return html.table("", self.header, self.data) |
| |
| |
| class Menu1st: |
| engineering = "Engineering" |
| summary = "Summary" |
| per_job = "Per Job" |
| |
| |
| class Menu2ndEng: |
| iops_time = "IOPS(time)" |
| hist = "IOPS/lat overall histogram" |
| lat_time = "Lat(time)" |
| |
| |
| class Menu2ndSumm: |
| io_lat_qd = "IO & Lat vs QD" |
| |
| |
| menu_1st_order = [Menu1st.summary, Menu1st.engineering, Menu1st.per_job] |
| |
| |
| # -------------------- REPORTS -------------------------------------------------------------------------------------- |
| |
| |
| class Reporter(metaclass=abc.ABCMeta): |
| suite_types = set() # type: Set[str] |
| |
| @abc.abstractmethod |
| def get_divs(self, suite: SuiteConfig, storage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| pass |
| |
| |
| class JobReporter(metaclass=abc.ABCMeta): |
| suite_type = set() # type: Set[str] |
| |
| @abc.abstractmethod |
| def get_divs(self, |
| suite: SuiteConfig, |
| job: JobConfig, |
| storage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| pass |
| |
| |
| # Main performance report |
| class PerformanceSummary(Reporter): |
| """Aggregated summary fro storage""" |
| |
| |
| # Main performance report |
| class IO_QD(Reporter): |
| """Creates graph, which show how IOPS and Latency depend on QD""" |
| suite_types = {'fio'} |
| |
| def get_divs(self, suite: SuiteConfig, rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| ts_map = defaultdict(list) # type: Dict[FioJobParams, List[Tuple[SuiteConfig, FioJobConfig]]] |
| str_summary = {} # type: Dict[FioJobParams, List[IOSummary]] |
| for job in rstorage.iter_job(suite): |
| fjob = cast(FioJobConfig, job) |
| fjob_no_qd = cast(FioJobParams, fjob.params.copy(qd=None)) |
| str_summary[fjob_no_qd] = (fjob_no_qd.summary, fjob_no_qd.long_summary) |
| ts_map[fjob_no_qd].append((suite, fjob)) |
| |
| for tpl, suites_jobs in ts_map.items(): |
| if len(suites_jobs) > StyleProfile.min_iops_vs_qd_jobs: |
| iosums = [make_iosum(rstorage, suite, job) for suite, job in suites_jobs] |
| iosums.sort(key=lambda x: x.qd) |
| summary, summary_long = str_summary[tpl] |
| ds = DataSource(suite_id=suite.storage_id, |
| job_id=summary, |
| node_id=AGG_TAG, |
| sensor="fio", |
| dev=AGG_TAG, |
| metric="io_over_qd", |
| tag="svg") |
| |
| title = "IOPS, BW, Lat vs. QD.\n" + summary_long |
| fpath = io_chart(rstorage, ds, title=title, legend="IOPS/BW", iosums=iosums) # type: str |
| yield Menu1st.summary, Menu2ndSumm.io_lat_qd, HTMLBlock(html.img(fpath)) |
| |
| |
| # Linearization report |
| class IOPS_Bsize(Reporter): |
| """Creates graphs, which show how IOPS and Latency depend on block size""" |
| |
| |
| # IOPS/latency distribution |
| class StatInfo(JobReporter): |
| """Statistic info for job results""" |
| suite_types = {'fio'} |
| |
| def get_divs(self, suite: SuiteConfig, job: JobConfig, |
| rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| |
| fjob = cast(FioJobConfig, job) |
| io_sum = make_iosum(rstorage, suite, fjob) |
| |
| summary_data = [ |
| ["Summary", job.params.long_summary], |
| ] |
| |
| res = html.H2(html.center("Test summary")) |
| res += html.table("Test info", None, summary_data) |
| stat_data_headers = ["Name", "Average ~ Dev", "Conf interval", "Mediana", "Mode", "Kurt / Skew", "95%", "99%"] |
| |
| bw_target_units = 'Bps' |
| bw_coef = float(unit_conversion_coef(io_sum.bw.units, bw_target_units)) |
| |
| bw_data = ["Bandwidth", |
| "{}{} ~ {}{}".format(b2ssize(io_sum.bw.average * bw_coef), bw_target_units, |
| b2ssize(io_sum.bw.deviation * bw_coef), bw_target_units), |
| b2ssize(io_sum.bw.confidence * bw_coef) + bw_target_units, |
| b2ssize(io_sum.bw.perc_50 * bw_coef) + bw_target_units, |
| "-", |
| "{:.2f} / {:.2f}".format(io_sum.bw.kurt, io_sum.bw.skew), |
| b2ssize(io_sum.bw.perc_5 * bw_coef) + bw_target_units, |
| b2ssize(io_sum.bw.perc_1 * bw_coef) + bw_target_units] |
| |
| iops_coef = float(unit_conversion_coef(io_sum.bw.units, 'KiBps')) / fjob.bsize |
| iops_data = ["IOPS", |
| "{}IOPS ~ {}IOPS".format(b2ssize_10(io_sum.bw.average * iops_coef), |
| b2ssize_10(io_sum.bw.deviation * iops_coef)), |
| b2ssize_10(io_sum.bw.confidence * iops_coef) + "IOPS", |
| b2ssize_10(io_sum.bw.perc_50 * iops_coef) + "IOPS", |
| "-", |
| "{:.2f} / {:.2f}".format(io_sum.bw.kurt, io_sum.bw.skew), |
| b2ssize_10(io_sum.bw.perc_5 * iops_coef) + "IOPS", |
| b2ssize_10(io_sum.bw.perc_1 * iops_coef) + "IOPS"] |
| |
| lat_target_unit = 's' |
| lat_coef = unit_conversion_coef(io_sum.lat.units, lat_target_unit) |
| # latency |
| lat_data = ["Latency", |
| "-", |
| "-", |
| b2ssize_10(io_sum.lat.perc_50 * lat_coef) + lat_target_unit, |
| "-", |
| "-", |
| b2ssize_10(io_sum.lat.perc_95 * lat_coef) + lat_target_unit, |
| b2ssize_10(io_sum.lat.perc_99 * lat_coef) + lat_target_unit] |
| |
| # sensor usage |
| stat_data = [iops_data, bw_data, lat_data] |
| res += html.table("Load stats info", stat_data_headers, stat_data) |
| |
| resource_headers = ["Resource", "Usage count", "Proportional to work done"] |
| |
| tot_io_coef = float(unit_conversion_coef(io_sum.bw.units, "KiBps")) |
| tot_ops_coef = tot_io_coef / fjob.bsize |
| |
| io_transfered = io_sum.bw.data.sum() * tot_io_coef |
| resource_data = [ |
| ["IO made", b2ssize_10(io_transfered * tot_ops_coef) + "OP", "-"], |
| ["Data transfered", b2ssize(io_transfered) + "B", "-"] |
| ] |
| |
| storage = rstorage.storage |
| nodes = storage.load_list(NodeInfo, 'all_nodes') # type: List[NodeInfo] |
| |
| ops_done = io_transfered * tot_ops_coef |
| |
| all_metrics = [ |
| ("Test nodes net send", 'net-io', 'send_bytes', b2ssize, ['testnode'], "B", io_transfered), |
| ("Test nodes net recv", 'net-io', 'recv_bytes', b2ssize, ['testnode'], "B", io_transfered), |
| |
| ("Test nodes disk write", 'block-io', 'sectors_written', b2ssize, ['testnode'], "B", io_transfered), |
| ("Test nodes disk read", 'block-io', 'sectors_read', b2ssize, ['testnode'], "B", io_transfered), |
| ("Test nodes writes", 'block-io', 'writes_completed', b2ssize_10, ['testnode'], "OP", ops_done), |
| ("Test nodes reads", 'block-io', 'reads_completed', b2ssize_10, ['testnode'], "OP", ops_done), |
| |
| ("Storage nodes net send", 'net-io', 'send_bytes', b2ssize, STORAGE_ROLES, "B", io_transfered), |
| ("Storage nodes net recv", 'net-io', 'recv_bytes', b2ssize, STORAGE_ROLES, "B", io_transfered), |
| |
| ("Storage nodes disk write", 'block-io', 'sectors_written', b2ssize, STORAGE_ROLES, "B", io_transfered), |
| ("Storage nodes disk read", 'block-io', 'sectors_read', b2ssize, STORAGE_ROLES, "B", io_transfered), |
| ("Storage nodes writes", 'block-io', 'writes_completed', b2ssize_10, STORAGE_ROLES, "OP", ops_done), |
| ("Storage nodes reads", 'block-io', 'reads_completed', b2ssize_10, STORAGE_ROLES, "OP", ops_done), |
| ] |
| |
| all_agg = {} |
| |
| for descr, sensor, metric, ffunc, roles, units, denom in all_metrics: |
| if not nodes: |
| continue |
| |
| res_ts = summ_sensors(rstorage, roles, sensor=sensor, metric=metric, time_range=job.reliable_info_range_s) |
| if res_ts is None: |
| continue |
| |
| agg = res_ts.data.sum() |
| resource_data.append([descr, ffunc(agg) + units, "{:.1f}".format(agg / denom)]) |
| all_agg[descr] = agg |
| |
| cums = [ |
| ("Test nodes writes", "Test nodes reads", "Total test ops", b2ssize_10, "OP", ops_done), |
| ("Storage nodes writes", "Storage nodes reads", "Total storage ops", b2ssize_10, "OP", ops_done), |
| ("Storage nodes disk write", "Storage nodes disk read", "Total storage IO size", b2ssize, |
| "B", io_transfered), |
| ("Test nodes disk write", "Test nodes disk read", "Total test nodes IO size", b2ssize, "B", io_transfered), |
| ] |
| |
| for name1, name2, descr, ffunc, units, denom in cums: |
| if name1 in all_agg and name2 in all_agg: |
| agg = all_agg[name1] + all_agg[name2] |
| resource_data.append([descr, ffunc(agg) + units, "{:.1f}".format(agg / denom)]) |
| |
| res += html.table("Resources usage", resource_headers, resource_data) |
| |
| yield Menu1st.per_job, job.summary, HTMLBlock(res) |
| |
| |
| # CPU load |
| class CPULoadPlot(JobReporter): |
| def get_divs(self, |
| suite: SuiteConfig, |
| job: JobConfig, |
| rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| |
| # plot CPU time |
| for rt, roles in [('storage', STORAGE_ROLES), ('test', ['testnode'])]: |
| cpu_ts = {} |
| cpu_metrics = "idle guest iowait irq nice sirq steal sys user".split() |
| for name in cpu_metrics: |
| cpu_ts[name] = summ_sensors(rstorage, roles, sensor='system-cpu', metric=name, |
| time_range=job.reliable_info_range_s) |
| |
| it = iter(cpu_ts.values()) |
| total_over_time = next(it).data.copy() |
| for ts in it: |
| total_over_time += ts.data |
| |
| fname = plot_simple_over_time(rstorage, |
| cpu_ts['idle'].source(job_id=job.storage_id, |
| suite_id=suite.storage_id, |
| metric='allcpu', tag=rt + '.plt.' + default_format), |
| tss=[(name, ts.data * 100 / total_over_time) for name, ts in cpu_ts.items()], |
| average=True, |
| ylabel="CPU time %", |
| title="{} nodes CPU usage".format(rt.capitalize())) |
| |
| yield Menu1st.per_job, job.summary, HTMLBlock(html.img(fname)) |
| |
| |
| # IO time and QD |
| class QDIOTimeHeatmap(JobReporter): |
| def get_divs(self, |
| suite: SuiteConfig, |
| job: JobConfig, |
| rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| |
| # TODO: fix this hardcode, need to track what devices are actually used on test and storage nodes |
| # use saved storage info in nodes |
| |
| journal_devs = None |
| storage_devs = None |
| test_nodes_devs = ['rbd0'] |
| |
| for node in find_nodes_by_roles(rstorage, STORAGE_ROLES): |
| cjd = set(node.params['ceph_journal_devs']) |
| if journal_devs is None: |
| journal_devs = cjd |
| else: |
| assert journal_devs == cjd, "{!r} != {!r}".format(journal_devs, cjd) |
| |
| csd = set(node.params['ceph_storage_devs']) |
| if storage_devs is None: |
| storage_devs = csd |
| else: |
| assert storage_devs == csd, "{!r} != {!r}".format(storage_devs, csd) |
| |
| trange = (job.reliable_info_range[0] // 1000, job.reliable_info_range[1] // 1000) |
| |
| for name, devs, roles in [('storage', storage_devs, STORAGE_ROLES), |
| ('journal', journal_devs, STORAGE_ROLES), |
| ('test', test_nodes_devs, ['testnode'])]: |
| # QD heatmap |
| ioq2d = find_sensors_to_2d(rstorage, roles, sensor='block-io', devs=devs, |
| metric='io_queue', time_range=trange) |
| fname = plot_hmap_from_2d(rstorage, DataSource(suite.storage_id, |
| job.storage_id, |
| AGG_TAG, |
| 'block-io', |
| name, |
| metric='io_queue', |
| tag="hmap." + default_format), |
| ioq2d, ylabel="IO QD", title=name.capitalize() + " devs QD", |
| bins=StyleProfile.qd_bins, |
| xlabel='Time') # type: str |
| yield Menu1st.per_job, job.summary, HTMLBlock(html.img(fname)) |
| |
| # Block size heatmap |
| wc2d = find_sensors_to_2d(rstorage, roles, sensor='block-io', devs=devs, |
| metric='writes_completed', time_range=trange) |
| wc2d[wc2d < 1E-3] = 1 |
| sw2d = find_sensors_to_2d(rstorage, roles, sensor='block-io', devs=devs, |
| metric='sectors_written', time_range=trange) |
| data2d = sw2d / wc2d / 1024 |
| fname = plot_hmap_from_2d(rstorage, DataSource(suite.storage_id, |
| job.storage_id, |
| AGG_TAG, |
| 'block-io', |
| name, |
| metric='wr_block_size', |
| tag="hmap." + default_format), |
| data2d, ylabel="IO bsize, KiB", title=name.capitalize() + " write block size", |
| xlabel='Time', |
| bins=StyleProfile.block_size_bins) # type: str |
| yield Menu1st.per_job, job.summary, HTMLBlock(html.img(fname)) |
| |
| # iotime heatmap |
| wtime2d = find_sensors_to_2d(rstorage, roles, sensor='block-io', devs=devs, |
| metric='io_time', time_range=trange) |
| fname = plot_hmap_from_2d(rstorage, DataSource(suite.storage_id, |
| job.storage_id, |
| AGG_TAG, |
| 'block-io', |
| name, |
| metric='io_time', |
| tag="hmap." + default_format), |
| wtime2d, ylabel="IO time (ms) per second", |
| title=name.capitalize() + " iotime", |
| xlabel='Time', |
| bins=StyleProfile.iotime_bins) # type: str |
| yield Menu1st.per_job, job.summary, HTMLBlock(html.img(fname)) |
| |
| |
| # IOPS/latency distribution |
| class IOHist(JobReporter): |
| """IOPS.latency distribution histogram""" |
| suite_types = {'fio'} |
| |
| def get_divs(self, |
| suite: SuiteConfig, |
| job: JobConfig, |
| rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| |
| fjob = cast(FioJobConfig, job) |
| |
| yield Menu1st.per_job, fjob.summary, HTMLBlock(html.H2(html.center("Load histograms"))) |
| |
| # agg_lat = get_aggregated(rstorage, suite, fjob, "lat") |
| # # bins_edges = numpy.array(get_lat_vals(agg_lat.data.shape[1]), dtype='float32') / 1000 # convert us to ms |
| # lat_stat_prop = calc_histo_stat_props(agg_lat, bins_edges=None, rebins_count=StyleProfile.hist_lat_boxes) |
| # |
| # long_summary = cast(FioJobParams, fjob.params).long_summary |
| # |
| # title = "Latency distribution" |
| # units = "ms" |
| # |
| # fpath = plot_hist(rstorage, agg_lat.source(tag='hist.svg'), title, units, lat_stat_prop) # type: str |
| # yield Menu1st.per_job, fjob.summary, HTMLBlock(html.img(fpath)) |
| |
| agg_io = get_aggregated(rstorage, suite, fjob, "bw") |
| |
| if fjob.bsize >= LARGE_BLOCKS: |
| title = "BW distribution" |
| units = "MiBps" |
| agg_io.data //= int(unit_conversion_coef(units, agg_io.units)) |
| else: |
| title = "IOPS distribution" |
| agg_io.data //= (int(unit_conversion_coef("KiBps", agg_io.units)) * fjob.bsize) |
| units = "IOPS" |
| |
| io_stat_prop = calc_norm_stat_props(agg_io, bins_count=StyleProfile.hist_boxes) |
| fpath = plot_hist(rstorage, agg_io.source(tag='hist.' + default_format), |
| title, units, io_stat_prop) # type: str |
| yield Menu1st.per_job, fjob.summary, HTMLBlock(html.img(fpath)) |
| |
| |
| # IOPS/latency over test time for each job |
| class IOTime(JobReporter): |
| """IOPS/latency during test""" |
| suite_types = {'fio'} |
| |
| def get_divs(self, |
| suite: SuiteConfig, |
| job: JobConfig, |
| rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| |
| fjob = cast(FioJobConfig, job) |
| |
| agg_io = get_aggregated(rstorage, suite, fjob, "bw") |
| if fjob.bsize >= LARGE_BLOCKS: |
| title = "Fio measured Bandwidth over time" |
| units = "MiBps" |
| agg_io.data //= int(unit_conversion_coef(units, agg_io.units)) |
| else: |
| title = "Fio measured IOPS over time" |
| agg_io.data //= (int(unit_conversion_coef("KiBps", agg_io.units)) * fjob.bsize) |
| units = "IOPS" |
| |
| fpath = plot_v_over_time(rstorage, agg_io.source(tag='ts.' + default_format), title, units, agg_io) # type: str |
| yield Menu1st.per_job, fjob.summary, HTMLBlock(html.img(fpath)) |
| |
| agg_lat = get_aggregated(rstorage, suite, fjob, "lat").copy() |
| TARGET_UNITS = 'ms' |
| coef = unit_conversion_coef(agg_lat.units, TARGET_UNITS) |
| agg_lat.histo_bins = agg_lat.histo_bins.copy() * float(coef) |
| agg_lat.units = TARGET_UNITS |
| |
| fpath = plot_lat_over_time(rstorage, agg_lat.source(tag='ts.' + default_format), "Latency", |
| agg_lat, ylabel="Latency, " + agg_lat.units) # type: str |
| yield Menu1st.per_job, fjob.summary, HTMLBlock(html.img(fpath)) |
| |
| fpath = plot_histo_heatmap(rstorage, |
| agg_lat.source(tag='hmap.' + default_format), |
| "Latency heatmap", |
| agg_lat, |
| ylabel="Latency, " + agg_lat.units, |
| xlabel='Test time') # type: str |
| |
| yield Menu1st.per_job, fjob.summary, HTMLBlock(html.img(fpath)) |
| |
| |
| class ResourceUsage: |
| def __init__(self, io_r_ops: int, io_w_ops: int, io_r_kb: int, io_w_kb: int) -> None: |
| self.io_w_ops = io_w_ops |
| self.io_r_ops = io_r_ops |
| self.io_w_kb = io_w_kb |
| self.io_r_kb = io_r_kb |
| |
| self.cpu_used_user = None # type: int |
| self.cpu_used_sys = None # type: int |
| self.cpu_wait_io = None # type: int |
| |
| self.net_send_packets = None # type: int |
| self.net_recv_packets = None # type: int |
| self.net_send_kb = None # type: int |
| self.net_recv_kb = None # type: int |
| |
| |
| # Cluster load over test time |
| class ClusterLoad(JobReporter): |
| """IOPS/latency during test""" |
| |
| # TODO: units should came from sensor |
| storage_sensors = [ |
| ('block-io', 'reads_completed', "Read", 'iop'), |
| ('block-io', 'writes_completed', "Write", 'iop'), |
| ('block-io', 'sectors_read', "Read", 'KiB'), |
| ('block-io', 'sectors_written', "Write", 'KiB'), |
| ] |
| |
| def get_divs(self, |
| suite: SuiteConfig, |
| job: JobConfig, |
| rstorage: ResultStorage) -> Iterator[Tuple[str, str, HTMLBlock]]: |
| yield Menu1st.per_job, job.summary, HTMLBlock(html.H2(html.center("Cluster load"))) |
| |
| for sensor, metric, op, units in self.storage_sensors: |
| ts = summ_sensors(rstorage, ['testnode'], sensor, metric, job.reliable_info_range_s) |
| ds = DataSource(suite_id=suite.storage_id, |
| job_id=job.storage_id, |
| node_id="test_nodes", |
| sensor=sensor, |
| dev=AGG_TAG, |
| metric=metric, |
| tag="ts." + default_format) |
| |
| data = ts.data if units != 'KiB' else ts.data * float(unit_conversion_coef(ts.units, 'KiB')) |
| ts = TimeSeries(name="", |
| times=numpy.arange(*job.reliable_info_range_s), |
| data=data, |
| raw=None, |
| units=units if ts.units is None else ts.units, |
| time_units=ts.time_units, |
| source=ds, |
| histo_bins=ts.histo_bins) |
| |
| sensor_title = "{} {}".format(op, units) |
| fpath = plot_v_over_time(rstorage, ds, sensor_title, sensor_title, ts=ts) # type: str |
| yield Menu1st.per_job, job.summary, HTMLBlock(html.img(fpath)) |
| |
| |
| # Ceph cluster summary |
| class ResourceConsumption(Reporter): |
| """Resources consumption report, only text""" |
| |
| |
| # Node load over test time |
| class NodeLoad(Reporter): |
| """IOPS/latency during test""" |
| |
| |
| # Ceph cluster summary |
| class CephClusterSummary(Reporter): |
| """IOPS/latency during test""" |
| |
| |
| # TODO: Ceph operation breakout report |
| # TODO: Resource consumption for different type of test |
| |
| |
| # ------------------------------------------ REPORT STAGES ----------------------------------------------------------- |
| |
| |
| class HtmlReportStage(Stage): |
| priority = StepOrder.REPORT |
| |
| def run(self, ctx: TestRun) -> None: |
| rstorage = ResultStorage(ctx.storage) |
| |
| job_reporters = [StatInfo(), IOTime(), IOHist(), ClusterLoad(), CPULoadPlot(), |
| QDIOTimeHeatmap()] # type: List[JobReporter] |
| reporters = [] |
| |
| # reporters = [IO_QD()] # type: List[Reporter] |
| # job_reporters = [ClusterLoad()] |
| |
| root_dir = os.path.dirname(os.path.dirname(wally.__file__)) |
| doc_templ_path = os.path.join(root_dir, "report_templates/index.html") |
| report_template = open(doc_templ_path, "rt").read() |
| css_file_src = os.path.join(root_dir, "report_templates/main.css") |
| css_file = open(css_file_src, "rt").read() |
| |
| menu_block = [] |
| content_block = [] |
| link_idx = 0 |
| |
| # matplotlib.rcParams.update(ctx.config.reporting.matplotlib_params.raw()) |
| # ColorProfile.__dict__.update(ctx.config.reporting.colors.raw()) |
| # StyleProfile.__dict__.update(ctx.config.reporting.style.raw()) |
| |
| items = defaultdict(lambda: defaultdict(list)) # type: Dict[str, Dict[str, List[HTMLBlock]]] |
| DEBUG = False |
| # TODO: filter reporters |
| for suite in rstorage.iter_suite(FioTest.name): |
| all_jobs = list(rstorage.iter_job(suite)) |
| all_jobs.sort(key=lambda job: job.params) |
| for job in all_jobs: |
| for reporter in job_reporters: |
| logger.debug("Start reporter %s on job %s suite %s", |
| reporter.__class__.__name__, job.summary, suite.test_type) |
| for block, item, html in reporter.get_divs(suite, job, rstorage): |
| items[block][item].append(html) |
| if DEBUG: |
| break |
| |
| for reporter in reporters: |
| logger.debug("Start reporter %s on suite %s", reporter.__class__.__name__, suite.test_type) |
| for block, item, html in reporter.get_divs(suite, rstorage): |
| items[block][item].append(html) |
| |
| if DEBUG: |
| break |
| |
| logger.debug("Generating result html") |
| |
| for idx_1st, menu_1st in enumerate(sorted(items, key=lambda x: menu_1st_order.index(x))): |
| menu_block.append( |
| '<a href="#item{}" class="nav-group" data-toggle="collapse" data-parent="#MainMenu">{}</a>' |
| .format(idx_1st, menu_1st) |
| ) |
| menu_block.append('<div class="collapse" id="item{}">'.format(idx_1st)) |
| for menu_2nd in sorted(items[menu_1st]): |
| menu_block.append(' <a href="#content{}" class="nav-group-item">{}</a>' |
| .format(link_idx, menu_2nd)) |
| content_block.append('<div id="content{}">'.format(link_idx)) |
| content_block.extend(" " + x.data for x in items[menu_1st][menu_2nd]) |
| content_block.append('</div>') |
| link_idx += 1 |
| menu_block.append('</div>') |
| |
| report = report_template.replace("{{{menu}}}", ("\n" + " " * 16).join(menu_block)) |
| report = report.replace("{{{content}}}", ("\n" + " " * 16).join(content_block)) |
| report_path = rstorage.put_report(report, "index.html") |
| rstorage.put_report(css_file, "main.css") |
| logger.info("Report is stored into %r", report_path) |