Cfg-checker ceph benchmark & info updates and fixes
- Added collecting Ceph global stats while running benchmark
- Added collecting osd pg dump data
- Added page with active OSD nodes stats
- --report-only option, ceph info still collected
Fixes:
- fio-runner uses scheduled time when reporting errors
- proper ceph pv creation
- updated retry decorator timeouts for overloaded envs
- calculated volume size creation with size*1.3
- Proper maximum values indication
Related-PROD: PROD-36669
Change-Id: Ic518ddbb2ca0915b550e981d0b0fc7084000aa04
diff --git a/templates/ceph_bench_html.j2 b/templates/ceph_bench_html.j2
index c05a412..229a553 100644
--- a/templates/ceph_bench_html.j2
+++ b/templates/ceph_bench_html.j2
@@ -2,7 +2,7 @@
<html lang="en">
<head>
<meta charset="UTF-8">
- <title>Ceph cluster info</title>
+ <title>Ceph cluster benchmark</title>
{% include 'common_styles.j2' %}
{% include 'common_scripts.j2' %}
{% include 'bar_chart.j2' %}
@@ -21,7 +21,11 @@
float: none;
transform: translate(25%);
}
-
+ .inlineheader {
+ background-color: lightgray;
+ padding-left: 40px;
+ margin-bottom: 10px;
+ }
/* Node rows*/
.node {
font-family: "LaoSangamMN", Monaco, monospace;
@@ -30,6 +34,9 @@
background-color: white;
align-items: center;
}
+ .node:hover, .node:active {
+ background-color: #eda;
+ }
.collapsable {
font-family: "LaoSangamMN", Monaco, monospace;
font-size: 0.8em;
@@ -98,10 +105,10 @@
div.services > .collapsable.in {
display: table-row;
}
- tr:nth-child(even) {
+ .agents:nth-child(even) {
background-color: #eee;
}
- tr:nth-child(odd) {
+ .agents:nth-child(odd) {
background-color: #fff;
}
@@ -125,6 +132,21 @@
padding-right: 0px;
margin: 1px;
}
+ td > .osd_props_group {
+ display: grid;
+ grid-template-columns: 50px 50px 50px 50px;
+ padding-left: 0px;
+ padding-right: 0px;
+ margin: 1px;
+ }
+ td > .osd_stats_group {
+ display: grid;
+ grid-template-columns: 80px 110px 110px 110px 100px 100px 110px 140px;
+ padding-left: 0px;
+ padding-right: 0px;
+ margin: 1px;
+
+ }
td > .pg_group {
display: grid;
grid-template-columns: 50px 40px 60px 65px 60px 65px 65px;;
@@ -335,6 +357,33 @@
</style>
</head>
+
+{% macro put_osd_prop(prop, b, a, p) %}
+ {% if prop in p %}
+ <div class="item bench">{{ a[prop] | to_mb }} ({{ "%0.4f" | format(p[prop]|float) }})</div>
+ {% else %}
+ <div class="item bench">{{ a[prop] | to_mb }}</div>
+ {% endif %}
+{% endmacro %}
+
+{% macro put_osd_perc(prop, b, a, p) %}
+ {% if prop in p %}
+ <div class="item bench">{{ "%0.4f"|format(a[prop]|float) }} ({{ "%0.4f" | format(p[prop]|float) }})</div>
+ {% else %}
+ <div class="item bench">{{ "%0.4f"|format(a[prop]|float) }}</div>
+ {% endif %}
+{% endmacro %}
+
+{% macro summary_value(prop, s) %}
+ {% if prop in s %}
+ <div class="item bench">{{ s[prop] }} nodes</div>
+ {% else %}
+ <div class="item bench">0</div>
+ {% endif %}
+{% endmacro %}
+
+
+
<body onload="init()">
<div class="header">
@@ -348,6 +397,7 @@
<div class="bar">
<div class="bar-centered">
<button class="bar-item" onclick="openBar(event, 'bench')">Benchmark Results</button>
+ <button class="bar-item" onclick="openBar(event, 'osdstats')">OSD Stats</button>
<button class="bar-item" onclick="openBar(event, 'status')">Status</button>
<!-- <button class="bar-item" onclick="openBar(event, 'latency')">Latency</button> -->
</div>
@@ -400,7 +450,7 @@
<div class="item prop">{{ t["storage_class"] }}</div>
<div class="item pg">{{ t["storage_class_stats"]["num_pg"] }}</div>
<div class="item prop">{{ o["ioengine"] }}</div>
- <div class="item prop">{{ o["readwrite"] }} ({{ o["rwmixread"] }}/{{ 100-o["rwmixread"] }})</div>
+ <div class="item prop">{{ o["readwrite"] }} ({{ o["rwmixread"] }}/{{ 100 - (o["rwmixread"]|int) }})</div>
<div class="item prop">{{ o["bs"] }}</div>
<div class="item prop">{{ o["iodepth"] }}</div>
<div class="item prop">{{ o["size"] }}</div>
@@ -417,35 +467,118 @@
</div>
</td>
</tr>
+ {% set c = dt["ceph"] %}
<tr class="collapsable" id="timing_{{ tstripped }}_data"><td colspan=3>
+ <div class="inlineheader">Global READ stats, MB/s vs seconds. Measured maximum is <b>{{ c["max_rbl"][0] | to_mb }}</b> MB/sec</div>
<div class="bc-wrap">
<div class="bctimecol">
- <div class="bctime"><span class="bctimetext">110</span></div>
- <div class="bctime"><span class="bctimetext">75</span></div>
- <div class="bctime"><span class="bctimetext">50</span></div>
- <div class="bctime"><span class="bctimetext">15</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][1] | to_mb }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][2] | to_mb }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][3] | to_mb }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][4] | to_mb }}</span></div>
</div>
<div class="bc-container">
<div class="bc">
- <div class="bccol"><div class="bcbar" style="height: 75%;"></div><div class="bcfooter">2s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 25%;"></div><div class="bcfooter">4s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 55%;"></div><div class="bcfooter">6s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 65%;"></div><div class="bcfooter">8s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 15%;"></div><div class="bcfooter">10s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 16%;"></div><div class="bcfooter">12s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 17%;"></div><div class="bcfooter">14s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 18%;"></div><div class="bcfooter">16s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 19%;"></div><div class="bcfooter">18s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 20%;"></div><div class="bcfooter">20s</div></div>
- <div class="bccol"><div class="bcbar" style="height: 21%;"></div><div class="bcfooter">22s</div></div>
+ {% for sec, c_data in c["stats"].items() %}
+ {% set elapsed = sec | float %}
+ <div class="bccol">
+ {% if sec == c["max_rbl_time"] %}
+ <div class="bcheader">{{ c_data["read_bytes_sec"] | to_mb }}</div>
+ <div class="bcbar green-bar" style="height: {{ c_data["read_bytes_sec_perc"] }};"></div>
+ {% else %}
+ <div class="bcbar" style="height: {{ c_data["read_bytes_sec_perc"] }};"></div>
+ {% endif%}
+ <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
+ </div>
+ {% endfor %}
</div>
</div>
</div>
+ <div class="inlineheader">Global READ stats, IOPS vs seconds. Measured maximum is <b>{{ c["max_ril"][0] }}</b> op/sec</div>
+ <div class="bc-wrap">
+ <div class="bctimecol">
+ <div class="bctime"><span class="bctimetext">{{ c["max_ril"][1] }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_ril"][2] }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_ril"][3] }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_ril"][4] }}</span></div>
+ </div>
+
+ <div class="bc-container">
+ <div class="bc">
+ {% for sec, c_data in c["stats"].items() %}
+ {% set elapsed = sec | float %}
+ <div class="bccol">
+ {% if sec == c["max_ril_time"] %}
+ <div class="bcheader">{{ c_data["read_op_per_sec"] }}</div>
+ <div class="bcbar green-bar" style="height: {{ c_data["read_op_per_sec_perc"] }};"></div>
+ {% else %}
+ <div class="bcbar" style="height: {{ c_data["read_op_per_sec_perc"] }};"></div>
+ {% endif%}
+ <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
+ </div>
+ {% endfor %}
+ </div>
+ </div>
+ </div>
+ <div class="inlineheader">Global WRITE stats, MB/s vs seconds. Measured maximum is <b>{{ c["max_wbl"][0] | to_mb }}</b> MB/sec</div>
+ <div class="bc-wrap">
+ <div class="bctimecol">
+ <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][1] | to_mb }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][2] | to_mb }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][3] | to_mb }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][4] | to_mb }}</span></div>
+ </div>
+
+ <div class="bc-container">
+ <div class="bc">
+ {% for sec, c_data in c["stats"].items() %}
+ {% set elapsed = sec | float %}
+ <div class="bccol">
+ {% if sec == c["max_wbl_time"] %}
+ <div class="bcheader">{{ c_data["write_bytes_sec"] | to_mb }}</div>
+ <div class="bcbar green-bar" style="height: {{ c_data["write_bytes_sec_perc"] }};"></div>
+ {% else %}
+ <div class="bcbar" style="height: {{ c_data["write_bytes_sec_perc"] }};"></div>
+ {% endif%}
+ <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
+ </div>
+ {% endfor %}
+ </div>
+ </div>
+ </div>
+
+ <div class="inlineheader">Global WRITE stats, IOPS vs seconds. Measured maximum is <b>{{ c["max_wil"][0] }}</b> op/sec</div>
+ <div class="bc-wrap">
+ <div class="bctimecol">
+ <div class="bctime"><span class="bctimetext">{{ c["max_wil"][1] }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_wil"][2] }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_wil"][3] }}</span></div>
+ <div class="bctime"><span class="bctimetext">{{ c["max_wil"][4] }}</span></div>
+ </div>
+
+ <div class="bc-container">
+ <div class="bc">
+ {% for sec, c_data in c["stats"].items() %}
+ {% set elapsed = sec | float %}
+ <div class="bccol">
+ {% if sec == c["max_wil_time"] %}
+ <div class="bcheader">{{ c_data["write_op_per_sec"] }}</div>
+ <div class="bcbar green-bar" style="height: {{ c_data["write_op_per_sec_perc"] }};"></div>
+ {% else %}
+ <div class="bcbar" style="height: {{ c_data["write_op_per_sec_perc"] }};"></div>
+ {% endif%}
+ <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
+ </div>
+ {% endfor %}
+ </div>
+ </div>
+ </div>
+ <div class="inlineheader">Per Agent stats</div>
<table style="table-layout: auto;"><tbody>
{% for agent,ag_result in dt["agents"].items() %}
{% set j = ag_result["jobs"][0] %}
- <tr>
+ <tr class="agents">
<td class="status">{{ time }}</td>
<td class="status">{{ agent }}</td>
<td class="col_properties">
@@ -455,7 +588,7 @@
<div class="item prop">{{ t["storage_class"] }}</div>
<div class="item pg">{{ t["storage_class_stats"]["num_pg"] }}</div>
<div class="item prop">{{ o["ioengine"] }}</div>
- <div class="item prop">{{ o["readwrite"] }} ({{ o["rwmixread"] }}/{{ 100-o["rwmixread"] }})</div>
+ <div class="item prop">{{ o["readwrite"] }} ({{ o["rwmixread"] }}/{{ 100 - (o["rwmixread"]|int) }})</div>
<div class="item prop">{{ j["job options"]["bs"] }}</div>
<div class="item prop">{{ o["iodepth"] }}</div>
<div class="item prop">{{ j["job options"]["size"] }}</div>
@@ -481,6 +614,105 @@
</div>
{% endmacro %}
+<!-- OSD stats -->
+{% macro osds_page(results, id_label) %}
+<div id="{{ id_label }}" class="barcontent">
+ <h5>{{ caller() }}</h5>
+ <hr>
+ <table class="ceph_status">
+ <tr class="node">
+ <td class="status">Time started</td>
+ <td class="status">Data point</td>
+ <td class="col_properties">
+ <div class="osd_props_group">
+ <div class="item prop">Status</div>
+ <div class="item prop">Class</div>
+ <div class="item prop">Weight</div>
+ <div class="item pg">PGs</div>
+ </div>
+ </td>
+ <td class="col_bench">
+ <div class="osd_stats_group">
+ <div class="item bench">Total, GB</div>
+ <div class="item bench">Avail., GB</div>
+ <div class="item bench">Used, GB</div>
+ <div class="item bench">Data, GB</div>
+ <div class="item bench">OMAP, GB</div>
+ <div class="item bench">Meta, GB</div>
+ <div class="item bench">Utilized, %</div>
+ <div class="item bench">Variance, %</div>
+ </div>
+ </td>
+ </tr>
+ {% for time,dt in results.items() %}
+ {% set b = dt["osd_summary"]["before"] %}
+ {% set a = dt["osd_summary"]["after"] %}
+ {% set s = dt["osd_summary"]["active"] %}
+ {% set tstripped = time | tstrip %}
+ <tr class="node" onclick="toggleClassByID('timing_{{ tstripped }}_osds')" id="timing_{{ tstripped }}_button">
+ <td class="status">{{ time }}</td>
+ <td class="status">Active nodes</td>
+ <td class="col_properties">
+ <div class="osd_props_group">
+ <div class="item prop">{{ s["status"] }}</div>
+ <div class="item prop">{{ s["device_class"] }}</div>
+ <div class="item prop">−</div>
+ <div class="item pg">{{ s["pgs"] }}</div>
+ </div>
+ </td>
+ <td class="col_bench">
+ <div class="osd_stats_group">
+ <div class="item bench">{{ a["total_kb"] | to_mb }}</div>
+ {{ summary_value("kb_avail", s) }}
+ {{ summary_value("kb_used", s) }}
+ {{ summary_value("kb_used_data", s) }}
+ {{ summary_value("kb_used_omap", s) }}
+ {{ summary_value("kb_used_meta", s) }}
+ {{ summary_value("utilization", s) }}
+ <div class="item bench">{{ s["var_down"] }}↓ / {{ s["var_up"] }}↑</div>
+ </div>
+ </td>
+ </tr>
+ <tr class="collapsable" id="timing_{{ tstripped }}_osds"><td colspan=3>
+ <table style="table-layout: auto;"><tbody>
+ {% for osd in dt["osds"].keys() | sort %}
+ {% set n = dt["osds"][osd] %}
+ {% set b = n["before"] %}
+ {% set a = n["after"] %}
+ {% set p = n["percent"] %}
+ <tr class="agents">
+ <td class="status">{{ time }}</td>
+ <td class="status">{{ osd }}</td>
+ <td class="col_properties">
+ <div class="osd_props_group">
+ <div class="item prop">{{ a["status"] }}</div>
+ <div class="item prop">{{ a["device_class"] }}</div>
+ <div class="item prop">{{ "%0.4f" | format(a["crush_weight"]|float) }}</div>
+ <div class="item pg">{{ a["pgs"] }}</div>
+ </div>
+ </td>
+ <td class="col_bench">
+ <div class="osd_stats_group">
+ <div class="item bench">{{ a["kb"] | to_mb }}</div>
+ {{ put_osd_prop("kb_avail", b, a, p) }}
+ {{ put_osd_prop("kb_used", b, a, p) }}
+ {{ put_osd_prop("kb_used_data", b, a, p) }}
+ {{ put_osd_prop("kb_used_omap", b, a, p) }}
+ {{ put_osd_prop("kb_used_meta", b, a, p) }}
+ {{ put_osd_perc("utilization", b, a, p) }}
+ {{ put_osd_perc("var", b, a, p) }}
+ </div>
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody></table>
+ </td></tr>
+ {% endfor %}
+ </table>
+</div>
+{% endmacro %}
+
+
<!-- Status page -->
{% macro status_page(info, id_label) %}
<div id="{{ id_label }}" class="barcontent">
@@ -671,6 +903,10 @@
Benchmark results
{% endcall %}
+{% call osds_page(results, "osdstats") %}
+ OSD nodes stats collected before and after each step
+{% endcall %}
+
{% call status_page(info, "status") %}
Cluster status
{% endcall %}