<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Ceph cluster benchmark</title>
    {% include 'common_styles.j2' %}
    {% include 'common_scripts.j2' %}
    {% include 'bar_chart.j2' %}
    <style>
        table.cluster_nodes {
            width: 100%;
            margin-left: 1%;
            margin-right: 1%;
        }
        .barcontent {
            margin: auto;
            width: 1350px;
            padding: 10px;
        }
        .bar-centered {
            float: none;
            transform: translate(25%);
        }
        .inlineheader {
            background-color: lightgray;
            padding-left: 40px;
            margin-bottom: 10px;
        }
        /* Node rows*/
		.node {
			font-family: "LaoSangamMN", Monaco, monospace;
			font-size: 0.8em;
			display: flex;
			background-color: white;
            align-items: center;
		}
        .node:hover, .node:active {
            background-color: #eda;
        }
		.collapsable {
			font-family: "LaoSangamMN", Monaco, monospace;
			font-size: 0.8em;
			display: none;
            background-color: white;
            visibility: hidden;
            width: 100%;
            border-style: dashed;
            border-width: 1px;
        }
        .collapsable.in {
            visibility: visible;
            display: inline-block;
        }

        .row_button {
            background-color: #468;
            color: #fff;
            cursor: pointer;
            padding: 5px;
            width: 100%;
            border: none;
            text-align: left;
            outline: none;
            font-size: 13px;
        }
        .row_button:after {
            content: '\02795'; /* Unicode character for "plus" sign (+) */
            font-size: 13px;
            color: white;
            float: left;
            margin-left: 5px;
        }
          
        .row_active:after {
            content: "\2796"; /* Unicode character for "minus" sign (-) */
            color: white
        }        

        .row_active, .row_button:hover {
            background-color: #68a;
            color: white
        }

        .cell_button {
            color: darkgreen;
            cursor: pointer;
            padding: 5px;
            width: 100%;
            border: none;
            text-align: center;
            outline: none;
        }
        .cell_button:hover {
            background-color: gray;
        }
        
        .row_content {
            padding: 0 18px;
            background-color: white;
            max-height: 0;
            overflow: hidden;
            transition: max-height 0.2s ease-out;
            border-width: 1px;
            border-color: #68a;
            border-style: solid;
        }

        div.services > .collapsable.in {
            display: table-row;
        }
        .agents:nth-child(even) {
            background-color: #eee;
        }
        .agents:nth-child(odd) {
            background-color: #fff;
        }
	
        tr.node > td, tr.collapsable > td {
            display: block;
            float: left;
            padding: 1px;
            margin: 2px;
        }
        td > .osd_group {
            display: grid;
            grid-template-columns: 40px 25px 25px 70px;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        td > .props_group {
            display: grid;
            grid-template-columns: 60px 80px 35px 45px 95px 110px;;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        td > .osd_props_group {
            display: grid;
            grid-template-columns: 50px 50px 50px 50px;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        td > .osd_stats_group {
            display: grid;
            grid-template-columns: 80px 110px 110px 110px 100px 100px 110px 140px;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;

        }
        td > .pg_group {
            display: grid;
            grid-template-columns: 50px 40px 60px 65px 60px 65px 65px;;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        td > .bench_run_group {
            display: grid;
            grid-template-columns: 80px 110px 80px 80px 110px 80px;;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        td > .bench_group {
            display: grid;
            grid-template-columns: 80px 80px 75px 75px;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        td > .meta_group {
            display: inline-block;
            grid-template-columns: repeat(4, auto);
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
        }
        .item {
  			border-width: 1px;
			border-style: solid;
			margin: 1px 1px 1px 1px;
            padding: 0px 1px 0px 1px;
        }

        .details-wrap { margin-left: 20px;}
	    .spacer { border-radius: 2px; width: 20px;}
        .bench_id { border-radius: 10px; width: 50px; text-align: center;}
        .time { border-radius: 10px; width: 160px; text-align: center;}
        .status { border-radius: 10px; width: 120px; text-align: center;}
	    .health_ok { background-color: #393; color: white;}
        .health_error { background-color: #933; color: white;}
        .health_warn { background-color: #eb3; color: #333;}
        .checks_code { border-radius: 2px; width: 20%; background-color: transparent; color: darkred;}

        .head { height: 18px; background-color: transparent; border-color: transparent; border: 0px;}
        .centered { text-align: center;}
        .right { text-align: right;}
	    .col_shortmessage {	min-width: 300px; }
        .col_longmessage {	width: auto; }
        .col_properties { width: auto; border-radius: 10px;}
        .col_bench { width: auto; border-radius: 10px;}

        .srv_name { width: 300px }
        .srv_path { width: 250px }
        .srv_timestamp { width: 250px }
        .srv_addr { width: 450px }

        .id { width: 30px }
        .bucket_name { width: 365px }
        .bucket_type { width: 50px }
        .bucket_params { width: 200px }
        .bucket_items { width: 630px }

        .df_name { width: 300px }
        .df_total { width: 150px }
        .df_avail { width: 150px }
        .df_used { width: 150px }
        .df_used_raw { width: 150px }
        .df_used_raw_rate { width: 150px }

        .rdf_name { width: 200px; }
        .rdf_obj { width: 75px; }
        .rdf_total { width: 100px; }
        .rdf_used { width: 100px; }
        .rdf_bench { width: 100px; }

        .dev_name { width: 300px; }
        .dev_param { width: 100px; }

        .mon_name { width: 100px }
        .mon_url { width: 500px }

        .meters {
            display: inline-block;
            margin: 1px;
        }
        .meters > .meter {
            display: block;
            float: left;
  			border-width: 1px;
			border-style: solid;
			margin: 0px 1px 0px 1px;
            padding: 0px 1px 0px 1px;
            
        }
        .meters > .warn {
			border-color: #d3a200;
			background-color: rgb(255, 216, 133);
        }
        .meters > .fail {
			border-color: #bb0000;
			background-color: rgb(250, 135, 135);
        }
        .osd { border-color: #a0c0a0; background-color: rgb(252, 248, 248); text-align: center;}
        .prop { border-color: #74c28b; background-color: rgb(252, 248, 248); text-align: center;}
        .pg { border-color: #c0c0a0; background-color: rgb(255, 255, 251); text-align: right; }
        .bench { border-color: #a0c0c0; background-color: rgb(255, 250, 250); text-align: right; }
        .lat_commit { border-color: #a0c0c0; background-color: rgb(255, 250, 250); text-align: right; width: 45px}
        .lat_apply { border-color: #a0c0c0; background-color: rgb(255, 250, 250); text-align: left;  width: 35px}
        .meta_name { border-color: #c4b890; background-color: #e7dbb6; text-align: left; width: 150px;}
        .meta_value { border-color: #c6c3ba;background-color: #d4d4d4; text-align: left; width: 480px;}

        .map_grid {
            display: grid;
            grid-template-columns: auto auto auto auto auto auto auto auto auto auto;
            grid-column-gap: 20px;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
            margin-left: 20px;

        }
        .map_item {
        	display: inline-grid;
            border-width: 0px;
			border-style: solid;
			margin: 1px 1px 1px 1px;
            padding: 0px 1px 0px 1px;
        }

        .map_grid > .ok {
            color: #80a080;
        }
        .map_grid > .warn {
            color: #d3a200;
        }
        .map_grid > .fail {
            color: #bb0000;
        }

        .modules {
			font-family: "LaoSangamMN", Monaco, monospace;
			font-size: 0.8em;
			background-color: white;
        }
        .module_node {
            margin-bottom: 2px;
            display: flex;
        }
        .module_name, .node_name {
            text-align: center;
            border-width: 0px;
			border-style: solid;
			margin: 1px 1px 1px 1px;
            padding: 0px 1px 0px 1px;
            min-width: 250px;
            border-radius: 10px;
        }
        .node_name {
            background-color: #ddd;
        }
        .module_grid {
            display: grid;
            grid-template-columns: repeat(8, 100px);
            grid-template-rows: repeat(6, auto);
            grid-auto-flow: column;
            grid-column-gap: 10px;
            padding-left: 0px;
            padding-right: 0px;
            margin: 1px;
            margin-left: 20px;
          }
        .module {
            display: inline-grid;
            text-align: center;
            border-width: 0px;
			border-style: solid;
			margin: 1px 1px 1px 1px;
            padding: 0px 1px 0px 1px;
            min-width: 100px;
            border-radius: 10px;
        }

        .module_grid > .on, .service_node > .ok {
            background-color: #8c8;
        }
        .module_grid > .off, .service_node > .off{
            background-color: #9aa;
        }
        .module_grid > .fail, .service_node > .fail {
            background-color: #a33;
        }
        .module_grid > .always, .service_node > .fail {
            background-color: #282;
        }

        .tooltip { border-bottom: 0px dotted black;}
        .tooltip .tooltiptext {
            font-size: 0.9em;
            width: 200px;

        }
        .tooltiptext {
            transform: translate(0px, 2px);
        }

        .console {
            background-color: black;
            font-family: "Lucida Console", Monaco, monospace;
            font-size: 0.5em;
            width: auto;
            color: #fff;
            border-radius: 6px;
            padding: 5px 5px;
        }
    
    </style>
</head>

{% macro put_osd_prop(prop, b, a, p) %}
    {% if prop in p %}
    <div class="item bench">{{ a[prop] | to_mb }} ({{ "%0.4f" | format(p[prop]|float) }})</div>
    {% else %}
    <div class="item bench">{{ a[prop] | to_mb }}</div>
    {% endif %}
{% endmacro %}

{% macro put_osd_perc(prop, b, a, p) %}
    {% if prop in p %}
    <div class="item bench">{{ "%0.4f"|format(a[prop]|float) }} ({{ "%0.4f" | format(p[prop]|float) }})</div>
    {% else %}
    <div class="item bench">{{ "%0.4f"|format(a[prop]|float) }}</div>
    {% endif %}
{% endmacro %}

{% macro summary_value(prop, s) %}
    {% if prop in s %}
    <div class="item bench">{{ s[prop] }} nodes</div>
    {% else %}
    <div class="item bench">0</div>
    {% endif %}
{% endmacro %}


<body onload="init()">

<div class="header">
    <div class="label">Ceph version:</div>
    <div class="text">{{ ceph_version }}</div>
    <div class="label">Image:</div>
    <div class="text">{{ cluster.image }}</div>
    <div class="label date">generated on: {{ gen_date }}</div>
</div>

<div class="bar">
    <div class="bar-centered">
        <button class="bar-item" onclick="openBar(event, 'bench')">Benchmark Results</button>
        <button class="bar-item" onclick="openBar(event, 'osdstats')">OSD Stats</button>
        <button class="bar-item" onclick="openBar(event, 'status')">Status</button>
        <!-- <button class="bar-item" onclick="openBar(event, 'latency')">Latency</button> -->
    </div>
</div>

<!-- Benchmarks -->
{% macro bench_page(results, id_label) %}
<div id="{{ id_label }}" class="barcontent">
    <h5>{{ caller() }}</h5>
    <div class="note">Graphs in detailed section shows value measured by internal Ceph profiler</div>
    <div class="note">'All agents' value shows theoretical load calculated client-side</div>
    <hr>
    <table class="ceph_status">
        <tr class="node">
            <td class="bench_id">N</td>
            <td class="time">Time started</td>
            <td class="status">Data point</td>
            <td class="col_properties">
                <div class="props_group">
                    <div class="item prop">Run Time</div>
                    <div class="item prop">Storage class</div>
                    <div class="item pg">PGs</div>
                    <div class="item prop">Engine</div>
                    <div class="item prop">Mode</div>
                    <div class="item prop">BS / IODepth / Size</div>
                </div>
            </td>
            <td class="col_bench">
                <div class="bench_run_group">
                    <div class="item bench">Read, MB/s</div>
                    <div class="item bench">Avg/cLat 95p, usec</div>
                    <div class="item bench">Read, op/s</div>
                    <div class="item bench">Write, MB/s</div>
                    <div class="item bench">Avg/cLat 95p, usec</div>
                    <div class="item bench">Write, op/s</div>
                </div>
            </td>
        </tr>
        {% for time,dt in results.items() %}
        {% set t = dt["totals"] %}
        {% set o = dt["input_options"] %}
        {% set tstripped = time | tstrip %}
        <tr class="node" onclick="toggleClassByID('timing_{{ tstripped }}_data')" id="timing_{{ tstripped }}_button">
            <td class="bench_id">{{ dt["id"] }}</td>
            <td class="time">{{ time }}</td>
            <td class="status">All agents</td>
            <td class="col_properties">
                <div class="props_group">
                    <div class="item prop">{{ o["ramp_time"] }}/{{ o["runtime"] }}</div>
                    <div class="item prop">{{ t["storage_class"] }}</div>
                    <div class="item pg">{{ t["storage_class_stats"]["num_pg"] }}</div>
                    <div class="item prop">{{ o["ioengine"] }}</div>
                    <div class="item prop">{{ o["readwrite"] }} ({{ o["rwmixread"] }}/{{ 100 - (o["rwmixread"]|int) }})</div>
                    <div class="item prop">{{ o["bs"] }}/{{ o["iodepth"] }}/{{ o["size"] }}</div>
                </div>
            </td>
            <td class="col_bench">
                <div class="bench_run_group">
                    <div class="item bench">{{ t["read_bw_bytes"] | to_mb }}</div>
                    <div class="item bench">{{ "%0.2f" | format(t["read_avg_lat_us"]|float) }} / {{ "%0.2f" | format(t["read_95p_clat_us"]|float) }}</div>
                    <div class="item bench">{{ "%0.2f" | format(t["read_iops"]|float) }}</div>
                    <div class="item bench">{{ t["write_bw_bytes"] | to_mb }}</div>
                    <div class="item bench">{{ "%0.2f" | format(t["write_avg_lat_us"]|float) }} / {{ "%0.2f" | format(t["write_95p_clat_us"]|float) }}</div>
                    <div class="item bench">{{ "%0.2f" | format(t["write_iops"]|float) }}</div>
                </div>
            </td>
        </tr>
        {% set c = dt["ceph"] %}
        <tr class="collapsable" id="timing_{{ tstripped }}_data"><td colspan=3>
            <div class="details-wrap">
            <div class="inlineheader">Global READ stats, MB/s vs seconds. Measured maximum is <b>{{ c["max_rbl"][0] | to_mb }}</b> MB/sec</div>
            <div class="bc-wrap">
                <div class="bctimecol">
                 <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][1] | to_mb }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][2] | to_mb }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][3] | to_mb }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_rbl"][4] | to_mb }}</span></div>
                </div>
                   
                <div class="bc-container">
                 <div class="bc">
                  {% for sec, c_data in c["stats"].items() %}
                  {% set elapsed = sec | float %}
                  <div class="bccol">
                      {% if sec == c["max_rbl_time"] %}
                      <div class="bcheader">{{ c_data["read_bytes_sec"] | to_mb }}</div>
                      <div class="bcbar green-bar" style="height: {{ c_data["read_bytes_sec_perc"] }};"></div>
                      {% else %}
                      <div class="bcbar" style="height: {{ c_data["read_bytes_sec_perc"] }};"></div>
                      {% endif%}
                      <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
                    </div>
                  {% endfor %}
                 </div>
                </div>
            </div>
            <div class="inlineheader">Global READ stats, IOPS vs seconds. Measured maximum is <b>{{ c["max_ril"][0] }}</b> op/sec</div>
            <div class="bc-wrap">
                <div class="bctimecol">
                 <div class="bctime"><span class="bctimetext">{{ c["max_ril"][1] }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_ril"][2] }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_ril"][3] }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_ril"][4] }}</span></div>
                </div>
                   
                <div class="bc-container">
                 <div class="bc">
                  {% for sec, c_data in c["stats"].items() %}
                  {% set elapsed = sec | float %}
                  <div class="bccol">
                      {% if sec == c["max_ril_time"] %}
                      <div class="bcheader">{{ c_data["read_op_per_sec"] }}</div>
                      <div class="bcbar green-bar" style="height: {{ c_data["read_op_per_sec_perc"] }};"></div>
                      {% else %}
                      <div class="bcbar" style="height: {{ c_data["read_op_per_sec_perc"] }};"></div>
                      {% endif%}
                      <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
                    </div>
                  {% endfor %}
                 </div>
                </div>
            </div>
            <div class="inlineheader">Global WRITE stats, MB/s vs seconds. Measured maximum is <b>{{ c["max_wbl"][0] | to_mb }}</b> MB/sec</div>
            <div class="bc-wrap">
                <div class="bctimecol">
                 <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][1] | to_mb }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][2] | to_mb }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][3] | to_mb }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_wbl"][4] | to_mb }}</span></div>
                </div>
                   
                <div class="bc-container">
                 <div class="bc">
                  {% for sec, c_data in c["stats"].items() %}
                  {% set elapsed = sec | float %}
                  <div class="bccol">
                      {% if sec == c["max_wbl_time"] %}
                      <div class="bcheader">{{ c_data["write_bytes_sec"] | to_mb }}</div>
                      <div class="bcbar green-bar" style="height: {{ c_data["write_bytes_sec_perc"] }};"></div>
                      {% else %}
                      <div class="bcbar" style="height: {{ c_data["write_bytes_sec_perc"] }};"></div>
                      {% endif%}
                      <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
                    </div>
                  {% endfor %}
                 </div>
                </div>
            </div>

            <div class="inlineheader">Global WRITE stats, IOPS vs seconds. Measured maximum is <b>{{ c["max_wil"][0] }}</b> op/sec</div>
            <div class="bc-wrap">
                <div class="bctimecol">
                 <div class="bctime"><span class="bctimetext">{{ c["max_wil"][1] }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_wil"][2] }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_wil"][3] }}</span></div>
                 <div class="bctime"><span class="bctimetext">{{ c["max_wil"][4] }}</span></div>
                </div>
                   
                <div class="bc-container">
                 <div class="bc">
                  {% for sec, c_data in c["stats"].items() %}
                  {% set elapsed = sec | float %}
                  <div class="bccol">
                      {% if sec == c["max_wil_time"] %}
                      <div class="bcheader">{{ c_data["write_op_per_sec"] }}</div>
                      <div class="bcbar green-bar" style="height: {{ c_data["write_op_per_sec_perc"] }};"></div>
                      {% else %}
                      <div class="bcbar" style="height: {{ c_data["write_op_per_sec_perc"] }};"></div>
                      {% endif%}
                      <div class="bcfooter">{{ "%0.1f" | format(elapsed) }}</div>
                    </div>
                  {% endfor %}
                 </div>
                </div>
            </div>
            <div class="inlineheader">Per Agent stats</div>
            <table style="table-layout: auto;"><tbody>
                {% for agent,ag_result in dt["agents"].items() %}
                {% set j = ag_result["jobs"][0] %}
                <tr class="agents">
                    <td class="time">{{ time }}</td>
                    <td class="status">{{ agent }}</td>
                    <td class="col_properties">
                        <div class="props_group">
                            <div class="item prop">{{ j["job options"]["ramp_time"] }}/{{ j["job options"]["runtime"] }}</div>
                            <div class="item prop">{{ t["storage_class"] }}</div>
                            <div class="item pg">{{ t["storage_class_stats"]["num_pg"] }}</div>
                            <div class="item prop">{{ o["ioengine"] }}</div>
                            <div class="item prop">{{ o["readwrite"] }} ({{ o["rwmixread"] }}/{{ 100 - (o["rwmixread"]|int) }})</div>
                            <div class="item prop">{{ j["job options"]["bs"] }}/{{ o["iodepth"] }}/{{ j["job options"]["size"] }}</div>
                        </div>
                    </td>
                    <td class="col_bench">
                        <div class="bench_run_group">
                            <div class="item bench">{{ j["read"]["bw_bytes"] | to_mb }}</div>
                            <div class="item bench">{{ "%0.2f" | format(j["read"]["lat_ns"]["mean"]|float / 1000) }} / {{ "%0.2f" | format(j["read"]["clat_ns"]["percentile"]["95.000000"]|float / 1000) }}</div>
                            <div class="item bench">{{ "%0.2f" | format(j["read"]["iops"]|float) }}</div>
                            <div class="item bench">{{ j["write"]["bw_bytes"] | to_mb }}</div>
                            <div class="item bench">{{ "%0.2f" | format(j["write"]["lat_ns"]["mean"]|float / 1000) }} / {{ "%0.2f" | format(j["write"]["clat_ns"]["percentile"]["95.000000"]|float / 1000) }}</div>
                            <div class="item bench">{{ "%0.2f" | format(j["write"]["iops"]|float) }}</div>
                        </div>
                    </td>
                </tr>
                        </tr>
                {% endfor %}
            </tbody></table>
            </div>
        </td></tr>
        {% endfor %}
    </table>
</div>
{% endmacro %}

<!-- OSD stats -->
{% macro osds_page(results, id_label) %}
<div id="{{ id_label }}" class="barcontent">
    <h5>{{ caller() }}</h5>
    <div class="note">Node counts is the number of nodes with parameter changed at the end of the testrun comparing to start of the testrun</div>
    <div class="note">Hover over column title for description</div>
    <hr>
    <table class="ceph_status">
        <tr class="node">
            <td class="bench_id">N</td>
            <td class="time">Time started</td>
            <td class="status">Data point</td>
            <td class="col_properties">
                <div class="osd_props_group">
                    <div class="tooltip">
                        <div class="item prop">Status</div>
                        <div class="tooltiptext">OSD nodes with 'up' status</div>
                    </div>
                    <div class="tooltip">
                        <div class="item prop">Class</div>
                        <div class="tooltiptext">OSD device class: 'hdd', 'ssd', 'nvme', etc</div>
                    </div>
                    <div class="tooltip">
                        <div class="item prop">Weight</div>
                        <div class="tooltiptext">The weight of the OSD in the CRUSH map</div>
                    </div>
                    <div class="tooltip">
                        <div class="item pg">PGs</div>
                        <div class="tooltiptext">The number of placement groups in the OSD</div>
                    </div>
                </div>
            </td>
            <td class="col_bench">
                <div class="osd_stats_group">
                    <div class="tooltip">
                        <div class="item bench">Total, GB</div>
                        <div class="tooltiptext">The total storage capacity of the OSD</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">Avail., GB</div>
                        <div class="tooltiptext">The amount of free space available on the OSD.</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">Used, GB</div>
                        <div class="tooltiptext">The OSD capacity used</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">Data, GB</div>
                        <div class="tooltiptext">The amount of OSD capacity that is used by user data</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">OMAP, GB</div>
                        <div class="tooltiptext">An estimate value of the bluefs storage that is being used to store object map (omap) data (key value pairs stored in rocksdb)</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">Meta, GB</div>
                        <div class="tooltiptext">The bluefs space allocated, or the value set in the bluestore_bluefs_min parameter, whichever is larger, for internal metadata which is calculated as the total space allocated in bluefs minus the estimated omap data size</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">Utilized, %</div>
                        <div class="tooltiptext">The notional percentage of storage used by the OSD</div>
                    </div>
                    <div class="tooltip">
                        <div class="item bench">Variance, %</div>
                        <div class="tooltiptext">The variation above or below average utilization</div>
                    </div>
                </div>
            </td>
        </tr>
        {% for time,dt in results.items() %}
        {% set b = dt["osd_summary"]["before"] %}
        {% set a = dt["osd_summary"]["after"] %}
        {% set s = dt["osd_summary"]["active"] %}
        {% set tstripped = time | tstrip %}
        <tr class="node" onclick="toggleClassByID('timing_{{ tstripped }}_osds')" id="timing_{{ tstripped }}_button">
            <td class="bench_id">{{ dt["id"] }}</td>
            <td class="time">{{ time }}</td>
            <td class="status">Active nodes</td>
            <td class="col_properties">
                <div class="osd_props_group">
                    <div class="item prop">{{ s["status"] }}</div>
                    <div class="item prop">{{ s["device_class"] }}</div>
                    <div class="item prop">&minus;</div>
                    <div class="item pg">{{ s["pgs"] }}</div>
                </div>
            </td>
            <td class="col_bench">
                <div class="osd_stats_group">
                    <div class="item bench">{{ a["total_kb"] | to_mb }}</div>
                    {{ summary_value("kb_avail", s) }}
                    {{ summary_value("kb_used", s) }}
                    {{ summary_value("kb_used_data", s) }}
                    {{ summary_value("kb_used_omap", s) }}
                    {{ summary_value("kb_used_meta", s) }}
                    {{ summary_value("utilization", s) }}
                    <div class="item bench">{{ s["var_down"] }}&darr; / {{ s["var_up"] }}&uarr;</div>
                </div>
            </td>
        </tr>
        <tr class="collapsable" id="timing_{{ tstripped }}_osds"><td colspan=3>
            <table style="table-layout: auto;"><tbody>
                {% for osd in dt["osds"].keys() | sort %}
                {% set n = dt["osds"][osd] %}
                {% set b = n["before"] %}
                {% set a = n["after"] %}
                {% set p = n["percent"] %}
                <tr class="agents">
                    <td class="time">{{ time }}</td>
                    <td class="status">{{ osd }}</td>
                    <td class="col_properties">
                        <div class="osd_props_group">
                            <div class="item prop">{{ a["status"] }}</div>
                            <div class="item prop">{{ a["device_class"] }}</div>
                            <div class="item prop">{{ "%0.4f" | format(a["crush_weight"]|float) }}</div>
                            <div class="item pg">{{ a["pgs"] }}</div>
                        </div>
                    </td>
                    <td class="col_bench">
                        <div class="osd_stats_group">
                            <div class="item bench">{{ a["kb"] | to_mb }}</div>
                            {{ put_osd_prop("kb_avail", b, a, p) }}
                            {{ put_osd_prop("kb_used", b, a, p) }}
                            {{ put_osd_prop("kb_used_data", b, a, p) }}
                            {{ put_osd_prop("kb_used_omap", b, a, p) }}
                            {{ put_osd_prop("kb_used_meta", b, a, p) }}
                            {{ put_osd_perc("utilization", b, a, p) }}
                            {{ put_osd_perc("var", b, a, p) }}
                        </div>
                    </td>
                </tr>
                {% endfor %}
            </tbody></table>
        </td></tr>
        {% endfor %}
    </table>
</div>
{% endmacro %}


<!-- Status page -->
{% macro status_page(info, id_label) %}
<div id="{{ id_label }}" class="barcontent">
    <h5>{{ caller() }}</h5>
    <hr>
    <table class="ceph_status">
        <tr class="node">
            <td class="status">Cluster status</td>
            <td class="col_shortmessage">Status summary</td>
            <td class="col_osd">
                <div class="osd_group">
                    <div class="item osd">OSDs</div>
                    <div class="item osd">Up</div>
                    <div class="item osd">In</div>
                    <div class="item osd">Remap PGs</div>
                </div>
            </td>
            <td class="col_pgs">
                <div class="pg_group">
                    <div class="item pg">PGs</div>
                    <div class="item pg">Pools</div>
                    <div class="item pg">Objects</div>
                    <div class="item pg">Data, GB</div>
                    <div class="item pg">Used, GB</div>
                    <div class="item pg">Avail, GB</div>
                    <div class="item pg">Total, GB</div>
                </div>
            </td>
            <td class="col_bench">
                <div class="bench_group">
                    <div class="item bench">Read, MB/sec</div>
                    <div class="item bench">Write, MB/sec</div>
                    <div class="item bench">Read, op/sec</div>
                    <div class="item bench">Write, op/sec</div>
                </div>
            </td>
        </tr>
        {% set cs = idle_status %}
        {% set osdmap = cs | get_osdmap %}
        <tr class="node" onclick="toggleClassByID('health_data')" id="health_data_button">
            <td class="status {{ health_detail["status"] | lower }}">{{ health_detail["status"] }}</td>
            <td class="col_shortmessage">
                {% for code,dt in health_detail["checks"].items() %}
                {{ dt["summary"]["message"] }}<br>
                {% endfor %}
            </td>
            <!-- background: linear-gradient(to right, gray 0% 20%, transparent 20% 100%); -->
            <td class="col_osd">
                <div class="osd_group">
                    <div class="item osd">{{ osdmap["num_osds"] }}</div>
                    <div class="item osd">{{ osdmap["num_up_osds"] }}</div>
                    <div class="item osd">{{ osdmap["num_in_osds"] }}</div>
                    <div class="item osd">{{ osdmap["num_remapped_pgs"] }}</div>
                </div>
            </td>
            {% set pgmap = cs["pgmap"] %}
            <td class="col_pgs">
                <div class="pg_group">
                    <div class="item pg">{{ pgmap["num_pgs"] }}</div>
                    <div class="item pg">{{ pgmap["num_pools"] }}</div>
                    <div class="item pg">{{ pgmap["num_objects"] }}</div>
                    <div class="item pg">{{ pgmap["data_bytes"] | to_gb }}</div>
                    <div class="item pg">{{ pgmap["bytes_used"] | to_gb }}</div>
                    <div class="item pg">{{ pgmap["bytes_avail"] | to_gb }}</div>
                    <div class="item pg">{{ pgmap["bytes_total"] | to_gb }}</div>
                </div>
            </td>
            <td class="col_bench">
                <div class="bench_group">
                    {% if "read_bytes_sec" in pgmap %}
                    <div class="item bench">{{ pgmap["read_bytes_sec"] | to_mb }}</div>
                    {% else %}
                    <div class="item bench">0</div>
                    {% endif %}
                    {% if "write_bytes_sec" in pgmap %}
                    <div class="item bench">{{ pgmap["write_bytes_sec"] | to_mb }}</div>
                    {% else %}
                    <div class="item bench">0</div>
                    {% endif %}
                    {% if "read_op_per_sec" in pgmap %}
                    <div class="item bench">{{ pgmap["read_op_per_sec"] }}</div>
                    {% else %}
                    <div class="item bench">0</div>
                    {% endif %}
                    {% if "write_op_per_sec" in pgmap %}
                    <div class="item bench">{{ pgmap["write_op_per_sec"] }}</div>
                    {% else %}
                    <div class="item bench">0</div>
                    {% endif %}
                </div>
            </td>
        </tr>
        <tr class="collapsable in" id="health_data"><td colspan=3>
            <table><tbody>
                {% for code,dt in health_detail["checks"].items() %}
                <tr>
                    <td class="spacer"></td>
                    <td class="status {{ dt["severity"] | lower }}">{{ dt["severity"] }}</td>
                    <td class="checks_code">{{ code }}</td>
                    <td class="col_longmessage">
                        <table><tbody>
                            {% for detail in dt["detail"] %}
                            <tr><td>{{ detail["message"] }}</td></tr>
                            {% endfor %}
                        </tbody></table>
                    </td>
                </tr>
                {% endfor %}
            </tbody></table>
        </td></tr>
    </table>
    <hr>
    <!-- Services -->
    {% set sm = idle_status["servicemap"] %}
    <h5>Services: {{ sm["services"] | count }} running. Last modification: {{ sm["modified"] }}</h5>
    <table class="ceph_status">
        <tr class="node">
            <td class="srv_name">Name</td>
            <td class="srv_path">Subpath</td>
            <td class="srv_timestamp">Start time</td>
            <td class="srv_addr">Address</td>
        </tr>
        {% for name, d1 in sm["services"].items() %}
        {% if "daemons" in d1 %}
            {% set d2 = d1["daemons"] %}
            {% for key, d3 in d2.items() %}
                {% if key.startswith("rgw.store") %}
        <tr class="node" onclick="toggleClassByID('{{ name }}_service_data')" id="{{ name }}_service_data_button">
            <td class="srv_name">{{ name }} ({{ d3["gid"] }})</td>
            <td class="srv_path">daemons:{{ key }}</td>
            <td class="srv_timestamp">{{ d3["start_stamp"] }}</td>
            <td class="srv_addr">{{ d3["addr"] }}</td>
        </tr>
        <tr class="collapsable in" id="{{ name}}_service_data"><td colspan=4>
            <table><tbody>
                <tr><td class="metadata">
                {% for mname, mvalue in d3["metadata"].items() %}
                    <div class="meta_group">
                        <div class="item meta_name">{{ mname }}</div>
                        <div class="item meta_value">{{ mvalue }}</div>
                    </div>
                {% endfor %}
                </td></tr>
            </tbody></table>
        </td></tr>
                {% endif %}
            {% endfor %}
        {% endif %}
        {% endfor %}
    </table>
    <hr>
    <!-- Modules -->
    {% set mgrmap = idle_status["mgrmap"] %}
    {% set mods = mgrmap["modules"] %}
    {% set avail = mgrmap["available_modules"] %}
    {% if "always_on_modules" in mgrmap %}
    {% set always_on = mgrmap["always_on_modules"].values() | list %}
    {% set always_on = always_on[0] %}
    {% else %}
    {% set always_on = [] %}
    {% endif %}
    <h5>Modules: {{ mods | count}} active. {{ always_on | count }} always on. {{ avail | count }} available.</h5>
    <div class="modules">
        <div class="module_grid">
            {% for mod in avail %}
                {% if mod["name"] in always_on %}
                <div class="module always">{{ mod["name"] }}</div>
                {% elif mod["name"] in mods %}
                <div class="module on">{{ mod["name"] }}</div>
                {% elif not mod["can_run"] %}
                <div class="module fail tooltip">
                    <div class="module fail">{{ mod["name"] }}</div>
                    <pre class="tooltiptext">{{ mod["error_string"] | linebreaks }}</pre>
                </div>
                {% else %}
                <div class="module">{{ mod["name"] }}</div>
                {% endif %}
            {% endfor %}
        </div>
    </div>
    <hr>
</div>
{% endmacro %}

<!-- ================================= -->
<!-- Cluster nodes page -->
{% call bench_page(results, "bench") %}
    Benchmark results
{% endcall %}

{% call osds_page(results, "osdstats") %}
    OSD nodes stats collected before and after each step
{% endcall %}

{% call status_page(info, "status") %}
    Cluster status
{% endcall %}

</body>
</html>