Add support for prometheus 2.0
New version changes:
* different alerts format
* rewritten storage (some config flags removed)
Closes-Bug: PROD-16609
Change-Id: I805fa322e4744e98177d6c3e29589ebc6fb917a2
diff --git a/README.rst b/README.rst
index cee3e90..95be7cc 100644
--- a/README.rst
+++ b/README.rst
@@ -15,6 +15,7 @@
prometheus:
server:
+ version: 2.0
enabled: true
dir:
config: /srv/volumes/prometheus
@@ -62,11 +63,7 @@
process_open_fds / process_max_fds
storage:
local:
- engine: "persisted"
retention: "360h"
- memory_chunks: 1048576
- max_chunks_to_persist: 524288
- num_fingerprint_mutexes: 4096
alertmanager:
notification_queue_capacity: 10000
config:
diff --git a/metadata/service/server/container.yml b/metadata/service/server/container.yml
index a3de78f..663092a 100644
--- a/metadata/service/server/container.yml
+++ b/metadata/service/server/container.yml
@@ -8,7 +8,6 @@
prometheus_region_label: "region1"
prometheus_scrape_interval: "15s"
prometheus_storage_retention: "360h"
- prometheus_storage_heap_size: 3221225472
prometheus:
server:
enabled: true
@@ -21,10 +20,7 @@
address: 0.0.0.0
storage:
local:
- engine: "persisted"
retention: ${_param:prometheus_storage_retention}
- target_heap_size: ${_param:prometheus_storage_heap_size}
- num_fingerprint_mutexes: 4096
alertmanager:
notification_queue_capacity: 10000
config:
diff --git a/prometheus/files/grafana_dashboards/prometheus_performances_prometheus.json b/prometheus/files/grafana_dashboards/server/1.7/prometheus_performances_prometheus.json
similarity index 100%
rename from prometheus/files/grafana_dashboards/prometheus_performances_prometheus.json
rename to prometheus/files/grafana_dashboards/server/1.7/prometheus_performances_prometheus.json
diff --git a/prometheus/files/grafana_dashboards/prometheus_stats_prometheus.json b/prometheus/files/grafana_dashboards/server/1.7/prometheus_stats_prometheus.json
similarity index 100%
rename from prometheus/files/grafana_dashboards/prometheus_stats_prometheus.json
rename to prometheus/files/grafana_dashboards/server/1.7/prometheus_stats_prometheus.json
diff --git a/prometheus/files/grafana_dashboards/server/2.0/prometheus_performances_prometheus.json b/prometheus/files/grafana_dashboards/server/2.0/prometheus_performances_prometheus.json
new file mode 100644
index 0000000..128d799
--- /dev/null
+++ b/prometheus/files/grafana_dashboards/server/2.0/prometheus_performances_prometheus.json
@@ -0,0 +1,2215 @@
+{% raw %}
+{
+ "annotations": {
+ "list": []
+ },
+ "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.0.x",
+ "editable": true,
+ "gnetId": 3834,
+ "graphTooltip": 0,
+ "hideControls": false,
+ "id": null,
+ "links": [],
+ "refresh": "1m",
+ "rows": [
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "Time series": "#70dbed"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_series{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Time series",
+ "metric": "prometheus_local_storage_memory_series",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Time series",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 26,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_active_appenders{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Time series",
+ "metric": "prometheus_local_storage_memory_series",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Active Appenders",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "samples/s": "#e5a8e2"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 1,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_samples_appended_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "samples/s",
+ "metric": "prometheus_local_storage_ingested_samples_total",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Samples Appended/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Maintenance",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250",
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "To persist": "#9AC48A"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 0,
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/Max.*/",
+ "fill": 0
+ }
+ ],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_chunks{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Chunks",
+ "metric": "prometheus_local_storage_memory_chunks",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Chunks",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_chunks_created_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Created",
+ "metric": "prometheus_local_storage_chunk_ops_total",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Chunks Created",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "Removed": "#e5ac0e"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 25,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_chunks_removed_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Removed",
+ "metric": "prometheus_local_storage_chunk_ops_total",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Chunks Removed",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Chunks",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 29,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head GC Time/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Queue length",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_blocks_loaded{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_indexing_batch_sizes_sum",
+ "refId": "A",
+ "step": 40
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Blocks Loaded",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Dashboard Row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Failed Reloads": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 30,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_reloads_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Reloads",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "expr": "rate(prometheus_tsdb_reloads_failures_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Failed Reloads",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TSDB Reloads/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 31,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(tsdb_wal_fsync_duration_seconds_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m]) / rate(tsdb_wal_fsync_duration_seconds_count{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "WAL Fsync Latency",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 32,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(tsdb_wal_corruptions_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "WAL Corruptions/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Checkpoints",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 19,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compactions_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Compactions",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "expr": "rate(prometheus_tsdb_compactions_failed_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Failed Compactions",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Compactions/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Allocated bytes": "#F9BA8F",
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "RSS": "#890F02"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 8,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compactions_triggered_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Checkpoint duration",
+ "metric": "last",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Compactions Triggered/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 33,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compaction_duration_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Compaction Time/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Dashboard Row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 27,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compaction_chunk_range_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_count{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "First Compaction, Avg Chunk Time Range",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "ms",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 34,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "First Compaction, Avg Chunk Samples",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 35,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compaction_chunk_size_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "First Compaction, Avg Bytes/Sample",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Dashboard Row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Allocated bytes": "#7EB26D",
+ "Allocated bytes - 1m max": "#BF1B00",
+ "Allocated bytes - 1m min": "#BF1B00",
+ "Allocated bytes - 5m max": "#BF1B00",
+ "Allocated bytes - 5m min": "#BF1B00",
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "RSS": "#447EBC"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "decimals": null,
+ "editable": true,
+ "error": false,
+ "fill": 0,
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/-/",
+ "fill": 0
+ }
+ ],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "process_resident_memory_bytes{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "RSS",
+ "metric": "process_resident_memory_bytes",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "expr": "prometheus_local_storage_target_heap_size_bytes{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Target heap size",
+ "metric": "go_memstats_alloc_bytes",
+ "refId": "D",
+ "step": 60
+ },
+ {
+ "expr": "go_memstats_next_gc_bytes{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Next GC",
+ "metric": "go_memstats_next_gc_bytes",
+ "refId": "C",
+ "step": 60
+ },
+ {
+ "expr": "go_memstats_alloc_bytes{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Allocated",
+ "metric": "go_memstats_alloc_bytes",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Allocated bytes": "#F9BA8F",
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "RSS": "#890F02"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 7,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(go_memstats_alloc_bytes_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Allocated Bytes/s",
+ "metric": "go_memstats_alloc_bytes",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Allocations",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 9,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "irate(process_cpu_seconds_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Irate",
+ "metric": "prometheus_local_storage_ingested_samples_total",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "expr": "rate(process_cpu_seconds_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "5m rate",
+ "metric": "prometheus_local_storage_ingested_samples_total",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": [
+ "avg"
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "RAM&CPU",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 250,
+ "panels": [
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 22,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_evaluator_iterations_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m]) - rate(prometheus_evaluator_iterations_skipped_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m]) - rate(prometheus_evaluator_iterations_missed_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Iterations performed/s",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "expr": "rate(prometheus_evaluator_iterations_skipped_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m]) ",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Iterations missed/s",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "expr": "rate(prometheus_evaluator_iterations_missed_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Iterations skipped/s",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "C",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Rule evaulation iterations",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 23,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_evaluator_duration_seconds_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m]) / rate(prometheus_evaluator_duration_seconds_count{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Rule evaluation duration",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Rule evaluation duration",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "description": "Time spent in each mode, per second",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "id": 24,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 4,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_engine_query_duration_seconds_sum{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{slice}}",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Query engine timings/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "Dashboard Row",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {},
+ "hide": 0,
+ "includeAll": false,
+ "label": null,
+ "multi": false,
+ "name": "Prometheus",
+ "options": [],
+ "query": "query_result(up{job=\"prometheus\"} == 1)",
+ "refresh": 1,
+ "regex": ".*instance=\"([^\"]+):9090\".*",
+ "sort": 0,
+ "tagValuesQuery": null,
+ "tags": [],
+ "tagsQuery": null,
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Prometheus Performances",
+ "version": 2
+}
+{% endraw %}
diff --git a/prometheus/files/grafana_dashboards/server/2.0/prometheus_stats_prometheus.json b/prometheus/files/grafana_dashboards/server/2.0/prometheus_stats_prometheus.json
new file mode 100644
index 0000000..f105549
--- /dev/null
+++ b/prometheus/files/grafana_dashboards/server/2.0/prometheus_stats_prometheus.json
@@ -0,0 +1,938 @@
+{% raw %}
+{
+ "annotations": {
+ "list": []
+ },
+ "description": "Prometheus Stats Dashboard",
+ "editable": true,
+ "gnetId": 358,
+ "graphTooltip": 0,
+ "hideControls": false,
+ "id": null,
+ "links": [],
+ "refresh": false,
+ "rows": [
+ {
+ "collapse": false,
+ "height": 178,
+ "panels": [
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 1,
+ "editable": true,
+ "error": false,
+ "format": "s",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 5,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 3,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "(time() - process_start_time_seconds{job=\"prometheus\", instance=~\"$Prometheus:[1-9][0-9]*\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A",
+ "step": 40
+ }
+ ],
+ "thresholds": "",
+ "title": "Uptime",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 6,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 3,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(prometheus_tsdb_head_series{instance=~\"$Prometheus:[1-9][0-9]*\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A",
+ "step": 40
+ }
+ ],
+ "thresholds": "1,5",
+ "title": "Total Series",
+ "type": "singlestat",
+ "valueFontSize": "70%",
+ "valueMaps": [],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 7,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 3,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(prometheus_tsdb_head_chunks{instance=~\"$Prometheus:[1-9][0-9]*\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A",
+ "step": 40
+ }
+ ],
+ "thresholds": "500,4000",
+ "title": "Memory Chunks",
+ "type": "singlestat",
+ "valueFontSize": "70%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "Empty",
+ "value": "0"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 16,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 3,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "prometheus_config_last_reload_successful{instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A",
+ "step": 40
+ }
+ ],
+ "thresholds": "0:1",
+ "title": "Successful Config Reload",
+ "type": "singlestat",
+ "valueFontSize": "70%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "Failure",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "Success",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "New row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": 227,
+ "panels": [
+ {
+ "aliasColors": {
+ "prometheus": "#C15C17",
+ "{instance=\"localhost:9090\",job=\"prometheus\"}": "#C15C17"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_samples_appended_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}",
+ "metric": "",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Appended Samples per Second",
+ "tooltip": {
+ "msResolution": false,
+ "ordering": "alphabetical",
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "short",
+ "label": "Samples / Second",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "fill": 1,
+ "id": 18,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(prometheus_tsdb_head_series_created_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m]))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "created",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "expr": "sum(increase(prometheus_tsdb_head_series_removed_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m]))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "removed",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Series Created / Removed",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "New row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {
+ "prometheus": "#F9BA8F",
+ "{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_target_interval_length_seconds_count{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Target Scrapes (last 5m)",
+ "tooltip": {
+ "msResolution": false,
+ "ordering": "alphabetical",
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\", instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{quantile}} ({{interval}})",
+ "metric": "",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Scrape Duration",
+ "tooltip": {
+ "msResolution": false,
+ "ordering": "alphabetical",
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "New row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "decimals": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "id": 12,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_evaluator_duration_seconds{quantile!=\"0.01\", quantile!=\"0.05\", instance=~\"$Prometheus:[1-9][0-9]*\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{quantile}}",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Rule Eval Duration",
+ "tooltip": {
+ "msResolution": false,
+ "ordering": "alphabetical",
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "New row",
+ "titleSize": "h6"
+ },
+ {
+ "collapse": false,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "id": 17,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(http_request_duration_microseconds_count{handler=~\"query|query_range\", instance=~\"$Prometheus:[1-9][0-9]*\"}[30s])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{handler}}",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Queries to prometheus",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "label": "duration",
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": false,
+ "title": "New row",
+ "titleSize": "h6"
+ }
+ ],
+ "schemaVersion": 14,
+ "style": "dark",
+ "tags": [
+ "prometheus"
+ ],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {},
+ "hide": 0,
+ "includeAll": false,
+ "label": "Prometheus host",
+ "multi": false,
+ "name": "Prometheus",
+ "options": [],
+ "query": "query_result(up{job=\"prometheus\"} == 1)",
+ "refresh": 1,
+ "regex": ".*instance=\"([^:]+):[1-9][0-9]*\".*",
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-30m",
+ "to": "now"
+ },
+ "timepicker": {
+ "now": true,
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "1m",
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Prometheus Stats",
+ "version": 1
+}
+{% endraw %}
diff --git a/prometheus/files/alerts.yml b/prometheus/files/server/1.7/alerts.yml
similarity index 100%
rename from prometheus/files/alerts.yml
rename to prometheus/files/server/1.7/alerts.yml
diff --git a/prometheus/files/server/2.0/alerts.yml b/prometheus/files/server/2.0/alerts.yml
new file mode 100644
index 0000000..7ca88e0
--- /dev/null
+++ b/prometheus/files/server/2.0/alerts.yml
@@ -0,0 +1,65 @@
+{%- from "prometheus/map.jinja" import server with context %}
+
+{%- set alerts = {} %}
+{%- set recordings = {} %}
+
+{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
+ {%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %}
+ {%- for recordingname, recording in server_grain.get('recording', {}).iteritems() %}
+ {%- do recordings.update({recordingname: recording}) %}
+ {%- endfor %}
+{%- endfor %}
+
+groups:
+{%- set recordings = salt['grains.filter_by']({'default': recordings}, merge=server.get('recording', {})) %}
+- name: recording.rules
+ rules:
+{%- for recordingname, recording in recordings.iteritems() %}
+ {%- if recording.get('enabled', True) %}
+ - record: {{ recordingname }}
+ expr: >-
+ {{ recording.query | indent(6, False) }}
+ {%- endif %}
+{%- endfor %}
+
+{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
+ {%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %}
+ {%- for alertname, alert in server_grain.get('alert', {}).iteritems() %}
+ {%- do alerts.update({alertname: alert}) %}
+ {%- endfor %}
+{%- endfor %}
+
+{%- set alerts = salt['grains.filter_by']({'default': alerts}, merge=server.get('alert', {})) %}
+- name: alert.rules
+ rules:
+{%- for alertname, alert in alerts.iteritems() %}
+ {%- if alert.get('enabled', True) %}
+ - alert: {{ alertname }}
+ expr: >-
+ {{ alert.if | indent(6, False) }}
+ {%- if alert.for is defined %}
+ for: {{ alert.for }}{%- endif %}
+ {#- Handle labels #}
+ {%- if alert.labels is not defined %}
+ {%- do alert.update({'labels': {}}) %}
+ {%- endif %}
+ {%- for label_name, label_value in server.get('config', {}).get('alert', {}).get('labels_add', {}).iteritems() %}
+ {%- if label_name not in alert.labels %}
+ {%- do alert.labels.update({label_name: label_value}) %}
+ {%- endif %}
+ {%- endfor %}
+ {%- if alert.labels %}
+ labels:
+ {%- for name, value in alert.labels.iteritems() %}
+ {{ name }}: "{{ value }}"
+ {%- endfor %}
+ {%- endif %}
+ {#- Handle annotations #}
+ {%- if alert.annotations is defined %}
+ annotations:
+ {%- for name, value in alert.annotations.iteritems() %}
+ {{ name }}: "{{ value }}"
+ {%- endfor %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
diff --git a/prometheus/files/prometheus.yml b/prometheus/files/server/prometheus.yml
similarity index 100%
rename from prometheus/files/prometheus.yml
rename to prometheus/files/server/prometheus.yml
diff --git a/prometheus/map.jinja b/prometheus/map.jinja
index dbb7803..d5c7d9b 100644
--- a/prometheus/map.jinja
+++ b/prometheus/map.jinja
@@ -1,5 +1,6 @@
{% set server = salt['grains.filter_by']({
'default': {
+ 'version': 2.0,
'target': {
'static': {
},
diff --git a/prometheus/meta/grafana.yml b/prometheus/meta/grafana.yml
index 399f284..0ed9c8a 100644
--- a/prometheus/meta/grafana.yml
+++ b/prometheus/meta/grafana.yml
@@ -1,8 +1,9 @@
+{% from "prometheus/map.jinja" import server with context %}
dashboard:
prometheus-performances_prometheus:
datasource: prometheus
format: json
- template: prometheus/files/grafana_dashboards/prometheus_performances_prometheus.json
+ template: prometheus/files/grafana_dashboards/server/{{server.version}}/prometheus_performances_prometheus.json
remote_storage_adapter_prometheus:
datasource: prometheus
format: json
@@ -10,7 +11,7 @@
prometheus_stats_prometheus:
datasource: prometheus
format: json
- template: prometheus/files/grafana_dashboards/prometheus_stats_prometheus.json
+ template: prometheus/files/grafana_dashboards/server/{{server.version}}/prometheus_stats_prometheus.json
alertmanager_prometheus:
datasource: prometheus
format: json
diff --git a/prometheus/meta/prometheus.yml b/prometheus/meta/prometheus.yml
index 89d5014..5655064 100644
--- a/prometheus/meta/prometheus.yml
+++ b/prometheus/meta/prometheus.yml
@@ -13,6 +13,9 @@
annotations:
summary: 'Prometheus endpoint {{ $labels.instance }} down'
description: 'The Prometheus target {{ $labels.instance }} is down for the job {{ $labels.job }}.'
+{% endraw %}
+{%- if server.version == 1.7 %}
+{% raw %}
PrometheusRushMode:
if: 'prometheus_local_storage_rushed_mode != 0'
for: 10m
@@ -24,6 +27,7 @@
description: 'The Prometheus {{ $labels.instance }} is in rush mode for 10m.'
{% endraw %}
{%- endif %}
+{%- endif %}
{%- if alertmanager.get('enabled', False) %}
AlertmanagerNotificationFailed:
{%- set threshold = monitoring.alertmanager.notification_failed_rate|float %}
diff --git a/prometheus/server.sls b/prometheus/server.sls
index d790553..388ca1d 100644
--- a/prometheus/server.sls
+++ b/prometheus/server.sls
@@ -14,14 +14,14 @@
{{server.dir.config}}/prometheus.yml:
file.managed:
- - source: salt://prometheus/files/prometheus.yml
+ - source: salt://prometheus/files/server/prometheus.yml
- template: jinja
- require:
- file: {{server.dir.config}}
{{server.dir.config}}/alerts.yml:
file.managed:
- - source: salt://prometheus/files/alerts.yml
+ - source: salt://prometheus/files/server/{{server.version}}/alerts.yml
- template: jinja
- require:
- file: {{server.dir.config}}
diff --git a/tests/pillar/server.sls b/tests/pillar/server.sls
index 1ef5abf..7b65e74 100644
--- a/tests/pillar/server.sls
+++ b/tests/pillar/server.sls
@@ -55,10 +55,7 @@
summary: 'Prometheus target down'
storage:
local:
- engine: "persisted"
retention: "360h"
- target_heap_size: 3221225472
- num_fingerprint_mutexes: 4096
alertmanager:
notification_queue_capacity: 10000
config: