Merge "Add tls support for dns endpoints" into release/2019.2.0
diff --git a/metadata/service/exporters/libvirt.yml b/metadata/service/exporters/libvirt.yml
index 2c910bb..e76a788 100644
--- a/metadata/service/exporters/libvirt.yml
+++ b/metadata/service/exporters/libvirt.yml
@@ -1,5 +1,7 @@
applications:
- prometheus
+classes:
+ - service.prometheus.support
parameters:
_param:
prometheus_exporter_libvirt_port: 9177
diff --git a/prometheus/files/grafana_dashboards/provisioning_kpi_prometheus.json b/prometheus/files/grafana_dashboards/provisioning_kpi_prometheus.json
new file mode 100644
index 0000000..dd8b032
--- /dev/null
+++ b/prometheus/files/grafana_dashboards/provisioning_kpi_prometheus.json
@@ -0,0 +1,737 @@
+{%- raw %}
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "prometheus",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 17,
+ "panels": [],
+ "title": "Today",
+ "type": "row"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "#299c46",
+ "rgba(237, 129, 40, 0.89)",
+ "#d44a3a"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 0,
+ "y": 1
+ },
+ "hideTimeOverride": true,
+ "id": 10,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(compute_instance_create_start_host_doc_count)",
+ "format": "time_series",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "30s",
+ "title": "Start",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "0",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "#299c46",
+ "rgba(237, 129, 40, 0.89)",
+ "#d44a3a"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 3,
+ "y": 1
+ },
+ "hideTimeOverride": true,
+ "id": 11,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(compute_instance_create_end_host_doc_count)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "30s",
+ "timeShift": null,
+ "title": "End",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#299c46",
+ "rgba(237, 129, 40, 0.89)",
+ "#d44a3a"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 6,
+ "y": 1
+ },
+ "hideTimeOverride": true,
+ "id": 12,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(compute_instance_create_error_host_doc_count)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "1",
+ "timeFrom": "30s",
+ "timeShift": null,
+ "title": "Error",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "0",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "percentunit",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 9,
+ "y": 1
+ },
+ "hideTimeOverride": true,
+ "id": 13,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "1 - sum(compute_instance_create_error_host_doc_count) / sum(compute_instance_create_start_host_doc_count)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.8",
+ "timeFrom": "30s",
+ "title": "VM creation success",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "100%",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 7
+ },
+ "id": 15,
+ "panels": [],
+ "title": "Daily (measured over the last 24h)",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 8,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Start",
+ "color": "#6ED0E0"
+ },
+ {
+ "alias": "End",
+ "color": "#7EB26D"
+ },
+ {
+ "alias": "Error",
+ "color": "#BF1B00"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "alias": "Start",
+ "bucketAggs": [
+ {
+ "field": "Timestamp",
+ "id": "2",
+ "settings": {
+ "interval": "1d",
+ "min_doc_count": 0,
+ "trimEdges": 0
+ },
+ "type": "date_histogram"
+ }
+ ],
+ "dsType": "elasticsearch",
+ "expr": "sum(compute_instance_create_start_host_doc_count)",
+ "format": "time_series",
+ "interval": "1d",
+ "intervalFactor": 2,
+ "legendFormat": "Start",
+ "metrics": [
+ {
+ "field": "select field",
+ "id": "1",
+ "type": "count"
+ }
+ ],
+ "query": "event_type:compute.instance.create.start",
+ "refId": "A",
+ "timeField": "Timestamp"
+ },
+ {
+ "alias": "End",
+ "bucketAggs": [
+ {
+ "field": "Timestamp",
+ "id": "2",
+ "settings": {
+ "interval": "1d",
+ "min_doc_count": 0,
+ "trimEdges": 0
+ },
+ "type": "date_histogram"
+ }
+ ],
+ "dsType": "elasticsearch",
+ "expr": "sum(compute_instance_create_end_host_doc_count)",
+ "format": "time_series",
+ "interval": "1d",
+ "intervalFactor": 2,
+ "legendFormat": "End",
+ "metrics": [
+ {
+ "field": "select field",
+ "id": "1",
+ "type": "count"
+ }
+ ],
+ "query": "event_type:compute.instance.create.end",
+ "refId": "B",
+ "timeField": "Timestamp"
+ },
+ {
+ "alias": "Error",
+ "bucketAggs": [
+ {
+ "field": "Timestamp",
+ "id": "2",
+ "settings": {
+ "interval": "1d",
+ "min_doc_count": 0,
+ "trimEdges": 0
+ },
+ "type": "date_histogram"
+ }
+ ],
+ "dsType": "elasticsearch",
+ "expr": "sum(compute_instance_create_error_host_doc_count)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "1d",
+ "intervalFactor": 2,
+ "legendFormat": "Error",
+ "metrics": [
+ {
+ "field": "select field",
+ "id": "1",
+ "type": "count"
+ }
+ ],
+ "query": "event_type:compute.instance.create.error",
+ "refId": "C",
+ "timeField": "Timestamp"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "VM instances",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "id": 3,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Start",
+ "color": "#6ED0E0"
+ },
+ {
+ "alias": "End",
+ "color": "#7EB26D"
+ },
+ {
+ "alias": "Error",
+ "color": "#BF1B00"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "alias": "Start",
+ "bucketAggs": [
+ {
+ "field": "Timestamp",
+ "id": "2",
+ "settings": {
+ "interval": "1d",
+ "min_doc_count": 0,
+ "trimEdges": 0
+ },
+ "type": "date_histogram"
+ }
+ ],
+ "dsType": "elasticsearch",
+ "expr": "1 - sum(compute_instance_create_error_host_doc_count) / (sum(compute_instance_create_start_host_doc_count > 0) or count(compute_instance_create_start_host_doc_count == 0))",
+ "format": "time_series",
+ "instant": false,
+ "interval": "1d",
+ "intervalFactor": 2,
+ "legendFormat": "success",
+ "metrics": [
+ {
+ "field": "select field",
+ "id": "1",
+ "type": "count"
+ }
+ ],
+ "query": "event_type:compute.instance.create.start",
+ "refId": "A",
+ "timeField": "Timestamp"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "VM creation success",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-30d",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Provisioning KPI",
+ "version": 1
+}
+{%- endraw %}
diff --git a/prometheus/meta/fluentd.yml b/prometheus/meta/fluentd.yml
index 3720b65..ee543bb 100644
--- a/prometheus/meta/fluentd.yml
+++ b/prometheus/meta/fluentd.yml
@@ -1,46 +1,34 @@
{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
-{% from "prometheus/map.jinja" import sf_notifier with context %}
-{%- if sf_notifier.get('enabled', False) %}
+ {%- if pillar.get('prometheus', {}).get('exporters', {}).libvirt is defined %}
+ {%- if grains.get('init') == 'systemd' %}
agent:
config:
label:
- sfnotifier:
- input:
- tail_sfnotifier:
- type: tail
- tag: sfnotifier.log
- path: {{ sf_notifier.dir.logs }}/sf-notifier.log
- pos_file: {{ pillar.fluentd.agent.dir.positiondb }}/sfnotifier.pos
- parser:
- type: multi_format
- patterns:
- # sfnotifier log format: https://regex101.com/r/JsYcJU/1
- - type: regexp
- time_key: Timestamp
- time_format: '%F %H:%M:%S'
- keep_time_key: false
- format: '/^\[(?<Timestamp>.*)\] (?<Severity>.*?) in (?<Module>.*?)\: (?<Payload>.+)$/'
- # http log format: https://regex101.com/r/8i6u1v/2
- - type: regexp
- time_key: Timestamp
- time_format: '%d/%b/%Y %H:%M:%S'
- keep_time_key: false
- format: '/^(?<http_client_ip_address>[^ ]*) - - \[(?<Timestamp>.*)\] "(?<http_method>\S+) (?<http_url>[^ ]*) \S*" (?<http_status>[^ ]*) -$/'
+ libvirt_exporter:
filter:
- sfnotifier_record:
+ enrich:
+ tag: 'libvirt-exporter.**'
type: record_transformer
- tag: sfnotifier.log
enable_ruby: true
- remove_keys: priority
record:
- - name: service
- value: sfnotifier
+ - name: programname
+ value: libvirt-exp
- name: severity_label
- value: '${ {"TRACE"=>8,"DEBUG"=>7,"INFO"=>6,"NOTICE"=>5,"WARNING"=>4,"ERROR"=>3,"CRITICAL"=>2,"ALERT"=>1,"EMERGENCY"=>0}.fetch(record["Severity"].to_i, 6) }'
+ value: ${ if record["Payload"].downcase.include? "error"; "ERROR"; else; "INFO"; end }
+ add_severity:
+ require:
+ - enrich
+ tag: 'libvirt-exporter.**'
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: '${ { 8=>"TRACE",7=>"DEBUG",6=>"INFO",5=>"NOTICE",4=>"WARNING",3=>"ERROR",2=>"CRITICAL",1=>"ALERT",0=>"EMERGENCY" }.key(record["severity_label"]) }'
match:
push_to_default:
- tag: sfnotifier.**
+ tag: 'libvirt-exporter.**'
type: relabel
label: default_output
-{%- endif %}
-{%- endif %}
+ {%- endif %}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/prometheus/meta/grafana.yml b/prometheus/meta/grafana.yml
index 9904974..f3bcb3b 100644
--- a/prometheus/meta/grafana.yml
+++ b/prometheus/meta/grafana.yml
@@ -1,4 +1,4 @@
-{% from "prometheus/map.jinja" import server, remote_storage_adapter, pushgateway, relay with context %}
+{% from "prometheus/map.jinja" import server, remote_storage_adapter, pushgateway, relay, elasticsearch_exporter with context %}
dashboard:
prometheus-performances_prometheus:
datasource: prometheus
@@ -30,3 +30,9 @@
format: json
template: prometheus/files/grafana_dashboards/relay_prometheus.json
{%- endif %}
+{%- if elasticsearch_exporter.get('enabled', False) and pillar._param.openstack_version is defined %}
+ provisioning_kpi_prometheus:
+ datasource: prometheus
+ format: json
+ template: prometheus/files/grafana_dashboards/provisioning_kpi_prometheus.json
+{%- endif %}