Add Ironic monitoring
- define fluentd metrics on ironic nodes
- api and conductor processes monitoring
- alerts for api and conductor downtime
- add Grafana dashboard
Related-PROD: PROD-30621
Related-PROD: PROD-32734
Related-PROD: PROD-33028
Change-Id: I4f89bcfe9fddf3bd85bc87a871073c2c2b149084
diff --git a/ironic/_common.sls b/ironic/_common.sls
index 446d653..0b636ec 100644
--- a/ironic/_common.sls
+++ b/ironic/_common.sls
@@ -1,13 +1,18 @@
{%- from "ironic/map.jinja" import api,conductor with context %}
{%- if api.get("enabled", False) %}
{%- set ironic, service_name = api, 'api' %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic, service_name = conductor, 'conductor' %}
{%- endif %}
include:
- ironic._ssl.mysql
- ironic._ssl.rabbitmq
+{%- if api.get("enabled", False) %}
+ - ironic.api
+{%- elif conductor.get("enabled", False) %}
+ - ironic.conductor
+{%- endif %}
ironic_common_pkgs:
pkg.installed:
@@ -25,3 +30,26 @@
- pkg: ironic_common_pkgs
- sls: ironic._ssl.mysql
- sls: ironic._ssl.rabbitmq
+
+{% if ironic.logging.log_appender %}
+ironic_general_logging_conf:
+ file.managed:
+ - name: /etc/ironic/logging.conf
+ - source: salt://oslo_templates/files/logging/_logging.conf
+ - template: jinja
+ - user: ironic
+ - group: ironic
+ - defaults:
+ service_name: ironic
+ _data: {{ ironic.logging }}
+ - watch_in:
+ - service: {{ ironic.service }}
+ironic_log_file:
+ file.managed:
+ - name: /var/log/ironic/ironic.log
+ - user: ironic
+ - group: ironic
+ - watch_in:
+ - service: {{ ironic.service }}
+{% endif %}
+
diff --git a/ironic/_ssl/mysql.sls b/ironic/_ssl/mysql.sls
index 9f13be7..28a0671 100644
--- a/ironic/_ssl/mysql.sls
+++ b/ironic/_ssl/mysql.sls
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor with context %}
{%- if api.get("enabled", False) %}
{%- set ironic, service_name = api, 'api' %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic, service_name = conductor, 'conductor' %}
{%- endif %}
@@ -79,4 +79,4 @@
- name: {{ ironic.database.ssl.get('cacert_file', ironic.cacert_file) }}
{%- endif %}
-{%- endif %}
\ No newline at end of file
+{%- endif %}
diff --git a/ironic/_ssl/rabbitmq.sls b/ironic/_ssl/rabbitmq.sls
index 2ce0069..078d832 100644
--- a/ironic/_ssl/rabbitmq.sls
+++ b/ironic/_ssl/rabbitmq.sls
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor with context %}
{%- if api.get("enabled", False) %}
{%- set ironic, service_name = api, 'api' %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic, service_name = conductor, 'conductor' %}
{%- endif %}
diff --git a/ironic/api.sls b/ironic/api.sls
index b1a6c6c..1afd4b5 100644
--- a/ironic/api.sls
+++ b/ironic/api.sls
@@ -12,6 +12,54 @@
- sls: ironic._common
- sls: ironic.db.offline_sync
+{%- if api.logging.log_handlers.get('fluentd',{}).get('enabled', False) %}
+ironic_api_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% if api.logging.log_appender %}
+ironic_api_logging_conf:
+ file.managed:
+ - name: /etc/ironic/logging/logging-{{ api.service }}.conf
+ - source: salt://oslo_templates/files/logging/_logging.conf
+ - template: jinja
+ - user: ironic
+ - group: ironic
+ - makedirs: True
+ - require_in:
+ - sls: ironic.db.offline_sync
+ - require:
+ - pkg: ironic_api_packages
+{%- if api.logging.log_handlers.get('fluentd',{}).get('enabled', False) %}
+ - pkg: ironic_api_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: ironic-api
+ _data: {{ api.logging }}
+ - watch_in:
+ - service: {{ api.service }}
+
+{{ api.service }}_default:
+ file.managed:
+ - name: /etc/default/{{ api.service }}
+ - source: salt://ironic/files/default
+ - template: jinja
+ - require:
+ - pkg: ironic_api_packages
+ - defaults:
+ service_name: {{ api.service }}
+ values: {{ api.logging }}
+ - watch_in:
+ - service: {{ api.service }}
+{%- else %}
+{{ api.service }}_default:
+ file.absent:
+ - name: /etc/default/{{ api.service }}
+ - watch_in:
+ - service: {{ api.service }}
+{%- endif %}
+
{{ api.service }}:
service.running:
- enable: true
diff --git a/ironic/conductor.sls b/ironic/conductor.sls
index 40b1e05..d58dadb 100644
--- a/ironic/conductor.sls
+++ b/ironic/conductor.sls
@@ -10,6 +10,54 @@
- require_in:
- sls: ironic._common
+{%- if conductor.logging.log_handlers.get('fluentd',{}).get('enabled', False) %}
+ironic_conductor_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% if conductor.logging.log_appender %}
+ironic_conductor_logging_conf:
+ file.managed:
+ - name: /etc/ironic/logging/logging-{{ conductor.service }}.conf
+ - source: salt://oslo_templates/files/logging/_logging.conf
+ - template: jinja
+ - user: ironic
+ - group: ironic
+ - makedirs: True
+ - require_in:
+ - sls: ironic.db.offline_sync
+ - require:
+ - pkg: ironic_conductor_packages
+{%- if conductor.logging.log_handlers.get('fluentd',{}).get('enabled', False) %}
+ - pkg: ironic_conductor_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: {{ conductor.service }}
+ _data: {{ conductor.logging }}
+ - watch_in:
+ - service: {{ conductor.service }}
+
+{{ conductor.service }}_default:
+ file.managed:
+ - name: /etc/default/{{ conductor.service }}
+ - source: salt://ironic/files/default
+ - template: jinja
+ - require:
+ - pkg: ironic_conductor_packages
+ - defaults:
+ service_name: {{ conductor.service }}
+ values: {{ conductor.logging }}
+ - watch_in:
+ - service: {{ conductor.service }}
+{%- else %}
+{{ conductor.service }}_default:
+ file.absent:
+ - name: /etc/default/{{ conductor.service }}
+ - watch_in:
+ - service: {{ conductor.service }}
+{%- endif %}
+
{%- if conductor.get('console', {}).get('enabled') %}
ironic_console_packages:
pkg.installed:
diff --git a/ironic/files/default b/ironic/files/default
new file mode 100644
index 0000000..09bfc2c
--- /dev/null
+++ b/ironic/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.log_appender %}
+DAEMON_ARGS="${DAEMON_ARGS} --log-config-append=/etc/ironic/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/ironic/files/grafana_dashboards/ironic_prometheus.json b/ironic/files/grafana_dashboards/ironic_prometheus.json
new file mode 100644
index 0000000..e863fe7
--- /dev/null
+++ b/ironic/files/grafana_dashboards/ironic_prometheus.json
@@ -0,0 +1,1904 @@
+{%- raw %}
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "prometheus",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "limit": 100,
+ "name": "Annotations & Alerts",
+ "showIn": 0,
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "iteration": 1568636224564,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 6,
+ "panels": [],
+ "title": "Cluster Status",
+ "type": "row"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": true,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 0,
+ "y": 1
+ },
+ "id": 9,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "max(openstack_api_check_status{name=~\"ironic.*\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "VIP API availability",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": true,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "percentunit",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 4,
+ "y": 1
+ },
+ "id": 8,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": " Up",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(http_response_status{name=\"ironic-api-deploy\"}) / count(http_response_status{name=\"ironic-api-deploy\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.3,0.6",
+ "title": "Hosts Deploy API availability",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 8,
+ "y": 1
+ },
+ "id": 14,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "max(haproxy_active_servers{proxy=~\"ironic_deploy\", sv=\"BACKEND\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.35,0.7",
+ "title": "Haproxy Deploy API backends",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": true,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "percentunit",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 12,
+ "y": 1
+ },
+ "id": 59,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": " Up",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(http_response_status{name=\"ironic-api-public\"}) / count(http_response_status{name=\"ironic-api-public\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.3,0.6",
+ "title": "Hosts Public API availability",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 16,
+ "y": 1
+ },
+ "id": 60,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "max(haproxy_active_servers{proxy=~\"ironic\", sv=\"BACKEND\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.35,0.7",
+ "title": "Haproxy Ironic API backends",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 6
+ },
+ "id": 37,
+ "panels": [],
+ "title": "Host API Status",
+ "type": "row"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 8,
+ "x": 0,
+ "y": 7
+ },
+ "id": 39,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "minSpan": 8,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "repeat": "host",
+ "repeatDirection": "h",
+ "scopedVars": {
+ "host": {
+ "selected": false,
+ "text": "bmt01",
+ "value": "bmt01"
+ }
+ },
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "http_response_status{host=~\"$host\",name=~\"ironic-api-.*\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "Ironic@$host",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 8,
+ "x": 8,
+ "y": 7
+ },
+ "id": 67,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "minSpan": 8,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "repeat": null,
+ "repeatDirection": "h",
+ "repeatIteration": 1568636224564,
+ "repeatPanelId": 39,
+ "scopedVars": {
+ "host": {
+ "selected": false,
+ "text": "bmt02",
+ "value": "bmt02"
+ }
+ },
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "http_response_status{host=~\"$host\",name=~\"ironic-api-.*\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "Ironic@$host",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 8,
+ "x": 16,
+ "y": 7
+ },
+ "id": 68,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "minSpan": 8,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "repeat": null,
+ "repeatDirection": "h",
+ "repeatIteration": 1568636224564,
+ "repeatPanelId": 39,
+ "scopedVars": {
+ "host": {
+ "selected": false,
+ "text": "bmt03",
+ "value": "bmt03"
+ }
+ },
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "http_response_status{host=~\"$host\",name=~\"ironic-api-.*\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "Ironic@$host",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 8,
+ "x": 0,
+ "y": 10
+ },
+ "id": 69,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "minSpan": 8,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "repeat": null,
+ "repeatDirection": "h",
+ "repeatIteration": 1568636224564,
+ "repeatPanelId": 39,
+ "scopedVars": {
+ "host": {
+ "selected": false,
+ "text": "ctl01",
+ "value": "ctl01"
+ }
+ },
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "http_response_status{host=~\"$host\",name=~\"ironic-api-.*\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "Ironic@$host",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 8,
+ "x": 8,
+ "y": 10
+ },
+ "id": 70,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "minSpan": 8,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "repeat": null,
+ "repeatDirection": "h",
+ "repeatIteration": 1568636224564,
+ "repeatPanelId": 39,
+ "scopedVars": {
+ "host": {
+ "selected": false,
+ "text": "ctl02",
+ "value": "ctl02"
+ }
+ },
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "http_response_status{host=~\"$host\",name=~\"ironic-api-.*\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "Ironic@$host",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": true,
+ "colorValue": false,
+ "colors": [
+ "#d44a3a",
+ "rgba(237, 129, 40, 0.89)",
+ "#299c46"
+ ],
+ "datasource": null,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 8,
+ "x": 16,
+ "y": 10
+ },
+ "id": 71,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "minSpan": 8,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "repeat": null,
+ "repeatDirection": "h",
+ "repeatIteration": 1568636224564,
+ "repeatPanelId": 39,
+ "scopedVars": {
+ "host": {
+ "selected": false,
+ "text": "ctl03",
+ "value": "ctl03"
+ }
+ },
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "http_response_status{host=~\"$host\",name=~\"ironic-api-.*\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "thresholds": "0.5,0.5",
+ "title": "Ironic@$host",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ },
+ {
+ "op": "=",
+ "text": "FAIL",
+ "value": "0"
+ },
+ {
+ "op": "=",
+ "text": "OK",
+ "value": "1"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 13
+ },
+ "id": 13,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "decimals": null,
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 0,
+ "y": 14
+ },
+ "hideTimeOverride": false,
+ "id": 16,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "minSpan": null,
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": null,
+ "repeatDirection": "v",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(openstack_http_response_times_count{service=\"ironic\",host=~\"$host\"}[$rate_interval])) by (http_status)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{ http_status }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Throughput@$host",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": false,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "decimals": null,
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "id": 17,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": null,
+ "repeatDirection": "v",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "avg(openstack_http_response_times{service=\"ironic\",quantile=\"0.9\",host=~\"^$host$\",http_status=~\"2..\"}) by (http_method)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{http_method}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Latency@$host",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "repeat": null,
+ "title": "API Performance",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 14
+ },
+ "id": 11,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 4,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 0,
+ "y": 15
+ },
+ "id": 19,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": null,
+ "repeatDirection": "h",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "count(procstat_running{process_name=~\"ironic-api\"} == 1)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "running",
+ "refId": "A"
+ },
+ {
+ "expr": "count(procstat_running{process_name=~\"ironic-api\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Ironic API",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": false,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 4,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 15
+ },
+ "id": 27,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatDirection": "h",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "count(procstat_running{process_name=~\"ironic-conductor\"} == 1)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "running",
+ "refId": "A"
+ },
+ {
+ "expr": "count(procstat_running{process_name=~\"ironic-conductor\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Ironic Conductor",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": false,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Ironic Services",
+ "type": "row"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 15
+ },
+ "id": 53,
+ "panels": [],
+ "title": "Nodes",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 19,
+ "x": 0,
+ "y": 16
+ },
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": null,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "openstack_ironic_nodes_total",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "refId": "A"
+ },
+ {
+ "expr": "openstack_ironic_nodes",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{provision_state}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Nodes",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 22
+ },
+ "id": 29,
+ "panels": [],
+ "title": "Drivers",
+ "type": "row"
+ },
+ {
+ "columns": [],
+ "datasource": null,
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 23
+ },
+ "id": 46,
+ "links": [],
+ "pageSize": null,
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 7,
+ "desc": false
+ },
+ "styles": [
+ {
+ "alias": "Time",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "link": false,
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "Value",
+ "thresholds": [],
+ "type": "hidden",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "environment",
+ "thresholds": [],
+ "type": "hidden",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "instance",
+ "thresholds": [],
+ "type": "hidden",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "job",
+ "thresholds": [],
+ "type": "hidden",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "__name__",
+ "thresholds": [],
+ "type": "hidden",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "mappingType": 1,
+ "pattern": "region",
+ "thresholds": [],
+ "type": "hidden",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "openstack_ironic_driver{host=~\"$host\", driver=~\"$driver\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": "1s",
+ "title": "$driver back-end drivers on @$host",
+ "transform": "table",
+ "type": "table"
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [
+ "openstack"
+ ],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "prometheus",
+ "hide": 0,
+ "includeAll": true,
+ "label": null,
+ "multi": true,
+ "name": "host",
+ "options": [],
+ "query": "label_values(http_response_status{service=~\"ironic-api.*\"},host)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "prometheus",
+ "hide": 0,
+ "includeAll": true,
+ "label": null,
+ "multi": true,
+ "name": "driver",
+ "options": [],
+ "query": "label_values(openstack_ironic_driver,driver)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": true,
+ "text": "3m",
+ "value": "3m"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": null,
+ "multi": false,
+ "name": "rate_interval",
+ "options": [
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": true,
+ "text": "3m",
+ "value": "3m"
+ },
+ {
+ "selected": false,
+ "text": "5m",
+ "value": "5m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "15m",
+ "value": "15m"
+ }
+ ],
+ "query": "1m,3m,5m,10m,15m",
+ "skipUrlSync": false,
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Ironic"
+}
+{%- endraw %}
diff --git a/ironic/files/newton/ironic.conf b/ironic/files/newton/ironic.conf
index 07f1015..3a1616e 100644
--- a/ironic/files/newton/ironic.conf
+++ b/ironic/files/newton/ironic.conf
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor with context -%}
{%- if api.get("enabled", False) %}
{%- set ironic = api %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic = conductor %}
{%- endif %}
[DEFAULT]
diff --git a/ironic/files/ocata/ironic.conf b/ironic/files/ocata/ironic.conf
index 55a6c2d..2f9ae00 100644
--- a/ironic/files/ocata/ironic.conf
+++ b/ironic/files/ocata/ironic.conf
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor with context -%}
{%- if api.get("enabled", False) %}
{%- set ironic = api %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic = conductor %}
{%- endif %}
[DEFAULT]
diff --git a/ironic/files/pike/ironic.conf b/ironic/files/pike/ironic.conf
index f8dc1eb..d3b1aa7 100644
--- a/ironic/files/pike/ironic.conf
+++ b/ironic/files/pike/ironic.conf
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor,upgrade with context -%}
{%- if api.get("enabled", False) %}
{%- set ironic = api %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic = conductor %}
{%- endif %}
diff --git a/ironic/files/queens/ironic.conf b/ironic/files/queens/ironic.conf
index 8eb5ff6..d2d08cc 100644
--- a/ironic/files/queens/ironic.conf
+++ b/ironic/files/queens/ironic.conf
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor,upgrade with context -%}
{%- if api.get("enabled", False) %}
{%- set ironic = api %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic = conductor %}
{%- endif %}
diff --git a/ironic/files/rocky/ironic.conf b/ironic/files/rocky/ironic.conf
index 4be00e2..f9e3809 100644
--- a/ironic/files/rocky/ironic.conf
+++ b/ironic/files/rocky/ironic.conf
@@ -1,7 +1,7 @@
{%- from "ironic/map.jinja" import api,conductor,upgrade with context -%}
{%- if api.get("enabled", False) %}
{%- set ironic = api %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic = conductor %}
{%- endif %}
diff --git a/ironic/map.jinja b/ironic/map.jinja
index a82a3c6..67a0e13 100644
--- a/ironic/map.jinja
+++ b/ironic/map.jinja
@@ -15,6 +15,21 @@
'grub_dir_name': 'grub',
'enabled': false
},
+ 'bind': {
+ 'protocol': 'http'
+ },
+ 'identity': {
+ 'protocol': 'http'
+ },
+ 'logging': {
+ 'app_name': 'ironic',
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ }
+ }
}
}, base='Common', merge=pillar.ironic.get('api', {})) %}
@@ -23,7 +38,19 @@
'service': 'ironic-conductor',
'ipxe_rom_files': ['undionly.kpxe', 'ipxe.efi'],
'cacert_file': cacert_file,
- 'notification': {}
+ 'notification': {},
+ 'identity': {
+ 'protocol': 'http'
+ },
+ 'logging': {
+ 'app_name': 'ironic',
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ }
+ }
},
'Debian': {
'pkgs': ['ipmitool', 'ironic-conductor', 'tftpd-hpa', 'syslinux-common', 'pxelinux', 'ipxe', 'gdisk'],
@@ -46,7 +73,7 @@
'src': '/usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed',
'dst': 'grubx64.efi'
},
- },
+ }
},
'RedHat': {
'pkgs': ['ipmitool', 'ironic-conductor', 'tftp-server', 'syslinux-extlinux', 'ipxe-bootimgs', 'gdisk'],
@@ -69,7 +96,7 @@
'src': '/boot/efi/EFI/centos/grubx64.efi',
'dst': 'grubx64.efi'
},
- },
+ }
},
}, base='Common', merge=pillar.ironic.get('conductor', {})) %}
diff --git a/ironic/meta/fluentd.yml b/ironic/meta/fluentd.yml
new file mode 100644
index 0000000..0191778
--- /dev/null
+++ b/ironic/meta/fluentd.yml
@@ -0,0 +1,111 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_ironic:
+ tag: openstack.ironic.**
+ type: relabel
+ label: openstack_ironic
+ openstack_ironic:
+ filter:
+ set_ironic_fields:
+ tag: openstack.ironic.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: ironic-${ tag_parts[2] }
+ parse_http_stats:
+ tag: openstack.ironic
+ require:
+ - set_ironic_fields
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s+(?<http_status>\d{3})\s+len:\s+(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.ironic.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.ironic
+ send_to_default:
+ tag: openstack.ironic
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.ironic_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.ironic_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ ironic_logs_per_severity:
+ tag: metric.ironic_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: ironic
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ ironic_openstack_http_response_times:
+ tag: metric.ironic_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: ironic
+ - name: host
+ value: ${Hostname}
+{% endif %}
diff --git a/ironic/meta/grafana.yml b/ironic/meta/grafana.yml
new file mode 100644
index 0000000..1713dc5
--- /dev/null
+++ b/ironic/meta/grafana.yml
@@ -0,0 +1,5 @@
+dashboard:
+ ironic_prometheus:
+ datasource: prometheus
+ format: json
+ template: ironic/files/grafana_dashboards/ironic_prometheus.json
diff --git a/ironic/meta/prometheus.yml b/ironic/meta/prometheus.yml
new file mode 100644
index 0000000..fdf6066
--- /dev/null
+++ b/ironic/meta/prometheus.yml
@@ -0,0 +1,113 @@
+{%- if pillar.ironic is defined %}
+{% from "ironic/map.jinja" import api with context %}
+{%- raw %}
+server:
+ alert:
+ IronicErrorLogsTooHigh:
+ if: >-
+ sum(rate(log_messages{service="ironic",level=~"(?i:(error|emergency|fatal))"}[5m])) without (level) > 0.2
+ labels:
+ severity: warning
+ service: ironic
+ annotations:
+ summary: High number of errors in Ironic logs
+ description: The average per-second rate of errors in Ironic logs on the {{ $labels.host }} node is {{ $value }} (as measured over the last 5 minutes).
+ IronicProcessDown:
+ if: >-
+ procstat_running{process_name=~"ironic-.*"} == 0
+ labels:
+ service: ironic
+ severity: minor
+ annotations:
+ summary: "{{ $labels.process_name }} process is down"
+ description: >-
+ The {{ $labels.process_name }} process on the {{ $labels.host }} node is down.
+ IronicProcessDownMinor:
+ if: >-
+ count(procstat_running{process_name=~"ironic-.*"} == 0) by (process_name) >= count(procstat_running{process_name=~"ironic-.*"}) by (process_name) * 0.33
+ labels:
+ service: ironic
+ severity: minor
+ annotations:
+ summary: "33% of {{ $labels.process_name }} processes are down"
+ description: >-
+ The {{ $labels.process_name }} process is down on 33% of nodes.
+ IronicProcessDownMajor:
+ if: >-
+ count(procstat_running{process_name=~"ironic-.*"} == 0) by (process_name) >= count(procstat_running{process_name=~"ironic-.*"}) by (process_name) * 0.66
+ labels:
+ service: ironic
+ severity: major
+ annotations:
+ summary: "66% of {{ $labels.process_name }} processes are down"
+ description: >-
+ The {{ $labels.process_name }} process is down on 66% of nodes.
+ IronicProcessOutage:
+ if: >-
+ count(procstat_running{process_name=~"ironic-.*"} == 0) by (process_name) == count(procstat_running{process_name=~"ironic-.*"}) by (process_name)
+ labels:
+ service: ironic
+ severity: critical
+ annotations:
+ summary: "{{ $labels.process_name }} process outage"
+ description: >-
+ The {{ $labels.process_name }} process is down on all nodes.
+ IronicDriverMissing:
+ if: >-
+ scalar(count(procstat_running{process_name=~"ironic-conductor"} == 1)) - count(openstack_ironic_driver) by (driver) > 0
+ labels:
+ severity: major
+ service: ironic
+ annotations:
+ summary: "ironic-conductor {{ $labels.driver }} back-end driver missing"
+ description: >-
+ The ironic-conductor {{ $labels.driver }} back-end driver is missing on {{ $value }} node(s).
+{%- endraw %}
+{%- if api.get('enabled', False) %}
+{%- raw %}
+ IronicApiEndpointDown:
+ if: >-
+ http_response_status{name=~"ironic-api.*"} == 0
+ for: 2m
+ labels:
+ severity: minor
+ service: ironic
+ annotations:
+ summary: "{{ $labels.name }} endpoint is not accessible"
+ description: >-
+ The {{ $labels.name }} endpoint on the {{ $labels.host }} node is not accessible for 2 minutes.
+ IronicApiEndpointsDownMajor:
+ if: >-
+ count(http_response_status{name=~"ironic-api.*"} == 0) by (name) >= count(http_response_status{name=~"ironic-api.*"}) by (name) * 0.5
+ for: 2m
+ labels:
+ severity: major
+ service: ironic
+ annotations:
+ summary: "50% of {{ $labels.name }} endpoints are not accessible"
+ description: >-
+ {{ $value }} of {{ $labels.name }} endpoints (>= 50%) are not accessible for 2 minutes.
+ IronicApiEndpointsOutage:
+ if: >-
+ count(http_response_status{name=~"ironic-api.*"} == 0) by (name) == count(http_response_status{name=~"ironic-api.*"}) by (name)
+ for: 2m
+ labels:
+ severity: critical
+ service: ironic
+ annotations:
+ summary: "{{ $labels.name }} endpoints outage"
+ description: All available {{ $labels.name }} endpoints are not accessible for 2 minutes.
+ IronicApiOutage:
+ if: >-
+ max(openstack_api_check_status{name="ironic"}) == 0
+ for: 2m
+ labels:
+ severity: critical
+ service: ironic
+ annotations:
+ summary: Ironic API outage
+ description: >-
+ Ironic API is not accessible for all available Ironic endpoints in the OpenStack service catalog for 2 minutes.
+{%- endraw %}
+{%- endif %}
+{%- endif %}
diff --git a/ironic/meta/telegraf.yml b/ironic/meta/telegraf.yml
new file mode 100644
index 0000000..b5a8df0
--- /dev/null
+++ b/ironic/meta/telegraf.yml
@@ -0,0 +1,17 @@
+{%- from "ironic/map.jinja" import api, conductor with context %}
+agent:
+ input:
+ procstat:
+ process:
+{%- if conductor.get('enabled', False) %}
+ ironic-conductor:
+ pattern: "ironic-conductor"
+{%- endif %}
+{%- if api.get('enabled', False) %}
+ ironic-api:
+ pattern: "ironic-api"
+ http_response:
+ ironic-api-{{ api.api_type}}:
+ address: "{{ api.bind.protocol }}://{{ api.bind.address|replace('0.0.0.0', '127.0.0.1') }}:{{ api.bind.port }}/"
+ expected_code: 200
+{%- endif %}
diff --git a/ironic/upgrade/pre/init.sls b/ironic/upgrade/pre/init.sls
index 59c7e14..0ad16c5 100644
--- a/ironic/upgrade/pre/init.sls
+++ b/ironic/upgrade/pre/init.sls
@@ -2,7 +2,7 @@
{%- if api.get("enabled", False) %}
{%- set ironic, service_name = api, 'api' %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic, service_name = conductor, 'conductor' %}
{%- endif %}
diff --git a/ironic/upgrade/render_config.sls b/ironic/upgrade/render_config.sls
index d8ddac4..68d0a19 100644
--- a/ironic/upgrade/render_config.sls
+++ b/ironic/upgrade/render_config.sls
@@ -2,7 +2,7 @@
{%- if api.get("enabled", False) %}
{%- set ironic, service_name = api, 'api' %}
-{%- elif conductor.get('enabled', False) %}
+{%- elif conductor.get("enabled", False) %}
{%- set ironic, service_name = conductor, 'conductor' %}
{%- endif %}
diff --git a/metadata/service/api/cluster.yml b/metadata/service/api/cluster.yml
index 5e07a47..1ccc55e 100644
--- a/metadata/service/api/cluster.yml
+++ b/metadata/service/api/cluster.yml
@@ -1,10 +1,25 @@
applications:
- ironic
+classes:
+ - service.ironic.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
+ openstack_ossyslog_handler_enabled: false
ironic:
api:
- api_type: ${_param:ironic_api_type}
enabled: true
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
+ ossyslog:
+ enabled: ${_param:openstack_ossyslog_handler_enabled}
+ api_type: ${_param:ironic_api_type}
version: ${_param:ironic_version}
bind:
address: ${_param:cluster_local_address}
diff --git a/metadata/service/api/single.yml b/metadata/service/api/single.yml
index 3765d85..bde3356 100644
--- a/metadata/service/api/single.yml
+++ b/metadata/service/api/single.yml
@@ -1,10 +1,25 @@
applications:
- ironic
+classes:
+ - service.ironic.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
+ openstack_ossyslog_handler_enabled: false
ironic:
api:
api_type: 'mixed'
enabled: true
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
+ ossyslog:
+ enabled: ${_param:openstack_ossyslog_handler_enabled}
version: ${_param:ironic_version}
bind:
address: ${_param:single_address}
diff --git a/metadata/service/conductor/cluster.yml b/metadata/service/conductor/cluster.yml
index ea97f73..79b0bc3 100644
--- a/metadata/service/conductor/cluster.yml
+++ b/metadata/service/conductor/cluster.yml
@@ -1,11 +1,25 @@
applications:
- ironic
+classes:
+ - service.ironic.support
parameters:
_param:
neutron_service_protocol: http
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
+ openstack_ossyslog_handler_enabled: false
ironic:
conductor:
enabled: true
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
+ ossyslog:
+ enabled: ${_param:openstack_ossyslog_handler_enabled}
uefi:
enabled: false
version: ${_param:ironic_version}
diff --git a/metadata/service/conductor/single.yml b/metadata/service/conductor/single.yml
index 7174d44..d6f89d9 100644
--- a/metadata/service/conductor/single.yml
+++ b/metadata/service/conductor/single.yml
@@ -1,9 +1,24 @@
applications:
- ironic
+classes:
+ - service.ironic.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
+ openstack_ossyslog_handler_enabled: false
ironic:
conductor:
enabled: true
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
+ ossyslog:
+ enabled: ${_param:openstack_ossyslog_handler_enabled}
uefi:
enabled: false
version: ${_param:ironic_version}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index f6446d5..1180747 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -5,7 +5,15 @@
enabled: false
heka:
enabled: false
+ grafana:
+ enabled: true
sensu:
enabled: true
sphinx:
enabled: true
+ fluentd:
+ enabled: true
+ prometheus:
+ enabled: true
+ telegraf:
+ enabled: true