diff --git a/.kitchen.yml b/.kitchen.yml
index 01c5fda..0c77d2e 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -8,7 +8,7 @@
   name: salt_solo
   salt_install: bootstrap
   salt_bootstrap_url: https://bootstrap.saltstack.com
-  salt_version: latest
+  salt_version: <%=ENV['SALT_VERSION'] || 'latest'%>
   require_chef: false
   log_level: error
   formula: cinder
diff --git a/.travis.yml b/.travis.yml
index 4f20546..07f3508 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,28 @@
   - bundle install
 
 env:
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=ceph-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=ceph-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=control-cluster
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=control-cluster
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=control-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=control-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=gpfs-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=gpfs-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=hp3par-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=hp3par-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=lefthand-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=lefthand-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=solidfire-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=solidfire-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=storwize-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=storwize-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=volume-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=volume-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=vsp-single
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=vsp-single
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=volume-single-barbican
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=volume-single-barbican
     - PLATFORM=trevorj/salty-whales:trusty OS_VERSION=mitaka SUITE=ceph-single
     - PLATFORM=trevorj/salty-whales:xenial OS_VERSION=ocata SUITE=ceph-single
     - PLATFORM=trevorj/salty-whales:trusty OS_VERSION=mitaka SUITE=control-cluster
@@ -37,8 +59,8 @@
     - PLATFORM=trevorj/salty-whales:xenial OS_VERSION=ocata SUITE=volume-single
     - PLATFORM=trevorj/salty-whales:trusty OS_VERSION=mitaka SUITE=vsp-single
     - PLATFORM=trevorj/salty-whales:xenial OS_VERSION=ocata SUITE=vsp-single
-    - PLATFORM=trevorj/salty-whales:trusty OS_VERSION=mitaka SUITE=volume-single-barbican
-    - PLATFORM=trevorj/salty-whales:xenial OS_VERSION=ocata SUITE=volume-single-barbican
+    - PLATFORM=trevorj/salty-whales:trusty-2017.7 OS_VERSION=mitaka SUITE=volume-single-barbican
+    - PLATFORM=trevorj/salty-whales:xenial-2017.7 OS_VERSION=ocata SUITE=volume-single-barbican
 
 before_script:
   - set -o pipefail
diff --git a/README.rst b/README.rst
index d3f0672..6c8590b 100644
--- a/README.rst
+++ b/README.rst
@@ -680,6 +680,40 @@
         barbican:
           enabled: true
 
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+  * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+  * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+  cinder:
+    controller:
+      logging:
+        log_appender: true
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: true
+
+    volume:
+      logging:
+        log_appender: true
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: true
+
 Documentation and Bugs
 ============================
 
diff --git a/cinder/controller.sls b/cinder/controller.sls
index ecf0786..e61b2ab 100644
--- a/cinder/controller.sls
+++ b/cinder/controller.sls
@@ -22,6 +22,97 @@
   - require:
     - pkg: cinder_controller_packages
 
+{%- if controller.backup.engine != None %}
+  {%- set cinder_log_services = controller.services + controller.backup.services %}
+{%- else %}
+  {%- set cinder_log_services = controller.services %}
+{%- endif %}
+
+{%- for service_name in cinder_log_services %}
+{{ service_name }}_default:
+  file.managed:
+    - name: /etc/default/{{ service_name }}
+    - source: salt://cinder/files/default
+    - template: jinja
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ controller }}
+    - require:
+      - pkg: cinder_controller_packages
+{%- if controller.backup.engine != None %}
+      - pkg: cinder_backup_packages
+{%- endif %}
+    - watch_in:
+      - service: cinder_controller_services
+{%- if controller.backup.engine != None %}
+      - pkg: cinder_backup_services
+{%- endif %}
+{%- endfor %}
+
+{% if controller.logging.log_appender %}
+
+{%- if controller.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+cinder_controller_fluentd_logger_package:
+  pkg.installed:
+    - name: python-fluent-logger
+{%- endif %}
+
+cinder_general_logging_conf:
+  file.managed:
+    - name: /etc/cinder/logging.conf
+    - source: salt://cinder/files/logging.conf
+    - template: jinja
+    - user: cinder
+    - group: cinder
+    - defaults:
+        service_name: cinder
+        values: {{ controller }}
+    - require:
+      - pkg: cinder_controller_packages
+{%- if controller.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+      - pkg: cinder_controller_fluentd_logger_package
+{%- endif %}
+    - watch_in:
+      - service: cinder_controller_services
+      - service: cinder_api_service
+
+/var/log/cinder/cinder.log:
+  file.managed:
+    - user: cinder
+    - group: cinder
+    - watch_in:
+      - service: cinder_controller_services
+      - service: cinder_api_service
+
+{% for service_name in cinder_log_services %}
+{{ service_name }}_logging_conf:
+  file.managed:
+    - name: /etc/cinder/logging/logging-{{ service_name }}.conf
+    - source: salt://cinder/files/logging.conf
+    - template: jinja
+    - makedirs: True
+    - user: cinder
+    - group: cinder
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ controller }}
+    - require:
+      - pkg: cinder_controller_packages
+{%- if controller.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+      - pkg: cinder_controller_fluentd_logger_package
+{%- endif %}
+{%- if controller.backup.engine != None %}
+      - pkg: cinder_backup_packages
+{%- endif %}
+    - watch_in:
+      - service: cinder_controller_services
+{%- if controller.backup.engine != None %}
+      - pkg: cinder_backup_services
+{%- endif %}
+{% endfor %}
+
+{% endif %}
+
 {%- for name, rule in controller.get('policy', {}).iteritems() %}
 
 {%- if rule != None %}
@@ -66,6 +157,10 @@
   - require:
     - pkg: cinder_controller_packages
 
+apache_enable_cinder_wsgi:
+  apache_conf.enabled:
+    - name: cinder-wsgi
+
 cinder_api_service:
   service.running:
   - name: apache2
diff --git a/cinder/files/default b/cinder/files/default
new file mode 100644
index 0000000..99f34b9
--- /dev/null
+++ b/cinder/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/cinder/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/cinder/files/grafana_dashboards/cinder_prometheus_fluentd.json b/cinder/files/grafana_dashboards/cinder_prometheus_fluentd.json
new file mode 100644
index 0000000..6c6561f
--- /dev/null
+++ b/cinder/files/grafana_dashboards/cinder_prometheus_fluentd.json
@@ -0,0 +1,2390 @@
+{% raw %}
+{
+  "annotations": {
+    "list": []
+  },
+  "description": "Monitors Cinder cluster using Prometheus. Shows overall cluster processes and usage.",
+  "editable": true,
+  "gnetId": 315,
+  "graphTooltip": 0,
+  "hideControls": false,
+  "id": null,
+  "links": [],
+  "refresh": "1m",
+  "rows": [
+    {
+      "collapse": false,
+      "height": "250px",
+      "panels": [
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": true,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 1,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 1,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 3,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "min(openstack_api_check_status{service=~\"cinder.*\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ service }}",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "0.5,1.5",
+          "title": "API Availability",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            },
+            {
+              "op": "=",
+              "text": "DOWN",
+              "value": "0"
+            },
+            {
+              "op": "=",
+              "text": "OK",
+              "value": "1"
+            },
+            {
+              "op": "=",
+              "text": "UNKNOWN",
+              "value": "2"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 2,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": " / sec",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 3,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "sum(irate(haproxy_http_response_5xx{proxy=~\"cinder.*\",sv=\"FRONTEND\"}[5m]))",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "per sec",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "HTTP 5xx errors",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 3,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 3,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "min(haproxy_active_servers{proxy=~\"cinder.api\", sv=\"BACKEND\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Cinder API backends",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "Service Status",
+      "titleSize": "h6"
+    },
+    {
+      "collapse": false,
+      "height": "250",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "fill": 1,
+          "id": 4,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(openstack_http_response_times_count{service=\"cinder\",host=~\"^$host$\"}[5m]))  by (http_status)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ http_status }}",
+              "refId": "A",
+              "step": 10
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Throughput",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "fill": 1,
+          "id": 5,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_http_response_times{service='cinder',quantile=\"0.9\",host=~\"^$host$\"}) by (http_method)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ http_method }}",
+              "refId": "A",
+              "step": 10
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Latency",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "API Performances",
+      "titleSize": "h6"
+    },
+    {
+      "collapse": false,
+      "height": "250px",
+      "panels": [
+        {
+          "content": "<br />\n<br />\n<br />\n<br />\n\n\n\n\n<h1 align=\"center\">Schedulers</h1>",
+          "id": 11,
+          "links": [],
+          "mode": "html",
+          "span": 2,
+          "title": "",
+          "type": "text"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(245, 54, 54, 0.9)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 12,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 1,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{state=\"disabled\", service=\"cinder-scheduler\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "1,1",
+          "title": "Disabled",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(245, 54, 54, 0.9)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 13,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 1,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{state=\"up\", service=\"cinder-scheduler\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "1,1",
+          "title": "Up",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(245, 54, 54, 0.9)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 14,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 1,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{state=\"down\", service=\"cinder-scheduler\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "1,1",
+          "title": "Down",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "decimals": 0,
+          "fill": 1,
+          "id": 15,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 7,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{service=\"cinder-scheduler\"}) by (state)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ state }}",
+              "metric": "openstack_cinder_services",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "History",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "content": "<br />\n<br />\n<br />\n<br />\n\n\n<h1 align=\"center\">Volumes</h1>",
+          "id": 16,
+          "links": [],
+          "mode": "html",
+          "span": 2,
+          "title": "",
+          "type": "text"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(245, 54, 54, 0.9)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 17,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 1,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{state=\"disabled\", service=\"cinder-volume\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "1,1",
+          "title": "Disabled",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(245, 54, 54, 0.9)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 18,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 1,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{state=\"up\", service=\"cinder-volume\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "1,1",
+          "title": "Up",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(50, 172, 45, 0.97)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(245, 54, 54, 0.9)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 19,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 1,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{state=\"down\", service=\"cinder-volume\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "1,1",
+          "title": "Down",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "decimals": 0,
+          "fill": 1,
+          "id": 20,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 7,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_services{service=\"cinder-volume\"}) by (state)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ state }}",
+              "metric": "openstack_cinder_services",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "History",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "Cinder Services",
+      "titleSize": "h6"
+    },
+    {
+      "collapse": false,
+      "height": "250px",
+      "panels": [
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 31,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes{status=\"available\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Available Volumes",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 32,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes{status=\"in-use\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Volumes in use",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 33,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes{status=\"error\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Volumes errored",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "decimals": 0,
+          "fill": 1,
+          "id": 34,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes) by (status)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ status }}",
+              "metric": "openstack_cinder_volumes",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Volumes",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 35,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes_size{status=\"available\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Available Volumes - size",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 36,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes_size{status=\"in-use\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Volumes in use - size",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 37,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes_size{status=\"error\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Volumes errored - size",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "decimals": 0,
+          "fill": 1,
+          "id": 38,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_volumes_size) by (status)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ status }}",
+              "metric": "openstack_cinder_volumes_size",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Volumes size",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "decbytes",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 39,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots{status=\"available\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Available Snapshots",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 40,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots{status=\"creating\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Creating Snapshots",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "none",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 41,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots{status=\"error\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Snapshots errored",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "decimals": 0,
+          "fill": 1,
+          "id": 42,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots) by (status)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ status }}",
+              "metric": "openstack_cinder_snapshots",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Snapshots",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 43,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots_size{status=\"available\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Available Snapshots - size",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 44,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots_size{status=\"creating\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Creating Snapshots - size",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": "prometheus",
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
+          "id": 45,
+          "interval": null,
+          "links": [],
+          "mappingType": 1,
+          "mappingTypes": [
+            {
+              "name": "value to text",
+              "value": 1
+            },
+            {
+              "name": "range to text",
+              "value": 2
+            }
+          ],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
+            {
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
+            }
+          ],
+          "span": 2,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "tableColumn": "",
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots_size{status=\"error\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 60
+            }
+          ],
+          "thresholds": "",
+          "title": "Snapshots errored - size",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "prometheus",
+          "decimals": 0,
+          "fill": 1,
+          "id": 46,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(openstack_cinder_snapshots_size) by (status)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{ status }}",
+              "metric": "openstack_cinder_snapshots_size",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Snapshots size",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "decbytes",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": true,
+      "title": "Resources",
+      "titleSize": "h6"
+    }
+  ],
+  "schemaVersion": 14,
+  "sharedCrosshair": true,
+  "style": "dark",
+  "tags": [
+    "cinder"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "prometheus",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "host",
+        "options": [],
+        "query": "label_values(openstack_http_response_times_count,host)",
+        "refresh": 1,
+        "refresh_on_load": true,
+        "regex": "",
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Cinder",
+  "version": 5
+}
+{% endraw %}
diff --git a/cinder/files/logging.conf b/cinder/files/logging.conf
new file mode 100644
index 0000000..7130ebf
--- /dev/null
+++ b/cinder/files/logging.conf
@@ -0,0 +1,77 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+  {%- if log_handler_attrs.get('enabled', False) %}
+    {%- do log_handlers.append(log_handler_name) -%}
+  {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, cinder
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_cinder]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = cinder
+propagate = 0
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither.  (Recommended for production systems.)
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".") }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/cinder/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/cinder/files/mitaka/cinder.conf.controller.Debian b/cinder/files/mitaka/cinder.conf.controller.Debian
index 852e974..8a66c11 100644
--- a/cinder/files/mitaka/cinder.conf.controller.Debian
+++ b/cinder/files/mitaka/cinder.conf.controller.Debian
@@ -83,6 +83,10 @@
 volumes_dir = /var/lib/cinder/volumes
 log_dir=/var/log/cinder
 
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/cinder/logging.conf
+{%- endif %}
+
 # Use syslog for logging. (boolean value)
 #use_syslog=false
 
diff --git a/cinder/files/newton/cinder.conf.controller.Debian b/cinder/files/newton/cinder.conf.controller.Debian
index 701f78f..6a0a6a2 100644
--- a/cinder/files/newton/cinder.conf.controller.Debian
+++ b/cinder/files/newton/cinder.conf.controller.Debian
@@ -82,6 +82,9 @@
 
 volumes_dir = /var/lib/cinder/volumes
 log_dir=/var/log/cinder
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/cinder/logging.conf
+{%- endif %}
 
 # Use syslog for logging. (boolean value)
 #use_syslog=false
diff --git a/cinder/files/ocata/cinder.conf.controller.Debian b/cinder/files/ocata/cinder.conf.controller.Debian
index 8bc04ef..a36f403 100644
--- a/cinder/files/ocata/cinder.conf.controller.Debian
+++ b/cinder/files/ocata/cinder.conf.controller.Debian
@@ -86,6 +86,9 @@
 
 volumes_dir = /var/lib/cinder/volumes
 log_dir=/var/log/cinder
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/cinder/logging.conf
+{%- endif %}
 
 # Use syslog for logging. (boolean value)
 #use_syslog=false
diff --git a/cinder/map.jinja b/cinder/map.jinja
index e04b232..35af31f 100644
--- a/cinder/map.jinja
+++ b/cinder/map.jinja
@@ -22,7 +22,15 @@
           'pkgs': ['cinder-backup'],
           'services': ['cinder-backup'],
           'engine': None
-        }
+        },
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
     },
     'RedHat': {
         'pkgs': ['openstack-cinder', 'python-cinder', 'lvm2', 'python-pycadf'],
@@ -39,8 +47,15 @@
           'pkgs': ['cinder-backup'],
           'services': ['cinder-backup'],
           'engine': None
-        }
-
+        },
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
     },
 }, merge=pillar.cinder.get('controller', {}), base='BaseDefaults') %}
 
@@ -60,8 +75,15 @@
           'pkgs': ['cinder-backup'],
           'services': ['cinder-backup'],
           'engine': None
-        }
-
+        },
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
     },
     'RedHat': {
         'pkgs': ['openstack-cinder', 'python-cinder', 'lvm2', 'sysfsutils', 'sg3_utils', 'device-mapper-multipath', 'device-mapper-multipath-libs', 'python-pycadf'],
@@ -77,7 +99,15 @@
           'pkgs': ['cinder-backup'],
           'services': ['cinder-backup'],
           'engine': None
-        }
+        },
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
     },
 }, merge=pillar.cinder.get('volume', {}), base='BaseDefaults') %}
 
diff --git a/cinder/meta/fluentd.yml b/cinder/meta/fluentd.yml
new file mode 100644
index 0000000..0951dd0
--- /dev/null
+++ b/cinder/meta/fluentd.yml
@@ -0,0 +1,171 @@
+{%- from "cinder/map.jinja" import controller with context %}
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+{%- set apache_wsgi = controller.get('enabled', False) and controller.version == 'ocata' %}
+agent:
+  config:
+    label:
+      forward_input:
+        input:
+          generic_forward_input:
+            type: forward
+            bind: 0.0.0.0
+            port: 24224
+        match:
+          route_openstack_cinder:
+            tag: openstack.cinder.**
+            type: relabel
+            label: openstack_cinder
+{%- if apache_wsgi %}
+      openstack_cinder_wsgi:
+        input:
+          cinder_api_wsgi_in_tail:
+            type: tail
+            path: /var/log/apache2/cinder.log
+            tag: openstack.cinder
+            pos_file: {{ positiondb }}/cinder.wsgi.pos
+            parser:
+              type: regexp
+              time_key: Timestamp
+              time_format: '%d/%b/%Y:%H:%M:%S %z'
+              keep_time_key: false
+              # Apache format: https://regex101.com/r/WeCT7s/5
+              format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+        filter:
+          add_cinder_wsgi_record_fields:
+            tag: openstack.cinder
+            type: record_transformer
+            enable_ruby: true
+            record:
+              - name: Severity
+                value: 6
+              - name: severity_label
+                value: INFO
+              - name: programname
+                value: cinder-wsgi
+              - name: http_response_time
+                value: ${ record['http_response_time'].to_i/100000.to_f }
+        match:
+          send_to_default:
+            tag: openstack.cinder
+            type: copy
+            store:
+              - type: relabel
+                label: default_output
+              - type: rewrite_tag_filter
+                rule:
+                  - name: severity_label
+                    regexp: '.'
+                    result: metric.cinder_log_messages
+              - type: rewrite_tag_filter
+                rule:
+                  - name: http_status
+                    regexp: '.'
+                    result: metric.cinder_openstack_http_response
+          push_to_metric:
+            tag: 'metric.**'
+            type: relabel
+            label: default_metric
+{%- endif %}
+      openstack_cinder:
+        filter:
+          set_programname:
+            tag: openstack.cinder.*
+            type: record_transformer
+            enable_ruby: true
+            record:
+              - name: programname
+                value: cinder-${ tag_parts[2] }
+        filter:
+          set_log_record_fields:
+            tag: openstack.cinder
+            type: record_transformer
+            enable_ruby: true
+            record:
+              - name: Severity
+                value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+              - name: severity_label
+                value: ${ record['level'] }
+              - name: Payload
+                value: ${ record['message'] }
+              - name: python_module
+                value: ${ record['name'] }
+              - name: programname
+                value: '${ record["programname"] ? record["programname"] : "cinder" }'
+          parse_http_stats:
+            tag: openstack.cinder
+            type: parser
+            key_name: Payload
+            reserve_data: true
+            emit_invalid_record_to_error: false
+            parser:
+              type: regexp
+              # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+              format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+              types: http_response_time:float
+        match:
+          unify_tag:
+            tag: openstack.cinder.*
+            type: rewrite_tag_filter
+            rule:
+              - name: level
+                regexp: '.*'
+                result: openstack.cinder
+          send_to_default:
+            tag: openstack.cinder
+            type: copy
+            store:
+              - type: relabel
+                label: default_output
+              - type: rewrite_tag_filter
+                rule:
+                  - name: severity_label
+                    regexp: '.'
+                    result: metric.cinder_log_messages
+              - type: rewrite_tag_filter
+                rule:
+                  - name: http_status
+                    regexp: '.'
+                    result: metric.cinder_openstack_http_response
+          push_to_metric:
+            tag: 'metric.**'
+            type: relabel
+            label: default_metric
+      default_metric:
+        filter:
+          cinder_logs_per_severity:
+            tag: metric.cinder_log_messages
+            require:
+              - add_general_fields
+            type: prometheus
+            metric:
+              - name: log_messages
+                type: counter
+                desc: Total number of log lines by severity
+            label:
+              - name: service
+                value: cinder
+              - name: level
+                value: ${severity_label}
+              - name: host
+                value: ${Hostname}
+          cinder_openstack_http_response_times:
+            tag: metric.cinder_openstack_http_response
+            require:
+              - add_general_fields
+            type: prometheus
+            metric:
+              - name: openstack_http_response_times
+                type: summary
+                desc: Total number of requests per method and status
+                key: http_response_time
+            label:
+              - name: http_method
+                value: ${http_method}
+              - name: http_status
+                value: ${http_status}
+              - name: service
+                value: cinder
+              - name: host
+                value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/cinder/meta/grafana.yml b/cinder/meta/grafana.yml
index 0d0ecc8..fadfad4 100644
--- a/cinder/meta/grafana.yml
+++ b/cinder/meta/grafana.yml
@@ -1,8 +1,15 @@
 dashboard:
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+  cinder_prometheus:
+    datasource: prometheus
+    format: json
+    template: cinder/files/grafana_dashboards/cinder_prometheus_fluentd.json
+{%- else %}
   cinder_prometheus:
     datasource: prometheus
     format: json
     template: cinder/files/grafana_dashboards/cinder_prometheus.json
+{%- endif %}
   cinder_influxdb:
     datasource: influxdb
     format: json
diff --git a/cinder/meta/prometheus.yml b/cinder/meta/prometheus.yml
index dca35fb..e3392b0 100644
--- a/cinder/meta/prometheus.yml
+++ b/cinder/meta/prometheus.yml
@@ -78,7 +78,7 @@
     CinderErrorLogsTooHigh:
       {%- set log_threshold = monitoring.error_log_rate|float %}
       if: >-
-        sum(rate(log_messages{service="cinder",level=~"error|emergency|fatal"}[5m])) without (level) > {{ log_threshold }}
+        sum(rate(log_messages{service="cinder",level=~"(?i:(error|emergency|fatal))"}[5m])) without (level) > {{ log_threshold }}
 {%- raw %}
       labels:
         severity: warning
diff --git a/cinder/meta/salt.yml b/cinder/meta/salt.yml
index b10f98d..7df3cf9 100644
--- a/cinder/meta/salt.yml
+++ b/cinder/meta/salt.yml
@@ -8,4 +8,9 @@
     priority: 550
     require:
     - salt: cinder.controller
-
+dependency:
+  {% from "cinder/map.jinja" import controller, client with context %}
+  {%- if client.get('enabled', False) or controller.get('enabled', False) %}
+  engine: pkg
+  pkgs: {{ client.pkgs }}
+  {%- endif %}
diff --git a/cinder/meta/sphinx.yml b/cinder/meta/sphinx.yml
index eebde48..f326ee9 100644
--- a/cinder/meta/sphinx.yml
+++ b/cinder/meta/sphinx.yml
@@ -34,7 +34,7 @@
           value: |
             {%- for pkg in controller.pkgs %}
             {%- set pkg_version = "dpkg -l "+pkg+" | grep "+pkg+" | awk '{print $3}'" %}
-            * {{ pkg }}: {{ salt['cmd.run'](pkg_version) }}
+            * {{ pkg }}: {{ salt['cmd.shell'](pkg_version) }}
             {%- endfor %}
         {%- if controller.backend is defined %}
         backends:
@@ -75,7 +75,7 @@
           value: |
             {%- for pkg in volume.pkgs %}
             {%- set pkg_version = "dpkg -l "+pkg+" | grep "+pkg+" | awk '{print $3}'" %}
-            * {{ pkg }}: {{ salt['cmd.run'](pkg_version) }}
+            * {{ pkg }}: {{ salt['cmd.shell'](pkg_version) }}
             {%- endfor %}
         {%- if volume.backend is defined %}
         backends:
diff --git a/cinder/volume.sls b/cinder/volume.sls
index 8fee662..06f13c3 100644
--- a/cinder/volume.sls
+++ b/cinder/volume.sls
@@ -65,6 +65,70 @@
     - pkg: cinder_volume_packages
 
 {%- if volume.backup.engine != None %}
+  {%- set cinder_log_services = volume.services + volume.backup.services %}
+{%- else %}
+  {%- set cinder_log_services = volume.services %}
+{%- endif %}
+
+{% for service_name in cinder_log_services %}
+{{ service_name }}_default:
+  file.managed:
+    - name: /etc/default/{{ service_name }}
+    - source: salt://cinder/files/default
+    - template: jinja
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ volume }}
+    - require:
+      - pkg: cinder_volume_packages
+{%- if volume.backup.engine != None %}
+      - pkg: cinder_backup_packages
+{%- endif %}
+    - watch_in:
+      - service: cinder_volume_services
+{%- if volume.backup.engine != None %}
+      - pkg: cinder_backup_services
+{%- endif %}
+{% endfor %}
+
+{% if volume.logging.log_appender %}
+
+{%- if volume.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+cinder_volume_fluentd_logger_package:
+  pkg.installed:
+    - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in cinder_log_services %}
+{{ service_name }}_logging_conf:
+  file.managed:
+    - name: /etc/cinder/logging/logging-{{ service_name }}.conf
+    - source: salt://cinder/files/logging.conf
+    - template: jinja
+    - makedirs: True
+    - user: cinder
+    - group: cinder
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ volume }}
+    - require:
+      - pkg: cinder_volume_packages
+{%- if volume.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+      - pkg: cinder_volume_fluentd_logger_package
+{%- endif %}
+{%- if controller.backup.engine != None %}
+      - pkg: cinder_backup_packages
+{%- endif %}
+    - watch_in:
+      - service: cinder_volume_services
+{%- if controller.backup.engine != None %}
+      - pkg: cinder_backup_services
+{%- endif %}
+{% endfor %}
+
+{% endif %}
+
+{%- if volume.backup.engine != None %}
 
 cinder_backup_packages:
   pkg.installed:
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index ffd2aac..3bc81cf 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_cinder_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   cinder:
     controller:
       enabled: true
@@ -30,6 +32,13 @@
       glance:
         host: ${_param:cluster_vip_address}
         port: 9292
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/cluster_control.yml b/metadata/service/control/cluster_control.yml
index ffd2aac..3bc81cf 100644
--- a/metadata/service/control/cluster_control.yml
+++ b/metadata/service/control/cluster_control.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_cinder_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   cinder:
     controller:
       enabled: true
@@ -30,6 +32,13 @@
       glance:
         host: ${_param:cluster_vip_address}
         port: 9292
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index 1bf8378..3ebd30d 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_cinder_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   cinder:
     controller:
       enabled: true
@@ -30,6 +32,13 @@
       glance:
         host: ${_param:single_address}
         port: 9292
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 9298460..f1a069b 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
     _support:
       collectd:
         enabled: true
+      fluentd:
+        enabled: true
       heka:
         enabled: true
       sensu:
diff --git a/metadata/service/volume/local.yml b/metadata/service/volume/local.yml
index 592f010..074ed9d 100644
--- a/metadata/service/volume/local.yml
+++ b/metadata/service/volume/local.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_cinder_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   cinder:
     volume:
       enabled: true
@@ -30,6 +32,13 @@
       glance:
         host: ${_param:single_address}
         port: 9292
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:single_address}
diff --git a/metadata/service/volume/single.yml b/metadata/service/volume/single.yml
index 65a5640..689f0a6 100644
--- a/metadata/service/volume/single.yml
+++ b/metadata/service/volume/single.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_cinder_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   cinder:
     volume:
       enabled: true
@@ -30,6 +32,13 @@
       glance:
         host: ${_param:cluster_vip_address}
         port: 9292
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:cluster_vip_address}