diff --git a/.kitchen.yml b/.kitchen.yml
index c6c5d32..68b7f56 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -35,26 +35,14 @@
   sudo: true
 
 docker_images:
-  - &xenial-20163 <%=ENV['IMAGE_XENIAL_20163'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-2016.3/salt:2018_11_19'%>
-  - &xenial-20177 <%=ENV['IMAGE_XENIAL_20177'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-2017.7/salt:2018_11_19'%>
-  - &xenial-stable <%=ENV['IMAGE_XENIAL_STABLE'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19'%>
+  - &xenial-20177 <%=ENV['IMAGE_XENIAL_20177'] || 'docker-dev-local.docker.mirantis.net/mirantis/drivetrain/salt-formulas-ci/salt-formulas-ci-xenial-2017.7:latest'%>
 
 platforms:
-  - name: xenial-2016.3
-    driver_config:
-      image: *xenial-20163
-      platform: ubuntu
-
   - name: xenial-2017.7
     driver_config:
       image: *xenial-20177
       platform: ubuntu
 
-  - name: xenial-stable
-    driver_config:
-      image: *xenial-stable
-      platform: ubuntu
-
 suites:
 
   - name: ceph_client_single
diff --git a/README.rst b/README.rst
index a0c3d3c..74ac462 100644
--- a/README.rst
+++ b/README.rst
@@ -525,10 +525,11 @@
             ...
 
 
-Ceph manage keyring keys
+Ceph manage clients keyring keys
 ------------------------
 
-Keyrings are dynamically generated unless specified by the following pillar.
+Keyrings are dynamically generated unless specified by the manage_keyring pillar.
+This settings has no effect on admin keyring.
 
 .. code-block:: yaml
 
@@ -543,6 +544,31 @@
               mon: "allow r"
               osd: "allow class-read object_prefix rdb_children, allow rwx pool=images"
 
+Ceph manage admin keyring
+--------------------------
+To use pre-defined admin key add manage_admin_keyring and admin keyring definition to ceph mon nodes in cluster_model/ceph/mon.yml
+
+ceph:
+  common:
+    manage_admin_keyring: true
+    keyring:
+      admin:
+        caps:
+          mds: "allow *"
+          mgr: "allow *"
+          mon: "allow *"
+          osd: "allow *"
+        key: AACf3ulZFFPNDxAAd2DWds3aEkHh4IklZVgIaQ==
+
+Specify alternative keyring path and username
+
+.. code-block:: yaml
+
+    ceph:
+      radosgw:
+        keyring_user: radosgw.gateway
+        keyring_path: /etc/ceph/keyring.radosgw.gateway
+
 
 Generate CRUSH map - Recommended way
 -----------------------------------
diff --git a/_grains/ceph.py b/_grains/ceph.py
index 45652e8..54bc97e 100644
--- a/_grains/ceph.py
+++ b/_grains/ceph.py
@@ -6,6 +6,7 @@
     from subprocess import check_output
     from subprocess import CalledProcessError
     import shlex
+    import json
     import os
     import re
 
@@ -28,44 +29,60 @@
 
         # osd
         if os.path.exists('/var/lib/ceph/osd'):
-            mount_path = check_output("df -h | awk '{print $6}' | grep ceph | grep -v lockbox | sed 's/-[0-9]*//g' | awk 'NR==1{print $1}'", shell=True).rstrip()
-            sed = 'sed \'s#{0}-##g\''.format(mount_path)
-            cmd = "lsblk -rp | awk '{print $1,$6,$7}' | grep -v lockbox | grep ceph | " + sed
+            cmd = "ceph-volume lvm list --format json"
             osd_output = check_output(cmd, shell=True)
+            osd_output = json.loads(osd_output)
+            dev_id = ''
+            devices = {}
             if osd_output:
+                for osd, params in osd_output.iteritems():
+                    dev_id = osd
+                    devices[dev_id] = {}
+                    devices[dev_id]['dev'] = params[0]['devices'][0]
+                    devices[dev_id]['path'] = params[0]['path']
+                    devices[dev_id]['uuid'] = params[0]['tags']['ceph.osd_fsid']
+
+                cmd = "ceph osd tree --format json"
+                osd_tree_output = check_output(cmd, shell=True)
+                osd_tree_output = json.loads(osd_tree_output)
+                for osd in osd_tree_output['nodes']:
+                    if 'type_id' in osd.keys():
+                        if str(osd['type_id']) == '0':
+                            for dev_id in devices.keys():
+                                if str(osd['id']) == str(dev_id):
+                                    devices[dev_id]['weight'] = osd['crush_weight']
+                                    devices[dev_id]['class'] = osd['device_class']
+                grain["ceph"]["ceph_volume"] = devices
+            else:
+                cmd = "ceph-disk list --format json"
+                osd_output = check_output(cmd, shell=True).decode("utf-8")
+                osd_output = json.loads(osd_output)
+                dev_id = ''
                 devices = {}
-                for line in osd_output.splitlines():
-                    device = line.split()
-                    encrypted = False
-                    if "crypt" in device[1]:
-                        output = check_output("lsblk -rp | grep -B1 " + device[0], shell=True)
-                        for l in output.splitlines():
-                            d = l.split()
-                            dev = re.sub("\d+", "", device[0])
-                            encrypted = True
-                            break
-                    else:
-                        dev = device[0].replace('1','')
-                        dev = re.sub("\d+", "", device[0])
-                    device[0] = device[2]
-                    try:
-                        devices[device[0]] = {}
-                        devices[device[0]]['dev'] = dev
-                        if encrypted:
-                            devices[device[0]]['dmcrypt'] = 'true'
-                        tline = check_output("ceph -c " + conf_file + " osd tree | awk '{print $1,$2,$3,$4}' | grep -w 'osd." + device[0] + "'", shell=True)
-                        osd = tline.split()
-                        if "osd" not in osd[2]:
-                            crush_class = osd[1]
-                            crush_weight = osd[2]
-                            devices[device[0]]['class'] = crush_class
-                            devices[device[0]]['weight'] = crush_weight
-                        else:
-                            crush_weight = osd[1]
-                            devices[device[0]]['weight'] = crush_weight
-                    except CalledProcessError:
-                        continue
-                grain["ceph"]["ceph_disk"] = devices
+                if osd_output:
+                    for line in osd_output:
+                        if "is_partition" not in line.keys():
+                            dev = line["path"]
+                            parts = line["partitions"]
+                            for p in parts:
+                                if "mount" in p.keys() and "ceph" in p["mount"]:
+                                    dev_id = p["whoami"]
+                                    devices[dev_id] = {}
+                                    devices[dev_id]['dev'] = dev
+                                    if len(p["dmcrypt"]) > 0:
+                                        devices[dev_id]['dmcrypt'] = 'true'
+
+                    cmd = "ceph osd tree --format json"
+                    osd_tree_output = check_output(cmd, shell=True).decode("utf-8")
+                    osd_tree_output = json.loads(osd_tree_output)
+                    for osd in osd_tree_output['nodes']:
+                        if 'type_id' in osd.keys():
+                            if str(osd['type_id']) == '0':
+                                for dev_id in devices.keys():
+                                    if str(osd['id']) == str(dev_id):
+                                        devices[dev_id]['weight'] = osd['crush_weight']
+                                        devices[dev_id]['class'] = osd['device_class']
+                    grain["ceph"]["ceph_disk"] = devices
 
         # keyrings
         directory = '/etc/ceph/'
diff --git a/ceph/backup.sls b/ceph/backup.sls
index 7963e5c..5067012 100644
--- a/ceph/backup.sls
+++ b/ceph/backup.sls
@@ -54,6 +54,8 @@
 {%- endif %}
 {%- if backup.client.backup_times.minute is defined %}
   - minute: {{ backup.client.backup_times.minute }}
+{%- else %}
+  - minute: 0
 {%- endif %}
 {%- elif backup.client.hours_before_incr is defined %}
   - minute: 0
@@ -155,6 +157,8 @@
 {%- endif %}
 {%- if backup.server.backup_times.minute is defined %}
   - minute: {{ backup.server.backup_times.minute }}
+{%- else %}
+  - minute: 0
 {%- endif %}
 {%- elif backup.server.hours_before_incr is defined %}
   - minute: 0
diff --git a/ceph/common.sls b/ceph/common.sls
index b445355..964dff7 100644
--- a/ceph/common.sls
+++ b/ceph/common.sls
@@ -37,6 +37,20 @@
 
 {%- if common.keyring is defined and common.keyring.admin is defined %}
 
+{%- if common.get("manage_admin_keyring", False) %}
+
+ceph_create_keyring_admin:
+  cmd.run:
+  - name: "ceph-authtool --create-keyring {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.admin.keyring --add-key {{ common.keyring.admin.key }} -n client.admin {%- for cap_name, cap in  common.keyring.admin.caps.iteritems() %} --cap {{ cap_name }} '{{ cap }}' {%- endfor %}"
+  - unless: "test -f {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.admin.keyring"
+  - require:
+  {% if not common.get('container_mode', False) %}
+    - pkg: common_packages
+  {%- endif %}
+    - file: common_config
+
+{%- else %}
+
 ceph_create_keyring_admin:
   cmd.run:
   - name: "ceph-authtool --create-keyring {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.admin.keyring --gen-key -n client.admin {%- for cap_name, cap in  common.keyring.admin.caps.iteritems() %} --cap {{ cap_name }} '{{ cap }}' {%- endfor %}"
@@ -49,6 +63,8 @@
 
 {%- endif %}
 
+{%- endif %}
+
 {%- for node_name, node_grains in salt['mine.get']('ceph:common:keyring:admin', 'grains.items', 'pillar').iteritems() %}
 
 {%- if node_grains.ceph is defined and node_grains.ceph.ceph_keyring is defined and node_grains.ceph.ceph_keyring.admin is defined and node_grains.ceph.get('fsid', '') == common.fsid %}
diff --git a/ceph/files/backup/ceph-backup-client-runner.sh b/ceph/files/backup/ceph-backup-client-runner.sh
index 971f944..bb04fc6 100644
--- a/ceph/files/backup/ceph-backup-client-runner.sh
+++ b/ceph/files/backup/ceph-backup-client-runner.sh
@@ -45,9 +45,13 @@
     rsync -arv --exclude=osd/{{ common.get('cluster_name', 'ceph') }}-*/current /var/lib/ceph $TMPDIR/{{ common.get('cluster_name', 'ceph') }}-$HOSTNAME/
 {%- elif mon.get('enabled', False) %}
     cp -a /etc/ceph/ $TMPDIR/
-    service ceph-mon@$HOSTNAME stop
-    cp -a /var/lib/ceph/ $TMPDIR/{{ common.get('cluster_name', 'ceph') }}-$HOSTNAME/
-    service ceph-mon@$HOSTNAME start
+    if systemctl status ceph-mon@$HOSTNAME >/dev/null; then
+        systemctl stop ceph-mon@$HOSTNAME
+        cp -a /var/lib/ceph/ $TMPDIR/{{ common.get('cluster_name', 'ceph') }}-$HOSTNAME/
+        systemctl start ceph-mon@$HOSTNAME
+    else
+        cp -a /var/lib/ceph/ $TMPDIR/{{ common.get('cluster_name', 'ceph') }}-$HOSTNAME/
+    fi
 {%- endif %}
 
     tar -cvzf $BACKUPDIR/$HOSTNAME/{{ common.get('cluster_name', 'ceph') }}-$HOSTNAME-$TIMESTAMP.tgz $TMPDIR
diff --git a/ceph/files/grafana_dashboards/ceph-cluster_prometheus.json b/ceph/files/grafana_dashboards/ceph-cluster_prometheus.json
index 2f54c74..5770143 100644
--- a/ceph/files/grafana_dashboards/ceph-cluster_prometheus.json
+++ b/ceph/files/grafana_dashboards/ceph-cluster_prometheus.json
@@ -124,6 +124,88 @@
       "colorBackground": false,
       "colorValue": false,
       "colors": [
+        "rgba(50, 128, 45, 0.9)",
+        "rgba(237, 129, 40, 0.9)",
+        "rgb(255, 0, 0)"
+      ],
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "format": "bytes",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 2,
+        "x": 2,
+        "y": 0
+      },
+      "hideTimeOverride": true,
+      "id": 67,
+      "interval": "1m",
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "span": 2,
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "sum(ceph_osd_stat_bytes{instance=~\"$instance\"})",
+          "format": "time_series",
+          "interval": "$interval",
+          "intervalFactor": 1,
+          "refId": "A",
+          "step": 60
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": "1m",
+      "title": "Total",
+      "transparent": false,
+      "type": "singlestat",
+      "valueFontSize": "50%",
+      "valueMaps": [],
+      "valueName": "current"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
         "#299c46",
         "rgba(237, 129, 40, 0.89)",
         "#d44a3a"
@@ -471,6 +553,170 @@
       }
     },
     {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "rgba(50, 128, 45, 0.9)",
+        "rgba(237, 129, 40, 0.9)",
+        "rgb(255, 0, 0)"
+      ],
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "format": "bytes",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 2,
+        "x": 0,
+        "y": 3
+      },
+      "hideTimeOverride": true,
+      "id": 68,
+      "interval": "1m",
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "span": 2,
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "sum(ceph_osd_stat_bytes{instance=~\"$instance\"})-sum(ceph_osd_stat_bytes_used{instance=~\"$instance\"})",
+          "format": "time_series",
+          "interval": "$interval",
+          "intervalFactor": 1,
+          "refId": "A",
+          "step": 60
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": "1m",
+      "title": "Free",
+      "transparent": false,
+      "type": "singlestat",
+      "valueFontSize": "50%",
+      "valueMaps": [],
+      "valueName": "current"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "rgba(50, 128, 45, 0.9)",
+        "rgba(237, 129, 40, 0.9)",
+        "rgb(255, 0, 0)"
+      ],
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "format": "bytes",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 2,
+        "x": 2,
+        "y": 3
+      },
+      "hideTimeOverride": true,
+      "id": 69,
+      "interval": "1m",
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "span": 2,
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "sum(ceph_osd_stat_bytes_used{instance=~\"$instance\"})",
+          "format": "time_series",
+          "interval": "$interval",
+          "intervalFactor": 1,
+          "refId": "A",
+          "step": 60
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": "1m",
+      "title": "Used",
+      "transparent": false,
+      "type": "singlestat",
+      "valueFontSize": "50%",
+      "valueMaps": [],
+      "valueName": "current"
+    },
+    {
       "aliasColors": {},
       "bars": false,
       "dashLength": 10,
@@ -512,14 +758,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(ceph_osd_op_w_in_bytes{instance=~\"$instance\"}[1m]))",
+          "expr": "sum(irate(ceph_osd_op_w_in_bytes{instance=~\"$instance\"}[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Writes",
           "refId": "A"
         },
         {
-          "expr": "sum(irate(ceph_osd_op_r_out_bytes{instance=~\"$instance\"}[1m]))",
+          "expr": "sum(irate(ceph_osd_op_r_out_bytes{instance=~\"$instance\"}[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Reads",
@@ -603,7 +849,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(deriv(ceph_pool_bytes_used{instance=~\"$instance\"}[1m]))",
+          "expr": "sum(deriv(ceph_pool_bytes_used{instance=~\"$instance\"}[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Bytes",
@@ -813,7 +1059,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(ceph_osd_recovery_ops[1m]))",
+          "expr": "sum(irate(ceph_osd_recovery_ops[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Op/s",
@@ -972,6 +1218,49 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "tags": [],
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
       }
     ]
   },
diff --git a/ceph/files/grafana_dashboards/hosts-overview_prometheus.json b/ceph/files/grafana_dashboards/hosts-overview_prometheus.json
index bdc6a90..921f886 100644
--- a/ceph/files/grafana_dashboards/hosts-overview_prometheus.json
+++ b/ceph/files/grafana_dashboards/hosts-overview_prometheus.json
@@ -577,7 +577,7 @@
       "tableColumn": "Value",
       "targets": [
         {
-          "expr": "avg (\n  ((irate(diskio_io_time[5m]) / 10 )\n  ) *\n  on(host, name) diskio_io_time{host=~\"($osd_hosts).*\"}\n)",
+          "expr": "avg ( (irate(diskio_io_time{host=~\"($osd_hosts).*\"}[5m])) )",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -660,7 +660,7 @@
       "tableColumn": "Value",
       "targets": [
         {
-          "expr": "sum (\n  irate(net_bytes_recv{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n  ) +\nsum (\n  irate(net_bytes_sent{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) \n  )",
+          "expr": "sum (\n  irate(net_bytes_recv{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[$rate_interval])\n  ) +\nsum (\n  irate(net_bytes_sent{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[$rate_interval]) \n  )",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -814,7 +814,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "topk(10, (sum by(host) (\n  (\n  irate(net_bytes_recv{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n  ) +\n  (\n  irate(net_bytes_sent{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n  ))\n  )\n)",
+          "expr": "topk(10, (sum by(host) (\n  (\n  irate(net_bytes_recv{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[$rate_interval])\n  ) +\n  (\n  irate(net_bytes_sent{host=~\"($osd_hosts|mon_hosts|rgw_hosts).*\",device!=\"lo\"}[$rate_interval])\n  ))\n  )\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{host}}",
@@ -1029,6 +1029,48 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
       }
     ]
   },
diff --git a/ceph/files/grafana_dashboards/osds-detail_prometheus.json b/ceph/files/grafana_dashboards/osds-detail_prometheus.json
index b9b950b..f02abbb 100644
--- a/ceph/files/grafana_dashboards/osds-detail_prometheus.json
+++ b/ceph/files/grafana_dashboards/osds-detail_prometheus.json
@@ -70,14 +70,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(ceph_osd_op_r_latency_sum{ceph_daemon=~\"osd.[[osd_id]]\"}[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m]) * 1000",
+          "expr": "irate(ceph_osd_op_r_latency_sum{ceph_daemon=~\"osd.[[osd_id]]\"}[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[$rate_interval]) * 1000",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "READs",
           "refId": "A"
         },
         {
-          "expr": "irate(ceph_osd_op_w_latency_sum{ceph_daemon=~\"osd.[[osd_id]]\"}[1m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[1m]) * 1000",
+          "expr": "irate(ceph_osd_op_w_latency_sum{ceph_daemon=~\"osd.[[osd_id]]\"}[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[$rate_interval]) * 1000",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "WRITEs",
@@ -161,14 +161,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(ceph_osd_op_r{ceph_daemon=~\"osd.[[osd_id]]\"}[1m])",
+          "expr": "irate(ceph_osd_op_r{ceph_daemon=~\"osd.[[osd_id]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Reads",
           "refId": "A"
         },
         {
-          "expr": "irate(ceph_osd_op_w{ceph_daemon=~\"osd.[[osd_id]]\"}[1m])",
+          "expr": "irate(ceph_osd_op_w{ceph_daemon=~\"osd.[[osd_id]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Writes",
@@ -252,14 +252,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(ceph_osd_op_r_out_bytes{ceph_daemon=~\"osd.[[osd_id]]\"}[1m])",
+          "expr": "irate(ceph_osd_op_r_out_bytes{ceph_daemon=~\"osd.[[osd_id]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Read Bytes",
           "refId": "A"
         },
         {
-          "expr": "irate(ceph_osd_op_w_in_bytes{ceph_daemon=~\"osd.[[osd_id]]\"}[1m])",
+          "expr": "irate(ceph_osd_op_w_in_bytes{ceph_daemon=~\"osd.[[osd_id]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Write Bytes",
@@ -356,14 +356,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "label_replace(label_replace(irate(diskio_read_time[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") / label_replace(label_replace(irate(diskio_reads[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_read_time[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") / label_replace(label_replace(irate(diskio_reads[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}}/{{device}} Reads",
           "refId": "A"
         },
         {
-          "expr": "label_replace(label_replace(irate(diskio_write_time[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") / label_replace(label_replace(irate(diskio_writes[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_write_time[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") / label_replace(label_replace(irate(diskio_writes[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}}/{{device}} Writes",
@@ -447,14 +447,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "label_replace(label_replace(irate(diskio_reads[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_reads[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}} {{device}} READS",
           "refId": "A"
         },
         {
-          "expr": "label_replace(label_replace(irate(diskio_writes[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_writes[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}} {{device}} WRITES",
@@ -538,14 +538,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "label_replace(label_replace(irate(diskio_read_bytes[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_read_bytes[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}} {{device}} READS",
           "refId": "A"
         },
         {
-          "expr": "label_replace(label_replace(irate(diskio_write_bytes[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_write_bytes[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}} {{device}} WRITES",
@@ -629,7 +629,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "label_replace(label_replace(irate(diskio_io_time[1m]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
+          "expr": "label_replace(label_replace(irate(diskio_io_time[$rate_interval]), \"instance\", \"$1\", \"host\", \"(.+)\"), \"device\", \"$1\", \"name\", \"(.+)\") and on (instance, device) ceph_disk_occupation{ceph_daemon=~\"osd.[[osd_id]]\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{instance}} {{device}}",
@@ -708,6 +708,48 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
       }
     ]
   },
diff --git a/ceph/files/grafana_dashboards/osds-overview_prometheus.json b/ceph/files/grafana_dashboards/osds-overview_prometheus.json
index c399fc0..3337014 100644
--- a/ceph/files/grafana_dashboards/osds-overview_prometheus.json
+++ b/ceph/files/grafana_dashboards/osds-overview_prometheus.json
@@ -58,21 +58,21 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "avg (irate(ceph_osd_op_r_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m]) * 1000)",
+          "expr": "avg (irate(ceph_osd_op_r_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[$rate_interval]) * 1000)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "AVG read",
           "refId": "A"
         },
         {
-          "expr": "max (irate(ceph_osd_op_r_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m]) * 1000)",
+          "expr": "max (irate(ceph_osd_op_r_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[$rate_interval]) * 1000)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "MAX read",
           "refId": "B"
         },
         {
-          "expr": "quantile(0.95,\n  (irate(ceph_osd_op_r_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m]) * 1000)\n)",
+          "expr": "quantile(0.95,\n  (irate(ceph_osd_op_r_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[$rate_interval]) * 1000)\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "@95%ile",
@@ -188,7 +188,7 @@
       ],
       "targets": [
         {
-          "expr": "topk(10,\n  (sort(\n    (irate(ceph_osd_op_r_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m]) * 1000)\n  ))\n)\n\n",
+          "expr": "topk(10,\n  (sort(\n    (irate(ceph_osd_op_r_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[$rate_interval]) * 1000)\n  ))\n)\n\n",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -239,21 +239,21 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "avg (irate(ceph_osd_op_w_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[1m]) * 1000)",
+          "expr": "avg (irate(ceph_osd_op_w_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[$rate_interval]) * 1000)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "AVG write",
           "refId": "A"
         },
         {
-          "expr": "max (irate(ceph_osd_op_w_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[1m]) * 1000)",
+          "expr": "max (irate(ceph_osd_op_w_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[$rate_interval]) * 1000)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "MAX write",
           "refId": "B"
         },
         {
-          "expr": "quantile(0.95,\n (irate(ceph_osd_op_w_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[1m]) * 1000)\n)",
+          "expr": "quantile(0.95,\n (irate(ceph_osd_op_w_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[$rate_interval]) * 1000)\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "@95%ile write",
@@ -369,7 +369,7 @@
       ],
       "targets": [
         {
-          "expr": "topk(10,\n  (sort(\n    (irate(ceph_osd_op_w_latency_sum[1m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[1m]) * 1000)\n  ))\n)\n\n",
+          "expr": "topk(10,\n  (sort(\n    (irate(ceph_osd_op_w_latency_sum[$rate_interval]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[$rate_interval]) * 1000)\n  ))\n)\n\n",
           "format": "table",
           "instant": true,
           "intervalFactor": 1,
@@ -383,7 +383,7 @@
     },
     {
       "aliasColors": {},
-      "bars": false,
+      "bars": true,
       "dashLength": 10,
       "dashes": false,
       "datasource": null,
@@ -405,11 +405,11 @@
         "max": false,
         "min": false,
         "rightSide": false,
-        "show": true,
+        "show": false,
         "total": false,
         "values": false
       },
-      "lines": true,
+      "lines": false,
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null",
@@ -427,7 +427,7 @@
         {
           "expr": "ceph_osd_numpg",
           "format": "time_series",
-          "instant": false,
+          "instant": true,
           "intervalFactor": 1,
           "legendFormat": "PGs per OSD",
           "refId": "A"
@@ -446,7 +446,7 @@
       "type": "graph",
       "xaxis": {
         "buckets": 20,
-        "mode": "time",
+        "mode": "histogram",
         "name": null,
         "show": true,
         "values": []
@@ -526,7 +526,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "round(sum(irate(ceph_pool_rd[1m])))",
+          "expr": "round(sum(irate(ceph_pool_rd[$rate_interval])))",
           "format": "time_series",
           "hide": false,
           "instant": false,
@@ -536,7 +536,7 @@
           "refId": "A"
         },
         {
-          "expr": "round(sum(irate(ceph_pool_wr[1m])))",
+          "expr": "round(sum(irate(ceph_pool_wr[$rate_interval])))",
           "format": "time_series",
           "instant": false,
           "interval": "",
@@ -594,7 +594,51 @@
     "osd"
   ],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "tags": [],
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
+      }
+    ]
   },
   "time": {
     "from": "now-1h",
diff --git a/ceph/files/grafana_dashboards/pool-overview_prometheus.json b/ceph/files/grafana_dashboards/pool-overview_prometheus.json
index a58e6a2..600ef46 100644
--- a/ceph/files/grafana_dashboards/pool-overview_prometheus.json
+++ b/ceph/files/grafana_dashboards/pool-overview_prometheus.json
@@ -16,8 +16,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": null,
-  "iteration": 1545072836850,
+  "iteration": 1574692480292,
   "links": [],
   "panels": [
     {
@@ -40,10 +39,10 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": null,
-      "fill": 5,
+      "fill": 2,
       "gridPos": {
         "h": 7,
-        "w": 24,
+        "w": 12,
         "x": 0,
         "y": 1
       },
@@ -68,11 +67,11 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": true,
+      "stack": false,
       "steppedLine": false,
       "targets": [
         {
-          "expr": "avg((rate(ceph_pool_rd{pool_id=~\"[[pool_id]]\"}[1m]) + rate(ceph_pool_wr{pool_id=~\"[[pool_id]]\"}[1m])) + on(pool_id,instance) group_left(name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance)",
+          "expr": "avg(rate(ceph_pool_rd{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id,instance) group_left(name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance)",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -83,7 +82,7 @@
       "thresholds": [],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Client IOPS by Pool",
+      "title": "Read IOPS by Pool",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -99,7 +98,7 @@
       },
       "yaxes": [
         {
-          "format": "none",
+          "format": "rps",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -126,10 +125,96 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": null,
-      "fill": 5,
+      "fill": 2,
       "gridPos": {
-        "h": 6,
-        "w": 24,
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 16,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "minSpan": 12,
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg(rate(ceph_pool_wr{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id,instance) group_left(name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance)",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{name}}",
+          "refId": "F"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Write IOPS by Pool",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "wps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 2,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
         "x": 0,
         "y": 8
       },
@@ -154,11 +239,11 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": true,
+      "stack": false,
       "steppedLine": false,
       "targets": [
         {
-          "expr": "avg((rate(ceph_pool_rd_bytes{pool_id=~\"[[pool_id]]\"}[1m]) + rate(ceph_pool_wr_bytes{pool_id=~\"[[pool_id]]\"}[1m])) + on(pool_id,instance) group_left(name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance)",
+          "expr": "avg(rate(ceph_pool_rd_bytes{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id,instance) group_left(name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{name}}",
@@ -169,7 +254,7 @@
       "thresholds": [],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Client Throughput by Pool",
+      "title": "Read Throughput by Pool",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -185,7 +270,93 @@
       },
       "yaxes": [
         {
-          "format": "decbytes",
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 2,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 17,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "minSpan": 12,
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg(rate(ceph_pool_wr_bytes{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id,instance) group_left(name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance)",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{name}}",
+          "refId": "A",
+          "textEditor": true
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Write Throughput by Pool",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -212,7 +383,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 14
+        "y": 15
       },
       "id": 15,
       "panels": [],
@@ -231,9 +402,9 @@
       "fontSize": "100%",
       "gridPos": {
         "h": 7,
-        "w": 8,
+        "w": 12,
         "x": 0,
-        "y": 15
+        "y": 16
       },
       "id": 3,
       "links": [],
@@ -242,7 +413,7 @@
       "scroll": true,
       "showHeader": true,
       "sort": {
-        "col": 6,
+        "col": 7,
         "desc": true
       },
       "styles": [
@@ -337,7 +508,7 @@
           "unit": "short"
         },
         {
-          "alias": "IOPS (R+W)",
+          "alias": "Read IOPS",
           "colorMode": null,
           "colors": [
             "rgba(245, 54, 54, 0.9)",
@@ -349,12 +520,12 @@
           "pattern": "Value",
           "thresholds": [],
           "type": "number",
-          "unit": "none"
+          "unit": "rps"
         }
       ],
       "targets": [
         {
-          "expr": "topk(5,(label_replace((irate(ceph_pool_rd{pool_id=~\"[[pool_id]]\"}[1m]) + irate(ceph_pool_wr{pool_id=~\"[[pool_id]]\"}[1m])),\"id\", \"$1\", \"pool_id\", \"(.*)\") + on(pool_id) group_left(instance,name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) )",
+          "expr": "topk(5,avg(rate(ceph_pool_rd{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance))",
           "format": "table",
           "instant": true,
           "intervalFactor": 2,
@@ -362,7 +533,7 @@
           "textEditor": true
         }
       ],
-      "title": "Top 5 Pools by Client IOPS",
+      "title": "Top 5 Pools by Read IOPS",
       "transform": "table",
       "type": "table"
     },
@@ -377,26 +548,35 @@
       "fontSize": "100%",
       "gridPos": {
         "h": 7,
-        "w": 8,
-        "x": 8,
-        "y": 15
+        "w": 12,
+        "x": 12,
+        "y": 16
       },
-      "id": 4,
+      "id": 20,
       "links": [],
       "minSpan": 12,
       "pageSize": null,
       "scroll": true,
       "showHeader": true,
       "sort": {
-        "col": 6,
+        "col": 7,
         "desc": true
       },
       "styles": [
         {
-          "alias": "Time",
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
           "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
           "pattern": "Time",
-          "type": "hidden"
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
         },
         {
           "alias": "",
@@ -474,7 +654,63 @@
           "unit": "short"
         },
         {
-          "alias": "Throughput",
+          "alias": "Write IOPS",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "pattern": "Value",
+          "thresholds": [],
+          "type": "number",
+          "unit": "wps"
+        }
+      ],
+      "targets": [
+        {
+          "expr": "topk(5,avg(rate(ceph_pool_wr{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance))",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "refId": "A",
+          "textEditor": true
+        }
+      ],
+      "title": "Top 5 Pools by Write IOPS",
+      "transform": "table",
+      "type": "table"
+    },
+    {
+      "columns": [
+        {
+          "text": "Current",
+          "value": "current"
+        }
+      ],
+      "datasource": null,
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 23
+      },
+      "id": 21,
+      "links": [],
+      "minSpan": 12,
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 7,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "",
           "colorMode": null,
           "colors": [
             "rgba(245, 54, 54, 0.9)",
@@ -483,15 +719,105 @@
           ],
           "dateFormat": "YYYY-MM-DD HH:mm:ss",
           "decimals": 2,
+          "pattern": "Time",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "id",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "instance",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "job",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "Pool Name",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "name",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "Pool ID",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "pool_id",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "Read Throughput",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
           "pattern": "Value",
           "thresholds": [],
           "type": "number",
-          "unit": "decbytes"
+          "unit": "Bps"
         }
       ],
       "targets": [
         {
-          "expr": "(label_replace((irate(ceph_pool_rd_bytes{pool_id=~\"[[pool_id]]\"}[1m]) + irate(ceph_pool_wr_bytes{pool_id=~\"[[pool_id]]\"}[1m])),\"id\", \"$1\", \"pool_id\", \"(.*)\") + on(pool_id) group_left(instance,name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) ",
+          "expr": "topk(5,avg(rate(ceph_pool_rd_bytes{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance))",
           "format": "table",
           "instant": true,
           "intervalFactor": 2,
@@ -499,7 +825,153 @@
           "textEditor": true
         }
       ],
-      "title": "Top 5 Pools by Throughput",
+      "title": "Top 5 Pools by Read Throughput",
+      "transform": "table",
+      "type": "table"
+    },
+    {
+      "columns": [
+        {
+          "text": "Current",
+          "value": "current"
+        }
+      ],
+      "datasource": null,
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 23
+      },
+      "id": 22,
+      "links": [],
+      "minSpan": 12,
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 7,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "Time",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "id",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "instance",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "job",
+          "thresholds": [],
+          "type": "hidden",
+          "unit": "short"
+        },
+        {
+          "alias": "Pool Name",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "name",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "Pool ID",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "pool_id",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "Write Throughput",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "pattern": "Value",
+          "thresholds": [],
+          "type": "number",
+          "unit": "Bps"
+        }
+      ],
+      "targets": [
+        {
+          "expr": "topk(5,avg(rate(ceph_pool_wr_bytes{pool_id=~\"[[pool_id]]\"}[$rate_interval]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{pool_id=~\"[[pool_id]]\"}) without (instance))",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "refId": "A",
+          "textEditor": true
+        }
+      ],
+      "title": "Top 5 Pools by Write Throughput",
       "transform": "table",
       "type": "table"
     },
@@ -509,9 +981,9 @@
       "fontSize": "100%",
       "gridPos": {
         "h": 7,
-        "w": 8,
-        "x": 16,
-        "y": 15
+        "w": 12,
+        "x": 0,
+        "y": 30
       },
       "id": 5,
       "links": [],
@@ -520,7 +992,7 @@
       "scroll": true,
       "showHeader": true,
       "sort": {
-        "col": 5,
+        "col": 6,
         "desc": true
       },
       "styles": [
@@ -620,7 +1092,7 @@
       ],
       "targets": [
         {
-          "expr": "topk(5,((ceph_pool_bytes_used / (ceph_pool_bytes_used + ceph_pool_max_avail)) * on(pool_id) group_left(name) ceph_pool_metadata))",
+          "expr": "topk(5,avg((ceph_pool_bytes_used / (ceph_pool_bytes_used + ceph_pool_max_avail)) * on(pool_id) group_left(name) ceph_pool_metadata) without (instance))",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -690,6 +1162,49 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "tags": [],
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
       }
     ]
   },
@@ -725,6 +1240,6 @@
   },
   "timezone": "browser",
   "title": "Ceph Pools Overview",
-  "version": 11
+  "version": 12
 }
 {%- endraw %}
diff --git a/ceph/files/grafana_dashboards/radosgw-detail_prometheus.json b/ceph/files/grafana_dashboards/radosgw-detail_prometheus.json
index fa95510..c536351 100644
--- a/ceph/files/grafana_dashboards/radosgw-detail_prometheus.json
+++ b/ceph/files/grafana_dashboards/radosgw-detail_prometheus.json
@@ -71,14 +71,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(ceph_rgw_get_initial_lat_sum{ceph_daemon=~\"($rgw_servers)\"}[1m]) / rate(ceph_rgw_get_initial_lat_count{ceph_daemon=~\"($rgw_servers)\"}[1m])",
+          "expr": "rate(ceph_rgw_get_initial_lat_sum{ceph_daemon=~\"($rgw_servers)\"}[$rate_interval]) / rate(ceph_rgw_get_initial_lat_count{ceph_daemon=~\"($rgw_servers)\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "GET {{ceph_daemon}}",
           "refId": "A"
         },
         {
-          "expr": "rate(ceph_rgw_put_initial_lat_sum{ceph_daemon=~\"($rgw_servers)\"}[1m]) / rate(ceph_rgw_put_initial_lat_count{ceph_daemon=~\"($rgw_servers)\"}[1m])",
+          "expr": "rate(ceph_rgw_put_initial_lat_sum{ceph_daemon=~\"($rgw_servers)\"}[$rate_interval]) / rate(ceph_rgw_put_initial_lat_count{ceph_daemon=~\"($rgw_servers)\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "PUT {{ceph_daemon}}",
@@ -162,14 +162,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(ceph_rgw_get_b{ceph_daemon=~\"[[rgw_servers]]\"}[1m])",
+          "expr": "rate(ceph_rgw_get_b{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "GETs {{ceph_daemon}}",
           "refId": "B"
         },
         {
-          "expr": "rate(ceph_rgw_put_b{ceph_daemon=~\"[[rgw_servers]]\"}[1m])",
+          "expr": "rate(ceph_rgw_put_b{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "PUTs {{ceph_daemon}}",
@@ -260,28 +260,28 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(ceph_rgw_failed_req{ceph_daemon=~\"[[rgw_servers]]\"}[1m])",
+          "expr": "rate(ceph_rgw_failed_req{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Requests Failed {{ceph_daemon}}",
           "refId": "B"
         },
         {
-          "expr": "rate(ceph_rgw_get{ceph_daemon=~\"[[rgw_servers]]\"}[1m])",
+          "expr": "rate(ceph_rgw_get{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "GETs  {{ceph_daemon}}",
           "refId": "C"
         },
         {
-          "expr": "rate(ceph_rgw_put{ceph_daemon=~\"[[rgw_servers]]\"}[1m])",
+          "expr": "rate(ceph_rgw_put{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "PUTs  {{ceph_daemon}}",
           "refId": "D"
         },
         {
-          "expr": "rate(ceph_rgw_req{ceph_daemon=~\"[[rgw_servers]]\"}[1m]) -\n  (rate(ceph_rgw_get{ceph_daemon=~\"[[rgw_servers]]\"}[1m]) +\n   rate(ceph_rgw_put{ceph_daemon=~\"[[rgw_servers]]\"}[1m]))",
+          "expr": "rate(ceph_rgw_req{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval]) -\n  (rate(ceph_rgw_get{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval]) +\n   rate(ceph_rgw_put{ceph_daemon=~\"[[rgw_servers]]\"}[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "Other  {{ceph_daemon}}",
@@ -361,6 +361,48 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
       }
     ]
   },
diff --git a/ceph/files/grafana_dashboards/radosgw-overview_prometheus.json b/ceph/files/grafana_dashboards/radosgw-overview_prometheus.json
index 1cafb7a..4a69c5c 100644
--- a/ceph/files/grafana_dashboards/radosgw-overview_prometheus.json
+++ b/ceph/files/grafana_dashboards/radosgw-overview_prometheus.json
@@ -70,14 +70,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "avg(rate(ceph_rgw_get_initial_lat_sum[1m]) / rate(ceph_rgw_get_initial_lat_count[1m]))",
+          "expr": "avg(rate(ceph_rgw_get_initial_lat_sum[$rate_interval]) / rate(ceph_rgw_get_initial_lat_count[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "GET AVG",
           "refId": "A"
         },
         {
-          "expr": "avg(rate(ceph_rgw_put_initial_lat_sum[1m]) / rate(ceph_rgw_put_initial_lat_count[1m]))",
+          "expr": "avg(rate(ceph_rgw_put_initial_lat_sum[$rate_interval]) / rate(ceph_rgw_put_initial_lat_count[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "PUT AVG",
@@ -161,7 +161,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum by(rgw_host) (label_replace(rate(ceph_rgw_req[1m]), \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"))",
+          "expr": "sum by(rgw_host) (label_replace(rate(ceph_rgw_req[$rate_interval]), \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{rgw_host}}",
@@ -247,7 +247,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "label_replace(rate(ceph_rgw_get_initial_lat_sum[1m]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\") / \nlabel_replace(rate(ceph_rgw_get_initial_lat_count[1m]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\")",
+          "expr": "label_replace(rate(ceph_rgw_get_initial_lat_sum[$rate_interval]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\") / \nlabel_replace(rate(ceph_rgw_get_initial_lat_count[$rate_interval]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\")",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{rgw_host}}",
@@ -333,14 +333,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(ceph_rgw_get_b[1m]))",
+          "expr": "sum(rate(ceph_rgw_get_b[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "GETs",
           "refId": "A"
         },
         {
-          "expr": "sum(rate(ceph_rgw_put_b[1m]))",
+          "expr": "sum(rate(ceph_rgw_put_b[$rate_interval]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "PUTs",
@@ -425,7 +425,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum by(rgw_host) (\n  (label_replace(rate(ceph_rgw_get_b[1m]), \"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\")) + \n  (label_replace(rate(ceph_rgw_put_b[1m]), \"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\"))\n)",
+          "expr": "sum by(rgw_host) (\n  (label_replace(rate(ceph_rgw_get_b[$rate_interval]), \"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\")) + \n  (label_replace(rate(ceph_rgw_put_b[$rate_interval]), \"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\"))\n)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{rgw_host}}",
@@ -510,7 +510,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "label_replace(rate(ceph_rgw_put_initial_lat_sum[1m]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\") / \nlabel_replace(rate(ceph_rgw_put_initial_lat_count[1m]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\")",
+          "expr": "label_replace(rate(ceph_rgw_put_initial_lat_sum[$rate_interval]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\") / \nlabel_replace(rate(ceph_rgw_put_initial_lat_count[$rate_interval]),\"rgw_host\",\"$1\",\"ceph_daemon\",\"rgw.(.*)\")",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{rgw_host}}",
@@ -591,6 +591,48 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "rate_interval",
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "skipUrlSync": false,
+        "type": "custom"
       }
     ]
   },
diff --git a/ceph/files/luminous/ceph.conf.Debian b/ceph/files/luminous/ceph.conf.Debian
index 43cb2f7..725109f 100644
--- a/ceph/files/luminous/ceph.conf.Debian
+++ b/ceph/files/luminous/ceph.conf.Debian
@@ -89,9 +89,17 @@
 
 {%- if pillar.ceph.radosgw is defined %}
 
+{%- if radosgw.keyring_user is defined %}
+[client.{{ radosgw.keyring_user }}]
+{%- else %}
 [client.rgw.{{ grains.host }}]
+{%- endif %}
 host = {{ grains.host }}
+{%- if radosgw.keyring_path is defined %}
+keyring = {{ radosgw.keyring_path }}
+{%- else %}
 keyring = /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.rgw.{{ grains.host }}.keyring
+{%- endif %}
 rgw socket path = /tmp/radosgw-{{ grains.host }}.sock
 log file = /var/log/ceph/{{ common.get('cluster_name', 'ceph') }}-rgw-{{ grains.host }}.log
 rgw data = /var/lib/ceph/radosgw/{{ common.get('cluster_name', 'ceph') }}-rgw.{{ grains.host }}
diff --git a/ceph/map.jinja b/ceph/map.jinja
index 9533c09..1833052 100644
--- a/ceph/map.jinja
+++ b/ceph/map.jinja
@@ -46,7 +46,11 @@
   - librados2
   services:
   {%- if grains.get('init', None) == 'systemd' %}
+    {%- if pillar.get('ceph',{}).get('radosgw',{}).keyring_user is defined %}
+  - ceph-radosgw@{{ pillar.ceph.radosgw.keyring_user }}
+    {%- else %}
   - ceph-radosgw@rgw.{{ grains.host }}
+    {%- endif %}
   {%- else %}
   - radosgw-all
   {%- endif %}
diff --git a/ceph/meta/logrotate.yml b/ceph/meta/logrotate.yml
new file mode 100644
index 0000000..8f2730c
--- /dev/null
+++ b/ceph/meta/logrotate.yml
@@ -0,0 +1,19 @@
+## Default: Daily rotation with 28 rotations kept
+{%- from "ceph/map.jinja" import common, mgr, mon, osd, radosgw with context %}
+
+{%- if mgr.get('enabled', False) or mon.get('enabled', False) or osd.get('enabled', False) or radosgw.get('enabled', False) %}
+job:
+  ceph-common:
+    - files:
+        - /var/log/ceph/*.log
+      options:
+        - {{ common.get('logrotate', {}).get('interval', 'daily') }}
+        - rotate: {{ common.get('logrotate', {}).get('rotate', 28) }}
+        - compress
+        - sharedscripts
+        - missingok
+        - notifempty
+        - su root ceph
+        - postrotate:
+              killall -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw || true
+{%- endif %}
diff --git a/ceph/osd/init.sls b/ceph/osd/init.sls
index 0813222..881c905 100644
--- a/ceph/osd/init.sls
+++ b/ceph/osd/init.sls
@@ -2,3 +2,4 @@
 - ceph.common
 - ceph.setup.keyring
 - ceph.osd.setup
+- ceph.osd.custom
diff --git a/ceph/osd/setup.sls b/ceph/osd/setup.sls
index 5b1a832..e028575 100644
--- a/ceph/osd/setup.sls
+++ b/ceph/osd/setup.sls
@@ -1,236 +1,11 @@
 {%- from "ceph/map.jinja" import osd, common with context %}
 
-ceph_osd_packages:
-  pkg.installed:
-  - names: {{ osd.pkgs }}
-
-/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf:
-  file.managed:
-  - source: salt://ceph/files/{{ common.version }}/ceph.conf.{{ grains.os_family }}
-  - template: jinja
-  - require:
-    - pkg: ceph_osd_packages
-
-{% set ceph_version = pillar.ceph.common.version %}
-
-{%- if osd.backend is defined %}
-
-{%- for backend_name, backend in osd.backend.iteritems() %}
-
-{%- for disk in backend.disks %}
-
-{%- if disk.get('enabled', True) %}
-
-{% set dev = disk.dev %}
-
-# for uniqueness
-{% set dev_device = dev + disk.get('data_partition_prefix', '') + disk.get('data_partition', 1)|string %}
-
-#{{ dev }}{{ disk.get('data_partition', 1) }}
-
-zap_disk_{{ dev_device }}:
-  cmd.run:
-  - name: "ceph-disk zap {{ dev }}"
-  - unless: "ceph-disk list | grep {{ dev }} | grep -e 'ceph' -e 'mounted'"
-  - require:
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
-
-{%- if disk.journal is defined %}
-
-zap_disk_journal_{{ disk.journal }}_for_{{ dev_device }}:
-  cmd.run:
-  - name: "ceph-disk zap {{ disk.journal }}"
-  - unless: "ceph-disk list | grep {{ disk.journal }} | grep -e 'ceph' -e 'mounted'"
-  - require:
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-    - cmd: zap_disk_{{ dev_device }}
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
-
+include:
+{%- if osd.zap_disks is defined and osd.zap_disks == true %}
+- ceph.osd.setup.partitioning
 {%- endif %}
-
-{%- if disk.block_db is defined %}
-
-zap_disk_blockdb_{{ disk.block_db }}_for_{{ dev_device }}:
-  cmd.run:
-  - name: "ceph-disk zap {{ disk.block_db }}"
-  - unless: "ceph-disk list | grep {{ disk.block_db }} | grep -e 'ceph' -e 'mounted'"
-  - require:
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-    - cmd: zap_disk_{{ dev_device }}
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
-
-{%- endif %}
-
-{%- if disk.block_wal is defined %}
-
-zap_disk_blockwal_{{ disk.block_wal }}_for_{{ dev_device }}:
-  cmd.run:
-  - name: "ceph-disk zap {{ disk.block_wal }}"
-  - unless: "ceph-disk list | grep {{ disk.block_wal }} | grep -e 'ceph' -e 'mounted'"
-  - require:
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-    - cmd: zap_disk_{{ dev_device }}
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
-
-{%- endif %}
-
-{%- set cmd = [] %}
-{%- do cmd.append('--cluster ' + common.get('cluster_name', 'ceph')) %}
-{%- do cmd.append('--cluster-uuid ' + common.fsid) %}
-{%- if disk.get('dmcrypt', False) %}
-  {%- do cmd.append('--dmcrypt') %}
-  {%- do cmd.append('--dmcrypt-key-dir ' + disk.get('dmcrypt_key_dir', '/etc/ceph/dmcrypt-keys')) %}
-{%- endif %}
-{%- if disk.lockbox_partition is defined %}
-  {%- do cmd.append('--lockbox-partition-number ' + disk.lockbox_partition|string) %}
-{%- endif %}
-{%- do cmd.append("--prepare-key /etc/ceph/" + common.get('cluster_name', 'ceph') + ".client.bootstrap-osd.keyring") %}
-{%- if disk.data_partition is defined %}
-  {%- do cmd.append('--data-partition-number ' + disk.data_partition|string) %}
-{%- endif %}
-{%- if disk.data_partition_size is defined %}
-  {%- do cmd.append('--data-partition-size ' + disk.data_partition_size|string) %}
-{%- endif %}
-{%- if backend_name == 'bluestore' %}
-  {%- do cmd.append('--bluestore') %}
-  {%- if disk.block_partition is defined %}
-    {%- do cmd.append('--block-partition-number ' + disk.block_partition|string) %}
-  {%- endif %}
-  {%- if disk.block_db is defined %}
-    {%- if disk.block_db_dmcrypt is defined and not disk.block_db_dmcrypt %}
-      {%- do cmd.append('--block-db-non-dmcrypt') %}
-    {%- elif disk.get('block_db_dmcrypt', False) %}
-      {%- do cmd.append('--block-db-dmcrypt') %}
-    {%- endif %}
-    {%- if disk.block_db_partition is defined %}
-      {%- do cmd.append('--block-db-partition-number ' + disk.block_db_partition|string) %}
-    {%- endif %}
-  {%- do cmd.append('--block.db ' + disk.block_db) %}
-  {%- endif %}
-  {%- if disk.block_wal is defined %}
-    {%- if disk.block_wal_dmcrypt is defined and not disk.block_wal_dmcrypt %}
-      {%- do cmd.append('--block-wal-non-dmcrypt') %}
-    {%- elif disk.get('block_wal_dmcrypt', False) %}
-      {%- do cmd.append('--block-wal-dmcrypt') %}
-    {%- endif %}
-    {%- if disk.block_wal_partition is defined %}
-      {%- do cmd.append('--block-wal-partition-number ' + disk.block_wal_partition|string) %}
-    {%- endif %}
-    {%- do cmd.append('--block.wal ' + disk.block_wal) %}
-  {%- endif %}
-  {%- do cmd.append(dev) %}
-{%- elif backend_name == 'filestore' and ceph_version not in ['kraken', 'jewel'] %}
-  {%- if disk.journal_dmcrypt is defined and not disk.journal_dmcrypt %}
-    {%- do cmd.append('--journal-non-dmcrypt') %}
-  {%- elif disk.get('journal_dmcrypt', False) %}
-    {%- do cmd.append('--journal-dmcrypt') %}
-  {%- endif %}
-  {%- if disk.journal_partition is defined %}
-    {%- do cmd.append('--journal-partition-number ' + disk.journal_partition|string) %}
-  {%- endif %}
-  {%- do cmd.append('--filestore') %}
-  {%- do cmd.append(dev) %}
-  {%- if disk.journal is defined %}
-    {%- do cmd.append(disk.journal) %}
-  {%- endif %}
-{%- elif backend_name == 'filestore' %}
-  {%- if disk.journal_dmcrypt is defined and not disk.journal_dmcrypt %}
-    {%- do cmd.append('--journal-non-dmcrypt') %}
-  {%- elif disk.get('journal_dmcrypt', False) %}
-    {%- do cmd.append('--journal-dmcrypt') %}
-  {%- endif %}
-  {%- if disk.journal_partition is defined %}
-    {%- do cmd.append('--journal-partition-number ' + disk.journal_partition|string) %}
-  {%- endif %}
-  {%- do cmd.append(dev) %}
-  {%- if disk.journal is defined %}
-    {%- do cmd.append(disk.journal) %}
-  {%- endif %}
-{%- endif %}
-
-prepare_disk_{{ dev_device }}:
-  cmd.run:
-  - name: "yes | ceph-disk prepare {{ cmd|join(' ') }}"
-  - unless: "ceph-disk list | grep {{ dev_device }} | grep -e 'ceph' -e 'mounted'"
-  - require:
-    - cmd: zap_disk_{{ dev_device }}
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
-
-reload_partition_table_{{ dev_device }}:
-  cmd.run:
-  - name: "partprobe"
-  - unless: "lsblk -p | grep {{ dev_device }} -A1 | grep -v lockbox | grep {{ dev_device }} | grep ceph | grep osd"
-  - require:
-    - cmd: prepare_disk_{{ dev_device }}
-    - cmd: zap_disk_{{ dev_device }}
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- else %}
-  - onlyif: ceph-disk list | grep {{ dev_device }} | grep ceph
-  {%- endif %}
-
-activate_disk_{{ dev_device }}:
-  cmd.run:
-{%- if disk.get('dmcrypt', False) %}
-  - name: "ceph-disk activate --dmcrypt --activate-key /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.bootstrap-osd.keyring {{ dev_device }}"
+{%- if osd.lvm_enabled is defined and osd.lvm_enabled == true %}
+- ceph.osd.setup.lvm
 {%- else %}
-  - name: "ceph-disk activate --activate-key /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.bootstrap-osd.keyring {{ dev_device }}"
+- ceph.osd.setup.disk
 {%- endif %}
-  - unless: "lsblk -p | grep {{ dev_device }} -A1 | grep -v lockbox | grep {{ dev_device }} | grep ceph | grep osd"
-  - require:
-    - cmd: prepare_disk_{{ dev_device }}
-    - cmd: zap_disk_{{ dev_device }}
-    - pkg: ceph_osd_packages
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- else %}
-  - onlyif: ceph-disk list | grep {{ dev_device }} | grep ceph
-  {%- endif %}
-
-{%- endif %}
-
-{%- endfor %}
-
-{%- endfor %}
-
-{%- endif %}
-
-osd_services_global:
-  service.running:
-  - enable: true
-  - names: ['ceph-osd.target']
-  - watch:
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
-
-osd_services:
-  service.running:
-  - enable: true
-  - names: ['ceph.target']
-  - watch:
-    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
-  {%- if grains.get('noservices') %}
-  - onlyif: /bin/false
-  {%- endif %}
diff --git a/ceph/osd/setup/disk.sls b/ceph/osd/setup/disk.sls
new file mode 100644
index 0000000..5b1a832
--- /dev/null
+++ b/ceph/osd/setup/disk.sls
@@ -0,0 +1,236 @@
+{%- from "ceph/map.jinja" import osd, common with context %}
+
+ceph_osd_packages:
+  pkg.installed:
+  - names: {{ osd.pkgs }}
+
+/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf:
+  file.managed:
+  - source: salt://ceph/files/{{ common.version }}/ceph.conf.{{ grains.os_family }}
+  - template: jinja
+  - require:
+    - pkg: ceph_osd_packages
+
+{% set ceph_version = pillar.ceph.common.version %}
+
+{%- if osd.backend is defined %}
+
+{%- for backend_name, backend in osd.backend.iteritems() %}
+
+{%- for disk in backend.disks %}
+
+{%- if disk.get('enabled', True) %}
+
+{% set dev = disk.dev %}
+
+# for uniqueness
+{% set dev_device = dev + disk.get('data_partition_prefix', '') + disk.get('data_partition', 1)|string %}
+
+#{{ dev }}{{ disk.get('data_partition', 1) }}
+
+zap_disk_{{ dev_device }}:
+  cmd.run:
+  - name: "ceph-disk zap {{ dev }}"
+  - unless: "ceph-disk list | grep {{ dev }} | grep -e 'ceph' -e 'mounted'"
+  - require:
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+
+{%- if disk.journal is defined %}
+
+zap_disk_journal_{{ disk.journal }}_for_{{ dev_device }}:
+  cmd.run:
+  - name: "ceph-disk zap {{ disk.journal }}"
+  - unless: "ceph-disk list | grep {{ disk.journal }} | grep -e 'ceph' -e 'mounted'"
+  - require:
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+    - cmd: zap_disk_{{ dev_device }}
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+
+{%- endif %}
+
+{%- if disk.block_db is defined %}
+
+zap_disk_blockdb_{{ disk.block_db }}_for_{{ dev_device }}:
+  cmd.run:
+  - name: "ceph-disk zap {{ disk.block_db }}"
+  - unless: "ceph-disk list | grep {{ disk.block_db }} | grep -e 'ceph' -e 'mounted'"
+  - require:
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+    - cmd: zap_disk_{{ dev_device }}
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+
+{%- endif %}
+
+{%- if disk.block_wal is defined %}
+
+zap_disk_blockwal_{{ disk.block_wal }}_for_{{ dev_device }}:
+  cmd.run:
+  - name: "ceph-disk zap {{ disk.block_wal }}"
+  - unless: "ceph-disk list | grep {{ disk.block_wal }} | grep -e 'ceph' -e 'mounted'"
+  - require:
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+    - cmd: zap_disk_{{ dev_device }}
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+
+{%- endif %}
+
+{%- set cmd = [] %}
+{%- do cmd.append('--cluster ' + common.get('cluster_name', 'ceph')) %}
+{%- do cmd.append('--cluster-uuid ' + common.fsid) %}
+{%- if disk.get('dmcrypt', False) %}
+  {%- do cmd.append('--dmcrypt') %}
+  {%- do cmd.append('--dmcrypt-key-dir ' + disk.get('dmcrypt_key_dir', '/etc/ceph/dmcrypt-keys')) %}
+{%- endif %}
+{%- if disk.lockbox_partition is defined %}
+  {%- do cmd.append('--lockbox-partition-number ' + disk.lockbox_partition|string) %}
+{%- endif %}
+{%- do cmd.append("--prepare-key /etc/ceph/" + common.get('cluster_name', 'ceph') + ".client.bootstrap-osd.keyring") %}
+{%- if disk.data_partition is defined %}
+  {%- do cmd.append('--data-partition-number ' + disk.data_partition|string) %}
+{%- endif %}
+{%- if disk.data_partition_size is defined %}
+  {%- do cmd.append('--data-partition-size ' + disk.data_partition_size|string) %}
+{%- endif %}
+{%- if backend_name == 'bluestore' %}
+  {%- do cmd.append('--bluestore') %}
+  {%- if disk.block_partition is defined %}
+    {%- do cmd.append('--block-partition-number ' + disk.block_partition|string) %}
+  {%- endif %}
+  {%- if disk.block_db is defined %}
+    {%- if disk.block_db_dmcrypt is defined and not disk.block_db_dmcrypt %}
+      {%- do cmd.append('--block-db-non-dmcrypt') %}
+    {%- elif disk.get('block_db_dmcrypt', False) %}
+      {%- do cmd.append('--block-db-dmcrypt') %}
+    {%- endif %}
+    {%- if disk.block_db_partition is defined %}
+      {%- do cmd.append('--block-db-partition-number ' + disk.block_db_partition|string) %}
+    {%- endif %}
+  {%- do cmd.append('--block.db ' + disk.block_db) %}
+  {%- endif %}
+  {%- if disk.block_wal is defined %}
+    {%- if disk.block_wal_dmcrypt is defined and not disk.block_wal_dmcrypt %}
+      {%- do cmd.append('--block-wal-non-dmcrypt') %}
+    {%- elif disk.get('block_wal_dmcrypt', False) %}
+      {%- do cmd.append('--block-wal-dmcrypt') %}
+    {%- endif %}
+    {%- if disk.block_wal_partition is defined %}
+      {%- do cmd.append('--block-wal-partition-number ' + disk.block_wal_partition|string) %}
+    {%- endif %}
+    {%- do cmd.append('--block.wal ' + disk.block_wal) %}
+  {%- endif %}
+  {%- do cmd.append(dev) %}
+{%- elif backend_name == 'filestore' and ceph_version not in ['kraken', 'jewel'] %}
+  {%- if disk.journal_dmcrypt is defined and not disk.journal_dmcrypt %}
+    {%- do cmd.append('--journal-non-dmcrypt') %}
+  {%- elif disk.get('journal_dmcrypt', False) %}
+    {%- do cmd.append('--journal-dmcrypt') %}
+  {%- endif %}
+  {%- if disk.journal_partition is defined %}
+    {%- do cmd.append('--journal-partition-number ' + disk.journal_partition|string) %}
+  {%- endif %}
+  {%- do cmd.append('--filestore') %}
+  {%- do cmd.append(dev) %}
+  {%- if disk.journal is defined %}
+    {%- do cmd.append(disk.journal) %}
+  {%- endif %}
+{%- elif backend_name == 'filestore' %}
+  {%- if disk.journal_dmcrypt is defined and not disk.journal_dmcrypt %}
+    {%- do cmd.append('--journal-non-dmcrypt') %}
+  {%- elif disk.get('journal_dmcrypt', False) %}
+    {%- do cmd.append('--journal-dmcrypt') %}
+  {%- endif %}
+  {%- if disk.journal_partition is defined %}
+    {%- do cmd.append('--journal-partition-number ' + disk.journal_partition|string) %}
+  {%- endif %}
+  {%- do cmd.append(dev) %}
+  {%- if disk.journal is defined %}
+    {%- do cmd.append(disk.journal) %}
+  {%- endif %}
+{%- endif %}
+
+prepare_disk_{{ dev_device }}:
+  cmd.run:
+  - name: "yes | ceph-disk prepare {{ cmd|join(' ') }}"
+  - unless: "ceph-disk list | grep {{ dev_device }} | grep -e 'ceph' -e 'mounted'"
+  - require:
+    - cmd: zap_disk_{{ dev_device }}
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+
+reload_partition_table_{{ dev_device }}:
+  cmd.run:
+  - name: "partprobe"
+  - unless: "lsblk -p | grep {{ dev_device }} -A1 | grep -v lockbox | grep {{ dev_device }} | grep ceph | grep osd"
+  - require:
+    - cmd: prepare_disk_{{ dev_device }}
+    - cmd: zap_disk_{{ dev_device }}
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- else %}
+  - onlyif: ceph-disk list | grep {{ dev_device }} | grep ceph
+  {%- endif %}
+
+activate_disk_{{ dev_device }}:
+  cmd.run:
+{%- if disk.get('dmcrypt', False) %}
+  - name: "ceph-disk activate --dmcrypt --activate-key /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.bootstrap-osd.keyring {{ dev_device }}"
+{%- else %}
+  - name: "ceph-disk activate --activate-key /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.bootstrap-osd.keyring {{ dev_device }}"
+{%- endif %}
+  - unless: "lsblk -p | grep {{ dev_device }} -A1 | grep -v lockbox | grep {{ dev_device }} | grep ceph | grep osd"
+  - require:
+    - cmd: prepare_disk_{{ dev_device }}
+    - cmd: zap_disk_{{ dev_device }}
+    - pkg: ceph_osd_packages
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- else %}
+  - onlyif: ceph-disk list | grep {{ dev_device }} | grep ceph
+  {%- endif %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endfor %}
+
+{%- endif %}
+
+osd_services_global:
+  service.running:
+  - enable: true
+  - names: ['ceph-osd.target']
+  - watch:
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+
+osd_services:
+  service.running:
+  - enable: true
+  - names: ['ceph.target']
+  - watch:
+    - file: /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
diff --git a/ceph/osd/setup/partitioning.sls b/ceph/osd/setup/partitioning.sls
new file mode 100644
index 0000000..16ced44
--- /dev/null
+++ b/ceph/osd/setup/partitioning.sls
@@ -0,0 +1,80 @@
+{%- from "ceph/map.jinja" import osd, common with context %}
+
+{%- set devs = [] %}
+{%- set dbs = [] %}
+{%- set wals = [] %}
+{%- for backend_name, backend in osd.backend.iteritems() %}
+{%- for disk in backend.disks %}
+{%- set dev = disk.dev %}
+
+{%- if disk.block_db is defined %}
+{%- set db = disk.block_db %}
+{%- do dbs.append(db) %}
+{%- endif %}
+{%- if disk.block_wal is defined %}
+{%- set wal = disk.block_wal %}
+{%- do wals.append(wal) %}
+{%- endif %}
+{%- do devs.append(dev) %}
+{%- endfor %}
+{%- endfor %}
+{%- set end_size = {} %}
+{%- set counter = {} %}
+
+{%- if dbs != [] %}
+{%- for db in dbs | unique %}
+{%- do end_size.update({db: 1048576}) %}
+{%- do counter.update({db: 1}) %}
+create_disk_label_{{ db }}:
+  module.run:
+  - name: partition.mklabel
+  - device: {{ db }}
+  - label_type: gpt
+  - unless: "fdisk -l {{ db }} | grep -i 'Disklabel type: gpt'"
+{%- endfor %}
+{%- for db in dbs %}
+create_partition_{{ db }}_{{ counter[db] }}:
+  module.run:
+  - name: partition.mkpart
+  - device: {{ db }}
+  - part_type: primary
+  - start: {{ end_size[db] }}B
+  - end: {{ end_size[db] + osd.bluestore_block_db_size }}B
+  - size: {{ osd.bluestore_block_db_size }}B
+  - unless: "blkid {{ db }}{{ counter[db] }} {{ db }}p{{ counter[db] }}"
+  - require:
+    - module: create_disk_label_{{ db }}
+
+{%- do counter.update({db: counter[db] + 1}) %}
+{%- do end_size.update({db: end_size[db] + osd.bluestore_block_db_size + 1048576}) %}
+{%- endfor %}
+{%- endif %}
+
+{%- if wals != [] %}
+{%- for wal in wals | unique %}
+{%- do end_size.update({wal: 1048576}) %}
+{%- do counter.update({wal: 1}) %}
+create_disk_label_{{ wal }}:
+  module.run:
+  - name: partition.mklabel
+  - device: {{ wal }}
+  - label_type: gpt
+  - unless: "fdisk -l {{ wal }} | grep -i 'Disklabel type: gpt'"
+{%- endfor %}
+{%- for wal in wals %}
+create_partition_{{ wal }}_{{ counter[wal] }}:
+  module.run:
+  - name: partition.mkpart
+  - device: {{ wal }}
+  - part_type: primary
+  - start: {{ end_size[wal] }}B
+  - end: {{ end_size[wal] + osd.bluestore_block_db_size }}B
+  - size: {{ osd.bluestore_block_wal_size }}B
+  - unless: "blkid {{ wal }}{{ counter[wal] }} {{ wal }}p{{ counter[wal] }}"
+  - require:
+    - module: create_disk_label_{{ wal }}
+
+{%- do counter.update({wal: counter[wal] + 1}) %}
+{%- do end_size.update({wal: end_size[wal] + osd.bluestore_block_wal_size + 1048576}) %}
+{%- endfor %}
+{% endif %}
diff --git a/ceph/setup/keyring.sls b/ceph/setup/keyring.sls
index f26c608..d51a109 100644
--- a/ceph/setup/keyring.sls
+++ b/ceph/setup/keyring.sls
@@ -2,18 +2,18 @@
 
 {% if not common.get('container_mode', False) %}
 
-{# run only if ceph cluster is present #}
-{%- for node_name, node_grains in salt['mine.get']('ceph:common:keyring:admin', 'grains.items', 'pillar').iteritems() %}
+  {# run only if ceph cluster is present #}
+  {%- for node_name, node_grains in salt['mine.get']('ceph:common:keyring:admin', 'grains.items', 'pillar').iteritems() %}
 
-{%- if node_grains.ceph is defined and node_grains.ceph.ceph_keyring is defined and node_grains.ceph.ceph_keyring.admin is defined and node_grains.ceph.get('fsid', '') == common.fsid %}
+    {%- if node_grains.ceph is defined and node_grains.ceph.ceph_keyring is defined and node_grains.ceph.ceph_keyring.admin is defined and node_grains.ceph.get('fsid', '') == common.fsid %}
 
-{%- if loop.index0 == 0 %}
+      {%- if loop.index0 == 0 %}
 
-{% for keyring_name, keyring in common.get('keyring', {}).iteritems() %}
+        {% for keyring_name, keyring in common.get('keyring', {}).iteritems() %}
 
-{%- if keyring.name is defined %}
+          {%- if keyring.name is defined %}
 
-{%- if keyring.name != 'admin' and keyring.key is defined and common.get("manage_keyring", False) %}
+            {%- if keyring.name != 'admin' and keyring.key is defined and common.get("manage_keyring", False) %}
 
 {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring:
   file.managed:
@@ -25,22 +25,43 @@
 
 ceph_import_keyring_{{ keyring.name }}:
   cmd.run:
-  - name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth import -i {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring"
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth import -i {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring
   - onchanges:
     - file: {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring
 
-{%- elif keyring.name != 'admin' %}
+ceph_update_caps_for_{{ keyring.name }}:
+  cmd.run:
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth caps client.{{ keyring.name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %}
+  - onchanges:
+    - file: {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring
+
+            {%- elif keyring.name != 'admin' %}
 
 ceph_create_keyring_{{ keyring.name }}:
   cmd.run:
-  - name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth get-or-create client.{{ keyring.name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %} > {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring"
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth get-or-create client.{{ keyring.name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %} > {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring
   - unless: "test -f {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring.name }}.keyring"
 
-{%- endif %}
+              {%- if salt['file.file_exists']('/usr/bin/ceph') %}
+                {%- set caps = salt['cmd.shell']('ceph auth list --format json') | load_json %}
+                {%- for client in caps['auth_dump'] %}
+                  {%- if client['entity'] == "client." + keyring.name %}
+                    {%- for cap_name, cap in  client.caps.iteritems() %}
+                      {%- if cap != keyring.caps[cap_name] %}
+ceph_update_caps_for_{{ keyring.name }}:
+  cmd.run:
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth caps client.{{ keyring.name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %}
+                      {%- endif %}
+                    {%- endfor %}
+                  {%- endif %}
+                {%- endfor %}
+              {%- endif %}
 
-{%- else %}
+            {%- endif %}
 
-{%- if keyring_name != 'admin' and keyring.key is defined and common.get("manage_keyring", False) %}
+          {%- else %}
+
+            {%- if keyring_name != 'admin' and keyring.key is defined and common.get("manage_keyring", False) %}
 
 {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring_name }}.keyring:
   file.managed:
@@ -56,23 +77,44 @@
   - onchanges:
     - file: {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring_name }}.keyring
 
-{%- elif keyring_name != 'admin' %}
+ceph_update_caps_for_{{ keyring_name }}:
+  cmd.run:
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth caps client.{{ keyring_name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %}
+  - onchanges:
+    - file: {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring_name }}.keyring
+
+            {%- elif keyring_name != 'admin' %}
 
 ceph_create_keyring_{{ keyring_name }}:
   cmd.run:
-  - name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth get-or-create client.{{ keyring_name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %} > {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring_name }}.keyring"
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth get-or-create client.{{ keyring_name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %} > {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring_name }}.keyring
   - unless: "test -f {{ common.prefix_dir }}/etc/ceph/{{ common.get('cluster_name', 'ceph') }}.client.{{ keyring_name }}.keyring"
 
-{%- endif %}
+              {%- if salt['file.file_exists']('/usr/bin/ceph') %}
+                {%- set caps = salt['cmd.shell']('ceph auth list --format json') | load_json %}
+                {%- for client in caps['auth_dump'] %}
+                  {%- if client['entity'] == "client." + keyring_name %}
+                    {%- for cap_name, cap in  client.caps.iteritems() %}
+                      {%- if cap != keyring.caps[cap_name] %}
+ceph_update_caps_for_{{ keyring_name }}:
+  cmd.run:
+  - name: ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf auth caps client.{{ keyring_name }} {%- for cap_name, cap in  keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %}
+                      {%- endif %}
+                    {%- endfor %}
+                  {%- endif %}
+                {%- endfor %}
+              {%- endif %}
 
-{%- endif %}
+            {%- endif %}
 
-{% endfor %}
+          {%- endif %}
 
-{%- endif %}
+        {% endfor %}
 
-{%- endif %}
+      {%- endif %}
 
-{%- endfor %}
+    {%- endif %}
+
+  {%- endfor %}
 
 {%- endif %}
diff --git a/metadata/service/common/cluster.yml b/metadata/service/common/cluster.yml
index 6974fe6..09ca6c2 100644
--- a/metadata/service/common/cluster.yml
+++ b/metadata/service/common/cluster.yml
@@ -7,6 +7,9 @@
     ceph_mon_node03_hostname: cmn03
   ceph:
     common:
+      config:
+        global:
+          mon_max_pg_per_osd: 600
       enabled: true
       version: ${_param:ceph_version}
       fsid: ${_param:ceph_cluster_fsid}
@@ -17,4 +20,3 @@
           host: ${_param:ceph_mon_node02_address}
         - name: ${_param:ceph_mon_node03_hostname}
           host: ${_param:ceph_mon_node03_address}
-
diff --git a/metadata/service/common/single.yml b/metadata/service/common/single.yml
index 7aec939..c81947b 100644
--- a/metadata/service/common/single.yml
+++ b/metadata/service/common/single.yml
@@ -5,6 +5,9 @@
     ceph_mon_node01_hostname: cmn01
   ceph:
     common:
+      config:
+        global:
+          mon_max_pg_per_osd: 600
       enabled: true
       version: ${_param:ceph_version}
       fsid: ${_param:ceph_cluster_fsid}
diff --git a/metadata/service/mon/cluster.yml b/metadata/service/mon/cluster.yml
index 94c484b..6a10da2 100644
--- a/metadata/service/mon/cluster.yml
+++ b/metadata/service/mon/cluster.yml
@@ -7,7 +7,3 @@
   ceph:
     mon:
       enabled: true
-    common:
-      config:
-        mon:
-          mon_max_pg_per_osd: 600
diff --git a/metadata/service/mon/single.yml b/metadata/service/mon/single.yml
index ea04ea0..8be5da5 100644
--- a/metadata/service/mon/single.yml
+++ b/metadata/service/mon/single.yml
@@ -12,9 +12,6 @@
           caps:
             mon: "allow *"
     common:
-      config:
-        mon:
-          mon_max_pg_per_osd: 600
       keyring:
         admin:
           caps:
diff --git a/tests/pillar/ceph_mon_single.sls b/tests/pillar/ceph_mon_single.sls
index 05ed121..5ad74c9 100644
--- a/tests/pillar/ceph_mon_single.sls
+++ b/tests/pillar/ceph_mon_single.sls
@@ -1,6 +1,6 @@
 ceph:
   common:
-    version: kraken
+    version: luminous
     cluster_name: ceph
     config:
       global:
@@ -27,7 +27,7 @@
           osd: "allow *"
   mon:
     enabled: true
-    version: kraken
+    version: luminous
     keyring:
       mon:
         key: AQAnQIhZ6in5KxAAdf467upoRMWFcVg5pbh1yg==
@@ -40,3 +40,24 @@
           mgr: "allow *"
           mon: "allow *"
           osd: "allow *"
+  radosgw:
+    enabled: true
+    hostname: gw.ceph.lab
+    keyring_user: radosgw.gateway
+    keyring_path: /etc/ceph/keyring.radosgw.gateway
+    bind:
+      address: 10.10.10.1
+      port: 8080
+    identity:
+      engine: keystone
+      api_version: 3
+      host: 10.10.10.100
+      port: 5000
+      user: admin
+      password: password
+      project: admin
+      domain: default
+    swift:
+      versioning:
+        enabled: true
+      enforce_content_length: true