diff --git a/README.rst b/README.rst
index b93b061..42ba8db 100644
--- a/README.rst
+++ b/README.rst
@@ -29,6 +29,7 @@
         cross_az_attach: false
         workers: 8
         report_interval: 60
+        dhcp_domain: novalocal
         bind:
           public_address: 10.0.0.122
           public_name: openstack.domain.com
@@ -1063,23 +1064,84 @@
 ---------------------
 By default communication between Nova and Galera is unsecure.
 
+nova:
+  controller:
+    database:
+      x509:
+        enabled: True
+
 You able to set custom certificates in pillar:
-controller:
-  database:
-    x509:
-      enabled: True
 
 nova:
   controller:
     database:
       x509:
-        cacert (certificate content)
-        cert (certificate content)
-        key (certificate content)
+        cacert: (certificate content)
+        cert: (certificate content)
+        key: (certificate content)
 
 You can read more about it here:
     https://docs.openstack.org/security-guide/databases/database-access-control.html
 
+Upgrades
+========
+
+Each openstack formula provide set of phases (logical bloks) that will help to
+build flexible upgrade orchestration logic for particular components. The list
+of phases might and theirs descriptions are listed in table below:
+
++-------------------------------+------------------------------------------------------+
+| State                         | Description                                          |
++===============================+======================================================+
+| <app>.upgrade.service_running | Ensure that all services for particular application  |
+|                               | are enabled for autostart and running                |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.service_stopped | Ensure that all services for particular application  |
+|                               | disabled for autostart and dead                      |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.pkgs_latest     | Ensure that packages used by particular application  |
+|                               | are installed to latest available version.           |
+|                               | This will not upgrade data plane packages like qemu  |
+|                               | and openvswitch as usually minimal required version  |
+|                               | in openstack services is really old. The data plane  |
+|                               | packages should be upgraded separately by `apt-get   |
+|                               | upgrade` or `apt-get dist-upgrade`                   |
+|                               | Applying this state will not autostart service.      |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.render_config   | Ensure configuration is rendered actual version.     +
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.pre             | We assume this state is applied on all nodes in the  |
+|                               | cloud before running upgrade.                        |
+|                               | Only non destructive actions will be applied during  |
+|                               | this phase. Perform service built in service check   |
+|                               | like (keystone-manage doctor and nova-status upgrade)|
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.upgrade.pre     | Mostly applicable for data plane nodes. During this  |
+|                               | phase resources will be gracefully removed from      |
+|                               | current node if it is allowed. Services for upgraded |
+|                               | application will be set to admin disabled state to   |
+|                               | make sure node will not participate in resources     |
+|                               | scheduling. For example on gtw nodes this will set   |
+|                               | all agents to admin disable state and will move all  |
+|                               | routers to other agents.                             |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.upgrade         | This state will basically upgrade application on     |
+|                               | particular target. Stop services, render             |
+|                               | configuration, install new packages, run offline     |
+|                               | dbsync (for ctl), start services. Data plane should  |
+|                               | not be affected, only OpenStack python services.     |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.upgrade.post    | Add services back to scheduling.                     |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.post            | This phase should be launched only when upgrade of   |
+|                               | the cloud is completed. Cleanup temporary files,     |
+|                               | perform other post upgrade tasks.                    |
++-------------------------------+------------------------------------------------------+
+| <app>.upgrade.verify          | Here we will do basic health checks (API CRUD        |
+|                               | operations, verify do not have dead network          |
+|                               | agents/compute services)                             |
++-------------------------------+------------------------------------------------------+
+
 Documentation and Bugs
 ======================
 
diff --git a/nova/_ssl/mysql.sls b/nova/_ssl/mysql.sls
index 3f44e2a..3d715e9 100644
--- a/nova/_ssl/mysql.sls
+++ b/nova/_ssl/mysql.sls
@@ -1,10 +1,14 @@
 {% from "nova/map.jinja" import controller with context %}
 
+nova_ssl_mysql:
+  test.show_notification:
+    - text: "Running nova._ssl.mysql"
+
 {%- if controller.database.get('x509',{}).get('enabled',False) %}
 
-  {%- set ca_file=controller.database.x509.get('ca_file') %}
-  {%- set key_file=controller.database.x509.get('key_file') %}
-  {%- set cert_file=controller.database.x509.get('cert_file') %}
+  {%- set ca_file=controller.database.x509.ca_file %}
+  {%- set key_file=controller.database.x509.key_file %}
+  {%- set cert_file=controller.database.x509.cert_file %}
 
 mysql_nova_ssl_x509_ca:
   {%- if controller.database.x509.cacert is defined %}
@@ -12,6 +16,8 @@
     - name: {{ ca_file }}
     - contents_pillar: nova:controller:database:x509:cacert
     - mode: 444
+    - user: nova
+    - group: nova
     - makedirs: true
   {%- else %}
   file.exists:
@@ -24,6 +30,8 @@
     - name: {{ cert_file }}
     - contents_pillar: nova:controller:database:x509:cert
     - mode: 440
+    - user: nova
+    - group: nova
     - makedirs: true
   {%- else %}
   file.exists:
@@ -36,13 +44,24 @@
     - name: {{ key_file }}
     - contents_pillar: nova:controller:database:x509:key
     - mode: 400
+    - user: nova
+    - group: nova
     - makedirs: true
   {%- else %}
   file.exists:
     - name: {{ key_file }}
   {%- endif %}
 
-{% elif controller.database.get('ssl',{}).get('enabled',False) %}
+mysql_nova_ssl_x509_set_user_and_group:
+  file.managed:
+    - names:
+      - {{ ca_file }}
+      - {{ cert_file }}
+      - {{ key_file }}
+    - user: nova
+    - group: nova
+
+  {% elif controller.database.get('ssl',{}).get('enabled',False) %}
 mysql_ca_nova_controller:
   {%- if controller.database.ssl.cacert is defined %}
   file.managed:
diff --git a/nova/compute.sls b/nova/compute.sls
index 1b8c149..4cfe44f 100644
--- a/nova/compute.sls
+++ b/nova/compute.sls
@@ -50,6 +50,19 @@
     - libvirtd
 {%- endif %}
 
+user_libvirt-qemu:
+  user.present:
+  - name: libvirt-qemu
+  - groups:
+    - nova
+
+/var/lib/nova:
+  file.directory:
+    - user: nova
+    - group: nova
+    - dir_mode: 0750
+    - makedirs: True
+
 /var/lib/nova/.ssh/id_rsa:
   file.managed:
   - user: nova
diff --git a/nova/controller.sls b/nova/controller.sls
index 19caa45..2b5d515 100644
--- a/nova/controller.sls
+++ b/nova/controller.sls
@@ -1,7 +1,5 @@
 {% from "nova/map.jinja" import controller with context %}
 
-{%- set mysql_x509_ssl_enabled = controller.database.get('x509',{}).get('enabled',False) or controller.database.get('ssl',{}).get('enabled',False) %}
-
 {%- if controller.get('enabled') %}
 
 include:
@@ -12,9 +10,7 @@
   # TODO(vsaienko) we need to run online dbsync only once after upgrade
   # Move to appropriate upgrade phase
   - nova.db.online_sync
-  {%- if mysql_x509_ssl_enabled %}
   - nova._ssl.mysql
-  {%- endif %}
 
 {%- if grains.os_family == 'Debian' %}
 debconf-set-prerequisite:
@@ -64,6 +60,7 @@
   - system: True
   - require_in:
     - pkg: nova_controller_packages
+    - sls: nova._ssl.mysql
 {%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
     - pkg: nova_placement_package
 {%- endif %}
@@ -173,6 +170,7 @@
   - template: jinja
   - require:
     - pkg: nova_controller_packages
+    - sls: nova._ssl.mysql
   - require_in:
     - sls: nova.db.offline_sync
     - sls: nova.db.online_sync
@@ -216,6 +214,8 @@
     - template: jinja
     - user: nova
     - group: nova
+    - require_in:
+      - sls: nova.db.offline_sync
     - require:
       - pkg: nova_controller_packages
 {%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
@@ -432,9 +432,7 @@
   {%- endif %}
   - require:
     - sls: nova.db.offline_sync
-    {%- if mysql_x509_ssl_enabled %}
     - sls: nova._ssl.mysql
-    {%- endif %}
   - watch:
     - file: /etc/nova/nova.conf
     - file: /etc/nova/api-paste.ini
@@ -451,11 +449,9 @@
   {%- endif %}
   - require:
     - sls: nova.db.offline_sync
+    - sls: nova._ssl.mysql
   - require_in:
     - sls: nova.db.online_sync
-    {%- if mysql_x509_ssl_enabled %}
-    - sls: nova._ssl.mysql
-    {%- endif %}
   - watch:
     - file: /etc/nova/nova.conf
     - file: /etc/nova/api-paste.ini
diff --git a/nova/files/grafana_dashboards/openstack_tenants_prometheus.json b/nova/files/grafana_dashboards/openstack_tenants_prometheus.json
new file mode 100644
index 0000000..607524e
--- /dev/null
+++ b/nova/files/grafana_dashboards/openstack_tenants_prometheus.json
@@ -0,0 +1,971 @@
+{%- raw %}
+{
+  "annotations": {
+    "list": [
+      {
+        "$$hashKey": "object:3956",
+        "builtIn": 1,
+        "datasource": "prometheus",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "limit": 100,
+        "name": "Annotations & Alerts",
+        "showIn": 0,
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1536224134023,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 38,
+      "panels": [],
+      "title": "Projects overview @ $project",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 36,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "$$hashKey": "object:4031",
+          "expr": "sum(libvirt_domain_info_virtual_cpus{project_name=~\"$project\"}) by (project_name)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{project_name}} total",
+          "refId": "A"
+        },
+        {
+          "$$hashKey": "object:4032",
+          "expr": "sum(rate(libvirt_domain_info_cpu_time_seconds_total{project_name=~\"$project\"}[$rate_interval])) by (project_name)",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{project_name}} usage",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "CPU",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 39,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(libvirt_domain_info_memory_usage_bytes{project_name=~\"$project\"}) by (project_name)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{project_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Memory Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "decbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 40,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(libvirt_domain_block_stats_read_bytes_total{project_name=~\"$project\"}[$rate_interval])) by (project_name)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{project_name}} read",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(libvirt_domain_block_stats_write_bytes_total{project_name=~\"$project\"}[$rate_interval])) by (project_name)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{project_name}} write",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Disks I/O",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 41,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/tx.*/",
+          "transform": "negative-Y"
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(libvirt_domain_interface_stats_receive_bytes_total{project_name=~\"$project\"}[$rate_interval])) by (project_name)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{project_name}} rx",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(libvirt_domain_interface_stats_transmit_bytes_total{project_name=~\"$project\"}[$rate_interval])) by (project_name)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{project_name}} tx",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Network RX/TX",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 11
+      },
+      "id": 43,
+      "panels": [],
+      "title": "Users overview @ $user",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 44,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "$$hashKey": "object:4174",
+          "expr": "sum(libvirt_domain_info_virtual_cpus{project_name=~\"$project\",user_name=~\"$user\"}) by (user_name)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{user_name}} total",
+          "refId": "A"
+        },
+        {
+          "$$hashKey": "object:4196",
+          "expr": "sum(rate(libvirt_domain_info_cpu_time_seconds_total{project_name=~\"$project\",user_name=~\"$user\"}[$rate_interval])) by (user_name)",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{user_name}} usage",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "CPU",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "id": 45,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(libvirt_domain_info_memory_usage_bytes{project_name=~\"$project\",user_name=~\"$user\"}) by (user_name)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{user_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Memory Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "decbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 17
+      },
+      "id": 46,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(libvirt_domain_block_stats_read_bytes_total{project_name=~\"$project\",user_name=~\"$user\"}[$rate_interval])) by (user_name)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{user_name}} read",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(libvirt_domain_block_stats_write_bytes_total{project_name=~\"$project\",user_name=~\"$user\"}[$rate_interval])) by (user_name)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{user_name}} write",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Disks I/O",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "id": 47,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/tx.*/",
+          "transform": "negative-Y"
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(libvirt_domain_interface_stats_receive_bytes_total{project_name=~\"$project\",user_name=~\"$user\"}[$rate_interval])) by (user_name)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{user_name}} rx",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(libvirt_domain_interface_stats_transmit_bytes_total{project_name=~\"$project\",user_name=~\"$user\"}[$rate_interval])) by (user_name)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{user_name}} tx",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Network RX/TX",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [
+    "openstack"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "tags": [],
+          "text": "admin",
+          "value": [
+            "admin"
+          ]
+        },
+        "datasource": "prometheus",
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": true,
+        "name": "project",
+        "options": [],
+        "query": "label_values(libvirt_domain_info_virtual_cpus,project_name)",
+        "refresh": 1,
+        "regex": "",
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "selected": false,
+          "tags": [],
+          "text": "admin",
+          "value": [
+            "admin"
+          ]
+        },
+        "datasource": "prometheus",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "user",
+        "options": [],
+        "query": "label_values(libvirt_domain_info_virtual_cpus{project_name=~\"$project\"},user_name)",
+        "refresh": 1,
+        "regex": "",
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "tags": [],
+          "text": "3m",
+          "value": "3m"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "rate_interval",
+        "options": [
+          {
+            "$$hashKey": "object:4409",
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "$$hashKey": "object:4410",
+            "selected": true,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "$$hashKey": "object:4411",
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "$$hashKey": "object:4412",
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "$$hashKey": "object:4413",
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "1m,3m,5m,10m,15m",
+        "type": "custom"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Openstack - Tenants",
+  "uid": null,
+  "version": 1
+}
+{%- endraw %}
diff --git a/nova/files/juno/nova-compute.conf.Debian b/nova/files/juno/nova-compute.conf.Debian
index b5693e7..7f2e4a0 100644
--- a/nova/files/juno/nova-compute.conf.Debian
+++ b/nova/files/juno/nova-compute.conf.Debian
@@ -100,8 +100,8 @@
 rpc_thread_pool_size = 70
 report_interval = 5
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 resume_guests_state_on_host_boot = True
 service_down_time = 90
diff --git a/nova/files/juno/nova-controller.conf.Debian b/nova/files/juno/nova-controller.conf.Debian
index 0cdffdc..2f886fe 100644
--- a/nova/files/juno/nova-controller.conf.Debian
+++ b/nova/files/juno/nova-controller.conf.Debian
@@ -127,8 +127,8 @@
 rpc_thread_pool_size = 70
 report_interval = 5
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 [keystone_authtoken]
 signing_dirname=/tmp/keystone-signing-nova
diff --git a/nova/files/kilo/nova-compute.conf.Debian b/nova/files/kilo/nova-compute.conf.Debian
index d76e9a1..8d17f05 100644
--- a/nova/files/kilo/nova-compute.conf.Debian
+++ b/nova/files/kilo/nova-compute.conf.Debian
@@ -89,8 +89,8 @@
 rpc_thread_pool_size = 70
 report_interval = {{ compute.get('report_interval', '60') }}
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 resume_guests_state_on_host_boot = True
 service_down_time = 90
diff --git a/nova/files/kilo/nova-controller.conf.Debian b/nova/files/kilo/nova-controller.conf.Debian
index 64a40d8..29ba5ca 100644
--- a/nova/files/kilo/nova-controller.conf.Debian
+++ b/nova/files/kilo/nova-controller.conf.Debian
@@ -116,8 +116,8 @@
 rpc_thread_pool_size = 70
 report_interval = 5
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 [oslo_concurrency]
 
diff --git a/nova/files/liberty/nova-compute.conf.Debian b/nova/files/liberty/nova-compute.conf.Debian
index de611f7..5369573 100644
--- a/nova/files/liberty/nova-compute.conf.Debian
+++ b/nova/files/liberty/nova-compute.conf.Debian
@@ -97,8 +97,8 @@
 rpc_thread_pool_size = 70
 report_interval = {{ compute.get('report_interval', '60') }}
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 resume_guests_state_on_host_boot = {{ compute.get('resume_guests_state_on_host_boot', True) }}
 service_down_time = 90
diff --git a/nova/files/liberty/nova-controller.conf.Debian b/nova/files/liberty/nova-controller.conf.Debian
index 64d8c85..392b48c 100644
--- a/nova/files/liberty/nova-controller.conf.Debian
+++ b/nova/files/liberty/nova-controller.conf.Debian
@@ -111,8 +111,8 @@
 rpc_thread_pool_size = 70
 report_interval = 5
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 [oslo_concurrency]
 
diff --git a/nova/files/mitaka/nova-compute.conf.Debian b/nova/files/mitaka/nova-compute.conf.Debian
index 4b22b31..0563049 100644
--- a/nova/files/mitaka/nova-compute.conf.Debian
+++ b/nova/files/mitaka/nova-compute.conf.Debian
@@ -74,8 +74,8 @@
 executor_thread_pool_size = 70
 report_interval = {{ compute.get('report_interval', '60') }}
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 resume_guests_state_on_host_boot = {{ compute.get('resume_guests_state_on_host_boot', True) }}
 service_down_time = 90
diff --git a/nova/files/mitaka/nova-controller.conf.Debian b/nova/files/mitaka/nova-controller.conf.Debian
index 5f568da..d5c4e65 100644
--- a/nova/files/mitaka/nova-controller.conf.Debian
+++ b/nova/files/mitaka/nova-controller.conf.Debian
@@ -15,6 +15,9 @@
 dhcpbridge = /usr/bin/nova-dhcpbridge
 dhcpbridge_flagfile = /etc/nova/nova.conf
 force_dhcp_release = True
+{%- if controller.dhcp_domain is defined %}
+dhcp_domain = {{ controller.dhcp_domain }}
+{%- endif %}
 injected_network_template = /usr/share/nova/interfaces.template
 libvirt_nonblocking = True
 vif_plugging_is_fatal = False
@@ -73,8 +76,8 @@
 rpc_thread_pool_size = 70
 report_interval = 5
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 {%- if controller.host is defined %}
 host={{ controller.host }}
diff --git a/nova/files/newton/nova-compute.conf.Debian b/nova/files/newton/nova-compute.conf.Debian
index f2f498d..7df9f31 100644
--- a/nova/files/newton/nova-compute.conf.Debian
+++ b/nova/files/newton/nova-compute.conf.Debian
@@ -89,8 +89,8 @@
 executor_thread_pool_size = 70
 report_interval = {{ compute.get('report_interval', '60') }}
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 resume_guests_state_on_host_boot = {{ compute.get('resume_guests_state_on_host_boot', True) }}
 service_down_time = 90
diff --git a/nova/files/newton/nova-controller.conf.Debian b/nova/files/newton/nova-controller.conf.Debian
index 26c38a1..124b5d7 100644
--- a/nova/files/newton/nova-controller.conf.Debian
+++ b/nova/files/newton/nova-controller.conf.Debian
@@ -15,6 +15,9 @@
 dhcpbridge = /usr/bin/nova-dhcpbridge
 dhcpbridge_flagfile = /etc/nova/nova.conf
 force_dhcp_release = True
+{%- if controller.dhcp_domain is defined %}
+dhcp_domain = {{ controller.dhcp_domain }}
+{%- endif %}
 injected_network_template = /usr/share/nova/interfaces.template
 libvirt_nonblocking = True
 vif_plugging_is_fatal = False
@@ -80,8 +83,8 @@
 rpc_thread_pool_size = 70
 report_interval = 5
 
-block_device_allocate_retries=600
-block_device_allocate_retries_interval=10
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 {%- set rabbit_port = controller.message_queue.get('port', 5671 if controller.message_queue.get('ssl',{}).get('enabled', False) else 5672) %}
 
diff --git a/nova/files/ocata/nova-compute.conf.Debian b/nova/files/ocata/nova-compute.conf.Debian
index 0b08583..dacdacb 100644
--- a/nova/files/ocata/nova-compute.conf.Debian
+++ b/nova/files/ocata/nova-compute.conf.Debian
@@ -738,7 +738,7 @@
 # * For any value > 0, total attempts are (value + 1)
 #  (integer value)
 #block_device_allocate_retries=60
-block_device_allocate_retries=600
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
 
 #
 # Number of greenthreads available for use to sync power states.
@@ -942,7 +942,7 @@
 #  (integer value)
 # Minimum value: 0
 #block_device_allocate_retries_interval=3
-block_device_allocate_retries_interval=10
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 #
 # Interval between sending the scheduler a list of current instance UUIDs to
@@ -6750,7 +6750,9 @@
 # * Any string representing mount options separated by commas.
 # * Example string: vers=3,lookupcache=pos
 #  (string value)
-#nfs_mount_options=<None>
+{%- if compute.nfs_mount_options is defined %}
+nfs_mount_options="{{ compute.nfs_mount_options }}"
+{%- endif %}
 
 #
 # Directory where the Quobyte volume is mounted on the compute node.
diff --git a/nova/files/ocata/nova-controller.conf.Debian b/nova/files/ocata/nova-controller.conf.Debian
index ac6b0e5..6c568f9 100644
--- a/nova/files/ocata/nova-controller.conf.Debian
+++ b/nova/files/ocata/nova-controller.conf.Debian
@@ -708,7 +708,7 @@
 # * For any value > 0, total attempts are (value + 1)
 #  (integer value)
 #block_device_allocate_retries=60
-block_device_allocate_retries=600
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
 
 #
 # Number of greenthreads available for use to sync power states.
@@ -910,7 +910,7 @@
 #  (integer value)
 # Minimum value: 0
 #block_device_allocate_retries_interval=3
-block_device_allocate_retries_interval=10
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 #
 # Interval between sending the scheduler a list of current instance UUIDs to
@@ -2272,6 +2272,9 @@
 # Reason:
 # nova-network is deprecated, as are any related configuration options.
 #dhcp_domain=novalocal
+{%- if controller.dhcp_domain is defined %}
+dhcp_domain = {{ controller.dhcp_domain }}
+{%- endif %}
 
 # DEPRECATED:
 # This option allows you to specify the L3 management library to be used.
@@ -6725,7 +6728,9 @@
 # * Any string representing mount options separated by commas.
 # * Example string: vers=3,lookupcache=pos
 #  (string value)
-#nfs_mount_options=<None>
+{%- if controller.nfs_mount_options is defined %}
+nfs_mount_options="{{ controller.nfs_mount_options }}"
+{%- endif %}
 
 #
 # Directory where the Quobyte volume is mounted on the compute node.
diff --git a/nova/files/pike/nova-compute.conf.Debian b/nova/files/pike/nova-compute.conf.Debian
index 3dee8e2..fc34f4c 100644
--- a/nova/files/pike/nova-compute.conf.Debian
+++ b/nova/files/pike/nova-compute.conf.Debian
@@ -742,7 +742,7 @@
 # * For any value > 0, total attempts are (value + 1)
 #  (integer value)
 #block_device_allocate_retries=60
-block_device_allocate_retries=600
+block_device_allocate_retries={{ compute.get('block_device_allocate_retries', '600') }}
 
 #
 # Number of greenthreads available for use to sync power states.
@@ -946,7 +946,7 @@
 #  (integer value)
 # Minimum value: 0
 #block_device_allocate_retries_interval=3
-block_device_allocate_retries_interval=10
+block_device_allocate_retries_interval={{ compute.get('block_device_allocate_retries_interval', '10') }}
 
 #
 # Interval between sending the scheduler a list of current instance UUIDs to
@@ -6792,7 +6792,9 @@
 # * Any string representing mount options separated by commas.
 # * Example string: vers=3,lookupcache=pos
 #  (string value)
-#nfs_mount_options=<None>
+{%- if compute.nfs_mount_options is defined %}
+nfs_mount_options="{{ compute.nfs_mount_options }}"
+{%- endif %}
 
 #
 # Directory where the Quobyte volume is mounted on the compute node.
diff --git a/nova/files/pike/nova-controller.conf.Debian b/nova/files/pike/nova-controller.conf.Debian
index 8321a8d..34451c2 100644
--- a/nova/files/pike/nova-controller.conf.Debian
+++ b/nova/files/pike/nova-controller.conf.Debian
@@ -2,7 +2,7 @@
 
 {%- set connection_x509_ssl_option = '' %}
 {%- if controller.database.get('x509',{}).get('enabled',False) %}
-  {%- set connection_x509_ssl_option = '&ssl_ca=' ~ controller.database.x509.get('ca_file') ~ '&ssl_cert=' ~ controller.database.x509.get('cert_file') ~ '&ssl_key=' ~ controller.database.x509.get('key_file') %}
+  {%- set connection_x509_ssl_option = '&ssl_ca=' ~ controller.database.x509.ca_file ~ '&ssl_cert=' ~ controller.database.x509.cert_file ~ '&ssl_key=' ~ controller.database.x509.key_file %}
 {%- elif controller.database.get('ssl',{}).get('enabled',False) %}
   {%- set connection_x509_ssl_option = '&ssl_ca=' ~ controller.database.ssl.get('cacert_file', controller.cacert_file) %}
 {%- endif %}
@@ -721,7 +721,7 @@
 # * For any value > 0, total attempts are (value + 1)
 #  (integer value)
 #block_device_allocate_retries=60
-block_device_allocate_retries=600
+block_device_allocate_retries={{ controller.get('block_device_allocate_retries', '600') }}
 
 #
 # Number of greenthreads available for use to sync power states.
@@ -923,7 +923,7 @@
 #  (integer value)
 # Minimum value: 0
 #block_device_allocate_retries_interval=3
-block_device_allocate_retries_interval=10
+block_device_allocate_retries_interval={{ controller.get('block_device_allocate_retries_interval', '10') }}
 
 #
 # Interval between sending the scheduler a list of current instance UUIDs to
@@ -3447,7 +3447,7 @@
 db_retry_interval = 1
 connection_debug = 10
 pool_timeout = 120
-connection = {{ controller.database.engine }}+pymysql://{{ controller.database.user }}:{{ controller.database.password }}@{{ controller.database.host }}/{{ controller.database.name }}_api?charset=utf8{{ connection_x509_ssl_option }}
+connection = {{ controller.database.engine }}+pymysql://{{ controller.database.user }}:{{ controller.database.password }}@{{ controller.database.host }}/{{ controller.database.name }}_api?charset=utf8{{ connection_x509_ssl_option|string }}
 
 # The SQLAlchemy connection string to use to connect to the database. (string
 # value)
@@ -4496,7 +4496,7 @@
 db_retry_interval = 1
 connection_debug = 10
 pool_timeout = 120
-connection = {{ controller.database.engine }}+pymysql://{{ controller.database.user }}:{{ controller.database.password }}@{{ controller.database.host }}/{{ controller.database.name }}?charset=utf8{{ connection_x509_ssl_option }}
+connection = {{ controller.database.engine }}+pymysql://{{ controller.database.user }}:{{ controller.database.password }}@{{ controller.database.host }}/{{ controller.database.name }}?charset=utf8{{ connection_x509_ssl_option|string }}
 
 # If True, SQLite uses synchronous mode. (boolean value)
 # Deprecated group/name - [DEFAULT]/sqlite_synchronous
@@ -6743,7 +6743,9 @@
 # * Any string representing mount options separated by commas.
 # * Example string: vers=3,lookupcache=pos
 #  (string value)
-#nfs_mount_options=<None>
+{%- if controller.nfs_mount_options is defined %}
+nfs_mount_options="{{ controller.nfs_mount_options }}"
+{%- endif %}
 
 #
 # Directory where the Quobyte volume is mounted on the compute node.
diff --git a/nova/files/queens/nova-compute.conf.Debian b/nova/files/queens/nova-compute.conf.Debian
index b2168a1..da35e8b 100644
--- a/nova/files/queens/nova-compute.conf.Debian
+++ b/nova/files/queens/nova-compute.conf.Debian
@@ -7392,7 +7392,9 @@
 # * Any string representing mount options separated by commas.
 # * Example string: vers=3,lookupcache=pos
 #  (string value)
-#nfs_mount_options = <None>
+{%- if compute.nfs_mount_options is defined %}
+nfs_mount_options="{{ compute.nfs_mount_options }}"
+{%- endif %}
 
 #
 # Directory where the Quobyte volume is mounted on the compute node.
diff --git a/nova/files/queens/nova-controller.conf.Debian b/nova/files/queens/nova-controller.conf.Debian
index c413d87..b456d31 100644
--- a/nova/files/queens/nova-controller.conf.Debian
+++ b/nova/files/queens/nova-controller.conf.Debian
@@ -7288,7 +7288,9 @@
 # * Any string representing mount options separated by commas.
 # * Example string: vers=3,lookupcache=pos
 #  (string value)
-#nfs_mount_options = <None>
+{%- if controller.nfs_mount_options is defined %}
+nfs_mount_options="{{ controller.nfs_mount_options }}"
+{%- endif %}
 
 #
 # Directory where the Quobyte volume is mounted on the compute node.
diff --git a/nova/map.jinja b/nova/map.jinja
index 651293a..7e0006f 100644
--- a/nova/map.jinja
+++ b/nova/map.jinja
@@ -19,7 +19,7 @@
 
 {%- if grains.os_family == "Debian" %}
 {%- set pkgs_list = [ 'nova-common', 'nova-consoleproxy', 'novnc', 'nova-api', 'nova-conductor', 'nova-consoleauth', 'nova-doc', 'nova-scheduler', 'python-novaclient', 'python-memcache', 'gettext-base', 'python-pycadf'] %}
-{%- set services_list = ['nova-api', 'nova-consoleauth', 'nova-scheduler', 'nova-conductor', 'nova-novncproxy'] %}
+{%- set services_list = ['nova-conductor', 'nova-api', 'nova-consoleauth', 'nova-scheduler', 'nova-novncproxy'] %}
 {%- if pillar.nova.controller is defined and pillar.nova.controller.get('version',{}) in ["juno", "kilo", "liberty", "mitaka"] %}
 {%- do pkgs_list.append('nova-cert') %}
 {%- do services_list.append('nova-cert') %}
@@ -28,7 +28,7 @@
 
 {%- if grains.os_family == "RedHat" %}
 {%- set pkgs_list = ['openstack-nova-novncproxy', 'python-nova', 'openstack-nova-api', 'openstack-nova-console', 'openstack-nova-scheduler', 'python-novaclient', 'openstack-nova-common', 'openstack-nova-conductor', 'python-pycadf'] %}
-{%- set services_list = ['openstack-nova-api', 'openstack-nova-consoleauth', 'openstack-nova-scheduler', 'openstack-nova-conductor', 'openstack-nova-novncproxy'] %}
+{%- set services_list = ['openstack-nova-conductor', 'openstack-nova-api', 'openstack-nova-consoleauth', 'openstack-nova-scheduler', 'openstack-nova-novncproxy'] %}
 {%- if pillar.nova.controller is defined and pillar.nova.controller.get('version',{}) in ["juno", "kilo", "liberty", "mitaka", "newton", "ocata"] %}
 {%- do pkgs_list.append('openstack-nova-cert') %}
 {%- do services_list.append('openstack-nova-cert') %}
@@ -111,6 +111,40 @@
     },
 }, merge=pillar.nova.get('controller', {}), base='BaseDefaults') %}
 
+{% set upgrade = pillar.get('nova', {}).get('upgrade', {}) %}
+{% set pin_level = 'auto' %}
+
+{% set upgrade_levels = salt['grains.filter_by']({
+    'Debian': {
+        'upgrade_levels': {
+          'compute': pin_level,
+          'cells': pin_level,
+          'intercell': pin_level,
+          'cert': pin_level,
+          'scheduler': pin_level,
+          'conductor': pin_level,
+          'console': pin_level,
+          'consoleauth': pin_level,
+          'network': pin_level,
+          'baseapi': pin_level,
+        },
+     },
+    'RedHat': {
+        'upgrade_levels': {
+          'compute': pin_level,
+          'cells': pin_level,
+          'intercell': pin_level,
+          'cert': pin_level,
+          'scheduler': pin_level,
+          'conductor': pin_level,
+          'console': pin_level,
+          'consoleauth': pin_level,
+          'network': pin_level,
+          'baseapi': pin_level,
+        },
+    },
+}) %}
+
 {% set client = salt['grains.filter_by']({
     'Debian': {
         'pkgs': ['python-novaclient']
@@ -250,6 +284,11 @@
 {%- endload %}
 {% set compute = salt["grains.filter_by"](compute_defaults, merge=pillar.nova.get("compute", {}), base='BaseDefaults') %}
 
+{%- if pillar.nova.get('upgrade',{}).get('upgrade_enabled',False) %}
+  {% do compute.update(upgrade_levels) %}
+  {% do controller.update(upgrade_levels) %}
+{%- endif %}
+
 {% set monitoring = salt['grains.filter_by']({
     'default': {
         'disk': {
@@ -272,3 +311,4 @@
         'endpoint_failed_major_threshold': 0.5,
     },
 }, grain='os_family', merge=salt['pillar.get']('nova:monitoring')) %}
+
diff --git a/nova/meta/grafana.yml b/nova/meta/grafana.yml
index 0fc3ac3..b189d5e 100644
--- a/nova/meta/grafana.yml
+++ b/nova/meta/grafana.yml
@@ -34,6 +34,10 @@
     datasource: prometheus
     format: json
     template: nova/files/grafana_dashboards/openstack_overview_prometheus.json
+  openstack_tenants_prometheus:
+    datasource: prometheus
+    format: json
+    template: nova/files/grafana_dashboards/openstack_tenants_prometheus.json
   nova_influxdb:
     datasource: influxdb
     format: json
diff --git a/nova/meta/salt.yml b/nova/meta/salt.yml
index ab7047e..f5d1f33 100644
--- a/nova/meta/salt.yml
+++ b/nova/meta/salt.yml
@@ -9,3 +9,8 @@
     require:
     - salt: keystone.server
 
+orchestration:
+  upgrade:
+    applications:
+      nova:
+        priority: 1100
diff --git a/nova/upgrade/pkgs_latest.sls b/nova/upgrade/pkgs_latest.sls
new file mode 100644
index 0000000..2c702ad
--- /dev/null
+++ b/nova/upgrade/pkgs_latest.sls
@@ -0,0 +1,44 @@
+{%- from "nova/map.jinja" import controller, compute, client, upgrade with context %}
+
+nova_task_pkgs_latest:
+  test.show_notification:
+    - name: "dump_message_pkgs_latest"
+    - text: "Running nova.upgrade.pkg_latest"
+
+policy-rc.d_present:
+  file.managed:
+    - name: /usr/sbin/policy-rc.d
+    - mode: 755
+    - contents: |
+        #!/bin/sh
+        exit 101
+
+{%- set pkgs = [] %}
+{%- if compute.get('enabled', false) %}
+  {%- do pkgs.extend(compute.pkgs) %}
+{%- endif %}
+
+{%- if client.get('enabled', false) %}
+  {%- do pkgs.extend(client.pkgs) %}
+{%- endif %}
+
+{%- if controller.get('enabled', false) %}
+  {%- do pkgs.extend(controller.pkgs) %}
+
+{%- if controller.get('version',{}) not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+  {%- do pkgs.append('nova-placement-api') %}
+{%- endif %}
+
+{%- endif %}
+
+nova_packages:
+  pkg.latest:
+  - names: {{ pkgs|unique }}
+  - require:
+    - file: policy-rc.d_present
+  - require_in:
+    - file: policy-rc.d_absent
+
+policy-rc.d_absent:
+  file.absent:
+    - name: /usr/sbin/policy-rc.d
diff --git a/nova/upgrade/post/init.sls b/nova/upgrade/post/init.sls
new file mode 100644
index 0000000..a29a06d
--- /dev/null
+++ b/nova/upgrade/post/init.sls
@@ -0,0 +1,21 @@
+{%- from "nova/map.jinja" import controller, compute, client, upgrade with context %}
+
+nova_post:
+  test.show_notification:
+    - name: "dump_message_post-upgrade"
+    - text: "Running nova.upgrade.post"
+
+{%- if controller.get('enabled') %}
+
+nova_status:
+  cmd.run:
+    - name: nova-status upgrade check
+  {%- if grains.get('noservices') or controller.get('role', 'primary') == 'secondary' %}
+    - onlyif: /bin/false
+  {%- endif %}
+
+{%- endif %}
+
+keystone_os_client_config_absent:
+  file.absent:
+    - name: /etc/openstack/clouds.yml
diff --git a/nova/upgrade/pre/init.sls b/nova/upgrade/pre/init.sls
new file mode 100644
index 0000000..9ed049a
--- /dev/null
+++ b/nova/upgrade/pre/init.sls
@@ -0,0 +1,44 @@
+{%- from "nova/map.jinja" import controller, compute, upgrade with context %}
+
+nova_pre:
+  test.show_notification:
+    - name: "dump_message_pre-upgrade_nova"
+    - text: "Running nova.upgrade.pre"
+
+{%- if controller.get('enabled', false) %}
+  {%- set _data = controller %}
+  {%- set type = 'controller' %}
+{%- elif compute.get('enabled', false) %}
+  {%- set _data = compute %}
+  {%- set type = 'compute' %}
+{%- endif %}
+
+/etc/nova/nova.conf:
+  file.managed:
+  - name: /etc/nova/nova.conf
+  - source: salt://nova/files/{{ _data.version }}/nova-{{ type }}.conf.{{ grains.os_family }}
+  - template: jinja
+
+{%- if controller.get('enabled') %}
+
+include:
+ - nova.db.online_sync
+
+nova_status:
+  cmd.run:
+    - name: nova-status upgrade check
+  {%- if grains.get('noservices') or controller.get('role', 'primary') == 'secondary' %}
+    - onlyif: /bin/false
+  {%- endif %}
+{%- endif %}
+
+{%- set os_content = salt['mine.get']('I@keystone:client:os_client_config:enabled:true', 'keystone_os_client_config', 'compound').values()[0] %}
+keystone_os_client_config:
+  file.managed:
+    - name: /etc/openstack/clouds.yml
+    - contents: |
+        {{ os_content |yaml(False)|indent(8) }}
+    - user: 'root'
+    - group: 'root'
+    - makedirs: True
+    - unless: test -f /etc/openstack/clouds.yml
diff --git a/nova/upgrade/render_config.sls b/nova/upgrade/render_config.sls
new file mode 100644
index 0000000..d963b42
--- /dev/null
+++ b/nova/upgrade/render_config.sls
@@ -0,0 +1,51 @@
+{%- from "nova/map.jinja" import controller, compute, client, upgrade with context %}
+
+nova_render_config:
+  test.show_notification:
+    - name: "dump_message_render_config_nova"
+    - text: "Running nova.upgrade.render_config"
+
+{%- if controller.get('enabled', false) %}
+  {%- set _data = controller %}
+  {%- set type = 'controller' %}
+{%- elif compute.get('enabled', false) %}
+  {%- set _data = compute %}
+  {%- set type = 'compute' %}
+{%- endif %}
+
+/etc/nova/nova.conf:
+  file.managed:
+  - source: salt://nova/files/{{ _data.version }}/nova-{{ type }}.conf.{{ grains.os_family }}
+  - template: jinja
+
+/etc/nova/api-paste.ini:
+  file.managed:
+  - source: salt://nova/files/{{ _data.version }}/api-paste.ini.{{ grains.os_family }}
+  - template: jinja
+
+{%- if controller.get('enabled', False) %}
+
+  {% for service_name in controller.services %}
+{{ service_name }}_default:
+  file.managed:
+    - name: /etc/default/{{ service_name }}
+    - source: salt://nova/files/default
+    - template: jinja
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ controller }}
+  {% endfor %}
+
+  {% if controller.get('policy', {}) and controller.version not in ['liberty', 'mitaka', 'newton'] %}
+{# nova no longer ships with a default policy.json #}
+
+/etc/nova/policy.json:
+  file.managed:
+    - contents: '{}'
+    - replace: False
+    - user: nova
+    - group: nova
+
+  {% endif %}
+
+{%- endif %}
diff --git a/nova/upgrade/service_running.sls b/nova/upgrade/service_running.sls
new file mode 100644
index 0000000..e2f2177
--- /dev/null
+++ b/nova/upgrade/service_running.sls
@@ -0,0 +1,33 @@
+{%- from "nova/map.jinja" import controller, compute, upgrade with context %}
+
+nova_task_service_running:
+  test.show_notification:
+    - name: "dump_message_service_running_nova"
+    - text: "Running nova.upgrade.service_running"
+
+{%- if controller.get('enabled', false) %}
+  {%- set _data = controller %}
+{%- elif compute.get('enabled', false) %}
+  {%- set _data = compute %}
+{%- endif %}
+
+{%- set nservices = [] %}
+{%- do nservices.extend(_data.services) %}
+
+{%- if controller.get('enabled') %}
+  {%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+    {%- do nservices.append('apache2') %}
+  {%- endif %}
+{%- endif %}
+
+{%- if nservices|unique|length > 0 %}
+  {%- for service in nservices|unique %}
+nova_service_running_{{ service }}:
+  service.running:
+  - enable: true
+  - name: {{ service }}
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+  {%- endfor %}
+{%- endif %}
diff --git a/nova/upgrade/service_stopped.sls b/nova/upgrade/service_stopped.sls
new file mode 100644
index 0000000..ca5fd5e
--- /dev/null
+++ b/nova/upgrade/service_stopped.sls
@@ -0,0 +1,46 @@
+{%- from "nova/map.jinja" import controller, compute, upgrade with context %}
+
+nova_task_service_stopped:
+  test.show_notification:
+    - name: "dump_message_service_stopped_nova"
+    - text: "Running nova.upgrade.service_stopped"
+
+{%- if controller.get('enabled', false) %}
+  {%- set _data = controller %}
+{%- elif compute.get('enabled', false) %}
+  {%- set _data = compute %}
+{%- endif %}
+
+{%- set nservices = [] %}
+{%- do nservices.extend(_data.services) %}
+
+{%- if controller.get('enabled') %}
+
+  {%- if upgrade.get('upgrade_enabled',false) %}
+    {%- do controller.update({'version': upgrade.old_release}) %}
+  {%- endif %}
+
+  {%- if controller.version in ["juno", "kilo", "liberty", "mitaka"] %}
+    {%- do nservices.append('nova-cert') %}
+  {%- endif %}
+
+{%- endif %}
+
+{%- if controller.get('enabled') %}
+  {%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+    {%- do nservices.append('apache2') %}
+  {%- endif %}
+{%- endif %}
+
+{%- if nservices|unique|length > 0 %}
+{%- for service in nservices|unique %}
+nova_service_stopped_{{ service }}:
+  service.dead:
+  - enable: false
+  - name: {{ service }}
+  {%- if grains.get('noservices') %}
+  - onlyif: /bin/false
+  {%- endif %}
+{%- endfor %}
+{%- endif %}
+
diff --git a/nova/upgrade/upgrade/init.sls b/nova/upgrade/upgrade/init.sls
new file mode 100644
index 0000000..31f23f8
--- /dev/null
+++ b/nova/upgrade/upgrade/init.sls
@@ -0,0 +1,13 @@
+{%- from "nova/map.jinja" import controller, compute, compute_driver_mapping with context %}
+
+nova_upgrade:
+  test.show_notification:
+    - name: "dump_message_upgrade_nova"
+    - text: "Running nova.upgrade.upgrade"
+
+include:
+ - nova.upgrade.service_stopped
+ - nova.upgrade.pkgs_latest
+ - nova.upgrade.render_config
+ - nova.db.offline_sync
+ - nova.upgrade.service_running
diff --git a/nova/upgrade/upgrade/post.sls b/nova/upgrade/upgrade/post.sls
new file mode 100644
index 0000000..92dc005
--- /dev/null
+++ b/nova/upgrade/upgrade/post.sls
@@ -0,0 +1,7 @@
+{%- from "nova/map.jinja" import controller, compute, compute_driver_mapping with context %}
+
+nova_upgrade_post:
+  test.show_notification:
+    - name: "dump_message_upgrade_nova_post"
+    - text: "Running nova.upgrade.upgrade.post"
+
diff --git a/nova/upgrade/upgrade/pre.sls b/nova/upgrade/upgrade/pre.sls
new file mode 100644
index 0000000..31be4d5
--- /dev/null
+++ b/nova/upgrade/upgrade/pre.sls
@@ -0,0 +1,7 @@
+{%- from "nova/map.jinja" import controller, compute, compute_driver_mapping with context %}
+
+nova_upgrade_pre:
+  test.show_notification:
+    - name: "dump_message_upgrade_nova_pre"
+    - text: "Running nova.upgrade.upgrade.pre"
+
diff --git a/nova/upgrade/verify/_api.sls b/nova/upgrade/verify/_api.sls
new file mode 100644
index 0000000..0cf99fd
--- /dev/null
+++ b/nova/upgrade/verify/_api.sls
@@ -0,0 +1,48 @@
+{%- from "keystone/map.jinja" import client as kclient with context %}
+
+nova_upgrade_verify_api:
+  test.show_notification:
+    - name: "dump_message_verify_api_nova"
+    - text: "Running nova.upgrade.verify.api"
+
+{%- if kclient.enabled and kclient.get('os_client_config', {}).get('enabled', False)  %}
+  {%- set Nova_Test_Flavor_ID = 'random' | uuid %}
+
+novav21_flavor_create:
+  module.run:
+    - name: novav21.flavor_create
+    - kwargs:
+        cloud_name: admin_identity
+        id: {{ Nova_Test_Flavor_ID }}
+        name: {{ Nova_Test_Flavor_ID }}
+        vcpus: 2
+        ram: 1
+        disk: 2
+
+novav21_flavor_list:
+  module.run:
+    - name: novav21.flavor_list
+    - kwargs:
+        cloud_name: admin_identity
+
+novav21_quota_list:
+  module.run:
+    - name: novav21.quota_list
+    - kwargs:
+        cloud_name: admin_identity
+        project_id: default
+
+novav21_flavor_absent:
+  novav21.flavor_absent:
+  - cloud_name: admin_identity
+  - name: {{ Nova_Test_Flavor_ID }}
+  - require:
+    - novav21_flavor_create
+
+novav21_server_list:
+  module.run:
+    - name: novav21.server_list
+    - kwargs:
+        cloud_name: admin_identity
+
+{%- endif %}
diff --git a/nova/upgrade/verify/init.sls b/nova/upgrade/verify/init.sls
new file mode 100644
index 0000000..a377c18
--- /dev/null
+++ b/nova/upgrade/verify/init.sls
@@ -0,0 +1,7 @@
+nova_upgrade_verify:
+  test.show_notification:
+    - name: "dump_message_upgrade_nova_verify"
+    - text: "Running nova.upgrade.verify"
+
+include:
+ - nova.upgrade.verify._api
diff --git a/tests/pillar/compute_single_config_drive_options.sls b/tests/pillar/compute_single_config_drive_options.sls
index f33faca..6351252 100644
--- a/tests/pillar/compute_single_config_drive_options.sls
+++ b/tests/pillar/compute_single_config_drive_options.sls
@@ -1,12 +1,13 @@
 nova:
   compute:
-    version: mitaka
+    version: queens
     enabled: true
     heal_instance_info_cache_interval: 60
     vncproxy_url: openstack:6080
     vnc_keymap: en-gb
     resume_guests_state_on_host_boot: False
     preallocate_images: space
+    nfs_mount_options: 'vers=3,lookupcache=pos'
     bind:
       vnc_address: 127.0.0.1
       vnc_port: 6080
diff --git a/tests/pillar/control_single.sls b/tests/pillar/control_single.sls
index 3b36463..eb91fd9 100644
--- a/tests/pillar/control_single.sls
+++ b/tests/pillar/control_single.sls
@@ -12,6 +12,7 @@
     ram_allocation_ratio: 1.5
     disk_allocation_ratio: 1.0
     workers: 8
+    nfs_mount_options: 'vers=3,lookupcache=pos'
     bind:
       private_address: 127.0.0.1
       public_address: 127.0.0.1