Merge "Set default TTL for memcache item expiration" into release/2019.2.0
diff --git a/README.rst b/README.rst
index 8eaa2e2..eec572b 100644
--- a/README.rst
+++ b/README.rst
@@ -512,6 +512,18 @@
libvirt:
volume_use_multipath: True
+To disable or enable StrictHostKeyChecking and discover
+compute nodes fingerprints the below pillar should be used:
+
+.. code-block:: yaml
+
+ nova:
+ compute:
+ ....
+ openssh:
+ stricthostkeychecking: True
+ discover_compute_hosts: True
+
Client role
-----------
@@ -718,6 +730,26 @@
libvirt:
cpu_model: IvyBridge
+RNG (Random Number Generator) device path
+----------------------
+
+The path to an RNG (Random Number Generator) device that will be used
+as the source of entropy on the host.
+The recommended source of entropy is /dev/urandom.
+Permitted options are: /dev/random, /dev/urandom or /dev/hwrng.
+Default: /dev/urandom
+
+.. code-block:: yaml
+
+ nova:
+ controller:
+ libvirt:
+ rng_dev_path: /dev/random
+
+ compute:
+ libvirt:
+ rng_dev_path: /dev/random
+
Nova compute workarounds
------------------------
@@ -1178,6 +1210,33 @@
You can read more about it here:
https://docs.openstack.org/security-guide/databases/database-access-control.html
+Configure nova to use service user tokens:
+========
+Long-running operations such as live migration or snapshot can sometimes overrun the
+expiry of the user token. In such cases, post operations such as cleaning up after a
+live migration can fail when the nova-compute service needs to cleanup resources in
+other services, such as in the block-storage (cinder) or networking (neutron) services.
+
+This patch enables nova to use service user tokens to supplement the regular user token
+used to initiate the operation. The identity service (keystone) will then authenticate
+a request using the service user token if the user token has already expired.
+
+.. code-block:: yaml
+
+ nova:
+ controller:
+ enabled: True
+ ...
+ service_user:
+ enabled: True
+ user_domain_id: default
+ project_domain_id: default
+ project_name: service
+ username: nova
+ password: pswd
+
+
+
Upgrades
========
diff --git a/_states/novav21.py b/_states/novav21.py
index b9f9dd0..7b6452b 100644
--- a/_states/novav21.py
+++ b/_states/novav21.py
@@ -361,6 +361,43 @@
return ret
+def instances_mapped_to_cell(name, timeout=60, runas='nova'):
+ """Ensure that all instances in the cell are mapped
+
+ :param name: cell name.
+ :param timeout: amount of time in seconds mapping process should finish in.
+ :param runas: username to run the shell commands under.
+ """
+ test = __opts__.get('test', False)
+ cell_uuid = __salt__['cmd.shell'](
+ "nova-manage cell_v2 list_cells 2>/dev/null | "
+ "awk '/%s/ {print $4}'" % name, runas=runas)
+ result = {'name': name, 'changes': {}, 'result': False}
+ if not cell_uuid:
+ result['comment'] = (
+ 'Failed to map all instances in cell {0}, it does not exist'
+ .format(name))
+ return result
+ start_time = time.time()
+ if not test:
+ while True:
+ rc = __salt__['cmd.retcode'](
+ 'nova-manage cell_v2 map_instances --cell_uuid %s' % cell_uuid,
+ runas=runas)
+ if rc == 0 or time.time() - start_time > timeout:
+ break
+ if rc != 0:
+ result['comment'] = (
+ 'Failed to map all instances in cell {0} in {1} seconds'
+ .format(name, timeout))
+ return result
+ result['comment'] = 'All instances mapped in cell {0}'.format(name)
+ if test:
+ result['comment'] = 'TEST: {}'.format(result['comment'])
+ result['result'] = True
+ return result
+
+
def _db_version_update(db, version, human_readable_resource_name):
existing_version = __salt__['cmd.shell'](
'nova-manage %s version 2>/dev/null' % db)
diff --git a/nova/compute.sls b/nova/compute.sls
index 6103c14..fb622c2 100644
--- a/nova/compute.sls
+++ b/nova/compute.sls
@@ -76,11 +76,31 @@
/var/lib/nova/.ssh/config:
file.managed:
- user: nova
- - contents: StrictHostKeyChecking no
+ - contents: StrictHostKeyChecking {% if compute.get('openssh',{}).get('stricthostkeychecking', False) %}yes{% else %}no{% endif %}
- mode: 400
- require:
- pkg: nova_compute_packages
+{%- if compute.get('openssh',{}).get('discover_compute_hosts', False) %}
+
+ {%- set cmp_nodenames = [] %}
+
+ {%- for cmp_nodes, node_grains in salt['mine.get']('I@nova:compute', 'grains.items', 'compound').items() %}
+ {%- if node_grains.fqdn is defined %}
+ {%- do cmp_nodenames.append(node_grains.fqdn) %}
+ {%- endif %}
+ {%- endfor %}
+
+ {%- for cmp_node in cmp_nodenames %}
+ssh_host_discover_{{ cmp_node }}_fingerprint:
+ ssh_known_hosts.present:
+ - name: {{ cmp_node }}
+ - user: nova
+ - require:
+ - file: /var/lib/nova/.ssh/config
+ {%- endfor %}
+{%- endif %}
+
{%- endif %}
{%- if not pillar.nova.get('controller',{}).get('enabled') %}
@@ -586,4 +606,23 @@
- name: 'systemd-tmpfiles --create'
{%- endif %}
+{%- if compute.get('libvirt', {}).rng_dev_path is defined and compute.libvirt.rng_dev_path == '/dev/hwrng' %}
+create_hwrng_udev_rule_compute:
+ file.managed:
+ - name: /etc/udev/rules.d/90-hwrng.rules
+ - source: salt://nova/files/90-hwrng.rules
+ - user: root
+ - group: root
+ - mode: 0644
+ - onlyif: test -c /dev/hwrng
+
+trigger_hwrng_udev_compute:
+ cmd.run:
+ - name: udevadm trigger /dev/hwrng
+ - onchanges:
+ - file: /etc/udev/rules.d/90-hwrng.rules
+
+{%- endif %}
+
+
{%- endif %}
diff --git a/nova/controller.sls b/nova/controller.sls
index 68a9bb4..013f40d 100644
--- a/nova/controller.sls
+++ b/nova/controller.sls
@@ -428,8 +428,9 @@
- sls: nova.db.offline_sync
nova_controller_map_instances:
- novang.map_instances:
+ novav21.instances_mapped_to_cell:
- name: 'cell1'
+ - timeout: {{ controller.get('mapped_instances_interval', 60) }}
{%- if grains.get('noservices') %}
- onlyif: /bin/false
{%- endif %}
@@ -486,4 +487,23 @@
{%- endif %}
+{%- if controller.get('libvirt', {}).rng_dev_path is defined and controller.libvirt.rng_dev_path == '/dev/hwrng' %}
+create_hwrng_udev_rule_controller:
+ file.managed:
+ - name: /etc/udev/rules.d/90-hwrng.rules
+ - source: salt://nova/files/90-hwrng.rules
+ - user: root
+ - group: root
+ - mode: 0644
+ - onlyif: test -c /dev/hwrng
+
+trigger_hwrng_udev_controller:
+ cmd.run:
+ - name: udevadm trigger /dev/hwrng
+ - onchanges:
+ - file: /etc/udev/rules.d/90-hwrng.rules
+
+{%- endif %}
+
+
{%- endif %}
diff --git a/nova/files/90-hwrng.rules b/nova/files/90-hwrng.rules
new file mode 100644
index 0000000..b817b2b
--- /dev/null
+++ b/nova/files/90-hwrng.rules
@@ -0,0 +1,2 @@
+SUBSYSTEM=="misc", KERNEL=="hw_random", NAME="hwrng", GROUP="kvm", MODE="0660"
+
diff --git a/nova/files/mitaka/nova-compute.conf.Debian b/nova/files/mitaka/nova-compute.conf.Debian
index b570570..05ec5d1 100644
--- a/nova/files/mitaka/nova-compute.conf.Debian
+++ b/nova/files/mitaka/nova-compute.conf.Debian
@@ -47,6 +47,10 @@
cpu_allocation_ratio = {{ compute.cpu_allocation_ratio }}
{%- endif %}
+{%- if compute.disk_allocation_ratio is defined %}
+disk_allocation_ratio = {{ compute.disk_allocation_ratio }}
+{%- endif %}
+
remove_unused_original_minimum_age_seconds=86400
image_service=nova.image.glance.GlanceImageService
diff --git a/nova/files/newton/nova-compute.conf.Debian b/nova/files/newton/nova-compute.conf.Debian
index 2540a43..c7ef466 100644
--- a/nova/files/newton/nova-compute.conf.Debian
+++ b/nova/files/newton/nova-compute.conf.Debian
@@ -63,6 +63,10 @@
cpu_allocation_ratio = {{ compute.cpu_allocation_ratio }}
{%- endif %}
+{%- if compute.disk_allocation_ratio is defined %}
+disk_allocation_ratio = {{ compute.disk_allocation_ratio }}
+{%- endif %}
+
#
# Defines which physical CPUs (pCPUs) can be used by instance
# virtual CPUs (vCPUs).
diff --git a/nova/files/ocata/nova-compute.conf.Debian b/nova/files/ocata/nova-compute.conf.Debian
index 3c069ed..44bf3f1 100644
--- a/nova/files/ocata/nova-compute.conf.Debian
+++ b/nova/files/ocata/nova-compute.conf.Debian
@@ -610,7 +610,11 @@
# * Any valid positive integer or float value
# (floating point value)
# Minimum value: 0
+{%- if compute.disk_allocation_ratio is defined %}
+disk_allocation_ratio = {{ compute.disk_allocation_ratio }}
+{%- else %}
#disk_allocation_ratio=0.0
+{%- endif %}
#
# Console proxy host to be used to connect to instances on this host. It is the
@@ -9046,6 +9050,24 @@
#
# From nova.conf
#
+{%- if compute.get('service_user', {}).get('enabled', True) %}
+send_service_user_token = True
+auth_type = password
+{%- set _data = {} %}
+{%- do _data.update(compute.get('identity', {})) %}
+{%- do _data.update(compute.get('service_user', {})) %}
+{%- if not _data.port == '5000' %}{% do _data.update({'port': '5000'}) %}{% endif %}
+{%- if 'cacert_file' not in _data.keys() %}{% do _data.update({'cacert_file': compute.cacert_file}) %}{% endif %}
+user_domain_id = {{ _data.get('domain', 'default') }}
+project_domain_id = {{ _data.get('domain', 'default') }}
+project_name = {{ _data.get('tenant', 'service') }}
+username = {{ _data.get('user', 'nova') }}
+password = {{ _data.password }}
+auth_url={{ _data.get('protocol', 'http') }}://{{ _data.host }}:{{ _data.port }}
+ {%- if _data.get('protocol', 'http') == 'https' %}
+cafile={{ _data.cacert_file }}
+ {%- endif %}
+{%- endif %}
#
# When True, if sending a user token to an REST API, also send a service token.
diff --git a/nova/files/ocata/nova-controller.conf.Debian b/nova/files/ocata/nova-controller.conf.Debian
index 23db54e..b7c38ff 100644
--- a/nova/files/ocata/nova-controller.conf.Debian
+++ b/nova/files/ocata/nova-controller.conf.Debian
@@ -9025,6 +9025,24 @@
#
# From nova.conf
#
+{%- if controller.get('service_user', {}).get('enabled', True) %}
+send_service_user_token = True
+auth_type = password
+{%- set _data = {} %}
+{%- do _data.update(controller.get('identity', {})) %}
+{%- do _data.update(controller.get('service_user', {})) %}
+{%- if not _data.port == '5000' %}{% do _data.update({'port': '5000'}) %}{% endif %}
+{%- if 'cacert_file' not in _data.keys() %}{% do _data.update({'cacert_file': controller.cacert_file}) %}{% endif %}
+user_domain_id = {{ _data.get('domain', 'default') }}
+project_domain_id = {{ _data.get('domain', 'default') }}
+project_name = {{ _data.get('tenant', 'service') }}
+username = {{ _data.get('user', 'nova') }}
+password = {{ _data.password }}
+auth_url={{ _data.get('protocol', 'http') }}://{{ _data.host }}:{{ _data.port }}
+ {%- if _data.get('protocol', 'http') == 'https' %}
+cafile={{ _data.cacert_file }}
+ {%- endif %}
+{%- endif %}
#
# When True, if sending a user token to an REST API, also send a service token.
diff --git a/nova/files/pike/nova-compute.conf.Debian b/nova/files/pike/nova-compute.conf.Debian
index 67bb430..314bf29 100644
--- a/nova/files/pike/nova-compute.conf.Debian
+++ b/nova/files/pike/nova-compute.conf.Debian
@@ -618,7 +618,11 @@
# * Any valid positive integer or float value
# (floating point value)
# Minimum value: 0
+{%- if compute.disk_allocation_ratio is defined %}
+disk_allocation_ratio = {{ compute.disk_allocation_ratio }}
+{%- else %}
#disk_allocation_ratio=0.0
+{%- endif %}
#
# Console proxy host to be used to connect to instances on this host. It is the
@@ -6550,6 +6554,9 @@
# A path to a device that will be used as source of entropy on the host.
# Permitted options are: /dev/random or /dev/hwrng (string value)
#rng_dev_path=<None>
+{%- if compute.get('libvirt', {}).rng_dev_path is defined %}
+rng_dev_path={{ compute.libvirt.rng_dev_path }}
+{%- endif %}
# For qemu or KVM guests, set this option to specify a default machine type per
# host architecture. You can find a list of supported machine types in your
@@ -9245,6 +9252,24 @@
#
# From nova.conf
#
+{%- if compute.get('service_user', {}).get('enabled', True) %}
+send_service_user_token = True
+auth_type = password
+{%- set _data = {} %}
+{%- do _data.update(compute.get('identity', {})) %}
+{%- do _data.update(compute.get('service_user', {})) %}
+{%- if not _data.port == '5000' %}{% do _data.update({'port': '5000'}) %}{% endif %}
+{%- if 'cacert_file' not in _data.keys() %}{% do _data.update({'cacert_file': compute.cacert_file}) %}{% endif %}
+user_domain_id = {{ _data.get('domain', 'default') }}
+project_domain_id = {{ _data.get('domain', 'default') }}
+project_name = {{ _data.get('tenant', 'service') }}
+username = {{ _data.get('user', 'nova') }}
+password = {{ _data.password }}
+auth_url={{ _data.get('protocol', 'http') }}://{{ _data.host }}:{{ _data.port }}
+ {%- if _data.get('protocol', 'http') == 'https' %}
+cafile={{ _data.cacert_file }}
+ {%- endif %}
+{%- endif %}
#
# When True, if sending a user token to an REST API, also send a service token.
diff --git a/nova/files/pike/nova-controller.conf.Debian b/nova/files/pike/nova-controller.conf.Debian
index a61c236..3bdcfd6 100644
--- a/nova/files/pike/nova-controller.conf.Debian
+++ b/nova/files/pike/nova-controller.conf.Debian
@@ -6525,6 +6525,9 @@
# A path to a device that will be used as source of entropy on the host.
# Permitted options are: /dev/random or /dev/hwrng (string value)
#rng_dev_path=<None>
+{%- if controller.get('libvirt', {}).rng_dev_path is defined %}
+rng_dev_path={{ controller.libvirt.rng_dev_path }}
+{%- endif %}
# For qemu or KVM guests, set this option to specify a default machine type per
# host architecture. You can find a list of supported machine types in your
@@ -9235,6 +9238,24 @@
#
# From nova.conf
#
+{%- if controller.get('service_user', {}).get('enabled', True) %}
+send_service_user_token = True
+auth_type = password
+{%- set _data = {} %}
+{%- do _data.update(controller.get('identity', {})) %}
+{%- do _data.update(controller.get('service_user', {})) %}
+{%- if not _data.port == '5000' %}{% do _data.update({'port': '5000'}) %}{% endif %}
+{%- if 'cacert_file' not in _data.keys() %}{% do _data.update({'cacert_file': controller.cacert_file}) %}{% endif %}
+user_domain_id = {{ _data.get('domain', 'default') }}
+project_domain_id = {{ _data.get('domain', 'default') }}
+project_name = {{ _data.get('tenant', 'service') }}
+username = {{ _data.get('user', 'nova') }}
+password = {{ _data.password }}
+auth_url={{ _data.get('protocol', 'http') }}://{{ _data.host }}:{{ _data.port }}
+ {%- if _data.get('protocol', 'http') == 'https' %}
+cafile={{ _data.cacert_file }}
+ {%- endif %}
+{%- endif %}
#
# When True, if sending a user token to an REST API, also send a service token.
diff --git a/nova/files/queens/nova-compute.conf.Debian b/nova/files/queens/nova-compute.conf.Debian
index 869bd94..4b85864 100644
--- a/nova/files/queens/nova-compute.conf.Debian
+++ b/nova/files/queens/nova-compute.conf.Debian
@@ -666,7 +666,11 @@
# * Any valid positive integer or float value
# (floating point value)
# Minimum value: 0
+{%- if compute.disk_allocation_ratio is defined %}
+disk_allocation_ratio = {{ compute.disk_allocation_ratio }}
+{%- else %}
#disk_allocation_ratio = 0.0
+{%- endif %}
#
# Console proxy host to be used to connect to instances on this host.
@@ -7058,6 +7062,9 @@
# host. Permitted options are: /dev/random or /dev/hwrng (string
# value)
#rng_dev_path = <None>
+{%- if compute.get('libvirt', {}).rng_dev_path is defined %}
+rng_dev_path={{ compute.libvirt.rng_dev_path }}
+{%- endif %}
# For qemu or KVM guests, set this option to specify a default machine
# type per host architecture. You can find a list of supported machine
@@ -9166,6 +9173,15 @@
# middleware.
# (boolean value)
#send_service_user_token = false
+{%- if compute.get('service_user', {}).get('enabled', True) %}
+send_service_user_token = True
+{%- set _data = {} %}
+{%- do _data.update(compute.get('identity', {})) %}
+{%- do _data.update(compute.get('service_user', {})) %}
+{%- if not _data.port == '5000' %}{% do _data.update({'port': '5000'}) %}{% endif %}
+{%- if 'cacert_file' not in _data.keys() %}{% do _data.update({'cacert_file': compute.cacert_file}) %}{% endif %}
+{%- include "oslo_templates/files/queens/keystoneauth/_type_"+ _data.get('auth_type','password') +".conf" %}
+{%- else %}
# PEM encoded Certificate Authority to use when verifying HTTPs
# connections. (string value)
@@ -9249,6 +9265,7 @@
# Tenant Name (string value)
#tenant_name = <None>
+{%- endif %}
[spice]
diff --git a/nova/files/queens/nova-controller.conf.Debian b/nova/files/queens/nova-controller.conf.Debian
index c90b047..621120d 100644
--- a/nova/files/queens/nova-controller.conf.Debian
+++ b/nova/files/queens/nova-controller.conf.Debian
@@ -655,7 +655,11 @@
# * Any valid positive integer or float value
# (floating point value)
# Minimum value: 0
+{%- if controller.disk_allocation_ratio is defined %}
+disk_allocation_ratio = {{ controller.disk_allocation_ratio }}
+{%- else %}
#disk_allocation_ratio = 0.0
+{%- endif %}
#
# Console proxy host to be used to connect to instances on this host.
@@ -6906,6 +6910,9 @@
# host. Permitted options are: /dev/random or /dev/hwrng (string
# value)
#rng_dev_path = <None>
+{%- if controller.get('libvirt', {}).rng_dev_path is defined %}
+rng_dev_path={{ controller.libvirt.rng_dev_path }}
+{%- endif %}
# For qemu or KVM guests, set this option to specify a default machine
# type per host architecture. You can find a list of supported machine
@@ -8882,6 +8889,15 @@
# middleware.
# (boolean value)
#send_service_user_token = false
+{%- if controller.get('service_user', {}).get('enabled', True) %}
+send_service_user_token = True
+{%- set _data = {} %}
+{%- do _data.update(controller.get('identity', {})) %}
+{%- do _data.update(controller.get('service_user', {})) %}
+{%- if not _data.port == '5000' %}{% do _data.update({'port': '5000'}) %}{% endif %}
+{%- if 'cacert_file' not in _data.keys() %}{% do _data.update({'cacert_file': controller.cacert_file}) %}{% endif %}
+{%- include "oslo_templates/files/queens/keystoneauth/_type_"+ _data.get('auth_type','password') +".conf" %}
+{%- else %}
# PEM encoded Certificate Authority to use when verifying HTTPs
# connections. (string value)
@@ -8965,6 +8981,7 @@
# Tenant Name (string value)
#tenant_name = <None>
+{%- endif %}
[spice]
diff --git a/nova/map.jinja b/nova/map.jinja
index 370f517..d9fea7b 100644
--- a/nova/map.jinja
+++ b/nova/map.jinja
@@ -295,20 +295,6 @@
'warn': '15%',
'crit': '5%',
},
- 'error_log_rate': {
- 'warn': 0.2,
- },
- 'services_failed_warning_threshold_percent': 0.3,
- 'services_failed_critical_threshold_percent': 0.6,
- 'computes_failed_warning_threshold_percent': 0.25,
- 'computes_failed_critical_threshold_percent': 0.5,
- 'cpu_minor_threshold': 0.85,
- 'cpu_major_threshold': 0.95,
- 'ram_major_threshold': 0.85,
- 'ram_critical_threshold': 0.95,
- 'disk_major_threshold': 0.85,
- 'disk_critical_threshold': 0.95,
- 'endpoint_failed_major_threshold': 0.5,
},
}, grain='os_family', merge=salt['pillar.get']('nova:monitoring')) %}
diff --git a/nova/meta/prometheus.yml b/nova/meta/prometheus.yml
index 001a9ea..f3e12ee 100644
--- a/nova/meta/prometheus.yml
+++ b/nova/meta/prometheus.yml
@@ -1,11 +1,10 @@
-{% from "nova/map.jinja" import controller, compute, monitoring with context %}
+{% from "nova/map.jinja" import controller, compute with context %}
{%- set is_controller = controller.get('enabled', False) %}
{%- set is_compute = compute.get('enabled', False) %}
{%- if is_controller or is_compute %}
-{%- if is_compute and
- exporters is defined %}
+{%- if is_compute and exporters is defined and compute.get('compute_driver', 'libvirt.LibvirtDriver') == 'libvirt.LibvirtDriver' %}
{%- set packages = exporters.get('libvirt', {}).get('packages', ('libvirt-exporter', )) %}
{%- load_yaml as new_exporters_cfg %}
exporters:
@@ -30,11 +29,6 @@
server:
alert:
{%- if is_controller %}
-{%- set minor_threshold = monitoring.services_failed_warning_threshold_percent|float %}
-{%- set major_threshold = monitoring.services_failed_critical_threshold_percent|float %}
-{%- set minor_compute_threshold = monitoring.computes_failed_warning_threshold_percent|float %}
-{%- set major_compute_threshold = monitoring.computes_failed_critical_threshold_percent|float %}
-{%- set major_endpoint_threshold = monitoring.endpoint_failed_major_threshold|float %}
{% raw %}
NovaApiOutage:
if: >-
@@ -67,18 +61,17 @@
summary: "nova-api endpoint is not accessible"
description: >-
The nova-api endpoint on the {{ $labels.host }} node is not accessible for 2 minutes.
-{%- endraw %}
NovaApiEndpointsDownMajor:
if: >-
- count(http_response_status{name=~"nova-api"} == 0) >= count(http_response_status{name=~"nova-api"}) * {{ major_endpoint_threshold }}
+ count(http_response_status{name=~"nova-api"} == 0) >= count(http_response_status{name=~"nova-api"}) * 0.6
for: 2m
labels:
severity: major
service: nova
annotations:
- summary: "{{major_endpoint_threshold * 100}}% of nova-api endpoints are not accessible"
+ summary: "60% of nova-api endpoints are not accessible"
description: >-
- {% raw %}{{ $value }} nova-api endpoints (>= {% endraw %} {{major_endpoint_threshold * 100}}{% raw %}%) are not accessible for 2 minutes.
+ More than 60% of nova-api endpoints are not accessible for 2 minutes.
NovaApiEndpointsOutage:
if: >-
count(http_response_status{name=~"nova-api"} == 0) == count(http_response_status{name=~"nova-api"})
@@ -100,47 +93,46 @@
summary: "{{ $labels.binary }} service is down"
description: >-
The {{ $labels.binary }} service on the {{ $labels.hostname }} node is down.
-{%- endraw %}
NovaServicesDownMinor:
if: >-
- count(openstack_nova_service_state{binary!~"nova-compute"} == 0) by (binary) >= on (binary) count(openstack_nova_service_state{binary!~"nova-compute"}) by (binary) * {{minor_threshold}}
+ count(openstack_nova_service_state{binary!~"nova-compute"} == 0) by (binary) >= on (binary) count(openstack_nova_service_state{binary!~"nova-compute"}) by (binary) * 0.3
labels:
severity: minor
service: nova
annotations:
- summary: "{{minor_threshold * 100}}%{%- raw %} of {{ $labels.binary }} services are down"
+ summary: "30% of {{ $labels.binary }} services are down"
description: >-
- {{ $value }} {{ $labels.binary }} services (>= {%- endraw %} {{minor_threshold * 100}}%) are down.
+ More than 30% {{ $labels.binary }} services are down.
NovaComputeServicesDownMinor:
if: >-
- count(openstack_nova_service_state{binary="nova-compute"} == 0) >= count(openstack_nova_service_state{binary="nova-compute"}) * {{minor_compute_threshold}}
+ count(openstack_nova_service_state{binary="nova-compute"} == 0) >= count(openstack_nova_service_state{binary="nova-compute"}) * 0.25
labels:
severity: minor
service: nova
annotations:
- summary: "{{minor_compute_threshold * 100}}%{%- raw %} of nova-compute services are down"
+ summary: "More than 25% of nova-compute services are down"
description: >-
- {{ $value }} nova-compute services (>= {%- endraw %} {{minor_compute_threshold * 100}}%) are down.
+ More than 25% of nova-compute services are down.
NovaServicesDownMajor:
if: >-
- count(openstack_nova_service_state{binary!~"nova-compute"} == 0) by (binary) >= on (binary) count(openstack_nova_service_state{binary!~"nova-compute"}) by (binary) * {{major_threshold}}
+ count(openstack_nova_service_state{binary!~"nova-compute"} == 0) by (binary) >= on (binary) count(openstack_nova_service_state{binary!~"nova-compute"}) by (binary) * 0.6
labels:
severity: major
service: nova
annotations:
- summary: "{{major_threshold * 100}}%{%- raw %} of {{ $labels.binary }} services are down"
+ summary: "More than 60% of {{ $labels.binary }} services are down"
description: >-
- {{ $value }} {{ $labels.binary }} services (>= {%- endraw %} {{major_threshold * 100}}%) are down.
+ More than 60% of {{ $labels.binary }} services are down.
NovaComputeServicesDownMajor:
if: >-
- count(openstack_nova_service_state{binary="nova-compute"} == 0) >= count(openstack_nova_service_state{binary="nova-compute"}) * {{major_compute_threshold}}
+ count(openstack_nova_service_state{binary="nova-compute"} == 0) >= count(openstack_nova_service_state{binary="nova-compute"}) * 0.5
labels:
severity: major
service: nova
annotations:
- summary: "{{major_compute_threshold * 100}}%{%- raw %} of nova-compute services are down"
+ summary: "More than 50% of nova-compute services are down"
description: >-
- {{ $value }} nova-compute services (>= {%- endraw %} {{major_compute_threshold * 100}}{%- raw %}%) are down.
+ More than 50% of nova-compute services are down.
NovaServiceOutage:
if: >-
count(openstack_nova_service_state == 0) by (binary) == on (binary) count(openstack_nova_service_state) by (binary)
@@ -152,170 +144,19 @@
description: >-
All {{ $labels.binary }} services are down.
{%- endraw %}
-{%- set cpu_minor_threshold = monitoring.cpu_minor_threshold|float %}
-{%- set cpu_major_threshold = monitoring.cpu_major_threshold|float %}
-{%- set ram_major_threshold = monitoring.ram_major_threshold|float %}
-{%- set ram_critical_threshold = monitoring.ram_critical_threshold|float %}
-{%- set disk_major_threshold = monitoring.disk_major_threshold|float %}
-{%- set disk_critical_threshold = monitoring.disk_critical_threshold|float %}
- NovaHypervisorVCPUsFullMinor:
- if: >-
- label_replace(system_load15, "hostname", "$1", "host", "(.*)") > on (hostname) openstack_nova_vcpus * {{ cpu_minor_threshold }}
- labels:
- severity: minor
- service: nova
- annotations:
- summary: "{{ cpu_minor_threshold * 100 }}% of hypervisor VCPUs are used"
- description: "{% raw %}{{ $value }} VCPUs on the {{ $labels.hostname }} node (> {% endraw %} {{ cpu_minor_threshold * 100 }}%) are used."
- NovaHypervisorVCPUsFullMajor:
- if: >-
- label_replace(system_load15, "hostname", "$1", "host", "(.*)") > on (hostname) openstack_nova_vcpus * {{ cpu_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ cpu_major_threshold * 100 }}% of hypervisor VCPUs are used"
- description: "{% raw %}{{ $value }} VCPUs on the {{ $labels.hostname }} node (> {% endraw %} {{ cpu_major_threshold * 100 }}%) are used."
- NovaHypervisorMemoryFullMajor:
- if: >-
- openstack_nova_used_ram > openstack_nova_ram * {{ ram_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ ram_major_threshold * 100 }}% of hypervisor RAM is used"
- description: "{% raw %}{{ $value }}MB of RAM on the {{ $labels.hostname }} node (> {% endraw %} {{ ram_major_threshold * 100 }}%) is used."
- NovaHypervisorMemoryFullCritical:
- if: >-
- openstack_nova_used_ram > openstack_nova_ram * {{ ram_critical_threshold }}
- labels:
- severity: critical
- service: nova
- annotations:
- summary: "{{ ram_critical_threshold * 100 }}% of hypervisor RAM is used"
- description: "{% raw %}{{ $value }}MB of RAM on the {{ $labels.hostname }} node (> {% endraw %} {{ ram_critical_threshold * 100 }}%) is used."
- NovaHypervisorDiskFullMajor:
- if: >-
- openstack_nova_used_disk > openstack_nova_disk * {{ disk_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ disk_major_threshold * 100 }}% of hypervisor disk space is used"
- description: "{% raw %}{{ $value }}GB of disk space on the {{ $labels.hostname }} node (> {% endraw %} {{ disk_major_threshold * 100 }}%) is used."
- NovaHypervisorDiskFullCritical:
- if: >-
- openstack_nova_used_disk > openstack_nova_disk * {{ disk_critical_threshold }}
- labels:
- severity: critical
- service: nova
- annotations:
- summary: "{{ disk_critical_threshold * 100 }}% of hypervisor disk space is used"
- description: "{% raw %}{{ $value }}GB of disk space on the {{ $labels.hostname }} node (> {% endraw %} {{ disk_critical_threshold * 100 }}%) is used."
- NovaAggregateMemoryFullMajor:
- if: >-
- openstack_nova_aggregate_used_ram > openstack_nova_aggregate_ram * {{ ram_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ ram_major_threshold * 100 }}% of aggregate RAM is used"
- description: "{% raw %}{{ $value }}MB of RAM on the {{ $labels.aggregate }} aggregate (> {% endraw %} {{ ram_major_threshold * 100 }}%) is used."
- NovaAggregateMemoryFullCritical:
- if: >-
- openstack_nova_aggregate_used_ram > openstack_nova_aggregate_ram * {{ ram_critical_threshold }}
- labels:
- severity: critical
- service: nova
- annotations:
- summary: "{{ ram_critical_threshold * 100 }}% of aggregate RAM is used"
- description: "{% raw %}{{ $value }}MB of RAM on the {{ $labels.aggregate }} aggregate (> {% endraw %} {{ ram_critical_threshold * 100 }}%) is used."
- NovaAggregateDiskFullMajor:
- if: >-
- openstack_nova_aggregate_used_disk > openstack_nova_aggregate_disk * {{ disk_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ disk_major_threshold * 100 }}% of aggregate disk space is used"
- description: "{% raw %}{{ $value }}GB of disk space on the {{ $labels.aggregate }} aggregate (> {% endraw %} {{ disk_major_threshold * 100 }}%) is used."
- NovaAggregateDiskFullCritical:
- if: >-
- openstack_nova_aggregate_used_disk > openstack_nova_aggregate_disk * {{ disk_critical_threshold }}
- labels:
- severity: critical
- service: nova
- annotations:
- summary: "{{ disk_critical_threshold * 100 }}% of aggregate disk space is used"
- description: "{% raw %}{{ $value }}GB of disk space on the {{ $labels.aggregate }} aggregate (> {% endraw %} {{ disk_critical_threshold * 100 }}%) is used."
- NovaTotalVCPUsFullMinor:
- if: >-
- sum(label_replace(system_load15, "hostname", "$1", "host", "(.*)") and on (hostname) openstack_nova_vcpus) > max(sum(openstack_nova_vcpus) by (instance)) * {{ cpu_minor_threshold }}
- labels:
- severity: minor
- service: nova
- annotations:
- summary: "{{ cpu_minor_threshold * 100 }}% of cloud VCPUs are used"
- description: "{% raw %}{{ $value }} VCPUs in the cloud (> {% endraw %} {{ cpu_minor_threshold * 100 }}%) are used."
- NovaTotalVCPUsFullMajor:
- if: >-
- sum(label_replace(system_load15, "hostname", "$1", "host", "(.*)") and on (hostname) openstack_nova_vcpus) > max(sum(openstack_nova_vcpus) by (instance)) * {{ cpu_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ cpu_major_threshold * 100 }}% of cloud VCPUs are used"
- description: "{% raw %}{{ $value }} VCPUs in the cloud (> {% endraw %} {{ cpu_major_threshold * 100 }}%) are used."
- NovaTotalMemoryFullMajor:
- if: >-
- openstack_nova_total_used_ram > openstack_nova_total_ram * {{ ram_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ ram_major_threshold * 100 }}% of cloud RAM is used"
- description: "{% raw %}{{ $value }}MB of RAM in the cloud (> {% endraw %} {{ ram_major_threshold * 100 }}%) is used."
- NovaTotalMemoryFullCritical:
- if: >-
- openstack_nova_total_used_ram > openstack_nova_total_ram * {{ ram_critical_threshold }}
- labels:
- severity: critical
- service: nova
- annotations:
- summary: "{{ ram_critical_threshold * 100 }}% of cloud RAM is used"
- description: "{% raw %}{{ $value }}MB of RAM in the cloud (> {% endraw %} {{ ram_critical_threshold * 100 }}%) is used."
- NovaTotalDiskFullMajor:
- if: >-
- openstack_nova_total_used_disk > openstack_nova_total_disk * {{ disk_major_threshold }}
- labels:
- severity: major
- service: nova
- annotations:
- summary: "{{ disk_major_threshold * 100 }}% of cloud disk space is used"
- description: "{% raw %}{{ $value }}GB of disk space in the cloud (> {% endraw %} {{ disk_major_threshold * 100 }}%) is used."
- NovaTotalDiskFullCritical:
- if: >-
- openstack_nova_total_used_disk > openstack_nova_total_disk * {{ disk_critical_threshold }}
- labels:
- severity: critical
- service: nova
- annotations:
- summary: "{{ disk_critical_threshold * 100 }}% of cloud disk space is used"
- description: "{% raw %}{{ $value }}GB of disk space in the cloud (> {% endraw %} {{ disk_critical_threshold * 100 }}%) is used."
{%- endif %}
- NovaErrorLogsTooHigh:
- {%- set log_threshold = monitoring.error_log_rate.warn|float %}
- if: >-
- sum(rate(log_messages{service="nova",level=~"(?i:(error|emergency|fatal))"}[5m])) without (level) > {{ log_threshold }}
{%- raw %}
+ NovaErrorLogsTooHigh:
+ if: >-
+ sum(rate(log_messages{service="nova",level=~"(?i:(error|emergency|fatal))"}[5m])) without (level) > 0.2
labels:
severity: warning
service: nova
annotations:
summary: "High number of errors in Nova logs"
- description: "The average per-second rate of errors in Nova logs on the {{ $labels.host }} node is {{ $value }} (as measured over the last 5 minutes)."
+ description: "The average rate of errors in Nova logs on the {{ $labels.host }} node is more than 0.2 error messages per second (as measured over the last 5 minutes)."
{%- endraw %}
-{%- if is_compute and exporters is defined %}
+{%- if is_compute and exporters is defined and compute.get('compute_driver', 'libvirt.LibvirtDriver') == 'libvirt.LibvirtDriver'%}
{%- raw %}
LibvirtDown:
if: >-
diff --git a/nova/upgrade/pre/init.sls b/nova/upgrade/pre/init.sls
index 9ed049a..0c5834e 100644
--- a/nova/upgrade/pre/init.sls
+++ b/nova/upgrade/pre/init.sls
@@ -16,7 +16,7 @@
/etc/nova/nova.conf:
file.managed:
- name: /etc/nova/nova.conf
- - source: salt://nova/files/{{ _data.version }}/nova-{{ type }}.conf.{{ grains.os_family }}
+ - source: salt://nova/files/{{ upgrade.old_release }}/nova-{{ type }}.conf.{{ grains.os_family }}
- template: jinja
{%- if controller.get('enabled') %}
diff --git a/tests/pillar/compute_cluster.sls b/tests/pillar/compute_cluster.sls
index 67e871a..6d38a36 100644
--- a/tests/pillar/compute_cluster.sls
+++ b/tests/pillar/compute_cluster.sls
@@ -13,6 +13,9 @@
vnc_keymap: en-gb
resume_guests_state_on_host_boot: True
preallocate_images: space
+ cpu_allocation_ratio: 16.0
+ ram_allocation_ratio: 1.5
+ disk_allocation_ratio: 1.0
bind:
vnc_address: 127.0.0.1
vnc_port: 6080
diff --git a/tests/pillar/compute_cluster_vmware.sls b/tests/pillar/compute_cluster_vmware.sls
index 80954dd..4e57b01 100644
--- a/tests/pillar/compute_cluster_vmware.sls
+++ b/tests/pillar/compute_cluster_vmware.sls
@@ -21,6 +21,13 @@
user: nova
password: password
tenant: service
+ service_user:
+ enabled: True
+ user_domain_id: default
+ project_domain_id: default
+ project_name: service
+ username: nova
+ password: pswd
logging:
log_appender: false
log_handlers:
diff --git a/tests/pillar/compute_single.sls b/tests/pillar/compute_single.sls
index f92fa3b..b8754f6 100644
--- a/tests/pillar/compute_single.sls
+++ b/tests/pillar/compute_single.sls
@@ -8,6 +8,9 @@
resume_guests_state_on_host_boot: False
preallocate_images: space
cpu_mode: custom
+ cpu_allocation_ratio: 16.0
+ ram_allocation_ratio: 1.5
+ disk_allocation_ratio: 1.0
libvirt:
cpu_model: IvyBridge
bind:
diff --git a/tests/pillar/control_cluster.sls b/tests/pillar/control_cluster.sls
index e3bf663..97e8e43 100644
--- a/tests/pillar/control_cluster.sls
+++ b/tests/pillar/control_cluster.sls
@@ -32,6 +32,14 @@
user: nova
password: password
tenant: service
+ service_user:
+ enabled: True
+ user_domain_id: default
+ project_domain_id: default
+ project_name: service
+ username: nova
+ password: pswd
+
logging:
log_appender: false
log_handlers:
diff --git a/tests/pillar/repo_mcp_openstack_queens.sls b/tests/pillar/repo_mcp_openstack_queens.sls
new file mode 100644
index 0000000..65fb320
--- /dev/null
+++ b/tests/pillar/repo_mcp_openstack_queens.sls
@@ -0,0 +1,12 @@
+linux:
+ system:
+ enabled: true
+ repo:
+ mirantis_openstack_repo:
+ source: "deb http://mirror.mirantis.com/nightly/openstack-queens/{{ grains.get('oscodename') }} {{ grains.get('oscodename') }} main"
+ architectures: amd64
+ key_url: "http://mirror.mirantis.com/nightly/openstack-queens/{{ grains.get('oscodename') }}/archive-queens.key"
+ pin:
+ - pin: 'release l=queens'
+ priority: 1050
+ package: '*'