Merge "Fix configuring log appender for cinder-volume on controllers"
diff --git a/_modules/cinderng.py b/_modules/cinderng.py
index 08aaad9..4d75116 100644
--- a/_modules/cinderng.py
+++ b/_modules/cinderng.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import logging
+import time
LOG = logging.getLogger(__name__)
@@ -45,6 +46,22 @@
}
return credentials
+def retries(func):
+ def func_wrapper(*args, **kwargs):
+ retries = kwargs.get('retries', 3)
+ timeout = kwargs.get('timeout', 5)
+ res = None
+ for i in range(retries):
+ try:
+ res = func(*args, **kwargs)
+ except Exception as e:
+ if i == retries - 1:
+ raise e
+ time.sleep(timeout)
+ else:
+ break
+ return res
+ return func_wrapper
def create_conn(cred=None):
"""
@@ -63,7 +80,7 @@
)
return nt
-
+@retries
def list_volumes(profile=None, **kwargs):
"""
Return list of cinder volumes.
@@ -72,7 +89,7 @@
nt = create_conn(cred)
return nt.volumes.list()
-
+@retries
def list_volume_type(profile=None, **kwargs):
"""
Return list of volume types
@@ -81,7 +98,7 @@
nt = create_conn(cred)
return nt.volume_types.list()
-
+@retries
def get_volume_type(type_name, profile=None, **kwargs):
"""
Returns id of the specified volume type name
@@ -103,7 +120,7 @@
else:
return
-
+@retries
def create_volume_type(type_name, profile=None, **kwargs):
"""
Create cinder volume type
@@ -120,7 +137,7 @@
else:
return 'exists'
-
+@retries
def get_keys_volume_type(type_name, profile=None, **kwargs):
"""
Return extra specs of the specified volume type.
@@ -135,7 +152,7 @@
else:
return
-
+@retries
def set_keys_volume_type(type_name, keys={}, profile=None, **kwargs):
"""
Set extra specs of the specified volume type.
diff --git a/cinder/meta/prometheus.yml b/cinder/meta/prometheus.yml
index 4894e50..133cc71 100644
--- a/cinder/meta/prometheus.yml
+++ b/cinder/meta/prometheus.yml
@@ -18,7 +18,7 @@
annotations:
summary: "Cinder API outage"
description: >-
- Cinder API check for the every available in the Openstack service catalog Cinder endpoint is down.
+ Cinder API is not accessible for all available Cinder endpoints in the OpenStack service catalog.
CinderAPIDown:
if: >-
openstack_api_check_status{name=~"cinder.*"} == 0
@@ -26,9 +26,9 @@
severity: major
service: cinder
annotations:
- summary: "'{{ $labels.name }}' endpoint is down"
+ summary: "{{ $labels.name }} endpoint is not accessible"
description: >-
- Cinder API check for the '{{ $labels.name }}' endpoint is down.
+ Cinder API is not accessible for the {{ $labels.name }} endpoint.
CinderAPIServiceDown:
if: >-
http_response_status{name=~"cinder-api"} == 0
@@ -37,9 +37,9 @@
severity: minor
service: cinder
annotations:
- summary: "The HTTP check for the 'cinder-api' is down"
+ summary: "Host cinder-api endpoint is not accessible"
description: >-
- The HTTP check for the 'cinder-api' on the '{{ $labels.host }}' is down for 2 minutes.
+ The host cinder-api endpoint on the {{ $labels.host }} node is not accessible for at least 2 minutes.
CinderServiceDown:
if: >-
openstack_cinder_service_state == 0
@@ -47,45 +47,40 @@
severity: minor
service: cinder
annotations:
- summary: "The '{{ $labels.binary }}' is in down state"
+ summary: "{{ $labels.binary }} service is down"
description: >-
- The '{{ $labels.binary }}' on the '{{ $labels.host }}' is in down state.
+ The {{ $labels.binary }} service on the {{ $labels.hostname }} node is down.
{%- endraw %}
CinderServicesDownMinor:
if: >-
count(openstack_cinder_service_state == 0) by (binary) >= on (binary) count(openstack_cinder_service_state) by (binary) * {{minor_threshold}} and count(openstack_cinder_service_state == 0) by (binary) < on (binary) count(openstack_cinder_service_state) by (binary) * {{major_threshold}}
-{%- raw %}
labels:
severity: minor
service: cinder
annotations:
- summary: "Medium percentage of '{{ $labels.binary }}' services are in down state"
+ summary: "{{minor_threshold * 100}}%{%- raw %} of {{ $labels.binary }} services are down"
description: >-
- {{ $value }} '{{ $labels.binary }}' services are in down state {%- endraw %}(More than {{minor_threshold * 100}}% and less than {{major_threshold * 100}}%).{%- raw %}
-{%- endraw %}
+ {{ $value }} {{ $labels.binary }} services are down {%- endraw %}(at least {{minor_threshold * 100}}%).
CinderServicesDownMajor:
if: >-
count(openstack_cinder_service_state == 0) by (binary) >= on (binary) count(openstack_cinder_service_state) by (binary) * {{major_threshold}}
-{%- raw %}
labels:
severity: major
service: cinder
annotations:
- summary: "High percentage of '{{ $labels.binary }}' services are in down state"
+ summary: "{{major_threshold * 100}}%{%- raw %} of {{ $labels.binary }} services are down"
description: >-
- {{ $value }} '{{ $labels.binary }}' services are in down state {%- endraw %}(More than {{major_threshold * 100}}%).{%- raw %}
-{%- endraw %}
+ {{ $value }} {{ $labels.binary }} services are down {%- endraw %}(at least {{major_threshold * 100}}%).{%- raw %}
CinderServiceOutage:
if: >-
count(openstack_cinder_service_state == 0) by (binary) == on (binary) count(openstack_cinder_service_state) by (binary)
-{%- raw %}
labels:
severity: critical
service: cinder
annotations:
- summary: "'{{ $labels.binary }}' service outage"
+ summary: "{{ $labels.binary }} service outage"
description: >-
- All '{{ $labels.binary }}' services are in down state.
+ All {{ $labels.binary }} services are down.
{%- endraw %}
{%- endif %}
CinderErrorLogsTooHigh:
@@ -97,6 +92,6 @@
severity: warning
service: cinder
annotations:
- summary: "There are too many errors in Cinder logs"
- description: "The rate of errors in Cinder logs over the last 5 minutes is too high on node '{{ $labels.host }}' (current value={{ $value }}, threshold={%- endraw %}{{ log_threshold }})."
+ summary: "High number of errors in Cinder logs"
+ description: "The rate of errors in Cinder logs over the last 5 minutes is too high on the {{ $labels.host }} node (current value={{ $value }}, threshold={%- endraw %}{{ log_threshold }})."
{%- endif %}