commit 40db1693edac0ca36f21da1134c85a2fae39c053
author Ksawery Dziekoński <kdziekonski@mirantis.com> Mon Mar 02 15:57:47 2020 +0100
committer Ivan Berezovskiy <iberezovskiy@mirantis.com> Wed Mar 11 16:25:58 2020 +0400
tree 72dc0117fb71cc7f00f655e3381d2e595a6090c6
parent 6bd210f4bb65d3c70baef11b2712da4484f80a82

Rework DockerServiceMonitoring* alerts

Fixes PROD-33667

Change-Id: Id34dba59e6a0d0eef41e680d8c526ddb96ff8fd9

docker/meta/prometheus.yml

diff --git a/docker/meta/prometheus.yml b/docker/meta/prometheus.yml
index 328085d..5aeca2f 100644
--- a/docker/meta/prometheus.yml
+++ b/docker/meta/prometheus.yml
@@ -44,7 +44,7 @@
       {%- if service.get('deploy', {}).get('replicas', 1) > 1 %}
     DockerService{{ camel_case_name }}ReplicasDownMinor:
       if: >-
-        {{ service.deploy.replicas }} - min(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) >= {{ service.deploy.replicas }} * {{ monitoring.replicas_failed_warning_threshold_percent }}
+        min(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) / min(docker_swarm_tasks_desired{{ '{' + label_selector + '}' }}) <= {{ 1 - monitoring.replicas_failed_warning_threshold_percent }}
   {%- raw %}
       for: 2m
       labels:
@@ -56,7 +56,7 @@
   {%- endraw %}
     DockerService{{ camel_case_name }}ReplicasDownMajor:
       if: >-
-        {{ service.deploy.replicas }} - min(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) >= {{ service.deploy.replicas }} * {{ monitoring.replicas_failed_critical_threshold_percent }}
+        min(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) / min(docker_swarm_tasks_desired{{ '{' + label_selector + '}' }}) <= {{ 1 - monitoring.replicas_failed_critical_threshold_percent }}
   {%- raw %}
       for: 2m
       labels:
@@ -69,7 +69,7 @@
       {%- endif %}
     DockerService{{ camel_case_name }}Outage:
       if: >-
-        docker_swarm_tasks_desired{{ '{' + label_selector + '}' }} > 0 and (docker_swarm_tasks_running{{ '{' + label_selector + '}' }} == 0 or absent(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) == 1)
+        min(docker_swarm_tasks_desired{{ '{' + label_selector + '}' }}) > 0 and (min(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) == 0 or absent(docker_swarm_tasks_running{{ '{' + label_selector + '}' }}) == 1)
   {%- raw %}
       for: 2m
       labels:

tests/test-requirements.txt

diff --git a/tests/test-requirements.txt b/tests/test-requirements.txt
index 7188b2b..bc7dc7f 100644
--- a/tests/test-requirements.txt
+++ b/tests/test-requirements.txt
@@ -1,3 +1,4 @@
 jsonschema
 reno
 setuptools<45.0.0
+msgpack<1.0.0