From: Dmitry Kalashnik <dkalashnik@mirantis.com>
Date: Fri, 13 Oct 2017 09:58:35 +0000 (+0400)
Subject: Add threshold to ElasticSearch alarms
X-Git-Url: https://gerrit.mcp.mirantis.com/gitweb?p=salt-formulas%2Felasticsearch.git;a=commitdiff_plain;h=a2d8557f849bcf876b0a6cc7a8816b7928f2b79e

Add threshold to ElasticSearch alarms

Change-Id: Id25f13cad6e5646a9cdc8f21fb8af86533ae6f7f
Partial-Bug: PROD-15203
---

diff --git a/elasticsearch/map.jinja b/elasticsearch/map.jinja
index 23e2a7b..94f12fa 100644
--- a/elasticsearch/map.jinja
+++ b/elasticsearch/map.jinja
@@ -26,12 +26,21 @@ Debian:
   server:
     host: 127.0.0.1
     port: 9200
+    binary_port: 9300
 RedHat:
   pkgs:
   - python-elasticsearch
   server:
     host: 127.0.0.1
     port: 9200
+    binary_port: 9300
 {%- endload %}
 
 {%- set client = salt['grains.filter_by'](client_defaults, merge=salt['pillar.get']('elasticsearch:client')) %}
+
+{% set monitoring = salt['grains.filter_by']({
+    'default': {
+        'service_failed_warning_threshold_percent': 0.3,
+        'service_failed_critical_threshold_percent': 0.6,
+    },
+}, grain='os_family', merge=salt['pillar.get']('elasticsearch:monitoring')) %}
diff --git a/elasticsearch/meta/prometheus.yml b/elasticsearch/meta/prometheus.yml
index f0aa983..0e4d7e0 100644
--- a/elasticsearch/meta/prometheus.yml
+++ b/elasticsearch/meta/prometheus.yml
@@ -1,5 +1,5 @@
 {%- if pillar.elasticsearch.server is defined or pillar.elasticsearch.client is defined %}
-{%- from "elasticsearch/map.jinja" import server, client with context %}
+{%- from "elasticsearch/map.jinja" import server, client, monitoring with context %}
 
 server:
   alert:
@@ -29,15 +29,42 @@ server:
 {%- endif %}
 {%- if server.get('enabled', False) %}
 {%- raw %}
-    ElasticsearchDown:
+    ElasticsearchInfo:
       if: >-
-        elasticsearch_up{host=~'.*'} != 1
+        elasticsearch_up{host=~'.*'} == 0
       labels:
-        severity: warning
+        severity: info
         service: elasticsearch
       annotations:
-        summary: 'Elasticsearch service down'
+        summary: 'Elasticsearch service is down'
         description: 'Elasticsearch service is down on node {{ $labels.host }}'
+    ElasticsearchWarning:
+      if: >-
+        count(elasticsearch_up{host=~'.*'} == 0) >= count(elasticsearch_up{host=~'.*'}) * {% endraw %} {{ monitoring.service_failed_warning_threshold_percent }} {% raw %}
+      labels:
+        severity: warning
+        service: elasticsearch
+      annotations:
+        summary: 'More than {%- endraw %} {{monitoring.service_failed_warning_threshold_percent*100}}%{%- raw %} of Elasticsearch services are down'
+        description: 'More than {%- endraw %} {{monitoring.service_failed_warning_threshold_percent*100}}%{%- raw %} of Elasticsearch services are down'
+    ElasticsearchCritical:
+      if: >-
+        count(elasticsearch_up{host=~'.*'} == 0) >= count(elasticsearch_up{host=~'.*'}) * {% endraw %} {{ monitoring.service_failed_critical_threshold_percent }} {% raw %}
+      labels:
+        severity: critical
+        service: elasticsearch
+      annotations:
+        summary: 'More than {%- endraw %} {{monitoring.service_failed_critical_threshold_percent*100}}%{%- raw %} of Elasticsearch services are down'
+        description: 'More than {%- endraw %} {{monitoring.service_failed_critical_threshold_percent*100}}%{%- raw %} of Elasticsearch services are down'
+    ElasticsearchDown:
+      if: >-
+        count(elasticsearch_up{host=~'.*'} == 0) == count(elasticsearch_up{host=~'.*'})
+      labels:
+        severity: down
+        service: elasticsearch
+      annotations:
+        summary: 'All Elasticsearch services are down'
+        description: 'All Elasticsearch services are down'
     ElasticsearchClusterDiskLowWaterMark:
       if: >-
           (max(elasticsearch_fs_total_total_in_bytes) by (host, instance) - max(elasticsearch_fs_total_available_in_bytes) by (host, instance)) / max(elasticsearch_fs_total_total_in_bytes)  by (host, instance) * 100.0 >= 85
@@ -49,7 +76,6 @@ server:
         summary: 'Elasticsearch low disk watermark [85%] exceeded on node {{ $labels.host}} instance {{ $labels.instance }}'
         description: >-
           Elasticsearch will not allocate new shards to node {{ $labels.host }}
-
     ElasticsearchClusterDiskHighWaterMark:
       if: >-
           (max(elasticsearch_fs_total_total_in_bytes) by (host, instance) - max(elasticsearch_fs_total_available_in_bytes) by (host, instance)) / max(elasticsearch_fs_total_total_in_bytes) by (host, instance) * 100.0 >= 90