From 674a72b302feda33f21fdebd63dfdacf8e85c8f0 Mon Sep 17 00:00:00 2001
From: Mateusz Matuszkowiak <mmatuszkowiak@mirantis.co>
Date: Tue, 22 Jan 2019 15:34:36 +0100
Subject: [PATCH] Added check for read-only indices in ESv6

Change-Id: I25a9e33497d5cc6205c269ba8cc0db427827486f
Related-Bug: PROD-26666
---
 elasticsearch/files/v6/es_check_read_only.sh |  5 +++++
 elasticsearch/meta/prometheus.yml            | 12 ++++++++++++
 elasticsearch/meta/telegraf.yml              |  6 ++++++
 elasticsearch/server/init.sls                |  9 +++++++++
 4 files changed, 32 insertions(+)
 create mode 100755 elasticsearch/files/v6/es_check_read_only.sh

diff --git a/elasticsearch/files/v6/es_check_read_only.sh b/elasticsearch/files/v6/es_check_read_only.sh
new file mode 100755
index 0000000..55098f3
--- /dev/null
+++ b/elasticsearch/files/v6/es_check_read_only.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+{%- from "elasticsearch/map.jinja" import server with context %}
+{%- set bind = server.get('bind', {}) %}
+val=$(curl --silent -XGET {{ bind.address|default('127.0.0.1') }}:{{ bind.port|default(9200) }}/_all/_settings?pretty=true|grep -c -e "\"read_only_allow_delete\" : \"true\"")
+echo "{ \"elasticsearch_indices_read_only_allow_delete\": ${val} }"
diff --git a/elasticsearch/meta/prometheus.yml b/elasticsearch/meta/prometheus.yml
index 46ab505..6d0445f 100644
--- a/elasticsearch/meta/prometheus.yml
+++ b/elasticsearch/meta/prometheus.yml
@@ -28,6 +28,18 @@ server:
         summary: "Elasticsearch cluster status is RED"
         description: "The Elasticsearch cluster status is RED for 2 minutes."
   {%- endraw %}
+    ElasticsearchIndicesInReadOnlyMode:
+      if: >-
+        exec_elasticsearch_indices_read_only_allow_delete > 0
+  {%- raw %}
+      for: 2m
+      labels:
+        severity: critical
+        service: elasticsearch
+      annotations:
+        summary: "Elasticsearch indices are read-only"
+        description: "The Elasticsearch service has indices in read-only mode for 2 minutes."
+  {%- endraw %}
 {%- endif %}
 {%- if server.get('enabled', False) %}
     ElasticsearchServiceDown:
diff --git a/elasticsearch/meta/telegraf.yml b/elasticsearch/meta/telegraf.yml
index 6ccac3f..2fc3166 100644
--- a/elasticsearch/meta/telegraf.yml
+++ b/elasticsearch/meta/telegraf.yml
@@ -13,6 +13,12 @@ agent:
       cluster_health: false
       cluster_stats: true
       local: true
+    exec:
+      commands:
+      - "/usr/local/bin/elasticsearch_check_read_only.sh"
+      interval: "30s"
+      timeout: "15s"
+      data_format: "json"
 {%- endif %}
 
 {%- if client.get('enabled', False) %}
diff --git a/elasticsearch/server/init.sls b/elasticsearch/server/init.sls
index e8ae754..273e214 100644
--- a/elasticsearch/server/init.sls
+++ b/elasticsearch/server/init.sls
@@ -42,6 +42,15 @@ elasticsearch_logging:
     - pkg: elasticsearch_packages
 {%- endif %}
 
+{%- if server.version == 6 %}
+elasticsearch_telegraf_exec_plugin_script:
+  file.managed:
+  - name: /usr/local/bin/elasticsearch_check_read_only.sh
+  - source: salt://elasticsearch/files/v{{ server.version }}/es_check_read_only.sh
+  - template: jinja
+  - mode: 755
+{%- endif %}
+
 {%- if server.version in [5, 6] %}
 elasticsearch_logging:
   file.managed:
-- 
2.32.7