commit 8bb2942eafdcae16b49daeb4f0deba600c389748
author Ales Komarek <akomarek@mirantis.com> Mon Sep 04 07:47:55 2017 +0000
committer Gerrit Code Review <gerrit2@b3fef64565ee> Mon Sep 04 07:47:55 2017 +0000
tree 54eea4ae3af0d16b68bc7d70d7cac5f0e8d8b53b
parent c6d3ec427113af29d5a6779d33028f07fd30567e
parent a77dcbbbf0f4462081e3eae92f1bcf788357100e

Merge "Add Prometheus monitoring"

heka/_common.sls

diff --git a/heka/_common.sls b/heka/_common.sls
index a46b820..fc1a651 100644
--- a/heka/_common.sls
+++ b/heka/_common.sls
@@ -26,6 +26,11 @@
   - require:
     - user: heka_user
 
+/usr/local/bin/monitor_heka_queues.sh:
+  file.managed:
+  - source: salt://heka/files/monitor_heka_queues.sh
+  - mode: 755
+
 heka_user:
   user.present:
   - name: heka

heka/files/monitor_heka_queues.sh

diff --git a/heka/files/monitor_heka_queues.sh b/heka/files/monitor_heka_queues.sh
new file mode 100755
index 0000000..7360440
--- /dev/null
+++ b/heka/files/monitor_heka_queues.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -e
+#
+# This script reports the size of each Heka output queue.
+
+for q in /var/cache/*/output_queue/*;
+do
+  DATA=$(du -sb "$q")
+  SIZE=$(echo $DATA | cut -d " " -f 1)
+  QUEUE=$(echo $DATA | cut -d " " -f 2)
+  if [[ -n "${SIZE}" && -n "${QUEUE}" ]]; then
+    echo "heka_output_queue_size,queue=${QUEUE} value=${SIZE}"
+  fi
+done

heka/meta/prometheus.yml

diff --git a/heka/meta/prometheus.yml b/heka/meta/prometheus.yml
new file mode 100644
index 0000000..daf3bd8
--- /dev/null
+++ b/heka/meta/prometheus.yml
@@ -0,0 +1,13 @@
+server:
+  alert:
+    HekaOutputQueueStalled:
+      if: >-
+        delta(heka_output_queue_size[1h]) == 0
+      labels:
+        severity: warning
+        service: heka
+      annotations:
+        summary: 'Heka queue stalled'
+{%- raw %}
+        description: 'The {{ $labels.queue }} queue is stalled on node {{ $labels.host }} for more than 1 hour. The corresponding Heka service is either down or stuck.'
+{%- endraw %}

heka/meta/telegraf.yml

diff --git a/heka/meta/telegraf.yml b/heka/meta/telegraf.yml
new file mode 100644
index 0000000..d8c7b13
--- /dev/null
+++ b/heka/meta/telegraf.yml
@@ -0,0 +1,8 @@
+agent:
+  input:
+    monitor_heka:
+      template: telegraf/files/input/exec.conf
+      commands:
+      - /usr/local/bin/monitor_heka_queues.sh
+      data_format: influx
+      interval: 60s

metadata/service/support.yml

diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 14a0a96..15b34cb 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -5,7 +5,11 @@
         enabled: true
       heka:
         enabled: true
+      prometheus:
+        enabled: true
       sensu:
         enabled: true
       sphinx:
         enabled: true
+      telegraf:
+        enabled: true