Heka stalled alert improvement
Constant queue size can be achieved by having services which are
not generating much logs
Change-Id: I52dd3487ce3ed3cee9dfbfc9c2fd16dd3be3eca1
diff --git a/heka/map.jinja b/heka/map.jinja
index e362fce..346671a 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -64,6 +64,7 @@
'sensu_notification_handler': default_sensu_notification_handler,
'sensu_port': default_sensu_port,
'sensu_watchdog_ttl': default_sensu_watchdog_ttl,
+ 'heka_queue_size_threshold': 134217728
}
}, merge=salt['pillar.get']('heka:log_collector')) %}
diff --git a/heka/meta/prometheus.yml b/heka/meta/prometheus.yml
index b772d6b..75b7d97 100644
--- a/heka/meta/prometheus.yml
+++ b/heka/meta/prometheus.yml
@@ -4,7 +4,7 @@
alert:
HekaOutputQueueStalled:
if: >-
- delta(heka_output_queue_size[1h]) == 0
+ heka_output_queue_size > {{ log_collector.heka_queue_size_threshold }}
labels:
severity: warning
service: heka