From a2af00ce2ca5a068b22b6b322bd165d35eb45d66 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Mon, 28 Nov 2016 17:39:25 +0100 Subject: [PATCH] Evaluate cluster alarms on the remote_collector This change also splits the collectd support metadata to collect node metrics from the local node(s) and cluster metrics from the remote collector node. --- .../files/collectd_elasticsearch_cluster.conf | 4 +- .../files/collectd_elasticsearch_node.conf | 6 +++ elasticsearch/meta/collectd.yml | 17 ++++++-- elasticsearch/meta/heka.yml | 42 ++++++++++++------- 4 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 elasticsearch/files/collectd_elasticsearch_node.conf diff --git a/elasticsearch/files/collectd_elasticsearch_cluster.conf b/elasticsearch/files/collectd_elasticsearch_cluster.conf index 49ddb85..bcd441f 100644 --- a/elasticsearch/files/collectd_elasticsearch_cluster.conf +++ b/elasticsearch/files/collectd_elasticsearch_cluster.conf @@ -1,6 +1,6 @@ -Import "elasticsearch_cluster" +Import "collectd_elasticsearch_cluster" - + Address "{{ plugin.address }}" Port "{{ plugin.port }}" diff --git a/elasticsearch/files/collectd_elasticsearch_node.conf b/elasticsearch/files/collectd_elasticsearch_node.conf new file mode 100644 index 0000000..a216f38 --- /dev/null +++ b/elasticsearch/files/collectd_elasticsearch_node.conf @@ -0,0 +1,6 @@ +Import "collectd_elasticsearch_node" + + + Address "{{ plugin.address }}" + Port "{{ plugin.port }}" + diff --git a/elasticsearch/meta/collectd.yml b/elasticsearch/meta/collectd.yml index 9da665f..c37f251 100644 --- a/elasticsearch/meta/collectd.yml +++ b/elasticsearch/meta/collectd.yml @@ -1,16 +1,25 @@ -{%- if pillar.elasticsearch.server is defined %} +{%- if pillar.elasticsearch.server is defined or pillar.elasticsearch.client is defined %} {%- from "elasticsearch/map.jinja" import server with context %} +{%- from "elasticsearch/map.jinja" import client with context %} {%- if server.get('enabled', False) %} local_plugin: - elasticsearch_cluster: + elasticsearch_node: plugin: python - template: elasticsearch/files/collectd_elasticsearch_cluster.conf - address: {{ server.get('bind', {}).get('address', '127.0.0.1') }} + template: elasticsearch/files/collectd_elasticsearch_node.conf + address: {{ server.get('bind', {}).get('address', '127.0.0.1')|replace('0.0.0.0', '127.0.0.1') }} port: {{ server.get('bind', {}).get('port', 9200) }} collectd_processes: process: elasticsearch: match: 'java.*elasticsearch' {%- endif %} +{%- if client.get('enabled', False) %} +remote_plugin: + elasticsearch_cluster: + plugin: python + template: elasticsearch/files/collectd_elasticsearch_cluster.conf + address: {{ client.server.host }} + port: {{ client.server.get('port', 9200) }} +{%- endif %} {%- endif %} diff --git a/elasticsearch/meta/heka.yml b/elasticsearch/meta/heka.yml index 90b3ae7..11e6c04 100644 --- a/elasticsearch/meta/heka.yml +++ b/elasticsearch/meta/heka.yml @@ -11,6 +11,25 @@ metric_collector: window: 60 periods: 0 function: last + alarm: + elasticsearch_check: + alerting: enabled + triggers: + - elasticsearch_check + dimension: + service: elasticsearch-node +remote_collector: + trigger: + elasticsearch_cluster_check: + description: 'Elasticsearch cluster cannot be checked' + severity: down + rules: + - metric: elasticsearch_cluster_check + relational_operator: '==' + threshold: 0 + window: 60 + periods: 0 + function: last elasticsearch_health_critical: description: 'Elasticsearch cluster health is critical' severity: critical @@ -30,19 +49,14 @@ metric_collector: window: 60 function: min alarm: - elasticsearch_check: - alerting: enabled - triggers: - - elasticsearch_check - dimension: - service: elasticsearch - elasticsearch_health: + elasticsearch_cluster_health: alerting: enabled triggers: + - elasticsearch_cluster_check - elasticsearch_health_critical - elasticsearch_health_warning dimension: - cluster: elasticsearch + service: elasticsearch-cluster aggregator: alarm_cluster: elasticsearch_service: @@ -50,27 +64,27 @@ aggregator: alerting: enabled group_by: hostname match: - service: elasticsearch + service: elasticsearch-node members: - elasticsearch_check dimension: - service: elasticsearch-cluster + service: elasticsearch nagios_host: 01-service-clusters elasticsearch_cluster: policy: highest_severity alerting: enabled match: - cluster: elasticsearch + service: elasticsearch-cluster members: - - elasticsearch_health + - elasticsearch_cluster_health dimension: - service: elasticsearch-cluster + service: elasticsearch nagios_host: 01-service-clusters elasticsearch: policy: highest_severity alerting: enabled_with_notification match: - service: elasticsearch-cluster + service: elasticsearch members: - elasticsearch_service - elasticsearch_cluster -- 2.32.7