commit a5645075a79398f3b81559467ae0a85180dfc5df
author Swann Croiset <scroiset@mirantis.com> Tue Jun 13 11:55:23 2017 +0200
committer Simon Pasquier <spasquier@mirantis.com> Mon Jun 19 11:58:16 2017 +0200
tree 902507eaa040f3abf444bb654465033e849f7785
parent 517ec618b35d09ece6e9c730edd736f1f2cf75e6

Add Telegraf output for log_collector

This sends metrics derived from logs to Telegraf instead
of the metric_collector service.

Change-Id: I596cf2c5700e57df056de57cbc3fea1079d79773

heka/files/toml/output/http.toml

diff --git a/heka/files/toml/output/http.toml b/heka/files/toml/output/http.toml
index fce5d97..586bd97 100644
--- a/heka/files/toml/output/http.toml
+++ b/heka/files/toml/output/http.toml
@@ -3,7 +3,7 @@
 message_matcher = "{{ output.message_matcher }}"
 encoder = "{{ output.encoder }}"
 address = "{{ output.address }}"
-{%- if output.username and output.password %}
+{%- if output.username is defined and output.password is defined %}
 username = "{{ output.username }}"
 password = "{{ output.password }}"
 {%- endif %}

heka/meta/heka.yml

diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index 3c4db74..81bea3a 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -6,6 +6,17 @@
 
 log_collector:
   filter:
+{%- if log_collector.telegraf_host is defined %}
+    influxdb_accumulator:
+      engine: sandbox
+      module_file: /usr/share/lma_collector/filters/influxdb_accumulator.lua
+      module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
+      preserve_data: false
+      message_matcher: "Type =~ /metric$/"
+      ticker_interval: 1
+      config:
+        time_precision: "ms"
+{%- endif %}
     aggregated_http_metrics:
       engine: sandbox
       module_file: /usr/share/lma_collector/filters/http_metrics_aggregator.lua
@@ -57,8 +68,14 @@
       config:
         service_name: "log_collector"
 {%- endif %}
-{%- if log_collector.elasticsearch_host is defined or log_collector.sensu_host is defined %}
+{%- if log_collector.elasticsearch_host is defined or log_collector.sensu_host is defined or log_collector.telegraf_host is defined%}
   encoder:
+  {%- if log_collector.telegraf_host is defined %}
+    influxdb:
+      engine: payload
+      append_newlines: false
+      prefix_ts: false
+  {%- endif %}
 {%- if log_collector.elasticsearch_host %}
     elasticsearch:
       engine: elasticsearch
@@ -75,11 +92,20 @@
 {%- endif %}
 {%- endif %}
   output:
+  {%- if log_collector.telegraf_host is defined %}
+    influxdb:
+      engine: http
+      address: "http://{{ log_collector.telegraf_host }}:{{ log_collector.telegraf_port }}/write?precision=ms"
+      message_matcher: "Fields[payload_type] == 'txt' && Fields[payload_name] == 'influxdb'"
+      encoder: influxdb_encoder
+      timeout: {{ log_collector.telegraf_timeout }}
+  {%- else %}
     metric_collector:
       engine: tcp
       host: {{ log_collector.metric_collector_host }}
       port: {{ log_collector.metric_collector_port }}
       message_matcher: "(Type == 'metric' || Type == 'heka.sandbox.metric' || Type == 'heka.sandbox.bulk_metric')"
+  {%- endif %}
 {%- if log_collector.sensu_host is defined %}
     sensu_watchdog:
       engine: udp

metadata/service/log_collector/output/telegraf.yml

diff --git a/metadata/service/log_collector/output/telegraf.yml b/metadata/service/log_collector/output/telegraf.yml
new file mode 100644
index 0000000..99c4d6d
--- /dev/null
+++ b/metadata/service/log_collector/output/telegraf.yml
@@ -0,0 +1,6 @@
+parameters:
+  heka:
+    log_collector:
+      telegraf_host: 127.0.0.1
+      telegraf_port: 8186
+      telegraf_timeout: 2000