Merge pull request #32 from elemoine/stacklight-lua-tests
Add Lua tests
diff --git a/heka/_service.sls b/heka/_service.sls
index 63516c6..22ca02a 100644
--- a/heka/_service.sls
+++ b/heka/_service.sls
@@ -1,3 +1,4 @@
+{%- macro load_grains_file(grains_fragment_file) %}{% include grains_fragment_file ignore missing %}{% endmacro %}
{%- set server = salt['pillar.get']('heka:'+service_name) %}
@@ -124,8 +125,6 @@
{%- for service_name, service in pillar.iteritems() %}
{%- if service.get('_support', {}).get('heka', {}).get('enabled', False) %}
-{%- macro load_grains_file(grains_fragment_file) %}{% include grains_fragment_file ignore missing %}{% endmacro %}
-
{%- set grains_fragment_file = service_name+'/meta/heka.yml' %}
{%- set grains_yaml = load_grains_file(grains_fragment_file)|load_yaml %}
{%- set service_grains = salt['grains.filter_by']({'default': service_grains}, merge=grains_yaml) %}
@@ -136,15 +135,19 @@
{%- endif %}
-{# Loading the other services' support metadata from salt-mine #}
-
{%- if service_name in ['remote_collector', 'aggregator'] %}
+{# Load the support metadata from heka/meta/heka.yml #}
+
+{%- set grains_fragment_file = 'heka/meta/heka.yml' %}
+{%- set grains_yaml = load_grains_file(grains_fragment_file)|load_yaml %}
+{%- set service_grains = salt['grains.filter_by']({'default': service_grains}, merge=grains_yaml) %}
+
+{# Load the other services' support metadata from salt-mine #}
+
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
{%- if node_grains.heka is defined %}
-
-{%- do service_grains.update(node_grains.heka) %}
-
+{%- set service_grains = salt['grains.filter_by']({'default': service_grains}, merge=node_grains.heka) %}
{%- endif %}
{%- endfor %}
@@ -264,9 +267,7 @@
- mode: 640
- group: heka
- require:
- - file: heka_{{ service_name }}_conf_dir
- - require_in:
- - file: heka_{{ service_name }}_conf_dir_clean
+ - file: /usr/share/lma_collector
- watch_in:
- service: heka_{{ service_name }}_service
- defaults:
diff --git a/heka/files/toml/decoder/sandbox.toml b/heka/files/toml/decoder/sandbox.toml
index f533ab4..c073991 100644
--- a/heka/files/toml/decoder/sandbox.toml
+++ b/heka/files/toml/decoder/sandbox.toml
@@ -1,3 +1,5 @@
+{%- from "heka/map.jinja" import server with context %}
+
[{{ decoder_name }}_decoder]
type = "SandboxDecoder"
filename = "{{ decoder.module_file }}"
@@ -17,9 +19,12 @@
output_limit = "{{ decoder.output_limit }}"
{%- endif %}
-{%- if decoder.config is defined %}
+{%- if decoder.config is defined or decoder.get('adjust_timezone', False) %}
[{{ decoder_name }}_decoder.config]
-{%- for config_param, config_value in decoder.config.iteritems() %}
+{%- for config_param, config_value in decoder.get('config', {}).iteritems() %}
{{ config_param }} = {% if config_value is string %}"{{ config_value }}"{% elif config_value in [True, False] %}{{ config_value|lower }}{% else %}{{ config_value }}{% endif %}
{%- endfor %}
+{%- if decoder.get('adjust_timezone') and server.timezone is defined %}
+tz = "{{ server.timezone }}"
+{%- endif %}
{%- endif %}
diff --git a/heka/files/toml/output/aggregator.toml b/heka/files/toml/output/aggregator.toml
new file mode 100644
index 0000000..1eedea7
--- /dev/null
+++ b/heka/files/toml/output/aggregator.toml
@@ -0,0 +1,7 @@
+{%- extends "heka/files/toml/output/tcp.toml" %}
+{%- block address -%}
+address = "{{ output.host }}:5565"
+{%- endblock %}
+{%- block message_matcher -%}
+message_matcher = "Fields[aggregator] == NIL && Type == 'heka.sandbox.afd_metric'"
+{%- endblock %}
diff --git a/heka/files/toml/output/tcp.toml b/heka/files/toml/output/tcp.toml
index dee7fa0..c4115df 100644
--- a/heka/files/toml/output/tcp.toml
+++ b/heka/files/toml/output/tcp.toml
@@ -1,8 +1,12 @@
[{{ output_name }}_output]
type="TcpOutput"
+{% block address %}
address = "{{ output.host }}:{{ output.port }}"
+{% endblock %}
encoder = "ProtobufEncoder"
+{% block message_matcher %}
message_matcher = "{{ output.message_matcher }}"
+{% endblock %}
use_buffering = true
[{{ output_name }}_output.buffering]
diff --git a/heka/map.jinja b/heka/map.jinja
index 841b34a..f7e9d6e 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -17,6 +17,9 @@
extra_fields:
environment_label: {{ grains.domain }}
influxdb_time_precision: ms
+ {%- if pillar.get('linux', {}).get('system', {}).timezone is defined %}
+ timezone: "{{ pillar.linux.system.timezone }}"
+ {%- endif %}
RedHat:
groups:
- adm
diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index 7fee988..fbd847a 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -97,20 +97,63 @@
port: 4353
ticker_interval: 30
remote_collector:
+ decoder:
+ collectd:
+ engine: sandbox
+ module_file: /usr/share/lma_collector/decoders/collectd.lua
+ module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
+ config:
+ hostname: '{{ grains.host }}'
+ input:
+ heka_collectd:
+ engine: http
+ address: 127.0.0.1
+ port: 8326
+ decoder: collectd_decoder
+ splitter: NullSplitter
+ filter:
+ influxdb_accumulator:
+ engine: sandbox
+ module_file: /usr/share/lma_collector/filters/influxdb_accumulator.lua
+ module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
+ preserve_data: false
+ message_matcher: "Type =~ /metric$/"
+ ticker_interval: 1
+ config:
+ tag_fields: "deployment_id environment_label tenant_id user_id"
+ time_precision: "{{ server.influxdb_time_precision }}"
+ encoder:
+ influxdb:
+ engine: payload
+ append_newlines: false
+ prefix_ts: false
output:
- metric_collector:
- engine: tcp
- host: 127.0.0.1
- port: 5567
remote_collector_dashboard:
engine: dashboard
host: 127.0.0.1
port: 4354
- ticker_interval: 30
+ ticker_interval: 30
aggregator:
- decoder: {}
- input: {}
- filter: {}
- output: {}
- splitter: {}
- encoder: {}
+ input:
+ heka_metric:
+ engine: tcp
+ address: 0.0.0.0
+ port: 5565
+ decoder: ProtobufDecoder
+ splitter: HekaFramingSplitter
+ filter:
+ influxdb_accumulator:
+ engine: sandbox
+ module_file: /usr/share/lma_collector/filters/influxdb_accumulator.lua
+ module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
+ preserve_data: false
+ message_matcher: "Type == 'heka.sandbox.gse_metric'"
+ ticker_interval: 1
+ config:
+ tag_fields: "deployment_id environment_label tenant_id user_id"
+ time_precision: "{{ server.influxdb_time_precision }}"
+ encoder:
+ influxdb:
+ engine: payload
+ append_newlines: false
+ prefix_ts: false
diff --git a/metadata/service/remote_collector/single.yml b/metadata/service/remote_collector/single.yml
index d0b6420..e141799 100644
--- a/metadata/service/remote_collector/single.yml
+++ b/metadata/service/remote_collector/single.yml
@@ -6,3 +6,4 @@
heka:
remote_collector:
enabled: true
+ influxdb_time_precision: ms