Merge "Remove legacy http_check metric decoder"
diff --git a/heka/files/lua/decoders/notification.lua b/heka/files/lua/decoders/notification.lua
index 3f10e0d..fc86283 100644
--- a/heka/files/lua/decoders/notification.lua
+++ b/heka/files/lua/decoders/notification.lua
@@ -17,6 +17,8 @@
local patt = require 'patterns'
local utils = require 'lma_utils'
+local hostname = read_config('hostname')
+
-- Mapping table from event_type prefixes to notification loggers
local logger_map = {
--cinder
@@ -150,7 +152,7 @@
end
function process_message()
- local msg = {Fields={}}
+ local msg = {Hostname=hostname, Fields={}}
local data = read_message("Payload")
local ok, notif = pcall(cjson.decode, data)
if not ok then
diff --git a/heka/files/lua/encoders/syslog.lua b/heka/files/lua/encoders/syslog.lua
new file mode 100644
index 0000000..eb59ce2
--- /dev/null
+++ b/heka/files/lua/encoders/syslog.lua
@@ -0,0 +1,46 @@
+-- Copyright 2017 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+require 'string'
+require 'os'
+local lma = require 'lma_utils'
+
+
+local default_facility = read_config('facility')
+if default_facility == nil or (default_facility + 0 < 0 and default_facility + 0 > 23) then
+ -- default to local0
+ default_facility = 16
+end
+
+-- Encodes Heka messages using the RFC5424 Syslog format
+-- https://tools.ietf.org/html/rfc5424
+-- A line feed is added at the end of the message to avoid truncated messages
+-- on the other side.
+function process_message()
+ local timestamp = os.date("%FT%TZ", read_message('Timestamp') / 1e9)
+ local hostname = read_message('Hostname')
+ local msg = read_message('Payload')
+ local pid = read_message('Pid')
+ if pid == nil or pid == 0 then
+ pid = '-'
+ end
+ local severity = read_message('Severity') or 7
+ local facility = read_message('Fields[syslogfacility]') or default_facility
+ local app = read_message('Fields[programname]') or read_message('Logger')
+
+ return lma.safe_inject_payload(
+ 'txt',
+ 'syslog',
+ string.format("<%d>1 %s %s %s %s - - %s\n", 8 * facility + severity, timestamp, hostname, app, pid, msg)
+ )
+end
diff --git a/heka/files/toml/encoder/sandbox.toml b/heka/files/toml/encoder/sandbox.toml
index 67c95a1..96ad77d 100644
--- a/heka/files/toml/encoder/sandbox.toml
+++ b/heka/files/toml/encoder/sandbox.toml
@@ -1,9 +1,7 @@
[{{ encoder_name }}_encoder]
type = "SandboxEncoder"
-filename = "{{ encoder.module_file }}"
-{%- if encoder.module_dir is defined %}
-module_directory = "{{ encoder.module_dir }}"
-{%- endif %}
+filename = "{{ encoder.module_file|default('/usr/share/lma_collector/encoders/' + encoder_name + '.lua' ) }}"
+module_directory = "{{ encoder.module_dir|default('/usr/share/lma_collector/common;/usr/share/heka/lua_modules') }}"
{%- if encoder.config is mapping %}
[{{ encoder_name }}_encoder.config]
diff --git a/heka/files/toml/output/_buffering.toml b/heka/files/toml/output/_buffering.toml
new file mode 100644
index 0000000..dd75cc3
--- /dev/null
+++ b/heka/files/toml/output/_buffering.toml
@@ -0,0 +1,9 @@
+{%- if output.get('use_buffering', True) %}
+use_buffering = true
+[{{ output_name }}_output.buffering]
+max_buffer_size = {{ output.buffering_max_buffer_size|default(268435456) }}
+max_file_size = {{ output.buffering_max_file_size|default(67108864) }}
+full_action = "{{ output.buffering_full_action|default('drop') }}"
+{%- else %}
+use_buffering = false
+{%- endif %}
diff --git a/heka/files/toml/output/tcp.toml b/heka/files/toml/output/tcp.toml
index dee7fa0..643f3ce 100644
--- a/heka/files/toml/output/tcp.toml
+++ b/heka/files/toml/output/tcp.toml
@@ -1,11 +1,7 @@
[{{ output_name }}_output]
type="TcpOutput"
address = "{{ output.host }}:{{ output.port }}"
-encoder = "ProtobufEncoder"
+encoder = "{{ output.encoder|default('ProtobufEncoder') }}"
message_matcher = "{{ output.message_matcher }}"
-use_buffering = true
-[{{ output_name }}_output.buffering]
-max_buffer_size = 268435456
-max_file_size = 67108864
-full_action = "drop"
+{%- include 'heka/files/toml/output/_buffering.toml' %}
diff --git a/heka/files/toml/output/udp.toml b/heka/files/toml/output/udp.toml
index 1b2854e..f649f7d 100644
--- a/heka/files/toml/output/udp.toml
+++ b/heka/files/toml/output/udp.toml
@@ -4,10 +4,4 @@
encoder = "{{ output.encoder }}"
message_matcher = "{{ output.message_matcher }}"
-{%- if output.get('use_buffering', True) %}
-use_buffering = true
-[{{ output_name }}_output.buffering]
-max_buffer_size = {{ output.buffering_max_buffer_size|default(268435456) }}
-max_file_size = {{ output.buffering_max_file_size|default(67108864) }}
-full_action = "{{ output.output.buffering_full_action|default("drop") }}"
-{%- endif %}
+{%- include 'heka/files/toml/output/_buffering.toml' %}
diff --git a/heka/map.jinja b/heka/map.jinja
index 0712436..14c5e4e 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -33,6 +33,7 @@
{% set default_elasticsearch_port = 9200 %}
{% set default_influxdb_port = 8086 %}
{% set default_influxdb_time_precision = 'ms' %}
+{% set default_influxdb_tag_fields = ['environment_label', 'region', 'tenant_id', 'user_id'] %}
{% set default_influxdb_timeout = 5000 %}
{% set default_aggregator_port = 5565 %}
{% set default_nagios_port = 8001 %}
@@ -63,6 +64,7 @@
'influxdb_port': default_influxdb_port,
'influxdb_time_precision': default_influxdb_time_precision,
'influxdb_timeout': default_influxdb_timeout,
+ 'influxdb_tag_fields': default_influxdb_tag_fields,
'aggregator_port': default_aggregator_port,
'nagios_port': default_nagios_port,
'poolsize': 100,
@@ -82,6 +84,7 @@
'influxdb_port': default_influxdb_port,
'influxdb_time_precision': default_influxdb_time_precision,
'influxdb_timeout': default_influxdb_timeout,
+ 'influxdb_tag_fields': default_influxdb_tag_fields,
'aggregator_port': default_aggregator_port,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
@@ -97,6 +100,7 @@
'influxdb_port': default_influxdb_port,
'influxdb_time_precision': default_influxdb_time_precision,
'influxdb_timeout': default_influxdb_timeout,
+ 'influxdb_tag_fields': default_influxdb_tag_fields,
'nagios_port': default_nagios_port,
'nagios_default_host_alarm_clusters': default_nagios_host_alarm_clusters,
'poolsize': 100,
diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index 60ac77e..12c0c63 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -146,7 +146,7 @@
message_matcher: "Type =~ /metric$/"
ticker_interval: 1
config:
- tag_fields: "deployment_id environment_label tenant_id user_id"
+ tag_fields: "{{ metric_collector.influxdb_tag_fields|join(' ') }}"
time_precision: "{{ metric_collector.influxdb_time_precision }}"
{%- endif %}
{%- endif %}
@@ -237,6 +237,7 @@
module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
config:
include_full_notification: false
+ hostname: '{{ grains.host }}'
{%- endif %}
input:
heka_collectd:
@@ -287,7 +288,7 @@
message_matcher: "Type =~ /metric$/"
ticker_interval: 1
config:
- tag_fields: "deployment_id environment_label tenant_id user_id"
+ tag_fields: "{{ remote_collector.influxdb_tag_fields|join(' ') }}"
time_precision: "{{ remote_collector.influxdb_time_precision }}"
{%- endif %}
{%- if remote_collector.amqp_host is defined %}
@@ -548,7 +549,7 @@
message_matcher: "Type == 'heka.sandbox.gse_metric' || Type == 'heka.sandbox.metric'"
ticker_interval: 1
config:
- tag_fields: "deployment_id environment_label tenant_id user_id"
+ tag_fields: "{{ aggregator.influxdb_tag_fields|join(' ') }}"
time_precision: "{{ aggregator.influxdb_time_precision }}"
influxdb_annotation:
engine: sandbox
diff --git a/metadata/service/log_collector/output/syslog.yml b/metadata/service/log_collector/output/syslog.yml
new file mode 100644
index 0000000..b7055d7
--- /dev/null
+++ b/metadata/service/log_collector/output/syslog.yml
@@ -0,0 +1,20 @@
+parameters:
+ _param:
+ log_collector_syslog_port: 514
+ # protocol should be either tcp or udp
+ log_collector_syslog_protocol: tcp
+ # set this parameter to false if using udp
+ log_collector_syslog_buffering: true
+ heka:
+ log_collector:
+ encoder:
+ syslog:
+ engine: sandbox
+ output:
+ syslog:
+ engine: ${_param:log_collector_syslog_protocol}
+ host: ${_param:log_collector_syslog_host}
+ port: ${_param:log_collector_syslog_port}
+ message_matcher: "Type == 'log'"
+ encoder: syslog_encoder
+ use_buffering: ${_param:log_collector_syslog_buffering}
diff --git a/metadata/service/remote_collector/output/syslog.yml b/metadata/service/remote_collector/output/syslog.yml
new file mode 100644
index 0000000..2709817
--- /dev/null
+++ b/metadata/service/remote_collector/output/syslog.yml
@@ -0,0 +1,20 @@
+parameters:
+ _param:
+ remote_collector_syslog_port: 514
+ # protocol should be either tcp or udp
+ remote_collector_syslog_protocol: tcp
+ # set this parameter to false if using udp
+ remote_collector_syslog_buffering: true
+ heka:
+ remote_collector:
+ encoder:
+ syslog:
+ engine: sandbox
+ output:
+ syslog:
+ engine: ${_param:remote_collector_syslog_protocol}
+ host: ${_param:remote_collector_syslog_host}
+ port: ${_param:remote_collector_syslog_port}
+ message_matcher: "Type == 'notification' || Type == 'audit'"
+ encoder: syslog_encoder
+ use_buffering: ${_param:remote_collector_syslog_buffering}