Merge pull request #68 from elemoine/stacklight-swap-size
Set actual swap_size in collectd decoder
diff --git a/heka/files/lua/common/java_patterns.lua b/heka/files/lua/common/java_patterns.lua
new file mode 100644
index 0000000..edbd130
--- /dev/null
+++ b/heka/files/lua/common/java_patterns.lua
@@ -0,0 +1,38 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt = require "date_time"
+local l = require 'lpeg'
+l.locale(l)
+
+local patt = require 'patterns'
+local utils = require 'lma_utils'
+
+local tonumber = tonumber
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+local message = l.Cg(patt.Message, "Message")
+
+-- Complete grammars
+
+-- Zookeeper
+-- 2016-11-21 06:38:43,081 - INFO [main:Environment@100] - Server environment:java.io.tmpdir=/tmp
+ZookeeperLogGrammar = l.Ct(patt.JavaTimestamp * patt.sp * patt.dash * patt.sp * patt.JavaSeverity * patt.sp^1 * message)
+
+-- Cassandra
+-- 2016-11-21 08:52:14,070 - DEBUG - run_command: nodetool -h 127.0.0.1 info | grep ID | awk '{print $3}'
+CassandraLogGrammar = l.Ct(patt.JavaTimestamp * patt.sp * patt.dash * patt.sp * patt.JavaSeverity * patt.sp^1 * patt.dash * patt.sp * message)
+
+return M
diff --git a/heka/files/lua/common/lma_utils.lua b/heka/files/lua/common/lma_utils.lua
index e7d3cc3..e79f934 100644
--- a/heka/files/lua/common/lma_utils.lua
+++ b/heka/files/lua/common/lma_utils.lua
@@ -41,6 +41,7 @@
label_to_severity_map = {
EMERGENCY = 0,
E = 0,
+ FATAL = 0,
ALERT = 1,
A = 1,
CRITICAL = 2,
@@ -62,6 +63,7 @@
I = 6,
DEBUG = 7,
D = 7,
+ TRACE = 7,
}
metric_type = {
diff --git a/heka/files/lua/common/patterns.lua b/heka/files/lua/common/patterns.lua
index e9acc18..09be695 100644
--- a/heka/files/lua/common/patterns.lua
+++ b/heka/files/lua/common/patterns.lua
@@ -144,4 +144,13 @@
-- Pattern used to match a number
Number = l.P"-"^-1 * l.xdigit^1 * (l.S(".,") * l.xdigit^1 )^-1 / tonumber
+-- Java/Log4J Timestamp patterns
+-- 2016-11-21 06:38:43,081 - INFO
+local time_secfrac = l.Cg(l.P"," * l.digit^1 / tonumber, "sec_frac")
+local ts_grammar = l.Ct(dt.date_fullyear * dash * dt.date_month * dash * dt.date_mday * sp * dt.rfc3339_partial_time * time_secfrac)
+JavaTimestamp = l.Cg(ts_grammar / dt.time_to_ns, "Timestamp")
+
+-- Java Severity
+JavaSeverity = l.Cg(SeverityLabel + l.P"WARN" + l.P"FATAL", "SeverityLabel")
+
return M
diff --git a/heka/files/lua/common/redis_patterns.lua b/heka/files/lua/common/redis_patterns.lua
new file mode 100644
index 0000000..ea879e3
--- /dev/null
+++ b/heka/files/lua/common/redis_patterns.lua
@@ -0,0 +1,58 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt = require "date_time"
+local l = require 'lpeg'
+local os = require 'os'
+l.locale(l)
+
+local patt = require 'patterns'
+local utils = require 'lma_utils'
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+-- Redis logs do not provide year in output
+local function extend_date (t)
+ if t['year'] == nil then
+ t['year'] = os.date("*t").year
+ end
+ return dt.time_to_ns(t)
+end
+
+local message = l.Cg(patt.Message / utils.chomp, "Message")
+
+-- Common patterns
+local delim = (l.P"#" + l.P"*")^1
+
+-- [11633 | signal handler] (1479710114) Received SIGTERM, scheduling shutdown...
+-- value between parenthesis is timestamp
+
+-- [11633] 21 Nov 06:35:14.648 [#*] blabla
+local redis_pid = l.P"[" * l.Cg(l.digit^1, "Pid") * l.P"]"
+local redis_sig_pid = l.P"[" * l.Cg(l.digit^1, "Pid") * patt.sp * l.P"| signal handler]"
+
+-- Timestamp patterns
+-- [11633] 21 Nov 06:35:14.648 [#*] blabla
+local ts_grammar = l.Ct(dt.date_mday * patt.sp * dt.date_mabbr * patt.sp * dt.rfc3339_partial_time)
+local redis_sig_timestamp = l.Cg(dt.build_strftime_grammar("(%s)") / dt.time_to_ns, "Timestamp")
+local redis_std_timestamp = l.Cg(ts_grammar / extend_date, "Timestamp")
+
+-- Complete grammars
+-- [11633] 21 Nov 06:35:00.925 [#*] blabla
+-- [11633 | signal handler] (1479710114) Received SIGTERM, scheduling shutdown...
+std_grammar = l.Ct(redis_pid * patt.sp * redis_std_timestamp * patt.sp * delim * patt.sp * message)
+sig_grammar = l.Ct(redis_sig_pid * patt.sp * redis_sig_timestamp * patt.sp * message)
+LogGrammar = std_grammar + sig_grammar
+
+return M
diff --git a/heka/files/lua/decoders/cassandra.lua b/heka/files/lua/decoders/cassandra.lua
new file mode 100644
index 0000000..7a80b9f
--- /dev/null
+++ b/heka/files/lua/decoders/cassandra.lua
@@ -0,0 +1,46 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local java = require 'java_patterns'
+
+local msg = {
+ Timestamp = nil,
+ Type = 'log',
+ Hostname = nil,
+ Payload = nil,
+ Pid = nil,
+ Fields = nil,
+ Severity = 6,
+}
+
+function process_message ()
+ local log = read_message("Payload")
+ local logger = read_message("Logger")
+ local m = java.CassandraLogGrammar:match(log)
+ if not m then
+ return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+ end
+ msg.Timestamp = m.Timestamp
+ msg.Payload = m.Message
+ msg.Pid = m.Pid
+ msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+ msg.Fields = {}
+ msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+ msg.Fields.programname = 'cassandra'
+ utils.inject_tags(msg)
+ return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/lua/decoders/collectd.lua b/heka/files/lua/decoders/collectd.lua
index ad2118a..085821c 100644
--- a/heka/files/lua/decoders/collectd.lua
+++ b/heka/files/lua/decoders/collectd.lua
@@ -148,7 +148,11 @@
msg['Fields']['fs'] = mount
table.insert(msg['Fields']['tag_fields'], 'fs')
elseif metric_source == 'disk' then
- msg['Fields']['name'] = metric_name
+ if sample['type'] == 'disk_io_time' then
+ msg['Fields']['name'] = 'disk' .. sep .. sample['dsnames'][i]
+ else
+ msg['Fields']['name'] = metric_name
+ end
msg['Fields']['device'] = sample['plugin_instance']
table.insert(msg['Fields']['tag_fields'], 'device')
elseif metric_source == 'cpu' then
diff --git a/heka/files/lua/decoders/redis.lua b/heka/files/lua/decoders/redis.lua
new file mode 100644
index 0000000..75c5916
--- /dev/null
+++ b/heka/files/lua/decoders/redis.lua
@@ -0,0 +1,45 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local redis = require 'redis_patterns'
+
+local msg = {
+ Timestamp = nil,
+ Type = 'log',
+ Hostname = nil,
+ Payload = nil,
+ Pid = nil,
+ Fields = nil,
+ Severity = 6,
+}
+
+function process_message ()
+ local log = read_message("Payload")
+ local logger = read_message("Logger")
+ local m = redis.LogGrammar:match(log)
+ if not m then
+ return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+ end
+ msg.Timestamp = m.Timestamp
+ msg.Payload = m.Message
+ msg.Pid = m.Pid
+ msg.Fields = {}
+ msg.Fields.severity_label = 'INFO'
+ msg.Fields.programname = 'redis'
+ utils.inject_tags(msg)
+ return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/lua/decoders/zookeeper.lua b/heka/files/lua/decoders/zookeeper.lua
new file mode 100644
index 0000000..ed3a338
--- /dev/null
+++ b/heka/files/lua/decoders/zookeeper.lua
@@ -0,0 +1,46 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local java = require 'java_patterns'
+
+local msg = {
+ Timestamp = nil,
+ Type = 'log',
+ Hostname = nil,
+ Payload = nil,
+ Pid = nil,
+ Fields = nil,
+ Severity = 6,
+}
+
+function process_message ()
+ local log = read_message("Payload")
+ local logger = read_message("Logger")
+ local m = java.ZookeeperLogGrammar:match(log)
+ if not m then
+ return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+ end
+ msg.Timestamp = m.Timestamp
+ msg.Payload = m.Message
+ msg.Pid = m.Pid
+ msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+ msg.Fields = {}
+ msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+ msg.Fields.programname = 'zookeeper'
+ utils.inject_tags(msg)
+ return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/lua/encoders/status_nagios.lua b/heka/files/lua/encoders/status_nagios.lua
index 0507a10..9957f70 100644
--- a/heka/files/lua/encoders/status_nagios.lua
+++ b/heka/files/lua/encoders/status_nagios.lua
@@ -19,7 +19,10 @@
local lma = require 'lma_utils'
local interp = require "msg_interpolate"
-local host = read_config('nagios_host')
+-- These 2 configurations are used only to encode GSE messages
+local default_host = read_config('default_nagios_host')
+local host_dimension_key = read_config('nagios_host_dimension_key')
+
-- Nagios CGI cannot accept 'plugin_output' parameter greater than 1024 bytes
-- See bug #1517917 for details.
-- With the 'cmd.cgi' re-implementation for the command PROCESS_SERVICE_CHECK_RESULT,
@@ -61,11 +64,12 @@
return -1
end
- if host then
- data['host'] = host
+ if host_dimension_key then
+ data['host'] = read_message(string.format('Fields[%s]', host_dimension_key)) or default_host
else
data['host'] = read_message('Fields[hostname]') or read_message('Hostname')
end
+
data['service'] = service_name
data['plugin_state'] = nagios_state_map[status]
diff --git a/heka/map.jinja b/heka/map.jinja
index 8b74f9d..7658623 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -72,7 +72,7 @@
'influxdb_time_precision': default_influxdb_time_precision,
'influxdb_timeout': default_influxdb_timeout,
'nagios_port': default_nagios_port,
- 'nagios_host_alarm_clusters': default_nagios_host_alarm_clusters,
+ 'nagios_default_host_alarm_clusters': default_nagios_host_alarm_clusters,
'poolsize': 100,
}
}, merge=salt['pillar.get']('heka:aggregator')) %}
diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index 3294684..4ce6298 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -396,7 +396,10 @@
module_file: /usr/share/lma_collector/encoders/status_nagios.lua
module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
config:
- nagios_host: "{{ aggregator.nagios_host_alarm_clusters }}"
+ default_nagios_host: "{{ aggregator.nagios_default_host_alarm_clusters }}"
+ {%- if aggregator.nagios_host_dimension_key is defined %}
+ nagios_host_dimension_key: "{{ aggregator.nagios_host_dimension_key }}"
+ {%- endif %}
{%- endif %}
output:
{%- if aggregator.influxdb_host is defined %}
diff --git a/metadata/service/aggregator/single.yml b/metadata/service/aggregator/single.yml
index 7e814c5..476536c 100644
--- a/metadata/service/aggregator/single.yml
+++ b/metadata/service/aggregator/single.yml
@@ -5,8 +5,10 @@
parameters:
_param:
aggregator_poolsize: 100
+ nagios_host_dimension_key: nagios_host
heka:
aggregator:
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:aggregator_poolsize}
+ nagios_host_dimension_key: ${_param:nagios_host_dimension_key}
diff --git a/metadata/service/metric_collector/single.yml b/metadata/service/metric_collector/single.yml
index ad183a2..2e4a442 100644
--- a/metadata/service/metric_collector/single.yml
+++ b/metadata/service/metric_collector/single.yml
@@ -5,8 +5,10 @@
parameters:
_param:
metric_collector_poolsize: 100
+ nagios_host_dimension_key: nagios_host
heka:
metric_collector:
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:metric_collector_poolsize}
+ nagios_host_dimension_key: ${_param:nagios_host_dimension_key}