Merge pull request #68 from elemoine/stacklight-swap-size

Set actual swap_size in collectd decoder
diff --git a/heka/files/lua/common/java_patterns.lua b/heka/files/lua/common/java_patterns.lua
new file mode 100644
index 0000000..edbd130
--- /dev/null
+++ b/heka/files/lua/common/java_patterns.lua
@@ -0,0 +1,38 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt     = require "date_time"
+local l      = require 'lpeg'
+l.locale(l)
+
+local patt   = require 'patterns'
+local utils  = require 'lma_utils'
+
+local tonumber = tonumber
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+local message   = l.Cg(patt.Message, "Message")
+
+-- Complete grammars
+
+-- Zookeeper
+-- 2016-11-21 06:38:43,081 - INFO  [main:Environment@100] - Server environment:java.io.tmpdir=/tmp
+ZookeeperLogGrammar = l.Ct(patt.JavaTimestamp * patt.sp * patt.dash * patt.sp * patt.JavaSeverity * patt.sp^1 * message)
+
+-- Cassandra
+-- 2016-11-21 08:52:14,070 - DEBUG - run_command: nodetool -h 127.0.0.1 info | grep ID | awk '{print $3}'
+CassandraLogGrammar = l.Ct(patt.JavaTimestamp * patt.sp * patt.dash * patt.sp * patt.JavaSeverity * patt.sp^1 * patt.dash * patt.sp * message)
+
+return M
diff --git a/heka/files/lua/common/lma_utils.lua b/heka/files/lua/common/lma_utils.lua
index e7d3cc3..e79f934 100644
--- a/heka/files/lua/common/lma_utils.lua
+++ b/heka/files/lua/common/lma_utils.lua
@@ -41,6 +41,7 @@
 label_to_severity_map = {
     EMERGENCY = 0,
     E = 0,
+    FATAL = 0,
     ALERT = 1,
     A = 1,
     CRITICAL = 2,
@@ -62,6 +63,7 @@
     I = 6,
     DEBUG = 7,
     D = 7,
+    TRACE = 7,
 }
 
 metric_type = {
diff --git a/heka/files/lua/common/patterns.lua b/heka/files/lua/common/patterns.lua
index e9acc18..09be695 100644
--- a/heka/files/lua/common/patterns.lua
+++ b/heka/files/lua/common/patterns.lua
@@ -144,4 +144,13 @@
 -- Pattern used to match a number
 Number = l.P"-"^-1 * l.xdigit^1 * (l.S(".,") * l.xdigit^1 )^-1 / tonumber
 
+-- Java/Log4J Timestamp patterns
+-- 2016-11-21 06:38:43,081 - INFO 
+local time_secfrac = l.Cg(l.P"," * l.digit^1 / tonumber, "sec_frac")
+local ts_grammar = l.Ct(dt.date_fullyear * dash * dt.date_month * dash * dt.date_mday * sp * dt.rfc3339_partial_time * time_secfrac)
+JavaTimestamp = l.Cg(ts_grammar / dt.time_to_ns, "Timestamp")
+
+-- Java Severity
+JavaSeverity = l.Cg(SeverityLabel + l.P"WARN" + l.P"FATAL", "SeverityLabel")
+
 return M
diff --git a/heka/files/lua/common/redis_patterns.lua b/heka/files/lua/common/redis_patterns.lua
new file mode 100644
index 0000000..ea879e3
--- /dev/null
+++ b/heka/files/lua/common/redis_patterns.lua
@@ -0,0 +1,58 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt     = require "date_time"
+local l      = require 'lpeg'
+local os     = require 'os'
+l.locale(l)
+
+local patt   = require 'patterns'
+local utils  = require 'lma_utils'
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+-- Redis logs do not provide year in output
+local function extend_date (t)
+    if t['year'] == nil then
+       t['year'] = os.date("*t").year
+    end
+    return dt.time_to_ns(t)
+end
+
+local message   = l.Cg(patt.Message / utils.chomp, "Message")
+
+-- Common patterns
+local delim = (l.P"#" + l.P"*")^1
+
+-- [11633 | signal handler] (1479710114) Received SIGTERM, scheduling shutdown...
+-- value between parenthesis is timestamp
+
+-- [11633] 21 Nov 06:35:14.648 [#*] blabla
+local redis_pid = l.P"[" * l.Cg(l.digit^1, "Pid") * l.P"]"
+local redis_sig_pid = l.P"[" * l.Cg(l.digit^1, "Pid") * patt.sp * l.P"| signal handler]"
+
+-- Timestamp patterns
+-- [11633] 21 Nov 06:35:14.648 [#*] blabla
+local ts_grammar = l.Ct(dt.date_mday * patt.sp * dt.date_mabbr * patt.sp * dt.rfc3339_partial_time)
+local redis_sig_timestamp = l.Cg(dt.build_strftime_grammar("(%s)") / dt.time_to_ns, "Timestamp")
+local redis_std_timestamp = l.Cg(ts_grammar / extend_date, "Timestamp")
+
+-- Complete grammars
+-- [11633] 21 Nov 06:35:00.925 [#*] blabla
+-- [11633 | signal handler] (1479710114) Received SIGTERM, scheduling shutdown...
+std_grammar = l.Ct(redis_pid * patt.sp * redis_std_timestamp * patt.sp * delim * patt.sp * message)
+sig_grammar = l.Ct(redis_sig_pid * patt.sp * redis_sig_timestamp * patt.sp * message)
+LogGrammar = std_grammar + sig_grammar
+
+return M
diff --git a/heka/files/lua/decoders/cassandra.lua b/heka/files/lua/decoders/cassandra.lua
new file mode 100644
index 0000000..7a80b9f
--- /dev/null
+++ b/heka/files/lua/decoders/cassandra.lua
@@ -0,0 +1,46 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l      = require 'lpeg'
+local utils  = require 'lma_utils'
+l.locale(l)
+
+local java   = require 'java_patterns'
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+function process_message ()
+    local log = read_message("Payload")
+    local logger = read_message("Logger")
+    local m = java.CassandraLogGrammar:match(log)
+    if not m then
+        return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+    end
+    msg.Timestamp = m.Timestamp
+    msg.Payload = m.Message
+    msg.Pid = m.Pid
+    msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+    msg.Fields = {}
+    msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+    msg.Fields.programname = 'cassandra'
+    utils.inject_tags(msg)
+    return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/lua/decoders/collectd.lua b/heka/files/lua/decoders/collectd.lua
index ad2118a..085821c 100644
--- a/heka/files/lua/decoders/collectd.lua
+++ b/heka/files/lua/decoders/collectd.lua
@@ -148,7 +148,11 @@
                 msg['Fields']['fs'] = mount
                 table.insert(msg['Fields']['tag_fields'], 'fs')
             elseif metric_source == 'disk' then
-                msg['Fields']['name'] = metric_name
+                if sample['type'] == 'disk_io_time' then
+                    msg['Fields']['name'] = 'disk' .. sep .. sample['dsnames'][i]
+                else
+                    msg['Fields']['name'] = metric_name
+                end
                 msg['Fields']['device'] = sample['plugin_instance']
                 table.insert(msg['Fields']['tag_fields'], 'device')
             elseif metric_source == 'cpu' then
diff --git a/heka/files/lua/decoders/redis.lua b/heka/files/lua/decoders/redis.lua
new file mode 100644
index 0000000..75c5916
--- /dev/null
+++ b/heka/files/lua/decoders/redis.lua
@@ -0,0 +1,45 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l      = require 'lpeg'
+local utils  = require 'lma_utils'
+l.locale(l)
+
+local redis  = require 'redis_patterns'
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+function process_message ()
+    local log = read_message("Payload")
+    local logger = read_message("Logger")
+    local m = redis.LogGrammar:match(log)
+    if not m then
+        return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+    end
+    msg.Timestamp = m.Timestamp
+    msg.Payload = m.Message
+    msg.Pid = m.Pid
+    msg.Fields = {}
+    msg.Fields.severity_label = 'INFO'
+    msg.Fields.programname = 'redis'
+    utils.inject_tags(msg)
+    return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/lua/decoders/zookeeper.lua b/heka/files/lua/decoders/zookeeper.lua
new file mode 100644
index 0000000..ed3a338
--- /dev/null
+++ b/heka/files/lua/decoders/zookeeper.lua
@@ -0,0 +1,46 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l      = require 'lpeg'
+local utils  = require 'lma_utils'
+l.locale(l)
+
+local java  = require 'java_patterns'
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+function process_message ()
+    local log = read_message("Payload")
+    local logger = read_message("Logger")
+    local m = java.ZookeeperLogGrammar:match(log)
+    if not m then
+        return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+    end
+    msg.Timestamp = m.Timestamp
+    msg.Payload = m.Message
+    msg.Pid = m.Pid
+    msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+    msg.Fields = {}
+    msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+    msg.Fields.programname = 'zookeeper'
+    utils.inject_tags(msg)
+    return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/lua/encoders/status_nagios.lua b/heka/files/lua/encoders/status_nagios.lua
index 0507a10..9957f70 100644
--- a/heka/files/lua/encoders/status_nagios.lua
+++ b/heka/files/lua/encoders/status_nagios.lua
@@ -19,7 +19,10 @@
 local lma = require 'lma_utils'
 local interp = require "msg_interpolate"
 
-local host = read_config('nagios_host')
+-- These 2 configurations are used only to encode GSE messages
+local default_host = read_config('default_nagios_host')
+local host_dimension_key = read_config('nagios_host_dimension_key')
+
 -- Nagios CGI cannot accept 'plugin_output' parameter greater than 1024 bytes
 -- See bug #1517917 for details.
 -- With the 'cmd.cgi' re-implementation for the command PROCESS_SERVICE_CHECK_RESULT,
@@ -61,11 +64,12 @@
         return -1
     end
 
-    if host then
-        data['host'] = host
+    if host_dimension_key then
+        data['host'] = read_message(string.format('Fields[%s]', host_dimension_key)) or default_host
     else
         data['host'] = read_message('Fields[hostname]') or read_message('Hostname')
     end
+
     data['service'] = service_name
     data['plugin_state'] = nagios_state_map[status]
 
diff --git a/heka/map.jinja b/heka/map.jinja
index 8b74f9d..7658623 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -72,7 +72,7 @@
     'influxdb_time_precision': default_influxdb_time_precision,
     'influxdb_timeout': default_influxdb_timeout,
     'nagios_port': default_nagios_port,
-    'nagios_host_alarm_clusters': default_nagios_host_alarm_clusters,
+    'nagios_default_host_alarm_clusters': default_nagios_host_alarm_clusters,
     'poolsize': 100,
   }
 }, merge=salt['pillar.get']('heka:aggregator')) %}
diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index 3294684..4ce6298 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -396,7 +396,10 @@
       module_file: /usr/share/lma_collector/encoders/status_nagios.lua
       module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
       config:
-        nagios_host: "{{ aggregator.nagios_host_alarm_clusters }}"
+        default_nagios_host: "{{ aggregator.nagios_default_host_alarm_clusters }}"
+        {%- if aggregator.nagios_host_dimension_key is defined %}
+        nagios_host_dimension_key: "{{ aggregator.nagios_host_dimension_key }}"
+        {%- endif %}
 {%- endif %}
   output:
 {%- if aggregator.influxdb_host is defined %}
diff --git a/metadata/service/aggregator/single.yml b/metadata/service/aggregator/single.yml
index 7e814c5..476536c 100644
--- a/metadata/service/aggregator/single.yml
+++ b/metadata/service/aggregator/single.yml
@@ -5,8 +5,10 @@
 parameters:
   _param:
     aggregator_poolsize: 100
+    nagios_host_dimension_key: nagios_host
   heka:
     aggregator:
       enabled: true
       influxdb_time_precision: ms
       poolsize: ${_param:aggregator_poolsize}
+      nagios_host_dimension_key: ${_param:nagios_host_dimension_key}
diff --git a/metadata/service/metric_collector/single.yml b/metadata/service/metric_collector/single.yml
index ad183a2..2e4a442 100644
--- a/metadata/service/metric_collector/single.yml
+++ b/metadata/service/metric_collector/single.yml
@@ -5,8 +5,10 @@
 parameters:
   _param:
     metric_collector_poolsize: 100
+    nagios_host_dimension_key: nagios_host
   heka:
     metric_collector:
       enabled: true
       influxdb_time_precision: ms
       poolsize: ${_param:metric_collector_poolsize}
+      nagios_host_dimension_key: ${_param:nagios_host_dimension_key}