Add zookeeper log parser
diff --git a/heka/files/lua/common/java_patterns.lua b/heka/files/lua/common/java_patterns.lua
new file mode 100644
index 0000000..73bca9b
--- /dev/null
+++ b/heka/files/lua/common/java_patterns.lua
@@ -0,0 +1,34 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt = require "date_time"
+local l = require 'lpeg'
+l.locale(l)
+
+local patt = require 'patterns'
+local utils = require 'lma_utils'
+
+local tonumber = tonumber
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+local message = l.Cg(patt.Message, "Message")
+
+-- Complete grammars
+
+-- Zookeeper
+-- 2016-11-21 06:38:43,081 - INFO [main:Environment@100] - Server environment:java.io.tmpdir=/tmp
+ZookeeperLogGrammar = l.Ct(patt.JavaTimestamp * patt.sp * patt.dash * patt.sp * patt.JavaSeverity * patt.sp^1 * message)
+
+return M
diff --git a/heka/files/lua/common/lma_utils.lua b/heka/files/lua/common/lma_utils.lua
index e7d3cc3..e79f934 100644
--- a/heka/files/lua/common/lma_utils.lua
+++ b/heka/files/lua/common/lma_utils.lua
@@ -41,6 +41,7 @@
label_to_severity_map = {
EMERGENCY = 0,
E = 0,
+ FATAL = 0,
ALERT = 1,
A = 1,
CRITICAL = 2,
@@ -62,6 +63,7 @@
I = 6,
DEBUG = 7,
D = 7,
+ TRACE = 7,
}
metric_type = {
diff --git a/heka/files/lua/common/patterns.lua b/heka/files/lua/common/patterns.lua
index e9acc18..09be695 100644
--- a/heka/files/lua/common/patterns.lua
+++ b/heka/files/lua/common/patterns.lua
@@ -144,4 +144,13 @@
-- Pattern used to match a number
Number = l.P"-"^-1 * l.xdigit^1 * (l.S(".,") * l.xdigit^1 )^-1 / tonumber
+-- Java/Log4J Timestamp patterns
+-- 2016-11-21 06:38:43,081 - INFO
+local time_secfrac = l.Cg(l.P"," * l.digit^1 / tonumber, "sec_frac")
+local ts_grammar = l.Ct(dt.date_fullyear * dash * dt.date_month * dash * dt.date_mday * sp * dt.rfc3339_partial_time * time_secfrac)
+JavaTimestamp = l.Cg(ts_grammar / dt.time_to_ns, "Timestamp")
+
+-- Java Severity
+JavaSeverity = l.Cg(SeverityLabel + l.P"WARN" + l.P"FATAL", "SeverityLabel")
+
return M
diff --git a/heka/files/lua/decoders/zookeeper.lua b/heka/files/lua/decoders/zookeeper.lua
new file mode 100644
index 0000000..ed3a338
--- /dev/null
+++ b/heka/files/lua/decoders/zookeeper.lua
@@ -0,0 +1,46 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local java = require 'java_patterns'
+
+local msg = {
+ Timestamp = nil,
+ Type = 'log',
+ Hostname = nil,
+ Payload = nil,
+ Pid = nil,
+ Fields = nil,
+ Severity = 6,
+}
+
+function process_message ()
+ local log = read_message("Payload")
+ local logger = read_message("Logger")
+ local m = java.ZookeeperLogGrammar:match(log)
+ if not m then
+ return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+ end
+ msg.Timestamp = m.Timestamp
+ msg.Payload = m.Message
+ msg.Pid = m.Pid
+ msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+ msg.Fields = {}
+ msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+ msg.Fields.programname = 'zookeeper'
+ utils.inject_tags(msg)
+ return utils.safe_inject_message(msg)
+end