Add Calico services log files parsers
Change-Id: Ib7d83de5954be01c007c1ed2288305132cdb5d75
diff --git a/heka/files/lua/common/calico_patterns.lua b/heka/files/lua/common/calico_patterns.lua
new file mode 100644
index 0000000..54d9aec
--- /dev/null
+++ b/heka/files/lua/common/calico_patterns.lua
@@ -0,0 +1,50 @@
+-- Copyright 2017 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+l.locale(l)
+
+local patt = require 'patterns'
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+local goid = l.Cg(patt.Pid, "GoId")
+local goline = l.Cg(patt.Pid, "GoLine")
+local goprog = l.Cg(patt.programname, "GoProg")
+local hostname = l.Cg(patt.programname, "Hostname")
+local message = l.Cg(patt.Message, "Message")
+local severity = l.Cg(patt.SeverityLabel, "SeverityLabel")
+local timestamp = l.Cg(patt.Timestamp, "Timestamp")
+
+-- Complete grammars
+
+-- Felix
+-- 2017-02-28 12:01:36.300 [INFO][96] table.go 416: Loading current iptables state and checking it is correct. ipVersion=0x4 table="raw"
+local FelixLogGrammar = l.Ct(timestamp * patt.sp^1 * "[" * severity * l.P"][" * goid * l.P"]" * patt.sp^1 * goprog * patt.sp^1 * goline * ":" * patt.sp^1 * message)
+
+-- Confd
+-- 2017-02-28T09:57:11Z cmp01 confd[92]: DEBUG Retrieving keys from store
+local ConfdLogGrammar = l.Ct(timestamp * patt.sp^1 * hostname * patt.sp^1 * l.P"confd[" * goid * l.P"]:" * patt.sp^1 * severity * patt.sp^1 * message)
+
+-- Bird
+-- 2017-02-28_09:57:11.84304 bird: device1: State changed to feed
+local BirdLogGrammar = l.Ct(timestamp * patt.sp^1 * l.P"bird:" * patt.sp^1 * message)
+
+patterns = {
+ felix = FelixLogGrammar,
+ bird = BirdLogGrammar,
+ confd = ConfdLogGrammar,
+}
+
+return M
diff --git a/heka/files/lua/common/patterns.lua b/heka/files/lua/common/patterns.lua
index be43d08..e330ee3 100644
--- a/heka/files/lua/common/patterns.lua
+++ b/heka/files/lua/common/patterns.lua
@@ -57,8 +57,8 @@
-- offset_min (number, can be nil)
--
-- The datetime string can be formatted as
--- 'YYYY-MM-DD( |T)HH:MM:SS(.ssssss)?(offset indicator)?'
-TimestampTable = l.Ct(dt.rfc3339_full_date * (sp + l.P"T") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1)
+-- 'YYYY-MM-DD( |T|_)HH:MM:SS(.ssssss)?(offset indicator)?'
+TimestampTable = l.Ct(dt.rfc3339_full_date * (sp + l.P"T" + l.P"_") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1)
-- Returns the parsed datetime converted to nanosec
Timestamp = TimestampTable / dt.time_to_ns
diff --git a/heka/files/lua/decoders/calico.lua b/heka/files/lua/decoders/calico.lua
new file mode 100644
index 0000000..588fa91
--- /dev/null
+++ b/heka/files/lua/decoders/calico.lua
@@ -0,0 +1,51 @@
+-- Copyright 2017 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local calico = require 'calico_patterns'
+local calico_service = read_config('calico_service') -- confd, bird or felix
+
+local msg = {
+ Timestamp = nil,
+ Type = 'log',
+ Hostname = nil,
+ Payload = nil,
+ Pid = nil,
+ Fields = nil,
+ Severity = 6,
+}
+
+function process_message ()
+ local log = read_message("Payload")
+ local logger = read_message("Logger")
+ local patt = calico.patterns[calico_service]
+ if not patt then
+ return -1, string.format("Unknown pattern (%s)!", calico_service)
+ end
+ local m = patt:match(log)
+ if not m then
+ return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+ end
+ msg.Timestamp = m.Timestamp
+ msg.Payload = m.Message
+ msg.Pid = m.Pid
+ msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+ msg.Fields = {}
+ msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+ msg.Fields.programname = m.Module or string.format("calico-%s", calico_service)
+ utils.inject_tags(msg)
+ return utils.safe_inject_message(msg)
+end