Add Calico services log files parsers

Change-Id: Ib7d83de5954be01c007c1ed2288305132cdb5d75
diff --git a/heka/files/lua/common/calico_patterns.lua b/heka/files/lua/common/calico_patterns.lua
new file mode 100644
index 0000000..54d9aec
--- /dev/null
+++ b/heka/files/lua/common/calico_patterns.lua
@@ -0,0 +1,50 @@
+-- Copyright 2017 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l      = require 'lpeg'
+l.locale(l)
+
+local patt   = require 'patterns'
+
+local M = {}
+setfenv(1, M) -- Remove external access to contain everything in the module
+
+local goid      = l.Cg(patt.Pid, "GoId")
+local goline    = l.Cg(patt.Pid, "GoLine")
+local goprog    = l.Cg(patt.programname, "GoProg")
+local hostname  = l.Cg(patt.programname, "Hostname")
+local message   = l.Cg(patt.Message, "Message")
+local severity  = l.Cg(patt.SeverityLabel, "SeverityLabel")
+local timestamp = l.Cg(patt.Timestamp, "Timestamp")
+
+-- Complete grammars
+
+-- Felix
+-- 2017-02-28 12:01:36.300 [INFO][96] table.go 416: Loading current iptables state and checking it is correct. ipVersion=0x4 table="raw"
+local FelixLogGrammar = l.Ct(timestamp * patt.sp^1 * "[" * severity * l.P"][" * goid * l.P"]" * patt.sp^1 * goprog * patt.sp^1 * goline * ":" * patt.sp^1 * message)
+
+-- Confd
+-- 2017-02-28T09:57:11Z cmp01 confd[92]: DEBUG Retrieving keys from store
+local ConfdLogGrammar = l.Ct(timestamp * patt.sp^1 * hostname * patt.sp^1 * l.P"confd[" * goid * l.P"]:" * patt.sp^1 * severity * patt.sp^1 * message)
+
+-- Bird
+-- 2017-02-28_09:57:11.84304 bird: device1: State changed to feed
+local BirdLogGrammar = l.Ct(timestamp * patt.sp^1 * l.P"bird:" * patt.sp^1 * message)
+
+patterns = {
+    felix = FelixLogGrammar,
+    bird = BirdLogGrammar,
+    confd = ConfdLogGrammar,
+}
+
+return M
diff --git a/heka/files/lua/common/patterns.lua b/heka/files/lua/common/patterns.lua
index be43d08..e330ee3 100644
--- a/heka/files/lua/common/patterns.lua
+++ b/heka/files/lua/common/patterns.lua
@@ -57,8 +57,8 @@
 --   offset_min (number, can be nil)
 --
 -- The datetime string can be formatted as
--- 'YYYY-MM-DD( |T)HH:MM:SS(.ssssss)?(offset indicator)?'
-TimestampTable = l.Ct(dt.rfc3339_full_date * (sp + l.P"T") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1)
+-- 'YYYY-MM-DD( |T|_)HH:MM:SS(.ssssss)?(offset indicator)?'
+TimestampTable = l.Ct(dt.rfc3339_full_date * (sp + l.P"T" + l.P"_") * dt.rfc3339_partial_time * (dt.rfc3339_time_offset + dt.timezone_offset)^-1)
 
 -- Returns the parsed datetime converted to nanosec
 Timestamp = TimestampTable / dt.time_to_ns
diff --git a/heka/files/lua/decoders/calico.lua b/heka/files/lua/decoders/calico.lua
new file mode 100644
index 0000000..588fa91
--- /dev/null
+++ b/heka/files/lua/decoders/calico.lua
@@ -0,0 +1,51 @@
+-- Copyright 2017 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l      = require 'lpeg'
+local utils  = require 'lma_utils'
+l.locale(l)
+
+local calico = require 'calico_patterns'
+local calico_service = read_config('calico_service') -- confd, bird or felix
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+function process_message ()
+    local log = read_message("Payload")
+    local logger = read_message("Logger")
+    local patt = calico.patterns[calico_service]
+    if not patt then
+        return -1, string.format("Unknown pattern (%s)!", calico_service)
+    end
+    local m = patt:match(log)
+    if not m then
+        return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+    end
+    msg.Timestamp = m.Timestamp
+    msg.Payload = m.Message
+    msg.Pid = m.Pid
+    msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+    msg.Fields = {}
+    msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+    msg.Fields.programname = m.Module or string.format("calico-%s", calico_service)
+    utils.inject_tags(msg)
+    return utils.safe_inject_message(msg)
+end