Updated logs parsing/fetching for Kubelet&K8s apiserver
Use systemd to read the logs for kubelet and apiserver.
Change-Id: I898f06a05fc6252a4a120deb174761d904917f75
Related-Bug: PROD-22233
Related-Bug: PROD-22235
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
index b29edb4..34fb625 100644
--- a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
@@ -11,6 +11,7 @@
@include forward-input.conf
@include general.conf
@include kubernetes-filter.conf
+ @include systemd-filter.conf
@include output.conf
forward-input.conf: |
@@ -49,10 +50,19 @@
<filter temp.kubernetes.**>
@type record_transformer
enable_ruby true
+ remove_keys log
<record>
kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
+ Payload ${record['log']}
</record>
</filter>
+ <filter temp.kubernetes.kube-system.**>
+ @type parser
+ format kubernetes
+ reserve_data true
+ key_name log
+ suppress_parse_error_log true
+ </filter>
<filter temp.kubernetes.container.**>
@type record_transformer
enable_ruby
@@ -62,11 +72,28 @@
programname ${ record['kubernetes']['container_name'] }
</record>
</filter>
- <filter temp.kubernetes.kube-system.**>
- @type parser
+
+ systemd-filter.conf: |
+ <match systemd.source>
+ @type rewrite_tag_filter
+ <rule>
+ key ident
+ pattern ^(.*)$
+ tag __TAG__.$1
+ </rule>
+ </match>
+ <filter systemd.source.kubelet>
+ type parser
format kubernetes
reserve_data true
- key_name log
+ key_name MESSAGE
+ suppress_parse_error_log true
+ </filter>
+ <filter systemd.source.docker>
+ type parser
+ format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+ reserve_data true
+ key_name MESSAGE
suppress_parse_error_log true
</filter>
@@ -79,15 +106,16 @@
tag kubernetes.container.$1
</rule>
</match>
- <match kubernetes.**>
+
+ <match **>
@type elasticsearch
+ log_level debug
host "#{ENV['FLUENTD_ELASTICSEARCH_HOST']}"
port "#{ENV['FLUENTD_ELASTICSEARCH_PORT']}"
scheme "#{ENV['FLUENTD_ELASTICSEARCH_SCHEME'] || 'http'}"
ssl_verify "#{ENV['FLUENTD_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
reload_connections "#{ENV['FLUENTD_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
type_name message
- tag_key Logger
include_tag_key true
time_key Timestamp
time_key_exclude_timestamp true
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml
index 59fcdab..a5c5764 100644
--- a/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml
@@ -50,6 +50,8 @@
- name: fluentd-logger-config
mountPath: /fluentd/etc
readOnly: false
+ - name: runlog
+ mountPath: /run/log
terminationGracePeriodSeconds: 30
volumes:
- name: varlog
@@ -61,3 +63,6 @@
- name: fluentd-logger-config
configMap:
name: fluentd-logger-cfg
+ - name: runlog
+ hostPath:
+ path: /run/log
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
index b5ceb87..3003504 100644
--- a/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
@@ -9,8 +9,8 @@
data:
fluent.conf: |
@include general.conf
- @include apiserver-audit-input.conf
@include kubernetes-input.conf
+ @include systemd-input.conf
@include forward-output.conf
general.conf: |
@@ -23,19 +23,6 @@
bind 0.0.0.0
</source>
- apiserver-audit-input.conf: |
- <source>
- @type tail
- format multiline
- multiline_flush_interval 5s
- format_firstline /^\S+\s+AUDIT:/
- format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
- time_format %FT%T.%L%Z
- path /var/log/kubernetes/kube-apiserver-audit.log
- pos_file /var/log/kube-apiserver-audit.log.pos
- tag temp.kubernetes.apiserver-audit.*
- </source>
-
kubernetes-input.conf: |
<source>
@type tail
@@ -45,6 +32,21 @@
tag temp.kubernetes.container.*
format json
read_from_head true
+ path_key log_path
+ </source>
+
+ systemd-input.conf: |
+ <source>
+ @type systemd
+ path /run/log/journal
+ pos_file /var/log/fluentd-journald-systemd.pos
+ tag systemd.source
+ <entry>
+ field_map {"MESSAGE": "Payload", "_CMDLINE": "process", "_COMM": "programname", "_PID": "Pid", "_SYSTEMD_UNIT": "service", "priority": "Severity", "syslog_identifier": "ident" }
+ field_map_strict true
+ fields_strip_underscores true
+ fields_lowercase true
+ </entry>
</source>
forward-output.conf: |