Added logging for all related k8s services
Change-Id: I54e9177d11e1c551b91d88a86ec46e7d1736518e
Related-Bug: PROD-22234
Related-Bug: PROD-22236
Related-Bug: PROD-22237
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
index 70b65c9..d76f290 100644
--- a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
@@ -48,13 +48,6 @@
merge_json_log true
preserve_json_log true
</filter>
- <filter temp.kubernetes.**>
- @type record_transformer
- enable_ruby true
- <record>
- kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
- </record>
- </filter>
<match temp.kubernetes.container>
@type rewrite_tag_filter
<rule>
@@ -63,6 +56,25 @@
tag __TAG__.$1
</rule>
</match>
+ <match temp.kubernetes.service>
+ @type rewrite_tag_filter
+ <rule>
+ key service
+ pattern (.*)
+ tag __TAG__.$1
+ </rule>
+ </match>
+ <filter temp.kubernetes.service.**>
+ @type record_transformer
+ enable_ruby
+ remove_keys severity
+ <record>
+ severity_label ${ {"ERROR"=>"E","INFO"=>"I","WARNING"=>"W","NOTICE"=>"N"}.key(record["severity"]) }
+ Severity ${ {3=>"E",4=>"W",5=>"N",6=>"I",}.key(record["severity"]) }
+ programname ${ record['service'] }
+ service ${record['service']}.service
+ </record>
+ </filter>
<filter temp.kubernetes.container.**>
@type record_transformer
enable_ruby
@@ -70,6 +82,7 @@
severity_label ${ {"ERROR"=>"stderr","INFO"=>"stdout"}.key(record["stream"]) }
Severity ${ {3=>"stderr",6=>"stdout"}.key(record["stream"]) }
programname ${ record['kubernetes']['container_name'] }
+ kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
</record>
</filter>
@@ -89,18 +102,6 @@
tag __TAG__.$1
</rule>
</match>
- <filter systemd.source.kubelet>
- @type parser
- format kubernetes
- reserve_data true
- key_name Payload
- </filter>
- <filter systemd.source.kube-apiserver>
- @type parser
- format kubernetes
- reserve_data true
- key_name Payload
- </filter>
<filter systemd.source.**>
@type record_transformer
remove_keys message, severity
@@ -115,6 +116,14 @@
tag kubernetes.container.$1
</rule>
</match>
+ <match temp.kubernetes.service.**>
+ @type rewrite_tag_filter
+ <rule>
+ key programname
+ pattern (.*)
+ tag kubernetes.service.$1
+ </rule>
+ </match>
<match **>
@type elasticsearch
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
index d7f7c15..621c0f1 100644
--- a/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
@@ -35,11 +35,26 @@
read_from_head true
path_key log_path
</source>
+ <source>
+ @type tail
+ path /var/log/kubernetes.log
+ pos_file /var/log/fluentd-kubernetes.log.pos
+ time_format {{ common.addons.fluentd.logger.config.kubernetes_input.get('time_format') }}
+ tag temp.kubernetes.service
+ format json
+ <parse>
+ @type regexp
+ expression /^(?<service>[^\[]*)\[.*\]:\s(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/
+ time_key time
+ </parse>
+ read_from_head true
+ path_key log_path
+ </source>
systemd-input.conf: |
<source>
@type systemd
- matches [{"_SYSTEMD_UNIT": "kubelet.service"}, {"_SYSTEMD_UNIT": "docker.service"}, {"_SYSTEMD_UNIT": "kube-apiserver.service"} ]
+ matches [{"_SYSTEMD_UNIT": "docker.service"}]
path /run/log/journal
pos_file /var/log/fluentd-journald-systemd.pos
tag systemd.source