Added fluentd service to k8s
Change-Id: Ia73090692c33da497380c9fa14dcfdb5a8fc90bb
Closes-Bug: PROD-21821
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-deploy.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-deploy.yaml
new file mode 100644
index 0000000..7cb8a38
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-deploy.yaml
@@ -0,0 +1,71 @@
+{%- from "kubernetes/map.jinja" import common with context -%}
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: fluentd-aggregator
+ namespace: stacklight
+ labels:
+ k8s-app: fluentd-aggregator
+ version: v1
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ k8s-app: fluentd-aggregator
+ template:
+ metadata:
+ labels:
+ k8s-app: fluentd-aggregator
+ version: v1
+ annotations:
+ scheduler.alpha.kubernetes.io/critical-pod: ''
+ scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+ spec:
+ serviceAccountName: fluentd
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ effect: NoSchedule
+ containers:
+ - name: fluentd-aggregator
+ image: fluent/fluentd-kubernetes-daemonset:v1.2-debian-elasticsearch
+ env:
+ - name: FLUENTD_ELASTICSEARCH_HOST
+ value: "{{ common.addons.fluentd.aggregator.es.get('host', '127.0.0.1') }}"
+ - name: FLUENTD_ELASTICSEARCH_PORT
+ value: "{{ common.addons.fluentd.aggregator.es.get('port', '9200') }}"
+ - name: FLUENTD_ELASTICSEARCH_SCHEME
+ value: "{{ common.addons.fluentd.aggregator.es.get('scheme', 'http') }}"
+ - name: FLUENTD_AGGREGATOR_BIND_PORT
+ value: "{{ common.addons.fluentd.aggregator.bind.get('port', '24224') }}"
+ - name: ENVIRONMENT_LABEL
+ value: "{{ grains.domain }}"
+ # - name: FLUENTD_OPT
+ # value: "--use-v1-config"
+ # TODO: a hack to pass the broken entrypoint in upstream docker image for k8s fluent when configmap is used
+ - name: FLUENT_ELASTICSEARCH_USER
+ value: "null"
+ - name: FLUENT_ELASTICSEARCH_PASSWORD
+ value: "null"
+ resources:
+ limits:
+ memory: 500Mi
+ requests:
+ memory: 500Mi
+ ports:
+ - containerPort: {{ common.addons.fluentd.aggregator.bind.get('port', '24224') }}
+ name: main-input
+ protocol: TCP
+ - containerPort: 9880
+ name: health-check
+ protocol: TCP
+ volumeMounts:
+ - name: fluentd-aggregator-config
+ mountPath: /fluentd/etc
+ readOnly: false
+ volumes:
+ - name: fluentd-aggregator-config
+ configMap:
+ name: fluentd-aggregator-cfg
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
new file mode 100644
index 0000000..b29edb4
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-fluent-conf.yaml
@@ -0,0 +1,104 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: fluentd-aggregator-cfg
+ namespace: stacklight
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+data:
+ fluent.conf: |
+ @include forward-input.conf
+ @include general.conf
+ @include kubernetes-filter.conf
+ @include output.conf
+
+ forward-input.conf: |
+ <source>
+ @type forward
+ port "#{ENV['FLUENTD_AGGREGATOR_BIND_PORT']}"
+ bind 0.0.0.0
+ </source>
+
+ general.conf: |
+ <match fluent.**>
+ @type null
+ </match>
+ <source>
+ @type http
+ port 9880
+ bind 0.0.0.0
+ </source>
+ <filter **>
+ @type record_transformer
+ enable_ruby
+ <record>
+ Type log
+ environment_label "#{ENV['ENVIRONMENT_LABEL']}"
+ Hostname ${ hostname }
+ Timestamp ${ time.strftime('%Y-%m-%dT%H:%M:%S.%N%z') }
+ </record>
+ </filter>
+
+ kubernetes-filter.conf: |
+ <filter temp.kubernetes.**>
+ @type kubernetes_metadata
+ merge_json_log true
+ preserve_json_log true
+ </filter>
+ <filter temp.kubernetes.**>
+ @type record_transformer
+ enable_ruby true
+ <record>
+ kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
+ </record>
+ </filter>
+ <filter temp.kubernetes.container.**>
+ @type record_transformer
+ enable_ruby
+ <record>
+ severity_label INFO
+ Severity 6
+ programname ${ record['kubernetes']['container_name'] }
+ </record>
+ </filter>
+ <filter temp.kubernetes.kube-system.**>
+ @type parser
+ format kubernetes
+ reserve_data true
+ key_name log
+ suppress_parse_error_log true
+ </filter>
+
+ output.conf: |
+ <match temp.kubernetes.container.**>
+ @type rewrite_tag_filter
+ <rule>
+ key log_path
+ pattern ^.*\/(.*)\.log$
+ tag kubernetes.container.$1
+ </rule>
+ </match>
+ <match kubernetes.**>
+ @type elasticsearch
+ host "#{ENV['FLUENTD_ELASTICSEARCH_HOST']}"
+ port "#{ENV['FLUENTD_ELASTICSEARCH_PORT']}"
+ scheme "#{ENV['FLUENTD_ELASTICSEARCH_SCHEME'] || 'http'}"
+ ssl_verify "#{ENV['FLUENTD_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
+ reload_connections "#{ENV['FLUENTD_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
+ type_name message
+ tag_key Logger
+ include_tag_key true
+ time_key Timestamp
+ time_key_exclude_timestamp true
+ logstash_format true
+ logstash_prefix k8s
+ logstash_dateformat %Y.%m.%d
+ request_timeout 10s
+ buffer_chunk_limit 2M
+ buffer_queue_limit 32
+ flush_interval 10s
+ max_retry_wait 30
+ disable_retry_limit
+ num_threads 8
+ </match>
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-svc.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-svc.yaml
new file mode 100644
index 0000000..7c58fd5
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-aggregator-svc.yaml
@@ -0,0 +1,19 @@
+{%- from "kubernetes/map.jinja" import common with context -%}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: fluentd-aggregator
+ namespace: stacklight
+ labels:
+ k8s-app: fluentd-aggregator
+ kubernetes.io/cluster-service: "true"
+ kubernetes.io/name: "fluentd-aggregator"
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ selector:
+ k8s-app: fluentd-aggregator
+ ports:
+ - name: fluentd-aggregator
+ port: {{ common.addons.fluentd.aggregator.bind.get('port', '24224') }}
+ protocol: TCP
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml
new file mode 100644
index 0000000..59fcdab
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-logger-ds.yaml
@@ -0,0 +1,63 @@
+{%- from "kubernetes/map.jinja" import common with context -%}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: fluentd-logger
+ namespace: stacklight
+ labels:
+ k8s-app: fluentd-logger
+ version: v1
+ kubernetes.io/cluster-service: "true"
+ beta.kubernetes.io/fluentd-ds-ready: "true"
+spec:
+ template:
+ metadata:
+ labels:
+ k8s-app: fluentd-logger
+ version: v1
+ kubernetes.io/cluster-service: "true"
+ # This annotation ensures that fluentd does not get evicted if the node
+ # supports critical pod annotation based priority scheme.
+ # Note that this does not guarantee admission on the nodes (#40573).
+ annotations:
+ scheduler.alpha.kubernetes.io/critical-pod: ''
+ seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+ spec:
+ priorityClassName: system-node-critical
+ serviceAccountName: fluentd
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ containers:
+ - name: fluentd-logger
+ image: fluent/fluentd-kubernetes-daemonset:v1.2-debian-stackdriver
+ env:
+ - name: FLUENTD_AGGREGATOR_HOST
+ value: "fluentd-aggregator"
+ - name: FLUENTD_AGGREGATOR_PORT
+ value: "{{ common.addons.fluentd.aggregator.bind.get('port', '24224') }}"
+ resources:
+ limits:
+ memory: 500Mi
+ requests:
+ memory: 500Mi
+ volumeMounts:
+ - name: varlog
+ mountPath: /var/log
+ - name: varlibdockercontainers
+ mountPath: /var/lib/docker/containers
+ readOnly: true
+ - name: fluentd-logger-config
+ mountPath: /fluentd/etc
+ readOnly: false
+ terminationGracePeriodSeconds: 30
+ volumes:
+ - name: varlog
+ hostPath:
+ path: /var/log
+ - name: varlibdockercontainers
+ hostPath:
+ path: /var/lib/docker/containers
+ - name: fluentd-logger-config
+ configMap:
+ name: fluentd-logger-cfg
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
new file mode 100644
index 0000000..b5ceb87
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-logger-fluent-conf.yaml
@@ -0,0 +1,74 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: fluentd-logger-cfg
+ namespace: stacklight
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+data:
+ fluent.conf: |
+ @include general.conf
+ @include apiserver-audit-input.conf
+ @include kubernetes-input.conf
+ @include forward-output.conf
+
+ general.conf: |
+ <match fluent.**>
+ @type null
+ </match>
+ <source>
+ @type http
+ port 9880
+ bind 0.0.0.0
+ </source>
+
+ apiserver-audit-input.conf: |
+ <source>
+ @type tail
+ format multiline
+ multiline_flush_interval 5s
+ format_firstline /^\S+\s+AUDIT:/
+ format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
+ time_format %FT%T.%L%Z
+ path /var/log/kubernetes/kube-apiserver-audit.log
+ pos_file /var/log/kube-apiserver-audit.log.pos
+ tag temp.kubernetes.apiserver-audit.*
+ </source>
+
+ kubernetes-input.conf: |
+ <source>
+ @type tail
+ path /var/log/containers/*.log
+ pos_file /var/log/fluentd-containers.log.pos
+ time_format %Y-%m-%dT%H:%M:%S.%NZ
+ tag temp.kubernetes.container.*
+ format json
+ read_from_head true
+ </source>
+
+ forward-output.conf: |
+ <match **>
+ @type forward
+ require_ack_response true
+ ack_response_timeout 30
+ recover_wait 10s
+ heartbeat_interval 1s
+ phi_threshold 16
+ send_timeout 10s
+ hard_timeout 10s
+ expire_dns_cache 15
+ heartbeat_type tcp
+ buffer_chunk_limit 2M
+ buffer_queue_limit 32
+ flush_interval 5s
+ max_retry_wait 15
+ disable_retry_limit
+ num_threads 8
+ <server>
+ name fluentd-aggregator
+ host fluentd-aggregator
+ port "#{ENV['FLUENTD_AGGREGATOR_PORT']}"
+ weight 60
+ </server>
+ </match>
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-ns.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-ns.yaml
new file mode 100644
index 0000000..1d454eb
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-ns.yaml
@@ -0,0 +1,8 @@
+---
+kind: Namespace
+apiVersion: v1
+metadata:
+ name: stacklight
+ labels:
+ k8s-app: fluentd
+ addonmanager.kubernetes.io/mode: Reconcile
diff --git a/kubernetes/files/kube-addons/fluentd/fluentd-sa.yaml b/kubernetes/files/kube-addons/fluentd/fluentd-sa.yaml
new file mode 100644
index 0000000..5d0b262
--- /dev/null
+++ b/kubernetes/files/kube-addons/fluentd/fluentd-sa.yaml
@@ -0,0 +1,42 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+ name: fluentd
+ namespace: stacklight
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ name: fluentd
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ - namespaces
+ verbs:
+ - "get"
+ - "watch"
+ - "list"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: fluentd
+ labels:
+ k8s-app: fluentd
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+ kind: ClusterRole
+ name: fluentd
+ apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+ name: fluentd
+ namespace: stacklight
diff --git a/kubernetes/master/kube-addons.sls b/kubernetes/master/kube-addons.sls
index a286266..86a6c4f 100644
--- a/kubernetes/master/kube-addons.sls
+++ b/kubernetes/master/kube-addons.sls
@@ -321,6 +321,52 @@
{% endif %}
+{%- if common.addons.get('fluentd', {}).get('enabled') %}
+
+/etc/kubernetes/addons/fluentd/fluentd-ns.yaml:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/fluentd/fluentd-ns.yaml
+ - template: jinja
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+
+/etc/kubernetes/addons/fluentd/fluentd-sa.yaml:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/fluentd/fluentd-sa.yaml
+ - template: jinja
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+
+{%- set fluentd_aggregator_resources = ['fluent-conf','deploy', 'svc'] %}
+{%- for resource in fluentd_aggregator_resources %}
+
+/etc/kubernetes/addons/fluentd/fluentd-aggregator-{{ resource }}.yaml:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/fluentd/fluentd-aggregator-{{ resource }}.yaml
+ - template: jinja
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+
+{%- endfor %}
+
+{%- set fluentd_logger_resources = ['fluent-conf', 'ds'] %}
+{%- for resource in fluentd_logger_resources %}
+
+/etc/kubernetes/addons/fluentd/fluentd-logger-{{ resource }}.yaml:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/fluentd/fluentd-logger-{{ resource }}.yaml
+ - template: jinja
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+
+{%- endfor %}
+
+{% endif %}
+
{%- if common.addons.get('dashboard', {'enabled': False}).enabled %}
{%- set dashboard_resources = ['deployment', 'secret', 'service', 'serviceaccount'] %}