Fluentd initial commit
Change-Id: Ibcc4f3ad5c9f14b218d771d835e9e1ca589903bb
diff --git a/fluentd/agent.sls b/fluentd/agent.sls
new file mode 100644
index 0000000..a45d5b6
--- /dev/null
+++ b/fluentd/agent.sls
@@ -0,0 +1,177 @@
+{% from "fluentd/map.jinja" import fluentd with context %}
+{%- if fluentd.get('enabled', False) %}
+
+fluentd_packages_agent:
+ pkg.installed:
+ - names: {{ fluentd.pkgs }}
+
+fluentd_gems_agent:
+ gem.installed:
+ - names: {{ fluentd.gems }}
+ - gem_bin: {{ fluentd.gem_path }}
+ - require:
+ - pkg: fluentd_packages_agent
+
+fluentd_config_d_dir:
+ file.directory:
+ - name: {{ fluentd.dir.config }}/config.d
+ - makedirs: True
+ - mode: 755
+ - require:
+ - pkg: fluentd_packages_agent
+
+fluentd_config_service:
+ file.managed:
+ - name: /etc/default/td-agent
+ - source: salt://fluentd/files/default-td-agent
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - context:
+ fluentd: {{ fluentd }}
+
+fluentd_config_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/td-agent.conf
+ - source: salt://fluentd/files/td-agent.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - context:
+ fluentd: {{ fluentd }}
+
+fluentd_grok_pattern_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/global.grok
+ - source: salt://fluentd/files/global.grok
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - context:
+ fluentd: {{ fluentd }}
+
+{%- set fluentd_config = fluentd.get('config', {}) %}
+{%- for name,values in fluentd_config.get('input', {}).iteritems() %}
+
+input_{{ name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/input-{{ name }}.conf
+ - source:
+ - salt://fluentd/files/input/_generate.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ name: {{ name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+{%- for name,values in fluentd_config.get('filter', {}).iteritems() %}
+
+filter_{{ name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/filter-{{ name }}.conf
+ - source:
+ - salt://fluentd/files/filter/_generate.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ name: {{ name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+{%- for name,values in fluentd_config.get('match', {}).iteritems() %}
+
+match_{{ name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/match-{{ name }}.conf
+ - source:
+ - salt://fluentd/files/match/_generate.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ name: {{ name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+{%- for label_name,values in fluentd_config.get('label', {}).iteritems() %}
+
+label_{{ label_name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/label-{{ label_name }}.conf
+ - source:
+ - salt://fluentd/files/label.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ label_name: {{ label_name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+fluentd_service_agent:
+ service.running:
+ - name: {{ fluentd.service_name }}
+ - enable: True
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: fluentd_config_agent
+
+{%- endif %}
diff --git a/fluentd/files/default-td-agent b/fluentd/files/default-td-agent
new file mode 100644
index 0000000..3f53213
--- /dev/null
+++ b/fluentd/files/default-td-agent
@@ -0,0 +1,2 @@
+TD_AGENT_USER={{ fluentd.get('user', 'root') }}
+TD_AGENT_GROUP={{ fluentd.get('group', 'root') }}
diff --git a/fluentd/files/filter/_generate.conf b/fluentd/files/filter/_generate.conf
new file mode 100644
index 0000000..67c0c70
--- /dev/null
+++ b/fluentd/files/filter/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Filter {{ name }}
+{%- if values.get('enabled', True) %}
+{% include 'fluentd/files/filter/' + values.get('type') + '.conf' %}
+{%- endif %}
+{%- endfor %}
diff --git a/fluentd/files/filter/parser.conf b/fluentd/files/filter/parser.conf
new file mode 100644
index 0000000..0579e32
--- /dev/null
+++ b/fluentd/files/filter/parser.conf
@@ -0,0 +1,11 @@
+<filter {{ values.tag }}>
+ @type parser
+ key_name {{ values.key_name }}
+ reserve_data {{ values.get('reserve_data', true) | lower }}
+ <parse>
+{%- with values=values.get('parser') %}
+ @type {{ values.get('type') }}
+{% include 'fluentd/files/parser/' + values.get('type') + '.conf' %}
+{%- endwith %}
+ </parse>
+</filter>
diff --git a/fluentd/files/filter/prometheus.conf b/fluentd/files/filter/prometheus.conf
new file mode 100644
index 0000000..0496d8d
--- /dev/null
+++ b/fluentd/files/filter/prometheus.conf
@@ -0,0 +1,21 @@
+<filter {{ values.tag }}>
+ @type prometheus
+ {%- if values.get('label') %}
+ <labels>
+ {%- for label in values.label %}
+ {%- if label.type == 'variable' %}
+ {{ label.name }} {%raw%}${{%-endraw%}{{ label.value }}{%raw%}}{%-endraw%}
+ {%- else %}
+ {{ label.name }} {{ label.value }}
+ {%- endif %}
+ {%- endfor %}
+ </labels>
+ {%- endif %}
+ {%- for metric in values.metric %}
+ <metric>
+ name {{ metric.name }}
+ type {{ metric.type }}
+ desc {{ metric.desc }}
+ </metric>
+ {%- endfor %}
+</filter>
diff --git a/fluentd/files/filter/record_transformer.conf b/fluentd/files/filter/record_transformer.conf
new file mode 100644
index 0000000..21f2198
--- /dev/null
+++ b/fluentd/files/filter/record_transformer.conf
@@ -0,0 +1,16 @@
+<filter {{ values.tag }}>
+ @type record_transformer
+ {%- if values.get('enable_ruby') %}
+ enable_ruby
+ {%- endif %}
+ {%- if values.get('remove_keys') %}
+ remove_keys {{ values.remove_keys }}
+ {%- endif %}
+ {%- if values.get('record') %}
+ <record>
+ {%- for record in values.record %}
+ {{ record.name }} ${ {{ record.value }} }
+ {%- endfor %}
+ </record>
+ {%- endif %}
+</filter>
diff --git a/fluentd/files/global.grok b/fluentd/files/global.grok
new file mode 100644
index 0000000..1ec0c73
--- /dev/null
+++ b/fluentd/files/global.grok
@@ -0,0 +1,15 @@
+# Openstack services logging
+
+REQUEST_ID req-[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+DASHLESS_UUID [A-Fa-f0-9]{32}
+
+REQUEST (?:(%{REQUEST_ID:request_id}|)%{SPACE}(%{DASHLESS_UUID:user_id}|)%{SPACE}(%{DASHLESS_UUID:tenant_id}|)%{SPACE}%{DATA})
+
+OPENSTACK_GENERIC (?:%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{NUMBER:pid}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NOTSPACE:api}%{SPACE}\[(%{REQUEST})\]%{SPACE}%{GREEDYDATA:message})
+
+KEYSTONE_APACHE_ACCESS (?:%{IPORHOST:clientip} %{USER} %{USER} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-))
+
+KEYSTONE_APACHE_ERROR (?:%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{GREEDYDATA:message})
+
+MYSQLERROR (?:%{NUMBER:pid}%{SPACE}%{TIME:timestamp}%{SPACE}%{GREEDYDATA:message})
\ No newline at end of file
diff --git a/fluentd/files/input/_general.conf b/fluentd/files/input/_general.conf
new file mode 100644
index 0000000..9b221c3
--- /dev/null
+++ b/fluentd/files/input/_general.conf
@@ -0,0 +1,8 @@
+ @type {{ values.type }}
+{%- set label_name = values.get('label') or label_name %}
+{%- if label_name is defined %}
+ @label @{{ label_name }}
+{%- endif %}
+{%- if values.get('tag') %}
+ tag {{ values.tag }}
+{%- endif %}
\ No newline at end of file
diff --git a/fluentd/files/input/_generate.conf b/fluentd/files/input/_generate.conf
new file mode 100644
index 0000000..91c1b52
--- /dev/null
+++ b/fluentd/files/input/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Input {{ name }}
+{%- if values.get('enabled', True) %}
+{% include ['fluentd/files/input/' + values.get('type') + '.conf', 'fluentd/files/input/generic.conf'] %}
+{%- endif %}
+{%- endfor %}
\ No newline at end of file
diff --git a/fluentd/files/input/forward.conf b/fluentd/files/input/forward.conf
new file mode 100644
index 0000000..95e3548
--- /dev/null
+++ b/fluentd/files/input/forward.conf
@@ -0,0 +1,5 @@
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+ port {{ values.port }}
+ bind {{ values.bind }}
+</source>
diff --git a/fluentd/files/input/generic.conf b/fluentd/files/input/generic.conf
new file mode 100644
index 0000000..fe840a7
--- /dev/null
+++ b/fluentd/files/input/generic.conf
@@ -0,0 +1,3 @@
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+</source>
diff --git a/fluentd/files/input/systemd.conf b/fluentd/files/input/systemd.conf
new file mode 100644
index 0000000..92556c1
--- /dev/null
+++ b/fluentd/files/input/systemd.conf
@@ -0,0 +1,23 @@
+{%- set plugin_parameters = ['pos_file', 'read_from_head', 'strip_underscores'] %}
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+ path {{ values.get("path", "/run/log/journal") }}
+{%- if values.get("filters") %}
+ filters [{%- for key, value in values.filters.iteritems() %}{"{{ key }}": "{{ value }}"}{{ ", " if not loop.last else '' }}{%- endfor %}]
+{%- endif %}
+{%- for parameter in plugin_parameters %}
+ {%- if values.get(parameter) %}
+ {{ parameter }} {{ values.get(parameter) }}
+ {%- endif %}
+{%- endfor %}
+ <entry>
+{%- if values.get("entry") %}
+ field_map { {%- for key, value in values.entry.field_map.iteritems() %}"{{ key }}": "{{ value }}"{{ ", " if not loop.last else '' }}{%- endfor %} }
+ {%- for parameter in ['field_map_strict', 'fields_strip_underscores', 'fields_lowercase', ] %}
+ {%- if values.entry.get(parameter) %}
+ {{ parameter }} {{ values.entry.get(parameter) | lower }}
+ {%- endif %}
+ {%- endfor %}
+ </entry>
+{%- endif %}
+</source>
diff --git a/fluentd/files/input/tail.conf b/fluentd/files/input/tail.conf
new file mode 100644
index 0000000..b836521
--- /dev/null
+++ b/fluentd/files/input/tail.conf
@@ -0,0 +1,25 @@
+{%- set plugin_parameters = ['refresh_interval', 'limit_recently_modified', 'encoding', 'from_encoding', 'read_lines_limit', 'multiline_flush_interval', 'pos_file', 'format', 'path_key', 'rotate_wait'] %}
+{%- set bool_plugin_parameters = ['read_from_head', 'enable_watch_timer', 'skip_refresh_on_startup', 'ignore_repeated_permission_error'] %}
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+ path {{ values.path }}
+{%- for parameter in plugin_parameters %}
+ {%- if values.get(parameter) %}
+ {{ parameter }} {{values.get(parameter)}}
+ {%- endif %}
+{%- endfor %}
+{%- for parameter in bool_plugin_parameters %}
+ {%- if values.get(parameter) %}
+ {{ parameter }} {{ values.get(parameter) | lower }}
+ {%- endif %}
+{%- endfor %}
+{%- if values.get('exclude_path') %}{# Replace 'for-if-else' with 'tojson' after updating to jinja 2.9 #}
+ exclude_path [{%- for path in values.exclude_path %}"{{ path }}"{{ ", " if not loop.last else '' }} {%- endfor %}]
+{%- endif %}
+ <parse>
+{%- with values=values.get('parser') %}
+ @type {{ values.get('type') }}
+{% include 'fluentd/files/parser/' + values.get('type') + '.conf' %}
+{%- endwith %}
+ </parse>
+</source>
\ No newline at end of file
diff --git a/fluentd/files/label.conf b/fluentd/files/label.conf
new file mode 100644
index 0000000..1285a18
--- /dev/null
+++ b/fluentd/files/label.conf
@@ -0,0 +1,20 @@
+# Label {{ label_name }}
+{%- if values.get('input') %}
+{%- with values=values.get('input') %}
+{%- include 'fluentd/files/input/_generate.conf' %}
+{%- endwith %}
+{%- endif %}
+
+<label @{{ label_name }}>
+{%- if values.get('filter') %}
+{%- with values=values.get('filter') %}
+{%- include 'fluentd/files/filter/_generate.conf' %}
+{%- endwith %}
+{%- endif %}
+
+{%- if values.get('match') %}
+{%- with values=values.get('match') %}
+{%- include 'fluentd/files/match/_generate.conf' %}
+{%- endwith %}
+{%- endif %}
+</label>
\ No newline at end of file
diff --git a/fluentd/files/match/_generate.conf b/fluentd/files/match/_generate.conf
new file mode 100644
index 0000000..3e9bc37
--- /dev/null
+++ b/fluentd/files/match/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Output {{ name }}
+{%- if values.get('enabled', True) %}
+{% include 'fluentd/files/match/' + values.get('type') + '.conf' %}
+{%- endif %}
+{%- endfor %}
diff --git a/fluentd/files/match/file.conf b/fluentd/files/match/file.conf
new file mode 100644
index 0000000..86e87ff
--- /dev/null
+++ b/fluentd/files/match/file.conf
@@ -0,0 +1,4 @@
+<match {{ values.tag }}>
+ @type file
+ path {{ values.path }}
+</match>
diff --git a/fluentd/files/match/relabel.conf b/fluentd/files/match/relabel.conf
new file mode 100644
index 0000000..88ca0c7
--- /dev/null
+++ b/fluentd/files/match/relabel.conf
@@ -0,0 +1,4 @@
+<match {{ values.tag }}>
+ @type relabel
+ @label @{{ values.label }}
+</match>
diff --git a/fluentd/files/match/rewrite_tag_filter.conf b/fluentd/files/match/rewrite_tag_filter.conf
new file mode 100644
index 0000000..b62f24f
--- /dev/null
+++ b/fluentd/files/match/rewrite_tag_filter.conf
@@ -0,0 +1,6 @@
+<match {{ values.tag }}>
+ @type rewrite_tag_filter
+ {%- for rule in values.rule %}
+ rewriterule{{ loop.index }} {{ rule.name }} {{ rule.regexp }} {{ rule.result }}
+ {%- endfor %}
+</match>
diff --git a/fluentd/files/parser/grok.conf b/fluentd/files/parser/grok.conf
new file mode 100644
index 0000000..421cc1a
--- /dev/null
+++ b/fluentd/files/parser/grok.conf
@@ -0,0 +1,9 @@
+ grok_failure_key grokfailure
+ {%- if values.get('custom_pattern_path') %}
+ custom_pattern_path {{ values.custom_pattern_path }}
+ {%- endif %}
+ {%- for grok_rule in values.get('rule', []) %}
+ <grok>
+ pattern {{ grok_rule.pattern }}
+ </grok>
+ {%- endfor %}
diff --git a/fluentd/files/parser/multiline.conf b/fluentd/files/parser/multiline.conf
new file mode 100644
index 0000000..7f1b3f5
--- /dev/null
+++ b/fluentd/files/parser/multiline.conf
@@ -0,0 +1,14 @@
+ time_key {{ values.get("time_key", "time") }}
+{%- if values.get('time_format') %}
+ time_format {{ values.time_format }}
+{%- endif %}
+ keep_time_key {{ values.get("keep_time_key", False) | json }}
+ format_firstline {{ values.format_firstline }}
+{%- if values.get("format") %}
+ format1 {{ values.format }}
+{%- endif %}
+{%- if values.get("formats") %}
+ {%- for format in values.formats %}
+ format{{ loop.index }} {{ format }}
+ {%- endfor %}
+{%- endif %}
\ No newline at end of file
diff --git a/fluentd/files/parser/multiline_grok.conf b/fluentd/files/parser/multiline_grok.conf
new file mode 100644
index 0000000..dcfabc5
--- /dev/null
+++ b/fluentd/files/parser/multiline_grok.conf
@@ -0,0 +1,10 @@
+ grok_failure_key grokfailure
+{%- if values.get('custom_pattern_path') %}
+ custom_pattern_path {{ values.custom_pattern_path }}
+{%- endif %}
+ multiline_start_regexp {{ values.multiline_start_regexp }}
+{%- for grok_rule in values.get('rule', []) %}
+ <grok>
+ pattern {{ grok_rule.pattern }}
+ </grok>
+{%- endfor %}
\ No newline at end of file
diff --git a/fluentd/files/parser/regexp.conf b/fluentd/files/parser/regexp.conf
new file mode 100644
index 0000000..14b3dbe
--- /dev/null
+++ b/fluentd/files/parser/regexp.conf
@@ -0,0 +1,2 @@
+ expression {{ values.format }}
+ time_format {{ values.time_format }}
diff --git a/fluentd/files/td-agent.conf b/fluentd/files/td-agent.conf
new file mode 100644
index 0000000..e69740c
--- /dev/null
+++ b/fluentd/files/td-agent.conf
@@ -0,0 +1 @@
+@include {{ fluentd.dir.config }}/config.d/*.conf
diff --git a/fluentd/init.sls b/fluentd/init.sls
new file mode 100644
index 0000000..2862a76
--- /dev/null
+++ b/fluentd/init.sls
@@ -0,0 +1,6 @@
+{%- if pillar.fluentd %}
+include:
+ {%- if pillar.fluentd is defined %}
+ - fluentd.agent
+ {%- endif %}
+{%- endif %}
diff --git a/fluentd/map.jinja b/fluentd/map.jinja
new file mode 100644
index 0000000..2531114
--- /dev/null
+++ b/fluentd/map.jinja
@@ -0,0 +1,31 @@
+{% set fluentd = salt['grains.filter_by']({
+ 'Debian': {
+ 'user': 'root',
+ 'group': 'root',
+ 'pkgs': ['td-agent', 'build-essential', 'ruby-dev'],
+ 'gems': ['fluent-plugin-prometheus', 'fluent-plugin-grok-parser', 'fluent-plugin-systemd'],
+ 'gem_path': 'td-agent-gem',
+ 'service_name': 'td-agent',
+ 'dir': {
+ 'config': '/etc/td-agent'
+ },
+ 'config': {
+ },
+ },
+}, merge=salt['pillar.get']('fluentd')) %}
+
+{# Collect configuration from */meta/fluentd.yml #}
+{%- set fluentd_grains = {'config': {'input': {}, 'filter': {}, 'match': {}, 'label': {}}} %}
+{%- for service_name, service in pillar.items() %}
+ {%- if service.get('_support', {}).get('fluentd', {}).get('enabled', False) %}
+ {%- set grains_fragment_file = service_name+'/meta/fluentd.yml' %}
+ {%- macro load_grains_file() %}{% include grains_fragment_file ignore missing %}{% endmacro %}
+ {%- set grains_yaml = load_grains_file()|load_yaml %}
+ {%- if grains_yaml is mapping %}
+ {%- set fluentd_grains = salt['grains.filter_by']({'default': fluentd_grains}, merge=grains_yaml) %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+
+{# Deep-merge the service configuration with the pillar data #}
+{%- do salt['defaults.merge'](fluentd, fluentd_grains) %}