Add creating of directory for pos files
Put everything under role `agent`
Change-Id: I0944e67c902790cdbdb5a88952d1f28482f70d7d
diff --git a/fluentd/agent.sls b/fluentd/agent.sls
index 02fafbe..463247f 100644
--- a/fluentd/agent.sls
+++ b/fluentd/agent.sls
@@ -1,20 +1,20 @@
-{% from "fluentd/map.jinja" import fluentd with context %}
-{%- if fluentd.get('enabled', False) %}
+{% from "fluentd/map.jinja" import fluentd_agent with context %}
+{%- if fluentd_agent.get('enabled', False) %}
fluentd_packages_agent:
pkg.installed:
- - names: {{ fluentd.pkgs }}
+ - names: {{ fluentd_agent.pkgs }}
fluentd_gems_agent:
gem.installed:
- - names: {{ fluentd.gems }}
- - gem_bin: {{ fluentd.gem_path }}
+ - names: {{ fluentd_agent.gems }}
+ - gem_bin: {{ fluentd_agent.gem_path }}
- require:
- pkg: fluentd_packages_agent
fluentd_config_d_dir:
file.directory:
- - name: {{ fluentd.dir.config }}/config.d
+ - name: {{ fluentd_agent.dir.config }}/config.d
- makedirs: True
- mode: 755
- require:
@@ -22,11 +22,21 @@
fluentd_config_d_dir_clean:
file.directory:
- - name: {{ fluentd.dir.config }}/config.d
+ - name: {{ fluentd_agent.dir.config }}/config.d
- clean: True
- watch_in:
- service: fluentd_service_agent
+fluentd_positiondb_dir:
+ file.directory:
+ - name: {{ fluentd_agent.dir.positiondb }}
+ - user: {{ fluentd_agent.user }}
+ - group: {{ fluentd_agent.group }}
+ - makedirs: True
+ - mode: 755
+ - require:
+ - pkg: fluentd_packages_agent
+
fluentd_config_service:
file.managed:
- name: /etc/default/td-agent
@@ -38,11 +48,11 @@
- require:
- pkg: fluentd_packages_agent
- context:
- fluentd: {{ fluentd }}
+ fluentd_agent: {{ fluentd_agent }}
fluentd_config_agent:
file.managed:
- - name: {{ fluentd.dir.config }}/td-agent.conf
+ - name: {{ fluentd_agent.dir.config }}/td-agent.conf
- source: salt://fluentd/files/td-agent.conf
- user: root
- group: root
@@ -51,11 +61,11 @@
- require:
- pkg: fluentd_packages_agent
- context:
- fluentd: {{ fluentd }}
+ fluentd_agent: {{ fluentd_agent }}
fluentd_grok_pattern_agent:
file.managed:
- - name: {{ fluentd.dir.config }}/config.d/global.grok
+ - name: {{ fluentd_agent.dir.config }}/config.d/global.grok
- source: salt://fluentd/files/global.grok
- user: root
- group: root
@@ -66,14 +76,14 @@
- require_in:
- file: fluentd_config_d_dir_clean
- context:
- fluentd: {{ fluentd }}
+ fluentd_agent: {{ fluentd_agent }}
-{%- set fluentd_config = fluentd.get('config', {}) %}
+{%- set fluentd_config = fluentd_agent.get('config', {}) %}
{%- for name,values in fluentd_config.get('input', {}).iteritems() %}
input_{{ name }}_agent:
file.managed:
- - name: {{ fluentd.dir.config }}/config.d/input-{{ name }}.conf
+ - name: {{ fluentd_agent.dir.config }}/config.d/input-{{ name }}.conf
- source:
- salt://fluentd/files/input/_generate.conf
- user: root
@@ -101,7 +111,7 @@
filter_{{ name }}_agent:
file.managed:
- - name: {{ fluentd.dir.config }}/config.d/filter-{{ name }}.conf
+ - name: {{ fluentd_agent.dir.config }}/config.d/filter-{{ name }}.conf
- source:
- salt://fluentd/files/filter/_generate.conf
- user: root
@@ -129,7 +139,7 @@
match_{{ name }}_agent:
file.managed:
- - name: {{ fluentd.dir.config }}/config.d/match-{{ name }}.conf
+ - name: {{ fluentd_agent.dir.config }}/config.d/match-{{ name }}.conf
- source:
- salt://fluentd/files/match/_generate.conf
- user: root
@@ -157,7 +167,7 @@
label_{{ label_name }}_agent:
file.managed:
- - name: {{ fluentd.dir.config }}/config.d/label-{{ label_name }}.conf
+ - name: {{ fluentd_agent.dir.config }}/config.d/label-{{ label_name }}.conf
- source:
- salt://fluentd/files/label.conf
- user: root
@@ -183,12 +193,14 @@
fluentd_service_agent:
service.running:
- - name: {{ fluentd.service_name }}
+ - name: {{ fluentd_agent.service_name }}
- enable: True
{%- if grains.get('noservices') %}
- onlyif: /bin/false
{%- endif %}
- watch:
- file: fluentd_config_agent
+ - require:
+ - file: fluentd_positiondb_dir
{%- endif %}
diff --git a/fluentd/files/buffer/_generate.conf b/fluentd/files/buffer/_generate.conf
deleted file mode 100644
index 40104f7..0000000
--- a/fluentd/files/buffer/_generate.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-{%- for name, values in values.iteritems() %}
-# Filter {{ name }}
-{%- if values.get('enabled', True) %}
-{% include 'fluentd/files/buffer/' + values.get('type') + '.conf' %}
-{%- endif %}
-{%- endfor %}
\ No newline at end of file
diff --git a/fluentd/files/default-td-agent b/fluentd/files/default-td-agent
index 3f53213..7343a45 100644
--- a/fluentd/files/default-td-agent
+++ b/fluentd/files/default-td-agent
@@ -1,2 +1,2 @@
-TD_AGENT_USER={{ fluentd.get('user', 'root') }}
-TD_AGENT_GROUP={{ fluentd.get('group', 'root') }}
+TD_AGENT_USER={{ fluentd_agent.get('user', 'root') }}
+TD_AGENT_GROUP={{ fluentd_agent.get('group', 'root') }}
diff --git a/fluentd/files/match/elasticsearch.conf b/fluentd/files/match/elasticsearch.conf
index 6f7434e..7e2b160 100644
--- a/fluentd/files/match/elasticsearch.conf
+++ b/fluentd/files/match/elasticsearch.conf
@@ -15,8 +15,8 @@
<buffer>
{%- with values=values.get("buffer") %}
@type {{ values.get("type") }}
-{%- include 'fluentd/files/buffer/_generate.conf' %}
+{% include 'fluentd/files/buffer/' + values.get('type') + '.conf' %}
{%- endwith %}
</buffer>
{%- endif %}
-</match>
\ No newline at end of file
+</match>
diff --git a/fluentd/files/td-agent.conf b/fluentd/files/td-agent.conf
index e69740c..45ef32e 100644
--- a/fluentd/files/td-agent.conf
+++ b/fluentd/files/td-agent.conf
@@ -1 +1 @@
-@include {{ fluentd.dir.config }}/config.d/*.conf
+@include {{ fluentd_agent.dir.config }}/config.d/*.conf
diff --git a/fluentd/map.jinja b/fluentd/map.jinja
index db4852b..1ced2a0 100644
--- a/fluentd/map.jinja
+++ b/fluentd/map.jinja
@@ -1,4 +1,4 @@
-{% set fluentd = salt['grains.filter_by']({
+{% set fluentd_agent = salt['grains.filter_by']({
'Debian': {
'user': 'root',
'group': 'root',
@@ -7,15 +7,16 @@
'gem_path': 'td-agent-gem',
'service_name': 'td-agent',
'dir': {
- 'config': '/etc/td-agent'
+ 'config': '/etc/td-agent',
+ 'positiondb': '/var/lib/fluentd/positiondb'
},
'config': {
},
},
-}, merge=salt['pillar.get']('fluentd')) %}
+}, merge=salt['pillar.get']('fluentd:agent')) %}
{# Collect configuration from */meta/fluentd.yml #}
-{%- set fluentd_grains = {'config': {'input': {}, 'filter': {}, 'match': {}, 'label': {}}} %}
+{%- set fluentd_grains = {'agent': {'config': {'input': {}, 'filter': {}, 'match': {}, 'label': {}}}} %}
{%- for service_name, service in pillar.items() %}
{%- if service.get('_support', {}).get('fluentd', {}).get('enabled', False) %}
{%- set grains_fragment_file = service_name+'/meta/fluentd.yml' %}
@@ -28,4 +29,4 @@
{%- endfor %}
{# Deep-merge the service configuration with the pillar data #}
-{%- do salt['defaults.merge'](fluentd, fluentd_grains) %}
+{%- do salt['defaults.merge'](fluentd_agent, fluentd_grains) %}
diff --git a/metadata/service/agent/single.yml b/metadata/service/agent/single.yml
index d9824e6..5b6c3d7 100644
--- a/metadata/service/agent/single.yml
+++ b/metadata/service/agent/single.yml
@@ -3,7 +3,8 @@
classes:
- service.fluentd.support
parameters:
- _param:
fluentd:
agent:
enabled: true
+ dir:
+ positiondb: /var/lib/fluentd/positiondb
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 88522d2..1fc7796 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -1,3 +1,4 @@
parameters:
fluentd:
_support:
+ enabled: true
diff --git a/tests/pillar/agent.sls b/tests/pillar/agent.sls
index 10e4cc1..a5afd84 100644
--- a/tests/pillar/agent.sls
+++ b/tests/pillar/agent.sls
@@ -1,28 +1,29 @@
fluentd:
- enabled: true
- config:
- label:
- monitoring:
- filter:
- parse_log:
- tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
- type: parser
- reserve_data: true
- key_name: log
- parser:
- type: regexp
- format: >-
- /^time="(?<time>[^ ]*)" level=(?<severity>[a-zA-Z]*) msg="(?<message>.+?)"/
- time_format: '%FT%TZ'
- remove_log_key:
- tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
- type: record_transformer
- remove_keys: log
- match:
- docker_log:
- tag: 'docker.**'
- type: file
- path: /tmp/flow-docker.log
+ agent:
+ enabled: true
+ config:
+ label:
+ monitoring:
+ filter:
+ parse_log:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: parser
+ reserve_data: true
+ key_name: log
+ parser:
+ type: regexp
+ format: >-
+ /^time="(?<time>[^ ]*)" level=(?<severity>[a-zA-Z]*) msg="(?<message>.+?)"/
+ time_format: '%FT%TZ'
+ remove_log_key:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: record_transformer
+ remove_keys: log
+ match:
+ docker_log:
+ tag: 'docker.**'
+ type: file
+ path: /tmp/flow-docker.log
docker:
host:
enabled: true