Merge "Add elasticsearch output & memory buffer"
diff --git a/fluentd/files/buffer/_generate.conf b/fluentd/files/buffer/_generate.conf
new file mode 100644
index 0000000..40104f7
--- /dev/null
+++ b/fluentd/files/buffer/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Filter {{ name }}
+{%- if values.get('enabled', True) %}
+{% include 'fluentd/files/buffer/' + values.get('type') + '.conf' %}
+{%- endif %}
+{%- endfor %}
\ No newline at end of file
diff --git a/fluentd/files/buffer/memory.conf b/fluentd/files/buffer/memory.conf
new file mode 100644
index 0000000..9133e05
--- /dev/null
+++ b/fluentd/files/buffer/memory.conf
@@ -0,0 +1,5 @@
+ chunk_limit_size {{ values.get("chunk_limit_size", "32m") }}
+ total_limit_size {{ values.get("total_limit_size", "512m") }}
+ queue_limit_length {{ values.get("queue_limit_length", 100) }}
+ chunk_limit_records {{ values.get("chunk_limit_records", 10000) }}
+ compress {{ values.get("compress", "gzip") }}
\ No newline at end of file
diff --git a/fluentd/files/match/elasticsearch.conf b/fluentd/files/match/elasticsearch.conf
new file mode 100644
index 0000000..6f7434e
--- /dev/null
+++ b/fluentd/files/match/elasticsearch.conf
@@ -0,0 +1,22 @@
+<match {{ values.tag }}>
+ @type elasticsearch
+ host {{ values.host }}
+ port {{ values.get("port", 9200) }}
+ type_name {{ values.get("es_type_name", "message") }}
+ tag_key Logger
+ include_tag_key true
+ time_key Timestamp
+ time_key_exclude_timestamp true
+ logstash_format true
+ logstash_prefix {{ values.get("es_index_name", "log") }}
+ logstash_dateformat %Y%m%d
+ flush_interval {{ values.get("flush_interval", "10s") }}
+{%- if values.get("buffer") %}
+ <buffer>
+ {%- with values=values.get("buffer") %}
+ @type {{ values.get("type") }}
+{%- include 'fluentd/files/buffer/_generate.conf' %}
+ {%- endwith %}
+ </buffer>
+{%- endif %}
+</match>
\ No newline at end of file
diff --git a/fluentd/map.jinja b/fluentd/map.jinja
index 2531114..db4852b 100644
--- a/fluentd/map.jinja
+++ b/fluentd/map.jinja
@@ -3,7 +3,7 @@
'user': 'root',
'group': 'root',
'pkgs': ['td-agent', 'build-essential', 'ruby-dev'],
- 'gems': ['fluent-plugin-prometheus', 'fluent-plugin-grok-parser', 'fluent-plugin-systemd'],
+ 'gems': ['fluent-plugin-prometheus', 'fluent-plugin-grok-parser', 'fluent-plugin-systemd', 'fluent-plugin-elasticsearch'],
'gem_path': 'td-agent-gem',
'service_name': 'td-agent',
'dir': {