Allow tuning buffers
- general buffer config
- add buffer, flushing, retry options in file buffer
- add flushing, retry options in mem buffer
Based on: https://docs.fluentd.org/configuration/buffer-section
Change-Id: I8fed7f97acc304c378ea3a5ba2d94304c5b39e1f
Related-bug: PROD-35697
(cherry picked from commit 6aa2672832631074efb5f95f0c1182bffbf01a4c)
diff --git a/fluentd/files/buffer/_general.conf b/fluentd/files/buffer/_general.conf
new file mode 100644
index 0000000..5ee8354
--- /dev/null
+++ b/fluentd/files/buffer/_general.conf
@@ -0,0 +1,55 @@
+ {%- if values.get("flush_at_shutdown") %}
+ flush_at_shutdown {{ values.get("flush_at_shutdown") }}
+ {%- endif %}
+ {%- if values.get("flush_mode") %}
+ flush_mode {{ values.get("flush_mode") }}
+ {%- endif %}
+ {%- if values.get("flush_interval") %}
+ flush_interval {{ values.get("flush_interval") }}
+ {%- endif %}
+ {%- if values.get("flush_thread_interval") %}
+ flush_thread_interval {{ values.get("flush_thread_interval") }}
+ {%- endif %}
+ {%- if values.get("flush_thread_burst_interval") %}
+ flush_thread_burst_interval {{ values.get("flush_thread_burst_interval") }}
+ {%- endif %}
+ {%- if values.get("delayed_commit_timeout") %}
+ delayed_commit_timeout {{ values.get("delayed_commit_timeout") }}
+ {%- endif %}
+ {%- if values.get("overflow_action") %}
+ overflow_action {{ values.get("overflow_action") }}
+ {%- endif %}
+
+ {%- if values.get("retry_timeout") %}
+ retry_timeout {{ values.get("retry_timeout") }}
+ {%- endif %}
+ {%- if values.get("retry_forever") %}
+ retry_forever {{ values.get("retry_forever") }}
+ {%- endif %}
+ {%- if values.get("retry_max_times") %}
+ retry_max_times {{ values.get("retry_max_times") }}
+ {%- endif %}
+ {%- if values.get("retry_secondary_threshold") %}
+ retry_secondary_threshold {{ values.get("retry_secondary_threshold") }}
+ {%- endif %}
+ {%- if values.get("retry_type") %}
+ retry_type {{ values.get("retry_type") }}
+ {%- endif %}
+ {%- if values.get("retry_wait") %}
+ retry_wait {{ values.get("retry_wait") }}
+ {%- endif %}
+ {%- if values.get("retry_exponential_backoff_base") %}
+ retry_exponential_backoff_base {{ values.get("retry_exponential_backoff_base") }}
+ {%- endif %}
+ {%- if values.get("retry_exponential_backoff_base") %}
+ retry_exponential_backoff_base {{ values.get("retry_exponential_backoff_base") }}
+ {%- endif %}
+ {%- if values.get("retry_max_interval") %}
+ retry_max_interval {{ values.get("retry_max_interval") }}
+ {%- endif %}
+ {%- if values.get("retry_randomize") %}
+ retry_randomize {{ values.get("retry_randomize") }}
+ {%- endif %}
+ {%- if values.get("disable_chunk_backup") %}
+ disable_chunk_backup {{ values.get("disable_chunk_backup") }}
+ {%- endif %}
diff --git a/fluentd/files/buffer/file.conf b/fluentd/files/buffer/file.conf
index aaa9691..b71370f 100644
--- a/fluentd/files/buffer/file.conf
+++ b/fluentd/files/buffer/file.conf
@@ -1 +1,23 @@
path {{ values.get("path", "/var/lib/fluentd/default.buffer") }}
+ {%- if values.get("chunk_limit_size") %}
+ chunk_limit_size {{ values.get("chunk_limit_size") }}
+ {%- endif %}
+ {%- if values.get("chunk_limit_records") %}
+ chunk_limit_records {{ values.get("chunk_limit_records") }}
+ {%- endif %}
+ {%- if values.get("total_limit_size") %}
+ total_limit_size {{ values.get("total_limit_size") }}
+ {%- endif %}
+ {%- if values.get("queue_limit_length") %}
+ queue_limit_length {{ values.get("queue_limit_length") }}
+ {%- endif %}
+ {%- if values.get("chunk_full_threshold") %}
+ chunk_full_threshold {{ values.get("chunk_full_threshold") }}
+ {%- endif %}
+ {%- if values.get("queued_chunks_limit_size") %}
+ queued_chunks_limit_size {{ values.get("queued_chunks_limit_size") }}
+ {%- endif %}
+ {%- if values.get("compress") %}
+ compress {{ values.get("compress") }}
+ {%- endif %}
+{% include "fluentd/files/buffer/_general.conf" %}
diff --git a/fluentd/files/buffer/memory.conf b/fluentd/files/buffer/memory.conf
index 9133e05..2b5e2e1 100644
--- a/fluentd/files/buffer/memory.conf
+++ b/fluentd/files/buffer/memory.conf
@@ -2,4 +2,11 @@
total_limit_size {{ values.get("total_limit_size", "512m") }}
queue_limit_length {{ values.get("queue_limit_length", 100) }}
chunk_limit_records {{ values.get("chunk_limit_records", 10000) }}
- compress {{ values.get("compress", "gzip") }}
\ No newline at end of file
+ compress {{ values.get("compress", "gzip") }}
+ {%- if values.get("chunk_full_threshold") %}
+ chunk_full_threshold {{ values.get("chunk_full_threshold") }}
+ {%- endif %}
+ {%- if values.get("queued_chunks_limit_size") %}
+ queued_chunks_limit_size {{ values.get("queued_chunks_limit_size") }}
+ {%- endif %}
+{% include "fluentd/files/buffer/_general.conf" %}