Add drop for sensitive and incompatible fields
Nested timestamp in openstack context contains data with
incompatible for elasticsearch format.
Drop all token related fields
Change-Id: If3b228a7d1f0cab5446bc08f4b3838b9b03b738f
diff --git a/fluentd/label/default_output/elasticsearch.yml b/fluentd/label/default_output/elasticsearch.yml
index 3c26d15..84b2dc1 100644
--- a/fluentd/label/default_output/elasticsearch.yml
+++ b/fluentd/label/default_output/elasticsearch.yml
@@ -9,6 +9,19 @@
config:
label:
default_output:
+ filter:
+ drop_nested_timestamp_and_sensitive_data:
+ tag: "openstack.**"
+ type: record_transformer
+ enable_ruby: true
+ remove_keys: '["_dummy_1", "_dummy_2", "_dummy_3"]'
+ record:
+ - name: _dummy_1
+ value: ${fluentd:dollar}{if record.has_key?("context"); record["context"].delete("timestamp") ; end; nil }
+ - name: _dummy_2
+ value: ${fluentd:dollar}{if record.has_key?("context"); record["context"].delete("auth_token"); end; nil}
+ - name: _dummy_3
+ value: ${fluentd:dollar}{if record.has_key?("context"); record["context"].delete("auth_token_info"); end; nil}
match:
elasticsearch_output:
host: ${_param:fluentd_elasticsearch_host}