Enable logging.conf & fluentd for ceilometer
This change introduce ability to use log_config_append parameter with
new FluentdHandler to send logs directly to Fluentd.
To save per binary log streams as it is in the default logging states generates
separate logging.conf files per service and use /etc/default/<service name>
to pass dedicated logging.conf to every service.
Change-Id: I65d2cc736e4475be0e23c02c234f4b3388900e7a
Related-Prod: PROD-16324
diff --git a/README.rst b/README.rst
index 44e9150..f165775 100644
--- a/README.rst
+++ b/README.rst
@@ -211,7 +211,39 @@
- host: 10.10.10.12
port: 11211
+Enhanced logging with logging.conf
+----------------------------------
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ ceilometer:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
+ agent:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
More Information
diff --git a/ceilometer/agent.sls b/ceilometer/agent.sls
index a574ce6..9370827 100644
--- a/ceilometer/agent.sls
+++ b/ceilometer/agent.sls
@@ -12,6 +12,52 @@
- require:
- pkg: ceilometer_agent_packages
+{% for service_name in agent.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://ceilometer/files/default
+ - template: jinja
+ - require:
+ - pkg: ceilometer_agent_packages
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ agent }}
+ - watch_in:
+ - service: ceilometer_agent_services
+{% endfor %}
+
+{% if agent.logging.log_appender -%}
+
+{%- if agent.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ceilometer_agent_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in agent.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/ceilometer_agent/logging/logging-{{ service_name }}.conf
+ - source: salt://ceilometer/files/logging.conf
+ - template: jinja
+ - user: ceilometer
+ - group: ceilometer
+ - require:
+ - pkg: ceilometer_agent_packages
+{%- if agent.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: ceilometer_agent_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ agent }}
+ - watch_in:
+ - service: ceilometer_agent_services
+{% endfor %}
+
+{% endif %}
+
{%- for publisher_name, publisher in agent.get('publisher', {}).iteritems() %}
{%- if publisher_name != "default" %}
diff --git a/ceilometer/files/default b/ceilometer/files/default
new file mode 100644
index 0000000..857d54b
--- /dev/null
+++ b/ceilometer/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/ceilometer/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/ceilometer/files/logging.conf b/ceilometer/files/logging.conf
new file mode 100644
index 0000000..dfd181c
--- /dev/null
+++ b/ceilometer/files/logging.conf
@@ -0,0 +1,49 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, ceilometer
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_ceilometer]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = ceilometer
+propagate = 0
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/ceilometer/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/ceilometer/files/mitaka/ceilometer-server.conf.Debian b/ceilometer/files/mitaka/ceilometer-server.conf.Debian
index 15895e9..e695903 100644
--- a/ceilometer/files/mitaka/ceilometer-server.conf.Debian
+++ b/ceilometer/files/mitaka/ceilometer-server.conf.Debian
@@ -13,6 +13,9 @@
{%- else %}
debug = False
{%- endif %}
+{%- if server.logging.log_appender %}
+log_config_append=/etc/ceilometer/logging.conf
+{%- endif %}
# Print more verbose output (set logging level to INFO instead
# of default WARNING level). (boolean value)
diff --git a/ceilometer/files/newton/ceilometer-server.conf.Debian b/ceilometer/files/newton/ceilometer-server.conf.Debian
index 82e1deb..a571039 100644
--- a/ceilometer/files/newton/ceilometer-server.conf.Debian
+++ b/ceilometer/files/newton/ceilometer-server.conf.Debian
@@ -25,6 +25,9 @@
# example, logging_context_format_string). (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/ceilometer/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/ceilometer/files/newton/ceilometer.apache2.conf.Debian b/ceilometer/files/newton/ceilometer.apache2.conf.Debian
index 988ab30..bf28270 100644
--- a/ceilometer/files/newton/ceilometer.apache2.conf.Debian
+++ b/ceilometer/files/newton/ceilometer.apache2.conf.Debian
@@ -10,5 +10,5 @@
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/ceilometer_error.log
- CustomLog /var/log/apache2/ceilometer_access.log combined
+ CustomLog /var/log/apache2/ceilometer_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
</VirtualHost>
diff --git a/ceilometer/files/ocata/ceilometer-server.conf.Debian b/ceilometer/files/ocata/ceilometer-server.conf.Debian
index 0316403..895cdad 100644
--- a/ceilometer/files/ocata/ceilometer-server.conf.Debian
+++ b/ceilometer/files/ocata/ceilometer-server.conf.Debian
@@ -161,6 +161,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/ceilometer/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/ceilometer/files/ocata/ceilometer.apache2.conf.Debian b/ceilometer/files/ocata/ceilometer.apache2.conf.Debian
index a0406fe..44cea9b 100644
--- a/ceilometer/files/ocata/ceilometer.apache2.conf.Debian
+++ b/ceilometer/files/ocata/ceilometer.apache2.conf.Debian
@@ -10,5 +10,5 @@
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/ceilometer_error.log
- CustomLog /var/log/apache2/ceilometer_access.log combined
+ CustomLog /var/log/apache2/ceilometer_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
</VirtualHost>
diff --git a/ceilometer/map.jinja b/ceilometer/map.jinja
index 6f40616..869f393 100644
--- a/ceilometer/map.jinja
+++ b/ceilometer/map.jinja
@@ -2,10 +2,26 @@
'Debian': {
'pkgs': ['ceilometer-agent-compute'],
'services': ['ceilometer-agent-compute'],
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-ceilometer-compute'],
'services': ['openstack-ceilometer-compute'],
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=salt['pillar.get']('ceilometer:agent')) %}
@@ -41,10 +57,21 @@
},
}) %}
-{%- set server = salt['grains.filter_by']({'default': {}}, merge=salt['pillar.get']('ceilometer:server')) %}
+{%- set server = salt['grains.filter_by']({
+ 'default': {
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
+ }
+}, merge=salt['pillar.get']('ceilometer:server')) %}
-# Mitaka and newer OpenStack releases don't need collector and alarm
-# packages/services, because alarming is implemented by Aodh and collector
+# Mitaka and newer OpenStack releases don't need collector and alarm
+# packages/services, because alarming is implemented by Aodh and collector
# stuff is implemented by the ceilometer_collector service.
# The ceilometer api for Ocata and newer is served by webserver.
{%- if salt['pillar.get']('ceilometer:server:version', 'mitaka') in ['liberty', 'juno', 'kilo'] %}
diff --git a/ceilometer/meta/fluentd.yml b/ceilometer/meta/fluentd.yml
new file mode 100644
index 0000000..d5d868e
--- /dev/null
+++ b/ceilometer/meta/fluentd.yml
@@ -0,0 +1,172 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- from "ceilometer/map.jinja" import server with context %}
+{%- set apache_wsgi = server.get('enabled') and server.version not in ['mitaka'] %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_ceilometer:
+ tag: openstack.ceilometer.**
+ type: relabel
+ label: openstack_ceilometer
+{%- if apache_wsgi %}
+ openstack_ceilometer_wsgi:
+ input:
+ ceilometer_api_wsgi_in_tail:
+ type: tail
+ path: /var/log/apache2/ceilometer_access.log
+ tag: openstack.ceilometer
+ pos_file: {{ positiondb }}/ceilometer.wsgi.pos
+ parser:
+ type: regexp
+ time_key: Timestamp
+ time_format: '%d/%b/%Y:%H:%M:%S %z'
+ keep_time_key: false
+ # Apache format: https://regex101.com/r/WeCT7s/5
+ format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+ filter:
+ add_ceilometer_wsgi_record_fields:
+ tag: openstack.ceilometer
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: 6
+ - name: severity_label
+ value: INFO
+ - name: programname
+ value: ceilometer-wsgi
+ - name: http_response_time
+ value: ${ record['http_response_time'].to_i/100000.to_f }
+ match:
+ send_to_default:
+ tag: openstack.ceilometer
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.ceilometer_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.ceilometer_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+{%- endif %}
+ openstack_ceilometer:
+ filter:
+ set_ceilometer_programname:
+ tag: openstack.ceilometer.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: ceilometer-${ tag_parts[2] }
+ set_log_record_fields:
+ tag: openstack.ceilometer
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: programname
+ value: ceilometer-${ tag_parts[2] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "ceilometer" }'
+ parse_http_stats:
+ tag: openstack.ceilometer
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.ceilometer.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.ceilometer
+ send_to_default:
+ tag: openstack.ceilometer
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.ceilometer_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.ceilometer_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ ceilometer_logs_per_severity:
+ tag: metric.ceilometer_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: ceilometer
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ ceilometer_openstack_http_response_times:
+ tag: metric.ceilometer_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: ceilometer
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/ceilometer/server.sls b/ceilometer/server.sls
index d0f1fc6..f2bb9d0 100644
--- a/ceilometer/server.sls
+++ b/ceilometer/server.sls
@@ -12,6 +12,81 @@
- require:
- pkg: ceilometer_server_packages
+{%- for service_name in server.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://ceilometer/files/default
+ - template: jinja
+ - require:
+ - pkg: ceilometer_server_packages
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - watch_in:
+ - service: ceilometer_server_services
+{%- endfor %}
+
+{%- if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ceilometer_server_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+ceilometer_general_logging_conf:
+ file.managed:
+ - name: /etc/ceilometer/logging.conf
+ - source: salt://ceilometer/files/logging.conf
+ - template: jinja
+ - user: ceilometer
+ - group: ceilometer
+ - require:
+ - pkg: ceilometer_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: ceilometer_server_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: ceilometer
+ values: {{ server }}
+ - watch_in:
+ - service: ceilometer_server_services
+{%- if server.version not in ['liberty', 'juno', 'kilo', 'mitaka'] %}
+ - service: ceilometer_apache_restart
+{%- endif %}
+
+/var/log/ceilometer/ceilometer.log:
+ file.managed:
+ - user: ceilometer
+ - group: ceilometer
+ - watch_in:
+ - service: ceilometer_server_services
+{%- if server.version not in ['liberty', 'juno', 'kilo', 'mitaka'] %}
+ - service: ceilometer_apache_restart
+{%- endif %}
+
+{%- for service_name in server.get('services', []) %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/ceilometer_server/logging/logging-{{ service_name }}.conf
+ - source: salt://ceilometer/files/logging.conf
+ - template: jinja
+ - require:
+ - pkg: ceilometer_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: ceilometer_server_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - watch_in:
+ - service: ceilometer_server_services
+{%- endfor %}
+
+{%- endif %}
+
{%- for name, rule in server.get('policy', {}).iteritems() %}
{%- if rule != None %}
diff --git a/metadata/service/agent/cluster.yml b/metadata/service/agent/cluster.yml
index 1e56978..91cf7d9 100644
--- a/metadata/service/agent/cluster.yml
+++ b/metadata/service/agent/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_ceilometer_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
ceilometer:
agent:
enabled: true
@@ -20,6 +22,13 @@
user: ceilometer
password: ${_param:keystone_ceilometer_password}
endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/agent/single.yml b/metadata/service/agent/single.yml
index dfffecf..23ee9ee 100644
--- a/metadata/service/agent/single.yml
+++ b/metadata/service/agent/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_ceilometer_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
ceilometer:
agent:
enabled: true
@@ -20,6 +22,13 @@
user: ceilometer
password: ${_param:keystone_ceilometer_password}
endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml
index eaedfa1..5b190a8 100644
--- a/metadata/service/server/cluster.yml
+++ b/metadata/service/server/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_ceilometer_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
ceilometer:
server:
enabled: true
@@ -25,6 +27,13 @@
user: ceilometer
password: ${_param:keystone_ceilometer_password}
endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/single.yml b/metadata/service/server/single.yml
index f150a61..d246599 100644
--- a/metadata/service/server/single.yml
+++ b/metadata/service/server/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_ceilometer_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
ceilometer:
server:
enabled: true
@@ -24,6 +26,13 @@
user: ceilometer
password: ${_param:keystone_ceilometer_password}
endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 24f09b4..d423400 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu: