Enable logging.conf & fluentd for glance
This change introduce ability to use log_config_append parameter with
new FluentdHandler to send logs directly to Fluentd.
To save per binary log streams as it is in the default logging states generates
separate logging.conf files per service and use /etc/default/<service name>
to pass dedicated logging.conf to every service.
Change-Id: Ie592faf9c91c71d5cc551bac62b25a19f04eb691
Related-Prod: PROD-16324
diff --git a/README.rst b/README.rst
index 415c31f..f693d68 100644
--- a/README.rst
+++ b/README.rst
@@ -322,6 +322,30 @@
protected: false
location: http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-i386-disk.img
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ glance:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
Usage
=====
diff --git a/glance/files/default b/glance/files/default
new file mode 100644
index 0000000..815ede9
--- /dev/null
+++ b/glance/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/nova/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/glance/files/logging.conf b/glance/files/logging.conf
new file mode 100644
index 0000000..37d54d2
--- /dev/null
+++ b/glance/files/logging.conf
@@ -0,0 +1,77 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, glance
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_glance]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = glance
+propagate = 0
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither. (Recommended for production systems.)
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".") }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/glance/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/glance/map.jinja b/glance/map.jinja
index e9a3f3d..fc912f3 100644
--- a/glance/map.jinja
+++ b/glance/map.jinja
@@ -16,7 +16,15 @@
'enabled': false
},
'glance_uid': 302,
- 'glance_gid': 302
+ 'glance_gid': 302,
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-glance', 'python-glanceclient','python-pycadf'],
@@ -27,7 +35,15 @@
'enabled': false
},
'glance_uid': 302,
- 'glance_gid': 302
+ 'glance_gid': 302,
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.glance.get('server', {}), base='BaseDefaults') %}
diff --git a/glance/meta/fluentd.yml b/glance/meta/fluentd.yml
new file mode 100644
index 0000000..7b1fc2b
--- /dev/null
+++ b/glance/meta/fluentd.yml
@@ -0,0 +1,110 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_glance:
+ tag: openstack.glance.*
+ type: relabel
+ label: openstack_glance
+ openstack_glance:
+ filter:
+ set_glance_fields:
+ tag: openstack.glance.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: programname
+ value: glance-${ tag_parts[2] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ parse_http_stats:
+ tag: openstack.glance
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.glance.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.glance
+ send_to_default:
+ tag: openstack.glance
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.glance_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.glance_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ glance_logs_per_severity:
+ tag: metric.glance_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: glance
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ glance_openstack_http_response_times:
+ tag: metric.glance_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: glance
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/glance/server.sls b/glance/server.sls
index 9812e27..b160b7c 100644
--- a/glance/server.sls
+++ b/glance/server.sls
@@ -107,6 +107,73 @@
{%- endif %}
{%- endif %}
+{%- set glance_glare_available = server.version in ['newton', 'ocata'] %}
+{%- if glance_glare_available %}
+ {%- set glance_services_list = server.services %}
+{%- else %}
+ {%- set glance_services_list = server.services + ['glance-glare'] %}
+{%- endif %}
+
+{%- if not grains.get('noservices', False) %}
+{%- for service_name in glance_services_list %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://glance/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: glance_packages
+{%- if glance_glare_available %}
+ - pkg: glance_glare_package
+{%- endif %}
+ - watch_in:
+ - service: glance_services
+{%- if glance_glare_available %}
+ - service: glance_glare_service
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+
+{%- if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+glance_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in glance_services_list %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/glance/logging/logging-{{ service_name }}.conf
+ - source: salt://glance/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - user: glance
+ - group: glance
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: glance_packages
+{%- if glance_glare_available %}
+ - pkg: glance_glare_package
+{%- endif %}
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: glance_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: glance_services
+{%- if glance_glare_available %}
+ - service: glance_glare_service
+{%- endif %}
+{%- endfor %}
+
+{%- endif %}
+
{% if server.storage.get('swift', {}).get('store', {}).get('references', {}) %}
/etc/glance/swift-stores.conf:
file.managed:
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index 2ede4d7..857f2e9 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_glance_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
glance:
server:
enabled: true
@@ -34,6 +36,13 @@
region: RegionOne
tenant: service
endpoint_type: ${_param:keystone_glance_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index 2abc3bd..a0858c3 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_glance_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
glance:
server:
enabled: true
@@ -32,6 +34,13 @@
region: RegionOne
tenant: service
endpoint_type: ${_param:keystone_glance_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index a711054..bf29b91 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu: