Enable logging.conf & fluentd for heat
This change introduce ability to use log_config_append parameter with
new FluentdHandler to send logs directly to Fluentd.
To save per binary log streams as it is in the default logging states generates
separate logging.conf files per service and use /etc/default/<service name>
to pass dedicated logging.conf to every service.
Change-Id: I283e95bf7c2902b2c7830db54de5c49d4fda8fa0
Related-Prod: PROD-16324
diff --git a/README.rst b/README.rst
index 41b793b..24582ca 100644
--- a/README.rst
+++ b/README.rst
@@ -218,6 +218,31 @@
* The `message_queue.port` is set to **5671** (AMQPS) by default if `ssl.enabled=True`.
* Use `message_queue.ssl.version` if you need to specify protocol version. By default is TLSv1 for python < 2.7.9 and TLSv1_2 for version above.
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ heat:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
Documentation and Bugs
======================
diff --git a/heat/files/default b/heat/files/default
new file mode 100644
index 0000000..9284a0f
--- /dev/null
+++ b/heat/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/heat/logging/logging-{{ service_name }}.conf"
+{% endif %}
\ No newline at end of file
diff --git a/heat/files/logging.conf b/heat/files/logging.conf
new file mode 100644
index 0000000..9596673
--- /dev/null
+++ b/heat/files/logging.conf
@@ -0,0 +1,77 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, heat
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_heat]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = heat
+propagate = 0
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither. (Recommended for production systems.)
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/heat/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/heat/files/mitaka/heat.conf.Debian b/heat/files/mitaka/heat.conf.Debian
index 1c43235..50efa5d 100644
--- a/heat/files/mitaka/heat.conf.Debian
+++ b/heat/files/mitaka/heat.conf.Debian
@@ -21,6 +21,9 @@
# files, see the Python logging module documentation. (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+log_config_append=/etc/heat/logging.conf
+{%- endif %}
# DEPRECATED. A logging.Formatter log message format string which may use any
# of the available logging.LogRecord attributes. This option is deprecated.
diff --git a/heat/files/newton/heat.conf.Debian b/heat/files/newton/heat.conf.Debian
index 4c2bc4d..4e82cc4 100644
--- a/heat/files/newton/heat.conf.Debian
+++ b/heat/files/newton/heat.conf.Debian
@@ -21,6 +21,9 @@
# files, see the Python logging module documentation. (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+log_config_append=/etc/heat/logging.conf
+{%- endif %}
# DEPRECATED. A logging.Formatter log message format string which may use any
# of the available logging.LogRecord attributes. This option is deprecated.
diff --git a/heat/files/ocata/heat.conf.Debian b/heat/files/ocata/heat.conf.Debian
index d7ee431..58d1461 100644
--- a/heat/files/ocata/heat.conf.Debian
+++ b/heat/files/ocata/heat.conf.Debian
@@ -316,6 +316,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+log_config_append=/etc/heat/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/heat/map.jinja b/heat/map.jinja
index 6a1743e..4653825 100644
--- a/heat/map.jinja
+++ b/heat/map.jinja
@@ -12,14 +12,30 @@
'services': ['heat-api', 'heat-api-cfn', 'heat-api-cloudwatch', 'heat-engine'],
'notification': False,
'cors': {},
- 'clients': {}
+ 'clients': {},
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-heat-api', 'openstack-heat-api-cfn', 'openstack-heat-api-cloudwatch', 'openstack-heat-engine', 'openstack-heat-common'],
'services': ['openstack-heat-api', 'openstack-heat-api-cfn', 'openstack-heat-api-cloudwatch', 'openstack-heat-engine'],
'notification': False,
'cors': {},
- 'clients': {}
+ 'clients': {},
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.heat.get('server', {}), base='BaseDefaults') %}
diff --git a/heat/meta/fluentd.yml b/heat/meta/fluentd.yml
new file mode 100644
index 0000000..8e554a7
--- /dev/null
+++ b/heat/meta/fluentd.yml
@@ -0,0 +1,117 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_heat:
+ tag: openstack.heat.**
+ type: relabel
+ label: openstack_heat
+ openstack_heat:
+ filter:
+ set_programname:
+ tag: openstack.heat.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: heat-${ tag_parts[2] }
+ set_heat_fields:
+ tag: openstack.heat
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "heat" }'
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ parse_http_stats:
+ tag: openstack.heat
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.heat.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.heat
+ send_to_default:
+ tag: openstack.heat
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.heat_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.heat_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ heat_logs_per_severity:
+ tag: metric.heat_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: heat
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ heat_openstack_http_response_times:
+ tag: metric.heat_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: heat
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/heat/server.sls b/heat/server.sls
index 3d22ebd..bee5b3c 100644
--- a/heat/server.sls
+++ b/heat/server.sls
@@ -19,6 +19,78 @@
- require:
- pkg: heat_server_packages
+{%- for service_name in server.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://heat/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: heat_server_packages
+ - watch_in:
+ - service: heat_server_services
+{%- endfor %}
+
+
+{%- if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+heat_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+heat_general_logging_conf:
+ file.managed:
+ - name: /etc/heat/logging.conf
+ - source: salt://heat/files/logging.conf
+ - template: jinja
+ - user: heat
+ - group: heat
+ - defaults:
+ service_name: heat
+ values: {{ server }}
+ - require:
+ - pkg: heat_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: heat_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: heat_server_services
+
+/var/log/heat/heat.log:
+ file.managed:
+ - user: heat
+ - group: heat
+ - watch_in:
+ - service: heat_server_services
+
+{% for service_name in server.get('services', []) %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/heat/logging/logging-{{ service_name }}.conf
+ - source: salt://heat/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - user: heat
+ - group: heat
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: heat_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: heat_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: heat_server_services
+{% endfor %}
+
+{% endif %}
+
{%- for name, rule in server.get('policy', {}).iteritems() %}
{%- if rule != None %}
diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml
index 67ddc7f..8c8476f 100644
--- a/metadata/service/server/cluster.yml
+++ b/metadata/service/server/cluster.yml
@@ -6,6 +6,8 @@
_param:
heat_client_default_endpoint_type: internalURL
heat_client_heat_endpoint_type: publicURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
heat:
server:
stack_domain_admin:
@@ -50,6 +52,13 @@
password: ${_param:keystone_heat_password}
endpoint_type_default: ${_param:heat_client_default_endpoint_type}
endpoint_type_heat: ${_param:heat_client_heat_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/single.yml b/metadata/service/server/single.yml
index 0e60d6a..79be27a 100644
--- a/metadata/service/server/single.yml
+++ b/metadata/service/server/single.yml
@@ -6,6 +6,8 @@
_param:
heat_client_default_endpoint_type: internalURL
heat_client_heat_endpoint_type: publicURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
heat:
server:
stack_domain_admin:
@@ -50,6 +52,13 @@
password: ${_param:keystone_heat_password}
endpoint_type_default: ${_param:heat_client_default_endpoint_type}
endpoint_type_heat: ${_param:heat_client_heat_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index e39cbbd..a2c7a62 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu: