Enable logging.conf & fluentd for aodh
This change introduce ability to use log_config_append parameter with
new FluentdHandler to send logs directly to Fluentd.
To save per binary log streams as it is in the default logging states generates
separate logging.conf files per service and use /etc/default/<service name>
to pass dedicated logging.conf to every service.
New prometheus metrics now available:
* log_messages - total number of log lines per severity
* openstack_http_response_times - set of HTTP-stats metrics per HTTP method and status:
- three quantiles, 50%, 90% and 99% for response time
- sum of response time
- count of responses
Change-Id: I87a3ada5d954ef916dfb38b09cac916e96e9b356
Related-Prod: PROD-16324
diff --git a/README.rst b/README.rst
index 2633fbf..fe6ac21 100644
--- a/README.rst
+++ b/README.rst
@@ -5,7 +5,7 @@
Aodh is an alarming service for OpenStack. It used to be a part of Ceilometer, but starting from Mitaka it
is a separate project. Aodh supports several types of alarms like threshold, event, composite and gnocchi-specific.
-In cluster mode, coordination is enabled via tooz with Redis backend.
+In cluster mode, coordination is enabled via tooz with Redis backend.
MySQL is used as a data backend for alarms and alarm history.
Sample pillars
@@ -20,7 +20,7 @@
enabled: true
version: mitaka
ttl: 86400
- cluster: true
+ cluster: true
database:
engine: "mysql+pymysql"
host: 10.0.106.20
@@ -33,11 +33,11 @@
port: 8042
identity:
engine: keystone
- host: 10.0.106.20
+ host: 10.0.106.20
port: 35357
tenant: service
user: aodh
- password: password
+ password: password
message_queue:
engine: rabbitmq
port: 5672
@@ -54,6 +54,31 @@
port: 11211
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ aodh:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
Development and testing
=======================
diff --git a/aodh/files/default b/aodh/files/default
new file mode 100644
index 0000000..b0259c6
--- /dev/null
+++ b/aodh/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/aodh/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/aodh/files/logging.conf b/aodh/files/logging.conf
new file mode 100644
index 0000000..703530c
--- /dev/null
+++ b/aodh/files/logging.conf
@@ -0,0 +1,50 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+
+[loggers]
+keys = root, aodh
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_aodh]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = aodh
+propagate = 0
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/aodh/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/aodh/files/mitaka/aodh.conf.Debian b/aodh/files/mitaka/aodh.conf.Debian
index ef20b06..938e9d0 100644
--- a/aodh/files/mitaka/aodh.conf.Debian
+++ b/aodh/files/mitaka/aodh.conf.Debian
@@ -102,6 +102,9 @@
# logging_context_format_string). (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/aodh/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set.
diff --git a/aodh/files/newton/aodh.conf.Debian b/aodh/files/newton/aodh.conf.Debian
index 83c8e4c..7afacdb 100644
--- a/aodh/files/newton/aodh.conf.Debian
+++ b/aodh/files/newton/aodh.conf.Debian
@@ -68,6 +68,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/aodh/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/aodh/files/newton/apache-aodh.apache2.conf.Debian b/aodh/files/newton/apache-aodh.apache2.conf.Debian
index 25218a8..3a32f6d 100644
--- a/aodh/files/newton/apache-aodh.apache2.conf.Debian
+++ b/aodh/files/newton/apache-aodh.apache2.conf.Debian
@@ -10,7 +10,7 @@
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/aodh.log
- CustomLog /var/log/apache2/aodh_access.log combined
+ CustomLog /var/log/apache2/aodh_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
<Directory /usr/bin>
<IfVersion >= 2.4>
Require all granted
diff --git a/aodh/files/ocata/aodh.conf.Debian b/aodh/files/ocata/aodh.conf.Debian
index 83c8e4c..7afacdb 100644
--- a/aodh/files/ocata/aodh.conf.Debian
+++ b/aodh/files/ocata/aodh.conf.Debian
@@ -68,6 +68,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/aodh/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/aodh/files/ocata/apache-aodh.apache2.conf.Debian b/aodh/files/ocata/apache-aodh.apache2.conf.Debian
index 1822dbe..381468d 100644
--- a/aodh/files/ocata/apache-aodh.apache2.conf.Debian
+++ b/aodh/files/ocata/apache-aodh.apache2.conf.Debian
@@ -10,5 +10,5 @@
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/aodh_error.log
- CustomLog /var/log/apache2/aodh_access.log combined
+ CustomLog /var/log/apache2/aodh_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
</VirtualHost>
diff --git a/aodh/map.jinja b/aodh/map.jinja
index 6a578e4..1bc623c 100644
--- a/aodh/map.jinja
+++ b/aodh/map.jinja
@@ -4,11 +4,27 @@
'pkgs': ['aodh-api', 'aodh-evaluator', 'aodh-notifier', 'aodh-listener', 'python-aodhclient'],
'services': ['aodh-evaluator', 'aodh-notifier', 'aodh-listener'],
'webserved': ['aodh-api'],
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ },
+ }
+ },
},
'RedHat': {
'pkgs':['openstack-aodh-api', 'openstack-aodh-evaluator', 'openstack-aodh-notifier', 'openstack-aodh-listener', 'python-aodhclient'],
'services': ['openstack-aodh-evaluator', 'openstack-aodh-notifier', 'openstack-aodh-listener'],
'webserved': ['openstack-aodh-api'],
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ },
+ }
+ },
},
}, merge=salt['pillar.get']('aodh:server')) %}
diff --git a/aodh/meta/fluentd.yml b/aodh/meta/fluentd.yml
new file mode 100644
index 0000000..e5abc2a
--- /dev/null
+++ b/aodh/meta/fluentd.yml
@@ -0,0 +1,170 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- from "aodh/map.jinja" import server with context %}
+{%- set apache_wsgi = server.version not in ['mitaka'] %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_aodh:
+ tag: openstack.aodh.**
+ type: relabel
+ label: openstack_aodh
+{%- if apache_wsgi %}
+ openstack_aodh_wsgi:
+ input:
+ aodh_api_wsgi_in_tail:
+ type: tail
+ path: /var/log/apache2/aodh_access.log
+ tag: openstack.aodh
+ pos_file: {{ positiondb }}/aodh.wsgi.pos
+ parser:
+ type: regexp
+ time_key: Timestamp
+ time_format: '%d/%b/%Y:%H:%M:%S %z'
+ keep_time_key: false
+ # Apache format: https://regex101.com/r/WeCT7s/5
+ format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+ filter:
+ add_aodh_wsgi_record_fields:
+ tag: openstack.aodh
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: 6
+ - name: severity_label
+ value: INFO
+ - name: programname
+ value: aodh-wsgi
+ - name: http_response_time
+ value: ${ record['http_response_time'].to_i/100000.to_f }
+ match:
+ send_to_default:
+ tag: openstack.aodh
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.aodh_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.aodh_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+{%- endif %}
+ openstack_aodh:
+ filter:
+ set_programname:
+ tag: openstack.aodh.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: aodh-${ tag_parts[2] }
+ set_log_record_fields:
+ tag: openstack.aodh
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "aodh" }'
+ parse_http_stats:
+ tag: openstack.aodh
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.aodh.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.aodh
+ send_to_default:
+ tag: openstack.aodh
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.aodh_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.aodh_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ aodh_logs_per_severity:
+ tag: metric.aodh_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: aodh
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ aodh_openstack_http_response_times:
+ tag: metric.aodh_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: aodh
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/aodh/server.sls b/aodh/server.sls
index f42b279..8c1300a 100644
--- a/aodh/server.sls
+++ b/aodh/server.sls
@@ -14,6 +14,83 @@
- require:
- pkg: aodh_server_packages
+{% for service_name in server.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://aodh/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: aodh_server_packages
+ - watch_in:
+ - service: aodh_server_services
+{% endfor %}
+
+{% if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+aodh_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+aodh_general_logging_conf:
+ file.managed:
+ - name: /etc/aodh/logging.conf
+ - source: salt://aodh/files/logging.conf
+ - template: jinja
+ - user: aodh
+ - group: aodh
+ - require:
+ - pkg: aodh_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: aodh_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: aodh
+ values: {{ server }}
+ - watch_in:
+ - service: aodh_server_services
+{%- if server.version not in ['mitaka'] %}
+ - service: aodh_apache_restart
+{%- endif %}
+
+/var/log/aodh/aodh.log:
+ file.managed:
+ - user: aodh
+ - group: aodh
+ - watch_in:
+ - service: aodh_server_services
+{%- if server.version not in ['mitaka'] %}
+ - service: aodh_apache_restart
+{%- endif %}
+
+{% for service_name in server.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/aodh/logging/logging-{{ service_name }}.conf
+ - source: salt://aodh/files/logging.conf
+ - template: jinja
+ - user: aodh
+ - group: aodh
+ - require:
+ - pkg: aodh_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: aodh_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - watch_in:
+ - service: aodh_server_services
+{% endfor %}
+
+{% endif %}
+
aodh_syncdb:
cmd.run:
- name: aodh-dbsync
diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml
index 00bb18b..1bc8da2 100644
--- a/metadata/service/server/cluster.yml
+++ b/metadata/service/server/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_aodh_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
aodh:
server:
enabled: true
@@ -31,6 +33,13 @@
user: aodh
password: ${_param:keystone_aodh_password}
endpoint_type: ${_param:keystone_aodh_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/single.yml b/metadata/service/server/single.yml
index 8cd1b5b..b08c43d 100644
--- a/metadata/service/server/single.yml
+++ b/metadata/service/server/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_aodh_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
aodh:
server:
enabled: true
@@ -28,6 +30,13 @@
user: aodh
password: ${_param:keystone_aodh_password}
endpoint_type: ${_param:keystone_aodh_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbit
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index ab94857..4720848 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: false
+ fluentd:
+ enabled: true
heka:
enabled: false
sensu: