Merge "Enable logging.conf & fluentd for aodh"
diff --git a/README.rst b/README.rst
index 2633fbf..fe6ac21 100644
--- a/README.rst
+++ b/README.rst
@@ -5,7 +5,7 @@
Aodh is an alarming service for OpenStack. It used to be a part of Ceilometer, but starting from Mitaka it
is a separate project. Aodh supports several types of alarms like threshold, event, composite and gnocchi-specific.
-In cluster mode, coordination is enabled via tooz with Redis backend.
+In cluster mode, coordination is enabled via tooz with Redis backend.
MySQL is used as a data backend for alarms and alarm history.
Sample pillars
@@ -20,7 +20,7 @@
enabled: true
version: mitaka
ttl: 86400
- cluster: true
+ cluster: true
database:
engine: "mysql+pymysql"
host: 10.0.106.20
@@ -33,11 +33,11 @@
port: 8042
identity:
engine: keystone
- host: 10.0.106.20
+ host: 10.0.106.20
port: 35357
tenant: service
user: aodh
- password: password
+ password: password
message_queue:
engine: rabbitmq
port: 5672
@@ -54,6 +54,31 @@
port: 11211
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ aodh:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
Development and testing
=======================
diff --git a/aodh/files/default b/aodh/files/default
new file mode 100644
index 0000000..b0259c6
--- /dev/null
+++ b/aodh/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/aodh/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/aodh/files/logging.conf b/aodh/files/logging.conf
new file mode 100644
index 0000000..703530c
--- /dev/null
+++ b/aodh/files/logging.conf
@@ -0,0 +1,50 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+
+[loggers]
+keys = root, aodh
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_aodh]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = aodh
+propagate = 0
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/aodh/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/aodh/files/mitaka/aodh.conf.Debian b/aodh/files/mitaka/aodh.conf.Debian
index ef20b06..938e9d0 100644
--- a/aodh/files/mitaka/aodh.conf.Debian
+++ b/aodh/files/mitaka/aodh.conf.Debian
@@ -102,6 +102,9 @@
# logging_context_format_string). (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/aodh/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set.
diff --git a/aodh/files/newton/aodh.conf.Debian b/aodh/files/newton/aodh.conf.Debian
index 83c8e4c..7afacdb 100644
--- a/aodh/files/newton/aodh.conf.Debian
+++ b/aodh/files/newton/aodh.conf.Debian
@@ -68,6 +68,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/aodh/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/aodh/files/newton/apache-aodh.apache2.conf.Debian b/aodh/files/newton/apache-aodh.apache2.conf.Debian
index 25218a8..3a32f6d 100644
--- a/aodh/files/newton/apache-aodh.apache2.conf.Debian
+++ b/aodh/files/newton/apache-aodh.apache2.conf.Debian
@@ -10,7 +10,7 @@
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/aodh.log
- CustomLog /var/log/apache2/aodh_access.log combined
+ CustomLog /var/log/apache2/aodh_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
<Directory /usr/bin>
<IfVersion >= 2.4>
Require all granted
diff --git a/aodh/files/ocata/aodh.conf.Debian b/aodh/files/ocata/aodh.conf.Debian
index 83c8e4c..7afacdb 100644
--- a/aodh/files/ocata/aodh.conf.Debian
+++ b/aodh/files/ocata/aodh.conf.Debian
@@ -68,6 +68,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/aodh/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/aodh/files/ocata/apache-aodh.apache2.conf.Debian b/aodh/files/ocata/apache-aodh.apache2.conf.Debian
index 1822dbe..381468d 100644
--- a/aodh/files/ocata/apache-aodh.apache2.conf.Debian
+++ b/aodh/files/ocata/apache-aodh.apache2.conf.Debian
@@ -10,5 +10,5 @@
ErrorLogFormat "%{cu}t %M"
</IfVersion>
ErrorLog /var/log/apache2/aodh_error.log
- CustomLog /var/log/apache2/aodh_access.log combined
+ CustomLog /var/log/apache2/aodh_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
</VirtualHost>
diff --git a/aodh/map.jinja b/aodh/map.jinja
index 6a578e4..1bc623c 100644
--- a/aodh/map.jinja
+++ b/aodh/map.jinja
@@ -4,11 +4,27 @@
'pkgs': ['aodh-api', 'aodh-evaluator', 'aodh-notifier', 'aodh-listener', 'python-aodhclient'],
'services': ['aodh-evaluator', 'aodh-notifier', 'aodh-listener'],
'webserved': ['aodh-api'],
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ },
+ }
+ },
},
'RedHat': {
'pkgs':['openstack-aodh-api', 'openstack-aodh-evaluator', 'openstack-aodh-notifier', 'openstack-aodh-listener', 'python-aodhclient'],
'services': ['openstack-aodh-evaluator', 'openstack-aodh-notifier', 'openstack-aodh-listener'],
'webserved': ['openstack-aodh-api'],
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ },
+ }
+ },
},
}, merge=salt['pillar.get']('aodh:server')) %}
diff --git a/aodh/meta/fluentd.yml b/aodh/meta/fluentd.yml
new file mode 100644
index 0000000..e5abc2a
--- /dev/null
+++ b/aodh/meta/fluentd.yml
@@ -0,0 +1,170 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- from "aodh/map.jinja" import server with context %}
+{%- set apache_wsgi = server.version not in ['mitaka'] %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_aodh:
+ tag: openstack.aodh.**
+ type: relabel
+ label: openstack_aodh
+{%- if apache_wsgi %}
+ openstack_aodh_wsgi:
+ input:
+ aodh_api_wsgi_in_tail:
+ type: tail
+ path: /var/log/apache2/aodh_access.log
+ tag: openstack.aodh
+ pos_file: {{ positiondb }}/aodh.wsgi.pos
+ parser:
+ type: regexp
+ time_key: Timestamp
+ time_format: '%d/%b/%Y:%H:%M:%S %z'
+ keep_time_key: false
+ # Apache format: https://regex101.com/r/WeCT7s/5
+ format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+ filter:
+ add_aodh_wsgi_record_fields:
+ tag: openstack.aodh
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: 6
+ - name: severity_label
+ value: INFO
+ - name: programname
+ value: aodh-wsgi
+ - name: http_response_time
+ value: ${ record['http_response_time'].to_i/100000.to_f }
+ match:
+ send_to_default:
+ tag: openstack.aodh
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.aodh_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.aodh_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+{%- endif %}
+ openstack_aodh:
+ filter:
+ set_programname:
+ tag: openstack.aodh.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: aodh-${ tag_parts[2] }
+ set_log_record_fields:
+ tag: openstack.aodh
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "aodh" }'
+ parse_http_stats:
+ tag: openstack.aodh
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.aodh.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.aodh
+ send_to_default:
+ tag: openstack.aodh
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.aodh_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.aodh_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ aodh_logs_per_severity:
+ tag: metric.aodh_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: aodh
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ aodh_openstack_http_response_times:
+ tag: metric.aodh_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: aodh
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/aodh/server.sls b/aodh/server.sls
index f42b279..8c1300a 100644
--- a/aodh/server.sls
+++ b/aodh/server.sls
@@ -14,6 +14,83 @@
- require:
- pkg: aodh_server_packages
+{% for service_name in server.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://aodh/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: aodh_server_packages
+ - watch_in:
+ - service: aodh_server_services
+{% endfor %}
+
+{% if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+aodh_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+aodh_general_logging_conf:
+ file.managed:
+ - name: /etc/aodh/logging.conf
+ - source: salt://aodh/files/logging.conf
+ - template: jinja
+ - user: aodh
+ - group: aodh
+ - require:
+ - pkg: aodh_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: aodh_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: aodh
+ values: {{ server }}
+ - watch_in:
+ - service: aodh_server_services
+{%- if server.version not in ['mitaka'] %}
+ - service: aodh_apache_restart
+{%- endif %}
+
+/var/log/aodh/aodh.log:
+ file.managed:
+ - user: aodh
+ - group: aodh
+ - watch_in:
+ - service: aodh_server_services
+{%- if server.version not in ['mitaka'] %}
+ - service: aodh_apache_restart
+{%- endif %}
+
+{% for service_name in server.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/aodh/logging/logging-{{ service_name }}.conf
+ - source: salt://aodh/files/logging.conf
+ - template: jinja
+ - user: aodh
+ - group: aodh
+ - require:
+ - pkg: aodh_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: aodh_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - watch_in:
+ - service: aodh_server_services
+{% endfor %}
+
+{% endif %}
+
aodh_syncdb:
cmd.run:
- name: aodh-dbsync
diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml
index 00bb18b..1bc8da2 100644
--- a/metadata/service/server/cluster.yml
+++ b/metadata/service/server/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_aodh_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
aodh:
server:
enabled: true
@@ -31,6 +33,13 @@
user: aodh
password: ${_param:keystone_aodh_password}
endpoint_type: ${_param:keystone_aodh_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/single.yml b/metadata/service/server/single.yml
index 8cd1b5b..b08c43d 100644
--- a/metadata/service/server/single.yml
+++ b/metadata/service/server/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_aodh_endpoint_type: internalURL
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
aodh:
server:
enabled: true
@@ -28,6 +30,13 @@
user: aodh
password: ${_param:keystone_aodh_password}
endpoint_type: ${_param:keystone_aodh_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbit
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index ab94857..4720848 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: false
+ fluentd:
+ enabled: true
heka:
enabled: false
sensu: