Enable logging.conf & fluentd for nova
This change introduce ability to use log_config_append parameter with
new FluentdHandler to send logs directly to Fluentd.
To save per binary log streams as it is in the default logging states generates
separate logging.conf files per service and use /etc/default/<service name>
to pass dedicated logging.conf to every service.
Change-Id: I93f1838592b34d703bb60e823d11dbf33b98e3d5
Related-Prod: PROD-16324
diff --git a/README.rst b/README.rst
index c3a67f1..d6283be 100644
--- a/README.rst
+++ b/README.rst
@@ -644,6 +644,39 @@
compute:
max_concurrent_live_migrations: 1 # (1 is the default)
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler is available.
+
+Also it is able to configure this with pillar:
+
+.. code-block:: yaml
+
+ nova:
+ controller:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
+ compute:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
Documentation and Bugs
======================
diff --git a/metadata/service/compute/cluster.yml b/metadata/service/compute/cluster.yml
index 2345193..77fed1a 100644
--- a/metadata/service/compute/cluster.yml
+++ b/metadata/service/compute/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
nova_compute_virtualization: kvm
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
compute:
version: ${_param:nova_version}
@@ -31,6 +33,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
port: 5672
diff --git a/metadata/service/compute/ironic.yml b/metadata/service/compute/ironic.yml
index c776dde..11e0cea 100644
--- a/metadata/service/compute/ironic.yml
+++ b/metadata/service/compute/ironic.yml
@@ -3,6 +3,9 @@
classes:
- service.nova.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
compute:
version: ${_param:nova_version}
@@ -26,6 +29,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/compute/kvm.yml b/metadata/service/compute/kvm.yml
index 7fa0036..0e71d06 100644
--- a/metadata/service/compute/kvm.yml
+++ b/metadata/service/compute/kvm.yml
@@ -3,6 +3,9 @@
classes:
- service.nova.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
compute:
version: ${_param:nova_version}
@@ -29,6 +32,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index 69dad66..b5436c5 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -6,6 +6,8 @@
_param:
nova_vncproxy_url: http://${_param:single_address}:6080
nova_networking: default
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
controller:
enabled: true
@@ -19,11 +21,6 @@
ram_allocation_ratio: 1.5
disk_allocation_ratio: 1.0
workers: 8
- logging:
- - engine: syslog
- facility: local0
- heka:
- enabled: true
bind:
private_address: ${_param:cluster_local_address}
public_address: ${_param:cluster_vip_address}
@@ -44,6 +41,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index 820c5ff..135bdaf 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -6,6 +6,8 @@
_param:
nova_vncproxy_url: http://${_param:single_address}:6080
nova_networking: default
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
controller:
enabled: true
@@ -19,11 +21,6 @@
ram_allocation_ratio: 1.5
disk_allocation_ratio: 1.0
workers: 1
- logging:
- - engine: syslog
- facility: local0
- heka:
- enabled: true
bind:
private_address: ${_param:single_address}
public_address: ${_param:single_address}
@@ -44,6 +41,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 949d7cc..a1f6414 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu:
diff --git a/nova/compute.sls b/nova/compute.sls
index eb3127e..4ab3d47 100644
--- a/nova/compute.sls
+++ b/nova/compute.sls
@@ -77,6 +77,53 @@
- pkg: nova_compute_packages
{%- endif %}
+{% for service_name in compute.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://nova/files/default
+ - template: jinja
+ - require:
+ - pkg: nova_compute_packages
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - watch_in:
+ - service: nova_compute_services
+{% endfor %}
+
+{% if compute.logging.log_appender -%}
+
+{% if compute.logging.log_handlers.get('fluentd').get('enabled', False) -%}
+nova_compute_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{% endif %}
+
+{% for service_name in compute.get('services', []) %}
+
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/nova/logging/logging-{{ service_name }}.conf
+ - source: salt://nova/files/logging.conf
+ - template: jinja
+ - user: nova
+ - group: nova
+ - require:
+ - pkg: nova_compute_packages
+{%- if compute.logging.log_handlers.get('fluentd').get('enabled', False) %}
+ - pkg: nova_compute_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - watch_in:
+ - service: nova_compute_services
+
+{% endfor %}
+{% endif %}
+
{%- if compute.message_queue.get('ssl',{}).get('enabled',False) %}
rabbitmq_ca_nova_compute:
{%- if compute.message_queue.ssl.cacert is defined %}
diff --git a/nova/controller.sls b/nova/controller.sls
index 4bc8035..07d7b29 100644
--- a/nova/controller.sls
+++ b/nova/controller.sls
@@ -85,6 +85,86 @@
- require:
- pkg: nova_controller_packages
+{% for service_name in controller.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://nova/files/default
+ - template: jinja
+ - require:
+ - pkg: nova_controller_packages
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ controller }}
+ - require:
+ - pkg: nova_controller_packages
+ - watch_in:
+ - service: nova_controller_services
+{% endfor %}
+
+{% if controller.logging.log_appender %}
+
+{%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
+nova_controller_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+nova_general_logging_conf:
+ file.managed:
+ - name: /etc/nova/logging.conf
+ - source: salt://nova/files/logging.conf
+ - template: jinja
+ - user: nova
+ - group: nova
+ - require:
+ - pkg: nova_controller_packages
+{%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
+ - pkg: nova_controller_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: nova
+ values: {{ controller }}
+ - watch_in:
+ - service: nova_controller_services
+
+/var/log/nova/nova.log:
+ file.managed:
+ - user: nova
+ - group: nova
+ - watch_in:
+ - service: nova_controller_services
+{%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+ - service: nova_apache_restart
+{%- endif %}
+
+{% for service_name in controller.services %}
+
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/nova/logging/logging-{{ service_name }}.conf
+ - source: salt://nova/files/logging.conf
+ - template: jinja
+ - user: nova
+ - group: nova
+ - require:
+ - pkg: nova_controller_packages
+{%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
+ - pkg: nova_controller_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ controller }}
+ - watch_in:
+ - service: nova_controller_services
+{%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+ - service: nova_apache_restart
+{%- endif %}
+
+{% endfor %}
+{% endif %}
+
{% if controller.get('policy', {}) and controller.version not in ['liberty', 'mitaka', 'newton'] %}
{# nova no longer ships with a default policy.json #}
diff --git a/nova/files/default b/nova/files/default
new file mode 100644
index 0000000..815ede9
--- /dev/null
+++ b/nova/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/nova/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/nova/files/logging.conf b/nova/files/logging.conf
new file mode 100644
index 0000000..fbc6ecb
--- /dev/null
+++ b/nova/files/logging.conf
@@ -0,0 +1,86 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+
+[loggers]
+keys = root, nova
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_nova]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = nova
+propagate = 0
+
+[logger_amqp]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqp
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither. (Recommended for production systems.)
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+# NOTE(mikal): suds is used by the vmware driver, removing this will
+# cause many extraneous log lines for their tempest runs. Refer to
+# https://review.openstack.org/#/c/219225/ for details.
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{% if values.logging.log_handlers.get('fluentd').get('enabled', False) -%}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{% if values.logging.log_handlers.watchedfile.enabled -%}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/nova/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{% if values.logging.log_handlers.get('fluentd').get('enabled', False) -%}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/nova/files/mitaka/nova-controller.conf.Debian b/nova/files/mitaka/nova-controller.conf.Debian
index d6f6c23..41afe87 100644
--- a/nova/files/mitaka/nova-controller.conf.Debian
+++ b/nova/files/mitaka/nova-controller.conf.Debian
@@ -7,6 +7,9 @@
{%- else %}
debug = False
{%- endif %}
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/nova/logging.conf
+{%- endif %}
state_path = /var/lib/nova
volumes_dir = /etc/nova/volumes
dhcpbridge = /usr/bin/nova-dhcpbridge
diff --git a/nova/files/newton/nova-controller.conf.Debian b/nova/files/newton/nova-controller.conf.Debian
index c8e0bee..5d9d7b7 100644
--- a/nova/files/newton/nova-controller.conf.Debian
+++ b/nova/files/newton/nova-controller.conf.Debian
@@ -7,6 +7,9 @@
{%- else %}
debug = False
{%- endif %}
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/nova/logging.conf
+{%- endif %}
state_path = /var/lib/nova
volumes_dir = /etc/nova/volumes
dhcpbridge = /usr/bin/nova-dhcpbridge
diff --git a/nova/files/ocata/nova-controller.conf.Debian b/nova/files/ocata/nova-controller.conf.Debian
index 9287087..ed0362f 100644
--- a/nova/files/ocata/nova-controller.conf.Debian
+++ b/nova/files/ocata/nova-controller.conf.Debian
@@ -2775,6 +2775,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append=<None>
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/nova/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/nova/map.jinja b/nova/map.jinja
index 51eb23f..c830f54 100644
--- a/nova/map.jinja
+++ b/nova/map.jinja
@@ -38,6 +38,14 @@
'audit': {
'enabled': false
},
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ }
+ },
},
'RedHat': {
'pkgs': pkgs_list,
@@ -48,6 +56,14 @@
'audit': {
'enabled': false
},
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ }
+ },
},
}, merge=pillar.nova.get('controller', {})) %}
@@ -108,6 +124,11 @@
region: RegionOne
network: {{ compute_network }}
heal_instance_info_cache_interval: '60'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
RedHat:
pkgs:
- openstack-nova-compute
@@ -131,6 +152,11 @@
region: RegionOne
network: {{ compute_network }}
heal_instance_info_cache_interval: '60'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
{%- endload %}
{% set compute = salt["grains.filter_by"](compute_defaults, merge=pillar.nova.get("compute", {})) %}
diff --git a/nova/meta/fluentd.yml b/nova/meta/fluentd.yml
new file mode 100644
index 0000000..87d5989
--- /dev/null
+++ b/nova/meta/fluentd.yml
@@ -0,0 +1,88 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- from "nova/map.jinja" import controller with context %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+{%- set apache_wsgi = controller.get('enabled') and controller.version not in ('juno', 'kilo', 'liberty', 'mitaka', 'newton') %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_inpit:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_nova:
+ tag: openstack.nova.**
+ type: relabel
+ label: openstack_nova
+{%- if apache_wsgi %}
+ openstack_nova_wsgi:
+ input:
+ nova_placement_api_wsgi_in_tail:
+ type: tail
+ path: /var/log/apache2/nova_placement_access.log
+ tag: openstack.nova
+ pos_file: {{ positiondb }}/nova_placement.wsgi.pos
+ parser:
+ type: regexp
+ time_key: Timestamp
+ time_format: '%d/%b/%Y:%H:%M:%S %z'
+ keep_time_key: false
+ # Apache access log custom format: https://regex101.com/r/WeCT7s/5
+ format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+ filter:
+ add_nova_palcement_wsgi_record_fields:
+ tag: openstack.nova
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: 6
+ - name: severity_label
+ value: INFO
+ - name: programname
+ value: nova-placement-wsgi
+ match:
+ send_to_default:
+ tag: openstack.nova
+ type: relabel
+ label: default_output
+{%- endif %}
+ openstack_nova:
+ filter:
+ set_nova_programname:
+ tag: openstack.nova.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: nova-${ tag_parts[2] }
+ set_nova_fields:
+ tag: openstack.nova
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "nova" }'
+ match:
+ unify_tag:
+ tag: openstack.nova.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.nova
+ send_to_default:
+ tag: openstack.nova
+ type: relabel
+ label: default_output
+{% endif %}
\ No newline at end of file