Merge "Enable logging.conf & fluentd for nova"
diff --git a/README.rst b/README.rst
index 3353131..2b4dd13 100644
--- a/README.rst
+++ b/README.rst
@@ -679,6 +679,39 @@
compute:
max_concurrent_live_migrations: 1 # (1 is the default)
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler is available.
+
+Also it is able to configure this with pillar:
+
+.. code-block:: yaml
+
+ nova:
+ controller:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
+ compute:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
Documentation and Bugs
======================
diff --git a/metadata/service/compute/cluster.yml b/metadata/service/compute/cluster.yml
index 2345193..77fed1a 100644
--- a/metadata/service/compute/cluster.yml
+++ b/metadata/service/compute/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
nova_compute_virtualization: kvm
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
compute:
version: ${_param:nova_version}
@@ -31,6 +33,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
port: 5672
diff --git a/metadata/service/compute/ironic.yml b/metadata/service/compute/ironic.yml
index 3cc8755..339099e 100644
--- a/metadata/service/compute/ironic.yml
+++ b/metadata/service/compute/ironic.yml
@@ -3,6 +3,9 @@
classes:
- service.nova.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
compute:
version: ${_param:nova_version}
@@ -27,6 +30,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/compute/kvm.yml b/metadata/service/compute/kvm.yml
index 7fa0036..0e71d06 100644
--- a/metadata/service/compute/kvm.yml
+++ b/metadata/service/compute/kvm.yml
@@ -3,6 +3,9 @@
classes:
- service.nova.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
compute:
version: ${_param:nova_version}
@@ -29,6 +32,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index 69dad66..b5436c5 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -6,6 +6,8 @@
_param:
nova_vncproxy_url: http://${_param:single_address}:6080
nova_networking: default
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
controller:
enabled: true
@@ -19,11 +21,6 @@
ram_allocation_ratio: 1.5
disk_allocation_ratio: 1.0
workers: 8
- logging:
- - engine: syslog
- facility: local0
- heka:
- enabled: true
bind:
private_address: ${_param:cluster_local_address}
public_address: ${_param:cluster_vip_address}
@@ -44,6 +41,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index 820c5ff..135bdaf 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -6,6 +6,8 @@
_param:
nova_vncproxy_url: http://${_param:single_address}:6080
nova_networking: default
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
nova:
controller:
enabled: true
@@ -19,11 +21,6 @@
ram_allocation_ratio: 1.5
disk_allocation_ratio: 1.0
workers: 1
- logging:
- - engine: syslog
- facility: local0
- heka:
- enabled: true
bind:
private_address: ${_param:single_address}
public_address: ${_param:single_address}
@@ -44,6 +41,13 @@
user: nova
password: ${_param:keystone_nova_password}
tenant: service
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 949d7cc..a1f6414 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu:
diff --git a/nova/compute.sls b/nova/compute.sls
index 9c9b03b..081b600 100644
--- a/nova/compute.sls
+++ b/nova/compute.sls
@@ -77,6 +77,53 @@
- pkg: nova_compute_packages
{%- endif %}
+{% for service_name in compute.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://nova/files/default
+ - template: jinja
+ - require:
+ - pkg: nova_compute_packages
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - watch_in:
+ - service: nova_compute_services
+{% endfor %}
+
+{% if compute.logging.log_appender -%}
+
+{% if compute.logging.log_handlers.get('fluentd').get('enabled', False) -%}
+nova_compute_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{% endif %}
+
+{% for service_name in compute.get('services', []) %}
+
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/nova/logging/logging-{{ service_name }}.conf
+ - source: salt://nova/files/logging.conf
+ - template: jinja
+ - user: nova
+ - group: nova
+ - require:
+ - pkg: nova_compute_packages
+{%- if compute.logging.log_handlers.get('fluentd').get('enabled', False) %}
+ - pkg: nova_compute_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - watch_in:
+ - service: nova_compute_services
+
+{% endfor %}
+{% endif %}
+
{%- if compute.message_queue.get('ssl',{}).get('enabled',False) %}
rabbitmq_ca_nova_compute:
{%- if compute.message_queue.ssl.cacert is defined %}
diff --git a/nova/controller.sls b/nova/controller.sls
index a55d037..24bb535 100644
--- a/nova/controller.sls
+++ b/nova/controller.sls
@@ -85,6 +85,86 @@
- require:
- pkg: nova_controller_packages
+{% for service_name in controller.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://nova/files/default
+ - template: jinja
+ - require:
+ - pkg: nova_controller_packages
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ controller }}
+ - require:
+ - pkg: nova_controller_packages
+ - watch_in:
+ - service: nova_controller_services
+{% endfor %}
+
+{% if controller.logging.log_appender %}
+
+{%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
+nova_controller_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+nova_general_logging_conf:
+ file.managed:
+ - name: /etc/nova/logging.conf
+ - source: salt://nova/files/logging.conf
+ - template: jinja
+ - user: nova
+ - group: nova
+ - require:
+ - pkg: nova_controller_packages
+{%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
+ - pkg: nova_controller_fluentd_logger_package
+{%- endif %}
+ - defaults:
+ service_name: nova
+ values: {{ controller }}
+ - watch_in:
+ - service: nova_controller_services
+
+/var/log/nova/nova.log:
+ file.managed:
+ - user: nova
+ - group: nova
+ - watch_in:
+ - service: nova_controller_services
+{%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+ - service: nova_apache_restart
+{%- endif %}
+
+{% for service_name in controller.services %}
+
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/nova/logging/logging-{{ service_name }}.conf
+ - source: salt://nova/files/logging.conf
+ - template: jinja
+ - user: nova
+ - group: nova
+ - require:
+ - pkg: nova_controller_packages
+{%- if controller.logging.log_handlers.get('fluentd').get('enabled', False) %}
+ - pkg: nova_controller_fluentd_logger_package
+{%- endif %}
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ controller }}
+ - watch_in:
+ - service: nova_controller_services
+{%- if controller.version not in ["juno", "kilo", "liberty", "mitaka", "newton"] %}
+ - service: nova_apache_restart
+{%- endif %}
+
+{% endfor %}
+{% endif %}
+
{% if controller.get('policy', {}) and controller.version not in ['liberty', 'mitaka', 'newton'] %}
{# nova no longer ships with a default policy.json #}
diff --git a/nova/files/default b/nova/files/default
new file mode 100644
index 0000000..815ede9
--- /dev/null
+++ b/nova/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/nova/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/nova/files/logging.conf b/nova/files/logging.conf
new file mode 100644
index 0000000..fbc6ecb
--- /dev/null
+++ b/nova/files/logging.conf
@@ -0,0 +1,86 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+
+[loggers]
+keys = root, nova
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_nova]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = nova
+propagate = 0
+
+[logger_amqp]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqp
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither. (Recommended for production systems.)
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+# NOTE(mikal): suds is used by the vmware driver, removing this will
+# cause many extraneous log lines for their tempest runs. Refer to
+# https://review.openstack.org/#/c/219225/ for details.
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{% if values.logging.log_handlers.get('fluentd').get('enabled', False) -%}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{% if values.logging.log_handlers.watchedfile.enabled -%}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/nova/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{% if values.logging.log_handlers.get('fluentd').get('enabled', False) -%}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/nova/files/mitaka/nova-controller.conf.Debian b/nova/files/mitaka/nova-controller.conf.Debian
index a08900f..2ed73c4 100644
--- a/nova/files/mitaka/nova-controller.conf.Debian
+++ b/nova/files/mitaka/nova-controller.conf.Debian
@@ -7,6 +7,9 @@
{%- else %}
debug = False
{%- endif %}
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/nova/logging.conf
+{%- endif %}
state_path = /var/lib/nova
volumes_dir = /etc/nova/volumes
dhcpbridge = /usr/bin/nova-dhcpbridge
diff --git a/nova/files/newton/nova-controller.conf.Debian b/nova/files/newton/nova-controller.conf.Debian
index 841e99f..43a2ec6 100644
--- a/nova/files/newton/nova-controller.conf.Debian
+++ b/nova/files/newton/nova-controller.conf.Debian
@@ -7,6 +7,9 @@
{%- else %}
debug = False
{%- endif %}
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/nova/logging.conf
+{%- endif %}
state_path = /var/lib/nova
volumes_dir = /etc/nova/volumes
dhcpbridge = /usr/bin/nova-dhcpbridge
diff --git a/nova/files/ocata/nova-controller.conf.Debian b/nova/files/ocata/nova-controller.conf.Debian
index f6bc29c..ed21920 100644
--- a/nova/files/ocata/nova-controller.conf.Debian
+++ b/nova/files/ocata/nova-controller.conf.Debian
@@ -2775,6 +2775,9 @@
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append=<None>
+{%- if controller.logging.log_appender %}
+log_config_append=/etc/nova/logging.conf
+{%- endif %}
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/nova/map.jinja b/nova/map.jinja
index 8ad808e..cbae33f 100644
--- a/nova/map.jinja
+++ b/nova/map.jinja
@@ -41,6 +41,14 @@
'audit': {
'enabled': false
},
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ }
+ },
},
'RedHat': {
'pkgs': pkgs_list,
@@ -51,6 +59,14 @@
'audit': {
'enabled': false
},
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ }
+ },
},
}, merge=pillar.nova.get('controller', {}), base='BaseDefaults') %}
@@ -112,6 +128,11 @@
region: RegionOne
network: {{ compute_network }}
heal_instance_info_cache_interval: '60'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
RedHat:
pkgs:
- openstack-nova-compute
@@ -135,6 +156,11 @@
region: RegionOne
network: {{ compute_network }}
heal_instance_info_cache_interval: '60'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
{%- endload %}
{% set compute = salt["grains.filter_by"](compute_defaults, merge=pillar.nova.get("compute", {}), base='BaseDefaults') %}
diff --git a/nova/meta/fluentd.yml b/nova/meta/fluentd.yml
new file mode 100644
index 0000000..87d5989
--- /dev/null
+++ b/nova/meta/fluentd.yml
@@ -0,0 +1,88 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- from "nova/map.jinja" import controller with context %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+{%- set apache_wsgi = controller.get('enabled') and controller.version not in ('juno', 'kilo', 'liberty', 'mitaka', 'newton') %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_inpit:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_nova:
+ tag: openstack.nova.**
+ type: relabel
+ label: openstack_nova
+{%- if apache_wsgi %}
+ openstack_nova_wsgi:
+ input:
+ nova_placement_api_wsgi_in_tail:
+ type: tail
+ path: /var/log/apache2/nova_placement_access.log
+ tag: openstack.nova
+ pos_file: {{ positiondb }}/nova_placement.wsgi.pos
+ parser:
+ type: regexp
+ time_key: Timestamp
+ time_format: '%d/%b/%Y:%H:%M:%S %z'
+ keep_time_key: false
+ # Apache access log custom format: https://regex101.com/r/WeCT7s/5
+ format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+ filter:
+ add_nova_palcement_wsgi_record_fields:
+ tag: openstack.nova
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: 6
+ - name: severity_label
+ value: INFO
+ - name: programname
+ value: nova-placement-wsgi
+ match:
+ send_to_default:
+ tag: openstack.nova
+ type: relabel
+ label: default_output
+{%- endif %}
+ openstack_nova:
+ filter:
+ set_nova_programname:
+ tag: openstack.nova.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: nova-${ tag_parts[2] }
+ set_nova_fields:
+ tag: openstack.nova
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "nova" }'
+ match:
+ unify_tag:
+ tag: openstack.nova.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.nova
+ send_to_default:
+ tag: openstack.nova
+ type: relabel
+ label: default_output
+{% endif %}
\ No newline at end of file