Enable logging.conf & fluentd for neutron
This change introduce ability to use log_config_append parameter with
new FluentdHandler to send logs directly to Fluentd.
To save per binary log streams as it is in the default logging states generates
separate logging.conf files per service and use /etc/default/<service name>
to pass dedicated logging.conf to every service.
Change-Id: I2952c8c0abe76690114d3ac5d3815b992d48d1cb
Related-Prod: PROD-16324
diff --git a/README.rst b/README.rst
index 5d21c9e..434957a 100644
--- a/README.rst
+++ b/README.rst
@@ -1055,6 +1055,48 @@
protocol: http
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ neutron:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+ ....
+ compute:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+ ....
+ gateway:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
Documentation and Bugs
======================
diff --git a/metadata/service/compute/ovn/single.yml b/metadata/service/compute/ovn/single.yml
index 9bfb54d..c62fbc1 100644
--- a/metadata/service/compute/ovn/single.yml
+++ b/metadata/service/compute/ovn/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
ovn_external_bridge: br-floating
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
compute:
enabled: true
@@ -15,3 +17,10 @@
external_bridge: ${_param:ovn_external_bridge}
backend:
engine: ovn
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
diff --git a/metadata/service/compute/single.yml b/metadata/service/compute/single.yml
index 377ff39..1884bbd 100644
--- a/metadata/service/compute/single.yml
+++ b/metadata/service/compute/single.yml
@@ -3,6 +3,9 @@
classes:
- service.neutron.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
compute:
enabled: true
@@ -29,3 +32,10 @@
mechanism:
ovs:
driver: openvswitch
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index b1ac5ad..844d590 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_neutron_endpoint_type: internal
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
server:
enabled: true
@@ -31,6 +33,13 @@
password: ${_param:keystone_neutron_password}
tenant: service
endpoint_type: ${_param:keystone_neutron_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index 0f22b88..91fd911 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_neutron_endpoint_type: internal
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
server:
enabled: true
@@ -35,6 +37,13 @@
password: ${_param:keystone_neutron_password}
tenant: service
endpoint_type: ${_param:keystone_neutron_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/gateway/single.yml b/metadata/service/gateway/single.yml
index 1af2525..860e18a 100644
--- a/metadata/service/gateway/single.yml
+++ b/metadata/service/gateway/single.yml
@@ -3,6 +3,9 @@
classes:
- service.neutron.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
gateway:
enabled: true
@@ -29,3 +32,10 @@
mechanism:
ovs:
driver: openvswitch
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 6ad7d01..cde8934 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu:
diff --git a/neutron/compute.sls b/neutron/compute.sls
index cc1f1aa..d540fa7 100644
--- a/neutron/compute.sls
+++ b/neutron/compute.sls
@@ -118,6 +118,83 @@
- file: rabbitmq_ca_neutron_compute
{%- endif %}
+{%- set neutron_compute_services_list = compute.services %}
+{%- if compute.backend.sriov is defined %}
+ {%- do neutron_compute_services_list.append('neutron-sriov-agent') %}
+{%- endif %}
+{%- if compute.dvr %}
+ {%- do neutron_compute_services_list.extend(['neutron-l3-agent', 'neutron-metadata-agent']) %}
+{%- endif %}
+
+{%- for service_name in neutron_compute_services_list %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://neutron/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - require:
+ - pkg: neutron_compute_packages
+{% if compute.backend.sriov is defined %}
+ - pkg: neutron_sriov_package
+{% endif %}
+{% if compute.dvr %}
+ - pkg: neutron_dvr_packages
+{% endif %}
+ - watch_in:
+ - service: neutron_compute_services
+{% if compute.backend.sriov is defined %}
+ - service: neutron_sriov_service
+{% endif %}
+{% if compute.dvr %}
+ - service: neutron_dvr_agents
+{% endif %}
+{% endfor %}
+
+{%- if compute.logging.log_appender %}
+
+{%- if compute.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+neutron_compute_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in neutron_compute_services_list %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/neutron/logging/logging-{{ service_name }}.conf
+ - source: salt://neutron/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - user: neutron
+ - group: neutron
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - require:
+ - pkg: neutron_compute_packages
+{% if compute.backend.sriov is defined %}
+ - pkg: neutron_sriov_package
+{% endif %}
+{% if compute.dvr %}
+ - pkg: neutron_dvr_packages
+{% endif %}
+{%- if compute.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: neutron_compute_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: neutron_compute_services
+{% if compute.backend.sriov is defined %}
+ - service: neutron_sriov_service
+{% endif %}
+{% if compute.dvr %}
+ - service: neutron_dvr_agents
+{% endif %}
+{% endfor %}
+
+{% endif %}
{%- if compute.message_queue.get('ssl',{}).get('enabled', False) %}
rabbitmq_ca_neutron_compute:
diff --git a/neutron/files/default b/neutron/files/default
new file mode 100644
index 0000000..be6a4c1
--- /dev/null
+++ b/neutron/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{%- if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-{{ service_name }}.conf"
+{%- endif %}
diff --git a/neutron/files/logging.conf b/neutron/files/logging.conf
new file mode 100644
index 0000000..f9cff2c
--- /dev/null
+++ b/neutron/files/logging.conf
@@ -0,0 +1,73 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, neutron
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_neutron]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = neutron
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/neutron/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/neutron/files/mitaka/neutron-server b/neutron/files/mitaka/neutron-server
index 54f6ceb..59f0305 100644
--- a/neutron/files/mitaka/neutron-server
+++ b/neutron/files/mitaka/neutron-server
@@ -13,4 +13,8 @@
{%- if server.backend.engine == "contrail" %}
NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugins/opencontrail/ContrailPlugin.ini"
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+
+{%- if server.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-neutron-server.conf"
+{%- endif %}
diff --git a/neutron/files/newton/neutron-server b/neutron/files/newton/neutron-server
index 54f6ceb..59f0305 100644
--- a/neutron/files/newton/neutron-server
+++ b/neutron/files/newton/neutron-server
@@ -13,4 +13,8 @@
{%- if server.backend.engine == "contrail" %}
NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugins/opencontrail/ContrailPlugin.ini"
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+
+{%- if server.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-neutron-server.conf"
+{%- endif %}
diff --git a/neutron/files/ocata/neutron-server b/neutron/files/ocata/neutron-server
index 04830ca..5061cca 100644
--- a/neutron/files/ocata/neutron-server
+++ b/neutron/files/ocata/neutron-server
@@ -13,4 +13,8 @@
{%- if server.backend.engine == "contrail" %}
NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugins/opencontrail/ContrailPlugin.ini"
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-neutron-server.conf"
+{%- endif %}
diff --git a/neutron/gateway.sls b/neutron/gateway.sls
index 6046b96..bbbca27 100644
--- a/neutron/gateway.sls
+++ b/neutron/gateway.sls
@@ -49,6 +49,52 @@
- require:
- pkg: neutron_gateway_packages
+{%- for service_name in gateway.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://neutron/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ gateway }}
+ - require:
+ - pkg: neutron_gateway_packages
+ - watch_in:
+ - service: neutron_gateway_services
+{% endfor %}
+
+{%- if gateway.logging.log_appender %}
+
+{%- if gateway.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+neutron_gateway_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in gateway.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/neutron/logging/logging-{{ service_name }}.conf
+ - source: salt://neutron/files/logging.conf
+ - template: jinja
+ - makedirs: true
+ - user: neutron
+ - group: neutron
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ gateway }}
+ - require:
+ - pkg: neutron_gateway_packages
+{%- if gateway.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: neutron_gateway_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: neutron_gateway_services
+{% endfor %}
+
+{% endif %}
+
neutron_gateway_services:
service.running:
- names: {{ gateway.services }}
diff --git a/neutron/map.jinja b/neutron/map.jinja
index 4c95928..8221b89 100644
--- a/neutron/map.jinja
+++ b/neutron/map.jinja
@@ -15,7 +15,15 @@
'dpdk': false,
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-neutron-openvswitch', 'openvswitch', 'python-pycadf'],
@@ -25,7 +33,15 @@
'dpdk': false,
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.neutron.get('compute', {}), base='BaseDefaults') %}
@@ -34,12 +50,28 @@
'Debian': {
'pkgs': ['neutron-dhcp-agent', 'neutron-openvswitch-agent', 'neutron-l3-agent', 'openvswitch-common', 'neutron-metadata-agent'],
'services': ['neutron-openvswitch-agent', 'neutron-metadata-agent', 'neutron-l3-agent', 'neutron-dhcp-agent'],
- 'dpdk': false
+ 'dpdk': false,
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-neutron-openvswitch'],
'services': ['neutron-openvswitch-agent', 'neutron-metadata-agent', 'neutron-l3-agent', 'neutron-dhcp-agent'],
- 'dpdk': false
+ 'dpdk': false,
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.neutron.get('gateway', {}), base='BaseDefaults') %}
@@ -56,7 +88,15 @@
'cors': {},
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs_ml2': ['openstack-neutron-ml2', 'python-pycadf'],
@@ -69,7 +109,15 @@
'cors': {},
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.neutron.get('server', {}), base='BaseDefaults') %}
diff --git a/neutron/meta/fluentd.yml b/neutron/meta/fluentd.yml
new file mode 100644
index 0000000..de237e0
--- /dev/null
+++ b/neutron/meta/fluentd.yml
@@ -0,0 +1,110 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_neutron:
+ tag: openstack.neutron.*
+ type: relabel
+ label: openstack_neutron
+ openstack_neutron:
+ filter:
+ set_neutron_fields:
+ tag: openstack.neutron.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: programname
+ value: neutron-${ tag_parts[2] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ parse_http_stats:
+ tag: openstack.neutron
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.neutron.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.neutron
+ send_to_default:
+ tag: openstack.neutron
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.neutron_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.neutron_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ neutron_logs_per_severity:
+ tag: metric.neutron_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: neutron
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ neutron_openstack_http_response_times:
+ tag: metric.neutron_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: neutron
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/neutron/server.sls b/neutron/server.sls
index a6de469..b4d70bb 100644
--- a/neutron/server.sls
+++ b/neutron/server.sls
@@ -123,6 +123,54 @@
- require:
- pkg: neutron_server_packages
+{%- for service_name in server.get('services', []) %}
+{%- if service_name != 'neutron-server' %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://neutron/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: neutron_server_packages
+ - watch_in:
+ - service: neutron_server_services
+{%- endif %}
+{%- endfor %}
+
+{%- if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+neutron_server_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{%- for service_name in server.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/neutron/logging/logging-{{ service_name }}.conf
+ - source: salt://neutron/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - user: neutron
+ - group: neutron
+ - require:
+ - pkg: neutron_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: neutron_server_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: neutron_server_services
+{%- endfor %}
+
+{%- endif %}
+
{%- for name, rule in server.get('policy', {}).iteritems() %}
{%- if rule != None %}