Merge "Enable logging.conf & fluentd for neutron"
diff --git a/README.rst b/README.rst
index 570558c..9e37b32 100644
--- a/README.rst
+++ b/README.rst
@@ -1058,6 +1058,48 @@
protocol: http
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ neutron:
+ server:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+ ....
+ compute:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+ ....
+ gateway:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
Documentation and Bugs
======================
diff --git a/metadata/service/compute/ovn/single.yml b/metadata/service/compute/ovn/single.yml
index 9bfb54d..c62fbc1 100644
--- a/metadata/service/compute/ovn/single.yml
+++ b/metadata/service/compute/ovn/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
ovn_external_bridge: br-floating
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
compute:
enabled: true
@@ -15,3 +17,10 @@
external_bridge: ${_param:ovn_external_bridge}
backend:
engine: ovn
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
diff --git a/metadata/service/compute/single.yml b/metadata/service/compute/single.yml
index 377ff39..1884bbd 100644
--- a/metadata/service/compute/single.yml
+++ b/metadata/service/compute/single.yml
@@ -3,6 +3,9 @@
classes:
- service.neutron.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
compute:
enabled: true
@@ -29,3 +32,10 @@
mechanism:
ovs:
driver: openvswitch
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index b1ac5ad..844d590 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_neutron_endpoint_type: internal
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
server:
enabled: true
@@ -31,6 +33,13 @@
password: ${_param:keystone_neutron_password}
tenant: service
endpoint_type: ${_param:keystone_neutron_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:cluster_vip_address}
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index 0f22b88..91fd911 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -5,6 +5,8 @@
parameters:
_param:
keystone_neutron_endpoint_type: internal
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
server:
enabled: true
@@ -35,6 +37,13 @@
password: ${_param:keystone_neutron_password}
tenant: service
endpoint_type: ${_param:keystone_neutron_endpoint_type}
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
message_queue:
engine: rabbitmq
host: ${_param:single_address}
diff --git a/metadata/service/gateway/single.yml b/metadata/service/gateway/single.yml
index 1af2525..860e18a 100644
--- a/metadata/service/gateway/single.yml
+++ b/metadata/service/gateway/single.yml
@@ -3,6 +3,9 @@
classes:
- service.neutron.support
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
neutron:
gateway:
enabled: true
@@ -29,3 +32,10 @@
mechanism:
ovs:
driver: openvswitch
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 6ad7d01..cde8934 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
_support:
collectd:
enabled: true
+ fluentd:
+ enabled: true
heka:
enabled: true
sensu:
diff --git a/neutron/compute.sls b/neutron/compute.sls
index cc1f1aa..d540fa7 100644
--- a/neutron/compute.sls
+++ b/neutron/compute.sls
@@ -118,6 +118,83 @@
- file: rabbitmq_ca_neutron_compute
{%- endif %}
+{%- set neutron_compute_services_list = compute.services %}
+{%- if compute.backend.sriov is defined %}
+ {%- do neutron_compute_services_list.append('neutron-sriov-agent') %}
+{%- endif %}
+{%- if compute.dvr %}
+ {%- do neutron_compute_services_list.extend(['neutron-l3-agent', 'neutron-metadata-agent']) %}
+{%- endif %}
+
+{%- for service_name in neutron_compute_services_list %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://neutron/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - require:
+ - pkg: neutron_compute_packages
+{% if compute.backend.sriov is defined %}
+ - pkg: neutron_sriov_package
+{% endif %}
+{% if compute.dvr %}
+ - pkg: neutron_dvr_packages
+{% endif %}
+ - watch_in:
+ - service: neutron_compute_services
+{% if compute.backend.sriov is defined %}
+ - service: neutron_sriov_service
+{% endif %}
+{% if compute.dvr %}
+ - service: neutron_dvr_agents
+{% endif %}
+{% endfor %}
+
+{%- if compute.logging.log_appender %}
+
+{%- if compute.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+neutron_compute_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in neutron_compute_services_list %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/neutron/logging/logging-{{ service_name }}.conf
+ - source: salt://neutron/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - user: neutron
+ - group: neutron
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ compute }}
+ - require:
+ - pkg: neutron_compute_packages
+{% if compute.backend.sriov is defined %}
+ - pkg: neutron_sriov_package
+{% endif %}
+{% if compute.dvr %}
+ - pkg: neutron_dvr_packages
+{% endif %}
+{%- if compute.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: neutron_compute_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: neutron_compute_services
+{% if compute.backend.sriov is defined %}
+ - service: neutron_sriov_service
+{% endif %}
+{% if compute.dvr %}
+ - service: neutron_dvr_agents
+{% endif %}
+{% endfor %}
+
+{% endif %}
{%- if compute.message_queue.get('ssl',{}).get('enabled', False) %}
rabbitmq_ca_neutron_compute:
diff --git a/neutron/files/default b/neutron/files/default
new file mode 100644
index 0000000..be6a4c1
--- /dev/null
+++ b/neutron/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{%- if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-{{ service_name }}.conf"
+{%- endif %}
diff --git a/neutron/files/logging.conf b/neutron/files/logging.conf
new file mode 100644
index 0000000..f9cff2c
--- /dev/null
+++ b/neutron/files/logging.conf
@@ -0,0 +1,73 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, neutron
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_neutron]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = neutron
+
+[logger_amqplib]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+
+[logger_boto]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+[logger_suds]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/neutron/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/neutron/files/mitaka/neutron-server b/neutron/files/mitaka/neutron-server
index 54f6ceb..59f0305 100644
--- a/neutron/files/mitaka/neutron-server
+++ b/neutron/files/mitaka/neutron-server
@@ -13,4 +13,8 @@
{%- if server.backend.engine == "contrail" %}
NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugins/opencontrail/ContrailPlugin.ini"
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+
+{%- if server.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-neutron-server.conf"
+{%- endif %}
diff --git a/neutron/files/newton/neutron-server b/neutron/files/newton/neutron-server
index 54f6ceb..59f0305 100644
--- a/neutron/files/newton/neutron-server
+++ b/neutron/files/newton/neutron-server
@@ -13,4 +13,8 @@
{%- if server.backend.engine == "contrail" %}
NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugins/opencontrail/ContrailPlugin.ini"
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+
+{%- if server.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-neutron-server.conf"
+{%- endif %}
diff --git a/neutron/files/ocata/neutron-server b/neutron/files/ocata/neutron-server
index 04830ca..5061cca 100644
--- a/neutron/files/ocata/neutron-server
+++ b/neutron/files/ocata/neutron-server
@@ -13,4 +13,8 @@
{%- if server.backend.engine == "contrail" %}
NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugins/opencontrail/ContrailPlugin.ini"
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+DAEMON_ARGS="--log-config-append=/etc/neutron/logging/logging-neutron-server.conf"
+{%- endif %}
diff --git a/neutron/gateway.sls b/neutron/gateway.sls
index 6046b96..bbbca27 100644
--- a/neutron/gateway.sls
+++ b/neutron/gateway.sls
@@ -49,6 +49,52 @@
- require:
- pkg: neutron_gateway_packages
+{%- for service_name in gateway.services %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://neutron/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ gateway }}
+ - require:
+ - pkg: neutron_gateway_packages
+ - watch_in:
+ - service: neutron_gateway_services
+{% endfor %}
+
+{%- if gateway.logging.log_appender %}
+
+{%- if gateway.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+neutron_gateway_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in gateway.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/neutron/logging/logging-{{ service_name }}.conf
+ - source: salt://neutron/files/logging.conf
+ - template: jinja
+ - makedirs: true
+ - user: neutron
+ - group: neutron
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ gateway }}
+ - require:
+ - pkg: neutron_gateway_packages
+{%- if gateway.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: neutron_gateway_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: neutron_gateway_services
+{% endfor %}
+
+{% endif %}
+
neutron_gateway_services:
service.running:
- names: {{ gateway.services }}
diff --git a/neutron/map.jinja b/neutron/map.jinja
index 4c95928..8221b89 100644
--- a/neutron/map.jinja
+++ b/neutron/map.jinja
@@ -15,7 +15,15 @@
'dpdk': false,
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-neutron-openvswitch', 'openvswitch', 'python-pycadf'],
@@ -25,7 +33,15 @@
'dpdk': false,
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.neutron.get('compute', {}), base='BaseDefaults') %}
@@ -34,12 +50,28 @@
'Debian': {
'pkgs': ['neutron-dhcp-agent', 'neutron-openvswitch-agent', 'neutron-l3-agent', 'openvswitch-common', 'neutron-metadata-agent'],
'services': ['neutron-openvswitch-agent', 'neutron-metadata-agent', 'neutron-l3-agent', 'neutron-dhcp-agent'],
- 'dpdk': false
+ 'dpdk': false,
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs': ['openstack-neutron-openvswitch'],
'services': ['neutron-openvswitch-agent', 'neutron-metadata-agent', 'neutron-l3-agent', 'neutron-dhcp-agent'],
- 'dpdk': false
+ 'dpdk': false,
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.neutron.get('gateway', {}), base='BaseDefaults') %}
@@ -56,7 +88,15 @@
'cors': {},
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
'RedHat': {
'pkgs_ml2': ['openstack-neutron-ml2', 'python-pycadf'],
@@ -69,7 +109,15 @@
'cors': {},
'audit': {
'enabled': false
- }
+ },
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
},
}, merge=pillar.neutron.get('server', {}), base='BaseDefaults') %}
diff --git a/neutron/meta/fluentd.yml b/neutron/meta/fluentd.yml
new file mode 100644
index 0000000..de237e0
--- /dev/null
+++ b/neutron/meta/fluentd.yml
@@ -0,0 +1,110 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_neutron:
+ tag: openstack.neutron.*
+ type: relabel
+ label: openstack_neutron
+ openstack_neutron:
+ filter:
+ set_neutron_fields:
+ tag: openstack.neutron.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: programname
+ value: neutron-${ tag_parts[2] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ parse_http_stats:
+ tag: openstack.neutron
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.neutron.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.neutron
+ send_to_default:
+ tag: openstack.neutron
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.neutron_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.neutron_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ neutron_logs_per_severity:
+ tag: metric.neutron_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: neutron
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ neutron_openstack_http_response_times:
+ tag: metric.neutron_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: neutron
+ - name: host
+ value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/neutron/server.sls b/neutron/server.sls
index c7be3d7..64a3427 100644
--- a/neutron/server.sls
+++ b/neutron/server.sls
@@ -123,6 +123,54 @@
- require:
- pkg: neutron_server_packages
+{%- for service_name in server.get('services', []) %}
+{%- if service_name != 'neutron-server' %}
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://neutron/files/default
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - require:
+ - pkg: neutron_server_packages
+ - watch_in:
+ - service: neutron_server_services
+{%- endif %}
+{%- endfor %}
+
+{%- if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+neutron_server_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{%- for service_name in server.services %}
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/neutron/logging/logging-{{ service_name }}.conf
+ - source: salt://neutron/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ server }}
+ - user: neutron
+ - group: neutron
+ - require:
+ - pkg: neutron_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: neutron_server_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: neutron_server_services
+{%- endfor %}
+
+{%- endif %}
+
{%- for name, rule in server.get('policy', {}).iteritems() %}
{%- if rule != None %}