Add support loggin conf for manila
Add support logging conf openstack_log_appender param
with fluentd and syslog log handlers
Change-Id: Ib4000d368976c07ff8724069b09dedb38b664874
Related-Prod: PROD-19781
diff --git a/.travis.yml b/.travis.yml
index 8f62a35..e7b61ad 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,11 +27,13 @@
- PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 OS_VERSION=pike SUITE=share-nexenta
- PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 OS_VERSION=pike SUITE=share-glusterfs
- PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 OS_VERSION=pike SUITE=server-cluster-ssl
- - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=server-cluster
- - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=server-single
- - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=share-nexenta
- - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=share-glusterfs
- - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=server-cluster-ssl
+# Uncomment when salt 2018.3.1 released bug https://github.com/saltstack/salt/issues/46929
+# - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=server-cluster
+# - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=server-single
+# - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=share-nexenta
+# - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=share-glusterfs
+# - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 OS_VERSION=pike SUITE=server-cluster-ssl
+#
# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 OS_VERSION=pike SUITE=server-cluster
# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 OS_VERSION=pike SUITE=server-single
# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 OS_VERSION=pike SUITE=share-nexenta
diff --git a/README.rst b/README.rst
index e3e4bad..f7fcf2d 100644
--- a/README.rst
+++ b/README.rst
@@ -112,6 +112,31 @@
extra_specs:
driver_handles_share_servers: false
+Enhanced logging with logging.conf
+----------------------------------
+
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+ * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+ * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+ manila:
+ common:
+ logging:
+ log_appender: true
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: true
+
More information
================
diff --git a/manila/_logging.sls b/manila/_logging.sls
new file mode 100644
index 0000000..700f008
--- /dev/null
+++ b/manila/_logging.sls
@@ -0,0 +1,37 @@
+{%- if config.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+{{ service_name }}_fluentd_logger_package:
+ pkg.installed:
+ - name: python-fluent-logger
+{%- endif %}
+
+{{ service_name }}_logging_conf:
+ file.managed:
+ - name: /etc/manila/logging/logging-{{ service_name }}.conf
+ - source: salt://manila/files/logging.conf
+ - template: jinja
+ - makedirs: True
+ - user: manila
+ - group: manila
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ config }}
+ - require:
+ - pkg: {{ service_name }}_pkg
+{%- if config.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ - pkg: {{ service_name }}_fluentd_logger_package
+{%- endif %}
+ - watch_in:
+ - service: {{ service_name }}
+
+{{ service_name }}_default:
+ file.managed:
+ - name: /etc/default/{{ service_name }}
+ - source: salt://manila/files/default
+ - template: jinja
+ - require:
+ - pkg: {{ service_name }}_pkg
+ - defaults:
+ service_name: {{ service_name }}
+ values: {{ config }}
+ - watch_in:
+ - service: {{ service_name }}
\ No newline at end of file
diff --git a/manila/data.sls b/manila/data.sls
index b4864dc..0516b98 100644
--- a/manila/data.sls
+++ b/manila/data.sls
@@ -1,9 +1,9 @@
-{%- from "manila/map.jinja" import data with context %}
+{%- from "manila/map.jinja" import data, cfg with context %}
{%- if data.enabled %}
include:
- manila._common
-manila_data_packages:
+{{ data.service }}_pkg:
pkg.installed:
- names: {{ data.pkgs }}
@@ -16,4 +16,14 @@
- onlyif: /bin/false
{%- endif %}
-{%- endif %}
+{% if not data.get('logging', {}).get('log_appender', False) %}
+{%- do data.update({'logging': cfg.logging})%}
+{% endif %}
+
+{% if data.logging.log_appender == True %}
+{%- set service_name = data.service %}
+{%- set config = data %}
+{%- include "manila/_logging.sls" %}
+{% endif %}
+
+{%- endif %}
\ No newline at end of file
diff --git a/manila/files/default b/manila/files/default
new file mode 100644
index 0000000..5a1a7b1
--- /dev/null
+++ b/manila/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/manila/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/manila/files/logging.conf b/manila/files/logging.conf
new file mode 100644
index 0000000..eaa0b1e
--- /dev/null
+++ b/manila/files/logging.conf
@@ -0,0 +1,90 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+ {%- if log_handler_attrs.get('enabled', False) %}
+ {%- do log_handlers.append(log_handler_name) -%}
+ {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, manila
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default{% if values.logging.log_handlers.get('fluentd',{}).get('enabled', False) %}, fluentd{% endif %}
+
+[logger_root]
+level = {{ values.logging.get('loggers', {}).get('root', {}).get('level', 'WARNING') }}
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_amqp]
+level = {{ values.logging.get('loggers', {}).get('amqp', {}).get('level', 'WARNING') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqp
+
+[logger_manila]
+level = {{ values.logging.get('loggers', {}).get('manila', {}).get('level', 'INFO') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = manila
+propagate = 0
+
+[logger_amqplib]
+level = {{ values.logging.get('loggers', {}).get('amqplib', {}).get('level', 'WARNING') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = {{ values.logging.get('loggers', {}).get('sqlalchemy', {}).get('level', 'WARNING') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither. (Recommended for production systems.)
+
+[logger_boto]
+level = {{ values.logging.get('loggers', {}).get('boto', {}).get('level', 'WARNING') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = boto
+
+[logger_suds]
+level = {{ values.logging.get('loggers', {}).get('suds', {}).get('level', 'INFO') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = suds
+
+[logger_eventletwsgi]
+level = {{ values.logging.get('loggers', {}).get('eventletwsgi', {}).get('level', 'WARNING') }}
+handlers = {{ log_handlers | join(", ") }}
+qualname = eventlet.wsgi.server
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".") }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/manila/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+{% if values.logging.log_handlers.get('ossyslog', {}).get('enabled', False) -%}
+{%- set ossyslog_args = values.logging.log_handlers.ossyslog.get('args', {}) -%}
+[handler_ossyslog]
+class = oslo_log.handlers.OSSysLogHandler
+args = ( handlers.SysLogHandler.{{ ossyslog_args.get('facility', 'LOG_USER') }}, )
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
\ No newline at end of file
diff --git a/manila/map.jinja b/manila/map.jinja
index 80350ad..4b7aa93 100644
--- a/manila/map.jinja
+++ b/manila/map.jinja
@@ -4,7 +4,18 @@
})
%}
-{% set cfg = pillar.manila.get('common', {}) %}
+{% set cfg = salt['grains.filter_by']({
+ 'Debian': {
+ 'logging': {
+ 'log_appender': false,
+ 'log_handlers': {
+ 'watchedfile': {
+ 'enabled': true
+ }
+ },
+ },
+ }
+}, merge=pillar.manila.get('common', {})) %}
{% set api = salt['grains.filter_by']({
'Debian': {
diff --git a/manila/meta/fluentd.yml b/manila/meta/fluentd.yml
new file mode 100644
index 0000000..21f00be
--- /dev/null
+++ b/manila/meta/fluentd.yml
@@ -0,0 +1,171 @@
+{%- from "manila/map.jinja" import api with context %}
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+{%- set apache_wsgi = api.get('enabled', False) %}
+agent:
+ config:
+ label:
+ forward_input:
+ input:
+ generic_forward_input:
+ type: forward
+ bind: 0.0.0.0
+ port: 24224
+ match:
+ route_openstack_manila:
+ tag: openstack.manila.**
+ type: relabel
+ label: openstack_manila
+{%- if apache_wsgi %}
+ openstack_manila_wsgi:
+ input:
+ manila_api_wsgi_in_tail:
+ type: tail
+ path: /var/log/apache2/manila.access.log
+ tag: openstack.manila
+ pos_file: {{ positiondb }}/manila.wsgi.pos
+ parser:
+ type: regexp
+ time_key: Timestamp
+ time_format: '%d/%b/%Y:%H:%M:%S %z'
+ keep_time_key: false
+ # Apache format: https://regex101.com/r/WeCT7s/5
+ format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+ filter:
+ add_manila_wsgi_record_fields:
+ tag: openstack.manila
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: 6
+ - name: severity_label
+ value: INFO
+ - name: programname
+ value: manila-wsgi
+ - name: http_response_time
+ value: ${ record['http_response_time'].to_i/100000.to_f }
+ match:
+ send_to_default:
+ tag: openstack.manila
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.manila_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.manila_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+{%- endif %}
+ openstack_manila:
+ filter:
+ set_programname:
+ tag: openstack.manila.*
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: programname
+ value: manila-${ tag_parts[2] }
+ filter:
+ set_log_record_fields:
+ tag: openstack.manila
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: Severity
+ value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+ - name: severity_label
+ value: ${ record['level'] }
+ - name: Payload
+ value: ${ record['message'] }
+ - name: python_module
+ value: ${ record['name'] }
+ - name: programname
+ value: '${ record["programname"] ? record["programname"] : "manila" }'
+ parse_http_stats:
+ tag: openstack.manila
+ type: parser
+ key_name: Payload
+ reserve_data: true
+ emit_invalid_record_to_error: false
+ parser:
+ type: regexp
+ # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+ format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+ types: http_response_time:float
+ match:
+ unify_tag:
+ tag: openstack.manila.*
+ type: rewrite_tag_filter
+ rule:
+ - name: level
+ regexp: '.*'
+ result: openstack.manila
+ send_to_default:
+ tag: openstack.manila
+ type: copy
+ store:
+ - type: relabel
+ label: default_output
+ - type: rewrite_tag_filter
+ rule:
+ - name: severity_label
+ regexp: '.'
+ result: metric.manila_log_messages
+ - type: rewrite_tag_filter
+ rule:
+ - name: http_status
+ regexp: '.'
+ result: metric.manila_openstack_http_response
+ push_to_metric:
+ tag: 'metric.**'
+ type: relabel
+ label: default_metric
+ default_metric:
+ filter:
+ manila_logs_per_severity:
+ tag: metric.manila_log_messages
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: log_messages
+ type: counter
+ desc: Total number of log lines by severity
+ label:
+ - name: service
+ value: manila
+ - name: level
+ value: ${severity_label}
+ - name: host
+ value: ${Hostname}
+ manila_openstack_http_response_times:
+ tag: metric.manila_openstack_http_response
+ require:
+ - add_general_fields
+ type: prometheus
+ metric:
+ - name: openstack_http_response_times
+ type: summary
+ desc: Total number of requests per method and status
+ key: http_response_time
+ label:
+ - name: http_method
+ value: ${http_method}
+ - name: http_status
+ value: ${http_status}
+ - name: service
+ value: manila
+ - name: host
+ value: ${Hostname}
+{% endif %}
diff --git a/manila/scheduler.sls b/manila/scheduler.sls
index 9089c90..ae4af1f 100644
--- a/manila/scheduler.sls
+++ b/manila/scheduler.sls
@@ -1,9 +1,9 @@
-{%- from "manila/map.jinja" import scheduler with context %}
+{%- from "manila/map.jinja" import scheduler, cfg with context %}
{%- if scheduler.enabled %}
include:
- manila._common
-manila_scheduler_packages:
+{{ scheduler.service }}_pkg:
pkg.installed:
- names: {{ scheduler.pkgs }}
@@ -16,4 +16,14 @@
- onlyif: /bin/false
{%- endif %}
+{% if not scheduler.get('logging', {}).get('log_appender', False) %}
+{%- do scheduler.update({'logging': cfg.logging})%}
+{% endif %}
+
+{% if scheduler.logging.log_appender == True %}
+{%- set service_name = scheduler.service %}
+{%- set config = scheduler %}
+{%- include "manila/_logging.sls" %}
+{% endif %}
+
{%- endif %}
diff --git a/manila/share.sls b/manila/share.sls
index 76228b0..c44f23f 100644
--- a/manila/share.sls
+++ b/manila/share.sls
@@ -1,9 +1,9 @@
-{%- from "manila/map.jinja" import share with context %}
+{%- from "manila/map.jinja" import share, cfg with context %}
{%- if share.enabled %}
include:
- manila._common
-manila_share_packages:
+{{ share.service }}_pkg:
pkg.installed:
- names: {{ share.pkgs }}
@@ -16,4 +16,14 @@
- onlyif: /bin/false
{%- endif %}
-{%- endif %}
+{% if not share.get('logging', {}).get('log_appender', False) %}
+{%- do share.update({'logging': cfg.logging})%}
+{% endif %}
+
+{% if share.logging.log_appender == True %}
+{%- set service_name = share.service %}
+{%- set config = share %}
+{%- include "manila/_logging.sls" %}
+{% endif %}
+
+{%- endif %}
\ No newline at end of file
diff --git a/metadata/service/common/cluster.yml b/metadata/service/common/cluster.yml
index e9e9b15..bcce15f 100644
--- a/metadata/service/common/cluster.yml
+++ b/metadata/service/common/cluster.yml
@@ -1,6 +1,10 @@
applications:
- manila
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
+ openstack_ossyslog_handler_enabled: false
manila:
common:
version: ${_param:openstack_version}
@@ -30,3 +34,12 @@
user_domain_id: default
project_domain_id: default
protocol: 'http'
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
+ ossyslog:
+ enabled: ${_param:openstack_ossyslog_handler_enabled}
diff --git a/metadata/service/common/single.yml b/metadata/service/common/single.yml
index b61b269..4919207 100644
--- a/metadata/service/common/single.yml
+++ b/metadata/service/common/single.yml
@@ -1,6 +1,10 @@
applications:
- manila
parameters:
+ _param:
+ openstack_log_appender: false
+ openstack_fluentd_handler_enabled: false
+ openstack_ossyslog_handler_enabled: false
manila:
common:
version: ${_param:openstack_version}
@@ -30,3 +34,12 @@
user_domain_id: default
project_domain_id: default
protocol: 'http'
+ logging:
+ log_appender: ${_param:openstack_log_appender}
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: ${_param:openstack_fluentd_handler_enabled}
+ ossyslog:
+ enabled: ${_param:openstack_ossyslog_handler_enabled}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 559dee2..6130e38 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -9,3 +9,5 @@
enabled: false
sphinx:
enabled: true
+ fluentd:
+ enabled: true
diff --git a/tests/pillar/server_cluster.sls b/tests/pillar/server_cluster.sls
index 368c199..bae2402 100644
--- a/tests/pillar/server_cluster.sls
+++ b/tests/pillar/server_cluster.sls
@@ -34,6 +34,15 @@
user: openstack
password: workshop
virtual_host: '/openstack'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: false
+ ossyslog:
+ enabled: false
apache:
server:
enabled: true
diff --git a/tests/pillar/server_cluster_ssl.sls b/tests/pillar/server_cluster_ssl.sls
index 425de7f..0df203d 100644
--- a/tests/pillar/server_cluster_ssl.sls
+++ b/tests/pillar/server_cluster_ssl.sls
@@ -38,6 +38,15 @@
user: openstack
password: workshop
virtual_host: '/openstack'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: false
+ ossyslog:
+ enabled: false
apache:
server:
enabled: true
diff --git a/tests/pillar/server_single.sls b/tests/pillar/server_single.sls
index 7a82b43..09e10be 100644
--- a/tests/pillar/server_single.sls
+++ b/tests/pillar/server_single.sls
@@ -28,6 +28,15 @@
user: openstack
password: workshop
virtual_host: '/openstack'
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: false
+ ossyslog:
+ enabled: false
apache:
server:
enabled: true
diff --git a/tests/pillar/share_glusterfs.sls b/tests/pillar/share_glusterfs.sls
index 41a9526..6b68be2 100644
--- a/tests/pillar/share_glusterfs.sls
+++ b/tests/pillar/share_glusterfs.sls
@@ -45,6 +45,15 @@
- server1
- server2
glusterfs_volume_pattern: manila-share-volume-d+$
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: false
+ ossyslog:
+ enabled: false
apache:
server:
enabled: true
diff --git a/tests/pillar/share_nexenta.sls b/tests/pillar/share_nexenta.sls
index 7d6dc28..6c28bbf 100644
--- a/tests/pillar/share_nexenta.sls
+++ b/tests/pillar/share_nexenta.sls
@@ -35,6 +35,15 @@
nexenta_password: secretpassword
nexenta_user: nonexistent
nexenta_volume: volume1
+ logging:
+ log_appender: false
+ log_handlers:
+ watchedfile:
+ enabled: true
+ fluentd:
+ enabled: false
+ ossyslog:
+ enabled: false
apache:
server:
enabled: true