Merge "Enable logging.conf & fluentd for ceilometer"
diff --git a/README.rst b/README.rst
index 44e9150..f165775 100644
--- a/README.rst
+++ b/README.rst
@@ -211,7 +211,39 @@
             - host: 10.10.10.12
               port: 11211
 
+Enhanced logging with logging.conf
+----------------------------------
 
+By default logging.conf is disabled.
+
+That is possible to enable per-binary logging.conf with new variables:
+  * openstack_log_appender - set it to true to enable log_config_append for all OpenStack services;
+  * openstack_fluentd_handler_enabled - set to true to enable FluentHandler for all Openstack services.
+
+Only WatchedFileHandler and FluentHandler are available.
+
+Also it is possible to configure this with pillar:
+
+.. code-block:: yaml
+
+  ceilometer:
+    server:
+      logging:
+        log_appender: true
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: true
+
+    agent:
+      logging:
+        log_appender: true
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: true
 
 
 More Information
diff --git a/ceilometer/agent.sls b/ceilometer/agent.sls
index a574ce6..9370827 100644
--- a/ceilometer/agent.sls
+++ b/ceilometer/agent.sls
@@ -12,6 +12,52 @@
   - require:
     - pkg: ceilometer_agent_packages
 
+{% for service_name in agent.services %}
+{{ service_name }}_default:
+  file.managed:
+    - name: /etc/default/{{ service_name }}
+    - source: salt://ceilometer/files/default
+    - template: jinja
+    - require:
+      - pkg: ceilometer_agent_packages
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ agent }}
+    - watch_in:
+      - service: ceilometer_agent_services
+{% endfor %}
+
+{% if agent.logging.log_appender -%}
+
+{%- if agent.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ceilometer_agent_fluentd_logger_package:
+  pkg.installed:
+    - name: python-fluent-logger
+{%- endif %}
+
+{% for service_name in agent.services %}
+{{ service_name }}_logging_conf:
+  file.managed:
+    - name: /etc/ceilometer_agent/logging/logging-{{ service_name }}.conf
+    - source: salt://ceilometer/files/logging.conf
+    - template: jinja
+    - user: ceilometer
+    - group: ceilometer
+    - require:
+      - pkg: ceilometer_agent_packages
+{%- if agent.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+      - pkg: ceilometer_agent_fluentd_logger_package
+{%- endif %}
+    - makedirs: True
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ agent }}
+    - watch_in:
+      - service: ceilometer_agent_services
+{% endfor %}
+
+{% endif %}
+
 {%- for publisher_name, publisher in agent.get('publisher', {}).iteritems() %}
 
 {%- if publisher_name != "default" %}
diff --git a/ceilometer/files/default b/ceilometer/files/default
new file mode 100644
index 0000000..857d54b
--- /dev/null
+++ b/ceilometer/files/default
@@ -0,0 +1,4 @@
+# Generated by Salt.
+{% if values.logging.log_appender %}
+DAEMON_ARGS="--log-config-append=/etc/ceilometer/logging/logging-{{ service_name }}.conf"
+{% endif %}
diff --git a/ceilometer/files/logging.conf b/ceilometer/files/logging.conf
new file mode 100644
index 0000000..dfd181c
--- /dev/null
+++ b/ceilometer/files/logging.conf
@@ -0,0 +1,49 @@
+{%- set log_handlers = [] -%}
+{%- for log_handler_name, log_handler_attrs in values.logging.log_handlers.items() %}
+  {%- if log_handler_attrs.get('enabled', False) %}
+    {%- do log_handlers.append(log_handler_name) -%}
+  {%- endif %}
+{%- endfor %}
+[loggers]
+keys = root, ceilometer
+
+[handlers]
+keys = {{ log_handlers | join(", ") }}
+
+[formatters]
+keys = context, default, fluentd
+
+[logger_root]
+level = WARNING
+handlers = {{ log_handlers | join(", ") }}
+
+[logger_ceilometer]
+level = INFO
+handlers = {{ log_handlers | join(", ") }}
+qualname = ceilometer
+propagate = 0
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[handler_fluentd]
+class = fluent.handler.FluentHandler
+args = ('openstack.{{ service_name | replace("-", ".", 1) }}', 'localhost', 24224)
+formatter = fluentd
+{%- endif %}
+
+{%- if values.logging.log_handlers.watchedfile.enabled %}
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('/var/log/ceilometer/{{ service_name }}.log',)
+formatter = context
+{%- endif %}
+
+[formatter_context]
+class = oslo_log.formatters.ContextFormatter
+
+[formatter_default]
+format = %(message)s
+
+{%- if values.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+[formatter_fluentd]
+class = oslo_log.formatters.FluentFormatter
+{%- endif %}
diff --git a/ceilometer/files/mitaka/ceilometer-server.conf.Debian b/ceilometer/files/mitaka/ceilometer-server.conf.Debian
index 15895e9..e695903 100644
--- a/ceilometer/files/mitaka/ceilometer-server.conf.Debian
+++ b/ceilometer/files/mitaka/ceilometer-server.conf.Debian
@@ -13,6 +13,9 @@
 {%- else %}
 debug = False
 {%- endif %}
+{%- if server.logging.log_appender %}
+log_config_append=/etc/ceilometer/logging.conf
+{%- endif %}
 
 # Print more verbose output (set logging level to INFO instead
 # of default WARNING level). (boolean value)
diff --git a/ceilometer/files/newton/ceilometer-server.conf.Debian b/ceilometer/files/newton/ceilometer-server.conf.Debian
index 82e1deb..a571039 100644
--- a/ceilometer/files/newton/ceilometer-server.conf.Debian
+++ b/ceilometer/files/newton/ceilometer-server.conf.Debian
@@ -25,6 +25,9 @@
 # example, logging_context_format_string). (string value)
 # Deprecated group/name - [DEFAULT]/log_config
 #log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/ceilometer/logging.conf
+{%- endif %}
 
 # Defines the format string for %%(asctime)s in log records. Default:
 # %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/ceilometer/files/newton/ceilometer.apache2.conf.Debian b/ceilometer/files/newton/ceilometer.apache2.conf.Debian
index 988ab30..bf28270 100644
--- a/ceilometer/files/newton/ceilometer.apache2.conf.Debian
+++ b/ceilometer/files/newton/ceilometer.apache2.conf.Debian
@@ -10,5 +10,5 @@
         ErrorLogFormat "%{cu}t %M"
     </IfVersion>
     ErrorLog /var/log/apache2/ceilometer_error.log
-    CustomLog /var/log/apache2/ceilometer_access.log combined
+    CustomLog /var/log/apache2/ceilometer_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
 </VirtualHost>
diff --git a/ceilometer/files/ocata/ceilometer-server.conf.Debian b/ceilometer/files/ocata/ceilometer-server.conf.Debian
index 0316403..895cdad 100644
--- a/ceilometer/files/ocata/ceilometer-server.conf.Debian
+++ b/ceilometer/files/ocata/ceilometer-server.conf.Debian
@@ -161,6 +161,9 @@
 # Note: This option can be changed without restarting.
 # Deprecated group/name - [DEFAULT]/log_config
 #log_config_append = <None>
+{%- if server.logging.log_appender %}
+log_config_append=/etc/ceilometer/logging.conf
+{%- endif %}
 
 # Defines the format string for %%(asctime)s in log records. Default:
 # %(default)s . This option is ignored if log_config_append is set. (string
diff --git a/ceilometer/files/ocata/ceilometer.apache2.conf.Debian b/ceilometer/files/ocata/ceilometer.apache2.conf.Debian
index a0406fe..44cea9b 100644
--- a/ceilometer/files/ocata/ceilometer.apache2.conf.Debian
+++ b/ceilometer/files/ocata/ceilometer.apache2.conf.Debian
@@ -10,5 +10,5 @@
         ErrorLogFormat "%{cu}t %M"
     </IfVersion>
     ErrorLog /var/log/apache2/ceilometer_error.log
-    CustomLog /var/log/apache2/ceilometer_access.log combined
+    CustomLog /var/log/apache2/ceilometer_access.log "%v:%p %h %l %u %t \"%r\" %>s %D %O \"%{Referer}i\" \"%{User-Agent}i\""
 </VirtualHost>
diff --git a/ceilometer/map.jinja b/ceilometer/map.jinja
index 6f40616..869f393 100644
--- a/ceilometer/map.jinja
+++ b/ceilometer/map.jinja
@@ -2,10 +2,26 @@
     'Debian': {
         'pkgs': ['ceilometer-agent-compute'],
         'services': ['ceilometer-agent-compute'],
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
     },
     'RedHat': {
         'pkgs': ['openstack-ceilometer-compute'],
         'services': ['openstack-ceilometer-compute'],
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
     },
 }, merge=salt['pillar.get']('ceilometer:agent')) %}
 
@@ -41,10 +57,21 @@
     },
 }) %}
 
-{%- set server = salt['grains.filter_by']({'default': {}}, merge=salt['pillar.get']('ceilometer:server')) %}
+{%- set server = salt['grains.filter_by']({
+    'default': {
+        'logging': {
+          'log_appender': false,
+          'log_handlers': {
+            'watchedfile': {
+              'enabled': true
+            }
+          },
+        },
+    }
+}, merge=salt['pillar.get']('ceilometer:server')) %}
 
-# Mitaka and newer OpenStack releases don't need collector and alarm 
-# packages/services, because alarming is implemented by Aodh and collector 
+# Mitaka and newer OpenStack releases don't need collector and alarm
+# packages/services, because alarming is implemented by Aodh and collector
 # stuff is implemented by the ceilometer_collector service.
 # The ceilometer api for Ocata and newer is served by webserver.
 {%- if salt['pillar.get']('ceilometer:server:version', 'mitaka') in ['liberty', 'juno', 'kilo'] %}
diff --git a/ceilometer/meta/fluentd.yml b/ceilometer/meta/fluentd.yml
new file mode 100644
index 0000000..d5d868e
--- /dev/null
+++ b/ceilometer/meta/fluentd.yml
@@ -0,0 +1,172 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- from "ceilometer/map.jinja" import server with context %}
+{%- set apache_wsgi = server.get('enabled') and server.version not in ['mitaka'] %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+agent:
+  config:
+    label:
+      forward_input:
+        input:
+          generic_forward_input:
+            type: forward
+            bind: 0.0.0.0
+            port: 24224
+        match:
+          route_openstack_ceilometer:
+            tag: openstack.ceilometer.**
+            type: relabel
+            label: openstack_ceilometer
+{%- if apache_wsgi %}
+      openstack_ceilometer_wsgi:
+        input:
+          ceilometer_api_wsgi_in_tail:
+            type: tail
+            path: /var/log/apache2/ceilometer_access.log
+            tag: openstack.ceilometer
+            pos_file: {{ positiondb }}/ceilometer.wsgi.pos
+            parser:
+              type: regexp
+              time_key: Timestamp
+              time_format: '%d/%b/%Y:%H:%M:%S %z'
+              keep_time_key: false
+              # Apache format: https://regex101.com/r/WeCT7s/5
+              format: '/(?<hostname>[\w\.\-]+)\:(?<port>\d+)\s(?<http_client_ip_address>[\d\.]+)\s\-\s\-\s\[(?<Timestamp>.*)\]\s(?<Payload>\"(?<http_method>[A-Z]+)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\s(?<http_status>\d{3})\s(?<http_response_time>\d+)\s(?<http_response_size>\d+)\s\"(?<http_referer>.*)\"\s\"(?<user_agent>.*)\")/'
+        filter:
+          add_ceilometer_wsgi_record_fields:
+            tag: openstack.ceilometer
+            type: record_transformer
+            enable_ruby: true
+            record:
+              - name: Severity
+                value: 6
+              - name: severity_label
+                value: INFO
+              - name: programname
+                value: ceilometer-wsgi
+              - name: http_response_time
+                value: ${ record['http_response_time'].to_i/100000.to_f }
+        match:
+          send_to_default:
+            tag: openstack.ceilometer
+            type: copy
+            store:
+              - type: relabel
+                label: default_output
+              - type: rewrite_tag_filter
+                rule:
+                  - name: severity_label
+                    regexp: '.'
+                    result: metric.ceilometer_log_messages
+              - type: rewrite_tag_filter
+                rule:
+                  - name: http_status
+                    regexp: '.'
+                    result: metric.ceilometer_openstack_http_response
+          push_to_metric:
+            tag: 'metric.**'
+            type: relabel
+            label: default_metric
+{%- endif %}
+      openstack_ceilometer:
+        filter:
+          set_ceilometer_programname:
+            tag: openstack.ceilometer.*
+            type: record_transformer
+            enable_ruby: true
+            record:
+              - name: programname
+                value: ceilometer-${ tag_parts[2] }
+          set_log_record_fields:
+            tag: openstack.ceilometer
+            type: record_transformer
+            enable_ruby: true
+            record:
+              - name: Severity
+                value: ${ {'TRACE'=>7,'DEBUG'=>7,'INFO'=>6,'AUDIT'=>6,'WARNING'=>4,'ERROR'=>3,'CRITICAL'=>2}[record['level']].to_i }
+              - name: severity_label
+                value: ${ record['level'] }
+              - name: programname
+                value: ceilometer-${ tag_parts[2] }
+              - name: Payload
+                value: ${ record['message'] }
+              - name: python_module
+                value: ${ record['name'] }
+              - name: programname
+                value: '${ record["programname"] ? record["programname"] : "ceilometer" }'
+          parse_http_stats:
+            tag: openstack.ceilometer
+            type: parser
+            key_name: Payload
+            reserve_data: true
+            emit_invalid_record_to_error: false
+            parser:
+              type: regexp
+              # Parse openstack http stats: https://regex101.com/r/Tf0XUK/1/
+              format: '\"(?<http_method>GET|POST|OPTIONS|DELETE|PUT|HEAD|TRACE|CONNECT|PATCH)\s(?<http_url>\S+)\s(?<http_version>[.\/\dHTFSP]+)\"\sstatus:\s(?<http_status>\d{3})\slen:\s(?<http_response_size>\d+)\stime:\s(?<http_response_time>\d+\.\d+)'
+              types: http_response_time:float
+        match:
+          unify_tag:
+            tag: openstack.ceilometer.*
+            type: rewrite_tag_filter
+            rule:
+              - name: level
+                regexp: '.*'
+                result: openstack.ceilometer
+          send_to_default:
+            tag: openstack.ceilometer
+            type: copy
+            store:
+              - type: relabel
+                label: default_output
+              - type: rewrite_tag_filter
+                rule:
+                  - name: severity_label
+                    regexp: '.'
+                    result: metric.ceilometer_log_messages
+              - type: rewrite_tag_filter
+                rule:
+                  - name: http_status
+                    regexp: '.'
+                    result: metric.ceilometer_openstack_http_response
+          push_to_metric:
+            tag: 'metric.**'
+            type: relabel
+            label: default_metric
+      default_metric:
+        filter:
+          ceilometer_logs_per_severity:
+            tag: metric.ceilometer_log_messages
+            require:
+              - add_general_fields
+            type: prometheus
+            metric:
+              - name: log_messages
+                type: counter
+                desc: Total number of log lines by severity
+            label:
+              - name: service
+                value: ceilometer
+              - name: level
+                value: ${severity_label}
+              - name: host
+                value: ${Hostname}
+          ceilometer_openstack_http_response_times:
+            tag: metric.ceilometer_openstack_http_response
+            require:
+              - add_general_fields
+            type: prometheus
+            metric:
+              - name: openstack_http_response_times
+                type: summary
+                desc: Total number of requests per method and status
+                key: http_response_time
+            label:
+              - name: http_method
+                value: ${http_method}
+              - name: http_status
+                value: ${http_status}
+              - name: service
+                value: ceilometer
+              - name: host
+                value: ${Hostname}
+{% endif %}
\ No newline at end of file
diff --git a/ceilometer/server.sls b/ceilometer/server.sls
index d0f1fc6..f2bb9d0 100644
--- a/ceilometer/server.sls
+++ b/ceilometer/server.sls
@@ -12,6 +12,81 @@
   - require:
     - pkg: ceilometer_server_packages
 
+{%- for service_name in server.services %}
+{{ service_name }}_default:
+  file.managed:
+    - name: /etc/default/{{ service_name }}
+    - source: salt://ceilometer/files/default
+    - template: jinja
+    - require:
+      - pkg: ceilometer_server_packages
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ server }}
+    - watch_in:
+      - service: ceilometer_server_services
+{%- endfor %}
+
+{%- if server.logging.log_appender %}
+
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+ceilometer_server_fluentd_logger_package:
+  pkg.installed:
+    - name: python-fluent-logger
+{%- endif %}
+
+ceilometer_general_logging_conf:
+  file.managed:
+    - name: /etc/ceilometer/logging.conf
+    - source: salt://ceilometer/files/logging.conf
+    - template: jinja
+    - user: ceilometer
+    - group: ceilometer
+    - require:
+      - pkg: ceilometer_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+      - pkg: ceilometer_server_fluentd_logger_package
+{%- endif %}
+    - defaults:
+        service_name: ceilometer
+        values: {{ server }}
+    - watch_in:
+      - service: ceilometer_server_services
+{%- if server.version not in ['liberty', 'juno', 'kilo', 'mitaka'] %}
+      - service: ceilometer_apache_restart
+{%- endif %}
+
+/var/log/ceilometer/ceilometer.log:
+  file.managed:
+    - user: ceilometer
+    - group: ceilometer
+    - watch_in:
+      - service: ceilometer_server_services
+{%- if server.version not in ['liberty', 'juno', 'kilo', 'mitaka'] %}
+      - service: ceilometer_apache_restart
+{%- endif %}
+
+{%- for service_name in server.get('services', []) %}
+{{ service_name }}_logging_conf:
+  file.managed:
+    - name: /etc/ceilometer_server/logging/logging-{{ service_name }}.conf
+    - source: salt://ceilometer/files/logging.conf
+    - template: jinja
+    - require:
+      - pkg: ceilometer_server_packages
+{%- if server.logging.log_handlers.get('fluentd', {}).get('enabled', False) %}
+      - pkg: ceilometer_server_fluentd_logger_package
+{%- endif %}
+    - makedirs: True
+    - defaults:
+        service_name: {{ service_name }}
+        values: {{ server }}
+    - watch_in:
+      - service: ceilometer_server_services
+{%- endfor %}
+
+{%- endif %}
+
 {%- for name, rule in server.get('policy', {}).iteritems() %}
 
 {%- if rule != None %}
diff --git a/metadata/service/agent/cluster.yml b/metadata/service/agent/cluster.yml
index 1e56978..91cf7d9 100644
--- a/metadata/service/agent/cluster.yml
+++ b/metadata/service/agent/cluster.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_ceilometer_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   ceilometer:
     agent:
       enabled: true
@@ -20,6 +22,13 @@
         user: ceilometer
         password: ${_param:keystone_ceilometer_password}
         endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:cluster_vip_address}
diff --git a/metadata/service/agent/single.yml b/metadata/service/agent/single.yml
index dfffecf..23ee9ee 100644
--- a/metadata/service/agent/single.yml
+++ b/metadata/service/agent/single.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_ceilometer_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   ceilometer:
     agent:
       enabled: true
@@ -20,6 +22,13 @@
         user: ceilometer
         password: ${_param:keystone_ceilometer_password}
         endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml
index eaedfa1..5b190a8 100644
--- a/metadata/service/server/cluster.yml
+++ b/metadata/service/server/cluster.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_ceilometer_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   ceilometer:
     server:
       enabled: true
@@ -25,6 +27,13 @@
         user: ceilometer
         password: ${_param:keystone_ceilometer_password}
         endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:cluster_vip_address}
diff --git a/metadata/service/server/single.yml b/metadata/service/server/single.yml
index f150a61..d246599 100644
--- a/metadata/service/server/single.yml
+++ b/metadata/service/server/single.yml
@@ -5,6 +5,8 @@
 parameters:
   _param:
     keystone_ceilometer_endpoint_type: internalURL
+    openstack_log_appender: false
+    openstack_fluentd_handler_enabled: false
   ceilometer:
     server:
       enabled: true
@@ -24,6 +26,13 @@
         user: ceilometer
         password: ${_param:keystone_ceilometer_password}
         endpoint_type: ${_param:keystone_ceilometer_endpoint_type}
+      logging:
+        log_appender: ${_param:openstack_log_appender}
+        log_handlers:
+          watchedfile:
+            enabled: true
+          fluentd:
+            enabled: ${_param:openstack_fluentd_handler_enabled}
       message_queue:
         engine: rabbitmq
         host: ${_param:single_address}
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 24f09b4..d423400 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -3,6 +3,8 @@
     _support:
       collectd:
         enabled: true
+      fluentd:
+        enabled: true
       heka:
         enabled: true
       sensu: