README.rst

===============
Fluentd Formula
===============

Many web/mobile applications generate huge amount of event logs
(c,f. login, logout, purchase, follow, etc). Analyzing these event
logs can be quite valuable for improving services. However, collecting
these logs easily and reliably is a challenging task.

Fluentd solves the problem by having: easy installation, small footprint,
plugins reliable buffering, log forwarding, etc.

**NOTE: WORK IN PROGRES**
NOTE: DESIGN OF THIS FORMULA IS NOT YET STABLE AND MAY CHANGE
NOTE: FORMULA NOT COMPATIBLE WITH OLD VERSION

Sample Pillars
==============

General pillar structure
------------------------

.. code-block:: yaml

  fluentd:
    agent:
      config:
        label:
          filename:
            input:
              input_name:
                params
            filter:
              filter_name:
                params
              filter_name2:
                params
            match:
              match_name:
                params
        input:
          filename:
            input_name:
              params
            input_name2:
              params
          filename2:
            input_name3:
              params
        filter:
          filename:
            filter_name:
              params
            filter_name2:
              params
          filename2:
            filter_name3:
              params
        match:
          filename:
            match_name:
              params

Example pillar
--------------
.. code-block:: yaml

  fluentd:
    enabled: true
    agent:
      multiworker:
        worker_count: 4
      config:
        label:
          elasticsearch_output:
            worker: 0
            match:
              elasticsearch_output:
                tag: "**"
                type: elasticsearch
                host: 10.100.0.1
                port: 9200
                buffer:
                  flush_thread_count: 8
          monitoring:
            worker: '0-2'
            filter:
              parse_log:
                tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
                type: parser
                reserve_data: true
                key_name: log
                parser:
                  type: regexp
                  format: >-
                    /^time="(?<time>[^ ]*)" level=(?<severity>[a-zA-Z]*) msg="(?<message>.+?)"/
                  time_format: '%FT%TZ'
              remove_log_key:
                tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
                type: record_transformer
                remove_keys: log
            match:
              docker_log:
                tag: 'docker.**'
                type: file
                path: /tmp/flow-docker.log
          grok_example:
            input:
              test_log:
                type: tail
                path: /var/log/test
                tag: test.test
                parser:
                  type: grok
                  custom_pattern_path: /etc/td-agent/config.d/global.grok
                  rule:
                    - pattern: >-
                        %{KEYSTONEACCESS}
          syslog:
            filter:
              add_severity:
                tag: 'syslog.*'
                type: record_transformer
                enable_ruby: true
                record:
                  - name: severity
                    value: 'record["pri"].to_i - (record["pri"].to_i / 8).floor * 8'
              severity_to_string:
                tag: 'syslog.*'
                type: record_transformer
                enable_ruby: true
                record:
                  - name: severity
                    value: '{"debug"=>7,"info"=>6,"notice"=>5,"warning"=>4,"error"=>3,"critical"=>2,"alert"=>1,"emerg"=>0}.key(record["severity"])'
              severity_for_telegraf:
                tag: 'syslog.*.telegraf'
                type: parser
                reserve_data: true
                key_name: message
                parser:
                  type: regexp
                  format: >-
                    /^(?<time>[^ ]*) (?<severity>[A-Z])! (?<message>.*)/
                  time_format: '%FT%TZ'
              severity_for_telegraf_string:
                tag: 'syslog.*.telegraf'
                type: record_transformer
                enable_ruby: true
                record:
                  - name: severity
                    value: '{"debug"=>"D","info"=>"I","notice"=>"N","warning"=>"W","error"=>"E","critical"=>"C","alert"=>"A","emerg"=>"E"}.key(record["severity"])'
              prometheus_metric:
                tag: 'syslog.*.*'
                type: prometheus
                label:
                  - name: ident
                    type: variable
                    value: ident
                  - name: severity
                    type: variable
                    value: severity
                metric:
                  - name: log_messages
                    type: counter
                    desc: The total number of log messages.
            match:
              rewrite_tag_key:
                tag: 'syslog.*'
                type: rewrite_tag_filter
                rule:
                  - name: ident
                    regexp: '^(.*)'
                    result: '__TAG__.$1'
              syslog_log:
                tag: 'syslog.*.*'
                type: file
                path: /tmp/syslog
        input:
          syslog:
            syslog_log:
              type: tail
              label: syslog
              path: /var/log/syslog
              tag: syslog.syslog
              parser:
                type: regexp
                format: >-
                  '/^\<(?<pri>[0-9]+)\>(?<time>[^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
                time_format: '%FT%T.%L%:z'
            auth_log:
              type: tail
              label: syslog
              path: /var/log/auth.log
              tag: syslog.auth
              parser:
                type: regexp
                format: >-
                  '/^\<(?<pri>[0-9]+)\>(?<time>[^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
                time_format: '%FT%T.%L%:z'
          prometheus:
            prometheus:
              type: prometheus
            prometheus_monitor:
              type: prometheus_monitor
            prometheus_output_monitor:
              type: prometheus_output_monitor
          forward:
            forward_listen:
              type: forward
              port: 24224
              bind: 0.0.0.0
        match:
          docker_monitoring:
            tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
            type: relabel
            label: monitoring