Merge branch 'develop' into 'master'
Develop
See merge request !1
diff --git a/README.rst b/README.rst
index 2e6b30d..90730cd 100644
--- a/README.rst
+++ b/README.rst
@@ -109,7 +109,7 @@
es-json:
engine: es-json
message_matcher: TRUE
- index = logfile-%{%Y.%m.%d}
+ index: logfile-%{%Y.%m.%d}
Read more
=========
diff --git a/VERSION b/VERSION
index 8a9ecc2..3b04cfb 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.0.1
\ No newline at end of file
+0.2
diff --git a/debian/changelog b/debian/changelog
index 3ed5139..f828603 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,8 @@
-salt-formula-heka (0.1) trusty; urgency=medium
+<<<<<<< HEAD
+=======
+salt-formula-heka (0.2) trusty; urgency=medium
- * Initial release
+ * First public release
- -- Jan Kaufman <jan.kaufman@tcpcloud.eu> Thu, 13 Aug 2015 23:23:41 +0200
+ -- Filip Pytloun <filip.pytloun@tcpcloud.eu> Wed, 07 Oct 2015 14:02:32 +0200
+
diff --git a/heka/files/00-hekad.toml b/heka/files/00-hekad.toml
index 08743c4..baecf51 100644
--- a/heka/files/00-hekad.toml
+++ b/heka/files/00-hekad.toml
@@ -1,6 +1,4 @@
-{%- from "heka/map.jinja" import router with context -%}
-
[hekad]
-# replace me with grains.items.num_cpus
-maxprocs = 2
+{%- set workers = grains.num_cpus + 1 %}
+maxprocs = {{ workers }}
diff --git a/heka/files/decoder/protobuf.toml b/heka/files/decoder/protobuf.toml
new file mode 100644
index 0000000..c856239
--- /dev/null
+++ b/heka/files/decoder/protobuf.toml
@@ -0,0 +1,3 @@
+[ProtoBufDecoder]
+type = "ProtobufDecoder"
+
diff --git a/heka/files/decoder/rsyslog.toml b/heka/files/decoder/rsyslog.toml
new file mode 100644
index 0000000..bb4ec51
--- /dev/null
+++ b/heka/files/decoder/rsyslog.toml
@@ -0,0 +1,9 @@
+[RsyslogDecoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/rsyslog.lua"
+
+[RsyslogDecoder.config]
+type = "RSYSLOG_TraditionalFileFormat"
+template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'
+tz = "Europe/Prague"
+
diff --git a/heka/files/encoder/RstEncoder.toml b/heka/files/encoder/RstEncoder.toml
new file mode 100644
index 0000000..268f355
--- /dev/null
+++ b/heka/files/encoder/RstEncoder.toml
@@ -0,0 +1,3 @@
+[RstEncoder]
+type = "RstEncoder"
+
diff --git a/heka/files/encoder/es-json.toml b/heka/files/encoder/es-json.toml
new file mode 100644
index 0000000..68ceecf
--- /dev/null
+++ b/heka/files/encoder/es-json.toml
@@ -0,0 +1,10 @@
+[es_json]
+type = "ESJsonEncoder"
+index = "{{ values.index }}"
+
+{% if values.es_index_from_timestamp == True %}
+es_index_from_timestamp = true
+{%- else -%}
+es_index_from_timestamp = false
+{%- endif -%}
+
diff --git a/heka/files/encoder/es_json.toml b/heka/files/encoder/es_json.toml
deleted file mode 100644
index ca909bd..0000000
--- a/heka/files/encoder/es_json.toml
+++ /dev/null
@@ -1,5 +0,0 @@
-[<%= @title %>_encoder]
-type = "ESJsonEncoder"
-index = "<%= @index %>"
-es_index_from_timestamp = <%= @es_index_from_timestamp %>
-
diff --git a/heka/files/input/amqp.toml b/heka/files/input/amqp.toml
index 947fd45..d05a6f6 100644
--- a/heka/files/input/amqp.toml
+++ b/heka/files/input/amqp.toml
@@ -1,13 +1,23 @@
-[<%= @title %>_amqp]
+[input_{{ name }}]
type = "AMQPInput"
-url = "amqp://<%= @user %>:<%= @password %>@<%= @host %>:<%= @port %>/"
-exchange = "<%= @exchange %>"
-exchange_type = "<%= @exchange_type %>"
-exchange_durability = <%= @exchange_durability %>
-exchange_auto_delete = <%= @exchange_auto_delete %>
-queue_auto_delete = <%= @queue_auto_delete %>
-queue = "<%= @queue %>"
-routing_key = "<%= @routing_key %>"
-decoder = "<%= @decoder %>_decoder"
-splitter = "NullSplitter"
+url = "amqp://{{ values.user }}:{{ values.password }}@{{ values.host }}:{{ values.port }}/{{ values.vhost }}"
+exchange = "{{ values.exchange }}"
+exchange_type = "{{ values.exchange_type }}"
+{%- if values.exchange_durability is defined -%}
+exchange_durability = "{{ values.exchange_durability }}"
+{% endif %}
+{%- if values.exchange_auto_delete is defined -%}
+exchange_auto_delete = "{{ values.exchange_auto_delete }}"
+{% endif %}
+{%- if values.queue_auto_delete is defined -%}
+queue_auto_delete = {{ values.queue_auto_delete }}
+{% endif %}
+{%- if values.queue is defined -%}
+queue = "{{ values.queue }}"
+{% endif %}
+{%- if values.routing_key is defined -%}
+routing_key = "{{ values.routing_key }}"
+{% endif %}
+decoder = "{{ values.decoder }}"
+splitter = "{{ values.splitter }}"
diff --git a/heka/files/input/logstreamer-rsyslog.toml b/heka/files/input/logstreamer-rsyslog.toml
deleted file mode 100644
index 7dbf4ae..0000000
--- a/heka/files/input/logstreamer-rsyslog.toml
+++ /dev/null
@@ -1,38 +0,0 @@
-[LogstreamerInputSyslog]
-type = "LogstreamerInput"
-log_directory = "/var/log"
-file_match = 'syslog\.?(?P<Index>\d+)?(.gz)?'
-priority = ["^Index"]
-decoder = "RsyslogDecoder"
-
-[LogstreamerInputAuthlog]
-type = "LogstreamerInput"
-log_directory = "/var/log"
-file_match = 'auth\.log\.?(?P<Index>\d+)?(.gz)?'
-priority = ["^Index"]
-decoder = "RsyslogDecoder"
-
-[LogstreamerInputAuthKernlog]
-type = "LogstreamerInput"
-log_directory = "/var/log"
-file_match = 'kern\.log\.?(?P<Index>\d+)?(.gz)?'
-priority = ["^Index"]
-decoder = "RsyslogDecoder"
-
-[LogstreamerInputAuthMaillog]
-type = "LogstreamerInput"
-log_directory = "/var/log"
-file_match = 'mail\.log?(?P<Index>\d+)?(.gz)?|mail\.err?(?P<Index>\d+)?(.gz)?'
-priority = ["^Index"]
-decoder = "RsyslogDecoder"
-
-[RsyslogDecoder]
-type = "SandboxDecoder"
-filename = "lua_decoders/rsyslog.lua"
-
-[RsyslogDecoder.config]
-type = "RSYSLOG_TraditionalFileFormat"
-template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'
-tz = "Europe/Prague"
-# hostname_keep = 'TRUE'
-
diff --git a/heka/files/input/logstreamer.toml b/heka/files/input/logstreamer.toml
index 109477d..527a19b 100644
--- a/heka/files/input/logstreamer.toml
+++ b/heka/files/input/logstreamer.toml
@@ -1,10 +1,14 @@
-[<%= @title %>_logstreamer]
+[logstreamer_{{ name }}]
type = "LogstreamerInput"
-log_directory = "<%= @log_directory %>"
-file_match = '<%= @file_match %>'
-<% if @differentiator -%>
-differentiator = <%= @differentiator %>
-<% end -%>
-decoder = "<%= @decoder %>_decoder"
-splitter = "<%= @splitter ? "#{ @splitter }_splitter" : "TokenSplitter" %>"
+log_directory = "{{ values.log_directory }}"
+file_match = '{{ values.file_match }}'
+{% if values.priority is defined %}
+priority = {{ values.priority }}
+{% endif %}
+{% if values.decoder is defined %}
+decoder = "{{ values.decoder }}"
+{% endif %}
+{% if values.splitter is defined %}
+splitter = {{ values.splitter }}
+{% endif %}
diff --git a/heka/files/output/95-output-amqp.toml b/heka/files/output/95-output-amqp.toml
deleted file mode 100644
index 8812ae2..0000000
--- a/heka/files/output/95-output-amqp.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[AMQPOutput]
-type = "AMQPOutput"
-url = "amqp://guest:guest@10.0.102.11/"
-exchange = "logs"
-exchange_type = "fanout"
-message_matcher = "TRUE"
-use_framing = true
-encoder = "ProtobufEncoder"
-
-[AMQPOutput.retries]
-max_delay = "30s"
-delay = "250ms"
-max_retries = 5
-
diff --git a/heka/files/output/amqp.toml b/heka/files/output/amqp.toml
new file mode 100644
index 0000000..58eb679
--- /dev/null
+++ b/heka/files/output/amqp.toml
@@ -0,0 +1,14 @@
+[AMQPOutput_{{ name }}]
+type = "AMQPOutput"
+url = "amqp://{{ values.user }}:{{ values.password }}@{{ values.host }}/{{ values.vhost }}"
+exchange = "{{ values.exchange }}"
+exchange_type = "{{ values.exchange_type }}"
+message_matcher = "{{ values.message_matcher }}"
+use_framing = true
+encoder = "{{ values.encoder }}"
+
+[AMQPOutput_{{ name }}.retries]
+max_delay = "30s"
+delay = "250ms"
+max_retries = 5
+
diff --git a/heka/files/output/dashboard.toml b/heka/files/output/dashboard.toml
index 154a6f1..30121f4 100644
--- a/heka/files/output/dashboard.toml
+++ b/heka/files/output/dashboard.toml
@@ -1,4 +1,6 @@
[DashboardOutput]
-address = "<%= @dashboard_address %>:<%= @dashboard_port %>"
-ticker_interval = <%= @ticker_interval %>
+{%- if values.dashboard_address is defined %}
+address = "{{ values.dashboard_address }}:{{ values.dashboard_port }}"
+{% endif %}
+ticker_interval = {{ values.ticker_interval }}
diff --git a/heka/files/output/elasticsearch.toml b/heka/files/output/elasticsearch.toml
index 2339da3..410e65b 100644
--- a/heka/files/output/elasticsearch.toml
+++ b/heka/files/output/elasticsearch.toml
@@ -1,22 +1,9 @@
-[<%= @title %>_output]
+[output_{{ name }}]
type = "ElasticSearchOutput"
-message_matcher = "<%= @message_matcher %>"
-encoder = "<%= @encoder %>_encoder"
+message_matcher = "{{ values.message_matcher }}"
+encoder = "{{ values.encoder }}"
+server = "http://{{ values.host }}:{{ values.port }}"
-<% # Heka expects milliseconds here -%>
-flush_interval = <%= @flush_interval.to_i() * 1000 %>
-flush_count = <%= @flush_count %>
-
-server = "http://<%= @server %>:<%= @port %>"
-
-use_buffering = <%= @use_buffering %>
-
-<% if @use_buffering -%>
-[<%= @title %>_output.buffering]
-max_buffer_size = <%= @queue_max_buffer_size %>
-<% if @max_file_size != :undef -%>
-max_file_size = <%= @max_file_size %>
-<% end -%>
-full_action = "<%= @queue_full_action %>"
-<% end -%>
+[output_{{ name }}.buffering]
+max_file_size = 268435456
diff --git a/heka/files/output/logoutput.toml b/heka/files/output/logoutput.toml
new file mode 100644
index 0000000..38bb08b
--- /dev/null
+++ b/heka/files/output/logoutput.toml
@@ -0,0 +1,4 @@
+[LogOutput]
+encoder = "{{ values.encoder }}"
+message_matcher = "{{ values.message_matcher }}"
+
diff --git a/heka/map.jinja b/heka/map.jinja
index 335cfd5..9b8eb7e 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -7,10 +7,18 @@
groups:
- adm
- syslog
+ service:
+ - heka
+ filter: {}
+ splitter: {}
+ encoder: {}
+ decoder: {}
RedHat:
groups:
- adm
+ service:
+ - heka
{%- endload %}
-{%- set server = salt['grains.filter_by'](server__defaults, merge=salt['pillar.get']('heka:server')) %}
+{%- set server = salt['grains.filter_by'](server_defaults, merge=salt['pillar.get']('heka:server')) %}
diff --git a/heka/server.sls b/heka/server.sls
index 84b9ab7..687e13b 100644
--- a/heka/server.sls
+++ b/heka/server.sls
@@ -3,42 +3,148 @@
heka_packages:
pkg.installed:
- - names: {{ heka.pkgs }}
+ - names: {{ server.pkgs }}
+
+purge-heka-conf-dir:
+ file.directory:
+ - name: /etc/heka/conf.d/
+ - clean: True
/etc/heka/conf.d/00-hekad.toml:
file.managed:
- source: salt://heka/files/00-hekad.toml
- template: jinja
- - mode: 755
+ - mode: 640
+ - group: heka
- require:
- pkg: heka_packages
- - watch_in:
- - service: heka_service
+ - file: purge-heka-conf-dir
heka_service:
service.running:
- enable: true
- - name: {{ heka.service }}
+ - name: heka
+ - watch:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - require:
+ - user: heka_user
heka_user:
user.present:
- name: heka
- shell: /bin/false
- - groups: {{ heka.groups }}
+ - groups: {{ server.groups }}
- require:
- pkg: heka_packages
-{%- for name,engine in server.input.iteritems() %}
+{%- for name,values in server.input.iteritems() %}
-/etc/heka/conf.d/10-input-{{ name }}.toml:
+/etc/heka/conf.d/15-input-{{ name }}-{{ values['engine'] }}.toml:
file.managed:
- - source: salt://heka/files/input/{{ engine }}.toml
+ - source: salt://heka/files/input/{{ values['engine'] }}.toml
- template: jinja
- - mode: 755
+ - mode: 640
+ - group: heka
- require:
- - file: /etc/heka/conf.d/00-hekad.toml
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
-{%- endfor %}
+{%- endfor %}
+
+{%- for name,values in server.output.iteritems() %}
+
+/etc/heka/conf.d/60-output-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/output/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+
+{%- for name,values in server.filter.iteritems() %}
+
+/etc/heka/conf.d/20-filter-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/filter/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.splitter.iteritems() %}
+
+/etc/heka/conf.d/30-splitter-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/splitter/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.encoder.iteritems() %}
+
+/etc/heka/conf.d/40-encoder-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/encoder/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.decoder.iteritems() %}
+
+/etc/heka/conf.d/10-decoder-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/decoder/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
{%- endif %}
diff --git a/metadata/service/server/amqp/compute.yml b/metadata/service/server/amqp/compute.yml
new file mode 100644
index 0000000..0d47ad7
--- /dev/null
+++ b/metadata/service/server/amqp/compute.yml
@@ -0,0 +1,97 @@
+applications:
+- heka
+_param:
+parameters:
+ heka:
+ server:
+ enabled: true
+ input:
+ rsyslog_syslog:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: syslog\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_auth:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_kern:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: kern\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ nova_compute:
+ engine: logstreamer
+ log_directory: /var/log/nova
+ file_match: nova-compute\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ["^Index"]
+ decoder: openstack
+ cinder_volume:
+ engine: logstreamer
+ log_directory: /var/log/cinder
+ file_match: cinder-volume\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ["^Index"]
+ decoder: openstack
+ ceilometer_agent:
+ engine: logstreamer
+ log_directory: /var/log/ceilometer
+ file_match: ceilometer-agent-compute\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ['^Index']
+ decoder: openstack
+ libvirtd:
+ engine: logstreamer
+ log_directory: /var/log/libvirt
+ file_match: libvirtd\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ['^Index']
+ decoder: libvirtd
+ decoder:
+ rsyslog:
+ engine: rsyslog
+ template: \%TIMESTAMP\% \%HOSTNAME\% \%syslogtag\%\%msg:::sp-if-no-1st-sp\%\%msg:::drop-last-lf\%\n
+ hostname_keep: TRUE
+ tz: Europe/Prague
+ openstack:
+ engine: multidecoder
+ subs: [ 'OpenStackLog','OpenStackLogTrace' ]
+ cascade_strategy: first-wins
+ log_sub_errors: false
+ OpenStackLog:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3})\s(?P<pid>\d+)\s(?P<severity>\w+)\s(?P<programname>\D+)\s\[(?P<id>.+)\]\s(?P<payload>.*)
+ timestamp_layout: 2015-10-06 11:34:37.243
+ OpenStackLogTrace:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3})\s(?P<pid>\d+)\s(?P<severity>TRACE)\s(?P<programname>.\S+)\s(?P<payload>.*)
+ timestamp_layout: 2015-10-06 11:34:37.243
+ libvirtd:
+ engine: multidecoder
+ subs: [ 'libvirtcommon', 'libvirtaudit' ]
+ cascade_strategy: firts-wins
+ log_sub_errors: false
+ libvirtcommon:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3}\+\d{4}):\s+(?P<pid>\d+):\s+(?P<severity>\w+)\s+:\s+(?P<qemutrace>\w+:\d+)\s+:\s+(?P<payload>.*)
+ timestamp_layout: 2015-09-01 15:56:14.675+0000
+ libvirtaudit:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3}\+\d{4}):\s+(?P<pid>\d+):\s+(?P<severity>\w+)\s+:\s+(?P<message>.+):\s(?P<payload>.+)
+ timestamp_layout: 2015-09-01 15:56:14.675+0000
+ output:
+ rabbitmq:
+ engine: amqp
+ host: ${_param:heka_shipper_output_host}
+ user: ${_param:heka_shipper_output_user}
+ password: ${_param:heka_shipper_output_password}
+ vhost: ${_param:heka_shipper_output_vhost}
+ exchange: ${_param:heka_shipper_output_exchange}
+ exchange_type: fanout
+ encoder: ProtobufEncoder
+ use_framing: true
+ message_matcher: "TRUE"
+ filter: {}
+ splitter: {}
+
diff --git a/metadata/service/server/router.yml b/metadata/service/server/amqp/router.yml
similarity index 80%
rename from metadata/service/server/router.yml
rename to metadata/service/server/amqp/router.yml
index 9549a6f..3cc679f 100644
--- a/metadata/service/server/router.yml
+++ b/metadata/service/server/amqp/router.yml
@@ -1,13 +1,6 @@
applications:
- heka
_param:
- heka_router_output_host: localhost
- heka_router_input_host: localhost
- heka_router_input_user: guest
- heka_router_input_password: guest
- heka_router_input_vhost: /heka
- heka_router_input_exchange: logs
-
parameters:
heka:
server:
@@ -19,17 +12,18 @@
user: ${_param:heka_router_input_user}
password: ${_param:heka_router_input_password}
vhost: ${_param:heka_router_input_vhost}
+ port: 5672
exchange: ${_param:heka_router_input_exchange}
exchange_type: fanout
decoder: ProtoBufDecoder
splitter: HekaFramingSplitter
- rsyslog-syslog:
+ rsyslog_syslog:
engine: logstreamer
log_directory: /var/log
file_match: syslog\.?(?P<Index>\d+)?(.gz)?
decoder: RsyslogDecoder
priority: ["^Index"]
- rsyslog-auth:
+ rsyslog_auth:
engine: logstreamer
log_directory: /var/log
file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
@@ -41,19 +35,21 @@
template: \%TIMESTAMP\% \%HOSTNAME\% \%syslogtag\%\%msg:::sp-if-no-1st-sp\%\%msg:::drop-last-lf\%\n
hostname_keep: TRUE
tz: Europe/Prague
+ ProtoBufDecoder:
+ engine: protobuf
output:
elasticsearch01:
engine: elasticsearch
host: ${_param:heka_router_output_host}
port: 9200
encoder: es_json
- message_matcher: TRUE
+ message_matcher: "Logger != 'hekad'"
dashboard01:
engine: dashboard
ticker_interval: 30
encoder:
- es-json:
+ es_json:
engine: es-json
- message_matcher: TRUE
index: logfile-%{%Y.%m.%d}
+ es_index_from_timestamp: true
diff --git a/metadata/service/server/shipper.yml b/metadata/service/server/amqp/shipper.yml
similarity index 71%
rename from metadata/service/server/shipper.yml
rename to metadata/service/server/amqp/shipper.yml
index 75e4496..24c3f18 100644
--- a/metadata/service/server/shipper.yml
+++ b/metadata/service/server/amqp/shipper.yml
@@ -1,29 +1,29 @@
applications:
- heka
_param:
- heka_shipper_output_host: localhost
- heka_shipper_input_host: localhost
- heka_shipper_input_user: guest
- heka_shipper_input_password: guest
- heka_shipper_input_vhost: /logs
- heka_shipper_input_exchange: logs
parameters:
heka:
server:
enabled: true
input:
- rsyslog-syslog:
+ rsyslog_syslog:
engine: logstreamer
log_directory: /var/log
file_match: syslog\.?(?P<Index>\d+)?(.gz)?
decoder: RsyslogDecoder
priority: ["^Index"]
- rsyslog-auth:
+ rsyslog_auth:
engine: logstreamer
log_directory: /var/log
file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
decoder: RsyslogDecoder
priority: ["^Index"]
+ rsyslog_kern:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: kern\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
decoder:
rsyslog:
engine: rsyslog
@@ -41,11 +41,7 @@
exchange_type: fanout
encoder: ProtobufEncoder
use_framing: true
- heka-logfile:
- engine: logoutput
- encoder: RstEncoder
- message_matcher: TRUE
- encoder:
- heka-logfile:
- engine: RstEncoder
+ message_matcher: "TRUE"
+ filter: {}
+ splitter: {}