Initial release
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 0000000..6c9d57c
--- /dev/null
+++ b/CHANGELOG.rst
@@ -0,0 +1,6 @@
+heka-formula
+==============
+
+0.0.1 (2015-08-03)
+
+- Initial formula setup
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..6f2b42f
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,13 @@
+Copyright (c) 2014-2015 tcp cloud a.s.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
\ No newline at end of file
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..2e6b30d
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,117 @@
+
+============
+Heka Formula
+============
+
+Heka is an open source stream processing software system developed by Mozilla. Heka is a Swiss Army Knife type tool for data processing
+
+Sample pillars
+==============
+
+Basic log shipper streaming decoded rsyslog's logfiles using amqp broker as transport.
+From every message there is one amqp message and it's also logged to heka's logfile in RST format.
+
+.. code-block:: yaml
+
+
+ heka:
+ server:
+ enabled: true
+ input:
+ rsyslog-syslog:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: syslog\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog-auth:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ decoder:
+ rsyslog:
+ engine: rsyslog
+ template: %TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n
+ hostname_keep: TRUE
+ tz: Europe/Prague
+ output:
+ rabbitmq:
+ engine: amqp
+ host: localhost
+ user: guest
+ password: guest
+ vhost: /logs
+ exchange: logs
+ exchange_type: fanout
+ encoder: ProtobufEncoder
+ use_framing: true
+ heka-logfile:
+ engine: logoutput
+ encoder: RstEncoder
+ message_matcher: TRUE
+ encoder:
+ heka-logfile:
+ engine: RstEncoder
+
+
+Heka acting as message router and dashboard.
+Messages are consumed from amqp and sent to elasticsearch server.
+
+
+.. code-block:: yaml
+
+
+ heka:
+ server:
+ enabled: true
+ input:
+ rabbitmq:
+ engine: amqp
+ host: localhost
+ user: guest
+ password: guest
+ vhost: /logs
+ exchange: logs
+ exchange_type: fanout
+ decoder: ProtoBufDecoder
+ splitter: HekaFramingSplitter
+ rsyslog-syslog:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: syslog\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog-auth:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ decoder:
+ rsyslog:
+ engine: rsyslog
+ template: %TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n
+ hostname_keep: TRUE
+ tz: Europe/Prague
+ output:
+ elasticsearch01:
+ engine: elasticsearch
+ host: localhost
+ port: 9200
+ encoder: es_json
+ message_matcher: TRUE
+ dashboard01:
+ engine: dashboard
+ ticker_interval: 30
+ encoder:
+ es-json:
+ engine: es-json
+ message_matcher: TRUE
+ index = logfile-%{%Y.%m.%d}
+
+Read more
+=========
+
+* https://hekad.readthedocs.org/en/latest/index.html
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..3b04cfb
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+0.2
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 0000000..f0b0069
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,11 @@
+salt-formula-heka (0.2) trusty; urgency=medium
+
+ * First public release
+
+ -- Filip Pytloun <filip.pytloun@tcpcloud.eu> Wed, 07 Oct 2015 14:02:32 +0200
+
+salt-formula-heka (0.1) trusty; urgency=medium
+
+ * Initial release
+
+ -- Jan Kaufman <jan.kaufman@tcpcloud.eu> Thu, 13 Aug 2015 23:23:41 +0200
diff --git a/debian/compat b/debian/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/debian/compat
@@ -0,0 +1 @@
+9
diff --git a/debian/control b/debian/control
new file mode 100644
index 0000000..ed8d832
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,15 @@
+Source: salt-formula-heka
+Maintainer: Jan Kaufman <jan.kaufman@tcpcloud.eu>
+Section: admin
+Priority: optional
+Build-Depends: debhelper (>= 9)
+Standards-Version: 3.9.6
+Homepage: http://www.tcpcloud.eu
+Vcs-Browser: https://github.com/tcpcloud/salt-formula-heka
+Vcs-Git: https://github.com/tcpcloud/salt-formula-heka.git
+
+Package: salt-formula-heka
+Architecture: all
+Depends: ${misc:Depends}, salt-master, reclass
+Description: Heka Salt formula
+ Install and configure Heka server and client.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 0000000..971f937
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,15 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: salt-formula-heka
+Upstream-Contact: Jan Kaufman <jan.kaufman@tcpcloud.eu>
+Source: https://github.com/tcpcloud/salt-formula-heka
+
+Files: *
+Copyright: 2014-2015 tcp cloud
+License: Apache-2.0
+ Copyright (C) 2014-2015 tcp cloud
+ .
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ .
+ On a Debian system you can find a copy of this license in
+ /usr/share/common-licenses/Apache-2.0.
diff --git a/debian/docs b/debian/docs
new file mode 100644
index 0000000..d585829
--- /dev/null
+++ b/debian/docs
@@ -0,0 +1,3 @@
+README.rst
+CHANGELOG.rst
+VERSION
diff --git a/debian/install b/debian/install
new file mode 100644
index 0000000..12adfe7
--- /dev/null
+++ b/debian/install
@@ -0,0 +1,2 @@
+heka/* /usr/share/salt-formulas/env/heka/
+metadata/service/* /usr/share/salt-formulas/reclass/service/heka/
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 0000000..abde6ef
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,5 @@
+#!/usr/bin/make -f
+
+%:
+ dh $@
+
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 0000000..89ae9db
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (native)
diff --git a/heka/files/00-hekad.toml b/heka/files/00-hekad.toml
new file mode 100644
index 0000000..baecf51
--- /dev/null
+++ b/heka/files/00-hekad.toml
@@ -0,0 +1,4 @@
+[hekad]
+{%- set workers = grains.num_cpus + 1 %}
+maxprocs = {{ workers }}
+
diff --git a/heka/files/80-output-elasticsearch.toml b/heka/files/80-output-elasticsearch.toml
new file mode 100644
index 0000000..ed318f1
--- /dev/null
+++ b/heka/files/80-output-elasticsearch.toml
@@ -0,0 +1,23 @@
+[ESJsonEncoder]
+#index = "%{Type}-%{%Y.%m.%d}"
+index = "logfile-%{%Y.%m.%d}"
+# es_index_from_timestamp = true
+#type_name = "%{Type}"
+type_name = "%{Type}-%{Hostname}"
+ [ESJsonEncoder.field_mappings]
+ timestamp = "@timestamp"
+ Severity = "level"
+
+[ElasticSearchOutput]
+# message_matcher = "TRUE"
+# message_matcher = "Type != 'heka.*'"
+message_matcher = "Type != 'heka.memstat' && Type != 'heka.all-report'"
+server = "http://10.0.151.204:9200"
+flush_interval = 2500
+# flush_interval = 5000
+flush_count = 10
+encoder = "ESJsonEncoder"
+
+[ElasticSearchOutput.buffering]
+max_file_size = 268435456
+
diff --git a/heka/files/85-output-slack.toml b/heka/files/85-output-slack.toml
new file mode 100644
index 0000000..fc06f14
--- /dev/null
+++ b/heka/files/85-output-slack.toml
@@ -0,0 +1,22 @@
+#[LogAggregator]
+#type = "SandboxFilter"
+#message_matcher = "TRUE"
+#filename = "lua_decoders/log_aggregator.lua"
+#ticker_interval = 15
+#can_exit = false
+
+#[SlackEncoder]
+#type = "SandboxEncoder"
+#script_type = "lua"
+#filename = "lua_encoders/slack_encoder.lua"
+
+# [SlackEncoder.config]
+# username = "heka"
+# channel = "monitoring"
+# icon_emoji = ""
+
+#[HttpOutput]
+# address = "https://hooks.slack.com/services/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"
+# message_matcher = "TRUE"
+# encoder = "SlackEncoder"
+
diff --git a/heka/files/98-output-heka-dashboard.toml b/heka/files/98-output-heka-dashboard.toml
new file mode 100644
index 0000000..5698238
--- /dev/null
+++ b/heka/files/98-output-heka-dashboard.toml
@@ -0,0 +1,24 @@
+[DashboardOutput]
+ticker_interval = 30
+
+#[stat_graph]
+#type = "SandboxFilter"
+#filename = "lua_filters/stat_graph.lua"
+#ticker_interval = 1
+#preserve_data = true
+#message_matcher = "Type == 'heka.statmetric'"
+
+# [stat_graph.config]
+# num_rows = 300
+# secs_per_row = 1
+# stats = "stats.counters.000000.count stats.counters.000001.count stats.counters.000002.count"
+# stat_labels = "counter_0 counter_1 counter_2"
+# preservation_version = 0
+
+#[HekaMemstat]
+#type = "SandboxFilter"
+#filename = "lua_filters/heka_memstat.lua"
+#ticker_interval = 60
+#preserve_data = true
+#message_matcher = "Type == 'heka.memstat'"
+
diff --git a/heka/files/99-output-heka-logfile.toml b/heka/files/99-output-heka-logfile.toml
new file mode 100644
index 0000000..e103731
--- /dev/null
+++ b/heka/files/99-output-heka-logfile.toml
@@ -0,0 +1,5 @@
+[RstEncoder]
+[LogOutput]
+encoder = "RstEncoder"
+message_matcher = "TRUE"
+
diff --git a/heka/files/decoder/protobuf.toml b/heka/files/decoder/protobuf.toml
new file mode 100644
index 0000000..c856239
--- /dev/null
+++ b/heka/files/decoder/protobuf.toml
@@ -0,0 +1,3 @@
+[ProtoBufDecoder]
+type = "ProtobufDecoder"
+
diff --git a/heka/files/decoder/rsyslog.toml b/heka/files/decoder/rsyslog.toml
new file mode 100644
index 0000000..bb4ec51
--- /dev/null
+++ b/heka/files/decoder/rsyslog.toml
@@ -0,0 +1,9 @@
+[RsyslogDecoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/rsyslog.lua"
+
+[RsyslogDecoder.config]
+type = "RSYSLOG_TraditionalFileFormat"
+template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'
+tz = "Europe/Prague"
+
diff --git a/heka/files/encoder/RstEncoder.toml b/heka/files/encoder/RstEncoder.toml
new file mode 100644
index 0000000..268f355
--- /dev/null
+++ b/heka/files/encoder/RstEncoder.toml
@@ -0,0 +1,3 @@
+[RstEncoder]
+type = "RstEncoder"
+
diff --git a/heka/files/encoder/es-json.toml b/heka/files/encoder/es-json.toml
new file mode 100644
index 0000000..68ceecf
--- /dev/null
+++ b/heka/files/encoder/es-json.toml
@@ -0,0 +1,10 @@
+[es_json]
+type = "ESJsonEncoder"
+index = "{{ values.index }}"
+
+{% if values.es_index_from_timestamp == True %}
+es_index_from_timestamp = true
+{%- else -%}
+es_index_from_timestamp = false
+{%- endif -%}
+
diff --git a/heka/files/input/amqp.toml b/heka/files/input/amqp.toml
new file mode 100644
index 0000000..d05a6f6
--- /dev/null
+++ b/heka/files/input/amqp.toml
@@ -0,0 +1,23 @@
+[input_{{ name }}]
+type = "AMQPInput"
+url = "amqp://{{ values.user }}:{{ values.password }}@{{ values.host }}:{{ values.port }}/{{ values.vhost }}"
+exchange = "{{ values.exchange }}"
+exchange_type = "{{ values.exchange_type }}"
+{%- if values.exchange_durability is defined -%}
+exchange_durability = "{{ values.exchange_durability }}"
+{% endif %}
+{%- if values.exchange_auto_delete is defined -%}
+exchange_auto_delete = "{{ values.exchange_auto_delete }}"
+{% endif %}
+{%- if values.queue_auto_delete is defined -%}
+queue_auto_delete = {{ values.queue_auto_delete }}
+{% endif %}
+{%- if values.queue is defined -%}
+queue = "{{ values.queue }}"
+{% endif %}
+{%- if values.routing_key is defined -%}
+routing_key = "{{ values.routing_key }}"
+{% endif %}
+decoder = "{{ values.decoder }}"
+splitter = "{{ values.splitter }}"
+
diff --git a/heka/files/input/logstreamer.toml b/heka/files/input/logstreamer.toml
new file mode 100644
index 0000000..527a19b
--- /dev/null
+++ b/heka/files/input/logstreamer.toml
@@ -0,0 +1,14 @@
+[logstreamer_{{ name }}]
+type = "LogstreamerInput"
+log_directory = "{{ values.log_directory }}"
+file_match = '{{ values.file_match }}'
+{% if values.priority is defined %}
+priority = {{ values.priority }}
+{% endif %}
+{% if values.decoder is defined %}
+decoder = "{{ values.decoder }}"
+{% endif %}
+{% if values.splitter is defined %}
+splitter = {{ values.splitter }}
+{% endif %}
+
diff --git a/heka/files/input/process.toml b/heka/files/input/process.toml
new file mode 100644
index 0000000..a7b4379
--- /dev/null
+++ b/heka/files/input/process.toml
@@ -0,0 +1,29 @@
+[<%= @title %>_process]
+type="ProcessInput"
+ticker_interval = <%= @ticker_interval %>
+decoder = "<%= @decoder %>_decoder"
+<% if @stdout -%>
+stdout = true
+<% else %>
+stdout = false
+<% end %>
+<% if @stderr -%>
+stderr = true
+<% else %>
+stderr = false
+<% end %>
+
+<% if @splitter -%>
+splitter = "<%= @splitter %>_splitter"
+<% else -%>
+splitter = "NullSplitter"
+<% end -%>
+
+<% @commands.each_with_index do |command, i| -%>
+ <% command.each do |cmd, args| -%>
+ [<%= @title %>_process.command.<%= i %>]
+ bin = "<%= cmd %>"
+ args = [ <%= args.collect{ |x| '"%s"' % x }.join(", ") %> ]
+ <%end%>
+<% end %>
+
diff --git a/heka/files/output/amqp.toml b/heka/files/output/amqp.toml
new file mode 100644
index 0000000..58eb679
--- /dev/null
+++ b/heka/files/output/amqp.toml
@@ -0,0 +1,14 @@
+[AMQPOutput_{{ name }}]
+type = "AMQPOutput"
+url = "amqp://{{ values.user }}:{{ values.password }}@{{ values.host }}/{{ values.vhost }}"
+exchange = "{{ values.exchange }}"
+exchange_type = "{{ values.exchange_type }}"
+message_matcher = "{{ values.message_matcher }}"
+use_framing = true
+encoder = "{{ values.encoder }}"
+
+[AMQPOutput_{{ name }}.retries]
+max_delay = "30s"
+delay = "250ms"
+max_retries = 5
+
diff --git a/heka/files/output/dashboard.toml b/heka/files/output/dashboard.toml
new file mode 100644
index 0000000..30121f4
--- /dev/null
+++ b/heka/files/output/dashboard.toml
@@ -0,0 +1,6 @@
+[DashboardOutput]
+{%- if values.dashboard_address is defined %}
+address = "{{ values.dashboard_address }}:{{ values.dashboard_port }}"
+{% endif %}
+ticker_interval = {{ values.ticker_interval }}
+
diff --git a/heka/files/output/elasticsearch.toml b/heka/files/output/elasticsearch.toml
new file mode 100644
index 0000000..410e65b
--- /dev/null
+++ b/heka/files/output/elasticsearch.toml
@@ -0,0 +1,9 @@
+[output_{{ name }}]
+type = "ElasticSearchOutput"
+message_matcher = "{{ values.message_matcher }}"
+encoder = "{{ values.encoder }}"
+server = "http://{{ values.host }}:{{ values.port }}"
+
+[output_{{ name }}.buffering]
+max_file_size = 268435456
+
diff --git a/heka/files/output/logoutput.toml b/heka/files/output/logoutput.toml
new file mode 100644
index 0000000..38bb08b
--- /dev/null
+++ b/heka/files/output/logoutput.toml
@@ -0,0 +1,4 @@
+[LogOutput]
+encoder = "{{ values.encoder }}"
+message_matcher = "{{ values.message_matcher }}"
+
diff --git a/heka/init.sls b/heka/init.sls
new file mode 100644
index 0000000..71d650a
--- /dev/null
+++ b/heka/init.sls
@@ -0,0 +1,5 @@
+include:
+{% if pillar.heka.server is defined %}
+- heka.server
+{% endif %}
+
diff --git a/heka/map.jinja b/heka/map.jinja
new file mode 100644
index 0000000..9b8eb7e
--- /dev/null
+++ b/heka/map.jinja
@@ -0,0 +1,24 @@
+{%- load_yaml as server_defaults %}
+Debian:
+ pkgs:
+ - heka
+ user:
+ - heka
+ groups:
+ - adm
+ - syslog
+ service:
+ - heka
+ filter: {}
+ splitter: {}
+ encoder: {}
+ decoder: {}
+RedHat:
+ groups:
+ - adm
+ service:
+ - heka
+{%- endload %}
+
+{%- set server = salt['grains.filter_by'](server_defaults, merge=salt['pillar.get']('heka:server')) %}
+
diff --git a/heka/server.sls b/heka/server.sls
new file mode 100644
index 0000000..8415e47
--- /dev/null
+++ b/heka/server.sls
@@ -0,0 +1,148 @@
+{%- from "heka/map.jinja" import server with context %}
+{%- if server.enabled %}
+
+heka_packages:
+ pkg.installed:
+ - names: {{ server.pkgs }}
+
+purge-heka-conf-dir:
+ file.directory:
+ - name: /etc/heka/conf.d/
+ - clean: True
+
+/etc/heka/conf.d/00-hekad.toml:
+ file.managed:
+ - source: salt://heka/files/00-hekad.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - pkg: heka_packages
+ - file: purge-heka-conf-dir
+
+heka_service:
+ service.running:
+ - enable: true
+ - name: heka
+ - watch:
+ - file: /etc/heka/conf.d/00-hekad.toml
+
+heka_user:
+ user.present:
+ - name: heka
+ - shell: /bin/false
+ - groups: {{ server.groups }}
+ - require:
+ - pkg: heka_packages
+
+{%- for name,values in server.input.iteritems() %}
+
+/etc/heka/conf.d/15-input-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/input/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.output.iteritems() %}
+
+/etc/heka/conf.d/60-output-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/output/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+
+{%- for name,values in server.filter.iteritems() %}
+
+/etc/heka/conf.d/20-filter-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/filter/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.splitter.iteritems() %}
+
+/etc/heka/conf.d/30-splitter-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/splitter/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.encoder.iteritems() %}
+
+/etc/heka/conf.d/40-encoder-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/encoder/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- for name,values in server.decoder.iteritems() %}
+
+/etc/heka/conf.d/10-decoder-{{ name }}-{{ values['engine'] }}.toml:
+ file.managed:
+ - source: salt://heka/files/decoder/{{ values['engine'] }}.toml
+ - template: jinja
+ - mode: 640
+ - group: heka
+ - require:
+ - file: /etc/heka/conf.d/00-hekad.toml
+ - watch_in:
+ - service: heka_service
+ - defaults:
+ name: {{ name }}
+ values: {{ values }}
+
+{%- endfor %}
+
+{%- endif %}
+
diff --git a/metadata/service/server/amqp/compute.yml b/metadata/service/server/amqp/compute.yml
new file mode 100644
index 0000000..0d47ad7
--- /dev/null
+++ b/metadata/service/server/amqp/compute.yml
@@ -0,0 +1,97 @@
+applications:
+- heka
+_param:
+parameters:
+ heka:
+ server:
+ enabled: true
+ input:
+ rsyslog_syslog:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: syslog\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_auth:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_kern:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: kern\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ nova_compute:
+ engine: logstreamer
+ log_directory: /var/log/nova
+ file_match: nova-compute\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ["^Index"]
+ decoder: openstack
+ cinder_volume:
+ engine: logstreamer
+ log_directory: /var/log/cinder
+ file_match: cinder-volume\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ["^Index"]
+ decoder: openstack
+ ceilometer_agent:
+ engine: logstreamer
+ log_directory: /var/log/ceilometer
+ file_match: ceilometer-agent-compute\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ['^Index']
+ decoder: openstack
+ libvirtd:
+ engine: logstreamer
+ log_directory: /var/log/libvirt
+ file_match: libvirtd\.log\\.?(?P<Index>\\d+)?(.gz)?
+ priority: ['^Index']
+ decoder: libvirtd
+ decoder:
+ rsyslog:
+ engine: rsyslog
+ template: \%TIMESTAMP\% \%HOSTNAME\% \%syslogtag\%\%msg:::sp-if-no-1st-sp\%\%msg:::drop-last-lf\%\n
+ hostname_keep: TRUE
+ tz: Europe/Prague
+ openstack:
+ engine: multidecoder
+ subs: [ 'OpenStackLog','OpenStackLogTrace' ]
+ cascade_strategy: first-wins
+ log_sub_errors: false
+ OpenStackLog:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3})\s(?P<pid>\d+)\s(?P<severity>\w+)\s(?P<programname>\D+)\s\[(?P<id>.+)\]\s(?P<payload>.*)
+ timestamp_layout: 2015-10-06 11:34:37.243
+ OpenStackLogTrace:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3})\s(?P<pid>\d+)\s(?P<severity>TRACE)\s(?P<programname>.\S+)\s(?P<payload>.*)
+ timestamp_layout: 2015-10-06 11:34:37.243
+ libvirtd:
+ engine: multidecoder
+ subs: [ 'libvirtcommon', 'libvirtaudit' ]
+ cascade_strategy: firts-wins
+ log_sub_errors: false
+ libvirtcommon:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3}\+\d{4}):\s+(?P<pid>\d+):\s+(?P<severity>\w+)\s+:\s+(?P<qemutrace>\w+:\d+)\s+:\s+(?P<payload>.*)
+ timestamp_layout: 2015-09-01 15:56:14.675+0000
+ libvirtaudit:
+ engine: payloadregex
+ match_regex: ^(?P<timestamp>\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}.\d{3}\+\d{4}):\s+(?P<pid>\d+):\s+(?P<severity>\w+)\s+:\s+(?P<message>.+):\s(?P<payload>.+)
+ timestamp_layout: 2015-09-01 15:56:14.675+0000
+ output:
+ rabbitmq:
+ engine: amqp
+ host: ${_param:heka_shipper_output_host}
+ user: ${_param:heka_shipper_output_user}
+ password: ${_param:heka_shipper_output_password}
+ vhost: ${_param:heka_shipper_output_vhost}
+ exchange: ${_param:heka_shipper_output_exchange}
+ exchange_type: fanout
+ encoder: ProtobufEncoder
+ use_framing: true
+ message_matcher: "TRUE"
+ filter: {}
+ splitter: {}
+
diff --git a/metadata/service/server/amqp/router.yml b/metadata/service/server/amqp/router.yml
new file mode 100644
index 0000000..3cc679f
--- /dev/null
+++ b/metadata/service/server/amqp/router.yml
@@ -0,0 +1,55 @@
+applications:
+- heka
+_param:
+parameters:
+ heka:
+ server:
+ enabled: true
+ input:
+ rabbitmq:
+ engine: amqp
+ host: ${_param:heka_router_input_host}
+ user: ${_param:heka_router_input_user}
+ password: ${_param:heka_router_input_password}
+ vhost: ${_param:heka_router_input_vhost}
+ port: 5672
+ exchange: ${_param:heka_router_input_exchange}
+ exchange_type: fanout
+ decoder: ProtoBufDecoder
+ splitter: HekaFramingSplitter
+ rsyslog_syslog:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: syslog\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_auth:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ decoder:
+ rsyslog:
+ engine: rsyslog
+ template: \%TIMESTAMP\% \%HOSTNAME\% \%syslogtag\%\%msg:::sp-if-no-1st-sp\%\%msg:::drop-last-lf\%\n
+ hostname_keep: TRUE
+ tz: Europe/Prague
+ ProtoBufDecoder:
+ engine: protobuf
+ output:
+ elasticsearch01:
+ engine: elasticsearch
+ host: ${_param:heka_router_output_host}
+ port: 9200
+ encoder: es_json
+ message_matcher: "Logger != 'hekad'"
+ dashboard01:
+ engine: dashboard
+ ticker_interval: 30
+ encoder:
+ es_json:
+ engine: es-json
+ index: logfile-%{%Y.%m.%d}
+ es_index_from_timestamp: true
+
diff --git a/metadata/service/server/amqp/shipper.yml b/metadata/service/server/amqp/shipper.yml
new file mode 100644
index 0000000..24c3f18
--- /dev/null
+++ b/metadata/service/server/amqp/shipper.yml
@@ -0,0 +1,47 @@
+applications:
+- heka
+_param:
+parameters:
+ heka:
+ server:
+ enabled: true
+ input:
+ rsyslog_syslog:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: syslog\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_auth:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: auth\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ rsyslog_kern:
+ engine: logstreamer
+ log_directory: /var/log
+ file_match: kern\.log\.?(?P<Index>\d+)?(.gz)?
+ decoder: RsyslogDecoder
+ priority: ["^Index"]
+ decoder:
+ rsyslog:
+ engine: rsyslog
+ template: \%TIMESTAMP\% \%HOSTNAME\% \%syslogtag\%\%msg:::sp-if-no-1st-sp\%\%msg:::drop-last-lf\%\n
+ hostname_keep: TRUE
+ tz: Europe/Prague
+ output:
+ rabbitmq:
+ engine: amqp
+ host: ${_param:heka_shipper_output_host}
+ user: ${_param:heka_shipper_output_user}
+ password: ${_param:heka_shipper_output_password}
+ vhost: ${_param:heka_shipper_output_vhost}
+ exchange: ${_param:heka_shipper_output_exchange}
+ exchange_type: fanout
+ encoder: ProtobufEncoder
+ use_framing: true
+ message_matcher: "TRUE"
+ filter: {}
+ splitter: {}
+