Merge pull request #58 from ityaptin/stacklight
Add an os_telemetry_collector service
diff --git a/heka/_service.sls b/heka/_service.sls
index 61d6e83..d098575 100644
--- a/heka/_service.sls
+++ b/heka/_service.sls
@@ -29,28 +29,29 @@
- clean: true
{%- if grains.get('init', None) == 'systemd' %}
+{%- set systemd_enabled = True %}
+{%- else %}
+{%- set systemd_enabled = False %}
+{%- endif %}
heka_{{ service_name }}_service_file:
file.managed:
+{%- if systemd_enabled %}
- name: /etc/systemd/system/{{ service_name }}.service
+{%- else %}
+ - name: /etc/init/{{ service_name }}.conf
+{%- endif %}
- source: salt://heka/files/heka.service
- user: root
- mode: 644
- defaults:
service_name: {{ service_name }}
+ systemd_enabled: {{ systemd_enabled|lower }}
+ max_open_files: 102400
+ automatic_starting: {{ server.automatic_starting }}
- template: jinja
-{%- else %}
-
-heka_{{ service_name }}_service_file:
- file.managed:
- - name: /etc/init/{{ service_name }}.conf
- - source: salt://heka/files/heka.service
- - user: root
- - mode: 644
- - defaults:
- service_name: {{ service_name }}
- - template: jinja
+{%- if not systemd_enabled %}
heka_{{ service_name }}_service_wrapper:
file.managed:
@@ -59,17 +60,22 @@
- user: root
- mode: 755
- defaults:
- service_name: {{ service_name }}
+ service_name: {{ service_name }}
- template: jinja
{%- endif %}
heka_{{ service_name }}_service:
+{%- if server.automatic_starting %}
service.running:
- - name: {{ service_name }}
- enable: True
- watch:
- - file: /usr/share/lma_collector
+ - file: /usr/share/lma_collector/*
+ - file: /etc/{{ service_name }}/*
+{%- else %}
+ service.disabled:
+{%- endif %}
+ - name: {{ service_name }}
{# Setup basic structure for all roles so updates can apply #}
@@ -197,8 +203,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
{%- set service_metadata = service_grains.get(service_name) %}
@@ -214,8 +218,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
decoder_name: {{ decoder_name }}
decoder: {{ decoder|yaml }}
@@ -234,8 +236,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
input_name: {{ input_name }}
input: {{ input|yaml }}
@@ -254,8 +254,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
alarm_name: {{ alarm_name }}
alarm: {{ alarm|yaml }}
@@ -269,8 +267,6 @@
- group: heka
- require:
- file: /usr/share/lma_collector
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
alarm_name: {{ alarm_name }}
alarm: {{ alarm|yaml }}
@@ -288,8 +284,6 @@
- group: heka
- require:
- file: /usr/share/lma_collector
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
policy: {{ policy|yaml }}
{%- endif %}
@@ -306,8 +300,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
alarm_cluster_name: {{ alarm_cluster_name }}
alarm_cluster: {{ alarm_cluster|yaml }}
@@ -320,8 +312,6 @@
- group: heka
- require:
- file: /usr/share/lma_collector
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
alarm_cluster_name: {{ alarm_cluster_name }}
alarm_cluster: {{ alarm_cluster|yaml }}
@@ -340,8 +330,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
filter_name: {{ filter_name }}
filter: {{ filter|yaml }}
@@ -360,8 +348,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
splitter_name: {{ splitter_name }}
splitter: {{ splitter|yaml }}
@@ -380,8 +366,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
encoder_name: {{ encoder_name }}
encoder: {{ encoder|yaml }}
@@ -400,8 +384,6 @@
- file: heka_{{ service_name }}_conf_dir
- require_in:
- file: heka_{{ service_name }}_conf_dir_clean
- - watch_in:
- - service: heka_{{ service_name }}_service
- defaults:
output_name: {{ output_name }}
output: {{ output|yaml }}
diff --git a/heka/files/heka.service b/heka/files/heka.service
index 935360a..02f6aed 100644
--- a/heka/files/heka.service
+++ b/heka/files/heka.service
@@ -1,4 +1,4 @@
-{%- if grains.get('init', None) == 'systemd' %}
+{%- if systemd_enabled %}
[Unit]
Description=heka {{ service_name }} - data collector and processor daemon
@@ -9,7 +9,7 @@
EnvironmentFile=-/etc/default/{{ service_name }}
User=heka
Group=heka
-LimitNOFILE=102400
+LimitNOFILE={{ max_open_files }}
ExecStart=/usr/bin/hekad -config=/etc/{{ service_name }}
# NOT SURE HEKA doesn't support reloading by signal
# ExecReload=/bin/kill -HUP $MAINPID
@@ -17,8 +17,10 @@
Restart=on-failure
StandardError=inherit
+{%- if automatic_starting %}
[Install]
WantedBy=multi-user.target
+{%- endif %}
{%- else %}
@@ -26,8 +28,10 @@
description "{{ service_name }}"
+{%- if automatic_starting %}
start on runlevel [2345]
stop on runlevel [!2345]
+{%- endif %}
respawn
@@ -38,7 +42,7 @@
script
# https://bugs.launchpad.net/lma-toolchain/+bug/1543289
- ulimit -n 102400
+ ulimit -n {{ max_open_files }}
exec start-stop-daemon --start --chuid heka --exec /usr/local/bin/{{ service_name }}_wrapper >> /var/log/{{ service_name }}.log 2>&1
end script
diff --git a/heka/files/lua/common/java_patterns.lua b/heka/files/lua/common/java_patterns.lua
index edbd130..ce81818 100644
--- a/heka/files/lua/common/java_patterns.lua
+++ b/heka/files/lua/common/java_patterns.lua
@@ -35,4 +35,8 @@
-- 2016-11-21 08:52:14,070 - DEBUG - run_command: nodetool -h 127.0.0.1 info | grep ID | awk '{print $3}'
CassandraLogGrammar = l.Ct(patt.JavaTimestamp * patt.sp * patt.dash * patt.sp * patt.JavaSeverity * patt.sp^1 * patt.dash * patt.sp * message)
+-- Ifmap
+-- 2016-11-24 10:11:32,457 - TRACE - [main] - MetadataTypeRepository: new MetadataType http://www.trustedcomputinggroup.org/2010/IFMAP-METADATA/2:discovered-by - singleValue
+IfmapLogGrammar = CassandraLogGrammar
+
return M
diff --git a/heka/files/lua/decoders/ifmap.lua b/heka/files/lua/decoders/ifmap.lua
new file mode 100644
index 0000000..b11c6b5
--- /dev/null
+++ b/heka/files/lua/decoders/ifmap.lua
@@ -0,0 +1,46 @@
+-- Copyright 2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local java = require 'java_patterns'
+
+local msg = {
+ Timestamp = nil,
+ Type = 'log',
+ Hostname = nil,
+ Payload = nil,
+ Pid = nil,
+ Fields = nil,
+ Severity = 6,
+}
+
+function process_message ()
+ local log = read_message("Payload")
+ local logger = read_message("Logger")
+ local m = java.IfmapLogGrammar:match(log)
+ if not m then
+ return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+ end
+ msg.Timestamp = m.Timestamp
+ msg.Payload = m.Message
+ msg.Pid = m.Pid
+ msg.Severity = utils.label_to_severity_map[m.SeverityLabel or 'INFO'] or 6
+ msg.Fields = {}
+ msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+ msg.Fields.programname = 'ifmap'
+ utils.inject_tags(msg)
+ return utils.safe_inject_message(msg)
+end
diff --git a/heka/files/service_wrapper b/heka/files/service_wrapper
index df3b5c9..f3532f1 100644
--- a/heka/files/service_wrapper
+++ b/heka/files/service_wrapper
@@ -2,5 +2,4 @@
HEKAD="/usr/bin/hekad"
-ulimit -n 102400
exec $HEKAD -config=/etc/{{ service_name }}
diff --git a/heka/map.jinja b/heka/map.jinja
index 3d7b9b6..908c5df 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -37,6 +37,7 @@
{% set default_aggregator_port = 5565 %}
{% set default_nagios_port = 8001 %}
{% set default_nagios_host_alarm_clusters = '00-clusters' %}
+{% set default_automatic_starting = True %}
{% set default_rabbit_port = 5672 %}
{% set default_rabbit_vhost = '/openstack' %}
@@ -46,6 +47,7 @@
'default': {
'elasticsearch_port': default_elasticsearch_port,
'poolsize': 100,
+ 'automatic_starting': default_automatic_starting,
}
}, merge=salt['pillar.get']('heka:log_collector')) %}
@@ -57,6 +59,7 @@
'aggregator_port': default_aggregator_port,
'nagios_port': default_nagios_port,
'poolsize': 100,
+ 'automatic_starting': default_automatic_starting,
}
}, merge=salt['pillar.get']('heka:metric_collector')) %}
@@ -67,6 +70,7 @@
'influxdb_timeout': default_influxdb_timeout,
'aggregator_port': default_aggregator_port,
'poolsize': 100,
+ 'automatic_starting': default_automatic_starting,
}
}, merge=salt['pillar.get']('heka:remote_collector')) %}
@@ -78,6 +82,7 @@
'nagios_port': default_nagios_port,
'nagios_default_host_alarm_clusters': default_nagios_host_alarm_clusters,
'poolsize': 100,
+ 'automatic_starting': default_automatic_starting,
}
}, merge=salt['pillar.get']('heka:aggregator')) %}
diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index 857add8..0f16d20 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -172,7 +172,7 @@
input:
heka_collectd:
engine: http
- address: 127.0.0.1
+ address: 0.0.0.0
port: 8326
decoder: collectd_decoder
splitter: NullSplitter
@@ -203,7 +203,7 @@
module_file: /usr/share/lma_collector/filters/influxdb_accumulator.lua
module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
preserve_data: false
- message_matcher: "Type == 'heka.sandbox.afd_metric'"
+ message_matcher: "Type =~ /metric$/"
ticker_interval: 1
config:
tag_fields: "deployment_id environment_label tenant_id user_id"
diff --git a/metadata/service/aggregator/cluster.yml b/metadata/service/aggregator/cluster.yml
new file mode 100644
index 0000000..e95243e
--- /dev/null
+++ b/metadata/service/aggregator/cluster.yml
@@ -0,0 +1,13 @@
+applications:
+- heka
+classes:
+- service.heka.support
+parameters:
+ _param:
+ aggregator_poolsize: 100
+ heka:
+ aggregator:
+ automatic_starting: false
+ enabled: true
+ influxdb_time_precision: ms
+ poolsize: ${_param:aggregator_poolsize}
diff --git a/metadata/service/aggregator/single.yml b/metadata/service/aggregator/single.yml
index 476536c..159e5bd 100644
--- a/metadata/service/aggregator/single.yml
+++ b/metadata/service/aggregator/single.yml
@@ -8,6 +8,7 @@
nagios_host_dimension_key: nagios_host
heka:
aggregator:
+ automatic_starting: true
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:aggregator_poolsize}
diff --git a/metadata/service/remote_collector/cluster.yml b/metadata/service/remote_collector/cluster.yml
new file mode 100644
index 0000000..9e60e86
--- /dev/null
+++ b/metadata/service/remote_collector/cluster.yml
@@ -0,0 +1,13 @@
+applications:
+- heka
+classes:
+- service.heka.support
+parameters:
+ _param:
+ remote_collector_poolsize: 100
+ heka:
+ remote_collector:
+ automatic_starting: false
+ enabled: true
+ influxdb_time_precision: ms
+ poolsize: ${_param:remote_collector_poolsize}
diff --git a/metadata/service/remote_collector/single.yml b/metadata/service/remote_collector/single.yml
index 120414c..8533ad2 100644
--- a/metadata/service/remote_collector/single.yml
+++ b/metadata/service/remote_collector/single.yml
@@ -7,6 +7,7 @@
remote_collector_poolsize: 100
heka:
remote_collector:
+ automatic_starting: true
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:remote_collector_poolsize}