Merge "Add Lua filter to get metrics from auth messages"
diff --git a/.gitignore b/.gitignore
index b80d7e8..aa8e42a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,5 @@
+.kitchen
tests/build/
*.swp
*.pyc
.ropeproject
-tests/lua/mocks/inspect.lua
-tests/lua/mocks/anomaly.lua
-tests/lua/mocks/annotation.lua
-tests/lua/mocks/date_time.lua
-tests/circular_buffer.so
diff --git a/Makefile b/Makefile
index fc83783..1043fbe 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,38 @@
DESTDIR=/
SALTENVDIR=/usr/share/salt-formulas/env
RECLASSDIR=/usr/share/salt-formulas/reclass
-FORMULANAME=$(shell grep name: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\-]*')
+FORMULANAME=$(shell grep name: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\-\_]*')
+VERSION=$(shell grep version: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\.\-\_]*')
+VERSION_MAJOR := $(shell echo $(VERSION)|cut -d . -f 1-2)
+VERSION_MINOR := $(shell echo $(VERSION)|cut -d . -f 3)
+
+NEW_MAJOR_VERSION ?= $(shell date +%Y.%m|sed 's,\.0,\.,g')
+NEW_MINOR_VERSION ?= $(shell /bin/bash -c 'echo $$[ $(VERSION_MINOR) + 1 ]')
+
+MAKE_PID := $(shell echo $$PPID)
+JOB_FLAG := $(filter -j%, $(subst -j ,-j,$(shell ps T | grep "^\s*$(MAKE_PID).*$(MAKE)")))
+
+ifneq ($(subst -j,,$(JOB_FLAG)),)
+JOBS := $(subst -j,,$(JOB_FLAG))
+else
+JOBS := 1
+endif
+
+KITCHEN_LOCAL_YAML?=.kitchen.yml
+KITCHEN_OPTS?="--concurrency=$(JOBS)"
+KITCHEN_OPTS_CREATE?=""
+KITCHEN_OPTS_CONVERGE?=""
+KITCHEN_OPTS_VERIFY?=""
+KITCHEN_OPTS_TEST?=""
all:
@echo "make install - Install into DESTDIR"
@echo "make test - Run tests"
+ @echo "make kitchen - Run Kitchen CI tests (create, converge, verify)"
@echo "make clean - Cleanup after tests run"
+ @echo "make release-major - Generate new major release"
+ @echo "make release-minor - Generate new minor release"
+ @echo "make changelog - Show changes since last release"
install:
# Formula
@@ -14,6 +40,7 @@
cp -a $(FORMULANAME) $(DESTDIR)/$(SALTENVDIR)/
[ ! -d _modules ] || cp -a _modules $(DESTDIR)/$(SALTENVDIR)/
[ ! -d _states ] || cp -a _states $(DESTDIR)/$(SALTENVDIR)/ || true
+ [ ! -d _grains ] || cp -a _grains $(DESTDIR)/$(SALTENVDIR)/ || true
# Metadata
[ -d $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) ] || mkdir -p $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
cp -a metadata/service/* $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
@@ -21,6 +48,71 @@
test:
[ ! -d tests ] || (cd tests; ./run_tests.sh)
+release-major: check-changes
+ @echo "Current version is $(VERSION), new version is $(NEW_MAJOR_VERSION)"
+ @[ $(VERSION_MAJOR) != $(NEW_MAJOR_VERSION) ] || (echo "Major version $(NEW_MAJOR_VERSION) already released, nothing to do. Do you want release-minor?" && exit 1)
+ echo "$(NEW_MAJOR_VERSION)" > VERSION
+ sed -i 's,version: .*,version: "$(NEW_MAJOR_VERSION)",g' metadata.yml
+ [ ! -f debian/changelog ] || dch -v $(NEW_MAJOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
+ make genchangelog-$(NEW_MAJOR_VERSION)
+ (git add -u; git commit -m "Version $(NEW_MAJOR_VERSION)")
+ git tag -s -m $(NEW_MAJOR_VERSION) $(NEW_MAJOR_VERSION)
+
+release-minor: check-changes
+ @echo "Current version is $(VERSION), new version is $(VERSION_MAJOR).$(NEW_MINOR_VERSION)"
+ echo "$(VERSION_MAJOR).$(NEW_MINOR_VERSION)" > VERSION
+ sed -i 's,version: .*,version: "$(VERSION_MAJOR).$(NEW_MINOR_VERSION)",g' metadata.yml
+ [ ! -f debian/changelog ] || dch -v $(VERSION_MAJOR).$(NEW_MINOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
+ make genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+ (git add -u; git commit -m "Version $(VERSION_MAJOR).$(NEW_MINOR_VERSION)")
+ git tag -s -m $(NEW_MAJOR_VERSION) $(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+
+check-changes:
+ @git log --pretty=oneline --decorate $(VERSION)..HEAD | grep -Eqc '.*' || (echo "No new changes since version $(VERSION)"; exit 1)
+
+changelog:
+ git log --pretty=short --invert-grep --grep="Merge pull request" --decorate $(VERSION)..HEAD
+
+genchangelog: genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+
+genchangelog-%:
+ $(eval NEW_VERSION := $(patsubst genchangelog-%,%,$@))
+ (echo "=========\nChangelog\n=========\n"; \
+ (echo $(NEW_VERSION);git tag) | sort -r | grep -E '^[0-9\.]+' | while read i; do \
+ cur=$$i; \
+ test $$i = $(NEW_VERSION) && i=HEAD; \
+ prev=`(echo $(NEW_VERSION);git tag)|sort|grep -E '^[0-9\.]+'|grep -B1 "$$cur\$$"|head -1`; \
+ echo "Version $$cur\n=============================\n"; \
+ git log --pretty=short --invert-grep --grep="Merge pull request" --decorate $$prev..$$i; \
+ echo; \
+ done) > CHANGELOG.rst
+
+kitchen-check:
+ @[ -e $(KITCHEN_LOCAL_YAML) ] || (echo "Kitchen tests not available, there's no $(KITCHEN_LOCAL_YAML)." && exit 1)
+
+kitchen: kitchen-check kitchen-create kitchen-converge kitchen-verify kitchen-list
+
+kitchen-create: kitchen-check
+ kitchen create ${KITCHEN_OPTS} ${KITCHEN_OPTS_CREATE}
+ [ "$(shell echo $(KITCHEN_LOCAL_YAML)|grep -Eo docker)" = "docker" ] || sleep 120
+
+kitchen-converge: kitchen-check
+ kitchen converge ${KITCHEN_OPTS} ${KITCHEN_OPTS_CONVERGE} &&\
+ kitchen converge ${KITCHEN_OPTS} ${KITCHEN_OPTS_CONVERGE}
+
+kitchen-verify: kitchen-check
+ [ ! -d tests/integration ] || kitchen verify -t tests/integration ${KITCHEN_OPTS} ${KITCHEN_OPTS_VERIFY}
+ [ -d tests/integration ] || kitchen verify ${KITCHEN_OPTS} ${KITCHEN_OPTS_VERIFY}
+
+kitchen-test: kitchen-check
+ [ ! -d tests/integration ] || kitchen test -t tests/integration ${KITCHEN_OPTS} ${KITCHEN_OPTS_TEST}
+ [ -d tests/integration ] || kitchen test ${KITCHEN_OPTS} ${KITCHEN_OPTS_TEST}
+
+kitchen-list: kitchen-check
+ kitchen list
+
clean:
+ [ ! -x "$(shell which kitchen)" ] || kitchen destroy
+ [ ! -d .kitchen ] || rm -rf .kitchen
[ ! -d tests/build ] || rm -rf tests/build
[ ! -d build ] || rm -rf build
diff --git a/README.rst b/README.rst
index 2aaef14..33a3bf4 100644
--- a/README.rst
+++ b/README.rst
@@ -180,3 +180,36 @@
=========
* https://hekad.readthedocs.org/en/latest/index.html
+
+Documentation and Bugs
+======================
+
+To learn how to install and update salt-formulas, consult the documentation
+available online at:
+
+ http://salt-formulas.readthedocs.io/
+
+In the unfortunate event that bugs are discovered, they should be reported to
+the appropriate issue tracker. Use Github issue tracker for specific salt
+formula:
+
+ https://github.com/salt-formulas/salt-formula-heka/issues
+
+For feature requests, bug reports or blueprints affecting entire ecosystem,
+use Launchpad salt-formulas project:
+
+ https://launchpad.net/salt-formulas
+
+You can also join salt-formulas-users team and subscribe to mailing list:
+
+ https://launchpad.net/~salt-formulas-users
+
+Developers wishing to work on the salt-formulas projects should always base
+their work on master branch and submit pull request against specific formula.
+
+ https://github.com/salt-formulas/salt-formula-heka
+
+Any questions or feedback is always welcome so feel free to join our IRC
+channel:
+
+ #salt-formulas @ irc.freenode.net
diff --git a/heka/files/lua/encoders/status_sensu.lua b/heka/files/lua/encoders/status_sensu.lua
new file mode 100644
index 0000000..2901dcf
--- /dev/null
+++ b/heka/files/lua/encoders/status_sensu.lua
@@ -0,0 +1,92 @@
+-- Copyright 2015 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+require 'table'
+require 'string'
+
+local afd = require 'afd'
+local consts = require 'gse_constants'
+local lma = require 'lma_utils'
+
+local source_dimension_field
+if read_config('sensu_source_dimension_key') then
+ source_dimension_field = string.format('Fields[%s]', read_config('sensu_source_dimension_key'))
+end
+
+-- mapping GSE statuses to Sensu states
+local sensu_state_map = {
+ [consts.OKAY]=0,
+ [consts.WARN]=1,
+ [consts.CRIT]=2,
+ [consts.DOWN]=2,
+ [consts.UNKW]=2
+}
+
+function process_message()
+
+ local data = {
+ source = nil,
+ name = nil,
+ status = nil,
+ output = nil,
+ }
+
+ local service_name = read_message('Fields[member]')
+ local status = afd.get_status()
+ local alarms = afd.alarms_for_human(afd.extract_alarms())
+ local msgtype = read_message("Type")
+
+ if not service_name or not sensu_state_map[status] or not alarms or not msgtype then
+ return -1
+ end
+
+ local source
+ if msgtype == "heka.sandbox.gse_metric" then
+ if source_dimension_field then
+ source = read_message(source_dimension_field) or "Unknown Source"
+ else
+ source = "Unknown source"
+ end
+ elseif msgtype == "heka.sandbox.afd_metric" then
+ source = read_message('Fields[hostname]') or read_message('Hostname')
+ else
+ -- Should not happen since we track only AFD and GSE plugins.
+ return -1
+ end
+
+ data['source'] = source
+ data['name'] = service_name
+ data['status'] = sensu_state_map[status]
+
+ local details = string.format('%s: ', consts.status_label(status))
+
+ if data['status'] ~= 0 then
+ if #alarms == 0 then
+ details = details .. 'No details\n'
+ else
+ for _, alarm in ipairs(alarms) do
+ details = details .. alarm .. '\n'
+ end
+ end
+ end
+
+ data['output'] = details
+
+ local payload = lma.safe_json_encode(data)
+
+ if not payload then
+ return -1
+ end
+
+ return lma.safe_inject_payload('json', 'sensu', payload)
+end
diff --git a/heka/map.jinja b/heka/map.jinja
index 1677ffc..86313f7 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -82,6 +82,7 @@
'influxdb_time_precision': default_influxdb_time_precision,
'influxdb_timeout': default_influxdb_timeout,
'nagios_port': default_nagios_port,
+ 'sensu_port': 3030,
'nagios_default_host_alarm_clusters': default_nagios_host_alarm_clusters,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
diff --git a/heka/meta/heka.yml b/heka/meta/heka.yml
index ba8fedd..637a870 100644
--- a/heka/meta/heka.yml
+++ b/heka/meta/heka.yml
@@ -4,7 +4,6 @@
{%- from "heka/map.jinja" import aggregator with context %}
{%- from "heka/map.jinja" import ceilometer_collector with context %}
-
log_collector:
filter:
aggregated_http_metrics:
@@ -457,7 +456,7 @@
preserve_data: false
message_matcher: "Type == 'heka.sandbox.gse_metric'"
{%- endif %}
-{%- if aggregator.influxdb_host is defined or aggregator.nagios_host is defined %}
+{%- if aggregator.influxdb_host is defined or aggregator.nagios_host is defined or aggregator.sensu_host is defined %}
encoder:
{%- if aggregator.influxdb_host is defined %}
influxdb:
@@ -477,8 +476,18 @@
nagios_host_dimension_key: "{{ aggregator.nagios_host_dimension_key }}"
{%- endif %}
{%- endif %}
+ {%- if aggregator.sensu_host is defined %}
+ sensu:
+ engine: sandbox
+ module_file: /usr/share/lma_collector/encoders/status_sensu.lua
+ module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
+ config:
+ {%- if aggregator.sensu_source_dimension_key is defined %}
+ sensu_source_dimension_key: "{{ aggregator.sensu_source_dimension_key }}"
+ {%- endif %}
+ {%- endif %}
{%- endif %}
-{%- if aggregator.influxdb_host is defined or aggregator.nagios_host is defined %}
+{%- if aggregator.influxdb_host is defined or aggregator.nagios_host is defined or aggregator.sensu_host is defined %}
output:
{%- if aggregator.influxdb_host is defined %}
influxdb:
@@ -506,6 +515,15 @@
max_file_size: 524288
full_action: drop
{%- endif %}
+ {%- if aggregator.sensu_host is defined %}
+ sensu_alarm_cluster:
+ engine: udp
+ host: "{{ aggregator.sensu_host }}"
+ port: "{{aggregator.sensu_port }}"
+ message_matcher: "(Type == 'heka.sandbox.gse_metric' || Type == 'heka.sandbox.afd_metric') && Fields[no_alerting] == NIL"
+ encoder: sensu_encoder
+ use_buffering: false
+ {%- endif %}
{%- endif %}
ceilometer_collector:
{%- if ceilometer_collector.amqp_host is defined %}
diff --git a/metadata/service/aggregator/output/nagios.yml b/metadata/service/aggregator/output/nagios.yml
new file mode 100644
index 0000000..7c30db1
--- /dev/null
+++ b/metadata/service/aggregator/output/nagios.yml
@@ -0,0 +1,5 @@
+parameters:
+ heka:
+ aggregator:
+ nagios_host_dimension_key: nagios_host
+ nagios_host: ${_param:nagios_host}
\ No newline at end of file
diff --git a/metadata/service/aggregator/output/sensu.yml b/metadata/service/aggregator/output/sensu.yml
new file mode 100644
index 0000000..bd5e55a
--- /dev/null
+++ b/metadata/service/aggregator/output/sensu.yml
@@ -0,0 +1,5 @@
+parameters:
+ heka:
+ aggregator:
+ sensu_source_dimension_key: nagios_host
+ sensu_host: 127.0.0.1
\ No newline at end of file
diff --git a/metadata/service/aggregator/single.yml b/metadata/service/aggregator/single.yml
index 159e5bd..bf2364e 100644
--- a/metadata/service/aggregator/single.yml
+++ b/metadata/service/aggregator/single.yml
@@ -5,11 +5,9 @@
parameters:
_param:
aggregator_poolsize: 100
- nagios_host_dimension_key: nagios_host
heka:
aggregator:
automatic_starting: true
enabled: true
influxdb_time_precision: ms
- poolsize: ${_param:aggregator_poolsize}
- nagios_host_dimension_key: ${_param:nagios_host_dimension_key}
+ poolsize: ${_param:aggregator_poolsize}
\ No newline at end of file