Merge pull request #5 from salt-formulas/pr_fix_missing_entity_before_deployment
fix missing rsyslog_util.syslog_file_match
diff --git a/.gitignore b/.gitignore
index 1bfce6e..aa8e42a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
+.kitchen
tests/build/
*.swp
*.pyc
-.ropeproject
\ No newline at end of file
+.ropeproject
diff --git a/Makefile b/Makefile
index fc83783..1043fbe 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,38 @@
DESTDIR=/
SALTENVDIR=/usr/share/salt-formulas/env
RECLASSDIR=/usr/share/salt-formulas/reclass
-FORMULANAME=$(shell grep name: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\-]*')
+FORMULANAME=$(shell grep name: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\-\_]*')
+VERSION=$(shell grep version: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\.\-\_]*')
+VERSION_MAJOR := $(shell echo $(VERSION)|cut -d . -f 1-2)
+VERSION_MINOR := $(shell echo $(VERSION)|cut -d . -f 3)
+
+NEW_MAJOR_VERSION ?= $(shell date +%Y.%m|sed 's,\.0,\.,g')
+NEW_MINOR_VERSION ?= $(shell /bin/bash -c 'echo $$[ $(VERSION_MINOR) + 1 ]')
+
+MAKE_PID := $(shell echo $$PPID)
+JOB_FLAG := $(filter -j%, $(subst -j ,-j,$(shell ps T | grep "^\s*$(MAKE_PID).*$(MAKE)")))
+
+ifneq ($(subst -j,,$(JOB_FLAG)),)
+JOBS := $(subst -j,,$(JOB_FLAG))
+else
+JOBS := 1
+endif
+
+KITCHEN_LOCAL_YAML?=.kitchen.yml
+KITCHEN_OPTS?="--concurrency=$(JOBS)"
+KITCHEN_OPTS_CREATE?=""
+KITCHEN_OPTS_CONVERGE?=""
+KITCHEN_OPTS_VERIFY?=""
+KITCHEN_OPTS_TEST?=""
all:
@echo "make install - Install into DESTDIR"
@echo "make test - Run tests"
+ @echo "make kitchen - Run Kitchen CI tests (create, converge, verify)"
@echo "make clean - Cleanup after tests run"
+ @echo "make release-major - Generate new major release"
+ @echo "make release-minor - Generate new minor release"
+ @echo "make changelog - Show changes since last release"
install:
# Formula
@@ -14,6 +40,7 @@
cp -a $(FORMULANAME) $(DESTDIR)/$(SALTENVDIR)/
[ ! -d _modules ] || cp -a _modules $(DESTDIR)/$(SALTENVDIR)/
[ ! -d _states ] || cp -a _states $(DESTDIR)/$(SALTENVDIR)/ || true
+ [ ! -d _grains ] || cp -a _grains $(DESTDIR)/$(SALTENVDIR)/ || true
# Metadata
[ -d $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) ] || mkdir -p $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
cp -a metadata/service/* $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
@@ -21,6 +48,71 @@
test:
[ ! -d tests ] || (cd tests; ./run_tests.sh)
+release-major: check-changes
+ @echo "Current version is $(VERSION), new version is $(NEW_MAJOR_VERSION)"
+ @[ $(VERSION_MAJOR) != $(NEW_MAJOR_VERSION) ] || (echo "Major version $(NEW_MAJOR_VERSION) already released, nothing to do. Do you want release-minor?" && exit 1)
+ echo "$(NEW_MAJOR_VERSION)" > VERSION
+ sed -i 's,version: .*,version: "$(NEW_MAJOR_VERSION)",g' metadata.yml
+ [ ! -f debian/changelog ] || dch -v $(NEW_MAJOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
+ make genchangelog-$(NEW_MAJOR_VERSION)
+ (git add -u; git commit -m "Version $(NEW_MAJOR_VERSION)")
+ git tag -s -m $(NEW_MAJOR_VERSION) $(NEW_MAJOR_VERSION)
+
+release-minor: check-changes
+ @echo "Current version is $(VERSION), new version is $(VERSION_MAJOR).$(NEW_MINOR_VERSION)"
+ echo "$(VERSION_MAJOR).$(NEW_MINOR_VERSION)" > VERSION
+ sed -i 's,version: .*,version: "$(VERSION_MAJOR).$(NEW_MINOR_VERSION)",g' metadata.yml
+ [ ! -f debian/changelog ] || dch -v $(VERSION_MAJOR).$(NEW_MINOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
+ make genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+ (git add -u; git commit -m "Version $(VERSION_MAJOR).$(NEW_MINOR_VERSION)")
+ git tag -s -m $(NEW_MAJOR_VERSION) $(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+
+check-changes:
+ @git log --pretty=oneline --decorate $(VERSION)..HEAD | grep -Eqc '.*' || (echo "No new changes since version $(VERSION)"; exit 1)
+
+changelog:
+ git log --pretty=short --invert-grep --grep="Merge pull request" --decorate $(VERSION)..HEAD
+
+genchangelog: genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+
+genchangelog-%:
+ $(eval NEW_VERSION := $(patsubst genchangelog-%,%,$@))
+ (echo "=========\nChangelog\n=========\n"; \
+ (echo $(NEW_VERSION);git tag) | sort -r | grep -E '^[0-9\.]+' | while read i; do \
+ cur=$$i; \
+ test $$i = $(NEW_VERSION) && i=HEAD; \
+ prev=`(echo $(NEW_VERSION);git tag)|sort|grep -E '^[0-9\.]+'|grep -B1 "$$cur\$$"|head -1`; \
+ echo "Version $$cur\n=============================\n"; \
+ git log --pretty=short --invert-grep --grep="Merge pull request" --decorate $$prev..$$i; \
+ echo; \
+ done) > CHANGELOG.rst
+
+kitchen-check:
+ @[ -e $(KITCHEN_LOCAL_YAML) ] || (echo "Kitchen tests not available, there's no $(KITCHEN_LOCAL_YAML)." && exit 1)
+
+kitchen: kitchen-check kitchen-create kitchen-converge kitchen-verify kitchen-list
+
+kitchen-create: kitchen-check
+ kitchen create ${KITCHEN_OPTS} ${KITCHEN_OPTS_CREATE}
+ [ "$(shell echo $(KITCHEN_LOCAL_YAML)|grep -Eo docker)" = "docker" ] || sleep 120
+
+kitchen-converge: kitchen-check
+ kitchen converge ${KITCHEN_OPTS} ${KITCHEN_OPTS_CONVERGE} &&\
+ kitchen converge ${KITCHEN_OPTS} ${KITCHEN_OPTS_CONVERGE}
+
+kitchen-verify: kitchen-check
+ [ ! -d tests/integration ] || kitchen verify -t tests/integration ${KITCHEN_OPTS} ${KITCHEN_OPTS_VERIFY}
+ [ -d tests/integration ] || kitchen verify ${KITCHEN_OPTS} ${KITCHEN_OPTS_VERIFY}
+
+kitchen-test: kitchen-check
+ [ ! -d tests/integration ] || kitchen test -t tests/integration ${KITCHEN_OPTS} ${KITCHEN_OPTS_TEST}
+ [ -d tests/integration ] || kitchen test ${KITCHEN_OPTS} ${KITCHEN_OPTS_TEST}
+
+kitchen-list: kitchen-check
+ kitchen list
+
clean:
+ [ ! -x "$(shell which kitchen)" ] || kitchen destroy
+ [ ! -d .kitchen ] || rm -rf .kitchen
[ ! -d tests/build ] || rm -rf tests/build
[ ! -d build ] || rm -rf build
diff --git a/README.rst b/README.rst
index c66fcb2..7c014b0 100644
--- a/README.rst
+++ b/README.rst
@@ -27,7 +27,7 @@
format:
name: TraditionalFormatWithPRI
template: '"%syslogpriority% %syslogfacility% %timestamp:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"'
- logfiles:
+ output:
file:
-/var/log/syslog:
filter: *.*;auth,authpriv.none
@@ -63,7 +63,44 @@
filter: *.emerg
"|/dev/xconsole":
filter: "daemon.*;mail.*; news.err; *.=debug;*.=info;*.=notice;*.=warn":
+ -/var/log/your-app.log:
+ filter: "if $programname startswith 'your-app' then"
+ owner: syslog
+ group: adm
+ createmode: 0640
+ umask: 0022
+ stop_processing: true
+Custom templates
+================
+
+It is possible to define a specific syslog template per output file instead of
+using the default one.
+
+.. code-block:: yaml
+
+ rsyslog:
+ output:
+ file:
+ /var/log/your-app.log:
+ template: ""%syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%\\n""
+ filter: "if $programname startswith 'your-app' then"
+
+Support metadata
+================
+
+If the *heka* support metadata is enabled, all output files are automatically
+parsed by the **log_collector** service.
+To skip the log_collector configuration, set the **skip_log_collector** to true.
+
+.. code-block:: yaml
+
+ rsyslog:
+ output:
+ file:
+ /var/log/your-app.log:
+ filter: "if $programname startswith 'your-app' then"
+ skip_log_collector: true
Read more
=========
@@ -71,3 +108,36 @@
http://www.rsyslog.com/
https://wiki.gentoo.org/wiki/Rsyslog
https://github.com/saz/puppet-rsyslog
+
+Documentation and Bugs
+======================
+
+To learn how to install and update salt-formulas, consult the documentation
+available online at:
+
+ http://salt-formulas.readthedocs.io/
+
+In the unfortunate event that bugs are discovered, they should be reported to
+the appropriate issue tracker. Use Github issue tracker for specific salt
+formula:
+
+ https://github.com/salt-formulas/salt-formula-rsyslog/issues
+
+For feature requests, bug reports or blueprints affecting entire ecosystem,
+use Launchpad salt-formulas project:
+
+ https://launchpad.net/salt-formulas
+
+You can also join salt-formulas-users team and subscribe to mailing list:
+
+ https://launchpad.net/~salt-formulas-users
+
+Developers wishing to work on the salt-formulas projects should always base
+their work on master branch and submit pull request against specific formula.
+
+ https://github.com/salt-formulas/salt-formula-rsyslog
+
+Any questions or feedback is always welcome so feel free to join our IRC
+channel:
+
+ #salt-formulas @ irc.freenode.net
diff --git a/_modules/rsyslog_util.py b/_modules/rsyslog_util.py
index d58a1c4..df21f07 100644
--- a/_modules/rsyslog_util.py
+++ b/_modules/rsyslog_util.py
@@ -15,7 +15,7 @@
"""
file_match = {}
for name, config in output.get('file', {}).items():
- if not config.get('enabled', False):
+ if not config.get('enabled') or config.get('skip_log_collector', False):
continue
logdir = os.path.dirname(name)
pattern = os.path.basename(name).replace('.', '\.')
diff --git a/rsyslog/files/rsyslog.default.conf b/rsyslog/files/rsyslog.default.conf
index edc2fca..ff7f917 100644
--- a/rsyslog/files/rsyslog.default.conf
+++ b/rsyslog/files/rsyslog.default.conf
@@ -30,7 +30,6 @@
$PrivDropToUser {{ global.run_user }}
$PrivDropToGroup {{ global.run_group }}
$WorkDirectory {{ global.spool_dir }}
-$IncludeConfig {{ global.rsyslog_d }}/*.conf
{% if global.format is defined -%}
$Template {{ global.format.name }}, "{{ global.format.template }}"
@@ -40,7 +39,27 @@
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
{% endif -%}
-{% for name,config in global.output.file.iteritems() if config.get('enabled', False) %}
+{# Additional configurations should be included after defining the global format otherwise they won't benefit from it #}
+$IncludeConfig {{ global.rsyslog_d }}/*.conf
+
+{%- set file_groups = {
+ 'stop': {},
+ 'other': {}
+} -%}
+
+{%- for name,config in global.output.file.iteritems() if config.get('enabled', False) %}
+{%- if config.get('stop_processing', False) -%}
+{%- do file_groups.stop.update({name: config}) -%}
+{%- else -%}
+{%- do file_groups.other.update({name: config}) -%}
+{%- endif -%}
+{%- if config.template is defined %}
+$Template {{ name|replace('/', '_') }}, "{{ config.template }}"
+{%- endif -%}
+{%- endfor -%}
+
+{%- macro rsyslog_output_file(files) -%}
+{%- for name, config in files.items() %}
{% if config.owner is defined -%}
$FileOwner {{ config['owner'] }}
{% endif -%}
@@ -53,8 +72,15 @@
{% if config.umask is defined -%}
$Umask {{ config['umask'] }}
{% endif -%}
-{{ config['filter'] }} {% if config.sync == true %}-{% endif %}{{ name }}
+{{ config['filter'] }} {% if config.sync == true %}-{% endif %}{{ name }}{%if config.template is defined %};{{ name|replace('/', '_') }}{% endif %}
+{%- if config.get('stop_processing', False) %}
+&stop
+{%- endif %}
{% endfor -%}
+{%- endmacro %}
+
+{{ rsyslog_output_file(file_groups.stop) }}
+{{ rsyslog_output_file(file_groups.other) }}
{% if global.output.console is defined %}
{% for name,config in global.output.console.iteritems() if config.get('enabled', False) -%}
diff --git a/rsyslog/map.jinja b/rsyslog/map.jinja
index a322661..7078ebc 100644
--- a/rsyslog/map.jinja
+++ b/rsyslog/map.jinja
@@ -106,44 +106,49 @@
sync: true
filter: "*.info;mail.none;authpriv.none;cron.none"
owner: root
- group: root
- createmode: "0600"
- umask: "0000"
+ group: adm
+ createmode: "0640"
+ umask: "0022"
enabled: true
/var/log/secure:
sync: true
filter: "authpriv.*"
owner: root
- group: root
- createmode: "0600"
+ group: adm
+ createmode: "0640"
+ umask: "0022"
enabled: true
/var/log/maillog:
sync: true
filter: "mail.*"
owner: root
- group: root
- createmode: "0600"
+ group: adm
+ createmode: "0640"
+ umask: "0022"
enabled: true
/var/log/cron:
sync: true
filter: "cron.*"
owner: root
- group: root
- createmode: "0600"
+ group: adm
+ createmode: "0640"
+ umask: "0022"
enabled: true
/var/log/spooler:
sync: true
filter: "uucp,news.crit"
owner: root
- group: root
- createmode: "0600"
+ group: adm
+ createmode: "0640"
+ umask: "0022"
enabled: true
/var/log/boot.log:
sync: false
filter: "local7.*"
owner: root
- group: root
- createmode: "0600"
+ group: adm
+ createmode: "0640"
+ umask: "0022"
enabled: true
{%- endload %}
{%- set global = salt['grains.filter_by'](base_defaults, merge=salt['pillar.get']('rsyslog:client')) %}
diff --git a/rsyslog/meta/heka.yml b/rsyslog/meta/heka.yml
index f0ad038..701369c 100644
--- a/rsyslog/meta/heka.yml
+++ b/rsyslog/meta/heka.yml
@@ -35,3 +35,30 @@
differentiator: [ 'system.', 'haproxy' ]
decoder: "syslog_decoder"
splitter: "TokenSplitter"
+ filter:
+ failed_logins:
+ engine: sandbox
+ module_file: /usr/share/lma_collector/filters/failed_logins.lua
+ module_dir: /usr/share/lma_collector/common;/usr/share/heka/lua_modules
+ message_matcher: "Type == 'log' && Logger == 'system.auth'"
+ ticker_interval: 60
+ config:
+ hostname: '{{ grains.host }}'
+ grace_interval: 30
+metric_collector:
+ trigger:
+ failed_logins_warning:
+ description: 'The rate of failed logins is too high.'
+ severity: warning
+ rules:
+ - metric: failed_logins_rate
+ relational_operator: '>='
+ threshold: 0.2
+ window: 120
+ periods: 0
+ function: avg
+ alarm:
+ failed_logins:
+ alerting: enabled
+ triggers:
+ - failed_logins_warning