Add specific logrotate script and crontab
The log rate produced by Heka may be high when the Elasticsearch output
queue is full. For that reason, we want to rotate log files every hour.
By default, logrotate runs daily on Ubuntu. So we use a specific
logrotate script and crontab.
diff --git a/heka/_service.sls b/heka/_service.sls
index 359e716..687b30d 100644
--- a/heka/_service.sls
+++ b/heka/_service.sls
@@ -53,7 +53,9 @@
- mode: 644
- replace: false
-/etc/logrotate.d/{{ service_name }}:
+{%- set logrotate_conf = "/etc/logrotate_" ~ service_name ~ ".conf" %}
+
+{{ logrotate_conf }}:
file.managed:
- source: salt://heka/files/heka_logrotate.conf
- template: jinja
@@ -63,6 +65,26 @@
- group: root
- mode: 644
+{%- set logrotate_command = "/usr/local/bin/" ~ service_name ~ "_logrotate" %}
+
+{{ logrotate_command }}:
+ file.managed:
+ - source: salt://heka/files/heka_logrotate.cron
+ - template: jinja
+ - defaults:
+ service_name: {{ service_name }}
+ logrotate_conf: {{ logrotate_conf }}
+ - user: root
+ - group: root
+ - mode: 755
+ - require:
+ - file: {{ logrotate_conf }}
+ cron.present:
+ - identifier: {{ service_name }}_logrotate_cron
+ - minute: '*/30'
+ - require:
+ - file: {{ logrotate_command }}
+
heka_{{ service_name }}_service_wrapper:
file.managed:
- name: /usr/local/bin/{{ service_name }}_wrapper
diff --git a/heka/files/heka_logrotate.cron b/heka/files/heka_logrotate.cron
new file mode 100644
index 0000000..cc3e249
--- /dev/null
+++ b/heka/files/heka_logrotate.cron
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+test -x /usr/sbin/logrotate || exit 0
+
+LOCK_FILE="/var/lock/logrotate.{{ service_name }}.lock"
+
+lock() {
+ exec 903>$LOCK_FILE
+ flock -n 903 && return 0 || return 1
+}
+
+unlock() {
+ flock -u 903
+}
+
+fail() {
+ if [ -z "$1" ]
+ then
+ MESSAGE="WARNING logrotate failed, no reason provided"
+ else
+ MESSAGE=$1
+ fi
+ /usr/bin/logger -t logrotate "{{ service_name }} ${MESSAGE}"
+ unlock
+ exit 1
+}
+
+lock || fail "WARNING {{ service_name }} logrotate flock failed, exiting"
+
+
+TMP_FILE=$(/bin/mktemp)
+nice ionice -c3 /usr/sbin/logrotate -s /var/lib/logrotate/{{ service_name }}.status {{ logrotate_conf }} >& $TMP_FILE
+EXITVALUE=$?
+
+if [ -f /etc/redhat-release ] || [ -f /etc/centos-release ];
+then
+ # Due to bug in logrotate on centos/rhel, it always returns 0. Use grep for
+ # detect errors; exit code 1 is considered a success as no errors were
+ # found.
+ grep -q error $TMP_FILE
+ EXITVALUE=$?
+ EXPECTEDVALUE=1
+else
+ EXPECTEDVALUE=0
+fi
+rm $TMP_FILE
+
+if [ "${EXITVALUE}" != "${EXPECTEDVALUE}" ]; then
+ fail "ALERT exited abnormally with [${EXITVALUE}] (${EXPECTEDVALUE} was expected)"
+fi
+
+unlock
+exit 0