Fluentd initial commit
Change-Id: Ibcc4f3ad5c9f14b218d771d835e9e1ca589903bb
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..aa8e42a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+.kitchen
+tests/build/
+*.swp
+*.pyc
+.ropeproject
diff --git a/.kitchen.yml b/.kitchen.yml
new file mode 100644
index 0000000..e516907
--- /dev/null
+++ b/.kitchen.yml
@@ -0,0 +1,52 @@
+---
+driver:
+ name: docker
+ hostname: fluentd.ci.local
+ use_sudo: false
+
+provisioner:
+ name: salt_solo
+ salt_install: bootstrap
+ salt_bootstrap_url: https://bootstrap.saltstack.com
+ salt_version: latest
+ require_chef: false
+ log_level: error
+ formula: fluentd
+ grains:
+ noservices: True
+ dependencies:
+ - name: linux
+ repo: git
+ source: https://github.com/salt-formulas/salt-formula-linux.git
+ state_top:
+ base:
+ "*":
+ - linux
+ - fluentd
+ pillars:
+ top.sls:
+ base:
+ "*":
+ - fluentd
+ - repo_tdagent
+ pillars-from-files:
+ repo_tdagent.sls: tests/pillar/repo_tdagent.sls
+
+verifier:
+ name: inspec
+ sudo: true
+
+
+platforms:
+ - name: <%=ENV['PLATFORM'] || 'ubuntu-xenial'%>
+ driver_config:
+ image: <%=ENV['PLATFORM'] || 'trevorj/salty-whales:xenial'%>
+ platform: ubuntu
+
+suites:
+
+ - name: agent
+ provisioner:
+ pillars-from-files:
+ fluentd.sls: tests/pillar/agent.sls
+# vim: ft=yaml sw=2 ts=2 sts=2 tw=125
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..15fd7bc
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,42 @@
+sudo: required
+services:
+ - docker
+
+install:
+ - git fetch --unshallow --tags
+ - pip install PyYAML
+ - pip install virtualenv
+ - pip install reno
+ - |
+ test -e Gemfile || cat <<EOF > Gemfile
+ source 'https://rubygems.org'
+ gem 'rake'
+ gem 'test-kitchen'
+ gem 'kitchen-docker'
+ gem 'kitchen-inspec'
+ gem 'inspec'
+ gem 'kitchen-salt', :git => 'https://github.com/salt-formulas/kitchen-salt.git'
+ - bundle install
+
+env:
+ - PLATFORM=trevorj/salty-whales:trusty
+ - PLATFORM=trevorj/salty-whales:xenial
+
+before_script:
+ - set -o pipefail
+ - make test | tail
+ - reno lint $PWD | tail
+
+script:
+ - test ! -e .kitchen.yml || bundle exec kitchen test -t tests/integration
+
+notifications:
+ webhooks:
+ urls:
+ - https://webhooks.gitter.im/e/6123573504759330786b
+ on_success: change # options: [always|never|change] default: always
+ on_failure: never # options: [always|never|change] default: always
+ on_start: never # options: [always|never|change] default: always
+ on_cancel: never # options: [always|never|change] default: always
+ on_error: never # options: [always|never|change] default: always
+ email: false
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..3b118ea
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,14 @@
+ Copyright (c) 2017 Mirantis et al.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..1043fbe
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,118 @@
+DESTDIR=/
+SALTENVDIR=/usr/share/salt-formulas/env
+RECLASSDIR=/usr/share/salt-formulas/reclass
+FORMULANAME=$(shell grep name: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\-\_]*')
+VERSION=$(shell grep version: metadata.yml|head -1|cut -d : -f 2|grep -Eo '[a-z0-9\.\-\_]*')
+VERSION_MAJOR := $(shell echo $(VERSION)|cut -d . -f 1-2)
+VERSION_MINOR := $(shell echo $(VERSION)|cut -d . -f 3)
+
+NEW_MAJOR_VERSION ?= $(shell date +%Y.%m|sed 's,\.0,\.,g')
+NEW_MINOR_VERSION ?= $(shell /bin/bash -c 'echo $$[ $(VERSION_MINOR) + 1 ]')
+
+MAKE_PID := $(shell echo $$PPID)
+JOB_FLAG := $(filter -j%, $(subst -j ,-j,$(shell ps T | grep "^\s*$(MAKE_PID).*$(MAKE)")))
+
+ifneq ($(subst -j,,$(JOB_FLAG)),)
+JOBS := $(subst -j,,$(JOB_FLAG))
+else
+JOBS := 1
+endif
+
+KITCHEN_LOCAL_YAML?=.kitchen.yml
+KITCHEN_OPTS?="--concurrency=$(JOBS)"
+KITCHEN_OPTS_CREATE?=""
+KITCHEN_OPTS_CONVERGE?=""
+KITCHEN_OPTS_VERIFY?=""
+KITCHEN_OPTS_TEST?=""
+
+all:
+ @echo "make install - Install into DESTDIR"
+ @echo "make test - Run tests"
+ @echo "make kitchen - Run Kitchen CI tests (create, converge, verify)"
+ @echo "make clean - Cleanup after tests run"
+ @echo "make release-major - Generate new major release"
+ @echo "make release-minor - Generate new minor release"
+ @echo "make changelog - Show changes since last release"
+
+install:
+ # Formula
+ [ -d $(DESTDIR)/$(SALTENVDIR) ] || mkdir -p $(DESTDIR)/$(SALTENVDIR)
+ cp -a $(FORMULANAME) $(DESTDIR)/$(SALTENVDIR)/
+ [ ! -d _modules ] || cp -a _modules $(DESTDIR)/$(SALTENVDIR)/
+ [ ! -d _states ] || cp -a _states $(DESTDIR)/$(SALTENVDIR)/ || true
+ [ ! -d _grains ] || cp -a _grains $(DESTDIR)/$(SALTENVDIR)/ || true
+ # Metadata
+ [ -d $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) ] || mkdir -p $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
+ cp -a metadata/service/* $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
+
+test:
+ [ ! -d tests ] || (cd tests; ./run_tests.sh)
+
+release-major: check-changes
+ @echo "Current version is $(VERSION), new version is $(NEW_MAJOR_VERSION)"
+ @[ $(VERSION_MAJOR) != $(NEW_MAJOR_VERSION) ] || (echo "Major version $(NEW_MAJOR_VERSION) already released, nothing to do. Do you want release-minor?" && exit 1)
+ echo "$(NEW_MAJOR_VERSION)" > VERSION
+ sed -i 's,version: .*,version: "$(NEW_MAJOR_VERSION)",g' metadata.yml
+ [ ! -f debian/changelog ] || dch -v $(NEW_MAJOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
+ make genchangelog-$(NEW_MAJOR_VERSION)
+ (git add -u; git commit -m "Version $(NEW_MAJOR_VERSION)")
+ git tag -s -m $(NEW_MAJOR_VERSION) $(NEW_MAJOR_VERSION)
+
+release-minor: check-changes
+ @echo "Current version is $(VERSION), new version is $(VERSION_MAJOR).$(NEW_MINOR_VERSION)"
+ echo "$(VERSION_MAJOR).$(NEW_MINOR_VERSION)" > VERSION
+ sed -i 's,version: .*,version: "$(VERSION_MAJOR).$(NEW_MINOR_VERSION)",g' metadata.yml
+ [ ! -f debian/changelog ] || dch -v $(VERSION_MAJOR).$(NEW_MINOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
+ make genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+ (git add -u; git commit -m "Version $(VERSION_MAJOR).$(NEW_MINOR_VERSION)")
+ git tag -s -m $(NEW_MAJOR_VERSION) $(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+
+check-changes:
+ @git log --pretty=oneline --decorate $(VERSION)..HEAD | grep -Eqc '.*' || (echo "No new changes since version $(VERSION)"; exit 1)
+
+changelog:
+ git log --pretty=short --invert-grep --grep="Merge pull request" --decorate $(VERSION)..HEAD
+
+genchangelog: genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+
+genchangelog-%:
+ $(eval NEW_VERSION := $(patsubst genchangelog-%,%,$@))
+ (echo "=========\nChangelog\n=========\n"; \
+ (echo $(NEW_VERSION);git tag) | sort -r | grep -E '^[0-9\.]+' | while read i; do \
+ cur=$$i; \
+ test $$i = $(NEW_VERSION) && i=HEAD; \
+ prev=`(echo $(NEW_VERSION);git tag)|sort|grep -E '^[0-9\.]+'|grep -B1 "$$cur\$$"|head -1`; \
+ echo "Version $$cur\n=============================\n"; \
+ git log --pretty=short --invert-grep --grep="Merge pull request" --decorate $$prev..$$i; \
+ echo; \
+ done) > CHANGELOG.rst
+
+kitchen-check:
+ @[ -e $(KITCHEN_LOCAL_YAML) ] || (echo "Kitchen tests not available, there's no $(KITCHEN_LOCAL_YAML)." && exit 1)
+
+kitchen: kitchen-check kitchen-create kitchen-converge kitchen-verify kitchen-list
+
+kitchen-create: kitchen-check
+ kitchen create ${KITCHEN_OPTS} ${KITCHEN_OPTS_CREATE}
+ [ "$(shell echo $(KITCHEN_LOCAL_YAML)|grep -Eo docker)" = "docker" ] || sleep 120
+
+kitchen-converge: kitchen-check
+ kitchen converge ${KITCHEN_OPTS} ${KITCHEN_OPTS_CONVERGE} &&\
+ kitchen converge ${KITCHEN_OPTS} ${KITCHEN_OPTS_CONVERGE}
+
+kitchen-verify: kitchen-check
+ [ ! -d tests/integration ] || kitchen verify -t tests/integration ${KITCHEN_OPTS} ${KITCHEN_OPTS_VERIFY}
+ [ -d tests/integration ] || kitchen verify ${KITCHEN_OPTS} ${KITCHEN_OPTS_VERIFY}
+
+kitchen-test: kitchen-check
+ [ ! -d tests/integration ] || kitchen test -t tests/integration ${KITCHEN_OPTS} ${KITCHEN_OPTS_TEST}
+ [ -d tests/integration ] || kitchen test ${KITCHEN_OPTS} ${KITCHEN_OPTS_TEST}
+
+kitchen-list: kitchen-check
+ kitchen list
+
+clean:
+ [ ! -x "$(shell which kitchen)" ] || kitchen destroy
+ [ ! -d .kitchen ] || rm -rf .kitchen
+ [ ! -d tests/build ] || rm -rf tests/build
+ [ ! -d build ] || rm -rf build
diff --git a/README.rst b/README.rst
old mode 100755
new mode 100644
index 421aff5..4ef4f61
--- a/README.rst
+++ b/README.rst
@@ -1,4 +1,3 @@
-
===============
Fluentd Formula
===============
@@ -11,37 +10,226 @@
Fluentd solves the problem by having: easy installation, small footprint,
plugins reliable buffering, log forwarding, etc.
+**NOTE: WORK IN PROGRES**
+NOTE: DESIGN OF THIS FORMULA IS NOT YET STABLE AND MAY CHANGE
+NOTE: FORMULA NOT COMPATIBLE WITH OLD VERSION
+
Sample Pillars
==============
+General pillar structure
+------------------------
+
.. code-block:: yaml
- fluentd:
- server:
- enabled: true
- plugins:
- - fluent-plugin-elasticsearch
- - fluent-plugin-mongo
- config:
- - name: forward
- type: input
- bind:
- port: 24224
- host: 0.0.0.0
- - name: elasticsearch
- type: output
- bind:
- port: 9200
- host: localhost
- - name: mongodb
- type: output
- bind:
- port: localhost
- host: localhost
+ fluentd:
+ config:
+ label:
+ filename:
+ input:
+ input_name:
+ params
+ filter:
+ filter_name:
+ params
+ filter_name2:
+ params
+ match:
+ match_name:
+ params
+ input:
+ filename:
+ input_name:
+ params
+ input_name2:
+ params
+ filename2:
+ input_name3:
+ params
+ filter:
+ filename:
+ filter_name:
+ params
+ filter_name2:
+ params
+ filename2:
+ filter_name3:
+ params
+ match:
+ filename:
+ match_name:
+ params
-More Information
-================
+Example pillar
+--------------
+.. code-block:: yaml
-* http://fluentd.org/
-* http://docs.fluentd.org/
-* http://docs.fluentd.org/categories/recipes
+ fluentd:
+ enabled: true
+ config:
+ label:
+ monitoring:
+ filter:
+ parse_log:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: parser
+ reserve_data: true
+ key_name: log
+ parser:
+ type: regexp
+ format: >-
+ /^time="(?<time>[^ ]*)" level=(?<severity>[a-zA-Z]*) msg="(?<message>.+?)"/
+ time_format: '%FT%TZ'
+ remove_log_key:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: record_transformer
+ remove_keys: log
+ match:
+ docker_log:
+ tag: 'docker.**'
+ type: file
+ path: /tmp/flow-docker.log
+ grok_example:
+ input:
+ test_log:
+ type: tail
+ path: /var/log/test
+ tag: test.test
+ parser:
+ type: grok
+ custom_pattern_path: /etc/td-agent/config.d/global.grok
+ rule:
+ - pattern: >-
+ %{KEYSTONEACCESS}
+ syslog:
+ filter:
+ add_severity:
+ tag: 'syslog.*'
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: severity
+ value: 'record["pri"].to_i - (record["pri"].to_i / 8).floor * 8'
+ severity_to_string:
+ tag: 'syslog.*'
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: severity
+ value: '{"debug"=>7,"info"=>6,"notice"=>5,"warning"=>4,"error"=>3,"critical"=>2,"alert"=>1,"emerg"=>0}.key(record["severity"])'
+ severity_for_telegraf:
+ tag: 'syslog.*.telegraf'
+ type: parser
+ reserve_data: true
+ key_name: message
+ parser:
+ type: regexp
+ format: >-
+ /^(?<time>[^ ]*) (?<severity>[A-Z])! (?<message>.*)/
+ time_format: '%FT%TZ'
+ severity_for_telegraf_string:
+ tag: 'syslog.*.telegraf'
+ type: record_transformer
+ enable_ruby: true
+ record:
+ - name: severity
+ value: '{"debug"=>"D","info"=>"I","notice"=>"N","warning"=>"W","error"=>"E","critical"=>"C","alert"=>"A","emerg"=>"E"}.key(record["severity"])'
+ prometheus_metric:
+ tag: 'syslog.*.*'
+ type: prometheus
+ label:
+ - name: ident
+ type: variable
+ value: ident
+ - name: severity
+ type: variable
+ value: severity
+ metric:
+ - name: log_messages
+ type: counter
+ desc: The total number of log messages.
+ match:
+ rewrite_tag_key:
+ tag: 'syslog.*'
+ type: rewrite_tag_filter
+ rule:
+ - name: ident
+ regexp: '^(.*)'
+ result: '__TAG__.$1'
+ syslog_log:
+ tag: 'syslog.*.*'
+ type: file
+ path: /tmp/syslog
+ input:
+ syslog:
+ syslog_log:
+ type: tail
+ label: syslog
+ path: /var/log/syslog
+ tag: syslog.syslog
+ parser:
+ type: regexp
+ format: >-
+ '/^\<(?<pri>[0-9]+)\>(?<time>[^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
+ time_format: '%FT%T.%L%:z'
+ auth_log:
+ type: tail
+ label: syslog
+ path: /var/log/auth.log
+ tag: syslog.auth
+ parser:
+ type: regexp
+ format: >-
+ '/^\<(?<pri>[0-9]+)\>(?<time>[^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
+ time_format: '%FT%T.%L%:z'
+ prometheus:
+ prometheus:
+ type: prometheus
+ prometheus_monitor:
+ type: prometheus_monitor
+ prometheus_output_monitor:
+ type: prometheus_output_monitor
+ forward:
+ forward_listen:
+ type: forward
+ port: 24224
+ bind: 0.0.0.0
+ match:
+ docker_monitoring:
+ docker_monitoring:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: relabel
+ label: monitoring
+
+Documentation and Bugs
+======================
+
+To learn how to install and update salt-formulas, consult the documentation
+available online at:
+
+ http://salt-formulas.readthedocs.io/
+
+In the unfortunate event that bugs are discovered, they should be reported to
+the appropriate issue tracker. Use Github issue tracker for specific salt
+formula:
+
+ https://github.com/salt-formulas/salt-formula-nova/issues
+
+For feature requests, bug reports or blueprints affecting entire ecosystem,
+use Launchpad salt-formulas project:
+
+ https://launchpad.net/salt-formulas
+
+You can also join salt-formulas-users team and subscribe to mailing list:
+
+ https://launchpad.net/~salt-formulas-users
+
+Developers wishing to work on the salt-formulas projects should always base
+their work on master branch and submit pull request against specific formula.
+
+ https://github.com/salt-formulas/salt-formula-nova
+
+Any questions or feedback is always welcome so feel free to join our IRC
+channel:
+
+ #salt-formulas @ irc.freenode.net
diff --git a/conf/td-agent.conf b/conf/td-agent.conf
deleted file mode 100755
index 3ee09cf..0000000
--- a/conf/td-agent.conf
+++ /dev/null
@@ -1,53 +0,0 @@
-{% from "fluentd/map.jinja" import server with context %}
-
-{%- for config in server.config %}
-
-{%- if config.type == "input" and config.name =="forward" %}
-<source>
- type forward
- port {{ config.bind.port }}
- bind {{ config.bind.host }}
-</source>
-{%- endif %}
-{%- if config.type == "output" and config.name == "elasticsearch" %}
-<match *>
- port {{ config.bind.port }}
- bind {{ config.bind.host }}
- type elasticsearch
- logstash_format true
- flush_interval 10s # for testing
-</match>
-{%- endif %}
-{%- if config.type == "output" and config.name == "mongodb" %}
-<match *>
- # plugin type
- type mongo
-
- # mongodb db + collection
- database apache
- collection access
-
- # mongodb host + port
- host localhost
- port 27017
-
- # interval
- flush_interval 10s
-</match>
-{%- endif %}
-
-{%- endfor %}
-
-<source>
- type syslog
- port 42185
- tag syslog
-</source>
-
-<source>
- type tail
- format apache2
- path /var/log/httpd/kibana_kibana.access.log
- pos_file /var/log/td-agent/kibana_kibana.access_log.pos
- tag mongo.apache.access
-</source>
\ No newline at end of file
diff --git a/conf/treasure-data.gpg b/conf/treasure-data.gpg
deleted file mode 100755
index 2ce022d..0000000
--- a/conf/treasure-data.gpg
+++ /dev/null
@@ -1,18 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.5 (GNU/Linux)
-
-mQGiBFKuy6MRBACJccrkWHfYSKSbchD4ywYLGkIKITSLzEXrlpy4Sp6Mr9G1OGIv
-buIUIxksnev6h6wlFgbFYTLngpod+Jn4DpcGgXqzZ3lhOY1fvAaQ4SJ4RUELm1JJ
-SEWE0uYHuoI4+TrLtod3U02ETD+Lf6LWO3IfU1AYsA467tDYpVXR0GHeAwCgqU/V
-M+ypNOYFHVs1/aX83wdg0K8D+wf+tcvTHb/i2lND5DVfTWtC54WYNIG6QO8DgGdu
-EYregcMKRRNM4nSfUISzCgrjnpLGjvB9NKONCrimDiuz4TH7KDcEsKGA+zTzJLCy
-mcE6tKEHV5yS7cK0inim4nxAKFB70306CkhHyL2TrB42ppNFJwG1igO4BJ+55b9X
-Dd03A/9o6ONFle6khG+UE82WYTayy2pAshKTSLnOlB7CIxwt3SrzFXTuNosajVgZ
-S0EIB8c6SjFN6P1jHjd7PYOmBxd2xptERTg8CJMoTUI+P7LypWuvmagIdEwe571d
-ZMQ26HNuHg7/IMg+FQxhl/DFU6Aq+2CBJ3q8gO8Us4/UYc1PhrRUVHJlYXN1cmUg
-RGF0YSwgSW5jIChUcmVhc3VyZSBBZ2VudCBPZmZpY2lhbCBTaWduaW5nIGtleSkg
-PHN1cHBvcnRAdHJlYXN1cmUtZGF0YS5jb20+iGAEExECACAFAlKuy6MCGwMGCwkI
-BwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAQk9tFoS4gb3HgAKCBCDNT/cWzkKRQxvnL
-LiGvPetVSACeLLKnt2twfiPc9ZEA/X8MKkPjWqc=
-=e5Fe
------END PGP PUBLIC KEY BLOCK-----
\ No newline at end of file
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 0000000..e229ea2
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,5 @@
+salt-formula-fluentd (0.1) trusty; urgency=medium
+
+ * Initial release
+
+ -- Filip Pytloun <filip@pytloun.cz> Mon, 20 Feb 2017 17:47:36 +0100
diff --git a/debian/compat b/debian/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/debian/compat
@@ -0,0 +1 @@
+9
diff --git a/debian/control b/debian/control
new file mode 100644
index 0000000..b646d8a
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,15 @@
+Source: salt-formula-fluentd
+Maintainer: Salt Formulas Community <salt-formulas@freelists.org>
+Section: admin
+Priority: optional
+Build-Depends: salt-master, python, python-yaml, debhelper (>= 9)
+Standards-Version: 3.9.6
+Homepage: https://github.com/salt-formulas/salt-formulas
+Vcs-Browser: https://github.com/salt-formulas/salt-formula-fluentd
+Vcs-Git: https://github.com/salt-formulas/salt-formula-fluentd.git
+
+Package: salt-formula-fluentd
+Architecture: all
+Depends: ${misc:Depends}, salt-master, reclass
+Description: Fluentd Salt formula
+ Install and configure fluentd
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 0000000..005847f
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,15 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: salt-formula-fluentd
+Upstream-Contact: Salt Formulas Community <salt-formulas@freelists.org>
+Source: https://github.com/salt-formulas/salt-formula-fluentd
+
+Files: *
+Copyright: 2017 Mirantis et al.
+License: Apache-2.0
+ Copyright (C) 2017 Mirantis et al.
+ .
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ .
+ On a Debian system you can find a copy of this license in
+ /usr/share/common-licenses/Apache-2.0.
diff --git a/debian/docs b/debian/docs
new file mode 100644
index 0000000..a1320b1
--- /dev/null
+++ b/debian/docs
@@ -0,0 +1 @@
+README.rst
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 0000000..abde6ef
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,5 @@
+#!/usr/bin/make -f
+
+%:
+ dh $@
+
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 0000000..89ae9db
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (native)
diff --git a/fluentd/agent.sls b/fluentd/agent.sls
new file mode 100644
index 0000000..a45d5b6
--- /dev/null
+++ b/fluentd/agent.sls
@@ -0,0 +1,177 @@
+{% from "fluentd/map.jinja" import fluentd with context %}
+{%- if fluentd.get('enabled', False) %}
+
+fluentd_packages_agent:
+ pkg.installed:
+ - names: {{ fluentd.pkgs }}
+
+fluentd_gems_agent:
+ gem.installed:
+ - names: {{ fluentd.gems }}
+ - gem_bin: {{ fluentd.gem_path }}
+ - require:
+ - pkg: fluentd_packages_agent
+
+fluentd_config_d_dir:
+ file.directory:
+ - name: {{ fluentd.dir.config }}/config.d
+ - makedirs: True
+ - mode: 755
+ - require:
+ - pkg: fluentd_packages_agent
+
+fluentd_config_service:
+ file.managed:
+ - name: /etc/default/td-agent
+ - source: salt://fluentd/files/default-td-agent
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - context:
+ fluentd: {{ fluentd }}
+
+fluentd_config_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/td-agent.conf
+ - source: salt://fluentd/files/td-agent.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - context:
+ fluentd: {{ fluentd }}
+
+fluentd_grok_pattern_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/global.grok
+ - source: salt://fluentd/files/global.grok
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - context:
+ fluentd: {{ fluentd }}
+
+{%- set fluentd_config = fluentd.get('config', {}) %}
+{%- for name,values in fluentd_config.get('input', {}).iteritems() %}
+
+input_{{ name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/input-{{ name }}.conf
+ - source:
+ - salt://fluentd/files/input/_generate.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ name: {{ name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+{%- for name,values in fluentd_config.get('filter', {}).iteritems() %}
+
+filter_{{ name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/filter-{{ name }}.conf
+ - source:
+ - salt://fluentd/files/filter/_generate.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ name: {{ name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+{%- for name,values in fluentd_config.get('match', {}).iteritems() %}
+
+match_{{ name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/match-{{ name }}.conf
+ - source:
+ - salt://fluentd/files/match/_generate.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ name: {{ name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+{%- for label_name,values in fluentd_config.get('label', {}).iteritems() %}
+
+label_{{ label_name }}_agent:
+ file.managed:
+ - name: {{ fluentd.dir.config }}/config.d/label-{{ label_name }}.conf
+ - source:
+ - salt://fluentd/files/label.conf
+ - user: root
+ - group: root
+ - mode: 644
+ - template: jinja
+ - require:
+ - pkg: fluentd_packages_agent
+ - file: fluentd_config_d_dir
+ - watch_in:
+ - service: fluentd_service_agent
+ - defaults:
+ label_name: {{ label_name }}
+{%- if values is mapping %}
+ values: {{ values | yaml }}
+{%- else %}
+ values: {}
+{%- endif %}
+
+{%- endfor %}
+
+fluentd_service_agent:
+ service.running:
+ - name: {{ fluentd.service_name }}
+ - enable: True
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: fluentd_config_agent
+
+{%- endif %}
diff --git a/fluentd/files/default-td-agent b/fluentd/files/default-td-agent
new file mode 100644
index 0000000..3f53213
--- /dev/null
+++ b/fluentd/files/default-td-agent
@@ -0,0 +1,2 @@
+TD_AGENT_USER={{ fluentd.get('user', 'root') }}
+TD_AGENT_GROUP={{ fluentd.get('group', 'root') }}
diff --git a/fluentd/files/filter/_generate.conf b/fluentd/files/filter/_generate.conf
new file mode 100644
index 0000000..67c0c70
--- /dev/null
+++ b/fluentd/files/filter/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Filter {{ name }}
+{%- if values.get('enabled', True) %}
+{% include 'fluentd/files/filter/' + values.get('type') + '.conf' %}
+{%- endif %}
+{%- endfor %}
diff --git a/fluentd/files/filter/parser.conf b/fluentd/files/filter/parser.conf
new file mode 100644
index 0000000..0579e32
--- /dev/null
+++ b/fluentd/files/filter/parser.conf
@@ -0,0 +1,11 @@
+<filter {{ values.tag }}>
+ @type parser
+ key_name {{ values.key_name }}
+ reserve_data {{ values.get('reserve_data', true) | lower }}
+ <parse>
+{%- with values=values.get('parser') %}
+ @type {{ values.get('type') }}
+{% include 'fluentd/files/parser/' + values.get('type') + '.conf' %}
+{%- endwith %}
+ </parse>
+</filter>
diff --git a/fluentd/files/filter/prometheus.conf b/fluentd/files/filter/prometheus.conf
new file mode 100644
index 0000000..0496d8d
--- /dev/null
+++ b/fluentd/files/filter/prometheus.conf
@@ -0,0 +1,21 @@
+<filter {{ values.tag }}>
+ @type prometheus
+ {%- if values.get('label') %}
+ <labels>
+ {%- for label in values.label %}
+ {%- if label.type == 'variable' %}
+ {{ label.name }} {%raw%}${{%-endraw%}{{ label.value }}{%raw%}}{%-endraw%}
+ {%- else %}
+ {{ label.name }} {{ label.value }}
+ {%- endif %}
+ {%- endfor %}
+ </labels>
+ {%- endif %}
+ {%- for metric in values.metric %}
+ <metric>
+ name {{ metric.name }}
+ type {{ metric.type }}
+ desc {{ metric.desc }}
+ </metric>
+ {%- endfor %}
+</filter>
diff --git a/fluentd/files/filter/record_transformer.conf b/fluentd/files/filter/record_transformer.conf
new file mode 100644
index 0000000..21f2198
--- /dev/null
+++ b/fluentd/files/filter/record_transformer.conf
@@ -0,0 +1,16 @@
+<filter {{ values.tag }}>
+ @type record_transformer
+ {%- if values.get('enable_ruby') %}
+ enable_ruby
+ {%- endif %}
+ {%- if values.get('remove_keys') %}
+ remove_keys {{ values.remove_keys }}
+ {%- endif %}
+ {%- if values.get('record') %}
+ <record>
+ {%- for record in values.record %}
+ {{ record.name }} ${ {{ record.value }} }
+ {%- endfor %}
+ </record>
+ {%- endif %}
+</filter>
diff --git a/fluentd/files/global.grok b/fluentd/files/global.grok
new file mode 100644
index 0000000..1ec0c73
--- /dev/null
+++ b/fluentd/files/global.grok
@@ -0,0 +1,15 @@
+# Openstack services logging
+
+REQUEST_ID req-[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+DASHLESS_UUID [A-Fa-f0-9]{32}
+
+REQUEST (?:(%{REQUEST_ID:request_id}|)%{SPACE}(%{DASHLESS_UUID:user_id}|)%{SPACE}(%{DASHLESS_UUID:tenant_id}|)%{SPACE}%{DATA})
+
+OPENSTACK_GENERIC (?:%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{NUMBER:pid}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NOTSPACE:api}%{SPACE}\[(%{REQUEST})\]%{SPACE}%{GREEDYDATA:message})
+
+KEYSTONE_APACHE_ACCESS (?:%{IPORHOST:clientip} %{USER} %{USER} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-))
+
+KEYSTONE_APACHE_ERROR (?:%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{GREEDYDATA:message})
+
+MYSQLERROR (?:%{NUMBER:pid}%{SPACE}%{TIME:timestamp}%{SPACE}%{GREEDYDATA:message})
\ No newline at end of file
diff --git a/fluentd/files/input/_general.conf b/fluentd/files/input/_general.conf
new file mode 100644
index 0000000..9b221c3
--- /dev/null
+++ b/fluentd/files/input/_general.conf
@@ -0,0 +1,8 @@
+ @type {{ values.type }}
+{%- set label_name = values.get('label') or label_name %}
+{%- if label_name is defined %}
+ @label @{{ label_name }}
+{%- endif %}
+{%- if values.get('tag') %}
+ tag {{ values.tag }}
+{%- endif %}
\ No newline at end of file
diff --git a/fluentd/files/input/_generate.conf b/fluentd/files/input/_generate.conf
new file mode 100644
index 0000000..91c1b52
--- /dev/null
+++ b/fluentd/files/input/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Input {{ name }}
+{%- if values.get('enabled', True) %}
+{% include ['fluentd/files/input/' + values.get('type') + '.conf', 'fluentd/files/input/generic.conf'] %}
+{%- endif %}
+{%- endfor %}
\ No newline at end of file
diff --git a/fluentd/files/input/forward.conf b/fluentd/files/input/forward.conf
new file mode 100644
index 0000000..95e3548
--- /dev/null
+++ b/fluentd/files/input/forward.conf
@@ -0,0 +1,5 @@
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+ port {{ values.port }}
+ bind {{ values.bind }}
+</source>
diff --git a/fluentd/files/input/generic.conf b/fluentd/files/input/generic.conf
new file mode 100644
index 0000000..fe840a7
--- /dev/null
+++ b/fluentd/files/input/generic.conf
@@ -0,0 +1,3 @@
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+</source>
diff --git a/fluentd/files/input/systemd.conf b/fluentd/files/input/systemd.conf
new file mode 100644
index 0000000..92556c1
--- /dev/null
+++ b/fluentd/files/input/systemd.conf
@@ -0,0 +1,23 @@
+{%- set plugin_parameters = ['pos_file', 'read_from_head', 'strip_underscores'] %}
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+ path {{ values.get("path", "/run/log/journal") }}
+{%- if values.get("filters") %}
+ filters [{%- for key, value in values.filters.iteritems() %}{"{{ key }}": "{{ value }}"}{{ ", " if not loop.last else '' }}{%- endfor %}]
+{%- endif %}
+{%- for parameter in plugin_parameters %}
+ {%- if values.get(parameter) %}
+ {{ parameter }} {{ values.get(parameter) }}
+ {%- endif %}
+{%- endfor %}
+ <entry>
+{%- if values.get("entry") %}
+ field_map { {%- for key, value in values.entry.field_map.iteritems() %}"{{ key }}": "{{ value }}"{{ ", " if not loop.last else '' }}{%- endfor %} }
+ {%- for parameter in ['field_map_strict', 'fields_strip_underscores', 'fields_lowercase', ] %}
+ {%- if values.entry.get(parameter) %}
+ {{ parameter }} {{ values.entry.get(parameter) | lower }}
+ {%- endif %}
+ {%- endfor %}
+ </entry>
+{%- endif %}
+</source>
diff --git a/fluentd/files/input/tail.conf b/fluentd/files/input/tail.conf
new file mode 100644
index 0000000..b836521
--- /dev/null
+++ b/fluentd/files/input/tail.conf
@@ -0,0 +1,25 @@
+{%- set plugin_parameters = ['refresh_interval', 'limit_recently_modified', 'encoding', 'from_encoding', 'read_lines_limit', 'multiline_flush_interval', 'pos_file', 'format', 'path_key', 'rotate_wait'] %}
+{%- set bool_plugin_parameters = ['read_from_head', 'enable_watch_timer', 'skip_refresh_on_startup', 'ignore_repeated_permission_error'] %}
+<source>
+{% include 'fluentd/files/input/_general.conf' %}
+ path {{ values.path }}
+{%- for parameter in plugin_parameters %}
+ {%- if values.get(parameter) %}
+ {{ parameter }} {{values.get(parameter)}}
+ {%- endif %}
+{%- endfor %}
+{%- for parameter in bool_plugin_parameters %}
+ {%- if values.get(parameter) %}
+ {{ parameter }} {{ values.get(parameter) | lower }}
+ {%- endif %}
+{%- endfor %}
+{%- if values.get('exclude_path') %}{# Replace 'for-if-else' with 'tojson' after updating to jinja 2.9 #}
+ exclude_path [{%- for path in values.exclude_path %}"{{ path }}"{{ ", " if not loop.last else '' }} {%- endfor %}]
+{%- endif %}
+ <parse>
+{%- with values=values.get('parser') %}
+ @type {{ values.get('type') }}
+{% include 'fluentd/files/parser/' + values.get('type') + '.conf' %}
+{%- endwith %}
+ </parse>
+</source>
\ No newline at end of file
diff --git a/fluentd/files/label.conf b/fluentd/files/label.conf
new file mode 100644
index 0000000..1285a18
--- /dev/null
+++ b/fluentd/files/label.conf
@@ -0,0 +1,20 @@
+# Label {{ label_name }}
+{%- if values.get('input') %}
+{%- with values=values.get('input') %}
+{%- include 'fluentd/files/input/_generate.conf' %}
+{%- endwith %}
+{%- endif %}
+
+<label @{{ label_name }}>
+{%- if values.get('filter') %}
+{%- with values=values.get('filter') %}
+{%- include 'fluentd/files/filter/_generate.conf' %}
+{%- endwith %}
+{%- endif %}
+
+{%- if values.get('match') %}
+{%- with values=values.get('match') %}
+{%- include 'fluentd/files/match/_generate.conf' %}
+{%- endwith %}
+{%- endif %}
+</label>
\ No newline at end of file
diff --git a/fluentd/files/match/_generate.conf b/fluentd/files/match/_generate.conf
new file mode 100644
index 0000000..3e9bc37
--- /dev/null
+++ b/fluentd/files/match/_generate.conf
@@ -0,0 +1,6 @@
+{%- for name, values in values.iteritems() %}
+# Output {{ name }}
+{%- if values.get('enabled', True) %}
+{% include 'fluentd/files/match/' + values.get('type') + '.conf' %}
+{%- endif %}
+{%- endfor %}
diff --git a/fluentd/files/match/file.conf b/fluentd/files/match/file.conf
new file mode 100644
index 0000000..86e87ff
--- /dev/null
+++ b/fluentd/files/match/file.conf
@@ -0,0 +1,4 @@
+<match {{ values.tag }}>
+ @type file
+ path {{ values.path }}
+</match>
diff --git a/fluentd/files/match/relabel.conf b/fluentd/files/match/relabel.conf
new file mode 100644
index 0000000..88ca0c7
--- /dev/null
+++ b/fluentd/files/match/relabel.conf
@@ -0,0 +1,4 @@
+<match {{ values.tag }}>
+ @type relabel
+ @label @{{ values.label }}
+</match>
diff --git a/fluentd/files/match/rewrite_tag_filter.conf b/fluentd/files/match/rewrite_tag_filter.conf
new file mode 100644
index 0000000..b62f24f
--- /dev/null
+++ b/fluentd/files/match/rewrite_tag_filter.conf
@@ -0,0 +1,6 @@
+<match {{ values.tag }}>
+ @type rewrite_tag_filter
+ {%- for rule in values.rule %}
+ rewriterule{{ loop.index }} {{ rule.name }} {{ rule.regexp }} {{ rule.result }}
+ {%- endfor %}
+</match>
diff --git a/fluentd/files/parser/grok.conf b/fluentd/files/parser/grok.conf
new file mode 100644
index 0000000..421cc1a
--- /dev/null
+++ b/fluentd/files/parser/grok.conf
@@ -0,0 +1,9 @@
+ grok_failure_key grokfailure
+ {%- if values.get('custom_pattern_path') %}
+ custom_pattern_path {{ values.custom_pattern_path }}
+ {%- endif %}
+ {%- for grok_rule in values.get('rule', []) %}
+ <grok>
+ pattern {{ grok_rule.pattern }}
+ </grok>
+ {%- endfor %}
diff --git a/fluentd/files/parser/multiline.conf b/fluentd/files/parser/multiline.conf
new file mode 100644
index 0000000..7f1b3f5
--- /dev/null
+++ b/fluentd/files/parser/multiline.conf
@@ -0,0 +1,14 @@
+ time_key {{ values.get("time_key", "time") }}
+{%- if values.get('time_format') %}
+ time_format {{ values.time_format }}
+{%- endif %}
+ keep_time_key {{ values.get("keep_time_key", False) | json }}
+ format_firstline {{ values.format_firstline }}
+{%- if values.get("format") %}
+ format1 {{ values.format }}
+{%- endif %}
+{%- if values.get("formats") %}
+ {%- for format in values.formats %}
+ format{{ loop.index }} {{ format }}
+ {%- endfor %}
+{%- endif %}
\ No newline at end of file
diff --git a/fluentd/files/parser/multiline_grok.conf b/fluentd/files/parser/multiline_grok.conf
new file mode 100644
index 0000000..dcfabc5
--- /dev/null
+++ b/fluentd/files/parser/multiline_grok.conf
@@ -0,0 +1,10 @@
+ grok_failure_key grokfailure
+{%- if values.get('custom_pattern_path') %}
+ custom_pattern_path {{ values.custom_pattern_path }}
+{%- endif %}
+ multiline_start_regexp {{ values.multiline_start_regexp }}
+{%- for grok_rule in values.get('rule', []) %}
+ <grok>
+ pattern {{ grok_rule.pattern }}
+ </grok>
+{%- endfor %}
\ No newline at end of file
diff --git a/fluentd/files/parser/regexp.conf b/fluentd/files/parser/regexp.conf
new file mode 100644
index 0000000..14b3dbe
--- /dev/null
+++ b/fluentd/files/parser/regexp.conf
@@ -0,0 +1,2 @@
+ expression {{ values.format }}
+ time_format {{ values.time_format }}
diff --git a/fluentd/files/td-agent.conf b/fluentd/files/td-agent.conf
new file mode 100644
index 0000000..e69740c
--- /dev/null
+++ b/fluentd/files/td-agent.conf
@@ -0,0 +1 @@
+@include {{ fluentd.dir.config }}/config.d/*.conf
diff --git a/fluentd/init.sls b/fluentd/init.sls
new file mode 100644
index 0000000..2862a76
--- /dev/null
+++ b/fluentd/init.sls
@@ -0,0 +1,6 @@
+{%- if pillar.fluentd %}
+include:
+ {%- if pillar.fluentd is defined %}
+ - fluentd.agent
+ {%- endif %}
+{%- endif %}
diff --git a/fluentd/map.jinja b/fluentd/map.jinja
new file mode 100644
index 0000000..2531114
--- /dev/null
+++ b/fluentd/map.jinja
@@ -0,0 +1,31 @@
+{% set fluentd = salt['grains.filter_by']({
+ 'Debian': {
+ 'user': 'root',
+ 'group': 'root',
+ 'pkgs': ['td-agent', 'build-essential', 'ruby-dev'],
+ 'gems': ['fluent-plugin-prometheus', 'fluent-plugin-grok-parser', 'fluent-plugin-systemd'],
+ 'gem_path': 'td-agent-gem',
+ 'service_name': 'td-agent',
+ 'dir': {
+ 'config': '/etc/td-agent'
+ },
+ 'config': {
+ },
+ },
+}, merge=salt['pillar.get']('fluentd')) %}
+
+{# Collect configuration from */meta/fluentd.yml #}
+{%- set fluentd_grains = {'config': {'input': {}, 'filter': {}, 'match': {}, 'label': {}}} %}
+{%- for service_name, service in pillar.items() %}
+ {%- if service.get('_support', {}).get('fluentd', {}).get('enabled', False) %}
+ {%- set grains_fragment_file = service_name+'/meta/fluentd.yml' %}
+ {%- macro load_grains_file() %}{% include grains_fragment_file ignore missing %}{% endmacro %}
+ {%- set grains_yaml = load_grains_file()|load_yaml %}
+ {%- if grains_yaml is mapping %}
+ {%- set fluentd_grains = salt['grains.filter_by']({'default': fluentd_grains}, merge=grains_yaml) %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+
+{# Deep-merge the service configuration with the pillar data #}
+{%- do salt['defaults.merge'](fluentd, fluentd_grains) %}
diff --git a/init.sls b/init.sls
deleted file mode 100755
index a6ebabe..0000000
--- a/init.sls
+++ /dev/null
@@ -1,4 +0,0 @@
-include:
-{%- if pillar.fluentd.get("server", {'enabled':False}).enabled %}
-- fluentd.server
-{%- endif %}
\ No newline at end of file
diff --git a/map.jinja b/map.jinja
deleted file mode 100755
index e29b050..0000000
--- a/map.jinja
+++ /dev/null
@@ -1,13 +0,0 @@
-
-{% set server = salt['grains.filter_by']({
- 'Debian': {
- 'pkgs': ['td-agent'],
- 'service': 'td-agent',
- 'config_path': '/etc/td-agent/td-agent.conf',
- },
- 'RedHat': {
- 'pkgs': ['td-agent'],
- 'service': 'td-agent',
- 'config_path': '/etc/td-agent/td-agent.conf',
- },
-}, merge=salt['pillar.get']('fluentd:server')) %}
\ No newline at end of file
diff --git a/metadata.yml b/metadata.yml
new file mode 100644
index 0000000..da0562d
--- /dev/null
+++ b/metadata.yml
@@ -0,0 +1,3 @@
+name: "fluentd"
+version: "1.0"
+source: "https://github.com/salt-formulas/salt-formula-fluentd"
diff --git a/metadata/service/agent/single.yml b/metadata/service/agent/single.yml
new file mode 100644
index 0000000..d9824e6
--- /dev/null
+++ b/metadata/service/agent/single.yml
@@ -0,0 +1,9 @@
+applications:
+ - fluentd
+classes:
+ - service.fluentd.support
+parameters:
+ _param:
+ fluentd:
+ agent:
+ enabled: true
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
new file mode 100644
index 0000000..88522d2
--- /dev/null
+++ b/metadata/service/support.yml
@@ -0,0 +1,3 @@
+parameters:
+ fluentd:
+ _support:
diff --git a/server.sls b/server.sls
deleted file mode 100755
index 1f00e57..0000000
--- a/server.sls
+++ /dev/null
@@ -1,59 +0,0 @@
-{% from "fluentd/map.jinja" import server with context %}
-
-fluentd_packages:
- pkg.installed:
- - names:
- - curl
- - libcurl-devel
-
-fluentd_install:
- cmd.run:
- - names:
- - curl -L http://toolbelt.treasuredata.com/sh/install-redhat.sh | sh
- - touch /root/fluentd_installed
- - cwd: /root
- - unless: "[ -f /root/fluentd_installed ]"
- require:
- - pkg: fluentd_packages
-
-{%- for plugin in server.get("plugins", []) %}
-fluentd_install_plugin_{{ plugin }}:
- cmd.run:
- - names:
- - /usr/lib64/fluent/ruby/bin/fluent-gem install plugin {{ plugin }}
- - touch /root/{{ plugin }}_installed
- - cwd: /root
- - unless: "[ -f /root/{{ plugin }}_installed ]"
- require:
- - pkg: fluentd_packages
-{% endfor %}
-
-{{ server.config_path }}:
- file.managed:
- - source: salt://fluentd/conf/td-agent.conf
- - template: jinja
- - require:
- - cmd: fluentd_install
-
-fluentd_service:
- service.running:
- - name: {{ server.service }}
- - enable: True
- - watch:
- - file: {{ server.config_path }}
-
-{#
-fluentd_repo:
- pkgrepo.managed:
- - human_name: TreasureData
- - comments:
- - '#http://packages.treasure-data.com/debian/RPM-GPG-KEY-td-agent'
- - name: TreasureData
- - baseurl: http://packages.treasure-data.com/redhat/\$basearch
- - file: /etc/apt/sources.list.d/fluentd.list
- - key_url: salt://fluentd/conf/treasure-data.gpg
- - gpgcheck: 1
-
-
- - deb http://packages.treasure-data.com/{{ grains.oscodename }}/ {{ grains.oscodename }} contrib
-#}
\ No newline at end of file
diff --git a/tests/pillar/agent.sls b/tests/pillar/agent.sls
new file mode 100644
index 0000000..10e4cc1
--- /dev/null
+++ b/tests/pillar/agent.sls
@@ -0,0 +1,34 @@
+fluentd:
+ enabled: true
+ config:
+ label:
+ monitoring:
+ filter:
+ parse_log:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: parser
+ reserve_data: true
+ key_name: log
+ parser:
+ type: regexp
+ format: >-
+ /^time="(?<time>[^ ]*)" level=(?<severity>[a-zA-Z]*) msg="(?<message>.+?)"/
+ time_format: '%FT%TZ'
+ remove_log_key:
+ tag: 'docker.monitoring.{alertmanager,remote_storage_adapter,prometheus}.*'
+ type: record_transformer
+ remove_keys: log
+ match:
+ docker_log:
+ tag: 'docker.**'
+ type: file
+ path: /tmp/flow-docker.log
+docker:
+ host:
+ enabled: true
+ experimental: true
+ insecure_registries:
+ - 127.0.0.1
+ log:
+ engine: json-file
+ size: 50m
diff --git a/tests/pillar/repo_tdagent.sls b/tests/pillar/repo_tdagent.sls
new file mode 100644
index 0000000..e6b069e
--- /dev/null
+++ b/tests/pillar/repo_tdagent.sls
@@ -0,0 +1,8 @@
+linux:
+ system:
+ enabled: true
+ repo:
+ td-agent:
+ source: 'deb http://packages.treasuredata.com.s3.amazonaws.com/2/ubuntu/{{ grains.get('oscodename') }} {{ grains.get('oscodename') }} contrib'
+ key_url: 'http://packages.treasuredata.com.s3.amazonaws.com/GPG-KEY-td-agent'
+
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
new file mode 100755
index 0000000..f6d5e40
--- /dev/null
+++ b/tests/run_tests.sh
@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+
+set -e
+[ -n "$DEBUG" ] && set -x
+
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+METADATA=${CURDIR}/../metadata.yml
+FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
+FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
+
+## Overrideable parameters
+PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
+BUILDDIR=${BUILDDIR:-${CURDIR}/build}
+VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
+MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
+DEPSDIR=${BUILDDIR}/deps
+
+SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
+SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
+SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
+SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
+
+SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
+
+if [ "x${SALT_VERSION}" != "x" ]; then
+ PIP_SALT_VERSION="==${SALT_VERSION}"
+fi
+
+## Functions
+log_info() {
+ echo "[INFO] $*"
+}
+
+log_err() {
+ echo "[ERROR] $*" >&2
+}
+
+setup_virtualenv() {
+ log_info "Setting up Python virtualenv"
+ virtualenv $VENV_DIR
+ source ${VENV_DIR}/bin/activate
+ pip install salt${PIP_SALT_VERSION}
+ pip install reno
+}
+
+setup_mock_bin() {
+ # If some state requires a binary, a lightweight replacement for
+ # such binary can be put into MOCK_BIN_DIR for test purposes
+ if [ -d "${MOCK_BIN_DIR}" ]; then
+ PATH="${MOCK_BIN_DIR}:$PATH"
+ export PATH
+ fi
+}
+
+setup_pillar() {
+ [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
+ echo "base:" > ${SALT_PILLAR_DIR}/top.sls
+ for pillar in ${PILLARDIR}/*; do
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ state_name=$(basename ${pillar%.sls})
+ echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
+ done
+}
+
+setup_salt() {
+ [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
+ [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
+ [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
+
+ echo "base:" > ${SALT_FILE_DIR}/top.sls
+ for pillar in ${PILLARDIR}/*.sls; do
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ state_name=$(basename ${pillar%.sls})
+ echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
+ done
+
+ cat << EOF > ${SALT_CONFIG_DIR}/minion
+file_client: local
+cachedir: ${SALT_CACHE_DIR}
+verify_env: False
+minion_id_caching: False
+
+file_roots:
+ base:
+ - ${SALT_FILE_DIR}
+ - ${CURDIR}/..
+ - /usr/share/salt-formulas/env
+
+pillar_roots:
+ base:
+ - ${SALT_PILLAR_DIR}
+ - ${PILLARDIR}
+EOF
+}
+
+fetch_dependency() {
+ dep_name="$(echo $1|cut -d : -f 1)"
+ dep_source="$(echo $1|cut -d : -f 2-)"
+ dep_root="${DEPSDIR}/$(basename $dep_source .git)"
+ dep_metadata="${dep_root}/metadata.yml"
+
+ [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0
+ [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0
+
+ log_info "Fetching dependency $dep_name"
+ [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
+ git clone $dep_source ${DEPSDIR}/$(basename $dep_source .git)
+ ln -s ${dep_root}/${dep_name} ${SALT_FILE_DIR}/${dep_name}
+
+ METADATA="${dep_metadata}" install_dependencies
+}
+
+install_dependencies() {
+ grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
+ (python - | while read dep; do fetch_dependency "$dep"; done) << EOF
+import sys,yaml
+for dep in yaml.load(open('${METADATA}', 'ro'))['dependencies']:
+ print '%s:%s' % (dep["name"], dep["source"])
+EOF
+}
+
+clean() {
+ log_info "Cleaning up ${BUILDDIR}"
+ [ -d ${BUILDDIR} ] && rm -rf ${BUILDDIR} || exit 0
+}
+
+salt_run() {
+ [ -e ${VEN_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
+ salt-call ${SALT_OPTS} $*
+}
+
+prepare() {
+ [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
+
+ which salt-call || setup_virtualenv
+ setup_mock_bin
+ setup_pillar
+ setup_salt
+ install_dependencies
+}
+
+lint_releasenotes() {
+ echo "placeholder"
+ #reno lint ${CURDIR}/../
+}
+
+lint() {
+ lint_releasenotes
+}
+
+run() {
+ for pillar in ${PILLARDIR}/*.sls; do
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ state_name=$(basename ${pillar%.sls})
+ salt_run grains.set 'noservices' False force=True
+
+ echo "Checking state ${FORMULA_NAME}.${state_name} ..."
+ salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+
+ # Check that all files in 'meta' folder can be rendered using any valid pillar
+ for meta in `find ${FORMULA_META_DIR} -type f`; do
+ meta_name=$(basename ${meta})
+ echo "Checking meta ${meta_name} ..."
+ salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
+ || (log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1)
+ cat ${SALT_CACHE_DIR}/${meta_name}
+ done
+ done
+}
+
+real_run() {
+ for pillar in ${PILLARDIR}/*.sls; do
+ state_name=$(basename ${pillar%.sls})
+ salt_run --id=${state_name} state.sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+ done
+}
+
+_atexit() {
+ RETVAL=$?
+ trap true INT TERM EXIT
+
+ if [ $RETVAL -ne 0 ]; then
+ log_err "Execution failed"
+ else
+ log_info "Execution successful"
+ fi
+ return $RETVAL
+}
+
+## Main
+trap _atexit INT TERM EXIT
+
+case $1 in
+ clean)
+ clean
+ ;;
+ prepare)
+ prepare
+ ;;
+ lint)
+ lint
+ ;;
+ run)
+ run
+ ;;
+ real-run)
+ real_run
+ ;;
+ *)
+ prepare
+ lint
+ run
+ ;;
+esac