Implement the remote collectd service
This change refactors the collectd formula to be able to install
another collectd instance in charge of running the remote plugins.
diff --git a/collectd/_common.sls b/collectd/_common.sls
new file mode 100644
index 0000000..7725520
--- /dev/null
+++ b/collectd/_common.sls
@@ -0,0 +1,34 @@
+{%- from "collectd/map.jinja" import client with context %}
+
+{%- if grains.os == 'Ubuntu' and (grains.osrelease in ['10.04', '12.04']) %}
+
+collectd_repo:
+ pkgrepo.managed:
+ - human_name: Collectd
+ - ppa: nikicat/collectd
+ - file: /etc/apt/sources.list.d/collectd.list
+ - require_in:
+ - pkg: collectd_client_packages
+
+collectd_amqp_packages:
+ pkg.installed:
+ - names:
+ - librabbitmq0
+
+{%- endif %}
+
+collectd_client_packages:
+ pkg.installed:
+ - names: {{ client.pkgs }}
+
+
+/usr/lib/collectd-python:
+ file.recurse:
+ - source: salt://collectd/files/plugin
+
+collectd_client_grains_dir:
+ file.directory:
+ - name: /etc/salt/grains.d
+ - mode: 700
+ - makedirs: true
+ - user: root
diff --git a/collectd/_service.sls b/collectd/_service.sls
new file mode 100644
index 0000000..90426fc
--- /dev/null
+++ b/collectd/_service.sls
@@ -0,0 +1,150 @@
+{%- if client.enabled %}
+
+{{ client.service }}_client_conf_dir:
+ file.directory:
+ - name: {{ client.config_dir }}
+ - user: root
+ - mode: 750
+ - makedirs: true
+
+{{ client.service }}_client_conf_dir_clean:
+ file.directory:
+ - name: {{ client.config_dir }}
+ - clean: true
+
+{%- for plugin_name, plugin in plugins.iteritems() %}
+
+{%- if plugin.get('plugin', 'native') not in ['python'] %}
+
+{{ client.config_dir }}/{{ plugin_name }}.conf:
+ file.managed:
+ {%- if plugin.template is defined %}
+ - source: salt://{{ plugin.template }}
+ - template: jinja
+ - defaults:
+ plugin: {{ plugin|yaml }}
+ {%- else %}
+ - contents: "<LoadPlugin {{ plugin.plugin }}>\n Globals false\n</LoadPlugin>\n"
+ {%- endif %}
+ - user: root
+ - mode: 660
+ - require:
+ - file: {{ client.service }}_client_conf_dir
+ - require_in:
+ - file: {{ client.service }}_client_conf_dir_clean
+ - watch_in:
+ - service: {{ client.service }}_service
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- if client.file_logging %}
+
+{{ client.config_dir }}/00_collectd_logfile.conf:
+ file.managed:
+ - source: salt://collectd/files/collectd_logfile.conf
+ - template: jinja
+ - defaults:
+ service_name: {{ client.service }}
+ - user: root
+ - group: root
+ - mode: 660
+ - watch_in:
+ - service: {{ client.service }}_service
+ - require:
+ - file: {{ client.service }}_client_conf_dir
+ - require_in:
+ - file: {{ client.service }}_client_conf_dir_clean
+
+{%- endif %}
+
+{{ client.config_dir }}/collectd_python.conf:
+ file.managed:
+ - source: salt://collectd/files/collectd_python.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 660
+ - defaults:
+ plugin: {{ plugins|yaml }}
+ - watch_in:
+ - service: {{ client.service }}_service
+ - require:
+ - file: {{ client.service }}_client_conf_dir
+ - require_in:
+ - file: {{ client.service }}_client_conf_dir_clean
+
+{{ client.config_file }}:
+ file.managed:
+ - source: salt://collectd/files/collectd.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 640
+ - defaults:
+ plugin: {{ plugins|yaml }}
+ client: {{ client|yaml }}
+ - watch_in:
+ - service: {{ client.service }}_service
+ - require:
+ - file: {{ client.service }}_client_conf_dir
+ - require_in:
+ - file: {{ client.service }}_client_conf_dir_clean
+
+{%- set network_backend = {} %}
+{%- for backend_name, backend in client.backend.iteritems() %}
+
+{%- if backend.engine not in ['network'] %}
+
+{{ client.config_dir }}/collectd_writer_{{ backend_name }}.conf:
+ file.managed:
+ - source: salt://collectd/files/backend/{{ backend.engine }}.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 660
+ - defaults:
+ backend: {{ backend|yaml }}
+ - watch_in:
+ - service: {{ client.service }}_service
+ - require:
+ - file: {{ client.service }}_client_conf_dir
+ - require_in:
+ - file: {{ client.service }}_client_conf_dir_clean
+
+{%- else %}
+
+{%- set network_backend = salt['grains.filter_by']({'default': network_backend}, merge={backend_name: backend}) %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- if network_backend|length > 0 %}
+
+{{ client.config_dir }}/collectd_writer_network.conf:
+ file.managed:
+ - source: salt://collectd/files/backend/network.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 660
+ - defaults:
+ backend: {{ backend|yaml }}
+ - watch_in:
+ - service: {{ client.service }}_service
+ - require:
+ - file: {{ client.service }}_client_conf_dir
+ - require_in:
+ - file: {{ client.service }}_client_conf_dir_clean
+
+{%- endif %}
+
+
+{{ client.service }}_service:
+ service.running:
+ - name: {{ client.service }}
+ - enable: true
+
+{%- endif %}
diff --git a/collectd/client.sls b/collectd/client.sls
index 5937f04..eca7071 100644
--- a/collectd/client.sls
+++ b/collectd/client.sls
@@ -1,26 +1,8 @@
{%- from "collectd/map.jinja" import client with context %}
{%- if client.enabled %}
-{%- if grains.os == 'Ubuntu' and (grains.osrelease in ['10.04', '12.04']) %}
-
-collectd_repo:
- pkgrepo.managed:
- - human_name: Collectd
- - ppa: nikicat/collectd
- - file: /etc/apt/sources.list.d/collectd.list
- - require_in:
- - pkg: collectd_client_packages
-
-collectd_amqp_packages:
- pkg.installed:
- - names:
- - librabbitmq0
-
-{%- endif %}
-
-collectd_client_packages:
- pkg.installed:
- - names: {{ client.pkgs }}
+include:
+- collectd._common
/etc/collectd:
file.directory:
@@ -30,31 +12,6 @@
- require:
- pkg: collectd_client_packages
-collectd_client_conf_dir:
- file.directory:
- - name: {{ client.config_dir }}
- - user: root
- - mode: 750
- - makedirs: true
- - require:
- - pkg: collectd_client_packages
-
-collectd_client_conf_dir_clean:
- file.directory:
- - name: {{ client.config_dir }}
- - clean: true
-
-collectd_client_grains_dir:
- file.directory:
- - name: /etc/salt/grains.d
- - mode: 700
- - makedirs: true
- - user: root
-
-/usr/lib/collectd-python:
- file.recurse:
- - source: salt://collectd/files/plugin
-
{%- set service_grains = {'collectd': {'remote_plugin': {}, 'local_plugin': {}}} %}
{%- for service_name, service in pillar.items() %}
@@ -71,22 +28,6 @@
{%- endif %}
{%- endfor %}
-{%- set remote_plugin = {} %}
-
-{%- if client.remote_collector %}
-
-{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
-
-{%- if node_grains.collectd is defined %}
-
-{%- set remote_plugin = salt['grains.filter_by']({'default': remote_plugin}, merge=node_grains.collectd.get('remote_plugin', {})) %}
-
-{%- endif %}
-
-{%- endfor %}
-
-{%- endif %}
-
collectd_client_grain:
file.managed:
- name: /etc/salt/grains.d/collectd
@@ -108,168 +49,7 @@
- watch:
- file: collectd_client_grain
-{%- for plugin_name, plugin in service_grains.collectd.local_plugin.iteritems() %}
-
-{%- if plugin.get('plugin', 'native') not in ['python'] %}
-
-{{ client.config_dir }}/{{ plugin_name }}.conf:
- file.managed:
- {%- if plugin.template is defined %}
- - source: salt://{{ plugin.template }}
- - template: jinja
- - defaults:
- plugin: {{ plugin|yaml }}
- {%- else %}
- - contents: "<LoadPlugin {{ plugin.plugin }}>\n Globals false\n</LoadPlugin>\n"
- {%- endif %}
- - user: root
- - mode: 660
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
- - watch_in:
- - service: collectd_service
-
-{%- endif %}
-
-{%- endfor %}
-
-{%- if client.remote_collector %}
-
-{%- for plugin_name, plugin in remote_plugin.iteritems() %}
-
-{%- if plugin.get('plugin', 'native') not in ['python'] %}
-
-{{ client.config_dir }}/{{ plugin_name }}.conf:
- file.managed:
- {%- if plugin.template is defined %}
- - source: salt://{{ plugin.template }}
- - template: jinja
- - defaults:
- plugin: {{ plugin|yaml }}
- {%- else %}
- - contents: "<LoadPlugin {{ plugin.plugin }}>\n Globals false\n</LoadPlugin>\n"
- {%- endif %}
- - user: root
- - mode: 660
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
- - watch_in:
- - service: collectd_service
-
-{%- endif %}
-
-{%- endfor %}
-
-{%- endif %}
-
-{%- if client.file_logging %}
-
-/etc/collectd/conf.d/00_collectd_logfile.conf:
- file.managed:
- - source: salt://collectd/files/collectd_logfile.conf
- - user: root
- - group: root
- - mode: 660
- - watch_in:
- - service: collectd_service
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
-
-{%- endif %}
-
-/etc/collectd/conf.d/collectd_python.conf:
- file.managed:
- - source: salt://collectd/files/collectd_python.conf
- - template: jinja
- - user: root
- - group: root
- - mode: 660
- - defaults:
- local_plugin: {{ service_grains.collectd.local_plugin|yaml }}
- remote_plugin: {{ remote_plugin|yaml }}
- - watch_in:
- - service: collectd_service
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
-
-/etc/collectd/filters.conf:
- file.managed:
- - source: salt://collectd/files/filters.conf
- - template: jinja
- - user: root
- - group: root
- - mode: 660
- - watch_in:
- - service: collectd_service
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
-
-/etc/collectd/thresholds.conf:
- file.managed:
- - source: salt://collectd/files/thresholds.conf
- - template: jinja
- - user: root
- - group: root
- - mode: 660
- - watch_in:
- - service: collectd_service
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
-
-{{ client.config_file }}:
- file.managed:
- - source: salt://collectd/files/collectd.conf
- - template: jinja
- - user: root
- - group: root
- - mode: 640
- - defaults:
- service_grains: {{ service_grains|yaml }}
- remote_plugin: {{ remote_plugin|yaml }}
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
- - watch_in:
- - service: collectd_service
-
-{%- for backend_name, backend in client.backend.iteritems() %}
-
-{{ client.config_dir }}/collectd_writer_{{ backend_name }}.conf:
- file.managed:
- - source: salt://collectd/files/backend/{{ backend.engine }}.conf
- - template: jinja
- - user: root
- - group: root
- - mode: 660
- - defaults:
- backend_name: "{{ backend_name }}"
- - require:
- - file: collectd_client_conf_dir
- - require_in:
- - file: collectd_client_conf_dir_clean
- - watch_in:
- - service: collectd_service
-
-{%- endfor %}
-
-collectd_service:
- service.running:
- - name: collectd
- - enable: true
- - require:
- - pkg: collectd_client_packages
+{%- set plugins = service_grains.collectd.local_plugin %}
+{%- include "collectd/_service.sls" %}
{%- endif %}
diff --git a/collectd/files/backend/amqp.conf b/collectd/files/backend/amqp.conf
index 606f446..ed2f968 100644
--- a/collectd/files/backend/amqp.conf
+++ b/collectd/files/backend/amqp.conf
@@ -1,4 +1,3 @@
-{%- set backend = salt['pillar.get']('collectd:client:backend:'+backend_name) %}
<LoadPlugin amqp>
Globals false
</LoadPlugin>
diff --git a/collectd/files/backend/carbon.conf b/collectd/files/backend/carbon.conf
index d39f4a4..7758f97 100644
--- a/collectd/files/backend/carbon.conf
+++ b/collectd/files/backend/carbon.conf
@@ -1,4 +1,3 @@
-{%- set backend = salt['pillar.get']('collectd:client:backend:'+backend_name) %}
<LoadPlugin write_graphite>
Globals false
</LoadPlugin>
diff --git a/collectd/files/backend/http.conf b/collectd/files/backend/http.conf
index 66401be..752df40 100644
--- a/collectd/files/backend/http.conf
+++ b/collectd/files/backend/http.conf
@@ -1,4 +1,3 @@
-{%- set backend = salt['pillar.get']('collectd:client:backend:'+backend_name) %}
<LoadPlugin write_http>
Globals false
</LoadPlugin>
diff --git a/collectd/files/backend/network.conf b/collectd/files/backend/network.conf
index 08fcd63..e97c530 100644
--- a/collectd/files/backend/network.conf
+++ b/collectd/files/backend/network.conf
@@ -1,9 +1,8 @@
-{%- from "collectd/map.jinja" import client with context %}
<LoadPlugin network>
Globals false
</LoadPlugin>
-{%- for backend_name in client.backend.iteritems() %}
+{%- for _, backend in client.backend.iteritems() %}
<Plugin network>
{%- if backend.mode == 'client' %}
Server "{{ backend.host }}" "{{ backend.port }}"
diff --git a/collectd/files/collectd.conf b/collectd/files/collectd.conf
index c839530..d815eb1 100644
--- a/collectd/files/collectd.conf
+++ b/collectd/files/collectd.conf
@@ -1,4 +1,3 @@
-{%- from "collectd/map.jinja" import client with context %}
{%- from "linux/map.jinja" import system with context %}
# Config file for collectd(1).
@@ -46,10 +45,12 @@
# accessed. #
##############################################################################
+{%- if client.syslog_logging %}
LoadPlugin syslog
<Plugin syslog>
LogLevel info
</Plugin>
+{%- endif %}
##############################################################################
# LoadPlugin section #
@@ -883,23 +884,14 @@
#</Plugin>
{%- if client.file_logging %}
-Include "/etc/collectd/conf.d/00_collectd_logfile.conf"
+Include "{{ client.config_dir }}/00_collectd_logfile.conf"
{%- endif %}
-{%- for plugin_name, plugin in service_grains.collectd.local_plugin.iteritems() %}
+{%- for plugin_name, plugin in plugin.iteritems() %}
{%- if plugin.get('plugin', 'native') not in ['python'] %}
Include "{{ client.config_dir }}/{{ plugin_name }}.conf"
{%- endif %}
{%- endfor %}
-{%- if client.remote_collector %}
-{%- for plugin_name, plugin in remote_plugin.iteritems() %}
-{%- if plugin.get('plugin', 'native') not in ['python'] %}
-{%- endif %}
-{%- endfor %}
-{%- endif %}
Include "{{ client.config_dir }}/collectd_python.conf"
{%- for backend_name, backend in client.backend.iteritems() %}
Include "{{ client.config_dir }}/collectd_writer_{{ backend_name }}.conf"
{%- endfor %}
-
-Include "/etc/collectd/filters.conf"
-Include "/etc/collectd/thresholds.conf"
diff --git a/collectd/files/collectd_logfile.conf b/collectd/files/collectd_logfile.conf
index 55bfbff..6d4fb6e 100644
--- a/collectd/files/collectd_logfile.conf
+++ b/collectd/files/collectd_logfile.conf
@@ -4,7 +4,7 @@
<Plugin logfile>
LogLevel warning
- File "/var/log/collectd.log"
+ File "/var/log/{{ service_name }}.log"
Timestamp true
PrintSeverity false
</Plugin>
diff --git a/collectd/files/collectd_python.conf b/collectd/files/collectd_python.conf
index 4d7cef8..53ad6f4 100644
--- a/collectd/files/collectd_python.conf
+++ b/collectd/files/collectd_python.conf
@@ -1,4 +1,3 @@
-{%- from "collectd/map.jinja" import client with context -%}
<LoadPlugin python>
Globals false
</LoadPlugin>
@@ -8,18 +7,11 @@
LogTraces false
Interactive false
- {%- for plugin_name, plugin in local_plugin.iteritems() %}
+ {%- for plugin_name, plugin in plugin.iteritems() %}
{%- if plugin.get('plugin', 'native') == 'python' %}
{% include plugin.template %}
{%- endif %}
- {%- endfor %}
- {%- if client.remote_collector %}
- {%- for plugin_name, plugin in remote_plugin.iteritems() %}
- {%- if plugin.get('plugin', 'native') == 'python' %}
- {% include plugin.template %}
- {%- endif %}
{%- endfor %}
- {%- endif %}
</Plugin>
diff --git a/collectd/files/collectd_systemd.service b/collectd/files/collectd_systemd.service
new file mode 100644
index 0000000..88c61d8
--- /dev/null
+++ b/collectd/files/collectd_systemd.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Statistics collection and monitoring daemon
+After=local-fs.target network.target
+Requires=local-fs.target network.target
+ConditionPathExists={{ config_file }}
+Documentation=man:collectd(1)
+Documentation=man:collectd.conf(5)
+Documentation=https://collectd.org
+
+[Service]
+Type=notify
+NotifyAccess=main
+EnvironmentFile=-/etc/default/{{ service_name }}
+ExecStartPre=/usr/sbin/collectd -t -C {{ config_file }}
+ExecStart=/usr/sbin/collectd -C {{ config_file }}
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
diff --git a/collectd/files/collectd_upstart.service b/collectd/files/collectd_upstart.service
new file mode 100644
index 0000000..c45f28e
--- /dev/null
+++ b/collectd/files/collectd_upstart.service
@@ -0,0 +1,10 @@
+# {{ service_name }}
+
+description "{{ service_name }}"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec /usr/bin/collectd -f -C {{ config_file }}
diff --git a/collectd/init.sls b/collectd/init.sls
index b794478..febb675 100644
--- a/collectd/init.sls
+++ b/collectd/init.sls
@@ -1,4 +1,9 @@
{%- if pillar.collectd is defined %}
include:
+{%- if pillar.collectd.client is defined %}
- collectd.client
-{%- endif %}
\ No newline at end of file
+{%- endif %}
+{%- if pillar.collectd.remote_client is defined %}
+- collectd.remote_client
+{%- endif %}
+{%- endif %}
diff --git a/collectd/map.jinja b/collectd/map.jinja
index 66c6974..701e444 100644
--- a/collectd/map.jinja
+++ b/collectd/map.jinja
@@ -7,7 +7,7 @@
'config_dir': '/etc/collectd.d',
'read_interval': 60,
'file_logging': True,
- 'remote_collector': False,
+ 'syslog_logging': True,
'use_fqdn': True,
},
'Debian': {
@@ -17,7 +17,7 @@
'config_dir': '/etc/collectd/conf.d',
'read_interval': 60,
'file_logging': True,
- 'remote_collector': False,
+ 'syslog_logging': True,
'use_fqdn': True,
},
'RedHat': {
@@ -27,7 +27,19 @@
'config_dir': '/etc/collectd.d',
'read_interval': 60,
'file_logging': True,
- 'remote_collector': False,
+ 'syslog_logging': True,
'use_fqdn': True,
},
}, merge=salt['pillar.get']('collectd:client')) %}
+
+{% set remote_client = salt['grains.filter_by']({
+ 'default': {
+ 'service': 'remote_collectd',
+ 'config_file': '/etc/remote_collectd/collectd.conf',
+ 'config_dir': '/etc/remote_collectd/conf.d',
+ 'read_interval': 60,
+ 'file_logging': True,
+ 'syslog_logging': False,
+ 'use_fqdn': True,
+ }
+}, merge=salt['pillar.get']('collectd:remote_client')) %}
diff --git a/collectd/meta/collectd.yml b/collectd/meta/collectd.yml
index ea0297a..dc349b5 100644
--- a/collectd/meta/collectd.yml
+++ b/collectd/meta/collectd.yml
@@ -1,17 +1,26 @@
+{%- from "collectd/map.jinja" import client with context %}
+{%- from "collectd/map.jinja" import remote_client with context %}
local_plugin:
collectd_processes:
plugin: processes
template: collectd/files/collectd_processes.conf
process:
- collectd: {}
- {%- if pillar.collectd.client.get('check', {}).curl is defined %}
+ {%- if pillar.collectd.get('client', {}).get('enabled', False) %}
+ collectd:
+ match: '(collectd.*{{ client.config_file }}|collectd$)'
+ {%- endif %}
+ {%- if pillar.collectd.get('remote_client', {}).get('enabled', False) %}
+ remote_collectd:
+ match: 'collectd.*{{ remote_client.config_file }}'
+ {%- endif %}
+ {%- if pillar.collectd.get('client', {}).get('check', {}).curl is defined %}
collectd_curl:
plugin: curl
execution: local
template: collectd/files/collectd_curl.conf
data: {{ pillar.collectd.client.check.curl|yaml }}
{%- endif %}
- {%- if pillar.collectd.client.get('check', {}).ping is defined %}
+ {%- if pillar.collectd.get('client', {}).get('check', {}).ping is defined %}
collectd_ping:
plugin: ping
execution: local
diff --git a/collectd/remote_client.sls b/collectd/remote_client.sls
new file mode 100644
index 0000000..0f95747
--- /dev/null
+++ b/collectd/remote_client.sls
@@ -0,0 +1,36 @@
+{%- from "collectd/map.jinja" import remote_client with context %}
+{%- if remote_client.enabled %}
+
+include:
+- collectd._common
+
+{# Collect all remote plugins from Salt mine #}
+{%- set plugins = {} %}
+{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
+{%- if node_grains.collectd is defined %}
+{%- set plugins = salt['grains.filter_by']({'default': plugins}, merge=node_grains.collectd.get('remote_plugin', {})) %}
+{%- endif %}
+{%- endfor %}
+
+{%- set client = remote_client %}
+{%- include "collectd/_service.sls" %}
+
+{{ remote_client.service }}_service_file:
+ file.managed:
+{%- if grains.get('init', None) == 'systemd' %}
+ - name: /etc/systemd/system/{{ remote_client.service }}.service
+ - source: salt://collectd/files/collectd_systemd.service
+{%- else %}
+ - name: /etc/init/{{ remote_client.service }}
+ - source: salt://collectd/files/collectd_upstart.service
+{%- endif %}
+ - user: root
+ - mode: 644
+ - defaults:
+ service_name: {{ remote_client.service }}
+ config_file: {{ remote_client.config_file }}
+ - template: jinja
+ - require_in:
+ - service: {{ remote_client.service }}_service
+
+{%- endif %}
diff --git a/metadata/service/remote_client/init.yml b/metadata/service/remote_client/init.yml
new file mode 100644
index 0000000..163f0c4
--- /dev/null
+++ b/metadata/service/remote_client/init.yml
@@ -0,0 +1,10 @@
+applications:
+- collectd
+classes:
+- service.collectd.support
+parameters:
+ collectd:
+ remote_client:
+ enabled: true
+ read_interval: 10
+ use_fqdn: false