Merge "Add Prometheus monitoring"
diff --git a/README.rst b/README.rst
index 33a3bf4..355c68f 100644
--- a/README.rst
+++ b/README.rst
@@ -21,6 +21,7 @@
metric_collector_host: 127.0.0.1
metric_collector_port: 5567
poolsize: 100
+ max_message_size: 262144
Default values:
@@ -30,6 +31,7 @@
* ``metric_collector_host: 127.0.0.1``
* ``metric_collector_port: 5567``
* ``poolsize: 100``
+* ``max_message_size: 262144``
Local Metric collector service
@@ -54,6 +56,7 @@
nagios_port: 5601
nagios_username: nagiosadmin
poolsize: 100
+ max_message_size: 262144
Default values:
@@ -65,6 +68,7 @@
* ``influxdb_timeout: 5000``
* ``nagios_port: 8001``
* ``poolsize: 100``
+* ``max_message_size: 262144``
Remote Metric Collector service
@@ -91,6 +95,7 @@
influxdb_time_precision: ms
influxdb_username: lma
poolsize: 100
+ max_message_size: 262144
Default values:
@@ -105,6 +110,7 @@
* ``influxdb_time_precision: ms``
* ``influxdb_timeout: 5000``
* ``poolsize: 100``
+* ``max_message_size: 262144``
Aggregator service
@@ -127,6 +133,7 @@
nagios_port: 5601
nagios_username: nagiosadmin
poolsize: 100
+ max_message_size: 262144
Default values:
@@ -138,6 +145,7 @@
* ``nagios_port: 8001``
* ``nagios_default_host_alarm_clusters: 00-clusters``
* ``poolsize: 100``
+* ``max_message_size: 262144``
Ceilometer service
diff --git a/heka/_service.sls b/heka/_service.sls
index 7af3101..c421697 100644
--- a/heka/_service.sls
+++ b/heka/_service.sls
@@ -220,6 +220,7 @@
- defaults:
service_name: {{ service_name }}
poolsize: {{ server.poolsize }}
+ max_message_size: {{ server.max_message_size }}
- require:
- file: heka_{{ service_name }}_conf_dir
- require_in:
diff --git a/heka/files/toml/global.toml b/heka/files/toml/global.toml
index 30d460d..bdc2a81 100644
--- a/heka/files/toml/global.toml
+++ b/heka/files/toml/global.toml
@@ -6,7 +6,7 @@
hostname="{{ grains.fqdn.split('.')[0] }}"
-max_message_size = 262144
+max_message_size = {{ max_message_size }}
max_process_inject = 1
max_timer_inject = 10
poolsize = {{ poolsize }}
diff --git a/heka/map.jinja b/heka/map.jinja
index 297705f..07e39a2 100644
--- a/heka/map.jinja
+++ b/heka/map.jinja
@@ -54,6 +54,7 @@
'emit_rates': True,
'prefix_dir': default_prefix_dir,
'elasticsearch_port': default_elasticsearch_port,
+ 'max_message_size': 262144,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
'metric_collector_host': '127.0.0.1',
@@ -76,6 +77,7 @@
'influxdb_tag_fields': default_influxdb_tag_fields,
'aggregator_port': default_aggregator_port,
'nagios_port': default_nagios_port,
+ 'max_message_size': 262144,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
'sensu_noop_handler': default_sensu_noop_handler,
@@ -99,6 +101,7 @@
'influxdb_timeout': default_influxdb_timeout,
'influxdb_tag_fields': default_influxdb_tag_fields,
'aggregator_port': default_aggregator_port,
+ 'max_message_size': 262144,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
'sensu_noop_handler': default_sensu_noop_handler,
@@ -119,6 +122,7 @@
'influxdb_tag_fields': default_influxdb_tag_fields,
'nagios_port': default_nagios_port,
'nagios_default_host_alarm_clusters': default_nagios_host_alarm_clusters,
+ 'max_message_size': 262144,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
'sensu_noop_handler': default_sensu_noop_handler,
@@ -141,6 +145,7 @@
'amqp_vhost': '/openstack',
'amqp_queue': 'metering.sample',
'resource_decoding': False,
+ 'max_message_size': 262144,
'poolsize': 100,
'automatic_starting': default_automatic_starting,
'sensu_noop_handler': default_sensu_noop_handler,
diff --git a/metadata/service/aggregator/cluster.yml b/metadata/service/aggregator/cluster.yml
index e95243e..35e854b 100644
--- a/metadata/service/aggregator/cluster.yml
+++ b/metadata/service/aggregator/cluster.yml
@@ -5,9 +5,11 @@
parameters:
_param:
aggregator_poolsize: 100
+ aggregator_max_message_size: 262144
heka:
aggregator:
automatic_starting: false
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:aggregator_poolsize}
+ max_message_size: ${_param:aggregator_max_message_size}
\ No newline at end of file
diff --git a/metadata/service/aggregator/single.yml b/metadata/service/aggregator/single.yml
index bf2364e..dc53ba8 100644
--- a/metadata/service/aggregator/single.yml
+++ b/metadata/service/aggregator/single.yml
@@ -5,9 +5,11 @@
parameters:
_param:
aggregator_poolsize: 100
+ aggregator_max_message_size: 262144
heka:
aggregator:
automatic_starting: true
enabled: true
influxdb_time_precision: ms
- poolsize: ${_param:aggregator_poolsize}
\ No newline at end of file
+ poolsize: ${_param:aggregator_poolsize}
+ max_message_size: ${_param:aggregator_max_message_size}
\ No newline at end of file
diff --git a/metadata/service/ceilometer_collector/single.yml b/metadata/service/ceilometer_collector/single.yml
index eacb11d..16f8050 100644
--- a/metadata/service/ceilometer_collector/single.yml
+++ b/metadata/service/ceilometer_collector/single.yml
@@ -5,9 +5,11 @@
parameters:
_param:
ceilometer_collector_poolsize: 100
+ ceilometer_collector_max_message_size: 262144
heka:
ceilometer_collector:
automatic_starting: true
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:ceilometer_collector_poolsize}
+ max_message_size: ${_param:ceilometer_collector_max_message_size}
diff --git a/metadata/service/log_collector/single.yml b/metadata/service/log_collector/single.yml
index 5160c35..d7a53fd 100644
--- a/metadata/service/log_collector/single.yml
+++ b/metadata/service/log_collector/single.yml
@@ -5,7 +5,9 @@
parameters:
_param:
log_collector_poolsize: 100
+ log_collector_max_message_size: 262144
heka:
log_collector:
enabled: true
poolsize: ${_param:log_collector_poolsize}
+ max_message_size: ${_param:log_collector_max_message_size}
diff --git a/metadata/service/metric_collector/single.yml b/metadata/service/metric_collector/single.yml
index ad183a2..420f630 100644
--- a/metadata/service/metric_collector/single.yml
+++ b/metadata/service/metric_collector/single.yml
@@ -5,8 +5,10 @@
parameters:
_param:
metric_collector_poolsize: 100
+ metric_collector_max_message_size: 262144
heka:
metric_collector:
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:metric_collector_poolsize}
+ max_message_size: ${_param:metric_collector_max_message_size}
diff --git a/metadata/service/remote_collector/cluster.yml b/metadata/service/remote_collector/cluster.yml
index 9e60e86..daa7e64 100644
--- a/metadata/service/remote_collector/cluster.yml
+++ b/metadata/service/remote_collector/cluster.yml
@@ -5,9 +5,11 @@
parameters:
_param:
remote_collector_poolsize: 100
+ remote_collector_max_message_size: 262144
heka:
remote_collector:
automatic_starting: false
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:remote_collector_poolsize}
+ max_message_size: ${_param:remote_collector_max_message_size}
diff --git a/metadata/service/remote_collector/single.yml b/metadata/service/remote_collector/single.yml
index 8533ad2..acc2853 100644
--- a/metadata/service/remote_collector/single.yml
+++ b/metadata/service/remote_collector/single.yml
@@ -5,9 +5,11 @@
parameters:
_param:
remote_collector_poolsize: 100
+ remote_collector_max_message_size: 262144
heka:
remote_collector:
automatic_starting: true
enabled: true
influxdb_time_precision: ms
poolsize: ${_param:remote_collector_poolsize}
+ max_message_size: ${_param:remote_collector_max_message_size}
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 3f42101..29fb975 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -6,11 +6,13 @@
CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
METADATA=${CURDIR}/../metadata.yml
FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
+FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
## Overrideable parameters
PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
BUILDDIR=${BUILDDIR:-${CURDIR}/build}
VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
+MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
DEPSDIR=${BUILDDIR}/deps
SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
@@ -18,7 +20,7 @@
SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
-SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR}"
+SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
if [ "x${SALT_VERSION}" != "x" ]; then
PIP_SALT_VERSION="==${SALT_VERSION}"
@@ -40,10 +42,20 @@
pip install salt${PIP_SALT_VERSION}
}
+setup_mock_bin() {
+ # If some state requires a binary, a lightweight replacement for
+ # such binary can be put into MOCK_BIN_DIR for test purposes
+ if [ -d "${MOCK_BIN_DIR}" ]; then
+ PATH="${MOCK_BIN_DIR}:$PATH"
+ export PATH
+ fi
+}
+
setup_pillar() {
[ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
echo "base:" > ${SALT_PILLAR_DIR}/top.sls
for pillar in ${PILLARDIR}/*; do
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
state_name=$(basename ${pillar%.sls})
echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
done
@@ -56,6 +68,7 @@
echo "base:" > ${SALT_FILE_DIR}/top.sls
for pillar in ${PILLARDIR}/*.sls; do
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
state_name=$(basename ${pillar%.sls})
echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
done
@@ -64,6 +77,7 @@
file_client: local
cachedir: ${SALT_CACHE_DIR}
verify_env: False
+minion_id_caching: False
file_roots:
base:
@@ -118,6 +132,7 @@
[ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
which salt-call || setup_virtualenv
+ setup_mock_bin
setup_pillar
setup_salt
install_dependencies
@@ -125,8 +140,28 @@
run() {
for pillar in ${PILLARDIR}/*.sls; do
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
state_name=$(basename ${pillar%.sls})
+ salt_run grains.set 'noservices' False force=True
+
+ echo "Checking state ${FORMULA_NAME}.${state_name} ..."
salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+
+ # Check that all files in 'meta' folder can be rendered using any valid pillar
+ for meta in `find ${FORMULA_META_DIR} -type f`; do
+ meta_name=$(basename ${meta})
+ echo "Checking meta ${meta_name} ..."
+ salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
+ || (log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1)
+ cat ${SALT_CACHE_DIR}/${meta_name}
+ done
+ done
+}
+
+real_run() {
+ for pillar in ${PILLARDIR}/*.sls; do
+ state_name=$(basename ${pillar%.sls})
+ salt_run --id=${state_name} state.sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
done
}
@@ -155,6 +190,9 @@
run)
run
;;
+ real-run)
+ real_run
+ ;;
*)
prepare
run