Merge "Add support for jinja2 renderer options. (2018.2)"
diff --git a/.kitchen.yml b/.kitchen.yml
index 75a6540..28e03cf 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -20,7 +20,7 @@
url: http://apt.mirantis.com/xenial
key_url: http://apt.mirantis.com/public.gpg
components: salt
- distribution: testing
+ distribution: nightly
state_top:
base:
"*":
@@ -59,9 +59,9 @@
sudo: true
platforms:
- - name: <%=ENV['PLATFORM'] || 'ubuntu-xenial'%>
+ - name: <%=ENV['PLATFORM'] || 'saltstack-ubuntu-xenial-salt-stable' %>
driver_config:
- image: <%=ENV['PLATFORM'] || 'trevorj/salty-whales:xenial'%>
+ image: <%=ENV['PLATFORM'] || 'epcim/salt-formulas:saltstack-ubuntu-xenial-salt-stable'%>
platform: ubuntu
suites:
diff --git a/.travis.yml b/.travis.yml
index 315c896..82d4591 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,28 +18,18 @@
env:
- - PLATFORM=trevorj/salty-whales:trusty-2017.7 SUITE=control-default
- - PLATFORM=trevorj/salty-whales:trusty-2017.7 SUITE=master-default
- - PLATFORM=trevorj/salty-whales:trusty-2017.7 SUITE=minion-default
- - PLATFORM=trevorj/salty-whales:trusty-2017.7 SUITE=minion-multi-master-failover
- - PLATFORM=trevorj/salty-whales:xenial-2017.7 SUITE=control-default
- - PLATFORM=trevorj/salty-whales:xenial-2017.7 SUITE=master-default
- - PLATFORM=trevorj/salty-whales:xenial-2017.7 SUITE=minion-default
- - PLATFORM=trevorj/salty-whales:xenial-2017.7 SUITE=minion-multi-master-failover
-# TODO, once 18.04 LTS Bionic Beaver is released, rename trusty to bionic
- - PLATFORM=trevorj/salty-whales:bionic-2017.7 SUITE=master-default
- - PLATFORM=trevorj/salty-whales:bionic-2017.7 SUITE=minion-default
- - PLATFORM=trevorj/salty-whales:trusty SUITE=control-default
- - PLATFORM=trevorj/salty-whales:trusty SUITE=master-default
- - PLATFORM=trevorj/salty-whales:trusty SUITE=minion-default
- - PLATFORM=trevorj/salty-whales:trusty SUITE=minion-multi-master-failover
- - PLATFORM=trevorj/salty-whales:xenial SUITE=control-default
- - PLATFORM=trevorj/salty-whales:xenial SUITE=master-default
- - PLATFORM=trevorj/salty-whales:xenial SUITE=minion-default
- - PLATFORM=trevorj/salty-whales:xenial SUITE=minion-multi-master-failover
-# TODO, once 18.04 LTS Bionic Beaver is released, rename trusty to bionic
- - PLATFORM=trevorj/salty-whales:bionic SUITE=master-default
- - PLATFORM=trevorj/salty-whales:bionic SUITE=minion-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=control-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=master-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=minion-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=minion-multi-master-failover
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=control-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=master-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=minion-default
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=minion-multi-master-failover
+# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=control-default
+# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=master-default
+# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=minion-default
+# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=minion-multi-master-failover
before_script:
- set -o pipefail
diff --git a/Makefile b/Makefile
index 1043fbe..d166862 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,7 @@
all:
@echo "make install - Install into DESTDIR"
+ @echo "make lint - Run lint tests"
@echo "make test - Run tests"
@echo "make kitchen - Run Kitchen CI tests (create, converge, verify)"
@echo "make clean - Cleanup after tests run"
@@ -45,6 +46,9 @@
[ -d $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) ] || mkdir -p $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
cp -a metadata/service/* $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
+lint:
+ [ ! -d tests ] || (cd tests; ./run_tests.sh lint)
+
test:
[ ! -d tests ] || (cd tests; ./run_tests.sh)
@@ -65,7 +69,7 @@
[ ! -f debian/changelog ] || dch -v $(VERSION_MAJOR).$(NEW_MINOR_VERSION) -m --force-distribution -D `dpkg-parsechangelog -S Distribution` "New version"
make genchangelog-$(VERSION_MAJOR).$(NEW_MINOR_VERSION)
(git add -u; git commit -m "Version $(VERSION_MAJOR).$(NEW_MINOR_VERSION)")
- git tag -s -m $(NEW_MAJOR_VERSION) $(VERSION_MAJOR).$(NEW_MINOR_VERSION)
+ git tag -s -m $(VERSION_MAJOR).$(NEW_MINOR_VERSION) $(VERSION_MAJOR).$(NEW_MINOR_VERSION)
check-changes:
@git log --pretty=oneline --decorate $(VERSION)..HEAD | grep -Eqc '.*' || (echo "No new changes since version $(VERSION)"; exit 1)
diff --git a/README.rst b/README.rst
index 684a871..0b5160f 100644
--- a/README.rst
+++ b/README.rst
@@ -225,6 +225,7 @@
backup: true
initial_data:
engine: backupninja
+ home_dir: remote-backup-home-dir
source: backup-node-host
host: original-salt-master-id
diff --git a/_engines/architect.py b/_engines/architect.py
index 4096a73..a6d6bed 100644
--- a/_engines/architect.py
+++ b/_engines/architect.py
@@ -1,19 +1,17 @@
# -*- coding: utf-8 -*-
"""
-Salt engine for intercepting state jobs and forwarding to the Architect
-service.
+Salt engine for intercepting state jobs and forwarding to the Architect.
"""
# Import python libs
from __future__ import absolute_import
-import json
import logging
+from architect_client.libarchitect import ArchitectClient
# Import salt libs
import salt.utils.event
-import salt.utils.http
-log = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
def start(project='default',
@@ -22,14 +20,11 @@
username=None,
password=None):
'''
- Listen to state jobs events and forward Salt states
+ Listen to state jobs events and forward state functions and node info
'''
- url = "{}://{}:{}/salt/{}/event/{}".format('http',
- host,
- port,
- 'v1',
- project)
- target_functions = ['state.sls', 'state.apply', 'state.highstate']
+ state_functions = ['state.sls', 'state.apply', 'state.highstate']
+ model_functions = ['architect.node_info']
+ class_tag = 'architect/minion/classify'
if __opts__['__role'] == 'master':
event_bus = salt.utils.event.get_master_event(__opts__,
@@ -43,20 +38,21 @@
sock_dir=__opts__['sock_dir'],
listen=True)
- log.info('Salt Architect engine initialised')
+ logger.info('Architect Engine initialised')
while True:
event = event_bus.get_event()
- if event and event.get('fun', None) in target_functions:
+ if event and event.get('fun', None) in state_functions:
is_test_run = 'test=true' in [arg.lower() for arg in event.get('fun_args', [])]
if not is_test_run:
- data = salt.utils.http.query(url=url,
- method='POST',
- decode=False,
- data=json.dumps(event))
- if 'OK' in data.get('body', ''):
- log.info("Architect Engine request to '{}'"
- " was successful".format(url))
- else:
- log.warning("Problem with Architect Engine"
- " request to '{}' ({})".format(url, data))
+ output = ArchitectClient().push_event(event)
+ logger.info("Sent Architect state function {}".format(output))
+ if event and event.get('fun', None) in model_functions:
+ output = ArchitectClient().push_node_info({event['id']: event['return']})
+ logger.info("Sent Architect node info function {}".format(output))
+ if event and event.get('tag', None) == class_tag:
+ output = ArchitectClient().classify_node({
+ 'name': event['id'],
+ 'data': event['data']
+ })
+ logger.info("Sent Architect node classification {}".format(output))
diff --git a/_modules/architect.py b/_modules/architect.py
new file mode 100644
index 0000000..dce22df
--- /dev/null
+++ b/_modules/architect.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+'''
+Salt modules to work with the Architect service.
+'''
+
+# Import python libs
+from __future__ import absolute_import
+import logging
+
+__virtualname__ = 'architect'
+
+logger = logging.getLogger(__name__)
+
+
+def __virtual__():
+ return __virtualname__
+
+
+def node_info():
+ '''
+ Get Salt minion metadata and forward it to the Architect master.
+
+ CLI Examples:
+
+ .. code-block:: bash
+
+ salt-call architect.minion_info
+ '''
+ data = {
+ 'pillar': __salt__['pillar.data'](),
+ 'grain': __salt__['grains.items'](),
+ 'lowstate': __salt__['state.show_lowstate'](),
+ }
+ return data
diff --git a/salt/files/restore_master.sh b/salt/files/restore_master.sh
index ef77ec2..92b94f2 100644
--- a/salt/files/restore_master.sh
+++ b/salt/files/restore_master.sh
@@ -4,8 +4,8 @@
{%- if master.initial_data is defined %}
mkdir -p /etc/salt/pki.bak
mv /etc/salt/pki/* /etc/salt/pki.bak
-scp -r backupninja@{{ master.initial_data.source }}:/srv/backupninja/{{ master.initial_data.host }}/etc/salt/pki/pki.0/* /etc/salt/pki
+scp -r backupninja@{{ master.initial_data.source }}:{{ master.initial_data.get('home_dir', '/srv/backupninja') }}/{{ master.initial_data.host }}/etc/salt/pki/pki.0/* /etc/salt/pki
{%- if master.pillar.engine == 'reclass' or (master.pillar.engine == 'composite' and master.pillar.reclass is defined) %}
-scp -r backupninja@{{ master.initial_data.source }}:/srv/backupninja/{{ master.initial_data.host }}/srv/salt/reclass/nodes/_generated/_generated.0/* /srv/salt/reclass/nodes/_generated
+scp -r backupninja@{{ master.initial_data.source }}:{{ master.initial_data.get('home_dir', '/srv/backupninja') }}/{{ master.initial_data.host }}/srv/salt/reclass/reclass.0/* /srv/salt/reclass
{%- endif %}
{%- endif %}
diff --git a/salt/map.jinja b/salt/map.jinja
index 38906bb..24d02ac 100644
--- a/salt/map.jinja
+++ b/salt/map.jinja
@@ -166,6 +166,7 @@
default:
pkgs:
- salt-api
+ - python-cherrypy3
service: salt-api
{%- endload %}
diff --git a/salt/master/api.sls b/salt/master/api.sls
deleted file mode 100644
index c60f24a..0000000
--- a/salt/master/api.sls
+++ /dev/null
@@ -1,22 +0,0 @@
-{%- from "salt/map.jinja" import master with context %}
-{%- if master.enabled %}
-
-include:
-- salt.master.service
-
-salt_api_packages:
- pkg.latest
- - names:
- - salt-api
- - require:
- - {{ master.install_state }}
-
-salt_api_service:
- service.running:
- - name: salt-api
- - require:
- - pkg: salt_api_packages
- - watch:
- - file: /etc/salt/master
-
-{%- endif %}
diff --git a/salt/meta/backupninja.yml b/salt/meta/backupninja.yml
index 0a98f44..1948dd8 100644
--- a/salt/meta/backupninja.yml
+++ b/salt/meta/backupninja.yml
@@ -6,7 +6,7 @@
{%- if master.get('backup', False) %}
fs_includes:
{%- if master.pillar.engine == 'reclass' or (master.pillar.engine == 'composite' and master.pillar.reclass is defined) %}
- - /srv/salt/reclass/nodes/_generated
+ - /srv/salt/reclass
{%- endif %}
- /etc/salt/pki
{%- if minion.get('backup', False) %}
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 9451611..316692b 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -37,7 +37,7 @@
log_info "Setting up Python virtualenv"
virtualenv $VENV_DIR
source ${VENV_DIR}/bin/activate
- pip install salt${PIP_SALT_VERSION}
+ python -m pip install salt${PIP_SALT_VERSION}
}
setup_pillar() {
@@ -111,7 +111,7 @@
salt_run() {
[ -e ${VENV_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
- salt-call ${SALT_OPTS} $*
+ python $(which salt-call) ${SALT_OPTS} $*
}
prepare() {