allow-multiple-ext-pillars-and-reclass-options
Change-Id: I336b0825712218e2381e1654251765a24581da00
diff --git a/.kitchen.yml b/.kitchen.yml
index eac1011..20f4fff 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -118,6 +118,16 @@
master_ssh_root.sls: tests/pillar/master_ssh_minion_root.sls
master_formulas.sls: tests/pillar/master_formulas.sls
+ - name: master-extpillar-composite
+ provisioner:
+ pillars-from-files:
+ salt.sls: tests/pillar/master_single_extpillars.sls
+
+ - name: master-extpillar-reclass
+ provisioner:
+ pillars-from-files:
+ salt.sls: tests/pillar/master_single_extreclass.sls
+
- name: control-default
provisioner:
grains:
diff --git a/README.rst b/README.rst
index 7ec1dfd..e4317ee 100644
--- a/README.rst
+++ b/README.rst
@@ -28,6 +28,11 @@
.. literalinclude:: tests/pillar/master_single_reclass.sls
:language: yaml
+Salt master with multiple ext_pillars
+
+.. literalinclude:: tests/pillar/master_single_extpillars.sls
+ :language: yaml
+
Salt master with API
.. literalinclude:: tests/pillar/master_api.sls
@@ -163,6 +168,7 @@
master:
state_output: changes
+
Salt synchronise node pillar and modules after start
.. code-block:: yaml
@@ -222,6 +228,80 @@
salt-call event.send 'reclass/minion/classify' "{'node_master_ip': '$config_host', 'node_ip': '${node_ip}', 'node_domain': '$node_domain', 'node_cluster': '$node_cluster', 'node_hostname': '$node_hostname', 'node_os': '$node_os'}"
+
+Encrypted pillars
+-----------------
+
+Note: NACL + below configuration will be available in Salt > 2017.7.
+
+External resources:
+
+- Tutorial to configure salt + reclass ext_pillar and nacl: http://apealive.net/post/2017-09-salt-nacl-ext-pillar/
+- Saltstack documentation: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.nacl.html
+
+Configure salt NACL module:
+
+.. code-block:: shell
+
+ pip install --upgrade libnacl===1.5.2
+ salt-call --local nacl.keygen /etc/salt/pki/master/nacl
+
+ local:
+ saved sk_file:/etc/salt/pki/master/nacl pk_file: /etc/salt/pki/master/nacl.pub
+
+
+.. code-block:: yaml
+
+ salt:
+ master:
+ pillar:
+ reclass: *reclass
+ nacl:
+ index: 99
+ nacl:
+ box_type: sealedbox
+ sk_file: /etc/salt/pki/master/nacl
+ pk_file: /etc/salt/pki/master/nacl.pub
+ #sk: None
+ #pk: None
+
+NACL encrypt secrets:
+
+ salt-call --local nacl.enc 'my_secret_value' pk_file=/etc/salt/pki/master/nacl.pub
+ hXTkJpC1hcKMS7yZVGESutWrkvzusXfETXkacSklIxYjfWDlMJmR37MlmthdIgjXpg4f2AlBKb8tc9Woma7q
+ # or
+ salt-run nacl.enc 'myotherpass'
+ ADDFD0Rav6p6+63sojl7Htfrncp5rrDVyeE4BSPO7ipq8fZuLDIVAzQLf4PCbDqi+Fau5KD3/J/E+Pw=
+
+
+NACL encrypted values on pillar:
+
+Use Boxed syntax `NACL[CryptedValue=]` to encode value on pillar:
+
+.. code-block:: yaml
+
+ my_pillar:
+ my_nacl:
+ key0: unencrypted_value
+ key1: NACL[hXTkJpC1hcKMS7yZVGESutWrkvzusXfETXkacSklIxYjfWDlMJmR37MlmthdIgjXpg4f2AlBKb8tc9Woma7q]
+
+NACL large files:
+
+.. code-block:: shell
+ salt-call nacl.enc_file /tmp/cert.crt out=/srv/salt/env/dev/cert.nacl
+ # or more advanced
+ cert=$(cat /tmp/cert.crt)
+ salt-call --out=newline_values_only nacl.enc_pub data="$cert" > /srv/salt/env/dev/cert.nacl
+
+
+NACL within template/native pillars:
+
+ pillarexample:
+ user: root
+ password1: {{salt.nacl.dec('DRB7Q6/X5gGSRCTpZyxS6hlbWj0llUA+uaVyvou3vJ4=')|json}}
+ cert_key: {{salt.nacl.dec_file('/srv/salt/env/dev/certs/example.com/cert.nacl')|json}}
+ cert_key2: {{salt.nacl.dec_file('salt:///certs/example.com/cert2.nacl')|json}}
+
Salt syndic
-----------
diff --git a/metadata/service/master/nacl/init.yml b/metadata/service/master/nacl/init.yml
new file mode 100644
index 0000000..48610b3
--- /dev/null
+++ b/metadata/service/master/nacl/init.yml
@@ -0,0 +1,6 @@
+salt:
+ master:
+ nacl:
+ box_type: sealedbox
+ sk_file: /etc/salt/pki/master/nacl
+ pk_file: /etc/salt/pki/master/nacl.pub
diff --git a/metadata/service/master/pillar/composite/gpg.yml b/metadata/service/master/pillar/composite/gpg.yml
new file mode 100644
index 0000000..92641e8
--- /dev/null
+++ b/metadata/service/master/pillar/composite/gpg.yml
@@ -0,0 +1,9 @@
+classes:
+- service.master.pillar.composite
+parameters:
+ salt:
+ master:
+ pillar:
+ gpg:
+ index: 99
+
diff --git a/metadata/service/master/pillar/composite/init.yml b/metadata/service/master/pillar/composite/init.yml
new file mode 100644
index 0000000..184469e
--- /dev/null
+++ b/metadata/service/master/pillar/composite/init.yml
@@ -0,0 +1,5 @@
+parameters:
+ salt:
+ master:
+ pillar:
+ engine: composite
diff --git a/metadata/service/master/pillar/composite/nacl.yml b/metadata/service/master/pillar/composite/nacl.yml
new file mode 100644
index 0000000..9aa76e6
--- /dev/null
+++ b/metadata/service/master/pillar/composite/nacl.yml
@@ -0,0 +1,11 @@
+classes:
+- service.master.nacl
+- service.master.pillar.composite
+parameters:
+ salt:
+ master:
+ pillar:
+ nacl:
+ # if order is provided 99 is used to compose "99-nacl" key name which is later used to order entries
+ index: 99
+
diff --git a/metadata/service/master/pillar/composite/reclass.yml b/metadata/service/master/pillar/composite/reclass.yml
new file mode 100644
index 0000000..93c8c0d
--- /dev/null
+++ b/metadata/service/master/pillar/composite/reclass.yml
@@ -0,0 +1,14 @@
+classes:
+- service.master.pillar.composite
+parameters:
+ salt:
+ master:
+ pillar:
+ reclass:
+ index: 1
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/salt/reclass
+ #class_mappings: []
+ propagate_pillar_data_to_reclass: False
+ ignore_class_notfound: False
+ #ignore_class_regexp: []
diff --git a/metadata/service/master/pillar/composite/saltclass.yml b/metadata/service/master/pillar/composite/saltclass.yml
new file mode 100644
index 0000000..6dfab2e
--- /dev/null
+++ b/metadata/service/master/pillar/composite/saltclass.yml
@@ -0,0 +1,8 @@
+classes:
+- service.master.pillar.composite
+parameters:
+ salt:
+ master:
+ pillar:
+ saltclass:
+ path: /srv/salt/saltclass
diff --git a/metadata/service/master/pillar/reclass.yml b/metadata/service/master/pillar/reclass.yml
new file mode 100644
index 0000000..f8ba995
--- /dev/null
+++ b/metadata/service/master/pillar/reclass.yml
@@ -0,0 +1,8 @@
+parameters:
+ salt:
+ master:
+ pillar:
+ engine: reclass
+ reclass:
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/salt/reclass
diff --git a/metadata/service/master/pillar/salt.yml b/metadata/service/master/pillar/salt.yml
new file mode 100644
index 0000000..83416bd
--- /dev/null
+++ b/metadata/service/master/pillar/salt.yml
@@ -0,0 +1,7 @@
+parameters:
+ salt:
+ master:
+ pillar:
+ engine: salt
+ salt:
+ path: /srv/salt/pillar
diff --git a/metadata/service/master/pillar/saltclass.yml b/metadata/service/master/pillar/saltclass.yml
new file mode 100644
index 0000000..44d682a
--- /dev/null
+++ b/metadata/service/master/pillar/saltclass.yml
@@ -0,0 +1,7 @@
+parameters:
+ salt:
+ master:
+ pillar:
+ engine: saltclass
+ saltclass:
+ path: /srv/salt/saltclass
diff --git a/metadata/service/minion/local.yml b/metadata/service/minion/local.yml
index 961cb1e..d1e178f 100644
--- a/metadata/service/minion/local.yml
+++ b/metadata/service/minion/local.yml
@@ -12,4 +12,3 @@
local: true
pillar:
engine: reclass
- data_dir: /srv/salt/reclass
diff --git a/releasenotes/config.yaml b/releasenotes/config.yaml
new file mode 100644
index 0000000..fe5bc11
--- /dev/null
+++ b/releasenotes/config.yaml
@@ -0,0 +1,59 @@
+---
+# Usage:
+#
+# reno list
+# reno new slug-title --edit
+# reno report --no-show-source
+
+# Change prelude_section_name to 'summary' from default value prelude
+prelude_section_name: summary
+show_source: False
+sections:
+ # summary/prelude section is always included
+ - [features, New Features]
+ - [issues, Known Issues]
+ - [fixes, Bug Fixes]
+ - [other, Other Notes]
+template: |
+ ---
+ # Author the following sections or remove the section if it is not related.
+ # Use one release note per a feature.
+ #
+ # If you miss a section from the list below, please first submit a review
+ # adding it to releasenotes/config.yaml.
+ #
+ summary: >
+ This section is not mandatory. Use it to highlight the change.
+
+ features:
+ - Use list to record summary of features.
+ - |
+ Provide detailed description with examples.
+ Format with reStructuredText.
+
+ .. code-block:: text
+
+ provide model/formula pillar snippets
+
+ issues:
+ - Use list to record known limitations.
+
+ fixes:
+ - Use list to record summary of fixes.
+ Quick and dirty `git log --oneline`.
+
+ other:
+ - Author additional notes for the release.
+ - Format with reStructuredText.
+ - |
+ Use this section if note is not related to one of the common sections:
+ features, issues, upgrade, deprecations, security, fixes, api, cli
+
+ * list item 1
+ * list item 2
+
+ .. code-block:: yaml
+
+ formula:
+ example:
+ enabled: true
diff --git a/releasenotes/notes/add-releasenotes-9c076c7ee8fbe2a4.yaml b/releasenotes/notes/add-releasenotes-9c076c7ee8fbe2a4.yaml
new file mode 100644
index 0000000..54deb69
--- /dev/null
+++ b/releasenotes/notes/add-releasenotes-9c076c7ee8fbe2a4.yaml
@@ -0,0 +1,18 @@
+---
+summary: >
+ Use "reno", an releasenotes configuration tool to record release notes.
+ Documentation: https://docs.openstack.org/reno/latest
+
+ To list/create/show release notes, run following commands:
+
+ .. code-block:: shell
+
+ reno list
+ reno new releasenote-slug-title --edit
+ # use favored $EDITOR to update the note
+ # git add/commit releasenotes/* as usual
+ reno report --no-show-source
+
+other:
+ - |
+ Added `reno <https://docs.openstack.org/reno/latest>_` configuration to the repository.
diff --git a/releasenotes/notes/allow-multiple-ext-pillars-885d28dc8a18ab99.yaml b/releasenotes/notes/allow-multiple-ext-pillars-885d28dc8a18ab99.yaml
new file mode 100644
index 0000000..44d33d3
--- /dev/null
+++ b/releasenotes/notes/allow-multiple-ext-pillars-885d28dc8a18ab99.yaml
@@ -0,0 +1,20 @@
+---
+features:
+ - |
+ Added option to define multiple ext_pillars.
+ Example usage:
+
+ .. code-block:: text
+
+ salt:
+ master:
+ pillar:
+ engine: composite
+ reclass:
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/salt/reclass_encrypted
+ nacl:
+ index: 99
+ nacl:
+ sk_file: /etc/salt/pki/master/nacl
+ pk_file: /etc/salt/pki/master/nacl.pub
diff --git a/salt/files/master.conf b/salt/files/master.conf
index 329ae0d..7fec1bd 100644
--- a/salt/files/master.conf
+++ b/salt/files/master.conf
@@ -53,48 +53,121 @@
{%- endif %}
{%- if master.pillar.engine == 'salt' %}
-
pillar_roots:
base:
- - /srv/salt/pillar
-
+ - {{ master.pillar.get('salt', {}).get('path', '/srv/salt/pillar') }}
{%- endif %}
-{%- if master.pillar.engine == 'reclass' %}
+{%- if master.pillar.engine == 'reclass' or (master.pillar.engine == 'composite' and master.pillar.reclass is defined) %}
reclass: &reclass
- storage_type: yaml_fs
- inventory_base_uri: /srv/salt/reclass
+ storage_type: {{ master.pillar.get('reclass', {'storage_type': 'yaml_fs'}).storage_type }}
+ inventory_base_uri: {{ master.pillar.get('reclass', {'inventory_base_uri': '/srv/salt/reclass'}).inventory_base_uri }}
+ {%- if master.pillar.reclass.reclass_source_path is defined %}
+ reclass_source_path: {{ master.pillar.reclass.reclass_source_path }}
+ {%- endif %}
+ {%- if master.pillar.reclass.get('class_mappings', [])|length > 0 %}
+ class_mappings:
+ {%- for mapping in master.pillar.reclass.class_mappings %}
+ - {{ mapping.target }} {{ mapping.class }}
+ {%- endfor %}
+ {%- endif %}
+ {%- if master.pillar.reclass.get('propagate_pillar_data_to_reclass', False) == True %}
+ propagate_pillar_data_to_reclass: {{ master.pillar.reclass.propagate_pillar_data_to_reclass }}
+ {%- endif %}
+ {%- if master.pillar.reclass.get('ignore_class_notfound', False) == True %}
+ # Below option is not available in upstream reclass, and require fork https://github.com/salt-formulas/reclass
+ ignore_class_notfound: {{ master.pillar.reclass.ignore_class_notfound }}
+ ignore_class_regexp: {{ master.pillar.reclass.ignore_class_regexp }}
+ {%- endif %}
+{%- endif %}
+
+{%- if master.pillar.engine == 'saltclass' or (master.pillar.engine == 'composite' and master.pillar.saltclass is defined ) %}
+
+saltclass: &saltclass
+ path: {{ master.pillar.saltclass.get('path', '/srv/salt/saltclass') }}
+{%- endif %}
+
+
+{%- if master.pillar.engine in ['composite', 'reclass', 'saltclass'] %}
+{# generate indexed list of ext_engines #}
+{# NONE: Might be rewritten, once proved to work properly, with filters: #}
+{# NONE: select('mapping')|selectattr('_index')|sort(attribute='_index') #}
+{%- set ext_engines = {} %}
+{%- for name,engine in master.pillar.iteritems() %}
+{%- if not engine is mapping %}{% continue %}{% endif %}
+{%- do engine.update({'name': engine.get('name', name) }) %}
+{%- set index = engine.get('index', '1')~'-'~name %}
+{%- do ext_engines.update({ index: engine }) %}
+{%- endfor %}
+{%- if ext_engines|length > 0 %}
ext_pillar:
+ {%- for name, engine in ext_engines|dictsort %}
+ {%- if master.pillar.engine in ['composite', 'reclass'] and engine.name == 'reclass' %}
- reclass: *reclass
+ {%- endif %}
+ {%- if master.pillar.engine in ['composite', 'saltclass'] and engine.name == 'saltclass' %}
+ - saltclass: *saltclass
+ {%- endif %}
+ {%- if engine.name == 'nacl' %}
+ - nacl: {}
+ {%- endif %}
+ {%- if engine.name == 'gpg' %}
+ - gpg: {}
+ {%- endif %}
+ {%- endfor %}
+{%- endif %}
+{%- endif %}
+
+{%- if master.pillar.engine in ['saltclass', 'reclass']
+ or (master.pillar.engine == 'composite' and
+ (master.pillar.saltclass is defined or
+ master.pillar.reclass is defined )) %}
master_tops:
+ {%- if master.pillar.engine in ['composite', 'reclass'] and master.pillar.get('reclass', False) %}
reclass: *reclass
-
+ {%- endif %}
+ {%- if master.pillar.engine in ['composite', 'saltclass'] and master.pillar.get('saltclass', False) %}
+ saltclass: *saltclass
+ {%- endif %}
{%- endif %}
{%- for handler in pillar.salt.minion.get("handlers", []) %}
-
{%- if handler.engine == "udp"%}
+
logstash_udp_handler:
host: {{ handler.host }}
port: {{ handler.port }}
version: 1
{%- endif %}
-
{%- if handler.engine == "zmq"%}
+
logstash_zmq_handler:
address: tcp://{{ handler.host }}:{{ handler.port }}
version: 1
{%- endif %}
-
{%- endfor %}
{%- if master.get('order_masters', False) %}
+
order_masters: True
{%- endif %}
+{%- if master.nacl is defined %}
+
+nacl.config:
+ box_type: {{ master.nacl.get('box_type', 'sealedbox') }}
+ {%- if master.nacl.sk is defined %}
+ sk: {{ master.nacl.sk }}
+ pk: {{ master.nacl.pk }}
+ {%- else %}
+ sk_file: {{ master.nacl.sk_file }}
+ pk_file: {{ master.nacl.pk_file }}
+ {%- endif %}
+{%- endif %}
+
{#-
vim: syntax=jinja
-#}
diff --git a/salt/master/pillar.sls b/salt/master/pillar.sls
index 3e2e715..806511d 100644
--- a/salt/master/pillar.sls
+++ b/salt/master/pillar.sls
@@ -1,4 +1,4 @@
-{%- from "salt/map.jinja" import master with context %}
+{%- from "salt/map.jinja" import master,storage with context %}
{%- if master.enabled %}
{%- if master.pillar.engine == 'salt' %}
@@ -38,6 +38,7 @@
/srv/salt/reclass/classes/service:
file.directory:
+ - makedirs: true
- require:
- file: reclass_data_dir
@@ -47,6 +48,7 @@
/srv/salt/reclass/classes/service/{{ formula_name }}:
file.symlink:
+ - makedirs: true
- target: /srv/salt/env/{{ master.system.environment }}/{{ formula_name }}/metadata/service
- require:
- file: /srv/salt/reclass/classes/service
@@ -55,7 +57,7 @@
{%- else %}
-{%- for environment_name, environment in master.environment.iteritems() %}
+{%- for environment_name, environment in master.get('environment', {}).iteritems() %}
{%- for formula_name, formula in environment.get('formula', {}).iteritems() %}
@@ -63,6 +65,7 @@
/srv/salt/reclass/classes/service/{{ formula_name }}:
file.symlink:
+ - makedirs: true
{%- if formula.source == 'pkg' %}
- target: /usr/share/salt-formulas/reclass/service/{{ formula_name }}
{%- else %}
diff --git a/salt/master/service.sls b/salt/master/service.sls
index 85f7eb5..5af786e 100644
--- a/salt/master/service.sls
+++ b/salt/master/service.sls
@@ -59,7 +59,13 @@
salt_master_service:
service.running:
- name: {{ master.service }}
- - enable: true
+ - enable: True
+ {%- if grains['saltversioninfo'][0] >= 2017 and grains['saltversioninfo'][1] >= 7 %}
+ - retry:
+ attempts: 2
+ interval: 5
+ splay: 5
+ {%- endif %}
/srv/salt/env:
file.directory:
diff --git a/tests/pillar/master_single_extpillars.sls b/tests/pillar/master_single_extpillars.sls
new file mode 100644
index 0000000..03521eb
--- /dev/null
+++ b/tests/pillar/master_single_extpillars.sls
@@ -0,0 +1,83 @@
+git:
+ client:
+ enabled: true
+linux:
+ system:
+ enabled: true
+reclass:
+ storage:
+ enabled: true
+ data_source:
+ engine: git
+ branch: master
+ address: 'https://github.com/salt-formulas/openstack-salt.git'
+salt:
+ master:
+ enabled: true
+ command_timeout: 5
+ worker_threads: 2
+ base_environment: prd
+ #environment:
+ # prd:
+ # formula:
+ # python:
+ # source: git
+ # address: 'https://github.com/salt-formulas/salt-formula-python.git'
+ # revision: master
+ pillar:
+ engine: composite
+ reclass:
+ # index: 1 is default value
+ index: 1
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/salt/reclass_encrypted
+ class_mappings:
+ - target: '/^cfg\d+/'
+ class: system.non-existing.class
+ ignore_class_notfound: True
+ ignore_class_regexp:
+ - 'service.*'
+ - '*.fluentd'
+ propagate_pillar_data_to_reclass: False
+ stack: # not yet implemented
+ # https://docs.saltstack.com/en/latest/ref/pillar/all/salt.pillar.stack.html
+ #option 1
+ #path:
+ # - /path/to/stack.cfg
+ #option 2
+ pillar:environment:
+ dev: path/to/dev/stasck.cfg
+ prod: path/to/prod/stasck.cfg
+ grains:custom:grain:
+ value:
+ - /path/to/stack1.cfg
+ - /path/to/stack2.cfg
+ saltclass:
+ path: /srv/salt/saltclass
+ nacl:
+ # if order is provided 99 is used to compose "99-nacl" key name which is later used to order entries
+ index: 99
+ gpg: {}
+ vault-1: # not yet implemented
+ name: vault
+ path: secret/salt
+ vault-2: # not yet implemented
+ name: vault
+ path: secret/root
+ vault: # not yet implemented
+ # https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.vault.html
+ name: myvault
+ url: https://vault.service.domain:8200
+ auth:
+ method: token
+ token: 11111111-2222-3333-4444-555555555555
+ policies:
+ - saltstack/minions
+ - saltstack/minion/{minion}
+ nacl:
+ # https://docs.saltstack.com/en/develop/ref/modules/all/salt.modules.nacl.html
+ box_type: sealedbox
+ sk_file: /etc/salt/pki/master/nacl
+ pk_file: /etc/salt/pki/master/nacl.pub
+ #sk: None
+ #pk: None
diff --git a/tests/pillar/master_single_extreclass.sls b/tests/pillar/master_single_extreclass.sls
new file mode 100644
index 0000000..7162dce
--- /dev/null
+++ b/tests/pillar/master_single_extreclass.sls
@@ -0,0 +1,41 @@
+git:
+ client:
+ enabled: true
+linux:
+ system:
+ enabled: true
+reclass:
+ storage:
+ enabled: true
+ data_source:
+ engine: git
+ branch: master
+ address: 'https://github.com/salt-formulas/openstack-salt.git'
+salt:
+ master:
+ enabled: true
+ command_timeout: 5
+ worker_threads: 2
+ base_environment: prd
+ #environment:
+ # prd:
+ # formula:
+ # python:
+ # source: git
+ # address: 'https://github.com/salt-formulas/salt-formula-python.git'
+ # revision: master
+ pillar:
+ engine: reclass
+ reclass:
+ # index: 1 is default value
+ index: 1
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/salt/reclass_encrypted
+ class_mappings:
+ - target: '/^cfg\d+/'
+ class: system.non-existing.class
+ ignore_class_notfound: True
+ ignore_class_regexp:
+ - 'service.*'
+ - '*.fluentd'
+ propagate_pillar_data_to_reclass: False
diff --git a/tests/pillar/master_single_reclass.sls b/tests/pillar/master_single_reclass.sls
index d6db522..19ebfa7 100644
--- a/tests/pillar/master_single_reclass.sls
+++ b/tests/pillar/master_single_reclass.sls
@@ -29,4 +29,8 @@
name: salt-formula-service02
pillar:
engine: reclass
- data_dir: /srv/salt/reclass
+ reclass:
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/salt/reclass
+ propagate_pillar_data_to_reclass: False
+ reclass_source_path: /tmp/reclass