Merge pull request #17 from michaelkuty/feature/api_ssl

Support letsencrypt certs for API.
diff --git a/salt/files/minion.conf b/salt/files/minion.conf
index 1323b2f..0f9ec33 100644
--- a/salt/files/minion.conf
+++ b/salt/files/minion.conf
@@ -57,6 +57,8 @@
 
 {%- endif %}
 
+log_level: {{ minion.log.level }}
+
 {%- if minion.sentry is defined %}
 sentry_handler:
 {% for server in minion.sentry.servers %}
@@ -128,16 +130,6 @@
 {%- if pillar.get('keystone', {}).get('server', {'enabled': False } ).enabled %}
 keystone.token: '{{ pillar.keystone.server.service_token }}'
 keystone.endpoint: 'http://{{ pillar.keystone.server.bind.private_address }}:{{ pillar.keystone.server.bind.private_port }}/v2.0'
-{%- elif pillar.get('keystone', {}).get('client', {'enabled': False } ).enabled %}
-{%- if pillar.keystone.client.server.service_token is defined %}
-keystone.token: '{{ pillar.keystone.client.server.service_token }}'
-keystone.endpoint: 'http://{{ pillar.keystone.client.server.host }}:{{ pillar.keystone.client.server.private_port }}/v2.0'
-{%- else %}
-keystone.user: '{{ pillar.keystone.client.server.user }}'
-keystone.password: '{{ pillar.keystone.client.server.password }}'
-keystone.tenant: '{{ pillar.keystone.client.server.tenant }}'
-keystone.auth_url: 'http://{{ pillar.keystone.client.server.host }}:{{ pillar.keystone.client.server.public_port }}/v2.0/'
-{%- endif %}
 {%- endif %}
 
 {%- for handler in pillar.salt.minion.get("handlers", []) %}
diff --git a/salt/files/orchestrate.sls b/salt/files/orchestrate.sls
index 39ed271..dde20cb 100644
--- a/salt/files/orchestrate.sls
+++ b/salt/files/orchestrate.sls
@@ -1,43 +1,113 @@
 {%- from "salt/map.jinja" import master with context %}
+
 {%- if master.enabled %}
+  {%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+    {%- if master.base_environment == environment_name %}
+      {%- set sorted_priorities = priorities|dictsort(false, 'value') %}
+      {#- Set debug = True to log simple state result - Fail/True - to /var/log/salt/ on both master and respective minions #}
+      {%- set debug = False %}
+      {#- Uncomment to print parsed metadata from formula_name/meta/salt.yml to output file %}
 
-{{ formula_dict }}
+PARSED METADATA: 
 
-{%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+---------------
 
-{%- if master.base_environment == environment_name %}
+Passed from salt/master/orchestrate.sls
+---
 
-{%- set formula_dict = environment.get('formula', {}) %}
-{%- set new_formula_dict = {} %}
+priorities: {{ priorities }}
 
-{%- for formula_name, formula in formula_dict.iteritems() %}
+sorted_priorities: {{ sorted_priorities }}
 
-{%- set _tmp = new_formula_dict.update({formula_name: formula.get('orchestrate_order', 100)}) %}
+args: {{ args }}
 
-{%- endfor %}
+---------------
 
-{%- set sorted_formula_list = new_formula_dict|dictsort(false, 'value') %}
-	
-{%- for formula in sorted_formula_list %}
+      #}
 
-{%- if salt['file.file_exists']('/srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate.sls') %}
+      {%- for state in sorted_priorities %}
+        {%- set formula = state.0.split('.') %}
 
-{{ salt['cmd.run']('cat /srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate.sls') }}
+        {%- if salt['file.directory_exists']('/srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate') and formula|length > 1 and salt['file.file_exists']('/srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate/'+formula.1+'.sls') %}
 
-{%- else %}
+{{ salt['cmd.run']('cat /srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate/'+formula.1+'.sls') }}
 
-{{ formula.0 }}:
+        {%- else %}
+          {%- if args[ state.0 ] is defined %}
+
+{{ state.0 }}:
   salt.state:
-    - tgt: 'services:{{ formula.0 }}'
-    - tgt_type: grain
-    - sls: {{ formula.0 }}
+    - tgt: '{{ state.0|replace(".", ":") }}'
+    - tgt_type: pillar
+    - queue: True
+    - sls: {{ state.0 }}
+    {{ args[ state.0 ]|yaml(false)|indent(4) }}
 
+            {%- if debug %}
+
+{{ state.0 }}.logok:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: OK" >> /var/log/salt/orchestrate_runner'
+    - require:
+      - salt: {{ state.0 }}
+
+{{ state.0 }}.logfail:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: FAIL" >> /var/log/salt/orchestrate_runner'
+    - onfail:
+      - salt: {{ state.0 }}
+
+            {%- endif %}
+          {%- else %}
+
+{{ state.0 }}:
+  salt.state:
+    - tgt: '{{ state.0|replace(".", ":") }}{%- if "." not in state.0 %}:*{%- endif %}'
+    - tgt_type: pillar
+    - queue: True
+    - sls: {{ state.0 }}
+
+            {%- if debug %}
+
+{{ state.0 }}.logok:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: OK" >> /var/log/salt/orchestrate_runner'
+    - require:
+      - salt: {{ state.0 }}
+
+{{ state.0 }}.logfail:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: FAIL" >> /var/log/salt/orchestrate_runner'
+    - onfail:
+      - salt: {{ state.0 }}
+
+            {%- endif %}
+          {%- endif %}
+        {%- endif %}
+
+      {%- endfor %}
+
+    {%- endif %}
+  {%- endfor %}
 {%- endif %}
 
-{%- endfor %}
-
-{%- endif %}
-
-{%- endfor %}
-
-{%- endif %}
\ No newline at end of file
diff --git a/salt/map.jinja b/salt/map.jinja
index 25d8195..ba8a816 100644
--- a/salt/map.jinja
+++ b/salt/map.jinja
@@ -55,6 +55,8 @@
   - python-pydot
   - python-yaml
   graph_states: False
+  log:
+    level: info
   service: salt-minion
   {%- if pillar.salt.get('minion', {}).get('source', {}).get('engine', 'pkg') == 'pkg' %}
   install_state: "pkg: salt_minion_packages"
@@ -76,7 +78,8 @@
 Debian:
   pkgs:
   - salt-minion
-  - m2crypto
+  - python-m2crypto
+  - python-psutil
 Gentoo:
   pkgs:
   - app-admin/salt
@@ -86,7 +89,8 @@
 RedHat:
   pkgs:
   - salt-minion
-  - m2crypto
+  - python-m2ext
+  - python-psutil
 {%- endload %}
 
 {%- if pillar.salt.minion is defined %}
diff --git a/salt/master/env.sls b/salt/master/env.sls
index 3822667..a486b99 100644
--- a/salt/master/env.sls
+++ b/salt/master/env.sls
@@ -153,12 +153,12 @@
   - target: /usr/share/salt-formulas/env/_formulas/{{ formula_name }}
   {% if formula.get("revision", "").split("/")[0] == "refs" %}
   - rev: {{ formula.branch|default("master") }}
-  {%- if grains['saltversion'] < "2015.8.0" %}
+  {%- if grains['saltversion'] >= "2015.8.0" %}
   - branch: {{ formula.branch|default("master") }}
   {%- endif %}
   {% else %}
   - rev: {{ formula.revision|default(formula.branch) }}
-  {%- if grains['saltversion'] < "2015.8.0" %}
+  {%- if grains['saltversion'] >= "2015.8.0" %}
   - branch: {{ formula.branch|default(formula.revision) }}
   {%- endif %}
   {% endif %}
diff --git a/salt/master/orchestrate.sls b/salt/master/orchestrate.sls
index 4aae098..80be824 100644
--- a/salt/master/orchestrate.sls
+++ b/salt/master/orchestrate.sls
@@ -1,32 +1,54 @@
 {%- from "salt/map.jinja" import master with context %}
+
 {%- if master.enabled %}
+  {%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+    {%- if master.base_environment == environment_name %}
+      {%- set priorities = {} %}
+      {%- set args = {} %}
+      {%- set formulas = environment.get('formula', {}) %}
 
-{%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+      {%- for formula_name, formula in formulas.iteritems() %}
+        {%- if salt['file.file_exists'](master.dir.files+'/'+environment_name+'/'+formula_name+'/meta/salt.yml') %}
+          {%- set grains_fragment_file = formula_name+'/meta/salt.yml' %}
+          {%- macro load_grains_file() %}{% include grains_fragment_file %}{% endmacro %}
+          {%- set grains_yaml = load_grains_file()|load_yaml %}
 
-{%- if master.base_environment == environment_name %}
+          {%- for state, priority in grains_yaml['orchestrate'].iteritems() %}
+            {%- do priorities.update({ formula_name+'.'+state: grains_yaml['orchestrate'][state]['priority'] }) %}
+            {%- set arguments = [] %}
 
-{%- set formula_dict = {} %}
-{%- for formula_name, formula in formula_dict.iteritems() %}
+            {%- for arg_name, arg_value in grains_yaml['orchestrate'][state].iteritems() %}
+              {%- if 'priority' not in arg_name %}
+                {%- do arguments.append({arg_name: arg_value}) %}
+              {%- endif %}
+            {%- endfor %}
 
-{%- if salt['file.file_exists']('salt://'+formula_name+'/meta/salt.yml') %}
-{%- set grains_fragment_file = formula_name+'/meta/salt.yml' %}
-{%- macro load_grains_file() %}{% include grains_fragment_file %}{% endmacro %}
-{%- set grains_yaml = load_grains_file()|load_yaml %}
-{% _dummy = formula_dict.update{formula_name: grains_yaml.orchestrate }}
-{%- else %}
-{%- endif %}
-{%- endfor %}
+            {%- if arguments %}
+              {%- do args.update({ formula_name+'.'+state: arguments }) %}
+            {%- endif %}
+          {%- endfor %}
+        {%- else %}
+          {%- do priorities.update({ formula_name: 10000 }) %}
+        {%- endif %}
+      {%- endfor %}
 
-/srv/salt/env/{{ environment_name}}/orchestrate.sls:
+{{ master.dir.files }}/{{ environment_name }}/orchestrate:
+  file.directory:
+    - user: root
+    - group: root
+    - mode: 755
+    - makedirs: True
+
+{{ master.dir.files }}/{{ environment_name }}/orchestrate/init.sls:
   file.managed:
   - source: salt://salt/files/orchestrate.sls
   - user: root
   - template: jinja
   - defaults:
-      formula_dict: {{ formula_dict|yaml }}
+      priorities: {{ priorities }}
+      args: {{ args }}
 
+    {%- endif %}
+  {%- endfor %}
 {%- endif %}
 
-{%- endfor %}
-
-{%- endif %}
\ No newline at end of file
diff --git a/salt/master/pillar.sls b/salt/master/pillar.sls
index 45dec09..3e2e715 100644
--- a/salt/master/pillar.sls
+++ b/salt/master/pillar.sls
@@ -37,7 +37,9 @@
 - reclass.storage.data
 
 /srv/salt/reclass/classes/service:
-  file.directory
+  file.directory:
+  - require:
+    - file: reclass_data_dir
 
 {%- if master.system is defined %}
 
@@ -47,7 +49,6 @@
   file.symlink:
   - target: /srv/salt/env/{{ master.system.environment }}/{{ formula_name }}/metadata/service
   - require:
-    - git: reclass_data_source
     - file: /srv/salt/reclass/classes/service
 
 {%- endfor %}
@@ -68,7 +69,6 @@
   - target: /usr/share/salt-formulas/env/_formulas/{{ formula_name }}/metadata/service
   {%- endif %}
   - require:
-    - git: reclass_data_source
     - file: /srv/salt/reclass/classes/service
 
 {%- endif %}
@@ -81,4 +81,4 @@
 
 {%- endif %}
 
-{%- endif %}
\ No newline at end of file
+{%- endif %}
diff --git a/salt/master/service.sls b/salt/master/service.sls
index cba4f2c..3ca6d1c 100644
--- a/salt/master/service.sls
+++ b/salt/master/service.sls
@@ -28,7 +28,7 @@
   - watch_in:
     - service: salt_master_service
 
-{%- if master.peer is defined %}
+{%- if master.user is defined %}
 
 /etc/salt/master.d/_acl.conf:
   file.managed:
@@ -40,6 +40,10 @@
   - watch_in:
     - service: salt_master_service
 
+{%- endif %}
+
+{%- if master.peer is defined %}
+
 /etc/salt/master.d/_peer.conf:
   file.managed:
   - source: salt://salt/files/_peer.conf
@@ -50,7 +54,6 @@
   - watch_in:
     - service: salt_master_service
 
-
 {%- endif %}
 
 salt_master_service:
diff --git a/salt/meta/salt.yml b/salt/meta/salt.yml
index 4f3a32b..81225fe 100644
--- a/salt/meta/salt.yml
+++ b/salt/meta/salt.yml
@@ -1,9 +1,10 @@
 orchestrate:
   master:
-    priority: 10
-  minion:
-    priority: 20
-  syndic:
-    priority: 200
-  control:
+    priority: 60
+  minion: 
+    priority: 70
+  control: 
     priority: 400
+    require:
+    - salt: salt.master
+
diff --git a/salt/meta/sensu.yml b/salt/meta/sensu.yml
index cac0fdf..fc529c9 100644
--- a/salt/meta/sensu.yml
+++ b/salt/meta/sensu.yml
@@ -1,6 +1,6 @@
 check:
   local_salt_master_proc:
-    command: "PATH=$PATH:/usr/lib64/nagios/plugins:/usr/lib/nagios/plugins check_procs -C salt-master -u root -c 1:10"
+    command: "PATH=$PATH:/usr/lib64/nagios/plugins:/usr/lib/nagios/plugins check_procs -C salt-master -u root -c 1:15"
     interval: 60
     occurrences: 1
     subscribers:
diff --git a/salt/minion/cert.sls b/salt/minion/cert.sls
index 8dc2e45..adea4f6 100644
--- a/salt/minion/cert.sls
+++ b/salt/minion/cert.sls
@@ -69,7 +69,7 @@
     - watch:
       - x509: {{ cert_file }}
 
-{%- for ca_path,ca_cert in salt['mine.get'](cert.host, 'x509.get_pem_entries')[cert.host].iteritems() %}
+{%- for ca_path,ca_cert in salt['mine.get'](cert.host, 'x509.get_pem_entries').get(cert.host, {}).iteritems() %}
 
 {%- if '/etc/pki/ca/'+cert.authority in ca_path %}
 
diff --git a/salt/minion/grains.sls b/salt/minion/grains.sls
index 972577c..86e9987 100644
--- a/salt/minion/grains.sls
+++ b/salt/minion/grains.sls
@@ -13,25 +13,32 @@
   - require:
     - {{ minion.install_state }}
 
-salt_minion_grains_placeholder:
-  file.touch:
-  - name: /etc/salt/grains.d/placeholder
+salt_minion_grains_files:
+  file.managed:
+  - names:
+    - /etc/salt/grains
+    - /etc/salt/grains.d/placeholder
+  - replace: False
   - require:
     - file: salt_minion_grains_dir
 
+{%- set new_grains = salt['cmd.run']('cat /etc/salt/grains.d/*') %}
+{%- set old_grains = salt['cmd.run']('cat /etc/salt/grains') %}
+
+{%- if new_grains != old_grains %}
+
 salt_minion_grains_file:
   cmd.run:
   - name: cat /etc/salt/grains.d/* > /etc/salt/grains
   - require:
-    - file: salt_minion_grains_placeholder
-  - watch_in:
-    - service: salt_minion_service
+    - file: salt_minion_grains_files
 
 salt_minion_grains_publish:
   module.run:
-  - name: mine.send
-  - name: grains.items
+  - name: mine.update
   - require:
     - cmd: salt_minion_grains_file
 
 {%- endif %}
+
+{%- endif %}