Merge "README update"
diff --git a/Makefile b/Makefile
index d166862..3af6b3d 100644
--- a/Makefile
+++ b/Makefile
@@ -34,6 +34,7 @@
 	@echo "make release-major  - Generate new major release"
 	@echo "make release-minor  - Generate new minor release"
 	@echo "make changelog      - Show changes since last release"
+	@echo "make test-model-validate      - Run salt jsonschema validation"
 
 install:
 	# Formula
@@ -41,6 +42,7 @@
 	cp -a $(FORMULANAME) $(DESTDIR)/$(SALTENVDIR)/
 	[ ! -d _modules ] || cp -a _modules $(DESTDIR)/$(SALTENVDIR)/
 	[ ! -d _states ] || cp -a _states $(DESTDIR)/$(SALTENVDIR)/ || true
+	[ ! -d _engines ] || cp -a _engines $(DESTDIR)/$(SALTENVDIR)/ || true
 	[ ! -d _grains ] || cp -a _grains $(DESTDIR)/$(SALTENVDIR)/ || true
 	# Metadata
 	[ -d $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME) ] || mkdir -p $(DESTDIR)/$(RECLASSDIR)/service/$(FORMULANAME)
@@ -52,6 +54,10 @@
 test:
 	[ ! -d tests ] || (cd tests; ./run_tests.sh)
 
+test-model-validate:
+	# TODO make it actually fail
+	[ ! -d $(FORMULANAME)/schemas/ ] || (cd tests; ./run_tests.sh model-validate)
+
 release-major: check-changes
 	@echo "Current version is $(VERSION), new version is $(NEW_MAJOR_VERSION)"
 	@[ $(VERSION_MAJOR) != $(NEW_MAJOR_VERSION) ] || (echo "Major version $(NEW_MAJOR_VERSION) already released, nothing to do. Do you want release-minor?" && exit 1)
@@ -119,4 +125,4 @@
 	[ ! -x "$(shell which kitchen)" ] || kitchen destroy
 	[ ! -d .kitchen ] || rm -rf .kitchen
 	[ ! -d tests/build ] || rm -rf tests/build
-	[ ! -d build ] || rm -rf build
+	[ ! -d build ] || rm -rf build
\ No newline at end of file
diff --git a/README.rst b/README.rst
index 6122233..869ea5b 100644
--- a/README.rst
+++ b/README.rst
@@ -71,7 +71,9 @@
     jenkins:
       master:
         mode: EXCLUSIVE
-        # Do not manage config.xml from Salt, use UI instead
+        java_args: -Xms256m -Xmx1g
+        # Do not manage any xml config files via Salt, use UI instead
+        # Including config.xml and any plugin xml's.
         no_config: true
         slaves:
           - name: slave01
@@ -447,12 +449,16 @@
 
     jenkins:
       client:
+        plugin_remove_unwanted: false
+        plugin_force_remove: false
         plugin:
-          swarm:
-            restart: false
-          hipchat:
+          plugin1: 1.2.3
+          plugin2:
+          plugin3: {}
+          plugin4:
+            version: 3.2.1
             enabled: false
-            restart: true
+          plugin5: absent
 
 Adding plugin params to job:
 
@@ -471,11 +477,7 @@
                 categories:
                   - my_throuttle_category
         plugin:
-          swarm:
-            restart: false
-          hipchat:
-            enabled: false
-            restart: true
+          throttle-concurrents:
 
 LDAP configuration (depends on LDAP plugin):
 
@@ -806,6 +808,10 @@
             email: "jenkins@domain.local"
             auth_key_file: "/var/jenkins_home/.ssh/id_rsa"
             frontendURL: "https://gerrit.domain.local"
+            build_current_patches_only: true
+            abort_new_patchsets: false
+            abort_manual_patchsets: false
+            abort_same_topic: false
             authkey: |
               SOMESSHKEY
           server2:
@@ -815,6 +821,10 @@
             email: "jenkins@domain.local"
             auth_key_file: "/var/jenkins_home/.ssh/id_rsa"
             frontendURL: "https://gerrit2.domain.local"
+            build_current_patches_only: true
+            abort_new_patchsets: false
+            abort_manual_patchsets: false
+            abort_same_topic: false
             authkey: |
               SOMESSHKEY
 
diff --git a/_modules/jenkins_common.py b/_modules/jenkins_common.py
index 377a71f..225042f 100644
--- a/_modules/jenkins_common.py
+++ b/_modules/jenkins_common.py
@@ -111,10 +111,14 @@
     tokenReq = requests.get("%s/crumbIssuer/api/json" % jenkins_url,
                             auth=(jenkins_user, jenkins_password) if jenkins_user else None)
     if tokenReq.status_code == 200:
+        logger.debug("Got Jenkins API crumb: %s", tokenReq.json())
         return tokenReq.json()
-    elif tokenReq.status_code in [404, 401]:
+    elif tokenReq.status_code in [404, 401, 502, 503]:
         # 404 means CSRF security is disabled, so api crumb is not necessary,
         # 401 means unauthorized
+        # 50x means jenkins is unavailabe - fail in call_groovy_script, but
+        #     not here, to handle exception in state
+        logger.debug("Got error %s: %s", str(tokenReq.status_code), tokenReq.reason)
         return None
     else:
         raise Exception("Cannot obtain Jenkins API crumb. Status code: %s. Text: %s" %
diff --git a/_states/jenkins_artifactory.py b/_states/jenkins_artifactory.py
index daccc2e..d88b84a 100644
--- a/_states/jenkins_artifactory.py
+++ b/_states/jenkins_artifactory.py
@@ -8,6 +8,9 @@
 import org.jfrog.hudson.*
 def inst = Jenkins.getInstance()
 def desc = inst.getDescriptor("org.jfrog.hudson.ArtifactoryBuilder")
+if (! desc.useCredentialsPlugin ) {
+    desc.useCredentialsPlugin = true
+}
 // empty artifactory servers is not empty list but null, but find can be called on null
 def server =  desc.getArtifactoryServers().find{it -> it.name.equals("${name}")}
 if(server &&
diff --git a/_states/jenkins_gerrit.py b/_states/jenkins_gerrit.py
index 26b1259..a2b0e95 100644
--- a/_states/jenkins_gerrit.py
+++ b/_states/jenkins_gerrit.py
@@ -15,7 +15,10 @@
 
 
 def present(name, hostname, username, frontendurl, auth_key_file, authkey,
-            port="29418", auth_key_file_password=None, email="", proxy=""):
+            port="29418", build_current_patches_only="false",
+            abort_new_patchsets="false", abort_manual_patchsets="false",
+            abort_same_topic="false", auth_key_file_password=None, email="",
+            proxy=""):
     """
     Jenkins gerrit-trigger state method
 
@@ -26,6 +29,10 @@
     :param port: server ssh port
     :param proxy: proxy url (optional)
     :param frontendurl: server frontend URL
+    :param build_current_patches_only: build current patches only (optional)
+    :abort_new_patchsets: abort new patchsets (optional)
+    :abort_manual_patchsets: abort manual patchsets (optional)
+    :abort_same_topic: abort same topic (optional)
     :param auth_key_file: path to key file
     :param authkey: ssh key
     :param auth_key_file_password: password for keyfile (optional)
@@ -35,7 +42,7 @@
         'salt://jenkins/files/groovy/gerrit.template',
         __env__)
     return __salt__['jenkins_common.api_call'](name, template,
-                        ["CREATED", "EXISTS"],
+                        ["CREATED", "CHANGED", "SKIPPED"],
                         {
                             "name": name,
                             "hostname": hostname,
@@ -44,8 +51,12 @@
                             "username": username,
                             "email": email if email else "",
                             "frontendurl": frontendurl,
+                            "build_current_patches_only": build_current_patches_only if build_current_patches_only else "false",
+                            "abort_new_patchsets": abort_new_patchsets if abort_new_patchsets else "false",
+                            "abort_manual_patchsets": abort_manual_patchsets if abort_manual_patchsets else "false",
+                            "abort_same_topic": abort_same_topic if abort_same_topic else "false",
                             "auth_key_file": auth_key_file,
                             "authkey": authkey,
-                            "auth_key_file_password": auth_key_file_password if auth_key_file_password else None
+                            "auth_key_file_password": auth_key_file_password if auth_key_file_password else ""
                         },
                         "Gerrit server")
diff --git a/_states/jenkins_plugin.py b/_states/jenkins_plugin.py
index 0f80d51..a96afed 100644
--- a/_states/jenkins_plugin.py
+++ b/_states/jenkins_plugin.py
@@ -1,81 +1,9 @@
+import json
 import logging
+import time
 
-logger = logging.getLogger(__name__)
 
-install_plugin_groovy = """\
-import jenkins.model.*
-import java.util.logging.Logger
-
-def logger = Logger.getLogger("")
-def installed = false
-def exists = false
-def pluginName="${plugin}"
-def instance = Jenkins.getInstance()
-def pm = instance.getPluginManager()
-def uc = instance.getUpdateCenter()
-def needUpdateSites(maxOldInSec = 1800){
-  long oldestTs = 0
-  for (UpdateSite s : Jenkins.instance.updateCenter.siteList) {
-    if(oldestTs == 0 || s.getDataTimestamp()<oldestTs){
-       oldestTs = s.getDataTimestamp()
-    }
-  }
-   return (System.currentTimeMillis()-oldestTs)/1000 > maxOldInSec
-}
-
-if (!pm.getPlugin(pluginName)) {
-  if(needUpdateSites()) {
-     uc.updateAllSites()
-  }
-  def plugin = uc.getPlugin(pluginName)
-  if (plugin) {
-    plugin.deploy()
-    installed = true
-  }
-}else{
-    exists = true
-    print("EXISTS")
-}
-if (installed) {
-  instance.save()
-  if(${restart}){
-      instance.doSafeRestart()
-   }
-  print("INSTALLED")
-}else if(!exists){
-  print("FAILED")
-}
-"""  # noqa
-
-remove_plugin_groovy = """
-import jenkins.model.*
-import java.util.logging.Logger
-
-def logger = Logger.getLogger("")
-def installed = false
-def initialized = false
-
-def pluginName="${plugin}"
-def instance = Jenkins.getInstance()
-def pm = instance.getPluginManager()
-
-def actPlugin = pm.getPlugin(pluginName)
-if (!actPlugin) {
-   def pluginToInstall = Jenkins.instance.updateCenter.getPlugin(pluginName)
-   if(!pluginToInstall){
-      print("FAILED")
-   }else{
-      print("NOT PRESENT")
-   }
-} else {
-   actPlugin.disable()
-   actPlugin.archive.delete()
-   if({restart}){
-      instance.doSafeRestart()
-   }
-   print("REMOVED")
-}
-"""  # noqa
+log = logging.getLogger(__name__)
 
 
 def __virtual__():
@@ -85,63 +13,46 @@
     if 'jenkins_common.call_groovy_script' not in __salt__:
         return (
             False,
-            'The jenkins_plugin state module cannot be loaded: '
+            'The jenkins_node state module cannot be loaded: '
             'jenkins_common not found')
     return True
 
-
-def present(name, restart=False):
+def managed(name, plugins, remove_unwanted=False, force_remove=False):
     """
-    Jenkins plugin present state method, for installing plugins
+    Manage jenkins plugins
 
-    :param name: plugin name
-    :param restart: do you want to restart jenkins after plugin install?
+    :param name: salt resource name (usually 'jenkins_plugin_manage')
+    :param plugins: map containing plugin names and parameters
+    :param remove_unwanted: whether to remove not listed plugins
+    :param force_remove: force removing plugins recursively with all dependent plugins
     :returns: salt-specified state dict
     """
-    return _plugin_call(name, restart, install_plugin_groovy, [
-                        "INSTALLED", "EXISTS"])
+    log.info('Managing jenkins plugins')
+    template = __salt__['jenkins_common.load_template'](
+        'salt://jenkins/files/groovy/plugin.template',
+        __env__)
+    result = __salt__['jenkins_common.api_call'](name, template,
+                        [ 'UPDATED', 'NO CHANGES' ],
+                        {
+                            'plugin_list': json.dumps(plugins),
+                            'clean_unwanted': remove_unwanted,
+                            'force_remove': force_remove
+                        },
+                        'Manage Jenkins plugins')
+    log.debug('Got result: ' + json.dumps(result))
 
+    log.info('Checking if restart is required...')
+    # While next code is successful, we should wait for jenkins shutdown
+    # either:
+    #   - false returned by isQuietingDown()
+    #   - any error meaning that jenkins is unavailable (restarting)
+    wait = { 'result': True }
+    while (wait['result']):
+        wait = __salt__['jenkins_common.api_call']('jenkins_restart_wait',
+                'println Jenkins.instance.isQuietingDown()', [ 'true' ], {},
+                'Wait for jenkins restart')
+        if (wait['result']):
+            log.debug('Jenkins restart is required. Waiting...')
+        time.sleep(5)
 
-def absent(name, restart=False):
-    """
-    Jenkins plugin absent state method, for removing plugins
-
-    :param name: plugin name
-    :param restart: do you want to restart jenkins after plugin remove?
-    :returns: salt-specified state dict
-    """
-    return _plugin_call(name, restart, remove_plugin_groovy, [
-                        "REMOVED", "NOT PRESENT"])
-
-
-def _plugin_call(name, restart, template, success_msgs):
-    test = __opts__['test']  # noqa
-    ret = {
-        'name': name,
-        'changes': {},
-        'result': False,
-        'comment': '',
-    }
-    result = False
-    if test:
-        status = success_msgs[0]
-        ret['changes'][name] = status
-        ret['comment'] = 'Jenkins plugin %s %s' % (name, status.lower())
-    else:
-        call_result = __salt__['jenkins_common.call_groovy_script'](
-            template, {"plugin": name, "restart": "true" if restart else "false"})
-        if call_result["code"] == 200 and call_result["msg"] in success_msgs:
-            status = call_result["msg"]
-            if status == success_msgs[0]:
-                ret['changes'][name] = status
-            ret['comment'] = 'Jenkins plugin %s %s%s' % (name, status.lower(
-            ), ", jenkins restarted" if status == success_msgs[0] and restart else "")
-            result = True
-        else:
-            status = 'FAILED'
-            logger.error(
-                "Jenkins plugin API call failure: %s", call_result["msg"])
-            ret['comment'] = 'Jenkins plugin API call failure: %s' % (call_result[
-                "msg"])
-    ret['result'] = None if test else result
-    return ret
+    return result
diff --git a/jenkins/client/gerrit.sls b/jenkins/client/gerrit.sls
index c540b20..24c4abf 100644
--- a/jenkins/client/gerrit.sls
+++ b/jenkins/client/gerrit.sls
@@ -11,6 +11,10 @@
   - email: {{ gerrit.get('email', '') }}
   - auth_key_file: {{ gerrit.get('auth_key_file', '') }}
   - frontendurl: {{ gerrit.get('frontendURL','') }}
+  - build_current_patches_only: {{ gerrit.get('build_current_patches_only', 'false') }}
+  - abort_new_patchsets: {{ gerrit.get('abort_new_patchsets', 'false') }}
+  - abort_manual_patchsets: {{ gerrit.get('abort_manual_patchsets', 'false') }}
+  - abort_same_topic: {{ gerrit.get('abort_same_topic', 'false') }}
   {%- if gerrit.authkey is defined %}
   - authkey: |
       {{ gerrit.get('authkey','')|indent(6) }}
diff --git a/jenkins/client/plugin.sls b/jenkins/client/plugin.sls
index f511ec0..bb888d2 100644
--- a/jenkins/client/plugin.sls
+++ b/jenkins/client/plugin.sls
@@ -1,15 +1,19 @@
 {% from "jenkins/map.jinja" import client with context %}
-{% for name, plug in client.get('plugin',{}).iteritems() %}
-{% if plug.get('enabled', True) %}
-plugin_{{ name }}:
-  jenkins_plugin.present:
-  - name: {{ plug.get('name', name) }}
-  - restart: {{ plug.get('restart', False) }}
-{% else %}
-plugin_{{ name }}_disable:
-   jenkins_plugin.absent:
-   - name: {{ plug.get('name', name) }}
-   - restart: {{ plug.get('restart', False) }}
-{% endif %}
-{% endfor %}
+{%- if client.plugin is defined %}
+jenkins_plugins:
+  jenkins_plugin.managed:
+  - plugins: {{ client.plugin }}
+  - remove_unwanted: {{ client.get('plugin_remove_unwanted', False) }}
+  - force_remove: {{ client.get('plugin_force_remove', False) }}
 
+jenkins_wait_functional:
+  cmd.script:
+  - source: salt://jenkins/files/wait4jenkins.sh
+  - shell: /bin/bash
+  - env:
+    - JENKINS_URL: "{{ client.master.get('proto', 'http') }}://{{ client.master.get('host', 'localhost') }}:{{ client.master.get('port', '8080') }}/login"
+    - WAIT_TIME: "300"
+    - INTERVAL: "5"
+  - require:
+    - jenkins_plugins
+{%- endif %}
diff --git a/jenkins/files/config.xml.user b/jenkins/files/config.xml.user
index 67c09b9..59a18a2 100644
--- a/jenkins/files/config.xml.user
+++ b/jenkins/files/config.xml.user
@@ -1,7 +1,7 @@
 {%- set user = pillar.jenkins.master.user.get(user_name) -%}
 <?xml version='1.0' encoding='UTF-8'?>
 <user>
-  <fullName>admin</fullName>
+  <fullName>{{ user_name }}</fullName>
   <properties>
     <hudson.model.PaneStatusProperties>
       <collapsed/>
diff --git a/jenkins/files/groovy/gerrit.template b/jenkins/files/groovy/gerrit.template
index 1895494..a6583a1 100644
--- a/jenkins/files/groovy/gerrit.template
+++ b/jenkins/files/groovy/gerrit.template
@@ -1,56 +1,91 @@
 #!groovy
-import jenkins.model.*;
-import net.sf.json.*;
-import com.sonyericsson.hudson.plugins.gerrit.trigger.*;
+import jenkins.model.Jenkins
+import com.sonyericsson.hudson.plugins.gerrit.trigger.GerritServer
+import com.sonyericsson.hudson.plugins.gerrit.trigger.PluginImpl
+import com.sonyericsson.hudson.plugins.gerrit.trigger.VerdictCategory
+import com.sonyericsson.hudson.plugins.gerrit.trigger.config.Config
 
-if ( Jenkins.instance.pluginManager.activePlugins.find { it.shortName == "gerrit-trigger" } != null ) {
-    def gerritPlugin = Jenkins.instance.getPlugin(com.sonyericsson.hudson.plugins.gerrit.trigger.PluginImpl.class)
-    gerritPlugin.getPluginConfig().setNumberOfReceivingWorkerThreads(3)
-    gerritPlugin.getPluginConfig().setNumberOfSendingWorkerThreads(1)
+def authkey = """${authkey}
+"""
+def serverName = "${name}"
 
-    def authkey = """${authkey}
-    """
+def newGerritConfig = new Config()
+newGerritConfig.with {
+setGerritHostName("${hostname}")
+setGerritUserName("${username}")
+setGerritFrontEndURL("${frontendurl}")
+setGerritSshPort(("${port}").toInteger())
+setGerritProxy("${proxy}")
+setGerritEMail("${email}")
+buildCurrentPatchesOnly.setEnabled(("${build_current_patches_only}").toBoolean())
+buildCurrentPatchesOnly.setAbortNewPatchsets(("${abort_new_patchsets}").toBoolean())
+buildCurrentPatchesOnly.setAbortManualPatchsets(("${abort_manual_patchsets}").toBoolean())
+buildCurrentPatchesOnly.setAbortSameTopic(("${abort_same_topic}").toBoolean())
+setGerritAuthKeyFilePassword("${auth_key_file_password}")
+}
 
-    def serverName = "${name}"
-    def gerritServer = new GerritServer(serverName)
+def gerritAuthKeyFile = new File("${auth_key_file}")
+gerritAuthKeyFile.write(authkey)
+if ( gerritAuthKeyFile.exists() ) {
+     newGerritConfig.setGerritAuthKeyFile(gerritAuthKeyFile)
+}
 
+def newCategories = newGerritConfig.getCategories()
+if (!newCategories) {
+    categories = new LinkedList<VerdictCategory>()
+}
+if (newCategories.isEmpty()) {
+    newCategories.add(new VerdictCategory('Code-Review', 'Code Review'))
+    newCategories.add(new VerdictCategory('Verified', 'Verified'))
+}
+
+Boolean compareObjects( Object a, b) {
+    String aXML = Jenkins.XSTREAM.toXML(a).replaceAll(/\{AQA[^\}]+\}/) {
+                      hudson.util.Secret.decrypt(it) }
+    String bXML = Jenkins.XSTREAM.toXML(b).replaceAll(/\{AQA[^\}]+\}/) {
+                      hudson.util.Secret.decrypt(it) }
+    return aXML == bXML
+}
+
+if ( Jenkins.instance.pluginManager.activePlugins.find { it.shortName == 'gerrit-trigger' } != null ) {
+    def gerritPlugin = Jenkins.instance.getPlugin(PluginImpl)
     def gerritTriggerPlugin = PluginImpl.getInstance()
     def gerritServers = gerritTriggerPlugin.getServerNames()
+
+    def gerritServer = gerritPlugin.getServer(serverName)
+    def newGerritServer = new GerritServer(serverName)
+
+    newGerritConfig.setCategories(newCategories)
+
     def gerritServerExists = false
     gerritServers.each {
         serverName = (String) it
-        if ( serverName == gerritServer.getName() ) {
+        if ( serverName == newGerritServer.getName() ) {
             gerritServerExists = true
         }
     }
-    if (!gerritServerExists){
 
-        def triggerConfig = new config.Config()
+    newGerritServer.setConfig(newGerritConfig)
 
-        triggerConfig.setGerritHostName("${hostname}")
-        triggerConfig.setGerritUserName("${username}")
-        triggerConfig.setGerritFrontEndURL("${frontendurl}")
-        triggerConfig.setGerritSshPort(${port})
-        triggerConfig.setGerritProxy("${proxy}")
-        triggerConfig.setGerritEMail("${email}")
-
-        def gerritAuthKeyFile = new File("${auth_key_file}")
-        gerritAuthKeyFile.write(authkey)
-        if ( gerritAuthKeyFile.exists() ) {
-            triggerConfig.setGerritAuthKeyFile(gerritAuthKeyFile)
+    if (gerritServerExists){
+        if (compareObjects(gerritServer, newGerritServer)) {
+            print('SKIPPED')
+        } else {
+            if (gerritServer) {
+                gerritServer.setConfig(newGerritServer.getConfig())
+            } else {
+                gerritPlugin.addServer(newGerritServer)
+            }
+            newGerritServer.start()
+            newGerritServer.startConnection()
+            Jenkins.instance.save()
+            print('CHANGED')
         }
-
-        triggerConfig.setGerritAuthKeyFilePassword("${auth_key_file_password}")
-
-        gerritServer.setConfig(triggerConfig)
-
-        gerritPlugin.addServer(gerritServer)
-        gerritServer.start()
-        gerritServer.startConnection()
+    } else {
+        gerritPlugin.addServer(newGerritServer)
+        print('CREATED')
+        newGerritServer.start()
+        newGerritServer.startConnection()
         Jenkins.instance.save()
-        print("CREATED")
-    }
-    else {
-        print("EXISTS")
     }
 }
diff --git a/jenkins/files/groovy/plugin.template b/jenkins/files/groovy/plugin.template
new file mode 100644
index 0000000..24591c4
--- /dev/null
+++ b/jenkins/files/groovy/plugin.template
@@ -0,0 +1,163 @@
+#!groovy
+
+import groovy.json.JsonSlurper
+
+// List of plugins to manage
+String pluginListJson = '''${plugin_list}'''
+LinkedHashMap pluginList = new JsonSlurper().parseText(pluginListJson)
+
+// Whether to remove plugins not listed in pluginList
+boolean cleanUnwanted = "${clean_unwanted}".toBoolean()
+
+// Whether to recursively remove plugins with all dependent plugins
+boolean forceRemove = "${force_remove}".toBoolean()
+
+// Removing and enabling/disabling plugins doesn't set isRestartRequiredForCompletion()
+boolean needRestart = false
+
+def pm = Jenkins.instance.pluginManager
+def uc = Jenkins.instance.updateCenter
+
+ArrayList pluginsToInstall = []
+ArrayList pluginsToRemove = []
+
+LinkedHashMap allPluginDeps = [:]
+
+// Compare version strings and return
+//   -1 if first parameter is less than second one
+//    0 if both parameters are equal
+//    1 if first parameter is greather than second one
+int versionCmp(String versionOne, String versionTwo){
+
+    List verA = versionOne.tokenize('.')
+    List verB = versionTwo.tokenize('.')
+
+    int commonIndices = Math.min(verA.size(), verB.size())
+
+    for (int i = 0; i < commonIndices; ++i){
+      int numA = verA[i].toInteger()
+      int numB = verB[i].toInteger()
+      if (numA != numB) {
+        return numA <=> numB
+      }
+    }
+
+    // If we got this far then all the common indices are identical, so whichever version is longer must be more recent
+    return verA.size() <=> verB.size()
+}
+
+// Return all dependency plugins for the specified one (recursiverly)
+LinkedHashMap getPluginDeps(String pluginName, String pluginVersion = null){
+  LinkedHashMap pluginDeps = [:]
+  def pluginToProbe = Jenkins.instance.updateCenter.getPlugin(pluginName)
+  pluginToProbe.dependencies.each { p, v ->
+    pluginDeps << getPluginDeps(p, v)
+  }
+  // FIXME: need to get minimal version because Jenkins senses specified version as 'at least'
+  pluginDeps[pluginName] = pluginVersion
+  return pluginDeps
+}
+
+// Remove plugin with all dependent plugins (!!! DANGEROUS !!!)
+ArrayList<String> removePluginRecursive(String pluginName, boolean force = false){
+  ArrayList removedPlugins = []
+  def pluginToRemove = Jenkins.instance.pluginManager.getPlugin(pluginName)
+  if (pluginToRemove) {
+    // Get plugin dependants
+    Set pluginsDependent = pluginToRemove.getDependants()
+    // ... and remove it all
+    if (pluginsDependent && force){
+      pluginsDependent.each { removedPlugins += removePluginRecursive(it) }
+    }
+    // Remove requested plugin if it is still not deleted
+    // and if requested plugin does not have dependants
+    if (! pluginToRemove.isDeleted() && ! pluginToRemove.getDependants()) {
+      pluginToRemove.doDoUninstall()
+      removedPlugins << pluginName
+    }
+  }
+  return removedPlugins
+}
+
+String pluginName
+def pluginInfo
+def pluginAvailable
+def pluginInstalled
+
+for (plugin in pluginList) {
+  pluginName = plugin.key
+  pluginInfo = plugin.value ?: new LinkedHashMap()
+
+  // Guess contents of pluginInfo if it is string
+  if (pluginInfo.getClass() == String){
+    if (pluginInfo == 'absent') {
+    // remove plugin
+      pluginsToRemove << pluginName
+      continue
+    } else if (pluginInfo ==~ /^[0-9.]+$/){
+    // pluginInfo is version
+        pluginInfo = new LinkedHashMap([ version: pluginInfo ])
+    } else {
+    // clean incorrect pluginInfo
+        pluginInfo = new LinkedHashMap()
+    }
+  }
+
+  pluginAvailable = uc.getPlugin(pluginName)
+  pluginInstalled = pluginAvailable.getInstalled()
+  // If plugin installed
+  if (pluginInstalled){
+    // ... and pluginInfo contains version
+    if (pluginInfo.get('version')){
+      // ... and installed plugin version is lower than version from pluginInfo
+      if (versionCmp(pluginInstalled.getVersion(), pluginInfo.get('version')) < 0){
+        // upgrade plugin
+        pluginsToInstall << pluginAvailable
+      }
+    }
+    // ... plugin is active and pluginInfo has enable=false
+    if (pluginInstalled.isActive() && ! pluginInfo.get('enabled', true).toBoolean()){
+      // disable plugin
+      pluginInstalled.disable()
+      needRestart = true
+    }
+    // ... plugin is not active and pluginInfo has enabled=true
+    if (! pluginInstalled.isActive() && pluginInfo.get('enabled', true).toBoolean()){
+      // enable plugin
+      pluginInstalled.enable()
+      needRestart = true
+    }
+  } else {
+    // install plugin
+    pluginsToInstall << pluginAvailable
+  }
+
+  // Collect all plugin dependencies to decide about unwanted installed plugins
+  allPluginDeps << getPluginDeps(pluginName, pluginInfo ? pluginInfo.get('version') : null)
+}
+
+// Deploy plugins by list if any
+pluginsToInstall.each {
+  it.deploy(false).get()
+}
+
+// Remove unwanted plugins
+if (cleanUnwanted){
+  pluginsToRemove += pm.getPlugins()*.getShortName()
+}
+
+// Remove plugins by list if any keeping required by some others
+(pluginsToRemove - allPluginDeps.keySet()).each {
+  if (removePluginRecursive(it, forceRemove)) {
+    needRestart = true
+  }
+}
+
+// Restart Jenkins if needed
+if (uc.isRestartRequiredForCompletion() || needRestart){
+  Jenkins.instance.doSafeRestart()
+  println 'UPDATED'
+} else {
+  // No changes
+  println 'NO CHANGES'
+}
diff --git a/jenkins/files/jenkins b/jenkins/files/jenkins
index 022627b..e311282 100644
--- a/jenkins/files/jenkins
+++ b/jenkins/files/jenkins
@@ -1,3 +1,4 @@
+{%- from "jenkins/map.jinja" import master with context -%}
 # defaults for jenkins continuous integration server
 
 # pulled in from the init script; makes things easier.
@@ -8,7 +9,7 @@
 
 # arguments to pass to java
 #JAVA_ARGS="-Xmx256m"
-JAVA_ARGS="-Djava.net.preferIPv4Stack=true -Djenkins.install.runSetupWizard=false" # make jenkins listen on IPv4 address and disable setup wizard
+JAVA_ARGS="{{ master.get('java_args', '') }} -Djava.net.preferIPv4Stack=true -Djenkins.install.runSetupWizard=false" # make jenkins listen on IPv4 address and disable setup wizard
 
 PIDFILE=/var/run/jenkins/jenkins.pid
 
diff --git a/jenkins/files/jobs/workflow-scm.xml b/jenkins/files/jobs/workflow-scm.xml
index 2aca88c..5998ff2 100644
--- a/jenkins/files/jobs/workflow-scm.xml
+++ b/jenkins/files/jobs/workflow-scm.xml
@@ -31,7 +31,17 @@
       </branches>
       <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
       <submoduleCfg class="list"/>
-      <extensions/>
+      <extensions>
+        {%- if job.scm.get('clone') %}
+        <hudson.plugins.git.extensions.impl.CloneOption>
+          <shallow>{{ job.scm.clone.get('shallow', False)|lower }}</shallow>
+          <noTags>{{ job.scm.clone.get('no_tags', False)|lower }}</noTags>
+          <reference>{{ job.scm.clone.get('reference', "") }}</reference>
+          <depth>{{ job.scm.clone.get('depth', 0) }}</depth>
+          <honorRefspec>{{ job.scm.clone.get('honor_refspec', False)|lower }}</honorRefspec>
+        </hudson.plugins.git.extensions.impl.CloneOption>
+        {%- endif %}
+      </extensions>
     </scm>
     {%- endif %}
     <scriptPath>{{ job.scm.script|default('Jenkinsfile') }}</scriptPath>
diff --git a/jenkins/files/wait4jenkins.sh b/jenkins/files/wait4jenkins.sh
new file mode 100755
index 0000000..f8eae77
--- /dev/null
+++ b/jenkins/files/wait4jenkins.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+: "${JENKINS_URL:=http://localhost:8080}"
+: "${WAIT_TIME:=300}"
+: "${INTERVAL:=5}"
+
+while [ $((WAIT_TIME -= INTERVAL)) -ge 0 ]; do
+    if curl -fs -o /dev/null "$JENKINS_URL"; then
+        STATUS=0
+        break
+    fi
+    sleep $INTERVAL
+done
+
+exit ${STATUS:-124}
diff --git a/jenkins/master/service.sls b/jenkins/master/service.sls
index c3e717b..845cff0 100644
--- a/jenkins/master/service.sls
+++ b/jenkins/master/service.sls
@@ -19,6 +19,7 @@
     - pkg: jenkins_packages
 
 {%- if master.get('no_config', False) == False %}
+
 {{ master.home }}/config.xml:
   file.managed:
   - source: salt://jenkins/files/config.xml
@@ -26,7 +27,6 @@
   - user: jenkins
   - watch_in:
     - service: jenkins_master_service
-{%- endif %}
 
 {%- if master.update_site_url is defined %}
 
@@ -64,6 +64,8 @@
 
 {%- endif %}
 
+{%- endif %}
+
 {%- if master.get('sudo', false) %}
 
 /etc/sudoers.d/99-jenkins-user:
@@ -86,9 +88,14 @@
     - file: {{ master.home }}/hudson.model.UpdateCenter.xml
 
 jenkins_service_running:
-  cmd.wait:
-    - name: "i=0; while true; do curl -s -f http://localhost:{{ master.http.port }}/login >/dev/null && exit 0; [ $i -gt 60 ] && exit 1; sleep 5; done"
-    - watch:
-      - service: jenkins_master_service
+  cmd.script:
+  - source: salt://jenkins/files/wait4jenkins.sh
+  - shell: /bin/bash
+  - env:
+    - JENKINS_URL: "http://localhost:{{ master.http.port }}/login"
+    - WAIT_TIME: "300"
+    - INTERVAL: "5"
+  - onchanges:
+    - service: jenkins_master_service
 
 {%- endif %}
diff --git a/jenkins/meta/config.yml b/jenkins/meta/config.yml
index ae730c6..23c8c85 100644
--- a/jenkins/meta/config.yml
+++ b/jenkins/meta/config.yml
@@ -7,7 +7,6 @@
     path: {{ master.home }}/config.xml
     source: "salt://jenkins/files/config.xml"
     template: jinja
-  {%- endif %}
 
   {%- if master.update_site_url is defined %}
   hudson.model.UpdateCenter.xml:
@@ -41,6 +40,8 @@
 
   {%- endif %}
 
+  {%- endif %}
+
 
 {#-
   vim: syntax=jinja
diff --git a/jenkins/meta/salt.yml b/jenkins/meta/salt.yml
index 4cad34d..3be4783 100644
--- a/jenkins/meta/salt.yml
+++ b/jenkins/meta/salt.yml
@@ -13,9 +13,9 @@
   {%- endif %}
 dependency:
   {%- if pillar.get('jenkins', {}).get('client') %}
-  {% from "jenkins/map.jinja" import client with context %}
-  {%- if client.enabled %}
+    {% from "jenkins/map.jinja" import client with context %}
+    {%- if client.enabled %}
   engine: pkg
   pkgs: {{ client.pkgs }}
-  {%- endif %}
+    {%- endif %}
   {%- endif %}
diff --git a/metadata.yml b/metadata.yml
index dcda74c..0954897 100644
--- a/metadata.yml
+++ b/metadata.yml
@@ -1,3 +1,8 @@
 name: "jenkins"
 version: "2017.8"
 source: "https://github.com/salt-formulas/salt-formula-jenkins"
+dependencies:
+- name: java
+  source: "https://github.com/salt-formulas/salt-formula-java"
+- name: linux
+  source: "https://github.com/salt-formulas/salt-formula-linux"
\ No newline at end of file
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 4a6a4cc..0e33c9e 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -12,7 +12,7 @@
       backupninja:
         enabled: true
       config:
-        enabled: true
+        enabled: false
       grafana:
         enabled: true
       prometheus:
diff --git a/tests/pillar/client.sls b/tests/pillar/client.sls
new file mode 100644
index 0000000..d1078a5
--- /dev/null
+++ b/tests/pillar/client.sls
@@ -0,0 +1,69 @@
+jenkins:
+  client:
+    enabled: true
+    master:
+      host: jenkins.example.com
+      port: 80
+      protocol: http
+    job:
+      jobname:
+        type: workflow
+        param:
+          bool_param:
+            type: boolean
+            description: true/false
+            default: true
+          string_param:
+            type: string
+            description: 1 liner
+            default: default_string
+          text_param:
+            type: text
+            description: multi-liner
+            default: default_text
+      jobname_scm:
+        type: workflow-scm
+        concurrent: false
+        scm:
+          type: git
+          url: https://github.com/jenkinsci/docker.git
+          branch: master
+          script: Jenkinsfile
+          github:
+            url: https://github.com/jenkinsci/docker
+            name: "Jenkins Docker Image"
+        trigger:
+          timer:
+            spec: "H H * * *"
+          github:
+          pollscm:
+            spec: "H/15 * * * *"
+          reverse:
+            projects:
+              - test1
+              - test2
+            state: SUCCESS
+        param:
+          bool_param:
+            type: boolean
+            description: true/false
+            default: true
+          string_param:
+            type: string
+            description: 1 liner
+            default: default_string
+          text_param:
+            type: text
+            description: multi-liner
+            default: default_text
+    approved_scripts:
+      - method groovy.json.JsonSlurperClassic parseText java.lang.String
+java:
+  environment:
+    enabled: true
+    version: '10'
+    release: '0.1'
+    build: '10'
+    oracle_hash: 'fb4372174a714e6b8c52526dc134031e'
+    platform: oracle-java
+    development: true
\ No newline at end of file
diff --git a/tests/pillar/master.sls b/tests/pillar/master.sls
new file mode 100644
index 0000000..00f0cdd
--- /dev/null
+++ b/tests/pillar/master.sls
@@ -0,0 +1,53 @@
+jenkins:
+  master:
+    enabled: true
+    mode: EXCLUSIVE
+    no_config: false
+    slaves:
+      - name: slave01
+        label: pbuilder
+        executors: 2
+      - name: slave02
+        label: image_builder
+        mode: EXCLUSIVE
+        executors: 2
+    views:
+      - name: "Package builds"
+        regex: "debian-build-.*"
+      - name: "Contrail builds"
+        regex: "contrail-build-.*"
+      - name: "Aptly"
+        regex: "aptly-.*"
+    plugins:
+    - name: slack
+    - name: extended-choice-parameter
+    - name: rebuild
+    - name: test-stability
+    email:
+      engine: "smtp"
+      host: "smtp.domain.com"
+      user: "user@domain.cz"
+      password: "smtp-password"
+      port: 25
+    http:
+      port: 80
+    approved_scripts:
+      - method groovy.json.JsonSlurperClassic parseText java.lang.String
+    user:
+      admin:
+        api_token: xxxxxxxxxx
+        password: admin_password
+        email: admin@domain.com
+      user01:
+        api_token: xxxxxxxxxx
+        password: user_password
+        email: user01@domain.com
+java:
+  environment:
+    enabled: true
+    version: '10'
+    release: '0.1'
+    build: '10'
+    oracle_hash: 'fb4372174a714e6b8c52526dc134031e'
+    platform: oracle-java
+    development: true
\ No newline at end of file
diff --git a/tests/pillar/slave.sls b/tests/pillar/slave.sls
new file mode 100644
index 0000000..fe00f46
--- /dev/null
+++ b/tests/pillar/slave.sls
@@ -0,0 +1,39 @@
+jenkins:
+  slave:
+    enabled: true
+    master:
+      host: jenkins.example.com
+      port: 80
+      protocol: http
+    user:
+      name: jenkins_slave
+      password: dexiech6AepohthaiHook2iesh7ol5ook4Ov3leid3yek6daid2ooNg3Ee2oKeYo
+    gpg:
+      keypair_id: A76882D3
+      public_key: |
+        -----BEGIN PGP PUBLIC KEY BLOCK-----
+        ...
+      private_key: |
+        -----BEGIN PGP PRIVATE KEY BLOCK-----
+        ...
+java:
+  environment:
+    enabled: true
+    version: '10'
+    release: '0.1'
+    build: '10'
+    oracle_hash: 'fb4372174a714e6b8c52526dc134031e'
+    platform: oracle-java
+    development: true
+linux:
+  system:
+    enabled: true
+    user:
+      not_a_jenkins:
+        enabled: true
+        name: notJenkins
+        sudo: false
+        uid: 9991
+        full_name: Not A. Jenkins
+        home: /home/notjenkins
+        home_dir_mode: 755
\ No newline at end of file
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
new file mode 100755
index 0000000..4a7ce6e
--- /dev/null
+++ b/tests/run_tests.sh
@@ -0,0 +1,274 @@
+#!/usr/bin/env bash
+
+###
+# Script requirments:
+#apt-get install -y python-yaml virtualenv git
+
+set -e
+[ -n "$DEBUG" ] && set -x
+
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+METADATA=${CURDIR}/../metadata.yml
+FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
+FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
+
+## Overrideable parameters
+PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
+BUILDDIR=${BUILDDIR:-${CURDIR}/build}
+VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
+MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
+DEPSDIR=${BUILDDIR}/deps
+SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
+
+SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
+SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
+SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
+SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
+SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
+
+SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
+
+if [ "x${SALT_VERSION}" != "x" ]; then
+    PIP_SALT_VERSION="==${SALT_VERSION}"
+fi
+
+## Functions
+log_info() {
+    echo -e "[INFO] $*"
+}
+
+log_err() {
+    echo -e "[ERROR] $*" >&2
+}
+
+setup_virtualenv() {
+    log_info "Setting up Python virtualenv"
+    dependency_check virtualenv
+    virtualenv $VENV_DIR
+    source ${VENV_DIR}/bin/activate
+    python -m pip install salt${PIP_SALT_VERSION}
+    python -m pip install jsonschema
+    if [[ -f ${CURDIR}/pip_requirements.txt ]]; then
+       python -m pip install -r ${CURDIR}/pip_requirements.txt
+    fi
+}
+
+setup_mock_bin() {
+    # If some state requires a binary, a lightweight replacement for
+    # such binary can be put into MOCK_BIN_DIR for test purposes
+    if [ -d "${MOCK_BIN_DIR}" ]; then
+        PATH="${MOCK_BIN_DIR}:$PATH"
+        export PATH
+    fi
+}
+
+setup_pillar() {
+    [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
+    echo "base:" > ${SALT_PILLAR_DIR}/top.sls
+    for pillar in ${PILLARDIR}/*; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+        state_name=$(basename ${pillar%.sls})
+        echo -e "  ${state_name}:\n    - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
+    done
+}
+
+setup_salt() {
+    [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
+    [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
+    [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
+    [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
+
+    echo "base:" > ${SALT_FILE_DIR}/top.sls
+    for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+        state_name=$(basename ${pillar%.sls})
+        echo -e "  ${state_name}:\n    - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
+    done
+
+    cat << EOF > ${SALT_CONFIG_DIR}/minion
+file_client: local
+cachedir: ${SALT_CACHE_DIR}
+extension_modules:  ${SALT_CACHE_EXTMODS_DIR}
+verify_env: False
+minion_id_caching: False
+
+file_roots:
+  base:
+  - ${SALT_FILE_DIR}
+  - ${CURDIR}/..
+
+pillar_roots:
+  base:
+  - ${SALT_PILLAR_DIR}
+  - ${PILLARDIR}
+EOF
+}
+
+fetch_dependency() {
+    # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
+    dep_name="$(echo $1|cut -d : -f 1)"
+    dep_source="$(echo $1|cut -d : -f 2-)"
+    dep_root="${DEPSDIR}/$(basename $dep_source .git)"
+    dep_metadata="${dep_root}/metadata.yml"
+
+    dependency_check git
+    [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
+
+    log_info "Fetching dependency $dep_name"
+    [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
+    git clone $dep_source ${DEPSDIR}/$(basename $dep_source .git)
+    ln -s ${dep_root}/${dep_name} ${SALT_FILE_DIR}/${dep_name}
+
+    METADATA="${dep_metadata}" install_dependencies
+}
+
+link_modules(){
+    # Link modules *.py files to temporary salt-root
+    local SALT_ROOT=${1:-$SALT_FILE_DIR}
+    local SALT_ENV=${2:-$DEPSDIR}
+
+    mkdir -p "${SALT_ROOT}/_modules/"
+    # from git, development versions
+    find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
+      ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
+    done
+    salt_run saltutil.sync_all
+}
+
+install_dependencies() {
+    grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
+    (python - | while read dep; do fetch_dependency "$dep"; done) << EOF
+import sys,yaml
+for dep in yaml.load(open('${METADATA}', 'ro'))['dependencies']:
+    print '%s:%s' % (dep["name"], dep["source"])
+EOF
+}
+
+clean() {
+    log_info "Cleaning up ${BUILDDIR}"
+    [ -d ${BUILDDIR} ] && rm -rf ${BUILDDIR} || exit 0
+}
+
+salt_run() {
+    [ -e ${VENV_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
+    python $(which salt-call) ${SALT_OPTS} $*
+}
+
+prepare() {
+    [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
+
+    [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+    setup_mock_bin
+    setup_pillar
+    setup_salt
+    install_dependencies
+    link_modules
+}
+
+lint_releasenotes() {
+    [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+    source ${VENV_DIR}/bin/activate
+    python -m pip install reno
+    reno lint ${CURDIR}/../
+}
+
+lint() {
+#    lint_releasenotes
+    log_err "TODO: lint_releasenotes"
+}
+
+run() {
+    for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+        state_name=$(basename ${pillar%.sls})
+        salt_run grains.set 'noservices' False force=True
+
+        echo "Checking state ${FORMULA_NAME}.${state_name} ..."
+        salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+
+        # Check that all files in 'meta' folder can be rendered using any valid pillar
+        for meta in `find ${FORMULA_META_DIR} -type f`; do
+            meta_name=$(basename ${meta})
+            echo "Checking meta ${meta_name} ..."
+            salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
+              || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
+            cat ${SALT_CACHE_DIR}/${meta_name}
+        done
+    done
+}
+
+real_run() {
+    for pillar in ${PILLARDIR}/*.sls; do
+        state_name=$(basename ${pillar%.sls})
+        salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
+    done
+}
+
+run_model_validate(){
+    if [ -d ${SCHEMARDIR} ]; then
+      # model validator require py modules
+      fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
+      link_modules
+      # Rendered Example:
+      # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
+      for role in ${SCHEMARDIR}/*.yaml; do
+          state_name=$(basename "${role%*.yaml}")
+          minion_id="${state_name}"
+          salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
+          salt_run -m ${DEPSDIR}/salt-formula-salt --id=${minion_id} modelschema.model_validate ${FORMULA_NAME} ${state_name} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1 ; }
+      done
+    else
+      log_info "${SCHEMARDIR} not found!";
+    fi
+}
+
+dependency_check() {
+  local DEPENDENCY_COMMANDS=$*
+
+  for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
+    which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
+  done
+}
+
+_atexit() {
+    RETVAL=$?
+    trap true INT TERM EXIT
+
+    if [ $RETVAL -ne 0 ]; then
+        log_err "Execution failed"
+    else
+        log_info "Execution successful"
+    fi
+    return $RETVAL
+}
+
+## Main
+trap _atexit INT TERM EXIT
+
+case $1 in
+    clean)
+        clean
+        ;;
+    prepare)
+        prepare
+        ;;
+    lint)
+        lint
+        ;;
+    run)
+        run
+        ;;
+    real-run)
+        real_run
+        ;;
+    model-validate)
+       prepare
+       run_model_validate
+        ;;
+    *)
+        prepare
+#        lint
+        run
+        run_model_validate
+        ;;
+esac
\ No newline at end of file