Merge "Added 'description' arg to jobResultNotification"
diff --git a/src/com/mirantis/mcp/MCPArtifactory.groovy b/src/com/mirantis/mcp/MCPArtifactory.groovy
index de4f81f..b10c1be 100644
--- a/src/com/mirantis/mcp/MCPArtifactory.groovy
+++ b/src/com/mirantis/mcp/MCPArtifactory.groovy
@@ -309,3 +309,68 @@
     }
     sh "rm -v ${queryFile}"
 }
+
+/**
+ * Save job artifacts to Artifactory server if available.
+ * Returns link to Artifactory repo, where saved job artifacts.
+ *
+ * @param config LinkedHashMap which contains next parameters:
+ *   @param artifactory String, Artifactory server id
+ *   @param artifactoryRepo String, repo to save job artifacts
+ *   @param buildProps ArrayList, additional props for saved artifacts. Optional, default: []
+ *   @param artifactory_not_found_fail Boolean, whether to fail if provided artifactory
+ *          id is not found or just print warning message. Optional, default: false
+ */
+def uploadJobArtifactsToArtifactory(LinkedHashMap config) {
+    def common = new com.mirantis.mk.Common()
+    def artifactsDescription = ''
+    def artifactoryServer
+    try {
+        artifactoryServer = Artifactory.server(config.get('artifactory'))
+    } catch (Exception e) {
+        if (config.get('artifactory_not_found_fail', false)) {
+            throw e
+        } else {
+            common.warningMsg(e)
+            return "Artifactory server is not found. Can't save artifacts in Artifactory."
+        }
+    }
+    def artifactDir = 'cur_build_artifacts'
+    def user = ''
+    wrap([$class: 'BuildUser']) {
+        user = env.BUILD_USER_ID
+    }
+    dir(artifactDir) {
+        try {
+            unarchive(mapping: ['**/*' : '.'])
+            // Mandatory and additional properties
+            def properties = getBinaryBuildProperties(config.get('buildProps', []) << "buildUser=${user}")
+
+            // Build Artifactory spec object
+            def uploadSpec = """{
+                "files":
+                    [
+                        {
+                            "pattern": "*",
+                            "target": "${config.get('artifactoryRepo')}/",
+                            "flat": false,
+                            "props": "${properties}"
+                        }
+                    ]
+                }"""
+
+            artifactoryServer.upload(uploadSpec, newBuildInfo())
+            def linkUrl = "${artifactoryServer.getUrl()}/artifactory/${config.get('artifactoryRepo')}"
+            artifactsDescription = "Job artifacts uploaded to Artifactory: <a href=\"${linkUrl}\">${linkUrl}</a>"
+        } catch (Exception e) {
+            if (e =~ /no artifacts/) {
+                artifactsDescription = 'Build has no artifacts saved.'
+            } else {
+                throw e
+            }
+        } finally {
+            deleteDir()
+        }
+    }
+    return artifactsDescription
+}
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index ec2d568..79efa08 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -56,7 +56,8 @@
         entry_point = '--entrypoint /bin/bash'
     }
     salt.cmdRun(master, target, "docker run -tid --net=host --name=${name} " +
-                                "-u root ${entry_point} ${variables} ${dockerImageLink}")
+                                "-u root ${entry_point} ${variables} " +
+                                "-v /srv/salt/pki/${cluster_name}/:/etc/certs ${dockerImageLink}")
 }
 
 
@@ -98,9 +99,11 @@
         keystone.add("OS_AUTH_URL=${_pillar.auth.auth_url}/v3")
         keystone.add("OS_REGION_NAME=${_pillar.region_name}")
         keystone.add("OS_IDENTITY_API_VERSION=${_pillar.identity_api_version}")
-        keystone.add("OS_ENDPOINT_TYPE=admin")
+        keystone.add("OS_ENDPOINT_TYPE=internal")
         keystone.add("OS_PROJECT_DOMAIN_NAME=${_pillar.auth.project_domain_name}")
         keystone.add("OS_USER_DOMAIN_NAME=${_pillar.auth.user_domain_name}")
+        // we mount /srv/salt/pki/${cluster_name}/:/etc/certs with certs for cvp container
+        keystone.add("OS_CACERT='/etc/certs/proxy-with-chain.crt'")
         return keystone
     }
     else {
@@ -414,29 +417,103 @@
  * @param scenarios         Directory inside repo with specific scenarios
  * @param sl_scenarios      Directory inside repo with specific scenarios for stacklight
  * @param tasks_args_file   Argument file that is used for throttling settings
+ * @param db_connection_str Rally-compliant external DB connection string
+ * @param tags              Additional tags used for tagging tasks or building trends
+ * @param trends            Build rally trends if enabled
  * @param ext_variables     The list of external variables
  * @param results           The reports directory
  */
-def runRallyTests(master, target, dockerImageLink, platform, output_dir, config_repo, config_branch, plugins_repo, plugins_branch, scenarios, sl_scenarios = '', tasks_args_file = '', ext_variables = [], results = '/root/qa_results', skip_list = '') {
+def runRallyTests(
+        master, target, dockerImageLink,
+        platform, output_dir, config_repo,
+        config_branch, plugins_repo, plugins_branch,
+        scenarios, sl_scenarios = '', tasks_args_file = '',
+        db_connection_str = '', tags = [],
+        trends = false, ext_variables = [],
+        results = '/root/qa_results', skip_list = ''
+        ) {
+
     def salt = new com.mirantis.mk.Salt()
     def output_file = 'docker-rally.log'
     def dest_folder = '/home/rally/qa_results'
     def env_vars = []
+    def work_dir = 'test_config'
+
+    // compile rally deployment name from env name, platform name,
+    // date, cmp nodes count
+    def deployment_name = ''
+    def cluster_name = salt.getPillar(
+        master, 'I@salt:master', '_param:cluster_name'
+    )['return'][0].values()[0]
+    def rcs_str_node = salt.getPillar(
+        master, 'I@salt:master', 'reclass:storage:node'
+    )['return'][0].values()[0]
+    def date = new Date()
+    date = date.format("yyyyMMddHHmm")
+    def cmp_count
+
+    if (platform['type'] == 'openstack') {
+        cmp_count = rcs_str_node.openstack_compute_rack01['repeat']['count']
+    } else if (platform['type'] == 'k8s') {
+        cmp_count = rcs_str_node.kubernetes_compute_rack01['repeat']['count']
+    } else {
+      throw new Exception("Platform ${platform} is not supported yet")
+    }
+    deployment_name = "env=${cluster_name}:platform=${platform.type}:" +
+        "date=${date}:cmp=${cmp_count}"
+
+    // set up rally cmds
     def rally_extra_args = ''
     def cmd_rally_plugins =
-          "git clone -b ${plugins_branch ?: 'master'} ${plugins_repo} /tmp/plugins; " +
-          "sudo pip install --upgrade /tmp/plugins; "
-    def cmd_rally_init = ''
-    def cmd_rally_checkout = "git clone -b ${config_branch ?: 'master'} ${config_repo} test_config; "
+        "git clone -b ${plugins_branch ?: 'master'} ${plugins_repo} /tmp/plugins; " +
+        "sudo pip install --upgrade /tmp/plugins; "
+    def cmd_rally_init = 'rally db ensure; '
+    if (db_connection_str) {
+        cmd_rally_init = "sudo sed -i -e " +
+            "'s#connection=.*#connection=${db_connection_str}#' " +
+            "/etc/rally/rally.conf; "
+    }
+    def cmd_rally_checkout = "git clone -b ${config_branch ?: 'master'} ${config_repo} ${work_dir}; "
     def cmd_rally_start = ''
     def cmd_rally_task_args = ''
     def cmd_rally_stacklight = ''
-    def cmd_rally_report = ''
+    def cmd_rally_report = "rally task export " +
+        "--uuid \\\$(rally task list --uuids-only --status finished) " +
+        "--type junit-xml --to ${dest_folder}/report-rally.xml; " +
+        "rally task report --uuid \\\$(rally task list --uuids-only --status finished) " +
+        "--out ${dest_folder}/report-rally.html"
+    def cmd_filter_tags = ''
+
+    // build rally trends if required
+    if (trends && db_connection_str) {
+        if (tags) {
+            cmd_filter_tags = "--tag " + tags.join(' ')
+        }
+        cmd_rally_report += "; rally task trends --tasks " +
+            "\\\$(rally task list " + cmd_filter_tags +
+            " --all-deployments --uuids-only --status finished) " +
+            "--out ${dest_folder}/trends-rally.html"
+    }
+
+    // add default env tags for inserting into rally tasks
+    tags = tags + [
+        "env=${cluster_name}",
+        "platform=${platform.type}",
+        "cmp=${cmp_count}"
+    ]
+
+    // create results directory
     salt.runSaltProcessStep(master, target, 'file.remove', ["${results}"])
     salt.runSaltProcessStep(master, target, 'file.mkdir', ["${results}", "mode=777"])
+
+    // get required OS data
     if (platform['type'] == 'openstack') {
-      def _pillar = salt.getPillar(master, 'I@keystone:server', 'keystone')
-      def keystone = _pillar['return'][0].values()[0]
+      cmd_rally_init += "rally deployment create --name='${deployment_name}' --fromenv; " +
+          "rally deployment check; "
+
+      def keystone = salt.getPillar(
+          master, 'I@keystone:server', 'keystone'
+      )['return'][0].values()[0]
       env_vars = ( ['tempest_version=15.0.0',
           "OS_USERNAME=${keystone.server.admin_name}",
           "OS_PASSWORD=${keystone.server.admin_password}",
@@ -445,88 +522,121 @@
           "/v${keystone.client.os_client_config.cfgs.root.content.clouds.admin_identity.identity_api_version}",
           "OS_REGION_NAME=${keystone.server.region}",
           'OS_ENDPOINT_TYPE=admin'] + ext_variables ).join(' -e ')
-      cmd_rally_init = 'rally db create; ' +
-          'rally deployment create --fromenv --name=existing; ' +
-          'rally deployment config; '
+
+      // get required SL data
       if (platform['stacklight_enabled'] == true) {
-        def _pillar_grafana = salt.getPillar(master, 'I@grafana:client', 'grafana:client:server')
-        def grafana = _pillar_grafana['return'][0].values()[0]
-        cmd_rally_stacklight = bundle_up_scenarios(sl_scenarios, skip_list, "scenarios_${platform.type}_stacklight.yaml")
+        def grafana = salt.getPillar(
+            master, 'I@grafana:client', 'grafana:client:server'
+        )['return'][0].values()[0]
+        cmd_rally_stacklight = bundle_up_scenarios(
+            work_dir + '/' + sl_scenarios,
+            skip_list,
+            "scenarios_${platform.type}_stacklight.yaml"
+        )
+        tags.add('stacklight')
         cmd_rally_stacklight += "sed -i 's/grafana_password: .*/grafana_password: ${grafana.password}/' " +
-            "test_config/job-params-stacklight.yaml; " +
-            "rally $rally_extra_args task start scenarios_${platform.type}_stacklight.yaml " +
-            "--task-args-file test_config/job-params-stacklight.yaml; "
+            "${work_dir}/job-params-stacklight.yaml; " +
+            "rally $rally_extra_args task start --tag " + tags.join(' ') +
+            " --task scenarios_${platform.type}_stacklight.yaml " +
+            "--task-args-file ${work_dir}/job-params-stacklight.yaml; "
       }
+
+    // get required K8S data
     } else if (platform['type'] == 'k8s') {
+      cmd_rally_init += "rally env create --name='${deployment_name}' --from-sysenv; " +
+          "rally env check; "
       rally_extra_args = "--debug --log-file ${dest_folder}/task.log"
-      def _pillar = salt.getPillar(master, 'I@kubernetes:master and *01*', 'kubernetes:master')
-      def kubernetes = _pillar['return'][0].values()[0]
+
+      def kubernetes = salt.getPillar(
+          master, 'I@kubernetes:master and *01*', 'kubernetes:master'
+      )['return'][0].values()[0]
       env_vars = [
           "KUBERNETES_HOST=http://${kubernetes.apiserver.vip_address}" +
           ":${kubernetes.apiserver.insecure_port}",
           "KUBERNETES_CERT_AUTH=${dest_folder}/k8s-ca.crt",
           "KUBERNETES_CLIENT_KEY=${dest_folder}/k8s-client.key",
           "KUBERNETES_CLIENT_CERT=${dest_folder}/k8s-client.crt"].join(' -e ')
-      def k8s_ca = salt.getReturnValues(salt.runSaltProcessStep(master,
-                        'I@kubernetes:master and *01*', 'cmd.run',
-                        ["cat /etc/kubernetes/ssl/ca-kubernetes.crt"]))
-      def k8s_client_key = salt.getReturnValues(salt.runSaltProcessStep(master,
-                        'I@kubernetes:master and *01*', 'cmd.run',
-                        ["cat /etc/kubernetes/ssl/kubelet-client.key"]))
-      def k8s_client_crt = salt.getReturnValues(salt.runSaltProcessStep(master,
-                        'I@kubernetes:master and *01*', 'cmd.run',
-                        ["cat /etc/kubernetes/ssl/kubelet-client.crt"]))
+
+      def k8s_ca = salt.getFileContent(
+          master, 'I@kubernetes:master and *01*', '/etc/kubernetes/ssl/ca-kubernetes.crt'
+      )
+      def k8s_client_key = salt.getFileContent(
+          master, 'I@kubernetes:master and *01*', '/etc/kubernetes/ssl/kubelet-client.key'
+      )
+      def k8s_client_crt = salt.getFileContent(
+          master, 'I@kubernetes:master and *01*', '/etc/kubernetes/ssl/kubelet-client.crt'
+      )
       def tmp_dir = '/tmp/kube'
+
       salt.runSaltProcessStep(master, target, 'file.mkdir', ["${tmp_dir}"])
-      salt.runSaltProcessStep(master, target, 'file.write', ["${tmp_dir}/k8s-ca.crt", "${k8s_ca}"])
-      salt.runSaltProcessStep(master, target, 'file.write', ["${tmp_dir}/k8s-client.key", "${k8s_client_key}"])
-      salt.runSaltProcessStep(master, target, 'file.write', ["${tmp_dir}/k8s-client.crt", "${k8s_client_crt}"])
+      salt.runSaltProcessStep(
+          master, target, 'file.write', ["${tmp_dir}/k8s-ca.crt","${k8s_ca}"]
+      )
+      salt.runSaltProcessStep(
+          master, target, 'file.write', ["${tmp_dir}/k8s-client.key", "${k8s_client_key}"]
+      )
+      salt.runSaltProcessStep(
+          master, target, 'file.write', ["${tmp_dir}/k8s-client.crt", "${k8s_client_crt}"]
+      )
       salt.cmdRun(master, target, "mv ${tmp_dir}/* ${results}/")
       salt.runSaltProcessStep(master, target, 'file.rmdir', ["${tmp_dir}"])
-      cmd_rally_init = "rally db recreate; " +
-          "rally env create --name k8s --from-sysenv; " +
-          "rally env check k8s; "
+
     } else {
       throw new Exception("Platform ${platform} is not supported yet")
     }
+
+    // set up rally task args file
     switch(tasks_args_file) {
       case 'none':
         cmd_rally_task_args = '; '
         break
       case '':
-        cmd_rally_task_args = '--task-args-file test_config/job-params-light.yaml; '
+        cmd_rally_task_args = "--task-args-file ${work_dir}/job-params-light.yaml; "
         break
       default:
-        cmd_rally_task_args = "--task-args-file ${tasks_args_file}; "
+        cmd_rally_task_args = "--task-args-file ${work_dir}/${tasks_args_file}; "
       break
     }
     if (platform['type'] == 'k8s') {
-      cmd_rally_start = "for task in \\\$(" + bundle_up_scenarios(scenarios, skip_list) +
+      cmd_rally_start = "for task in \\\$(" +
+          bundle_up_scenarios(work_dir + '/' + scenarios, skip_list) +
           "); do " +
-          "rally $rally_extra_args task start \\\$task ${cmd_rally_task_args}" +
+          "rally $rally_extra_args task start --tag " + tags.join(' ') +
+          " --task \\\$task ${cmd_rally_task_args}" +
           "done; "
     } else {
-      cmd_rally_checkout += bundle_up_scenarios(scenarios, skip_list, "scenarios_${platform.type}.yaml")
-      cmd_rally_start = "rally $rally_extra_args task start " +
-          "scenarios_${platform.type}.yaml ${cmd_rally_task_args}"
+      cmd_rally_checkout += bundle_up_scenarios(
+          work_dir + '/' + scenarios,
+          skip_list,
+          "scenarios_${platform.type}.yaml"
+      )
+      cmd_rally_start = "rally $rally_extra_args task start --tag " + tags.join(' ') +
+          " --task scenarios_${platform.type}.yaml ${cmd_rally_task_args}"
     }
-    cmd_rally_report= "rally task export --uuid \\\$(rally task list --uuids-only --status finished) " +
-        "--type junit-xml --to ${dest_folder}/report-rally.xml; " +
-        "rally task report --uuid \\\$(rally task list --uuids-only --status finished) " +
-        "--out ${dest_folder}/report-rally.html"
 
+    // compile full rally cmd
     full_cmd = 'set -xe; ' + cmd_rally_plugins +
         cmd_rally_init + cmd_rally_checkout +
         'set +e; ' + cmd_rally_start +
-        cmd_rally_stacklight +
-        cmd_rally_report
+        cmd_rally_stacklight + cmd_rally_report
+
+    // run rally
     salt.runSaltProcessStep(master, target, 'file.touch', ["${results}/rally.db"])
     salt.cmdRun(master, target, "chmod 666 ${results}/rally.db")
-    salt.cmdRun(master, target, "docker run -w /home/rally -i --rm --net=host -e ${env_vars} " +
+    salt.cmdRun(
+        master, target,
+        "docker run -w /home/rally -i --rm --net=host -e ${env_vars} " +
         "-v ${results}:${dest_folder} " +
         "-v ${results}/rally.db:/home/rally/data/rally.db " +
         "--entrypoint /bin/bash ${dockerImageLink} " +
         "-c \"${full_cmd}\" > ${results}/${output_file}")
+
+    // remove k8s secrets
+    if (platform['type'] == 'k8s') {
+        salt.cmdRun(master, target, "rm ${results}/k8s-*")
+    }
+
+    // save artifacts
     addFiles(master, target, results, output_dir)
 }
 
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index 1cb3d6e..d0f3598 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -730,7 +730,9 @@
                     description += "<li><a href=\"${httpWS}/old/${item}/*view*/\">${item}</a></li>"
                 }
             }
-
+            def cwd = sh(script: 'basename $(pwd)', returnStdout: true).trim()
+            sh "tar -cf old_${cwd}.tar.gz old/ && rm -rf old/"
+            sh "tar -cf new_${cwd}.tar.gz new/ && rm -rf new/"
         }
     }
 
diff --git a/src/com/mirantis/mk/Galera.groovy b/src/com/mirantis/mk/Galera.groovy
new file mode 100644
index 0000000..5a15823
--- /dev/null
+++ b/src/com/mirantis/mk/Galera.groovy
@@ -0,0 +1,338 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Galera functions
+ *
+ */
+
+
+/**
+ * Returns parameters from mysql.status output on given target node
+ *
+ * @param env           Salt Connection object or pepperEnv
+ * @param target        Targeted node
+ * @param parameters    Parameters to be retruned (String or list of Strings). If no parameters are provided or is set to '[]', it returns all of them.
+ * @return result       List of parameters with its values
+ */
+
+def getWsrepParameters(env, target, parameters=[], print=false) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    result = [:]
+    out = salt.runSaltProcessStep(env, "${target}", "mysql.status", [], null, false)
+    outlist = out['return'][0]
+    resultYaml = outlist.get(outlist.keySet()[0]).sort()
+    if (print) {
+        common.prettyPrint(resultYaml)
+    }
+    if (parameters instanceof String) {
+        parameters = [parameters]
+    }
+    if (parameters == [] || parameters == ['']) {
+        result = resultYaml
+    } else {
+        for (String param in parameters) {
+            value = resultYaml[param]
+            if (value instanceof String && value.isBigDecimal()) {
+                value = value.toBigDecimal()
+            }
+            result[param] = value
+        }
+    }
+    return result
+}
+
+/**
+ * Verifies Galera database
+ *
+ * This function checks for Galera master, tests connection and if reachable, it obtains the result
+ *      of Salt mysql.status function. The result is then parsed, validated and outputed to the user.
+ *
+ * @param env           Salt Connection object or pepperEnv
+ * @param slave         Boolean value to enable slave checking (if master in unreachable)
+ * @param checkTimeSync Boolean value to enable time sync check
+ * @return resultCode   int values used to determine exit status in the calling function
+ */
+def verifyGaleraStatus(env, slave=false, checkTimeSync=false) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def out = ""
+    def status = "unknown"
+    def testNode = ""
+    if (!slave) {
+        try {
+            galeraMaster = salt.getMinions(env, "I@galera:master")
+            common.infoMsg("Current Galera master is: ${galeraMaster}")
+            salt.minionsReachable(env, "I@salt:master", "I@galera:master")
+            testNode = "I@galera:master"
+        } catch (Exception e) {
+            common.errorMsg('Galera master is not reachable.')
+            return 128
+        }
+    } else {
+        try {
+            galeraSlaves = salt.getMinions(env, "I@galera:slave")
+            common.infoMsg("Testing Galera slave minions: ${galeraSlaves}")
+        } catch (Exception e) {
+            common.errorMsg("Cannot obtain Galera slave minions list.")
+            return 129
+        }
+        for (minion in galeraSlaves) {
+            try {
+                salt.minionsReachable(env, "I@salt:master", minion)
+                testNode = minion
+                break
+            } catch (Exception e) {
+                common.warningMsg("Slave '${minion}' is not reachable.")
+            }
+        }
+    }
+    if (!testNode) {
+        common.errorMsg("No Galera slave was reachable.")
+        return 130
+    }
+    if (checkTimeSync && !salt.checkClusterTimeSync(env, "I@galera:master or I@galera:slave")) {
+        common.errorMsg("Time in cluster is desynchronized or it couldn't be detemined. You should fix this issue manually before proceeding.")
+        return 131
+    }
+    try {
+        out = salt.runSaltProcessStep(env, "${testNode}", "mysql.status", [], null, false)
+    } catch (Exception e) {
+        common.errorMsg('Could not determine mysql status.')
+        return 256
+    }
+    if (out) {
+        try {
+            status = validateAndPrintGaleraStatusReport(env, out, testNode)
+        } catch (Exception e) {
+            common.errorMsg('Could not parse the mysql status output. Check it manually.')
+            return 1
+        }
+    } else {
+        common.errorMsg("Mysql status response unrecognized or is empty. Response: ${out}")
+        return 1024
+    }
+    if (status == "OK") {
+        common.infoMsg("No errors found - MySQL status is ${status}.")
+        return 0
+    } else if (status == "unknown") {
+        common.warningMsg('MySQL status cannot be detemined')
+        return 1
+    } else {
+        common.errorMsg("Errors found.")
+        return 2
+    }
+}
+
+/** Validates and prints result of verifyGaleraStatus function
+@param env      Salt Connection object or pepperEnv
+@param out      Output of the mysql.status Salt function
+@return status  "OK", "ERROR" or "uknown" depending on result of validation
+*/
+
+def validateAndPrintGaleraStatusReport(env, out, minion) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    if (minion == "I@galera:master") {
+        role = "master"
+    } else {
+        role = "slave"
+    }
+    sizeOut = salt.getReturnValues(salt.getPillar(env, minion, "galera:${role}:members"))
+    expected_cluster_size = sizeOut.size()
+    outlist = out['return'][0]
+    resultYaml = outlist.get(outlist.keySet()[0]).sort()
+    common.prettyPrint(resultYaml)
+    parameters = [
+        wsrep_cluster_status: [title: 'Cluster status', expectedValues: ['Primary'], description: ''],
+        wsrep_cluster_size: [title: 'Current cluster size', expectedValues: [expected_cluster_size], description: ''],
+        wsrep_ready: [title: 'Node status', expectedValues: ['ON', true], description: ''],
+        wsrep_local_state_comment: [title: 'Node status comment', expectedValues: ['Joining', 'Waiting on SST', 'Joined', 'Synced', 'Donor'], description: ''],
+        wsrep_connected: [title: 'Node connectivity', expectedValues: ['ON', true], description: ''],
+        wsrep_local_recv_queue_avg: [title: 'Average size of local reveived queue', expectedThreshold: [warn: 0.5, error: 1.0], description: '(Value above 0 means that the node cannot apply write-sets as fast as it receives them, which can lead to replication throttling)'],
+        wsrep_local_send_queue_avg: [title: 'Average size of local send queue', expectedThreshold: [warn: 0.5, error: 1.0], description: '(Value above 0 indicate replication throttling or network throughput issues, such as a bottleneck on the network link.)']
+        ]
+    for (key in parameters.keySet()) {
+        value = resultYaml[key]
+        if (value instanceof String && value.isBigDecimal()) {
+            value = value.toBigDecimal()
+        }
+        parameters.get(key) << [actualValue: value]
+    }
+    for (key in parameters.keySet()) {
+        param = parameters.get(key)
+        if (key == 'wsrep_local_recv_queue_avg' || key == 'wsrep_local_send_queue_avg') {
+            if (param.get('actualValue') == null || (param.get('actualValue') > param.get('expectedThreshold').get('error'))) {
+                param << [match: 'error']
+            } else if (param.get('actualValue') > param.get('expectedThreshold').get('warn')) {
+                param << [match: 'warn']
+            } else {
+                param << [match: 'ok']
+            }
+        } else {
+            for (expValue in param.get('expectedValues')) {
+                if (expValue == param.get('actualValue')) {
+                    param << [match: 'ok']
+                    break
+                } else {
+                    param << [match: 'error']
+                }
+            }
+        }
+    }
+    cluster_info_report = []
+    cluster_warning_report = []
+    cluster_error_report = []
+    for (key in parameters.keySet()) {
+        param = parameters.get(key)
+        if (param.containsKey('expectedThreshold')) {
+            expValues = "below ${param.get('expectedThreshold').get('warn')}"
+        } else {
+            if (param.get('expectedValues').size() > 1) {
+                expValues = param.get('expectedValues').join(' or ')
+            } else {
+                expValues = param.get('expectedValues')[0]
+            }
+        }
+        reportString = "${param.title}: ${param.actualValue} (Expected: ${expValues}) ${param.description}"
+        if (param.get('match').equals('ok')) {
+            cluster_info_report.add("[OK     ] ${reportString}")
+        } else if (param.get('match').equals('warn')) {
+            cluster_warning_report.add("[WARNING] ${reportString}")
+        } else {
+            cluster_error_report.add("[  ERROR] ${reportString})")
+        }
+    }
+    common.infoMsg("CLUSTER STATUS REPORT: ${cluster_info_report.size()} expected values, ${cluster_warning_report.size()} warnings and ${cluster_error_report.size()} error found:")
+    if (cluster_info_report.size() > 0) {
+        common.infoMsg(cluster_info_report.join('\n'))
+    }
+    if (cluster_warning_report.size() > 0) {
+        common.warningMsg(cluster_warning_report.join('\n'))
+    }
+    if (cluster_error_report.size() > 0) {
+        common.errorMsg(cluster_error_report.join('\n'))
+        return "ERROR"
+    } else {
+        return "OK"
+    }
+}
+
+/** Returns last shutdown node of Galera cluster
+@param env      Salt Connection object or pepperEnv
+@param nodes    List of nodes to check only (defaults to []). If not provided, it will check all nodes.
+                Use this parameter if the cluster splits to several components and you only want to check one fo them.
+@return status  ip address or hostname of last shutdown node
+*/
+
+def getGaleraLastShutdownNode(env, nodes = []) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    members = []
+    lastNode = [ip: '', seqno: -2]
+    try {
+        if (nodes) {
+            nodes = salt.getIPAddressesForNodenames(env, nodes)
+            for (node in nodes) {
+                members = [host: "${node.get(node.keySet()[0])}"] + members
+            }
+        } else {
+            members = salt.getReturnValues(salt.getPillar(env, "I@galera:master", "galera:master:members"))
+        }
+    } catch (Exception er) {
+        common.errorMsg('Could not retrieve members list')
+        return 'I@galera:master'
+    }
+    if (members) {
+        for (member in members) {
+            try {
+                salt.minionsReachable(env, 'I@salt:master', "S@${member.host}")
+                out = salt.getReturnValues(salt.cmdRun(env, "S@${member.host}", 'cat /var/lib/mysql/grastate.dat | grep "seqno" | cut -d ":" -f2', true, null, false))
+                seqno = out.tokenize('\n')[0].trim()
+                if (seqno.isNumber()) {
+                    seqno = seqno.toInteger()
+                } else {
+                    seqno = -2
+                }
+                highestSeqno = lastNode.get('seqno')
+                if (seqno > highestSeqno) {
+                    lastNode << [ip: "${member.host}", seqno: seqno]
+                }
+            } catch (Exception er) {
+                common.warningMsg("Could not determine 'seqno' value for node ${member.host} ")
+            }
+        }
+    }
+    if (lastNode.get('ip') != '') {
+        return "S@${lastNode.ip}"
+    } else {
+        return "I@galera:master"
+    }
+}
+
+/**
+ * Restores Galera database
+ * @param env Salt Connection object or pepperEnv
+ * @return output of salt commands
+ */
+def restoreGaleraDb(env) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    try {
+        salt.runSaltProcessStep(env, 'I@galera:slave', 'service.stop', ['mysql'])
+    } catch (Exception er) {
+        common.warningMsg('Mysql service already stopped')
+    }
+    try {
+        salt.runSaltProcessStep(env, 'I@galera:master', 'service.stop', ['mysql'])
+    } catch (Exception er) {
+        common.warningMsg('Mysql service already stopped')
+    }
+    lastNodeTarget = getGaleraLastShutdownNode(env)
+    try {
+        salt.cmdRun(env, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
+    } catch (Exception er) {
+        common.warningMsg('Files are not present')
+    }
+    try {
+        salt.cmdRun(env, 'I@galera:slave', "rm  /var/lib/mysql/grastate.dat")
+    } catch (Exception er) {
+        common.warningMsg('Files are not present')
+    }
+    try {
+        salt.cmdRun(env, lastNodeTarget, "mkdir /root/mysql/mysql.bak")
+    } catch (Exception er) {
+        common.warningMsg('Directory already exists')
+    }
+    try {
+        salt.cmdRun(env, lastNodeTarget, "rm -rf /root/mysql/mysql.bak/*")
+    } catch (Exception er) {
+        common.warningMsg('Directory already empty')
+    }
+    try {
+        salt.cmdRun(env, lastNodeTarget, "mv /var/lib/mysql/* /root/mysql/mysql.bak")
+    } catch (Exception er) {
+        common.warningMsg('Files were already moved')
+    }
+    try {
+        salt.runSaltProcessStep(env, lastNodeTarget, 'file.remove', ["/var/lib/mysql/.galera_bootstrap"])
+    } catch (Exception er) {
+        common.warningMsg('File is not present')
+    }
+    salt.cmdRun(env, lastNodeTarget, "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
+    def backup_dir = salt.getReturnValues(salt.getPillar(env, lastNodeTarget, 'xtrabackup:client:backup_dir'))
+    if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
+    salt.runSaltProcessStep(env, lastNodeTarget, 'file.remove', ["${backup_dir}/dbrestored"])
+    salt.cmdRun(env, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
+    salt.enforceState(env, lastNodeTarget, 'galera')
+    // wait until mysql service on galera master is up
+    try {
+        salt.commandStatus(env, lastNodeTarget, 'service mysql status', 'running')
+    } catch (Exception er) {
+        input message: "Database is not running please fix it first and only then click on PROCEED."
+    }
+
+    salt.runSaltProcessStep(env, "I@galera:master and not ${lastNodeTarget}", 'service.start', ['mysql'])
+    salt.runSaltProcessStep(env, "I@galera:slave and not ${lastNodeTarget}", 'service.start', ['mysql'])
+}
diff --git a/src/com/mirantis/mk/Git.groovy b/src/com/mirantis/mk/Git.groovy
index d20c159..575fb80 100644
--- a/src/com/mirantis/mk/Git.groovy
+++ b/src/com/mirantis/mk/Git.groovy
@@ -99,12 +99,17 @@
  *
  * @param path            Path to the git repository
  * @param message         A commit message
+ * @param global          Use global config
  */
-def commitGitChanges(path, message, gitEmail='jenkins@localhost', gitName='jenkins-slave') {
+def commitGitChanges(path, message, gitEmail='jenkins@localhost', gitName='jenkins-slave', global=false) {
     def git_cmd
+    def global_arg = ''
+    if (global) {
+        global_arg = '--global'
+    }
     dir(path) {
-        sh "git config --global user.email '${gitEmail}'"
-        sh "git config --global user.name '${gitName}'"
+        sh "git config ${global_arg} user.email '${gitEmail}'"
+        sh "git config ${global_arg} user.name '${gitName}'"
 
         sh(
             script: 'git add -A',
@@ -206,3 +211,24 @@
     }
     sh "git remote rm target"
 }
+
+
+/**
+ * Return all branches for the defined git repository that match the matcher.
+ *
+ * @param repoUrl        URL of git repository
+ * @param branchMatcher  matcher to filter out the branches (If '' or '*', returns all branches without filtering)
+ * @return branchesList  list of branches
+ */
+
+def getBranchesForGitRepo(repoUrl, branchMatcher = ''){
+
+    if (branchMatcher.equals("*")) {
+        branchMatcher = ''
+    }
+    branchesList = sh (
+                script: "git ls-remote --heads ${repoUrl} | cut -f2 | grep -e '${branchMatcher}' | sed 's/refs\\/heads\\///g'",
+                returnStdout: true
+        ).trim()
+    return branchesList.tokenize('\n')
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index af697da..9ce6aef 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -435,22 +435,41 @@
 /**
  * Return intersection of globally installed services and those are
  * defined on specific target according to theirs priorities.
+ * By default services are added to the result list only if
+ * <service>.upgrade.enabled pillar is set to "True". However if it
+ * is needed to obtain list of upgrade services regardless of
+ * <service>.upgrade.enabled pillar value it is needed to set
+ * "upgrade_condition" param to "False".
  *
  * @param env     Salt Connection object or env
- * @param target  The target node to get list of apps for.
+ * @param target  The target node to get list of apps for
+ * @param upgrade_condition  Whether to take "upgrade:enabled"
+ *                           service pillar into consideration
+ *                           when obtaining list of upgrade services
 **/
-def getOpenStackUpgradeServices(env, target){
+def getOpenStackUpgradeServices(env, target, upgrade_condition=true){
     def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
 
     def global_apps = salt.getConfig(env, 'I@salt:master:enabled:true', 'orchestration.upgrade.applications')
     def node_apps = salt.getPillar(env, target, '__reclass__:applications')['return'][0].values()[0]
+    if (upgrade_condition) {
+        node_pillar = salt.getPillar(env, target)
+    }
     def node_sorted_apps = []
     if ( !global_apps['return'][0].values()[0].isEmpty() ) {
         Map<String,Integer> _sorted_apps = [:]
         for (k in global_apps['return'][0].values()[0].keySet()) {
             if (k in node_apps) {
-              _sorted_apps[k] = global_apps['return'][0].values()[0][k].values()[0].toInteger()
+                if (upgrade_condition) {
+                    if (node_pillar['return'][0].values()[k]['upgrade']['enabled'][0] != null) {
+                        if (node_pillar['return'][0].values()[k]['upgrade']['enabled'][0].toBoolean()) {
+                            _sorted_apps[k] = global_apps['return'][0].values()[0][k].values()[0].toInteger()
+                        }
+                    }
+                } else {
+                    _sorted_apps[k] = global_apps['return'][0].values()[0][k].values()[0].toInteger()
+                }
             }
         }
         node_sorted_apps = common.SortMapByValueAsc(_sorted_apps).keySet()
@@ -462,7 +481,6 @@
   return node_sorted_apps
 }
 
-
 /**
  * Run specified upgrade phase for all services on given node.
  *
@@ -505,280 +523,30 @@
     }
 }
 
-/**
- * Verifies Galera database
- *
- * This function checks for Galera master, tests connection and if reachable, it obtains the result
- *      of Salt mysql.status function. The result is then parsed, validated and outputed to the user.
- *
- * @param env           Salt Connection object or pepperEnv
- * @param slave         Boolean value to enable slave checking (if master in unreachable)
- * @param checkTimeSync Boolean value to enable time sync check
- * @return resultCode   int values used to determine exit status in the calling function
- */
 def verifyGaleraStatus(env, slave=false, checkTimeSync=false) {
-    def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
-    def out = ""
-    def status = "unknown"
-    def testNode = ""
-    if (!slave) {
-        try {
-            galeraMaster = salt.getMinions(env, "I@galera:master")
-            common.infoMsg("Current Galera master is: ${galeraMaster}")
-            salt.minionsReachable(env, "I@salt:master", "I@galera:master")
-            testNode = "I@galera:master"
-        } catch (Exception e) {
-            common.errorMsg('Galera master is not reachable.')
-            return 128
-        }
-    } else {
-        try {
-            galeraMinions = salt.getMinions(env, "I@galera:slave")
-            common.infoMsg("Testing Galera slave minions: ${galeraMinions}")
-        } catch (Exception e) {
-            common.errorMsg("Cannot obtain Galera slave minions list.")
-            return 129
-        }
-        for (minion in galeraMinions) {
-            try {
-                salt.minionsReachable(env, "I@salt:master", minion)
-                testNode = minion
-                break
-            } catch (Exception e) {
-                common.warningMsg("Slave '${minion}' is not reachable.")
-            }
-        }
-    }
-    if (!testNode) {
-        common.errorMsg("No Galera slave was reachable.")
-        return 130
-    }
-    if (checkTimeSync && !salt.checkClusterTimeSync(env, "I@galera:master or I@galera:slave")) {
-        common.errorMsg("Time in cluster is desynchronized or it couldn't be detemined. You should fix this issue manually before proceeding.")
-        return 131
-    }
-    try {
-        out = salt.cmdRun(env, "I@salt:master", "salt -C '${testNode}' mysql.status")
-    } catch (Exception e) {
-        common.errorMsg('Could not determine mysql status.')
-        return 256
-    }
-    if (out) {
-        try {
-            status = validateAndPrintGaleraStatusReport(env, out, testNode)
-        } catch (Exception e) {
-            common.errorMsg('Could not parse the mysql status output. Check it manually.')
-            return 1
-        }
-    } else {
-        common.errorMsg("Mysql status response unrecognized or is empty. Response: ${out}")
-        return 1024
-    }
-    if (status == "OK") {
-        common.infoMsg("No errors found - MySQL status is ${status}.")
-        return 0
-    } else if (status == "unknown") {
-        common.warningMsg('MySQL status cannot be detemined')
-        return 1
-    } else {
-        common.errorMsg("Errors found.")
-        return 2
-    }
+    def galera = new com.mirantis.mk.Galera()
+    common.warningMsg("verifyGaleraStatus method was moved to Galera class. Please change your calls accordingly.")
+    return galera.verifyGaleraStatus(env, slave, checkTimeSync)
 }
 
-/** Validates and prints result of verifyGaleraStatus function
-@param env      Salt Connection object or pepperEnv
-@param out      Output of the mysql.status Salt function
-@return status  "OK", "ERROR" or "uknown" depending on result of validation
-*/
-
 def validateAndPrintGaleraStatusReport(env, out, minion) {
-    def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
-    if (minion == "I@galera:master") {
-        role = "master"
-    } else {
-        role = "slave"
-    }
-    sizeOut = salt.getReturnValues(salt.getPillar(env, minion, "galera:${role}:members"))
-    expected_cluster_size = sizeOut.size()
-    outlist = out['return'][0]
-    resultString = outlist.get(outlist.keySet()[0]).replace("\n        ", " ").replace("    ", "").replace("Salt command execution success", "").replace("----------", "").replace(": \n", ": no value\n")
-    resultYaml = readYaml text: resultString
-    parameters = [
-        wsrep_cluster_status: [title: 'Cluster status', expectedValues: ['Primary'], description: ''],
-        wsrep_cluster_size: [title: 'Current cluster size', expectedValues: [expected_cluster_size], description: ''],
-        wsrep_ready: [title: 'Node status', expectedValues: ['ON', true], description: ''],
-        wsrep_local_state_comment: [title: 'Node status comment', expectedValues: ['Joining', 'Waiting on SST', 'Joined', 'Synced', 'Donor'], description: ''],
-        wsrep_connected: [title: 'Node connectivity', expectedValues: ['ON', true], description: ''],
-        wsrep_local_recv_queue_avg: [title: 'Average size of local reveived queue', expectedThreshold: [warn: 0.5, error: 1.0], description: '(Value above 0 means that the node cannot apply write-sets as fast as it receives them, which can lead to replication throttling)'],
-        wsrep_local_send_queue_avg: [title: 'Average size of local send queue', expectedThreshold: [warn: 0.5, error: 1.0], description: '(Value above 0 indicate replication throttling or network throughput issues, such as a bottleneck on the network link.)']
-        ]
-    for (key in parameters.keySet()) {
-        value = resultYaml[key]
-        parameters.get(key) << [actualValue: value]
-    }
-    for (key in parameters.keySet()) {
-        param = parameters.get(key)
-        if (key == 'wsrep_local_recv_queue_avg' || key == 'wsrep_local_send_queue_avg') {
-            if (param.get('actualValue') > param.get('expectedThreshold').get('error')) {
-                param << [match: 'error']
-            } else if (param.get('actualValue') > param.get('expectedThreshold').get('warn')) {
-                param << [match: 'warn']
-            } else {
-                param << [match: 'ok']
-            }
-        } else {
-            for (expValue in param.get('expectedValues')) {
-                if (expValue == param.get('actualValue')) {
-                    param << [match: 'ok']
-                    break
-                } else {
-                    param << [match: 'error']
-                }
-            }
-        }
-    }
-    cluster_info_report = []
-    cluster_warning_report = []
-    cluster_error_report = []
-    for (key in parameters.keySet()) {
-        param = parameters.get(key)
-        if (param.containsKey('expectedThreshold')) {
-            expValues = "below ${param.get('expectedThreshold').get('warn')}"
-        } else {
-            if (param.get('expectedValues').size() > 1) {
-                expValues = param.get('expectedValues').join(' or ')
-            } else {
-                expValues = param.get('expectedValues')[0]
-            }
-        }
-        reportString = "${param.title}: ${param.actualValue} (Expected: ${expValues}) ${param.description}"
-        if (param.get('match').equals('ok')) {
-            cluster_info_report.add("[OK     ] ${reportString}")
-        } else if (param.get('match').equals('warn')) {
-            cluster_warning_report.add("[WARNING] ${reportString}")
-        } else {
-            cluster_error_report.add("[  ERROR] ${reportString})")
-        }
-    }
-    common.infoMsg("CLUSTER STATUS REPORT: ${cluster_info_report.size()} expected values, ${cluster_warning_report.size()} warnings and ${cluster_error_report.size()} error found:")
-    if (cluster_info_report.size() > 0) {
-        common.infoMsg(cluster_info_report.join('\n'))
-    }
-    if (cluster_warning_report.size() > 0) {
-        common.warningMsg(cluster_warning_report.join('\n'))
-    }
-    if (cluster_error_report.size() > 0) {
-        common.errorMsg(cluster_error_report.join('\n'))
-        return "ERROR"
-    } else {
-        return "OK"
-    }
+    def galera = new com.mirantis.mk.Galera()
+    common.warningMsg("validateAndPrintGaleraStatusReport method was moved to Galera class. Please change your calls accordingly.")
+    return galera.validateAndPrintGaleraStatusReport(env, out, minion)
 }
 
 def getGaleraLastShutdownNode(env) {
-    def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
-    members = ''
-    lastNode = [ip: '', seqno: -2]
-    try {
-        members = salt.getReturnValues(salt.getPillar(env, "I@galera:master", "galera:master:members"))
-    } catch (Exception er) {
-        common.errorMsg('Could not retrieve members list')
-        return 'I@galera:master'
-    }
-    if (members) {
-        for (member in members) {
-            try {
-                salt.minionsReachable(env, 'I@salt:master', "S@${member.host}")
-                out = salt.getReturnValues(salt.cmdRun(env, "S@${member.host}", 'cat /var/lib/mysql/grastate.dat | grep "seqno" | cut -d ":" -f2', true, null, false))
-                seqno = out.tokenize('\n')[0].trim()
-                if (seqno.isNumber()) {
-                    seqno = seqno.toInteger()
-                } else {
-                    seqno = -2
-                }
-                highestSeqno = lastNode.get('seqno')
-                if (seqno > highestSeqno) {
-                    lastNode << [ip: "${member.host}", seqno: seqno]
-                }
-            } catch (Exception er) {
-                common.warningMsg("Could not determine 'seqno' value for node ${member.host} ")
-            }
-        }
-    }
-    if (lastNode.get('ip') != '') {
-        return "S@${lastNode.ip}"
-    } else {
-        return "I@galera:master"
-    }
+    def galera = new com.mirantis.mk.Galera()
+    common.warningMsg("getGaleraLastShutdownNode method was moved to Galera class. Please change your calls accordingly.")
+    return galera.getGaleraLastShutdownNode(env)
 }
 
-/**
- * Restores Galera database
- * @param env Salt Connection object or pepperEnv
- * @return output of salt commands
- */
 def restoreGaleraDb(env) {
-    def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
-    try {
-        salt.runSaltProcessStep(env, 'I@galera:slave', 'service.stop', ['mysql'])
-    } catch (Exception er) {
-        common.warningMsg('Mysql service already stopped')
-    }
-    try {
-        salt.runSaltProcessStep(env, 'I@galera:master', 'service.stop', ['mysql'])
-    } catch (Exception er) {
-        common.warningMsg('Mysql service already stopped')
-    }
-    lastNodeTarget = getGaleraLastShutdownNode(env)
-    try {
-        salt.cmdRun(env, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
-    } catch (Exception er) {
-        common.warningMsg('Files are not present')
-    }
-    try {
-        salt.cmdRun(env, 'I@galera:slave', "rm  /var/lib/mysql/grastate.dat")
-    } catch (Exception er) {
-        common.warningMsg('Files are not present')
-    }
-    try {
-        salt.cmdRun(env, lastNodeTarget, "mkdir /root/mysql/mysql.bak")
-    } catch (Exception er) {
-        common.warningMsg('Directory already exists')
-    }
-    try {
-        salt.cmdRun(env, lastNodeTarget, "rm -rf /root/mysql/mysql.bak/*")
-    } catch (Exception er) {
-        common.warningMsg('Directory already empty')
-    }
-    try {
-        salt.cmdRun(env, lastNodeTarget, "mv /var/lib/mysql/* /root/mysql/mysql.bak")
-    } catch (Exception er) {
-        common.warningMsg('Files were already moved')
-    }
-    try {
-        salt.runSaltProcessStep(env, lastNodeTarget, 'file.remove', ["/var/lib/mysql/.galera_bootstrap"])
-    } catch (Exception er) {
-        common.warningMsg('File is not present')
-    }
-    salt.cmdRun(env, lastNodeTarget, "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
-    def backup_dir = salt.getReturnValues(salt.getPillar(env, lastNodeTarget, 'xtrabackup:client:backup_dir'))
-    if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
-    salt.runSaltProcessStep(env, lastNodeTarget, 'file.remove', ["${backup_dir}/dbrestored"])
-    salt.cmdRun(env, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
-    salt.runSaltProcessStep(env, lastNodeTarget, 'service.start', ['mysql'])
-
-    // wait until mysql service on galera master is up
-    try {
-        salt.commandStatus(env, lastNodeTarget, 'service mysql status', 'running')
-    } catch (Exception er) {
-        input message: "Database is not running please fix it first and only then click on PROCEED."
-    }
-
-    salt.runSaltProcessStep(env, "I@galera:master and not ${lastNodeTarget}", 'service.start', ['mysql'])
-    salt.runSaltProcessStep(env, "I@galera:slave and not ${lastNodeTarget}", 'service.start', ['mysql'])
+    def galera = new com.mirantis.mk.Galera()
+    common.warningMsg("restoreGaleraDb method was moved to Galera class. Please change your calls accordingly.")
+    return galera.restoreGaleraDb(env)
 }
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 0e2c239..7ef7964 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -650,12 +650,12 @@
         // Run k8s on first node without master.setup and master.kube-addons
         salt.enforceStateWithExclude([saltId: master, target: "${first_target} ${extra_tgt}", state: "kubernetes.master", excludedStates: "kubernetes.master.setup,kubernetes.master.kube-addons"])
         // Run k8s without master.setup and master.kube-addons
-        salt.enforceStateWithExclude([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: "kubernetes", excludedStates: "kubernetes.master.setup,kubernetes.master.kube-addons"])
+        salt.enforceStateWithExclude([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: "kubernetes", excludedStates: "kubernetes.master.setup,kubernetes.master.kube-addons,kubernetes.client"])
     } else {
         // Run k8s on first node without master.setup and master.kube-addons
         salt.enforceStateWithExclude([saltId: master, target: "${first_target} ${extra_tgt}", state: "kubernetes.master", excludedStates: "kubernetes.master.setup"])
         // Run k8s without master.setup
-        salt.enforceStateWithExclude([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: "kubernetes", excludedStates: "kubernetes.master.setup"])
+        salt.enforceStateWithExclude([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: "kubernetes", excludedStates: "kubernetes.master.setup,kubernetes.client"])
     }
 
     // Run k8s master setup
@@ -689,6 +689,13 @@
     salt.runSaltProcessStep(master, "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", 'service.restart', ['kubelet'])
 }
 
+def installKubernetesClient(master, extra_tgt = '') {
+    def salt = new com.mirantis.mk.Salt()
+
+    // Install kubernetes client
+    salt.enforceStateWithTest([saltId: master, target: "I@kubernetes:client ${extra_tgt}", state: 'kubernetes.client'])
+}
+
 
 def installDockerSwarm(master, extra_tgt = '') {
     def salt = new com.mirantis.mk.Salt()
@@ -722,8 +729,8 @@
 def installCicd(master, extra_tgt = '') {
     def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
-    def gerrit_compound = "I@gerrit:client and ci* ${extra_tgt}"
-    def jenkins_compound = "I@jenkins:client and ci* ${extra_tgt}"
+    def gerrit_compound = "I@gerrit:client ${extra_tgt}"
+    def jenkins_compound = "I@jenkins:client ${extra_tgt}"
 
     salt.fullRefresh(master, gerrit_compound)
     salt.fullRefresh(master, jenkins_compound)
@@ -731,13 +738,13 @@
     // Temporary exclude cfg node from docker.client state (PROD-24934)
     def dockerClientExclude = !salt.getPillar(master, 'I@salt:master', 'docker:client:stack:jenkins').isEmpty() ? 'and not I@salt:master' : ''
     // Pull images first if any
-    def listCIMinions = salt.getMinions(master, "ci* ${dockerClientExclude} ${extra_tgt}")
+    def listCIMinions = salt.getMinions(master, "* ${dockerClientExclude} ${extra_tgt}")
     for (int i = 0; i < listCIMinions.size(); i++) {
         if (!salt.getReturnValues(salt.getPillar(master, listCIMinions[i], 'docker:client:images')).isEmpty()) {
-            salt.enforceState([saltId: master, target: listCIMinions[i], state: 'docker.client.images', retries: 2])
+            salt.enforceStateWithTest([saltId: master, target: listCIMinions[i], state: 'docker.client.images', retries: 2])
         }
     }
-    salt.enforceState([saltId: master, target: "I@docker:swarm:role:master and I@jenkins:client ${dockerClientExclude} ${extra_tgt}", state: 'docker.client', retries: 2])
+    salt.enforceStateWithTest([saltId: master, target: "I@docker:swarm:role:master and I@jenkins:client ${dockerClientExclude} ${extra_tgt}", state: 'docker.client', retries: 2])
 
     // API timeout in minutes
     def wait_timeout = 10
@@ -760,6 +767,7 @@
       def gerrit_host
       def gerrit_http_port
       def gerrit_http_scheme
+      def gerrit_http_prefix
 
       def host_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:host')
       gerrit_host = salt.getReturnValues(host_pillar)
@@ -770,7 +778,10 @@
       def scheme_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:protocol')
       gerrit_http_scheme = salt.getReturnValues(scheme_pillar)
 
-      gerrit_master_url = gerrit_http_scheme + '://' + gerrit_host + ':' + gerrit_http_port
+      def prefix_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:url_prefix')
+      gerrit_http_prefix = salt.getReturnValues(prefix_pillar)
+
+      gerrit_master_url = gerrit_http_scheme + '://' + gerrit_host + ':' + gerrit_http_port + gerrit_http_prefix
 
     }
 
@@ -783,7 +794,9 @@
     // Jenkins
     def jenkins_master_host_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_host')
     def jenkins_master_port_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_port')
-    jenkins_master_url = "http://${salt.getReturnValues(jenkins_master_host_pillar)}:${salt.getReturnValues(jenkins_master_port_pillar)}"
+    def jenkins_master_url_prefix_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_url_prefix')
+
+    jenkins_master_url = "http://${salt.getReturnValues(jenkins_master_host_pillar)}:${salt.getReturnValues(jenkins_master_port_pillar)}${salt.getReturnValues(jenkins_master_url_prefix_pillar)}"
 
     timeout(wait_timeout) {
       common.infoMsg('Waiting for Jenkins to come up..')
@@ -798,7 +811,7 @@
     withEnv(['ASK_ON_ERROR=false']){
         retry(2){
             try{
-                salt.enforceState([saltId: master, target: "I@gerrit:client ${extra_tgt}", state: 'gerrit'])
+                salt.enforceStateWithTest([saltId: master, target: "I@gerrit:client ${extra_tgt}", state: 'gerrit'])
             }catch(e){
                 salt.fullRefresh(master, "I@gerrit:client ${extra_tgt}")
                 throw e //rethrow for retry handler
@@ -806,7 +819,7 @@
         }
         retry(2){
             try{
-                salt.enforceState([saltId: master, target: "I@jenkins:client ${extra_tgt}", state: 'jenkins'])
+                salt.enforceStateWithTest([saltId: master, target: "I@jenkins:client ${extra_tgt}", state: 'jenkins'])
             }catch(e){
                 salt.fullRefresh(master, "I@jenkins:client ${extra_tgt}")
                 throw e //rethrow for retry handler
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index 39bd6f1..c326bf6 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -9,17 +9,17 @@
 /**
  * Install python virtualenv
  *
- * @param path          Path to virtualenv
- * @param python        Version of Python (python/python3)
- * @param reqs          Environment requirements in list format
- * @param reqs_path     Environment requirements path in str format
+ * @param path Path to virtualenv
+ * @param python Version of Python (python/python3)
+ * @param reqs Environment requirements in list format
+ * @param reqs_path Environment requirements path in str format
  */
-def setupVirtualenv(path, python = 'python2', reqs=[], reqs_path=null, clean=false, useSystemPackages=false) {
+def setupVirtualenv(path, python = 'python2', reqs = [], reqs_path = null, clean = false, useSystemPackages = false) {
     def common = new com.mirantis.mk.Common()
 
     def offlineDeployment = env.getEnvironment().containsKey("OFFLINE_DEPLOYMENT") && env["OFFLINE_DEPLOYMENT"].toBoolean()
     def virtualenv_cmd = "virtualenv ${path} --python ${python}"
-    if (useSystemPackages){
+    if (useSystemPackages) {
         virtualenv_cmd += " --system-site-packages"
     }
     if (clean) {
@@ -27,19 +27,19 @@
         sh("rm -rf \"${path}\"")
     }
 
-    if(offlineDeployment){
-       virtualenv_cmd+=" --no-download"
+    if (offlineDeployment) {
+        virtualenv_cmd += " --no-download"
     }
     common.infoMsg("[Python ${path}] Setup ${python} environment")
     sh(returnStdout: true, script: virtualenv_cmd)
-    if(!offlineDeployment){
-      try {
-          runVirtualenvCommand(path, "pip install -U setuptools pip")
-      } catch(Exception e) {
-          common.warningMsg("Setuptools and pip cannot be updated, you might be offline but OFFLINE_DEPLOYMENT global property not initialized!")
-      }
+    if (!offlineDeployment) {
+        try {
+            runVirtualenvCommand(path, "pip install -U setuptools pip")
+        } catch (Exception e) {
+            common.warningMsg("Setuptools and pip cannot be updated, you might be offline but OFFLINE_DEPLOYMENT global property not initialized!")
+        }
     }
-    if (reqs_path==null) {
+    if (reqs_path == null) {
         def args = ""
         for (req in reqs) {
             args = args + "${req}\n"
@@ -53,15 +53,15 @@
 /**
  * Run command in specific python virtualenv
  *
- * @param path   Path to virtualenv
- * @param cmd    Command to be executed
+ * @param path Path to virtualenv
+ * @param cmd Command to be executed
  * @param silent dont print any messages (optional, default false)
  */
-def runVirtualenvCommand(path, cmd, silent=false) {
+def runVirtualenvCommand(path, cmd, silent = false) {
     def common = new com.mirantis.mk.Common()
 
     virtualenv_cmd = "set +x; . ${path}/bin/activate; ${cmd}"
-    if(!silent){
+    if (!silent) {
         common.infoMsg("[Python ${path}] Run command ${cmd}")
     }
     output = sh(
@@ -71,15 +71,14 @@
     return output
 }
 
-
 /**
  * Install docutils in isolated environment
  *
- * @param path        Path where virtualenv is created
+ * @param path Path where virtualenv is created
  */
 def setupDocutilsVirtualenv(path) {
     requirements = [
-      'docutils',
+        'docutils',
     ]
     setupVirtualenv(path, 'python2', requirements)
 }
@@ -93,9 +92,9 @@
 /**
  * Parse content from markup-text tables to variables
  *
- * @param tableStr   String representing the table
- * @param mode       Either list (1st row are keys) or item (key, value rows)
- * @param format     Format of the table
+ * @param tableStr String representing the table
+ * @param mode Either list (1st row are keys) or item (key, value rows)
+ * @param format Format of the table
  */
 def parseTextTable(tableStr, type = 'item', format = 'rest', path = none) {
     parserFile = "${env.WORKSPACE}/textTableParser.py"
@@ -226,9 +225,8 @@
     cmd = "python ${parserFile} --file '${tableFile}' --type ${type}"
     if (path) {
         rawData = runVirtualenvCommand(path, cmd)
-    }
-    else {
-        rawData = sh (
+    } else {
+        rawData = sh(
             script: cmd,
             returnStdout: true
         ).trim()
@@ -241,7 +239,7 @@
 /**
  * Install cookiecutter in isolated environment
  *
- * @param path        Path where virtualenv is created
+ * @param path Path where virtualenv is created
  */
 def setupCookiecutterVirtualenv(path) {
     requirements = [
@@ -265,19 +263,10 @@
     def common = new com.mirantis.mk.Common()
     configFile = "default_config.yaml"
     writeFile file: configFile, text: context
-    if (fileExists(new File(templatePath, 'tox.ini').toString())) {
-        withEnv(["CONFIG_FILE=$configFile",
-                 "OUTPUT_DIR=$outputDir",
-                 "TEMPLATE=$template"
-        ]) {
-            output = sh(returnStdout: true, script: "tox -ve generate")
-        }
-    } else {
-        common.warningMsg('Old Cookiecutter env detected!')
-        command = ". ${path}/bin/activate; if [ -f ${templatePath}/generate.py ]; then python ${templatePath}/generate.py --config-file ${configFile} --template ${template} --output-dir ${outputDir}; else cookiecutter --config-file ${configFile} --output-dir ${outputDir} --overwrite-if-exists --verbose --no-input ${template}; fi"
-        output = sh(returnStdout: true, script: command)
-    }
-    common.infoMsg("[Cookiecutter build] Result: ${output}")
+    common.warningMsg('Old Cookiecutter env detected!')
+    command = ". ${path}/bin/activate; if [ -f ${templatePath}/generate.py ]; then python ${templatePath}/generate.py --config-file ${configFile} --template ${template} --output-dir ${outputDir}; else cookiecutter --config-file ${configFile} --output-dir ${outputDir} --overwrite-if-exists --verbose --no-input ${template}; fi"
+    output = sh(returnStdout: true, script: command)
+    common.infoMsg('[Cookiecutter build] Result:' + output)
 }
 
 /**
@@ -300,54 +289,66 @@
     def templateDir = "${templateEnvDir}/dir"
     def templateOutputDir = templateBaseDir
     dir(templateEnvDir) {
-        common.infoMsg("Generating model from context ${contextName}")
-        def productList = ["infra", "cicd", "kdt", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
-        for (product in productList) {
-            // get templateOutputDir and productDir
-            templateOutputDir = "${templateEnvDir}/output/${product}"
-            productDir = product
-            templateDir = "${templateEnvDir}/cluster_product/${productDir}"
-            // Bw for 2018.8.1 and older releases
-            if (product.startsWith("stacklight") && (!fileExists(templateDir))) {
-                common.warningMsg("Old release detected! productDir => 'stacklight2' ")
-                productDir = "stacklight2"
+        if (fileExists(new File(templateEnvDir, 'tox.ini').toString())) {
+            def tempContextFile = new File(templateEnvDir, 'tempContext.yaml').toString()
+            writeFile file: tempContextFile, text: context
+            common.warningMsg('Generating models using context:\n')
+            print(context)
+            withEnv(["CONFIG_FILE=$tempContextFile",
+                     "OUTPUT_DIR=${modelEnv}",
+            ]) {
+                print('[Cookiecutter build] Result:\n' +
+                    sh(returnStdout: true, script: 'tox -ve generate_auto'))
+            }
+        } else {
+            common.warningMsg("Old format: Generating model from context ${contextName}")
+            def productList = ["infra", "cicd", "kdt", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
+            for (product in productList) {
+                // get templateOutputDir and productDir
+                templateOutputDir = "${templateEnvDir}/output/${product}"
+                productDir = product
                 templateDir = "${templateEnvDir}/cluster_product/${productDir}"
-            }
-            // generate infra unless its explicitly disabled
-            if ((product == "infra" && templateContext.default_context.get("infra_enabled", "True").toBoolean())
-                 || (templateContext.default_context.get(product + "_enabled", "False").toBoolean())) {
+                // Bw for 2018.8.1 and older releases
+                if (product.startsWith("stacklight") && (!fileExists(templateDir))) {
+                    common.warningMsg("Old release detected! productDir => 'stacklight2' ")
+                    productDir = "stacklight2"
+                    templateDir = "${templateEnvDir}/cluster_product/${productDir}"
+                }
+                // generate infra unless its explicitly disabled
+                if ((product == "infra" && templateContext.default_context.get("infra_enabled", "True").toBoolean())
+                    || (templateContext.default_context.get(product + "_enabled", "False").toBoolean())) {
 
-                common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
+                    common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
 
-                sh "rm -rf ${templateOutputDir} || true"
-                sh "mkdir -p ${templateOutputDir}"
-                sh "mkdir -p ${outputDestination}"
+                    sh "rm -rf ${templateOutputDir} || true"
+                    sh "mkdir -p ${templateOutputDir}"
+                    sh "mkdir -p ${outputDestination}"
 
-                buildCookiecutterTemplate(templateDir, context, templateOutputDir, virtualenv, templateBaseDir)
-                sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-            } else {
-                common.warningMsg("Product " + product + " is disabled")
-            }
-        }
-
-        def localRepositories = templateContext.default_context.local_repositories
-        localRepositories = localRepositories ? localRepositories.toBoolean() : false
-        def offlineDeployment = templateContext.default_context.offline_deployment
-        offlineDeployment = offlineDeployment ? offlineDeployment.toBoolean() : false
-        if (localRepositories && !offlineDeployment) {
-            def mcpVersion = templateContext.default_context.mcp_version
-            def aptlyModelUrl = templateContext.default_context.local_model_url
-            def ssh = new com.mirantis.mk.Ssh()
-            dir(path: modelEnv) {
-                ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
-                if (!(mcpVersion in ["nightly", "testing", "stable"])) {
-                    ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
+                    buildCookiecutterTemplate(templateDir, context, templateOutputDir, virtualenv, templateBaseDir)
+                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+                } else {
+                    common.warningMsg("Product " + product + " is disabled")
                 }
             }
-        }
 
-        def nodeFile = "${generatedModel}/nodes/${saltMasterName}.${clusterDomain}.yml"
-        def nodeString = """classes:
+            def localRepositories = templateContext.default_context.local_repositories
+            localRepositories = localRepositories ? localRepositories.toBoolean() : false
+            def offlineDeployment = templateContext.default_context.offline_deployment
+            offlineDeployment = offlineDeployment ? offlineDeployment.toBoolean() : false
+            if (localRepositories && !offlineDeployment) {
+                def mcpVersion = templateContext.default_context.mcp_version
+                def aptlyModelUrl = templateContext.default_context.local_model_url
+                def ssh = new com.mirantis.mk.Ssh()
+                dir(path: modelEnv) {
+                    ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
+                    if (!(mcpVersion in ["nightly", "testing", "stable"])) {
+                        ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
+                    }
+                }
+            }
+
+            def nodeFile = "${generatedModel}/nodes/${saltMasterName}.${clusterDomain}.yml"
+            def nodeString = """classes:
 - cluster.${clusterName}.infra.config
 parameters:
   _param:
@@ -358,20 +359,21 @@
       name: ${saltMasterName}
       domain: ${clusterDomain}
     """
-        sh "mkdir -p ${generatedModel}/nodes/"
-        writeFile(file: nodeFile, text: nodeString)
+            sh "mkdir -p ${generatedModel}/nodes/"
+            writeFile(file: nodeFile, text: nodeString)
+        }
     }
 }
 
 /**
  * Install jinja rendering in isolated environment
  *
- * @param path        Path where virtualenv is created
+ * @param path Path where virtualenv is created
  */
 def setupJinjaVirtualenv(path) {
     requirements = [
-      'jinja2-cli',
-      'pyyaml',
+        'jinja2-cli',
+        'pyyaml',
     ]
     setupVirtualenv(path, 'python2', requirements)
 }
@@ -379,9 +381,9 @@
 /**
  * Generate the Jinja templates with given context
  *
- * @param path        Path where virtualenv is created
+ * @param path Path where virtualenv is created
  */
-def jinjaBuildTemplate (template, context, path = none) {
+def jinjaBuildTemplate(template, context, path = none) {
     contextFile = "jinja_context.yml"
     contextString = ""
     for (parameter in context) {
@@ -389,7 +391,7 @@
     }
     writeFile file: contextFile, text: contextString
     cmd = "jinja2 ${template} ${contextFile} --format=yaml"
-    data = sh (returnStdout: true, script: cmd)
+    data = sh(returnStdout: true, script: cmd)
     echo(data)
     return data
 }
@@ -397,9 +399,9 @@
 /**
  * Install salt-pepper in isolated environment
  *
- * @param path        Path where virtualenv is created
- * @param url         SALT_MASTER_URL
- * @param credentialsId        Credentials to salt api
+ * @param path Path where virtualenv is created
+ * @param url SALT_MASTER_URL
+ * @param credentialsId Credentials to salt api
  */
 def setupPepperVirtualenv(path, url, credentialsId) {
     def common = new com.mirantis.mk.Common()
@@ -426,10 +428,10 @@
 /**
  * Install devops in isolated environment
  *
- * @param path        Path where virtualenv is created
- * @param clean       Define to true is the venv have to cleaned up before install a new one
+ * @param path Path where virtualenv is created
+ * @param clean Define to true is the venv have to cleaned up before install a new one
  */
-def setupDevOpsVenv(venv, clean=false) {
+def setupDevOpsVenv(venv, clean = false) {
     requirements = ['git+https://github.com/openstack/fuel-devops.git']
     setupVirtualenv(venv, 'python2', requirements, null, false, clean)
 }
diff --git a/src/com/mirantis/mk/Ruby.groovy b/src/com/mirantis/mk/Ruby.groovy
index dc08ce5..7802a11 100644
--- a/src/com/mirantis/mk/Ruby.groovy
+++ b/src/com/mirantis/mk/Ruby.groovy
@@ -9,12 +9,13 @@
  * @param rubyVersion target ruby version (optional, default 2.2.3)
  */
 def ensureRubyEnv(rubyVersion="2.4.1"){
-    if(!fileExists("/var/lib/jenkins/.rbenv/versions/${rubyVersion}/bin/ruby")){
+    def ruby_build_root = "${env.WORKSPACE}/.rbenv/plugins/ruby-build"
+    if (!fileExists("/var/lib/jenkins/.rbenv/versions/${rubyVersion}/bin/ruby")){
         //XXX: patch ruby-build because debian package is quite old
-        sh "git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build"
-        sh "rbenv install ${rubyVersion}";
+        sh "rbenv install ${rubyVersion} -sv";
     }
     sh "rbenv local ${rubyVersion};rbenv exec gem update --system"
+    sh "rm -rf ${ruby_build_root}"
 }
 
 /**
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 006df14..7688c0e 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -485,6 +485,39 @@
 }
 
 /**
+ * You can call this function when need to check that all minions are available, free and ready for command execution
+ * @param config LinkedHashMap config parameter, which contains next:
+ *   @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+ *   @param target unique identification of a minion or group of salt minions
+ *   @param target_reachable unique identification of a minion or group of salt minions to check availability
+ *   @param wait timeout between retries to check target minions (default 5)
+ *   @param retries finite number of iterations to check minions (default 10)
+ *   @param timeout timeout for the salt command if minions do not return (default 5)
+ *   @param availability check that minions also are available before checking readiness (default true)
+ */
+def checkTargetMinionsReady(LinkedHashMap config) {
+    def common = new com.mirantis.mk.Common()
+    def saltId = config.get('saltId')
+    def target = config.get('target')
+    def target_reachable = config.get('target_reachable', target)
+    def wait = config.get('wait', 30)
+    def retries = config.get('retries', 10)
+    def timeout = config.get('timeout', 5)
+    def checkAvailability = config.get('availability', true)
+    common.retry(retries, wait) {
+        if (checkAvailability) {
+            minionsReachable(saltId, 'I@salt:master', target_reachable)
+        }
+        def running = salt.runSaltProcessStep(saltId, target, 'saltutil.running', [], null, true, timeout)
+        for (value in running.get("return")[0].values()) {
+            if (value != []) {
+                throw new Exception("Not all salt-minions are ready for execution")
+            }
+        }
+    }
+}
+
+/**
  * Run command on salt minion (salt cmd.run wrapper)
  * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
  * @param target Get pillar target
@@ -1163,3 +1196,33 @@
         return false
     }
 }
+
+/**
+* Finds out IP address of the given node or a list of nodes
+*
+* @param saltId     Salt Connection object or pepperEnv (the command will be sent using the selected method)
+* @param nodes      Targeted node hostnames to be checked (String or List of strings)
+* @param useGrains  If the, the value will be taken from grains. If false, it will be taken from 'hostname' command.
+* @return Map       Return result Map in format ['nodeName1': 'ipAdress1', 'nodeName2': 'ipAdress2', ...]
+*/
+
+def getIPAddressesForNodenames(saltId, nodes = [], useGrains = true) {
+    result = [:]
+
+    if (nodes instanceof String) {
+        nodes = [nodes]
+    }
+
+    if (useGrains) {
+        for (String node in nodes) {
+            ip = getReturnValues(getGrain(saltId, node, "fqdn_ip4"))["fqdn_ip4"][0]
+            result[node] = ip
+        }
+    } else {
+        for (String node in nodes) {
+            ip = getReturnValues(cmdRun(saltId, node, "hostname -i")).readLines()[0]
+            result[node] = ip
+        }
+    }
+    return result
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 2dd9b38..0e49a42 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -50,17 +50,21 @@
     if (baseRepoPreConfig) {
         // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release
         def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main"
+        def releaseVersionQ4 = '2018.11.0'
+        def oldRelease = false
         try {
             def releaseNaming = 'yyyy.MM.dd'
             def repoDateUsed = new Date().parse(releaseNaming, distribRevision)
-            def extraAvailableFrom = new Date().parse(releaseNaming, '2018.11.0')
+            def extraAvailableFrom = new Date().parse(releaseNaming, releaseVersionQ4)
             if (repoDateUsed < extraAvailableFrom) {
                 extraRepoSource = "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
+                oldRelease = true
             }
         } catch (Exception e) {
             common.warningMsg(e)
             if (!(distribRevision in ['nightly', 'proposed', 'testing'])) {
                 extraRepoSource = "deb [arch=amd64] http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
+                oldRelease = true
             }
         }
 
@@ -96,6 +100,12 @@
         def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'override')
         def extraRepos = config.get('extraRepos', [:])
         def defaultRepos = readYaml text: defaultExtraReposYaml
+        if (! oldRelease && distribRevision != releaseVersionQ4) {
+            defaultRepos['repo']['mcp_saltformulas_update'] = [
+                'source': "deb [arch=amd64]  http://mirror.mirantis.com/update/${distribRevision}/salt-formulas/xenial xenial main",
+                'repo_key': "http://mirror.mirantis.com/update/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
+            ]
+        }
         if (extraRepoMergeStrategy == 'merge') {
             extraReposConfig = common.mergeMaps(defaultRepos, extraRepos)
         } else {
@@ -268,6 +278,7 @@
 def testNode(LinkedHashMap config) {
     def common = new com.mirantis.mk.Common()
     def dockerHostname = config.get('dockerHostname')
+    def domain = config.get('domain')
     def reclassEnv = config.get('reclassEnv')
     def clusterName = config.get('clusterName', "")
     def formulasSource = config.get('formulasSource', 'pkg')
@@ -276,10 +287,11 @@
     def aptRepoUrl = config.get('aptRepoUrl', "")
     def aptRepoGPG = config.get('aptRepoGPG', "")
     def testContext = config.get('testContext', 'test')
+    def nodegenerator = config.get('nodegenerator', false)
     config['envOpts'] = [
         "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5",
-        "MASTER_HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
-        "MINION_ID=${dockerHostname}", "FORMULAS_SOURCE=${formulasSource}",
+        "HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
+        "DOMAIN=${domain}", "FORMULAS_SOURCE=${formulasSource}",
         "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
         "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1",
         "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}"
@@ -293,6 +305,7 @@
         '002_Prepare_something'          : {
             sh('''#!/bin/bash -x
               rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2  salt' >> /etc/hosts
+              echo "127.0.0.1 ${HOSTNAME}.${DOMAIN}" >> /etc/hosts
               if [ -f '/srv/salt/reclass/salt_master_pillar.asc' ] ; then
                 mkdir -p /etc/salt/gpgkeys
                 chmod 700 /etc/salt/gpgkeys
@@ -340,6 +353,31 @@
             archiveArtifacts artifacts: "nodesinfo.tar.gz"
         }
     ]
+    // this tool should be tested in master branch only
+    // and not for all jobs, as pilot will be used cc-reclass-chunk
+    if (nodegenerator) {
+        config['runCommands']['005_Test_new_nodegenerator'] = {
+            try {
+                sh('''#!/bin/bash
+                new_generated_dir=/srv/salt/_new_generated
+                mkdir -p ${new_generated_dir}
+                nodegenerator -b /srv/salt/reclass/classes/ -o ${new_generated_dir} ${CLUSTER_NAME}
+                diff -r /srv/salt/reclass/nodes/_generated ${new_generated_dir} > /tmp/nodegenerator.diff
+                tar -czf /tmp/_generated.tar.gz /srv/salt/reclass/nodes/_generated/
+                tar -czf /tmp/_new_generated.tar.gz ${new_generated_dir}/
+                tar -czf /tmp/_model.tar.gz /srv/salt/reclass/classes/cluster/*
+                ''')
+            } catch (Exception e) {
+                print "Test new nodegenerator tool is failed: ${e}"
+            }
+        }
+        config['runFinally']['002_Archive_nodegenerator_artefact'] = {
+            sh(script: "cd /tmp; [ -f nodegenerator.diff ] && tar -czf ${env.WORKSPACE}/nodegenerator.tar.gz nodegenerator.diff _generated.tar.gz _new_generated.tar.gz _model.tar.gz", returnStatus: true)
+            if (fileExists('nodegenerator.tar.gz')) {
+                archiveArtifacts artifacts: "nodegenerator.tar.gz"
+            }
+        }
+    }
     testResult = setupDockerAndTest(config)
     if (testResult) {
         common.infoMsg("Node test for context: ${testContext} model: ${reclassEnv} finished: SUCCESS")