Merge "Fixed backward compatibility lab pipeline code"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 799b806..b34db68 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -19,6 +19,7 @@
  *
  *   AWS_STACK_REGION           CloudFormation AWS region
  *   AWS_API_CREDENTIALS        AWS Access key ID with  AWS secret access key
+ *   AWS_SSH_KEY                AWS key pair name (used for SSH access)
  *
  *   HEAT_STACK_ENVIRONMENT     Heat stack environmental parameters
  *   HEAT_STACK_ZONE            Heat stack availability zone
@@ -41,6 +42,7 @@
 common = new com.mirantis.mk.Common()
 git = new com.mirantis.mk.Git()
 openstack = new com.mirantis.mk.Openstack()
+aws = new com.mirantis.mk.Aws()
 orchestrate = new com.mirantis.mk.Orchestrate()
 salt = new com.mirantis.mk.Salt()
 test = new com.mirantis.mk.Test()
@@ -108,23 +110,66 @@
                                 'instance_zone': HEAT_STACK_ZONE,
                                 'public_net': HEAT_STACK_PUBLIC_NET
                             ]
-                            openstack.createHeatStack(openstackCloud, STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+                            openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
                         }
                     }
 
                     // get SALT_MASTER_URL
                     saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, STACK_NAME, 'salt_master_ip', openstackEnv)
-                    currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
+                    currentBuild.description = "${STACK_NAME} ${saltMasterHost}"
 
                     SALT_MASTER_URL = "http://${saltMasterHost}:6969"
-                }
+                } else if (STACK_TYPE == 'aws') {
 
-                if (STACK_TYPE == 'aws') {
-                    saltMasterHost = ''
-                    currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
+                    def venv_path = 'aws_venv'
+
+                    if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
+                        error("If you want to reuse existing stack you need to provide it's name")
+                    }
+
+                    if (STACK_REUSE.toBoolean() == false) {
+                        // Don't allow to set custom stack name
+                        wrap([$class: 'BuildUser']) {
+                            if (env.BUILD_USER_ID) {
+                                STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
+                            } else {
+                                STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
+                            }
+                        }
+
+                        // no underscore in STACK_NAME
+                        STACK_NAME = STACK_NAME.replaceAll('_', '-')
+                    }
+
+                    // set description
+                    currentBuild.description = STACK_NAME
+
+                    // prepare configuration
+                    def env_vars = aws.getEnvVars(AWS_API_CREDENTIALS, AWS_STACK_REGION)
+
+                    if (STACK_REUSE.toBoolean() == false) {
+                        // get templates
+                        git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
+
+                        // setup environment
+                        aws.setupVirtualEnv(venv_path)
+
+                        // start stack
+                        def stack_params = ["ParameterKey=KeyName,ParameterValue=" + AWS_SSH_KEY]
+                        aws.createStack(venv_path, env_vars, STACK_TEMPLATE, STACK_NAME, stack_params)
+                    }
+
+                    // wait for stack to be ready
+                    aws.waitForStatus(venv_path, env_vars, STACK_NAME, 'CREATE_COMPLETE')
+
+                    // get outputs
+                    saltMasterHost = aws.getOutputs(venv_path, env_vars, STACK_NAME, 'SaltMasterIP')
+                    currentBuild.description = "${STACK_NAME} ${saltMasterHost}"
                     SALT_MASTER_URL = "http://${saltMasterHost}:6969"
-                }
 
+                } else {
+                    throw new Exception("STACK_TYPE ${STACK_TYPE} is not supported")
+                }
             }
 
             //
@@ -299,7 +344,7 @@
                 if (STACK_DELETE.toBoolean() == true) {
                     common.errorMsg('Heat job cleanup triggered')
                     stage('Trigger cleanup job') {
-                        build job: STACK_CLEANUP_JOB, parameters: [[$class: 'StringParameterValue', name: 'STACK_NAME', value: STACK_NAME]]
+                        build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'STACK_NAME', value: STACK_NAME]]
                     }
                 } else {
                     if (currentBuild.result == 'FAILURE') {
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index a9ca48f..e64abda 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -17,11 +17,13 @@
 common = new com.mirantis.mk.Common()
 git = new com.mirantis.mk.Git()
 python = new com.mirantis.mk.Python()
+saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 
 timestamps {
-    node() {
+    node("python&&docker") {
         def templateEnv = "${env.WORKSPACE}/template"
         def modelEnv = "${env.WORKSPACE}/model"
+        def testEnv = "${env.WORKSPACE}/test"
 
         try {
             def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
@@ -48,68 +50,20 @@
                 }
             }
 
-            stage('Generate base infrastructure') {
-                templateDir = "${templateEnv}/cluster_product/infra"
-                templateOutputDir = "${env.WORKSPACE}/template/output/infra"
-                sh "mkdir -p ${templateOutputDir}"
-                sh "mkdir -p ${outputDestination}"
-                python.setupCookiecutterVirtualenv(cutterEnv)
-                python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-            }
-
-            stage('Generate product CI/CD') {
-                if (templateContext.default_context.cicd_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/cicd"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/cicd"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product OpenContrail') {
-                if (templateContext.default_context.opencontrail_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/opencontrail"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/opencontrail"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product Kubernetes') {
-                if (templateContext.default_context.kubernetes_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/kubernetes"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/kubernetes"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product OpenStack') {
-                if (templateContext.default_context.openstack_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/openstack"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/openstack"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product StackLight') {
-                if (templateContext.default_context.stacklight_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/stacklight"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/stacklight"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+            def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+            for (product in productList) {
+                def stagename = (product == "infra") ? "Generate base infrastructure" : "Generate product ${product}"
+                stage(stagename) {
+                    if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+                        && templateContext.default_context["${product}_enabled"].toBoolean())) {
+                        templateDir = "${templateEnv}/cluster_product/${product}"
+                        templateOutputDir = "${env.WORKSPACE}/template/output/${product}"
+                        sh "mkdir -p ${templateOutputDir}"
+                        sh "mkdir -p ${outputDestination}"
+                        python.setupCookiecutterVirtualenv(cutterEnv)
+                        python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
+                        sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+                    }
                 }
             }
 
@@ -130,6 +84,15 @@
                 writeFile(file: nodeFile, text: nodeString)
             }
 
+            stage("Test") {
+                if (RECLASS_MODEL_URL == "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
+                    sh("cp -r ${modelEnv} ${testEnv}")
+                    def defaultReclassModel = "ssh://jenkins-mk@gerrit.mcp.mirantis.net:29418/salt-models/reclass-system"
+                    git.checkoutGitRepository("${testEnv}/classes/system", defaultReclassModel, RECLASS_MODEL_BRANCH, RECLASS_MODEL_CREDENTIALS)
+                    saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", testEnv)
+                }
+            }
+
             stage ('Save changes to Reclass model') {
                 if (env.getEnvironment().containsKey('COMMIT_CHANGES') && COMMIT_CHANGES.toBoolean() && RECLASS_MODEL_URL != null && RECLASS_MODEL_URL != "") {
                     git.changeGitBranch(modelEnv, targetBranch)
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index 7ddfbe8..1158a74 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -41,6 +41,8 @@
  *   CALICO_CNI_IMAGE            Docker repository and tag for calico CNI image
  *   CALICO_NODE_IMAGE           Docker repository and tag for calico node image
  *   CALICOCTL_IMAGE             Docker repository and tag for calicoctl image
+ *   NETCHECKER_AGENT_IMAGE      Docker repository and tag for netchecker agent image
+ *   NETCHECKER_SERVER_IMAGE      Docker repository and tag for netchecker server image
  *
  */
 
@@ -189,6 +191,15 @@
                       salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_calicoctl_image: ${CALICOCTL_IMAGE}")
                     }
 
+                    // Overwrite netchecker vars if specified
+                    if (env.getEnvironment().containsKey("NETCHECKER_AGENT_IMAGE")) {
+                      salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_netchecker_agent_image: ${NETCHECKER_AGENT_IMAGE}")
+                    }
+                    if (env.getEnvironment().containsKey("NETCHECKER_SERVER_IMAGE")) {
+                      salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_netchecker_server_image: ${NETCHECKER_SERVER_IMAGE}")
+                    }
+
+
                     orchestrate.installKubernetesControl(saltMaster)
                 }
 
diff --git a/mk-k8s-simple-deploy-pipeline.groovy b/mk-k8s-simple-deploy-pipeline.groovy
index 48bd064..4aae816 100644
--- a/mk-k8s-simple-deploy-pipeline.groovy
+++ b/mk-k8s-simple-deploy-pipeline.groovy
@@ -1,5 +1,5 @@
 /**
- * DO NOT USE THIS OUTDATED PIPELINE - add your steps to mk-lab-pipeline
+ * DO NOT USE THIS OUTDATED PIPELINE - add your steps to lab-pipeline
  *
  * Launch heat stack with basic k8s
  *
diff --git a/mk-lab-pipeline.groovy b/mk-lab-pipeline.groovy
deleted file mode 100644
index 9e9a28d..0000000
--- a/mk-lab-pipeline.groovy
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- *
- * Launch heat stack with basic k8s
- *
- * Expected parameters:
- *   HEAT_TEMPLATE_URL          URL to git repo with Heat templates
- *   HEAT_TEMPLATE_CREDENTIALS  Credentials to the Heat templates repo
- *   HEAT_TEMPLATE_BRANCH       Heat templates repo branch
- *   HEAT_STACK_NAME            Heat stack name
- *   HEAT_STACK_TEMPLATE        Heat stack HOT template
- *   HEAT_STACK_ENVIRONMENT     Heat stack environmental parameters
- *   HEAT_STACK_ZONE            Heat stack availability zone
- *   HEAT_STACK_PUBLIC_NET      Heat stack floating IP pool
- *   HEAT_STACK_DELETE          Delete Heat stack when finished (bool)
- *   HEAT_STACK_CLEANUP_JOB     Name of job for deleting Heat stack
- *   HEAT_STACK_REUSE           Reuse Heat stack (don't create one)
- *
- *   SALT_MASTER_CREDENTIALS    Credentials to the Salt API
- *
- *   OPENSTACK_API_URL          OpenStack API address
- *   OPENSTACK_API_CREDENTIALS  Credentials to the OpenStack API
- *   OPENSTACK_API_PROJECT      OpenStack project to connect to
- *   OPENSTACK_API_CLIENT       Versions of OpenStack python clients
- *   OPENSTACK_API_VERSION      Version of the OpenStack API (2/3)
- *
- *   K8S_API_SERVER             Kubernetes API address
- *   K8S_CONFORMANCE_IMAGE      Path to docker image with conformance e2e tests
- *
- *   TEMPEST_IMAGE_LINK         Tempest image link
- *
- *   INSTALL                    What should be installed (k8s, openstack, ...)
- *   TESTS                      Run tests (bool)
- */
-
-git = new com.mirantis.mk.Git()
-openstack = new com.mirantis.mk.Openstack()
-salt = new com.mirantis.mk.Salt()
-orchestrate = new com.mirantis.mk.Orchestrate()
-test = new com.mirantis.mk.Test()
-
-node {
-
-    // connection objects
-    def openstackCloud
-    def saltMaster
-
-    // value defaults
-    def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
-    def openstackEnv = "${env.WORKSPACE}/venv"
-
-    if (HEAT_STACK_NAME == '') {
-        HEAT_STACK_NAME = BUILD_TAG
-    }
-
-    //
-    // Bootstrap
-    //
-
-    stage ('Download Heat templates') {
-        git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
-    }
-
-    stage('Install OpenStack CLI') {
-        openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
-    }
-
-    stage('Connect to OpenStack cloud') {
-        openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
-        openstack.getKeystoneToken(openstackCloud, openstackEnv)
-    }
-
-    if (HEAT_STACK_REUSE == 'false') {
-        stage('Launch new Heat stack') {
-            envParams = [
-                'instance_zone': HEAT_STACK_ZONE,
-                'public_net': HEAT_STACK_PUBLIC_NET
-            ]
-            openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
-        }
-    }
-
-    stage('Connect to Salt master') {
-        saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
-        saltMasterUrl = "http://${saltMasterHost}:8088"
-        saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
-    }
-
-    //
-    // Install
-    //
-
-    stage('Install core infra') {
-        // salt.master, reclass
-        // refresh_pillar
-        // sync_all
-        // linux,openssh,salt.minion.ntp
-
-        orchestrate.installFoundationInfra(saltMaster)
-        orchestrate.validateFoundationInfra(saltMaster)
-    }
-
-
-    if (INSTALL.toLowerCase().contains('k8s')) {
-        stage('Install Kubernetes infra') {
-            orchestrate.installOpenstackMcpInfra(saltMaster)
-        }
-
-        stage('Install Kubernetes control') {
-            orchestrate.installOpenstackMcpControl(saltMaster)
-        }
-
-        if (TESTS.toLowerCase().contains('k8s')) {
-            stage('Run k8s bootstrap tests') {
-                test.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
-            }
-
-            stage('Run k8s conformance e2e tests') {
-                test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
-            }
-
-            stage("Copy k8s e2e test output to config node ") {
-                test.copyTestsOutput(saltMaster,K8S_CONFORMANCE_IMAGE)
-            }
-        }
-    }
-
-    if (INSTALL.toLowerCase().contains('openstack')) {
-        // install Infra and control, tests, ...
-
-        stage('Install OpenStack infra') {
-            orchestrate.installOpenstackMkInfra(saltMaster)
-        }
-
-        stage('Install OpenStack control') {
-            orchestrate.installOpenstackMkControl(saltMaster)
-        }
-
-        stage('Install OpenStack network') {
-            orchestrate.installOpenstackMkNetwork(saltMaster)
-        }
-
-        stage('Install OpenStack compute') {
-            orchestrate.installOpenstackMkCompute(saltMaster)
-        }
-
-        if (TESTS.toLowerCase().contains('openstack')) {
-            stage('Run OpenStack tests') {
-                test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
-            }
-
-            stage('Copy Tempest results to config node') {
-                test.copyTempestResults(saltMaster)
-            }
-        }
-    }
-
-    //
-    // Cleanup
-    //
-
-    if (HEAT_STACK_DELETE == 'true') {
-        stage('Trigger cleanup job') {
-            build job: 'deploy_heat_cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
-        }
-    }
-}
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 60fa160..7dd2ea1 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -30,12 +30,20 @@
                 try {
                     salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
                 } catch (Exception e) {
-                    common.warningMsg(" Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
+                    common.warningMsg("Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
                 }
 
-                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
-                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
+                try {
+                    salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
+                } catch (Exception e) {
+                    common.warningMsg("No response from some minions. We should continue to run")
+                }
                 
+                try {
+                    salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
+                } catch (Exception e) {
+                    common.warningMsg("No response from some minions. We should continue to run")
+                }
 
                 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
                 def domain = _pillar['return'][0].values()[0].values()[0]
@@ -110,7 +118,7 @@
                 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
                 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
                 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
-                salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
+                salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
 
 
                 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
@@ -262,7 +270,7 @@
 
 
                 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
-                salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
+                salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
 
                 try {
                     salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index fad3d4e..3ed8f61 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -11,6 +11,7 @@
 def gerrit = new com.mirantis.mk.Gerrit()
 def ssh = new com.mirantis.mk.Ssh()
 def git = new com.mirantis.mk.Git()
+def saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 
 def gerritRef
 try {
@@ -71,6 +72,7 @@
     }
 
     stage("test-nodes") {
+      def workspace = common.getWorkspace()
       def nodes = sh(script: "find ./nodes -type f -name 'cfg*.yml'", returnStdout: true).tokenize()
       def buildSteps = [:]
       if(nodes.size() > 1){
@@ -78,7 +80,7 @@
             common.infoMsg("Found <=3  cfg nodes, running parallel test")
              for(int i=0; i < nodes.size();i++){
                def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
-               buildSteps.put("node-${basename}", { setupAndTestNode(basename) })
+               buildSteps.put("node-${basename}", { saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace) })
              }
              parallel buildSteps
           }else{
@@ -89,7 +91,7 @@
               buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
               for(int k=0; k < partition.size;k++){
                   def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
-                  buildSteps.get("partition-${i}").put(basename, { setupAndTestNode(basename) })
+                  buildSteps.get("partition-${i}").put(basename, { saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace) })
               }
             }
             common.serial(buildSteps)
@@ -97,7 +99,7 @@
       }else{
           common.infoMsg("Found one cfg node, running single test")
           def basename = sh(script: "basename ${nodes[0]} .yml", returnStdout: true).trim()
-          setupAndTestNode(basename)
+          saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace)
       }
     }
 
@@ -110,53 +112,6 @@
   }
 }
 
-def setupAndTestNode(masterName) {
-  def img = docker.image("ubuntu:latest")
-  def saltOpts = "--retcode-passthrough --force-color"
-  def common = new com.mirantis.mk.Common()
-  def workspace = common.getWorkspace()
-
-  img.inside("-u root:root --hostname=${masterName}") {
-    wrap([$class: 'AnsiColorBuildWrapper']) {
-      sh("mkdir -p /srv/salt/ || true")
-      sh("cp -r ${workspace} /srv/salt/reclass")
-      sh("apt-get update && apt-get install -y curl subversion git python-pip sudo python-pip python-dev zlib1g-dev git")
-      sh("svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts")
-      sh("git config --global user.email || git config --global user.email 'ci@ci.local'")
-      sh("git config --global user.name || git config --global user.name 'CI'")
-      sh("pip install git+https://github.com/epcim/reclass.git@pr/fix/fix_raise_UndefinedVariableError")
-      sh("ls -lRa /srv/salt/reclass")
-
-      // setup iniot and verify salt master and minions
-      withEnv(["FORMULAS_SOURCE=pkg", "EXTRA_FORMULAS=${EXTRA_FORMULAS}", "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "MINION_ID=${masterName}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
-          sh("bash -c 'echo $MASTER_HOSTNAME'")
-          sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && system_config'")
-          sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_bootstrap'")
-          sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_init'")
-      }
-      sh("ls -lRa /srv/salt/reclass/classes/service/")
-
-      def nodes
-      if (DEFAULT_GIT_URL.contains("mk-ci")) {
-        nodes = sh script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
-      } else {
-        nodes = sh script:"find /srv/salt/reclass/nodes/_generated -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
-      }
-      for (minion in nodes.tokenize()) {
-        def basename = sh script: "basename ${minion} .yml", returnStdout: true
-        if (!basename.trim().contains(masterName)) {
-          testMinion(basename.trim())
-        }
-      }
-    }
-  }
-}
-
-def testMinion(minionName)
-{
-  sh("service salt-master restart && service salt-minion restart && sleep 5 && bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'")
-}
-
 @NonCPS
 def _getRunningTriggeredTestsBuildNumbers(jobName, gerritChangeNumber, excludePatchsetNumber){
   def gerrit = new com.mirantis.mk.Gerrit()