Merge "Extend git checkout function by timeout possibility."
diff --git a/src/com/mirantis/mcp/Calico.groovy b/src/com/mirantis/mcp/Calico.groovy
index 2ed3563..0d7428c 100644
--- a/src/com/mirantis/mcp/Calico.groovy
+++ b/src/com/mirantis/mcp/Calico.groovy
@@ -7,10 +7,11 @@
  * @param config LinkedHashMap
  *        config includes next parameters:
  *          - project_name String, Calico project to clone
- *          - projectNamespace String, gerrit namespace (optional)
- *          - commit String, Git commit to checkout
- *          - credentialsId String, gerrit credentials ID (optional)
  *          - host String, gerrit host
+ *          - projectNamespace String, gerrit namespace (optional)
+ *          - commit String, Git commit to checkout (optional)
+ *          - credentialsId String, gerrit credentials ID (optional)
+ *          - refspec String, remote refs to be retrieved (optional)
  *
  * Usage example:
  *
@@ -28,15 +29,16 @@
 
   def project_name = config.get('project_name')
   def projectNamespace = config.get('projectNamespace', 'projectcalico')
-  def commit = config.get('commit')
+  def commit = config.get('commit', '*')
   def host = config.get('host')
   def credentialsId = config.get('credentialsId', 'mcp-ci-gerrit')
+  def refspec = config.get('refspec')
 
   if (!project_name) {
     throw new RuntimeException("Parameter 'project_name' must be set for checkoutCalico() !")
   }
-  if (!commit) {
-    throw new RuntimeException("Parameter 'commit' must be set for checkoutCalico() !")
+  if (!host) {
+    throw new RuntimeException("Parameter 'host' must be set for checkoutCalico() !")
   }
 
   stage ("Checkout ${project_name}"){
@@ -46,6 +48,7 @@
       host : host,
       project : "${projectNamespace}/${project_name}",
       withWipeOut : true,
+      refspec : refspec,
     ])
   }
 }
@@ -462,6 +465,60 @@
 
 
 /**
+ * Run Calico system tests stage
+ *
+ * @param nodeImage String, docker image for calico/node container
+ * @param ctlImage String, docker image with calicoctl binary
+ * @param failOnErrors Boolean, raise exception if some tests fail (default true)
+ *
+ * Usage example:
+ *
+ * def calico = new com.mirantis.mcp.Calico()
+ * calico.systestCalico('calico/node:latest', 'calico/ctl:latest')
+ *
+ */
+def systestCalico(nodeImage, ctlImage, failOnErrors = true) {
+  stage ('Run Calico system tests'){
+    try {
+      // create fake targets to avoid execution of unneeded operations
+      sh """
+        mkdir -p vendor
+      """
+      // pull calico/ctl image and extract calicoctl binary from it
+      sh """
+        mkdir -p dist
+        docker run --rm -u \$(id -u):\$(id -g) --entrypoint /bin/cp -v \$(pwd)/dist:/dist ${ctlImage} /calicoctl /dist/calicoctl
+        touch dist/calicoctl dist/calicoctl-linux-amd64
+      """
+      // pull calico/node image and extract required binaries
+      sh """
+        mkdir -p calico_node/filesystem/bin
+        for calico_binary in startup allocate-ipip-addr calico-felix bird calico-bgp-daemon confd libnetwork-plugin; do
+          docker run --rm -u \$(id -u):\$(id -g) --entrypoint /bin/cp -v \$(pwd)/calico_node/filesystem/bin:/calicobin ${nodeImage} /bin/\${calico_binary} /calicobin/
+        done
+        cp calico_node/filesystem/bin/startup dist/
+        cp calico_node/filesystem/bin/allocate-ipip-addr dist/
+        touch calico_node/filesystem/bin/*
+        touch calico_node/.calico_node.created
+      """
+      sh "NODE_CONTAINER_NAME=${nodeImage} make st"
+    } catch (Exception e) {
+      sh "make stop-etcd"
+      // FIXME: cleaning has to be done by make stop/clean targets
+      sh """
+        for dc in calico-felix cali-st-ext-nginx cali-st-host cali-st-gw host1 host2 host3; do
+          docker rm -f "\${dc}" || :
+        done
+      """
+      if (failOnErrors) {
+        throw e
+      }
+    }
+  }
+}
+
+
+/**
  * Build Calico containers stages
  *
  * @param config LinkedHashMap
diff --git a/src/com/mirantis/mcp_qa/Common.groovy b/src/com/mirantis/mcp_qa/Common.groovy
index 7d9fd89..22dcf81 100644
--- a/src/com/mirantis/mcp_qa/Common.groovy
+++ b/src/com/mirantis/mcp_qa/Common.groovy
@@ -97,3 +97,87 @@
     }
     return jobSetParameters
 }
+
+/**
+ * Upload tests results to TestRail
+ *
+ * @param config LinkedHashMap
+ *        config includes next parameters:
+ *          - junitXml String, path to XML file with tests results
+ *          - testPlanName String, name of test plan in TestRail
+ *          - testSuiteName String, name of test suite in TestRail
+ *          - testrailMilestone String, milestone name in TestRail
+ *          - tesPlanDesc String, description of test plan in TestRail (optional)
+ *          - jobURL String, URL of job build with tests (optional)
+ *          - testrailURL String, TestRail URL (optional)
+ *          - testrailProject String, project name in TestRail (optional)
+ *
+ *
+ * Usage example:
+ *
+ * uploadResultsTestRail([
+ *   junitXml: './nosetests.xml',
+ *   testPlanName: 'MCP test plan #1',
+ *   testSuiteName: 'Calico component tests',
+ *   jobURL: 'jenkins.example.com/job/tests.mcp/1',
+ * ])
+ *
+ */
+def uploadResultsTestRail(config) {
+  def venvPath = 'testrail-venv'
+  // TODO: install 'testrail_reporter' pypi when new version with eee508d commit is released
+  def testrailReporterPackage = 'git+git://github.com/gdyuldin/testrail_reporter.git'
+  def testrailReporterVersion = 'eee508d'
+
+  def requiredArgs = ['junitXml', 'testPlanName', 'testSuiteName', 'testrailMilestone']
+  def missingArgs = []
+  for (i in requiredArgs) { if (!config.containsKey(i)) { missingArgs << i }}
+  if (missingArgs) { println "Required arguments are missing for '${funcName}': ${missingArgs.join(', ')}" }
+
+  def junitXml = config.get('junitXml')
+  def testPlanName = config.get('testPlanName')
+  def testSuiteName = config.get('testSuiteName')
+  def testrailMilestone = config.get('testrailMilestone')
+  def testrailURL = config.get('testrailURL', 'https://mirantis.testrail.com')
+  def testrailProject = config.get('testrailProject', 'Mirantis Cloud Platform')
+  def tesPlanDesc = config.get('tesPlanDesc')
+  def jobURL = config.get('jobURL')
+
+  def reporterOptions = [
+    "--verbose",
+    "--testrail-run-update",
+    "--testrail-url '${testrailURL}'",
+    "--testrail-user \"\${TESTRAIL_USER}\"",
+    "--testrail-password \"\${TESTRAIL_PASSWORD}\"",
+    "--testrail-project '${testrailProject}'",
+    "--testrail-plan-name '${testPlanName}'",
+    "--testrail-milestone '${testrailMilestone}'",
+    "--testrail-suite '${testSuiteName}'",
+    "--xunit-name-template '{methodname}'",
+    "--testrail-name-template '{custom_test_group}'",
+  ]
+
+  if (tesPlanDesc) { reporterOptions << "--env-description '${tesPlanDesc}'" }
+  if (jobURL) { reporterOptions << "--test-results-link '${jobURL}'" }
+
+  // Install testrail reporter
+  sh """
+    virtualenv ${venvPath}
+    . ${venvPath}/bin/activate
+    pip install --upgrade ${testrailReporterPackage}@${testrailReporterVersion}
+  """
+
+  def script = """
+    . ${venvPath}/bin/activate
+    report ${reporterOptions.join(' ')} ${junitXml}
+  """
+
+  withCredentials([
+             [$class          : 'UsernamePasswordMultiBinding',
+             credentialsId   : 'testrail',
+             passwordVariable: 'TESTRAIL_PASSWORD',
+             usernameVariable: 'TESTRAIL_USER']
+  ]) {
+    return sh(script: script, returnStdout: true).trim().split().last()
+  }
+}
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index 881f706..ffa72d3 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -17,17 +17,6 @@
 }
 
 /**
- * Parse HEAD of current directory and return commit hash
- */
-def getGitCommit() {
-    git_commit = sh (
-        script: 'git rev-parse HEAD',
-        returnStdout: true
-    ).trim()
-    return git_commit
-}
-
-/**
  * Return workspace.
  * Currently implemented by calling pwd so it won't return relevant result in
  * dir context
@@ -218,106 +207,6 @@
 
     throw new Exception("Could not find credentials for ID ${id}")
 }
-/**
- * Setup ssh agent and add private key
- *
- * @param credentialsId Jenkins credentials name to lookup private key
- */
-def prepareSshAgentKey(credentialsId) {
-    c = getSshCredentials(credentialsId)
-    sh("test -d ~/.ssh || mkdir -m 700 ~/.ssh")
-    sh('pgrep -l -u $USER -f | grep -e ssh-agent\$ >/dev/null || ssh-agent|grep -v "Agent pid" > ~/.ssh/ssh-agent.sh')
-    sh("set +x; echo '${c.getPrivateKey()}' > ~/.ssh/id_rsa_${credentialsId} && chmod 600 ~/.ssh/id_rsa_${credentialsId}; set -x")
-    agentSh("ssh-add ~/.ssh/id_rsa_${credentialsId}")
-}
-
-/**
- * Execute command with ssh-agent
- *
- * @param cmd   Command to execute
- */
-def agentSh(cmd) {
-    sh(". ~/.ssh/ssh-agent.sh && ${cmd}")
-}
-
-/**
- * Ensure entry in SSH known hosts
- *
- * @param url   url of remote host
- */
-def ensureKnownHosts(url) {
-    def hostArray = getKnownHost(url)
-    sh "test -f ~/.ssh/known_hosts && grep ${hostArray[0]} ~/.ssh/known_hosts || ssh-keyscan -p ${hostArray[1]} ${hostArray[0]} >> ~/.ssh/known_hosts"
-}
-
-@NonCPS
-def getKnownHost(url){
-     // test for git@github.com:organization/repository like URLs
-    def p = ~/.+@(.+\..+)\:{1}.*/
-    def result = p.matcher(url)
-    def host = ""
-    if (result.matches()) {
-        host = result.group(1)
-        port = 22
-    } else {
-        parsed = new URI(url)
-        host = parsed.host
-        port = parsed.port && parsed.port > 0 ? parsed.port: 22
-    }
-    return [host,port]
-}
-
-/**
- * Mirror git repository, merge target changes (downstream) on top of source
- * (upstream) and push target or both if pushSource is true
- *
- * @param sourceUrl      Source git repository
- * @param targetUrl      Target git repository
- * @param credentialsId  Credentials id to use for accessing source/target
- *                       repositories
- * @param branches       List or comma-separated string of branches to sync
- * @param followTags     Mirror tags
- * @param pushSource     Push back into source branch, resulting in 2-way sync
- * @param pushSourceTags Push target tags into source or skip pushing tags
- * @param gitEmail       Email for creation of merge commits
- * @param gitName        Name for creation of merge commits
- */
-def mirrorGit(sourceUrl, targetUrl, credentialsId, branches, followTags = false, pushSource = false, pushSourceTags = false, gitEmail = 'jenkins@localhost', gitName = 'Jenkins') {
-    if (branches instanceof String) {
-        branches = branches.tokenize(',')
-    }
-
-    prepareSshAgentKey(credentialsId)
-    ensureKnownHosts(targetUrl)
-    sh "git config user.email '${gitEmail}'"
-    sh "git config user.name '${gitName}'"
-
-    sh "git remote | grep target || git remote add target ${TARGET_URL}"
-    agentSh "git remote update --prune"
-
-    for (i=0; i < branches.size; i++) {
-        branch = branches[i]
-        sh "git branch | grep ${branch} || git checkout -b ${branch} origin/${branch}"
-        sh "git branch | grep ${branch} && git checkout ${branch} && git reset --hard origin/${branch}"
-
-        sh "git ls-tree target/${branch} && git merge --no-edit --ff target/${branch} || echo 'Target repository is empty, skipping merge'"
-        followTagsArg = followTags ? "--follow-tags" : ""
-        agentSh "git push ${followTagsArg} target HEAD:${branch}"
-
-        if (pushSource == true) {
-            followTagsArg = followTags && pushSourceTags ? "--follow-tags" : ""
-            agentSh "git push ${followTagsArg} origin HEAD:${branch}"
-        }
-    }
-
-    if (followTags == true) {
-        agentSh "git push target --tags"
-
-        if (pushSourceTags == true) {
-            agentSh "git push origin --tags"
-        }
-    }
-}
 
 /**
  * Tests Jenkins instance for existence of plugin with given name
@@ -415,4 +304,4 @@
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/com/mirantis/mk/Git.groovy b/src/com/mirantis/mk/Git.groovy
index 5e89c71..5292033 100644
--- a/src/com/mirantis/mk/Git.groovy
+++ b/src/com/mirantis/mk/Git.groovy
@@ -47,6 +47,20 @@
 }
 
 /**
+ * Get remote URL
+ *
+ * @param name  Name of remote (default any)
+ * @param type  Type (fetch or push, default fetch)
+ */
+def getGitRemote(name = '', type = 'fetch') {
+    gitRemote = sh (
+        script: "git remote -v | grep '${name}' | grep ${type} | awk '{print \$2}' | head -1",
+        returnStdout: true
+    ).trim()
+    return gitRemote
+}
+
+/**
  * Change actual working branch of repo
  *
  * @param path            Path to the git repository
@@ -125,27 +139,54 @@
 }
 
 /**
- * Mirror git repository
+ * Mirror git repository, merge target changes (downstream) on top of source
+ * (upstream) and push target or both if pushSource is true
+ *
+ * @param sourceUrl      Source git repository
+ * @param targetUrl      Target git repository
+ * @param credentialsId  Credentials id to use for accessing source/target
+ *                       repositories
+ * @param branches       List or comma-separated string of branches to sync
+ * @param followTags     Mirror tags
+ * @param pushSource     Push back into source branch, resulting in 2-way sync
+ * @param pushSourceTags Push target tags into source or skip pushing tags
+ * @param gitEmail       Email for creation of merge commits
+ * @param gitName        Name for creation of merge commits
  */
-def mirrorReporitory(sourceUrl, targetUrl, credentialsId, branches, followTags = false, gitEmail = 'jenkins@localhost', gitUsername = 'Jenkins') {
-    def ssl = new com.mirantis.mk.Ssl()
+def mirrorGit(sourceUrl, targetUrl, credentialsId, branches, followTags = false, pushSource = false, pushSourceTags = false, gitEmail = 'jenkins@localhost', gitName = 'Jenkins') {
     if (branches instanceof String) {
         branches = branches.tokenize(',')
     }
-    ssl.prepareSshAgentKey(credentialsId)
-    ssl.ensureKnownHosts(targetUrl)
 
-    sh "git remote | grep target || git remote add target ${targetUrl}"
-    agentSh "git remote update --prune"
+    def ssh = new com.mirantis.mk.Ssh()
+    ssh.prepareSshAgentKey(credentialsId)
+    ssh.ensureKnownHosts(targetUrl)
+    sh "git config user.email '${gitEmail}'"
+    sh "git config user.name '${gitName}'"
+
+    sh "git remote | grep target || git remote add target ${TARGET_URL}"
+    ssh.agentSh "git remote update --prune"
+
     for (i=0; i < branches.size; i++) {
         branch = branches[i]
         sh "git branch | grep ${branch} || git checkout -b ${branch} origin/${branch}"
         sh "git branch | grep ${branch} && git checkout ${branch} && git reset --hard origin/${branch}"
 
-        sh "git config --global user.email '${gitEmail}'"
-        sh "git config --global user.name '${gitUsername}'"
         sh "git ls-tree target/${branch} && git merge --no-edit --ff target/${branch} || echo 'Target repository is empty, skipping merge'"
         followTagsArg = followTags ? "--follow-tags" : ""
-        agentSh "git push ${followTagsArg} target HEAD:${branch}"
+        ssh.agentSh "git push ${followTagsArg} target HEAD:${branch}"
+
+        if (pushSource == true) {
+            followTagsArg = followTags && pushSourceTags ? "--follow-tags" : ""
+            ssh.agentSh "git push ${followTagsArg} origin HEAD:${branch}"
+        }
+    }
+
+    if (followTags == true) {
+        ssh.agentSh "git push target --tags"
+
+        if (pushSourceTags == true) {
+            ssh.agentSh "git push origin --tags"
+        }
     }
 }
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 069442e..66bf44e 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -28,42 +28,60 @@
     salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.sync_all')
 
     salt.runSaltProcessStep(master, 'I@salt:control', 'state.sls', ['salt.minion,linux.system,linux.network,ntp'])
-    salt.runSaltProcessStep(master, 'I@salt:control', 'state.sls', ['libvirt'])
-    salt.runSaltProcessStep(master, 'I@salt:control', 'state.sls', ['salt.control'])
+    salt.enforceState(master, 'I@salt:control', 'libvirt', true)
+    salt.enforceState(master, 'I@salt:control', 'salt.control', true)
+
+    sleep(300)
+
+    salt.runSaltProcessStep(master, '* and not kvm*', 'saltutil.refresh_pillar')
+    salt.runSaltProcessStep(master, '* and not kvm*', 'saltutil.sync_all')
+
+    // workaround - install apt-transport-https
+    salt.runSaltProcessStep(master, '* and not kvm*', 'pkg.install', ['apt-transport-https'])
+
+    salt.runSaltProcessStep(master, '* and not kvm*', 'state.sls', ['linux,openssh,salt.minion,ntp'])
 }
 
-def installOpenstackMkInfra(master) {
+def installOpenstackMkInfra(master, physical = "false") {
     def salt = new com.mirantis.mk.Salt()
     // Install keepaliveds
     //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
-    salt.runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
+    salt.enforceState(master, 'ctl01*', 'keepalived', true)
+    salt.enforceState(master, 'I@keepalived:cluster', 'keepalived', true)
     // Check the keepalived VIPs
     salt.runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
     // Install glusterfs
-    salt.runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
+    salt.enforceState(master, 'I@glusterfs:server', 'glusterfs.server.service', true)
+
     //runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.setup'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
-    salt.runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
-    salt.runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
+    if (physical.equals("false")) {
+        salt.enforceState(master, 'ctl01*', 'glusterfs.server.setup', true)
+        salt.enforceState(master, 'ctl02*', 'glusterfs.server.setup', true)
+        salt.enforceState(master, 'ctl03*', 'glusterfs.server.setup', true)
+    } else {
+        salt.enforceState(master, 'kvm01*', 'glusterfs.server.setup', true)
+        salt.enforceState(master, 'kvm02*', 'glusterfs.server.setup', true)
+        salt.enforceState(master, 'kvm03*', 'glusterfs.server.setup', true)
+    }
     salt.runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
     salt.runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
+
     // Install rabbitmq
-    salt.runSaltProcessStep(master, 'I@rabbitmq:server', 'state.sls', ['rabbitmq'])
+    salt.enforceState(master, 'I@rabbitmq:server', 'rabbitmq', true)
     // Check the rabbitmq status
     salt.runSaltProcessStep(master, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
     // Install galera
-    salt.runSaltProcessStep(master, 'I@galera:master', 'state.sls', ['galera'])
-    salt.runSaltProcessStep(master, 'I@galera:slave', 'state.sls', ['galera'])
+    salt.enforceState(master, 'I@galera:master', 'galera', true)
+    salt.enforceState(master, 'I@galera:slave', 'galera', true)
     // Check galera status
     salt.runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
     salt.runSaltProcessStep(master, 'I@galera:slave', 'mysql.status')
     // Install haproxy
-    salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
+    salt.enforceState(master, 'I@haproxy:proxy', 'haproxy', true)
     salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
     salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
     // Install memcached
-    salt.runSaltProcessStep(master, 'I@memcached:server', 'state.sls', ['memcached'])
+    salt.enforceState(master, 'I@memcached:server', 'memcached', true)
 }
 
 
@@ -71,60 +89,67 @@
     def salt = new com.mirantis.mk.Salt()
     // setup keystone service
     //runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keystone.server'])
-    salt.runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
+    salt.enforceState(master, 'ctl01*', 'keystone.server', true)
+    salt.enforceState(master, 'I@keystone:server', 'keystone.server', true)
     // populate keystone services/tenants/roles/users
-    salt.runSaltProcessStep(master, 'I@keystone:client', 'state.sls', ['keystone.client'])
+
+    // keystone:client must be called locally
+    salt.runSaltProcessStep(master, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'])
+
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; keystone service-list'])
     // Install glance and ensure glusterfs clusters
     //runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glance.server'])
-    salt.runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'])
-    salt.runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glusterfs.client'])
+    salt.enforceState(master, 'ctl01*', 'glance.server', true)
+    salt.enforceState(master, 'I@glance:server', 'glance.server', true)
+    salt.enforceState(master, 'I@glance:server', 'glusterfs.client', true)
     // Update fernet tokens before doing request on keystone server
-    salt.runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
+    salt.enforceState(master, 'I@keystone:server', 'keystone.server', true)
     // Check glance service
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'])
     // Install and check nova service
     //runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['nova'])
-    salt.runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'])
+    salt.enforceState(master, 'ctl01*', 'nova', true)
+    salt.enforceState(master, 'I@nova:controller', 'nova', true)
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'])
     // Install and check cinder service
     //runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['cinder'])
-    salt.runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'])
+    salt.enforceState(master, 'ctl01*', 'cinder', true)
+    salt.enforceState(master, 'I@cinder:controller', 'cinder', true)
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'])
     // Install neutron service
     //runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['neutron'])
-    salt.runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'])
+    salt.enforceState(master, 'ctl01*', 'neutron', true)
+    salt.enforceState(master, 'I@neutron:server', 'neutron', true)
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'])
     // Install heat service
     //runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'], 1)
-    salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['heat'])
-    salt.runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'])
+    salt.enforceState(master, 'ctl01*', 'heat', true)
+    salt.enforceState(master, 'I@heat:server', 'heat', true)
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'])
     // Install horizon dashboard
-    salt.runSaltProcessStep(master, 'I@horizon:server', 'state.sls', ['horizon'])
-    salt.runSaltProcessStep(master, 'I@nginx:server', 'state.sls', ['nginx'])
+    salt.enforceState(master, 'I@horizon:server', 'horizon', true)
+    salt.enforceState(master, 'I@nginx:server', 'nginx', true)
 }
 
 
-def installOpenstackMkNetwork(master) {
+def installOpenstackMkNetwork(master, physical = "false") {
     def salt = new com.mirantis.mk.Salt()
     // Install opencontrail database services
     //runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
-    salt.runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail.database'])
-    salt.runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'])
+    salt.enforceState(master, 'ntw01*', 'opencontrail.database', true)
+    salt.enforceState(master, 'I@opencontrail:database', 'opencontrail.database', true)
     // Install opencontrail control services
     //runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
-    salt.runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail'])
-    salt.runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'])
+    salt.enforceState(master, 'ntw01*', 'opencontrail', true)
+    salt.enforceState(master, 'I@opencontrail:control', 'opencontrail', true)
+
     // Provision opencontrail control services
-    salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl01 --host_ip 172.16.10.101 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
-    salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl02 --host_ip 172.16.10.102 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
-    salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl03 --host_ip 172.16.10.103 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+    if (physical.equals("false")) {
+        salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl01 --host_ip 172.16.10.101 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+        salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl02 --host_ip 172.16.10.102 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+        salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl03 --host_ip 172.16.10.103 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+    }
+
     // Test opencontrail
     salt.runSaltProcessStep(master, 'I@opencontrail:control', 'cmd.run', ['contrail-status'])
     salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
@@ -132,13 +157,17 @@
 }
 
 
-def installOpenstackMkCompute(master) {
+def installOpenstackMkCompute(master, physical = "false") {
      def salt = new com.mirantis.mk.Salt()
     // Configure compute nodes
     salt.runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
     salt.runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
+
     // Provision opencontrail virtual routers
-    salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_vrouter.py --host_name cmp01 --host_ip 172.16.10.105 --api_server_ip 172.16.10.254 --oper add --admin_user admin --admin_password workshop --admin_tenant_name admin'])
+    if (physical.equals("false")) {
+        salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_vrouter.py --host_name cmp01 --host_ip 172.16.10.105 --api_server_ip 172.16.10.254 --oper add --admin_user admin --admin_password workshop --admin_tenant_name admin'])
+    }
+
     salt.runSaltProcessStep(master, 'I@nova:compute', 'system.reboot')
 }
 
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 10930a5..710d4a6 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -12,7 +12,7 @@
  * @param credentialsID       ID of credentials store entry
  */
 def connection(url, credentialsId = "salt") {
-    def common = new com.mirantis.mk.Common();
+    def common = new com.mirantis.mk.Common()
     params = [
         "url": url,
         "credentialsId": credentialsId,
@@ -48,7 +48,7 @@
  * @param target   Target specification, eg. for compound matches by Pillar
  *                 data: ['expression': 'I@openssh:server', 'type': 'compound'])
  * @param function Function to execute (eg. "state.sls")
- * @param batch 
+ * @param batch
  * @param args     Additional arguments to function
  * @param kwargs   Additional key-value arguments to function
  */
@@ -63,8 +63,8 @@
         'expr_form': target.type,
     ]
 
-    if (batch) {
-        data['batch'] = batch
+    if (batch == true) {
+        data['batch'] = "local_batch"
     }
 
     if (args) {
@@ -88,14 +88,19 @@
 }
 
 def enforceState(master, target, state, output = false) {
+    def common = new com.mirantis.mk.Common()
     def run_states
+
     if (state instanceof String) {
         run_states = state
     } else {
         run_states = state.join(',')
     }
 
+    common.infoMsg("Enforcing state ${run_states} on ${target}")
+
     def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', null, [run_states])
+
     try {
         checkResult(out)
     } finally {
@@ -107,6 +112,10 @@
 }
 
 def cmdRun(master, target, cmd) {
+    def common = new com.mirantis.mk.Common()
+
+    common.infoMsg("Running command ${cmd} on ${target}")
+
     def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.run', null, [cmd])
     return out
 }
@@ -143,14 +152,21 @@
     return runSaltCommand(master, 'runner', target, 'state.orchestrate', [orchestrate])
 }
 
-def runSaltProcessStep(master, tgt, fun, arg = [], batch = null) {
-    if (batch) {
-        result = runSaltCommand(master, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg)
+def runSaltProcessStep(master, tgt, fun, arg = [], batch = null, output = false) {
+    def common = new com.mirantis.mk.Common()
+    def out
+
+    common.infoMsg("Running step ${fun} on ${tgt}")
+
+    if (batch == true) {
+        out = runSaltCommand(master, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg)
+    } else {
+        out = runSaltCommand(master, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg)
     }
-    else {
-        result = runSaltCommand(master, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg)
+
+    if (output == true) {
+            printSaltCommandResult(out)
     }
-    echo("${result}")
 }
 
 /**
@@ -165,7 +181,7 @@
         }
         for (node in entry) {
             for (resource in node.value) {
-                if (resource instanceof String || resource.value.result.toString().toBoolean() != true) {
+                if (resource instanceof String || (resource instanceof Boolean && resource == false) || (resource instanceof HashMap && resource.value.result.toString().toBoolean() != true)) {
                     throw new Exception("Salt state on node ${node.key} failed: ${node.value}")
                 }
             }
@@ -198,7 +214,7 @@
     for (node in out) {
         if (node.value) {
             println "Node ${node.key} changes:"
-            print new groovy.json.JsonBuilder(node.value).toPrettyString()
+            print new groovy.json.JsonBuilder(node.value).toPrettyString().replace('\\n', System.getProperty('line.separator'))
         } else {
             println "No changes for node ${node.key}"
         }
diff --git a/src/com/mirantis/mk/Ssl.groovy b/src/com/mirantis/mk/Ssh.groovy
similarity index 66%
rename from src/com/mirantis/mk/Ssl.groovy
rename to src/com/mirantis/mk/Ssh.groovy
index c9bec04..4526d6d 100644
--- a/src/com/mirantis/mk/Ssl.groovy
+++ b/src/com/mirantis/mk/Ssh.groovy
@@ -2,7 +2,7 @@
 
 /**
  *
- * SSL functions
+ * SSH functions
  *
  */
 
@@ -12,10 +12,25 @@
  * @param url   url of remote host
  */
 def ensureKnownHosts(url) {
-    uri = new URI(url)
-    port = uri.port ?: 22
+    def hostArray = getKnownHost(url)
+    sh "test -f ~/.ssh/known_hosts && grep ${hostArray[0]} ~/.ssh/known_hosts || ssh-keyscan -p ${hostArray[1]} ${hostArray[0]} >> ~/.ssh/known_hosts"
+}
 
-    sh "test -f ~/.ssh/known_hosts && grep ${uri.host} ~/.ssh/known_hosts || ssh-keyscan -p ${port} ${uri.host} >> ~/.ssh/known_hosts"
+@NonCPS
+def getKnownHost(url){
+     // test for git@github.com:organization/repository like URLs
+    def p = ~/.+@(.+\..+)\:{1}.*/
+    def result = p.matcher(url)
+    def host = ""
+    if (result.matches()) {
+        host = result.group(1)
+        port = 22
+    } else {
+        parsed = new URI(url)
+        host = parsed.host
+        port = parsed.port && parsed.port > 0 ? parsed.port: 22
+    }
+    return [host,port]
 }
 
 /**
@@ -41,6 +56,15 @@
 }
 
 /**
+ * Execute command with ssh-agent (shortcut for runSshAgentCommand)
+ *
+ * @param cmd   Command to execute
+ */
+def agentSh(cmd) {
+    runSshAgentCommand(cmd)
+}
+
+/**
  * Setup ssh agent and add private key
  *
  * @param credentialsId Jenkins credentials name to lookup private key
diff --git a/src/com/mirantis/mk/Test.groovy b/src/com/mirantis/mk/Test.groovy
index c212c61..702fb76 100644
--- a/src/com/mirantis/mk/Test.groovy
+++ b/src/com/mirantis/mk/Test.groovy
@@ -14,5 +14,45 @@
  */
 def runConformanceTests(master, k8s_api, image) {
     def salt = new com.mirantis.mk.Salt()
-    salt = runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["docker run --rm --net=host -e API_SERVER=${k8s_api} ${image} >> e2e-conformance.log"])
-}
\ No newline at end of file
+    salt.runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["docker run --rm --net=host -e API_SERVER=${k8s_api} ${image} >> ${image}.output"])
+}
+
+/**
+ * Copy test output to cfg node
+ *
+ * @param image      Docker image with tests
+ */
+def copyTestsOutput(master, image) {
+    def salt = new com.mirantis.mk.Salt()
+    salt.runSaltProcessStep(master, 'cfg01*', 'cmd.run', ["scp ctl01:/root/${image}.output /home/ubuntu/"])
+}
+
+/**
+ * Execute tempest tests
+ *
+ * @param tempestLink   Docker image link with rally and tempest
+ */
+def runTempestTests(master, tempestLink) {
+    def salt = new com.mirantis.mk.Salt()
+    salt.runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["docker run --rm --net=host " +
+                                                          "-e TEMPEST_CONF=mcp.conf " +
+                                                          "-e SKIP_LIST=mcp_skip.list " +
+                                                          "-e SOURCE_FILE=keystonercv3 " +
+                                                          "-v /root/:/home/rally ${tempestLink} >> docker-tempest.log"])
+}
+
+/**
+ * Upload results to worker
+ *
+ */
+def copyTempestResults(master) {
+    def salt = new com.mirantis.mk.Salt()
+    salt.runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["scp /root/docker-tempest.log cfg01:/home/ubuntu/ && " +
+                                                          "find /root -name result.xml -exec scp {} cfg01:/home/ubuntu \\;"])
+}
+
+
+/**
+ * Upload results to testrail
+ *
+ */
\ No newline at end of file