Merge "Check that docker container exists/running before deleting/killing it"
diff --git a/src/com/mirantis/mk/Ceph.groovy b/src/com/mirantis/mk/Ceph.groovy
new file mode 100644
index 0000000..bb837b2
--- /dev/null
+++ b/src/com/mirantis/mk/Ceph.groovy
@@ -0,0 +1,92 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Ceph functions
+ *
+ */
+
+/**
+ * Ceph health check
+ *
+ */
+def waitForHealthy(master, target, flags=[], count=0, attempts=300) {
+    def common = new com.mirantis.mk.Common()
+    def salt = new com.mirantis.mk.Salt()
+    // wait for healthy cluster
+    while (count < attempts) {
+        def health = salt.cmdRun(master, target, 'ceph health')['return'][0].values()[0]
+        if (health.contains('HEALTH_OK')) {
+            common.infoMsg('Cluster is healthy')
+            break
+        } else {
+            for (flag in flags) {
+                if (health.contains(flag + ' flag(s) set') && !(health.contains('down'))) {
+                    common.infoMsg('Cluster is healthy')
+                    return
+                }
+            }
+        }
+        common.infoMsg("Ceph health status: ${health}")
+        count++
+        sleep(10)
+    }
+}
+
+/**
+ * Ceph remove partition
+ *
+ */
+def removePartition(master, target, partition_uuid, type='', id=-1) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def partition = ""
+    if (type == 'lockbox') {
+        try {
+            // umount - partition = /dev/sdi2
+            partition = salt.cmdRun(master, target, "lsblk -rp | grep -v mapper | grep ${partition_uuid} ")['return'][0].values()[0].split()[0]
+            salt.cmdRun(master, target, "umount ${partition}")
+        } catch (Exception e) {
+            common.warningMsg(e)
+        }
+    } else if (type == 'data') {
+        try {
+            // umount - partition = /dev/sdi2
+            partition = salt.cmdRun(master, target, "df | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0].split()[0]
+            salt.cmdRun(master, target, "umount ${partition}")
+        } catch (Exception e) {
+            common.warningMsg(e)
+        }
+        try {
+            // partition = /dev/sdi2
+            partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
+        } catch (Exception e) {
+            common.warningMsg(e)
+        }
+    } else {
+        try {
+            // partition = /dev/sdi2
+            partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
+        } catch (Exception e) {
+            common.warningMsg(e)
+        }
+    }
+    if (partition?.trim()) {
+        if (partition.contains("nvme")) {
+            // partition = /dev/nvme1n1p2
+            // dev = /dev/nvme1n1
+            def dev = partition.replaceAll('p\\d+$', "")
+            // part_id = 2
+            def part_id = partition.substring(partition.lastIndexOf("p") + 1).replaceAll("[^0-9]+", "")
+
+        } else {
+            // partition = /dev/sdi2
+            // dev = /dev/sdi
+            def dev = partition.replaceAll('\\d+$', "")
+            // part_id = 2
+            def part_id = partition.substring(partition.lastIndexOf("/") + 1).replaceAll("[^0-9]+", "")
+        }
+        salt.cmdRun(master, target, "Ignore | parted ${dev} rm ${part_id}")
+    }
+    return
+}
diff --git a/src/com/mirantis/mk/ReleaseWorkflow.groovy b/src/com/mirantis/mk/ReleaseWorkflow.groovy
index adb49a6..c27fa8a 100644
--- a/src/com/mirantis/mk/ReleaseWorkflow.groovy
+++ b/src/com/mirantis/mk/ReleaseWorkflow.groovy
@@ -69,10 +69,30 @@
 }
 
 /**
- * Update release metadata value and upload CR to release metadata repository
+ * Get release metadata value for given key
  *
  * @param key metadata key
- * @param value metadata value
+ * @param metadataDir metadata directory
+ * @param dirdepth the level at which YAML file should be created
+ */
+
+def precreateKeyReleaseMetadataFile(String key, String metadataDir, Integer dirdepth = 0) {
+    def keySize = key.split(':').size() - 1
+    if (dirdepth > 0 && dirdepth - 1 <= keySize) {
+        def dirPath = metadataDir + '/' + key.split(':')[0..dirdepth - 1].join('/')
+        sh "if ! test -d \"${dirPath}\" ; then mkdir -p \"${dirPath}\"; fi"
+        if (dirdepth - 1 != keySize) {
+            def pathToDummyFile = dirPath + '/' + key.split(':')[dirdepth] + '.yml'
+            sh "if ! test -f \"${pathToDummyFile}\" ; then touch \"${pathToDummyFile}\"; fi"
+        }
+    }
+}
+
+/**
+ * Update release metadata value and upload CR to release metadata repository
+ *
+ * @param key metadata key (Several keys could be passed joined by ';' character)
+ * @param value metadata value (Several values could be passed joined by ';' character)
  * @param params map with expected parameters:
  *    - metadataCredentialsId
  *    - metadataGitRepoUrl
@@ -138,18 +158,17 @@
             git.createGitBranch(repoDir, crTopic)
         }
 
-        def keySize = key.split(':').size() - 1
-        if (dirdepth > 0 && dirdepth - 1 <= keySize) {
-            def dirPath = metadataDir + '/' + key.split(':')[0..dirdepth - 1].join('/')
-            sh "if ! test -d \"${dirPath}\" ; then mkdir -p \"${dirPath}\"; fi"
-            if (dirdepth - 1 != keySize) {
-                def pathToDummyFile = dirPath + '/' + key.split(':')[dirdepth] + '.yml'
-                sh "if ! test -f \"${pathToDummyFile}\" ; then touch \"${pathToDummyFile}\"; fi"
+        def keyArr = key.split(';')
+        def valueArr = value.split(';')
+        if (keyArr.size() == valueArr.size()) {
+            for (i in 0..keyArr.size()-1) {
+                precreateKeyReleaseMetadataFile(keyArr[i], metadataDir, dirdepth)
+
+                cmdText = "python '${repoDir}/utils/app.py' --path '${metadataDir}' update --key '${keyArr[i]}' --value '${valueArr[i]}'"
+                python.runVirtualenvCommand(venvDir, cmdText)
             }
         }
 
-        cmdText = "python '${repoDir}/utils/app.py' --path '${metadataDir}' update --key '${key}' --value '${value}'"
-        python.runVirtualenvCommand(venvDir, cmdText)
         commitMessage =
                 """${comment}
 
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index dc639ec..c7afbb0 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -103,10 +103,11 @@
         def defaultRepos = readYaml text: defaultExtraReposYaml
         // Don't check for magic, if set explicitly
         if (updateSaltFormulas) {
+            def updateSaltFormulasRev = config.get('updateSaltFormulasRev', distribRevision)
             if (!oldRelease && distribRevision != releaseVersionQ4) {
                 defaultRepos['repo']['mcp_saltformulas_update'] = [
-                    'source'  : "deb [arch=amd64]  http://mirror.mirantis.com/update/${distribRevision}/salt-formulas/xenial xenial main",
-                    'repo_key': "http://mirror.mirantis.com/update/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
+                    'source'  : "deb [arch=amd64]  http://mirror.mirantis.com/update/${updateSaltFormulasRev}/salt-formulas/xenial xenial main",
+                    'repo_key': "http://mirror.mirantis.com/update/${updateSaltFormulasRev}/salt-formulas/xenial/archive-salt-formulas.key"
                 ]
             }
         }