support for live VM snapshots / new salt / openstack functions
PROD-17559
Change-Id: I828db6e310946bf9355799987264376637f7ccbc
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index ddd9bca..1105dc5 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -371,3 +371,80 @@
echo("[Stack ${name}] Servers: ${servers}")
return servers
}
+
+/**
+ * Stops all services that contain specific string (for example nova,heat, etc.)
+ * @param env Salt Connection object or pepperEnv
+ * @param probe single node on which to list service names
+ * @param target all targeted nodes
+ * @param services lists of type of services to be stopped
+ * @return output of salt commands
+ */
+def stopServices(env, probe, target, services=[]) {
+ def salt = new com.mirantis.mk.Salt()
+ for (s in services) {
+ def outputServicesStr = salt.getReturnValues(salt.cmdRun(env, "${probe}*", "service --status-all | grep ${s} | awk \'{print \$4}\'"))
+ def servicesList = outputServicesStr.tokenize("\n")
+ for (name in servicesList) {
+ if (!name.contains('Salt command')) {
+ runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+ }
+ }
+ }
+}
+
+/**
+ * Restores Galera database
+ * @param env Salt Connection object or pepperEnv
+ * @return output of salt commands
+ */
+def restoreGaleraDb(env) {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ try {
+ salt.runSaltProcessStep(env, 'I@galera:slave', 'service.stop', ['mysql'])
+ } catch (Exception er) {
+ common.warningMsg('Mysql service already stopped')
+ }
+ try {
+ salt.runSaltProcessStep(env, 'I@galera:master', 'service.stop', ['mysql'])
+ } catch (Exception er) {
+ common.warningMsg('Mysql service already stopped')
+ }
+ try {
+ salt.cmdRun(env, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
+ } catch (Exception er) {
+ common.warningMsg('Files are not present')
+ }
+ try {
+ salt.cmdRun(env, 'I@galera:master', "mkdir /root/mysql/mysql.bak")
+ } catch (Exception er) {
+ common.warningMsg('Directory already exists')
+ }
+ try {
+ salt.cmdRun(env, 'I@galera:master', "rm -rf /root/mysql/mysql.bak/*")
+ } catch (Exception er) {
+ common.warningMsg('Directory already empty')
+ }
+ try {
+ salt.cmdRun(env, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
+ } catch (Exception er) {
+ common.warningMsg('Files were already moved')
+ }
+ try {
+ salt.runSaltProcessStep(env, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"])
+ } catch (Exception er) {
+ common.warningMsg('File is not present')
+ }
+ salt.cmdRun(env, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
+ def backup_dir = salt.getReturnValues(salt.getPillar(env, "I@galera:master", 'xtrabackup:client:backup_dir'))
+ if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
+ salt.runSaltProcessStep(env, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"])
+ salt.cmdRun(env, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
+ salt.runSaltProcessStep(env, 'I@galera:master', 'service.start', ['mysql'])
+
+ // wait until mysql service on galera master is up
+ salt.commandStatus(env, 'I@galera:master', 'service mysql status', 'running')
+
+ salt.runSaltProcessStep(env, 'I@galera:slave', 'service.start', ['mysql'])
+}
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 7ace470..883cb61 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -294,6 +294,27 @@
}
/**
+ * Checks if salt minion is in a list of salt master's accepted keys
+ * @usage minionPresent(saltId, 'I@salt:master', 'I@salt:minion', true, null, true, 200, 3)
+ * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+ * @param target Performs tests on this target node
+ * @param target_minions all targeted minions to test (for ex. I@salt:minion)
+ * @param waitUntilPresent return after the minion becomes present (default true)
+ * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
+ * @param output print salt command (default true)
+ * @param maxRetries finite number of iterations to check status of a command (default 200)
+ * @param answers how many minions should return (optional, default 1)
+ * @return output of salt command
+ */
+def minionsPresent(saltId, target = 'I@salt:master', target_minions = '', waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
+ def target_hosts = getMinionsSorted(pepperEnv, target_minions)
+ for (t in target_hosts) {
+ def tgt = salt.stripDomainName(t)
+ salt.minionPresent(pepperEnv, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
+ }
+}
+
+/**
* You can call this function when salt-master already contains salt keys of the target_nodes
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target Should always be salt-master
@@ -473,6 +494,90 @@
return new ArrayList<String>(minionsRaw['return'][0].keySet())
}
+/**
+ * Get sorted running minions IDs according to the target
+ * @param saltId Salt Connection object or pepperEnv
+ * @param target Get minions target
+ * @return list of sorted active minions fitin
+ */
+def getMinionsSorted(saltId, target) {
+ return getMinions(saltId, target).sort()
+}
+
+/**
+ * Get first out of running minions IDs according to the target
+ * @param saltId Salt Connection object or pepperEnv
+ * @param target Get minions target
+ * @return first of active minions fitin
+ */
+def getFirstMinion(saltId, target) {
+ def minionsSorted = getMinionsSorted(saltId, target)
+ return minionsSorted[0].split("\\.")[0]
+}
+
+/**
+ * Get running salt minions IDs without it's domain name part and its numbering identifications
+ * @param saltId Salt Connection object or pepperEnv
+ * @param target Get minions target
+ * @return list of active minions fitin without it's domain name part name numbering
+ */
+def getMinionsGeneralName(saltId, target) {
+ def minionsSorted = getMinionsSorted(saltId, target)
+ return stripDomainName(minionsSorted[0]).replaceAll('\\d+$', "")
+}
+
+/**
+ * Get domain name of the env
+ * @param saltId Salt Connection object or pepperEnv
+ * @return domain name
+ */
+def getDomainName(saltId) {
+ return getReturnValues(getPillar(saltId, 'I@salt:master', '_param:cluster_domain'))
+}
+
+/**
+ * Remove domain name from Salt minion ID
+ * @param name String of Salt minion ID
+ * @return Salt minion ID without its domain name
+ */
+def stripDomainName(name) {
+ return name.split("\\.")[0]
+}
+
+/**
+ * Gets return values of a salt command
+ * @param output String of Salt minion ID
+ * @return Return values of a salt command
+ */
+def getReturnValues(output) {
+ if(output.containsKey("return") && !output.get("return").isEmpty()) {
+ return output['return'][0].values()[0]
+ }
+ def common = new com.mirantis.mk.Common()
+ common.errorMsg('output does not contain return key')
+ return ''
+}
+
+/**
+ * Get minion ID of one of KVM nodes
+ * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+ * @return Salt minion ID of one of KVM nodes in env
+ */
+def getKvmMinionId(saltId) {
+ return getReturnValues(getGrain(saltId, 'I@salt:control', 'id')).values()[0]
+}
+
+/**
+ * Get Salt minion ID of KVM node hosting 'name' VM
+ * @param saltId Salt Connection object or pepperEnv
+ * @param name Name of the VM (for ex. ctl01)
+ * @return Salt minion ID of KVM node hosting 'name' VM
+ */
+def getNodeProvider(saltId, name) {
+ def kvm = getKvmMinionId(saltId)
+ return getReturnValues(getPillar(saltId, "${kvm}", "salt:control:cluster:internal:node:${name}:provider"))
+}
+
/**
* Test if there are any minions to target
diff --git a/src/com/mirantis/mk/Virsh.groovy b/src/com/mirantis/mk/Virsh.groovy
new file mode 100644
index 0000000..c0314cd
--- /dev/null
+++ b/src/com/mirantis/mk/Virsh.groovy
@@ -0,0 +1,147 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Virsh functions
+ *
+ */
+
+/**
+ * Ensures that the live snapshot exists
+ *
+ * @param nodeProvider KVM node that hosts the VM
+ * @param target Unique identification of the VM being snapshoted without domain (for ex. ctl01)
+ * @param snapshotName Snapshot name
+ * @param path Path where snapshot image and dumpxml are being put
+ * @param diskName Disk name of the snapshot
+ */
+def liveSnapshotPresent(master, nodeProvider, target, snapshotName, path='/var/lib/libvirt/images', diskName='vda') {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def snapshotPresent = ""
+ def domain = salt.getDomainName(master)
+ try {
+ snapshotPresent = salt.getReturnValues(salt.cmdRun(master, "${nodeProvider}*", "virsh snapshot-list ${target}.${domain} | grep ${snapshotName}")).split("\n")[0]
+ } catch (Exception er) {
+ common.infoMsg('snapshot not present')
+ }
+ if (!snapshotPresent.contains(snapshotName)) {
+ def dumpxmlPresent = ''
+ try {
+ dumpxmlPresent = salt.getReturnValues(salt.cmdRun(master, "${nodeProvider}*", "ls -la ${path}/${target}.${domain}.xml")).split("\n")[0]
+ } catch (Exception er) {
+ common.infoMsg('dumpxml file not present')
+ }
+ if (!dumpxmlPresent?.trim()) {
+ salt.cmdRun(master, "${nodeProvider}*", "virsh dumpxml ${target}.${domain} > ${path}/${target}.${domain}.xml")
+ }
+ salt.cmdRun(master, "${nodeProvider}*", "virsh snapshot-create-as --domain ${target}.${domain} ${snapshotName} --diskspec ${diskName},file=${path}/${target}.${domain}.${snapshotName}.qcow2 --disk-only --atomic")
+ }
+}
+
+/**
+ * Ensures that the live snapshot does not exist
+ *
+ * @param nodeProvider KVM node that hosts the VM
+ * @param target Unique identification of the VM being snapshoted without domain (for ex. ctl01)
+ * @param snapshotName Snapshot name
+ * @param path Path where snapshot image and dumpxml are being put
+ */
+def liveSnapshotAbsent(master, nodeProvider, target, snapshotName, path='/var/lib/libvirt/images') {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def domain = salt.getDomainName(master)
+ try {
+ salt.cmdRun(master, "${nodeProvider}*", "virsh snapshot-delete ${target}.${domain} --metadata ${snapshotName}")
+ } catch (Exception e) {
+ common.warningMsg('Snapshot ${snapshotName} for ${target}.${domain} does not exist or failed to be removed')
+ }
+ try {
+ salt.runSaltProcessStep(master, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.${snapshotName}.qcow2"], null, true)
+ } catch (Exception e) {
+ common.warningMsg('Snapshot ${snapshotName} qcow2 file for ${target}.${domain} does not exist or failed to be removed')
+ }
+ try {
+ salt.runSaltProcessStep(master, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.xml"], null, true)
+ } catch (Exception e) {
+ common.warningMsg('Dumpxml file for ${target}.${domain} does not exist or failed to be removed')
+ }
+}
+
+/**
+ * Rollback
+ *
+ * @param nodeProvider KVM node that hosts the VM
+ * @param target Unique identification of the VM being snapshoted without domain (for ex. ctl01)
+ * @param snapshotName Snapshot name
+ * @param path Path where snapshot image and dumpxml are being put
+ */
+def liveSnapshotRollback(master, nodeProvider, target, snapshotName, path='/var/lib/libvirt/images') {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def domain = salt.getDomainName(master)
+ salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
+ salt.cmdRun(pepperEnv, "${nodeProvider}*", "virsh define ${path}/${target}.${domain}.xml")
+ liveSnapshotAbsent(master, nodeProvider, target, snapshotName, path)
+ salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.start', ["${target}.${domain}"], null, true)
+}
+
+/**
+ * Merge snapshot while instance is running
+ *
+ * @param nodeProvider KVM node that hosts the VM
+ * @param target Unique identification of the VM being snapshoted without domain (for ex. ctl01)
+ * @param snapshotName Snapshot name
+ * @param path Path where snapshot image and dumpxml are being put
+ * @param diskName Disk name of the snapshot
+ */
+def liveSnapshotMerge(master, nodeProvider, target, snapshotName, path='/var/lib/libvirt/images', diskName='vda') {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def domain = salt.getDomainName(master)
+ try {
+ salt.cmdRun(pepperEnv, "${nodeProvider}*", "virsh blockcommit ${target}.${domain} ${diskName} --active --verbose --pivot")
+ try {
+ salt.cmdRun(pepperEnv, "${nodeProvider}*", "virsh snapshot-delete ${target}.${domain} --metadata ${snapshotName}")
+ } catch (Exception e) {
+ common.warningMsg('Snapshot ${snapshotName} for ${target}.${domain} does not exist or failed to be removed')
+ }
+ try {
+ salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.${snapshotName}.qcow2"], null, true)
+ } catch (Exception e) {
+ common.warningMsg('Snapshot ${snapshotName} qcow2 file for ${target}.${domain} does not exist or failed to be removed')
+ }
+ try {
+ salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.xml"], null, true)
+ } catch (Exception e) {
+ common.warningMsg('Dumpxml file for ${target}.${domain} does not exist or failed to be removed')
+ }
+ } catch (Exception e) {
+ common.errorMsg("The live snapshoted VM ${target}.${domain} failed to be merged, trying to fix it")
+ checkLiveSnapshotMerge(master, nodeProvider, target, snapshotName, path, diskName)
+ }
+}
+
+
+/**
+ * Check live snapshot merge failure due to known qemu issue not receiving message about merge completion
+ *
+ * @param nodeProvider KVM node that hosts the VM
+ * @param target Unique identification of the VM being snapshoted without domain (for ex. ctl01)
+ * @param snapshotName Snapshot name
+ * @param path Path where snapshot image and dumpxml are being put
+ * @param diskName Disk name of the snapshot
+ */
+def checkLiveSnapshotMerge(master, nodeProvider, target, snapshotName, path='/var/lib/libvirt/images', diskName='vda') {
+ def salt = new com.mirantis.mk.Salt()
+ def domain = salt.getDomainName(master)
+ def out = salt.getReturnValues(salt.cmdRun(pepperEnv, "${nodeProvider}*", "virsh blockjob ${target}.${domain} ${diskName} --info"))
+ if (out.contains('Block Commit')) {
+ def blockJobs = salt.getReturnValues(salt.cmdRun(pepperEnv, "{nodeProvider}*", "virsh qemu-monitor-command ${target}.${domain} --pretty -- '{ \"execute\": \"query-block-jobs\" }'"))
+ if (blockJobs.contains('offset')) {
+ // if Block Commit hangs on 100 and check offset - len = 0, then it is safe to merge the image
+ input message: "Please check if offset - len = 0, If so run: virsh qemu-monitor-command ${target}.${domain} --pretty -- '{ \"execute\": \"block-job-complete\", \"arguments\": { \"device\": \"drive-virtio-disk0\" } }', then virsh define ${path}/${target}.${domain}.xml, then virsh snapshot-delete ${target}.${domain} --metadata ${snapshotName} and remove ${path}/${target}.${domain}.${snapshotName}.qcow2 file. When you resolve this issue click on PROCEED."
+ }
+ }
+}
+