Merge the tip of origin/release/proposed/2019.2.0 into origin/release/2019.2.0
e9a9d25 Fix mysql backup directory creation path for affected nodes
3a9bf69 Add 'mine.update' before runing Horizon state
a9132d0 Check that docker container exists/running before deleting/killing it
cd63dbe Proper fix for Ceph osd remove pipeline
d25fb78 Hide password from cvp-finc tests output
43755db Move common Ceph function to Ceph class
ca25083 Pin pip to latest version with Python2 support for Python 2.7
c796f4f [CVP] Added option to pytest to show skip reasons
Change-Id: I807c3614b76aa39468e1d6669b5c610cbab0f21e
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index 25f4465..dd53450 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -39,11 +39,12 @@
* @param env_var Environment variables to set in container
* @param entrypoint Set entrypoint to /bin/bash or leave default
* @param mounts Map with mounts for container
+ * @param output_replacing Maps with regex with should be hide from output (passwords, etc)
**/
def runContainer(Map params){
def common = new com.mirantis.mk.Common()
- defaults = ["name": "cvp", "env_var": [], "entrypoint": true]
+ defaults = ["name": "cvp", "env_var": [], "entrypoint": true, "mounts": [:], "output_replacing": []]
params = defaults + params
def salt = new com.mirantis.mk.Salt()
def variables = ''
@@ -69,8 +70,11 @@
params.mounts.each { local, container ->
mounts = mounts + " -v ${local}:${container}"
}
- salt.cmdRun(params.master, params.target, "docker run -tid --net=host --name=${params.name}" +
- "${mounts} -u root ${entry_point} ${variables} ${params.dockerImageLink}")
+ salt.cmdRun(params.master, params.target,
+ "docker run -tid --net=host --name=${params.name}" +
+ "${mounts} -u root ${entry_point} ${variables} ${params.dockerImageLink}",
+ true, null, true, [],
+ params.output_replacing)
}
def runContainer(master, target, dockerImageLink, name='cvp', env_var=[], entrypoint=true, mounts=[:]){
@@ -280,7 +284,7 @@
}
}
def script = ". ${env.WORKSPACE}/venv/bin/activate; ${settings}" +
- "pytest --junitxml ${output_dir}cvp_sanity.xml --tb=short -sv ${env.WORKSPACE}/cvp-sanity-checks/cvp_checks/tests/${test_set}"
+ "pytest --junitxml ${output_dir}cvp_sanity.xml --tb=short -rs -sv ${env.WORKSPACE}/cvp-sanity-checks/cvp_checks/tests/${test_set}"
withEnv(["SALT_USERNAME=${username}", "SALT_PASSWORD=${password}", "SALT_URL=${salt_url}"]) {
def statusCode = sh script:script, returnStatus:true
}
@@ -305,7 +309,7 @@
if (container_node != "") {
def saltMaster
saltMaster = salt.connection(salt_url, salt_credentials)
- def script = "pytest --junitxml ${xml_file} --tb=short -sv ${test_set}"
+ def script = "pytest --junitxml ${xml_file} --tb=short -rs -sv ${test_set}"
env_vars.addAll("SALT_USERNAME=${username}", "SALT_PASSWORD=${password}",
"SALT_URL=${salt_url}")
variables = ' -e ' + env_vars.join(' -e ')
@@ -318,7 +322,7 @@
variables = 'export ' + env_vars.join(';export ')
}
def script = ". ${env.WORKSPACE}/venv/bin/activate; ${variables}; " +
- "pytest --junitxml ${artifacts_dir}${xml_file} --tb=short -sv ${env.WORKSPACE}/${test_set}"
+ "pytest --junitxml ${artifacts_dir}${xml_file} --tb=short -rs -sv ${env.WORKSPACE}/${test_set}"
withEnv(["SALT_USERNAME=${username}", "SALT_PASSWORD=${password}", "SALT_URL=${salt_url}"]) {
def statusCode = sh script:script, returnStatus:true
}
@@ -348,7 +352,7 @@
}
}
def script = ". ${env.WORKSPACE}/venv/bin/activate; ${settings}" +
- "pytest --junitxml ${output_dir}report.xml --tb=short -sv ${env.WORKSPACE}/${test_set}"
+ "pytest --junitxml ${output_dir}report.xml --tb=short -rs -sv ${env.WORKSPACE}/${test_set}"
withEnv(["SALT_USERNAME=${username}", "SALT_PASSWORD=${password}", "SALT_URL=${salt_url}"]) {
def statusCode = sh script:script, returnStatus:true
}
diff --git a/src/com/mirantis/mk/Ceph.groovy b/src/com/mirantis/mk/Ceph.groovy
new file mode 100644
index 0000000..bb837b2
--- /dev/null
+++ b/src/com/mirantis/mk/Ceph.groovy
@@ -0,0 +1,92 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Ceph functions
+ *
+ */
+
+/**
+ * Ceph health check
+ *
+ */
+def waitForHealthy(master, target, flags=[], count=0, attempts=300) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+ // wait for healthy cluster
+ while (count < attempts) {
+ def health = salt.cmdRun(master, target, 'ceph health')['return'][0].values()[0]
+ if (health.contains('HEALTH_OK')) {
+ common.infoMsg('Cluster is healthy')
+ break
+ } else {
+ for (flag in flags) {
+ if (health.contains(flag + ' flag(s) set') && !(health.contains('down'))) {
+ common.infoMsg('Cluster is healthy')
+ return
+ }
+ }
+ }
+ common.infoMsg("Ceph health status: ${health}")
+ count++
+ sleep(10)
+ }
+}
+
+/**
+ * Ceph remove partition
+ *
+ */
+def removePartition(master, target, partition_uuid, type='', id=-1) {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def partition = ""
+ if (type == 'lockbox') {
+ try {
+ // umount - partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "lsblk -rp | grep -v mapper | grep ${partition_uuid} ")['return'][0].values()[0].split()[0]
+ salt.cmdRun(master, target, "umount ${partition}")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ } else if (type == 'data') {
+ try {
+ // umount - partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "df | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0].split()[0]
+ salt.cmdRun(master, target, "umount ${partition}")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ try {
+ // partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ } else {
+ try {
+ // partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ }
+ if (partition?.trim()) {
+ if (partition.contains("nvme")) {
+ // partition = /dev/nvme1n1p2
+ // dev = /dev/nvme1n1
+ def dev = partition.replaceAll('p\\d+$', "")
+ // part_id = 2
+ def part_id = partition.substring(partition.lastIndexOf("p") + 1).replaceAll("[^0-9]+", "")
+
+ } else {
+ // partition = /dev/sdi2
+ // dev = /dev/sdi
+ def dev = partition.replaceAll('\\d+$', "")
+ // part_id = 2
+ def part_id = partition.substring(partition.lastIndexOf("/") + 1).replaceAll("[^0-9]+", "")
+ }
+ salt.cmdRun(master, target, "Ignore | parted ${dev} rm ${part_id}")
+ }
+ return
+}
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index 296dd38..c8f2e05 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -147,6 +147,16 @@
}
/**
+ * Print informational message
+ *
+ * @param msg
+ * @param color Colorful output or not
+ */
+def infoSensitivityMsg(msg, color = true, replacing = []) {
+ printSensitivityMsg(msg, "cyan", replacing)
+}
+
+/**
* Print error message
*
* @param msg
@@ -215,6 +225,25 @@
}
/**
+ * Print sensitivity message
+ *
+ * @param msg Message to be printed
+ * @param color Color to use for output
+ * @param replacing List with maps for deletion (passwords, logins, etc).
+ * The first () matching is mandatory !
+ * Example:
+ * [/ (OS_PASSWORD=)(.*?)+ /,
+ * / (password = )(.*?)+ /,
+ * / (password )(.*?) / ]
+ */
+def printSensitivityMsg(msg, color, replacing = []) {
+ for (i in replacing) {
+ msg = msg.replaceAll(i, ' $1XXXXXX ')
+ }
+ printMsg(msg, color)
+}
+
+/**
* Traverse directory structure and return list of files
*
* @param path Path to search
diff --git a/src/com/mirantis/mk/Galera.groovy b/src/com/mirantis/mk/Galera.groovy
index e5ffe12..3b6f626 100644
--- a/src/com/mirantis/mk/Galera.groovy
+++ b/src/com/mirantis/mk/Galera.groovy
@@ -378,8 +378,8 @@
if (restoreDb) {
def timestamp = common.getDatetime()
- salt.cmdRun(env, lastNodeTarget, "mkdir -p /root/mysql")
- def bakDir = salt.getReturnValues(salt.cmdRun(env, lastNodeTarget, "mktemp -d --suffix='_${timestamp}' /root/mysql/mysql.bak.XXXXXX", false))
+ def bakDir = "/root/mysql/mysql.bak.${timestamp}".toString()
+ salt.cmdRun(env, lastNodeTarget, "mkdir -p ${bakDir}")
salt.cmdRun(env, lastNodeTarget, "mv /var/lib/mysql/* ${bakDir} || echo 'Nothing to backup from directory /var/lib/mysql/'")
}
if (total) {
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 8f5d189..1656594 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -444,6 +444,7 @@
}
// Install horizon dashboard
+ salt.runSaltProcessStep(master, "* ${extra_tgt}", 'mine.update')
salt.enforceStateWithTest([saltId: master, target: "I@horizon:server ${extra_tgt}", state: 'horizon'])
}
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index 82e9ee3..f009107 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -34,7 +34,12 @@
sh(returnStdout: true, script: virtualenv_cmd)
if(!offlineDeployment){
try {
- runVirtualenvCommand(path, "pip install -U setuptools pip")
+ def pipPackage = 'pip'
+ if (python == 'python2') {
+ pipPackage = "\"pip<=19.3.1\""
+ common.infoMsg("Pinning pip package due to end of life of Python2 to ${pipPackage} version.")
+ }
+ runVirtualenvCommand(path, "pip install -U setuptools ${pipPackage}")
} catch(Exception e) {
common.warningMsg("Setuptools and pip cannot be updated, you might be offline but OFFLINE_DEPLOYMENT global property not initialized!")
}
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 8c6384c..f4147e0 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -331,12 +331,13 @@
* @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
* @param output do you want to print output
* @param saltArgs additional salt args eq. ["runas=aptly"]
+ * @param replacing list with maps for deletion in info message (passwords, logins, etc)
* @return output of salt command
*/
-def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = []) {
+def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = [], replacing = []) {
def common = new com.mirantis.mk.Common()
def originalCmd = cmd
- common.infoMsg("Running command ${cmd} on ${target}")
+ common.infoSensitivityMsg("Running command ${cmd} on ${target}", true, replacing)
if (checkResponse) {
cmd = cmd + " && echo Salt command execution success"
}
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 7952980..893f10f 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -155,16 +155,16 @@
}
try {
- common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
- timeout(time: 10, unit: 'SECONDS') {
- sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
- }
- timeout(time: 10, unit: 'SECONDS') {
- sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
+ timeout(time: 30, unit: 'SECONDS') {
+ if (sh(script: "docker inspect ${dockerContainerName}", returnStatus: true) == 0) {
+ common.warningMsg("Verify that container is not running. Ignore further docker-daemon errors")
+ sh(script: "set -x; test \$(docker inspect -f '{{.State.Running}}' ${dockerContainerName} 2>/dev/null) = 'true' && docker kill ${dockerContainerName}", returnStdout: true)
+ sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
+ }
}
}
catch (Exception er) {
- common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
+ common.warningMsg("IgnoreMe:Timeout to delete test docker container with force! Message:\n" + er.toString())
}
if (TestMarkerResult) {