Merge the tip of origin/release/proposed/2019.2.0 into origin/release/2019.2.0
c3f75c1 Added lvm support for removing partitions
23e38d6 Pass batch param to debian.osUpgradeNode function
5ab6364 Add missed step for keycloak deploy on cicd stage
f14fc57 Fix type in requirements file
71754dc Pin pip packages to stay with python2
a56c1ab Pin setuptools<45.0.0 for setupOpenstackVirtualenv
a69bd10 [CVP] Fixed cleaning up cvp containers
Change-Id: I89a0f0129ca1756c09dd4b3d2d8258954d8bf25c
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index dd53450..9c08472 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -58,7 +58,7 @@
"/tmp/": "/tmp/",
"/etc/hosts": "/etc/hosts"]
params.mounts = default_mounts + params.mounts
- if ( salt.cmdRun(params.master, params.target, "docker ps -f name=${params.name} -q", false, null, false)['return'][0].values()[0] ) {
+ if ( salt.cmdRun(params.master, params.target, "docker ps -f name=^${params.name}\$ -q", false, null, false)['return'][0].values()[0] ) {
salt.cmdRun(params.master, params.target, "docker rm -f ${params.name}")
}
if (params.env_var.size() > 0) {
@@ -869,7 +869,7 @@
*/
def runCleanup(master, target, name='cvp') {
def salt = new com.mirantis.mk.Salt()
- if ( salt.cmdRun(master, target, "docker ps -f name=${name} -q", false, null, false)['return'][0].values()[0] ) {
+ if ( salt.cmdRun(master, target, "docker ps -f name=^${name}\$ -q", false, null, false)['return'][0].values()[0] ) {
salt.cmdRun(master, target, "docker rm -f ${name}")
}
}
diff --git a/src/com/mirantis/mk/Ceph.groovy b/src/com/mirantis/mk/Ceph.groovy
index bb837b2..5326321 100644
--- a/src/com/mirantis/mk/Ceph.groovy
+++ b/src/com/mirantis/mk/Ceph.groovy
@@ -41,52 +41,62 @@
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
def partition = ""
- if (type == 'lockbox') {
- try {
- // umount - partition = /dev/sdi2
- partition = salt.cmdRun(master, target, "lsblk -rp | grep -v mapper | grep ${partition_uuid} ")['return'][0].values()[0].split()[0]
- salt.cmdRun(master, target, "umount ${partition}")
- } catch (Exception e) {
- common.warningMsg(e)
+ def dev = ''
+ def lvm_enabled = salt.getPillar(master, "I@ceph:osd", "ceph:osd:lvm_enabled")['return'].first().containsValue(true)
+ if ( !lvm_enabled ){
+ if (type == 'lockbox') {
+ try {
+ // umount - partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "lsblk -rp | grep -v mapper | grep ${partition_uuid} ")['return'][0].values()[0].split()[0]
+ salt.cmdRun(master, target, "umount ${partition}")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ } else if (type == 'data') {
+ try {
+ // umount - partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "df | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0].split()[0]
+ salt.cmdRun(master, target, "umount ${partition}")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ try {
+ // partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ } else {
+ try {
+ // partition = /dev/sdi2
+ partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
}
- } else if (type == 'data') {
- try {
- // umount - partition = /dev/sdi2
- partition = salt.cmdRun(master, target, "df | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0].split()[0]
- salt.cmdRun(master, target, "umount ${partition}")
- } catch (Exception e) {
- common.warningMsg(e)
- }
- try {
- // partition = /dev/sdi2
- partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
- } catch (Exception e) {
- common.warningMsg(e)
- }
- } else {
- try {
- // partition = /dev/sdi2
- partition = salt.cmdRun(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
- } catch (Exception e) {
- common.warningMsg(e)
+ if (partition?.trim()) {
+ if (partition.contains("nvme")) {
+ // partition = /dev/nvme1n1p2
+ // dev = /dev/nvme1n1
+ dev = partition.replaceAll('p\\d+$', "")
+ // part_id = 2
+ def part_id = partition.substring(partition.lastIndexOf("p") + 1).replaceAll("[^0-9]+", "")
+
+ } else {
+ // partition = /dev/sdi2
+ // dev = /dev/sdi
+ dev = partition.replaceAll('\\d+$', "")
+ // part_id = 2
+ def part_id = partition.substring(partition.lastIndexOf("/") + 1).replaceAll("[^0-9]+", "")
+ }
}
}
- if (partition?.trim()) {
- if (partition.contains("nvme")) {
- // partition = /dev/nvme1n1p2
- // dev = /dev/nvme1n1
- def dev = partition.replaceAll('p\\d+$', "")
- // part_id = 2
- def part_id = partition.substring(partition.lastIndexOf("p") + 1).replaceAll("[^0-9]+", "")
-
- } else {
- // partition = /dev/sdi2
- // dev = /dev/sdi
- def dev = partition.replaceAll('\\d+$', "")
- // part_id = 2
- def part_id = partition.substring(partition.lastIndexOf("/") + 1).replaceAll("[^0-9]+", "")
- }
- salt.cmdRun(master, target, "Ignore | parted ${dev} rm ${part_id}")
+ if (lvm_enabled && type != 'lockbox') {
+ salt.cmdRun(master, target, "ceph-volume lvm zap ${partition_uuid} --destroy")
+ } else if (dev != '') {
+ salt.cmdRun(master, target, "parted ${dev} rm ${part_id}")
+ } else {
+ common.infoMsg("Did not found any device to be wiped.")
}
return
}
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 6bb1f50..5ca197b 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -258,15 +258,15 @@
* @param env Salt Connection object or env Salt command map
* @param target Salt target to upgrade packages on.
*/
-def osUpgrade(env, target){
- def common = new com.mirantis.mk.Common()
- def salt = new com.mirantis.mk.Salt()
+def osUpgrade(env, target, batch=null) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
- common.infoMsg("Running upgrade on ${target}")
+ common.infoMsg("Running upgrade on ${target}")
- salt.runSaltProcessStep(env, target, 'pkg.refresh_db', [], null, true)
- def cmd = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade'
- salt.cmdRun(env, target, cmd)
+ salt.runSaltProcessStep(env, target, 'pkg.refresh_db', [], batch, true)
+ def cmd = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade'
+ salt.cmdRun(env, target, cmd, true, batch)
}
/**
@@ -275,14 +275,14 @@
* @param env Salt Connection object or env Salt command map
* @param target Salt target to upgrade packages on.
*/
-def osDistUpgrade(env, target){
- def salt = new com.mirantis.mk.Salt()
- def common = new com.mirantis.mk.Common()
+def osDistUpgrade(env, target, batch=null) {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
- common.infoMsg("Running dist-upgrade on ${target}")
- salt.runSaltProcessStep(env, target, 'pkg.refresh_db', [], null, true)
- def cmd = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade'
- salt.cmdRun(env, target, cmd)
+ common.infoMsg("Running dist-upgrade on ${target}")
+ salt.runSaltProcessStep(env, target, 'pkg.refresh_db', [], batch, true)
+ def cmd = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade'
+ salt.cmdRun(env, target, cmd, true, batch)
}
/**
@@ -293,18 +293,18 @@
* @param timeout Sleep timeout when doing retries.
* @param attempts Number of attemps to wait for.
*/
-def osReboot(env, target, timeout=30, attempts=10){
- def salt = new com.mirantis.mk.Salt()
- def common = new com.mirantis.mk.Common()
+def osReboot(env, target, timeout=30, attempts=10) {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
- salt.runSaltProcessStep(env, target, 'cmd.run', ["touch /tmp/rebooting"])
- salt.runSaltProcessStep(env, target, 'system.reboot', [], null, true, 5)
+ salt.runSaltProcessStep(env, target, 'cmd.run', ["touch /tmp/rebooting"])
+ salt.runSaltProcessStep(env, target, 'system.reboot', [], null, true, 5)
- common.retry(timeout, attempts){
- if (salt.runSaltProcessStep(env, target, 'file.file_exists', ['/tmp/rebooting'], null, true, 5)['return'][0].values()[0].toBoolean()){
- error("The system is still rebooting...")
+ common.retry(timeout, attempts) {
+ if (salt.runSaltProcessStep(env, target, 'file.file_exists', ['/tmp/rebooting'], null, true, 5)['return'][0].values()[0].toBoolean()) {
+ error("The system is still rebooting...")
+ }
}
- }
}
/**
@@ -317,23 +317,23 @@
* @param timeout Sleep timeout when doing retries.
* @param attempts Number of attemps to wait for.
*/
-def osUpgradeNode(env, target, mode, postponeReboot=false, timeout=30, attempts=10){
+def osUpgradeNode(env, target, mode, postponeReboot=false, timeout=30, attempts=10, batch=null) {
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def rebootRequired = false
- if (mode == 'dist-upgrade'){
- osDistUpgrade(env, target)
- } else if (mode == 'upgrade'){
- osUpgrade(env, target)
+ if (mode == 'dist-upgrade') {
+ osDistUpgrade(env, target, batch)
+ } else if (mode == 'upgrade') {
+ osUpgrade(env, target, batch)
}
- rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], null, true, 5)['return'][0].values()[0].toBoolean()
+ rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
if (rebootRequired) {
- if (!postponeReboot){
- common.infoMsg("Reboot is required after upgrade on ${target} Rebooting...")
- osReboot(env, target, timeout, attempts)
- } else {
- common.infoMsg("Postponing reboot on node ${target}")
- }
+ if (!postponeReboot) {
+ common.infoMsg("Reboot is required after upgrade on ${target} Rebooting...")
+ osReboot(env, target, timeout, attempts)
+ } else {
+ common.infoMsg("Postponing reboot on node ${target}")
+ }
}
}
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index b11f628..98df398 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -22,9 +22,9 @@
* @param version Version of the OpenStack clients
*/
-def setupOpenstackVirtualenv(path, version = 'latest') {
- def python = new com.mirantis.mk.Python()
- python.setupDocutilsVirtualenv(path)
+def setupOpenstackVirtualenv(path, version = 'latest', python="python2") {
+ def pythonLib = new com.mirantis.mk.Python()
+ pythonLib.setupDocutilsVirtualenv(path)
def openstack_kilo_packages = [
//XXX: hack to fix https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1635463
@@ -41,7 +41,7 @@
'oslo.i18n>=2.3.0,<2.4.0',
'oslo.serialization>=1.8.0,<1.9.0',
'oslo.utils>=1.4.0,<1.5.0',
- 'docutils'
+ 'docutils==0.16'
]
def openstack_latest_packages = [
@@ -54,9 +54,11 @@
'cmd2<0.9.0;python_version=="2.7"',
'cmd2>=0.9.1;python_version=="3.4"',
'cmd2>=0.9.1;python_version=="3.5"',
- 'python-openstackclient',
- 'python-heatclient',
- 'docutils'
+ // NOTE: pin client packages to current latest to prevent
+ // downloading packages which are not support Python 2.7
+ 'python-openstackclient==4.0.0',
+ 'python-heatclient==1.18.0',
+ 'docutils==0.16'
]
if (version == 'kilo') {
@@ -68,7 +70,7 @@
} else {
requirements = openstack_latest_packages
}
- python.setupVirtualenv(path, 'python2', requirements, null, true)
+ pythonLib.setupVirtualenv(path, python, requirements, null, true)
}
/**
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 1656594..21635f5 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -194,10 +194,8 @@
// If galera is not enabled check if we need to install mysql:server
} else {
-
- salt.enforceStateWithTest([saltId: master, target: "I@mysql:server ${extra_tgt}", state: 'mysql.server'])
- salt.enforceStateWithTest([saltId: master, target: "I@mysql:client ${extra_tgt}", state: 'mysql.client'])
-
+ salt.enforceStateWithTest([saltId: master, target: "I@mysql:server ${extra_tgt}", state: 'mysql.server'])
+ salt.enforceStateWithTest([saltId: master, target: "I@mysql:client ${extra_tgt}", state: 'mysql.client'])
}
installBackup(master, 'mysql', extra_tgt)
@@ -804,6 +802,8 @@
salt.enforceStateWithTest([saltId: master, target: "I@openldap:client ${extra_tgt}", state: 'openldap', retries: 2])
+ salt.enforceStateWithTest([saltId: master, target: "( I@keycloak:server or I@keycloak:proxy ) ${extra_tgt}", state: 'keycloak', retries: 2])
+
salt.enforceStateWithTest([saltId: master, target: "I@python:environment ${extra_tgt}", state: 'python'])
withEnv(['ASK_ON_ERROR=false']){
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index f009107..29288f9 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -32,17 +32,18 @@
}
common.infoMsg("[Python ${path}] Setup ${python} environment")
sh(returnStdout: true, script: virtualenv_cmd)
- if(!offlineDeployment){
- try {
- def pipPackage = 'pip'
- if (python == 'python2') {
- pipPackage = "\"pip<=19.3.1\""
- common.infoMsg("Pinning pip package due to end of life of Python2 to ${pipPackage} version.")
- }
- runVirtualenvCommand(path, "pip install -U setuptools ${pipPackage}")
- } catch(Exception e) {
- common.warningMsg("Setuptools and pip cannot be updated, you might be offline but OFFLINE_DEPLOYMENT global property not initialized!")
- }
+ if (!offlineDeployment) {
+ try {
+ def pipPackage = 'pip'
+ if (python == 'python2') {
+ pipPackage = "\"pip<=19.3.1\""
+ common.infoMsg("Pinning pip package due to end of life of Python2 to ${pipPackage} version.")
+ }
+ // NOTE(vsaienko): pin setuptools explicitly for latest version that works with python2
+ runVirtualenvCommand(path, "pip install -U \"setuptools<45.0.0\" ${pipPackage}")
+ } catch (Exception e) {
+ common.warningMsg("Setuptools and pip cannot be updated, you might be offline but OFFLINE_DEPLOYMENT global property not initialized!")
+ }
}
if (reqs_path==null) {
def args = ""
@@ -87,11 +88,11 @@
*
* @param path Path where virtualenv is created
*/
-def setupDocutilsVirtualenv(path) {
+def setupDocutilsVirtualenv(path, python="python2") {
requirements = [
- 'docutils',
+ 'docutils==0.16',
]
- setupVirtualenv(path, 'python2', requirements)
+ setupVirtualenv(path, python, requirements)
}
@@ -370,8 +371,8 @@
*/
def setupJinjaVirtualenv(path) {
requirements = [
- 'jinja2-cli',
- 'pyyaml',
+ 'jinja2-cli==0.7.0',
+ 'pyyaml==5.3',
]
setupVirtualenv(path, 'python2', requirements)
}