Maintenance update pipeline for 2018.4.1
Change-Id: I8dd540944ceef4f77015f01facb68c3c7c15e055
Related-Prod: PROD-28502
Related-Prod: PROD-28549
diff --git a/component-maintenance-update.groovy b/component-maintenance-update.groovy
new file mode 100644
index 0000000..1b7264d
--- /dev/null
+++ b/component-maintenance-update.groovy
@@ -0,0 +1,298 @@
+/**
+ *
+ * Performs MCP component packages updates
+ *
+ * Expected parameters:
+ * SALT_MASTER_URL Url to Salt API
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ * TARGET_SERVERS (Optional) String containing list of Salt targets split by comma.
+ * NOTE: For if this parameter is set, it will be used to run packages updates on specified targets
+ * If it isn't set targets will be detected automatically.
+ * COMPONENTS String containing comma-separated list of supported for update components
+ * Currently only nova and ceph are supported.
+ *
+ */
+
+common = new com.mirantis.mk.Common()
+def python = new com.mirantis.mk.Python()
+salt = new com.mirantis.mk.Salt()
+
+pepperEnv = "pepperEnv"
+
+/**
+ * Execute shell command using salt
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Minion to execute on
+ * @param cmd Command to execute
+ * @return string with shell output
+ */
+
+def runShCommand(saltMaster, target, cmd) {
+ return salt.cmdRun(saltMaster, target, cmd)
+}
+
+/**
+ * Installs packages updates by running apt install with
+ * flag --only-upgrade on target
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Minion to execute on
+ * @param pkgs List of packages to update e.g nova*, salt*, ceph*
+ */
+
+def installPkgUpdate(saltMaster, target, pkgs) {
+ common.infoMsg("Installing ${pkgs} updates on ${target}")
+ runShCommand(saltMaster, target, "apt install --only-upgrade ${pkgs.join(' ')} -y")
+}
+
+/**
+ * Returns string with values from pillar
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Minion to execute on
+ * @param pillar Pillar path to get values (nova:controller)
+ * @return string with Pillar values
+ */
+def getPillarValues(saltMaster, target, pillar) {
+ return salt.getReturnValues(salt.getPillar(saltMaster, target, pillar))
+}
+
+/**
+ * Returns pillar value converted to boolean
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Minion to execute on
+ * @param pillar Pillar path to get values (nova:controller:enabled)
+ * @return Boolean as result of Pillar output string
+ */
+
+def getPillarBoolValues(saltMaster, target, pillar){
+ return getPillarValues(saltMaster, target, pillar).toBoolean()
+}
+
+/**
+ * Returns first minion from sorted in alphsberical
+ * order list of minions
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Criteria by which to choose minions
+ * @return string with minion id
+ */
+
+def getFirstMinion(saltMaster, target) {
+ def minionsSorted = salt.getMinionsSorted(saltMaster, target)
+ return minionsSorted[0]
+}
+
+/**
+ * Stops list of services one by one on target
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Criteria by which to choose minions
+ *
+ */
+
+def stopServices(saltMaster, target, services) {
+ common.infoMsg("Stopping ${services} on ${target}")
+ for (s in services){
+ runShCommand(saltMaster, target, "systemctl stop ${s}")
+ }
+}
+
+def waitForHealthy(saltMaster, count=0, attempts=100) {
+ // wait for healthy cluster
+ while (count<attempts) {
+ def health = runShCommand(saltMaster, "I@ceph:mon and I@ceph:common:keyring:admin", 'ceph health')['return'][0].values()[0]
+ if (health.contains('HEALTH_OK')) {
+ common.infoMsg('Cluster is healthy')
+ break;
+ }
+ count++
+ sleep(10)
+ }
+}
+
+/**
+ * Returns nova service status in as list of hashes e.g.
+ * [
+ * {
+ * "Status": "enabled",
+ * "Binary": "nova-conductor",
+ * "Zone": "internal",
+ * "State": "up",
+ * "Host": "ctl01",
+ * "Updated At": "2019-03-22T17:39:02.000000",
+ * "ID": 7
+ * }
+ * ]
+ *
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target on which to run openstack client command
+ * @param host for which to get service status e.g. cmp1
+ * @param service name to check e.g. nova-compute
+ * @return List of hashes with service status data
+ *
+ */
+
+def getServiceStatus(saltMaster, target, host, service){
+ def cmd = ". /root/keystonercv3; openstack compute service list --host ${host} --service ${service} -f json"
+ common.retry(3, 10) {
+ res = readJSON text: salt.cmdRun(saltMaster, target, cmd)['return'][0].values()[0].replaceAll('Salt command execution success','')
+ }
+ return res
+}
+
+/**
+ * Waits while services are back to up state in Nova api output, if state
+ * doesn't change to 'up' raises error
+ *
+ * @param saltMaster Object pointing to salt master
+ * @param target Criteria by which to choose hosts where to check services states
+ * @param clientTarget Criteria by which to choose minion where to run openstack commands
+ * @param binaries lsit of services to wait for
+ * @param retries number of tries to to get service status
+ * @param timeout number of seconds to wait between tries
+ *
+ */
+
+def waitForServices(saltMaster, target, clientTarget, binaries, retries=18, timeout=10) {
+ for (host in salt.getMinionsSorted(saltMaster, target)) {
+ for (b in binaries) {
+ common.retry(retries, timeout) {
+ def status = getServiceStatus(saltMaster, clientTarget, host.tokenize('.')[0], b)[0]
+ if (status['State'] == 'up') {
+ common.infoMsg("Service ${b} on host ${host} is UP and Running")
+ } else {
+ error("Service ${b} status check failed or service isn't running on host ${host}")
+ }
+ }
+ }
+ }
+}
+
+node(){
+ try {
+ stage('Setup virtualenv for Pepper') {
+ python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ def components = COMPONENTS.tokenize(',')
+
+ if ('ceph' in components) {
+ def monPillar = 'ceph:mon:enabled'
+ def commonPillar = 'ceph:common:enabled'
+ def osdPillar = 'ceph:osd:enabled'
+ def rgwPillar = 'ceph:radosgw:enabled'
+
+ def monTarget = "I@${monPillar}:true"
+ def commonTarget = "I@${commonPillar}:true"
+ def osdTarget = "I@${osdPillar}:true"
+ def rgwTarget = "I@${rgwPillar}:true"
+ def targets = TARGET_SERVERS.tokenize(',')
+
+ // If TARGET_SERVERS is empty
+ if (!targets) {
+ targets = salt.getMinionsSorted(pepperEnv, commonTarget) + salt.getMinionsSorted(pepperEnv, monTarget) + salt.getMinionsSorted(pepperEnv, rgwTarget) + salt.getMinionsSorted(pepperEnv, osdTarget)
+ }
+ // Ceph common and other roles can be combined, so making host list elements to be unique
+ targets = targets.toSet()
+
+ stage('Update Ceph configuration using new defaults') {
+ for (t in targets) {
+ if (getPillarBoolValues(pepperEnv, t, commonPillar)) {
+ salt.enforceState(pepperEnv, t, 'ceph.common', true)
+ }
+ }
+ }
+
+ stage('Restart Ceph services') {
+ for (t in targets) {
+ if (getPillarBoolValues(pepperEnv, t, monPillar)) {
+ def monitors = salt.getMinions(pepperEnv, t)
+ for (tgt in monitors) {
+ runShCommand(pepperEnv, tgt, "systemctl restart ceph-mon.target")
+ runShCommand(pepperEnv, tgt, "systemctl restart ceph-mgr.target")
+ waitForHealthy(pepperEnv)
+ }
+ }
+ }
+ for (t in targets) {
+ if (getPillarBoolValues(pepperEnv, t, rgwPillar)) {
+ runShCommand(pepperEnv, t, "systemctl restart ceph-radosgw.target")
+ waitForHealthy(pepperEnv)
+ }
+ }
+ for (t in targets) {
+ if (getPillarBoolValues(pepperEnv, t, osdPillar)) {
+ def nodes = salt.getMinions(pepperEnv, t)
+ for (tgt in nodes) {
+ salt.runSaltProcessStep(pepperEnv, tgt, 'saltutil.sync_grains', [], null, true, 5)
+ def ceph_disks = salt.getGrain(pepperEnv, tgt, 'ceph')['return'][0].values()[0].values()[0]['ceph_disk']
+
+ def osd_ids = []
+ for (i in ceph_disks) {
+ def osd_id = i.getKey().toString()
+ osd_ids.add('osd.' + osd_id)
+ }
+
+ for (i in osd_ids) {
+ runShCommand(pepperEnv, tgt, 'ceph osd set noout')
+ salt.runSaltProcessStep(pepperEnv, tgt, 'service.restart', ['ceph-osd@' + i.replaceAll('osd.', '')], null, true)
+ sleep(60)
+ runShCommand(pepperEnv, tgt, 'ceph osd unset noout')
+ // wait for healthy cluster
+ waitForHealthy(pepperEnv)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if ('nova' in components) {
+ def ctlPillar = 'nova:controller:enabled'
+ def cmpPillar = 'nova:compute:enabled'
+
+ def cmpTarget = "I@${cmpPillar}:true"
+ def ctlTarget = "I@${ctlPillar}:true"
+ // Target for colling openstack client containing keystonercv3
+ def clientTarget = getFirstMinion(pepperEnv, 'I@keystone:client:enabled:true')
+ def targets = TARGET_SERVERS.tokenize(',')
+ // If TARGET_SERVERS is empty
+ if (!targets) {
+ targets = salt.getMinionsSorted(pepperEnv, ctlTarget) + salt.getMinionsSorted(pepperEnv, cmpTarget)
+ }
+
+ for (t in targets){
+ if (getPillarBoolValues(pepperEnv, t, ctlPillar) || getPillarBoolValues(pepperEnv, t, cmpPillar)) {
+ def tservices = ['nova*']
+ def tbinaries = []
+ if (getPillarBoolValues(pepperEnv, t, ctlPillar)) {
+ tservices += ['apache2']
+ tbinaries += ['nova-consoleauth', 'nova-scheduler', 'nova-conductor']
+ }
+ if (getPillarBoolValues(pepperEnv, t, cmpPillar)) {
+ tbinaries += ['nova-compute']
+ }
+ // Stop component services to ensure that updated code is running
+ stopServices(pepperEnv, t, tservices)
+ // Update all installed nova packages
+ installPkgUpdate(pepperEnv, t, ['nova*', 'python-nova*'])
+ common.infoMsg("Applying component states on ${t}")
+ salt.enforceState(pepperEnv, t, 'nova')
+ waitForServices(pepperEnv, t, clientTarget, tbinaries)
+ } else {
+ // If no compute or controller pillar is detected just packages will be updated
+ installPkgUpdate(pepperEnv, t, ['nova*', 'python-nova*'])
+ }
+ }
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+ throw e
+ }
+}
\ No newline at end of file