Merge "Add check for user choosing of stages Related-Prod: 23318 (PROD:23318)"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index f05735a..9878a49 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -134,12 +134,19 @@
}
if (STACK_REUSE.toBoolean() == false) {
- // Don't allow to set custom heat stack name
+ // TODO(vsaienko): remove stack creation from this pipeline to separate job
+ // Allow to set custom stack name but user-id will be added anyway
+ // This will fix issue with cleanup when job is aborted by jenkins and
+ // still guarantee stack count per user.
+ def stackNameSuffix = "${JOB_NAME}-${BUILD_NUMBER}"
+ if (STACK_NAME != ''){
+ stackNameSuffix = STACK_NAME
+ }
wrap([$class: 'BuildUser']) {
if (env.BUILD_USER_ID) {
- STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
+ STACK_NAME = "${env.BUILD_USER_ID}-${stackNameSuffix}"
} else {
- STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
+ STACK_NAME = "jenkins-${stackNameSuffix}"
}
currentBuild.description = STACK_NAME
}
@@ -390,6 +397,9 @@
stage('Install Kubernetes control') {
orchestrate.installKubernetesControl(venvPepper, extra_tgt)
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
+ orchestrate.checkContrailApiReadiness(venvPepper, extra_tgt)
+ }
// collect artifacts (kubeconfig)
writeFile(file: 'kubeconfig', text: salt.getFileContent(venvPepper, "I@kubernetes:master and *01* ${extra_tgt}", '/etc/kubernetes/admin-kube-config'))
@@ -466,6 +476,7 @@
if (common.checkContains('STACK_INSTALL', 'contrail')) {
orchestrate.installContrailNetwork(venvPepper, extra_tgt)
+ orchestrate.checkContrailApiReadiness(venvPepper, extra_tgt)
} else if (common.checkContains('STACK_INSTALL', 'ovs')) {
orchestrate.installOpenstackNetwork(venvPepper, extra_tgt)
}
diff --git a/k8s-upgrade-pipeline.groovy b/k8s-upgrade-pipeline.groovy
index 79de867..fe7d189 100644
--- a/k8s-upgrade-pipeline.groovy
+++ b/k8s-upgrade-pipeline.groovy
@@ -28,6 +28,8 @@
* KUBERNETES_CALICO_CNI_IPAM_SOURCE_HASH Сalico/ipam binary hash. Should be null if update rolling via reclass-system level
* KUBERNETES_CALICO_KUBE_CONTROLLERS_IMAGE Target calico/kube-controllers image. May be null in case of reclass-system rollout.
* CALICO_UPGRADE_VERSION Version of "calico-upgrade" utility to be used ("v1.0.5" for Calico v3.1.3 target).
+ * KUBERNETES_ETCD_SOURCE Target etcd binary. May be null in case of reclass-system rollout.
+ * KUBERNETES_ETCD_SOURCE_HASH Target etcd binary checksum. May be null in case of reclass-system rollout.
*
**/
import groovy.json.JsonSlurper
@@ -87,6 +89,27 @@
}
}
+def overrideEtcdSource(pepperEnv) {
+ def salt = new com.mirantis.mk.Salt()
+
+ def k8sSaltOverrides = """
+ kubernetes_etcd_source: ${KUBERNETES_ETCD_SOURCE}
+ kubernetes_etcd_source_hash: ${KUBERNETES_ETCD_SOURCE_HASH}
+ """
+ stage("Override etcd binaries to target version") {
+ salt.setSaltOverrides(pepperEnv, k8sSaltOverrides)
+ }
+}
+
+def performEtcdUpdateAndServicesRestart(pepperEnv, target) {
+ def salt = new com.mirantis.mk.Salt()
+
+ stage("Performing etcd update and services restart on ${target}") {
+ salt.enforceState(pepperEnv, target, "etcd.server.service")
+ salt.cmdRun(pepperEnv, target, ". /var/lib/etcd/configenv && etcdctl cluster-health")
+ }
+}
+
def performKubernetesComputeUpdate(pepperEnv, target) {
def salt = new com.mirantis.mk.Salt()
@@ -100,7 +123,7 @@
def salt = new com.mirantis.mk.Salt()
stage("Execute Kubernetes control plane update on ${target}") {
- salt.enforceStateWithExclude(pepperEnv, target, "kubernetes", "kubernetes.master.setup")
+ salt.enforceStateWithExclude(pepperEnv, target, "kubernetes", "kubernetes.master.setup,kubernetes.master.kube-addons")
// Restart kubelet
salt.runSaltProcessStep(pepperEnv, target, 'service.restart', ['kubelet'])
}
@@ -207,6 +230,69 @@
}
}
+def buildDaemonsetMap(pepperEnv, target) {
+ def salt = new com.mirantis.mk.Salt()
+ def daemonset_lists
+ daemonset_lists = salt.cmdRun(pepperEnv, target, "kubectl get ds --all-namespaces | tail -n+2 | awk '{print \$2, \$1}'"
+ )['return'][0].values()[0].replaceAll('Salt command execution success','').tokenize("\n")
+ def daemonset_map = []
+ for (ds in daemonset_lists) {
+ a = ds.tokenize(" ")
+ daemonset_map << a
+ }
+ print("Built daemonset map")
+ print(daemonset_map)
+ return daemonset_map
+}
+
+def purgeDaemonsetPods(pepperEnv, target, daemonSetMap) {
+ def salt = new com.mirantis.mk.Salt()
+ def originalTarget = "I@kubernetes:master and not ${target}"
+ def nodeShortName = target.tokenize(".")[0]
+ firstTarget = salt.getFirstMinion(pepperEnv, originalTarget)
+
+ if (daemonSetMap) {
+ stage("Purging daemonset-managed pods on ${target}") {
+ for (ds in daemonSetMap) {
+ print("Purging "+ ds[0] +" inside "+ ds[1] +" namespace")
+ salt.cmdRun(pepperEnv, firstTarget, "kubectl get po -n ${ds[1]} -o wide | grep ${nodeShortName}" +
+ " | grep ${ds[0]} | awk '{print \$1}' | xargs --no-run-if-empty kubectl delete po -n ${ds[1]} --grace-period=0 --force")
+ }
+ }
+ }
+}
+
+def isNodeReady(pepperEnv, target) {
+ def salt = new com.mirantis.mk.Salt()
+ def originalTarget = "I@kubernetes:master and not ${target}"
+ def nodeShortName = target.tokenize(".")[0]
+ firstTarget = salt.getFirstMinion(pepperEnv, originalTarget)
+
+ status = salt.cmdRun(pepperEnv, firstTarget, "kubectl get no | grep ${nodeShortName} | awk '{print \$2}'"
+ )['return'][0].values()[0].replaceAll('Salt command execution success',''
+ ).replaceAll(',SchedulingDisabled','').trim()
+
+ if (status == "Ready") {
+ return true
+ } else {
+ return false
+ }
+}
+
+def rebootKubernetesNode(pepperEnv, target, times=15, delay=10) {
+ def common = new com.mirantis.mk.Common()
+ def debian = new com.mirantis.mk.Debian()
+
+ stage("Rebooting ${target}") {
+ debian.osReboot(pepperEnv, target)
+ common.retry(times, delay) {
+ if(!isNodeReady(pepperEnv, target)) {
+ error("Node still not in Ready state...")
+ }
+ }
+ }
+}
+
def upgradeDocker(pepperEnv, target) {
def salt = new com.mirantis.mk.Salt()
@@ -579,6 +665,9 @@
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+ def ctl_node = salt.getMinionsSorted(pepperEnv, CTL_TARGET)[0]
+ def daemonsetMap = buildDaemonsetMap(pepperEnv, ctl_node)
+
if (CONFORMANCE_RUN_BEFORE.toBoolean()) {
def target = CTL_TARGET
def mcp_repo = ARTIFACTORY_URL
@@ -623,9 +712,6 @@
* as Calico etcd schema has different formats for Calico v2.x and Calico v3.x.
*/
if (UPGRADE_CALICO_V2_TO_V3.toBoolean()) {
- // one CTL node will be used for running upgrade of Calico etcd schema
- def ctl_node = salt.getMinionsSorted(pepperEnv, CTL_TARGET)[0]
-
// get ETCD_ENDPOINTS in use by Calico
def ep_str = salt.cmdRun(pepperEnv, ctl_node, "cat /etc/calico/calicoctl.cfg | grep etcdEndpoints")['return'][0].values()[0]
ETCD_ENDPOINTS = ep_str.split("\n")[0].tokenize(' ')[1]
@@ -648,6 +734,17 @@
}
/*
+ * Execute etcd update
+ */
+ if ((common.validInputParam('KUBERNETES_ETCD_SOURCE')) && (common.validInputParam('KUBERNETES_ETCD_SOURCE_HASH'))) {
+ overrideEtcdSource(pepperEnv)
+ }
+ def targetHostsEtcd = salt.getMinionsSorted(pepperEnv, "I@etcd:server")
+ for (t in targetHostsEtcd) {
+ performEtcdUpdateAndServicesRestart(pepperEnv, t)
+ }
+
+ /*
* Execute k8s update
*/
if (updates.contains("ctl")) {
@@ -665,6 +762,10 @@
regenerateCerts(pepperEnv, t)
performKubernetesControlUpdate(pepperEnv, t)
updateAddonManager(pepperEnv, t)
+ if (daemonsetMap) {
+ purgeDaemonsetPods(pepperEnv, t, daemonsetMap)
+ rebootKubernetesNode(pepperEnv, t)
+ }
uncordonNode(pepperEnv, t)
}
}
@@ -693,6 +794,10 @@
drainNode(pepperEnv, t)
regenerateCerts(pepperEnv, t)
performKubernetesComputeUpdate(pepperEnv, t)
+ if (daemonsetMap) {
+ purgeDaemonsetPods(pepperEnv, t, daemonsetMap)
+ rebootKubernetesNode(pepperEnv, t)
+ }
uncordonNode(pepperEnv, t)
}
}
@@ -701,7 +806,6 @@
}
}
- def ctl_node = salt.getMinionsSorted(pepperEnv, CTL_TARGET)[0]
if (calicoEnabled(pepperEnv, ctl_node)) {
checkCalicoClusterState(pepperEnv, POOL)
}
@@ -732,4 +836,4 @@
throw e
}
}
-}
+}
\ No newline at end of file
diff --git a/update-ceph.groovy b/update-ceph.groovy
new file mode 100644
index 0000000..59c616e
--- /dev/null
+++ b/update-ceph.groovy
@@ -0,0 +1,138 @@
+/**
+ * Update packages on given nodes
+ *
+ * Expected parameters:
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
+ * SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
+ * TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
+ */
+
+pepperEnv = "pepperEnv"
+salt = new com.mirantis.mk.Salt()
+def common = new com.mirantis.mk.Common()
+def python = new com.mirantis.mk.Python()
+def targetLiveSubset
+def targetLiveAll
+def minions
+def result
+def packages
+def command
+def commandKwargs
+def selMinions = []
+
+def runCephCommand(master, target, cmd) {
+ return salt.cmdRun(master, target, cmd)
+}
+
+def waitForHealthy(master, tgt, attempts=100, timeout=10) {
+ // wait for healthy cluster
+ common = new com.mirantis.mk.Common()
+ common.retry(attempts, timeout){
+ def health = runCephCommand(master, tgt, 'ceph health')['return'][0].values()[0]
+ if (health.contains('HEALTH_OK') || health.contains('HEALTH_WARN noout flag(s) set\n')) {
+ common.infoMsg('Cluster is healthy')
+ return 0
+ } else {
+ common.infoMsg(health)
+ throw new Exception()
+ }
+ }
+}
+
+timeout(time: 12, unit: 'HOURS') {
+ node() {
+ try {
+
+ stage('Setup virtualenv for Pepper') {
+ python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ stage('List target servers') {
+ minions = salt.getMinions(pepperEnv, TARGET_SERVERS)
+
+ if (minions.isEmpty()) {
+ throw new Exception("No minion was targeted")
+ }
+
+ for (m in minions) {
+ if (m.startsWith("osd") || m.startsWith("cmn") || m.startsWith("rgw")) {
+ selMinions.add(m)
+ }
+ }
+ }
+
+
+
+ stage('Apply package upgrades on all nodes') {
+
+ for (tgt in selMinions) {
+ try {
+ if (tgt.startsWith("osd")) {
+ out = runCephCommand(pepperEnv, tgt, "apt install --only-upgrade ceph-osd -y")
+ salt.printSaltCommandResult(out)
+ } else if (tgt.startsWith("cmn")) {
+ out = runCephCommand(pepperEnv, tgt, "apt install --only-upgrade ceph-mon -y")
+ salt.printSaltCommandResult(out)
+ } else if (tgt.startsWith("rgw")) {
+ out = runCephCommand(pepperEnv, tgt, "apt install --only-upgrade radosgw -y")
+ salt.printSaltCommandResult(out)
+ }
+ } catch (Throwable e) {
+ if (e.message.contains("Unmet dependencies")) {
+ out = runCephCommand(pepperEnv, tgt, "apt -f install -y")
+ salt.printSaltCommandResult(out)
+ } else {
+ throw (e)
+ }
+ }
+ }
+ }
+
+ stage("Restart MONs and RGWs") {
+ for (tgt in selMinions) {
+ if (tgt.contains("cmn")) {
+ runCephCommand(pepperEnv, tgt, "systemctl restart ceph-mon.target")
+ waitForHealthy(pepperEnv, tgt)
+ } else if (tgt.contains("rgw")) {
+ runCephCommand(pepperEnv, tgt, "systemctl restart ceph-radosgw.target")
+ waitForHealthy(pepperEnv, tgt)
+ }
+ }
+ }
+
+ stage('Restart OSDs') {
+
+ for (tgt in selMinions) {
+ if (tgt.contains("osd")) {
+ salt.runSaltProcessStep(pepperEnv, tgt, 'saltutil.sync_grains', [], null, true, 5)
+ def ceph_disks = salt.getGrain(pepperEnv, tgt, 'ceph')['return'][0].values()[0].values()[0]['ceph_disk']
+
+ def osd_ids = []
+ for (i in ceph_disks) {
+ def osd_id = i.getKey().toString()
+ osd_ids.add('osd.' + osd_id)
+ }
+
+ runCephCommand(pepperEnv, tgt, 'ceph osd set noout')
+
+ for (i in osd_ids) {
+
+ salt.runSaltProcessStep(pepperEnv, tgt, 'service.restart', ['ceph-osd@' + i.replaceAll('osd.', '')], null, true)
+ // wait for healthy cluster
+ waitForHealthy(pepperEnv, tgt)
+ }
+
+ runCephCommand(pepperEnv, tgt, 'ceph osd unset noout')
+ }
+ }
+ }
+
+
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+ throw e
+ }
+ }
+}
\ No newline at end of file
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 14a746b..24ac15f 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -189,6 +189,14 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.salt/system.linux.system.repo.mcp.apt_mirantis.update/g'")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout ${reclassSystemBranch}")
+ // Add kubernetes-extra repo
+ if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
+ common.infoMsg("Add kubernetes-extra repo")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
+ "grep -q system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra' kubernetes/common.yml")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
+ "grep -q system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra' kubernetes/common.yml")
+ }
// Add new defaults
common.infoMsg("Add new defaults")
salt.cmdRun(venvPepper, 'I@salt:master', "grep '^ mcp_version: ' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
deleted file mode 100644
index b1d4a4e..0000000
--- a/xtrabackup-restore-mysql-db.groovy
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Update packages on given nodes
- *
- * Expected parameters:
- * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
- * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
- *
-**/
-
-def common = new com.mirantis.mk.Common()
-def salt = new com.mirantis.mk.Salt()
-def python = new com.mirantis.mk.Python()
-
-def pepperEnv = "pepperEnv"
-timeout(time: 12, unit: 'HOURS') {
- node() {
-
- stage('Setup virtualenv for Pepper') {
- python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
- }
-
- stage('Start restore') {
- // # actual upgrade
-
- stage('Ask for manual confirmation') {
- input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore mysql db?"
- }
- // database restore section
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
- } catch (Exception er) {
- common.warningMsg('Mysql service already stopped')
- }
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'service.stop', ['mysql'], null, true)
- } catch (Exception er) {
- common.warningMsg('Mysql service already stopped')
- }
- try {
- salt.cmdRun(pepperEnv, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
- } catch (Exception er) {
- common.warningMsg('Files are not present')
- }
- try {
- salt.cmdRun(pepperEnv, 'I@galera:master', "mkdir -p /root/mysql/mysql.bak")
- } catch (Exception er) {
- common.warningMsg('Directory already exists')
- }
- try {
- salt.cmdRun(pepperEnv, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
- } catch (Exception er) {
- common.warningMsg('Files were already moved')
- }
- try {
- salt.cmdRun(pepperEnv, 'I@galera:master', "rm -rf /var/lib/mysql/*")
- } catch (Exception er) {
- common.warningMsg('Directory already empty')
- }
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
- } catch (Exception er) {
- common.warningMsg('File is not present')
- }
- salt.cmdRun(pepperEnv, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
- _pillar = salt.getPillar(pepperEnv, "I@galera:master", 'xtrabackup:client:backup_dir')
- backup_dir = _pillar['return'][0].values()[0]
- if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
- print(backup_dir)
- salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
- salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'state.apply', ["xtrabackup.client.restore"], null, true)
- salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'service.start', ['mysql'], null, true)
-
- // wait until mysql service on galera master is up
- salt.commandStatus(pepperEnv, 'I@galera:master', 'service mysql status', 'running')
-
- salt.runSaltProcessStep(pepperEnv, 'I@galera:slave', 'service.start', ['mysql'], null, true)
- try {
- salt.commandStatus(pepperEnv, 'I@galera:slave', 'service mysql status', 'running')
- } catch (Exception er) {
- common.warningMsg('Either there are no galera slaves or something failed when starting mysql on galera slaves')
- }
- sleep(5)
- salt.cmdRun(pepperEnv, 'I@galera:master', "su root -c 'salt-call mysql.status | grep -A1 wsrep_cluster_size'")
-
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@galera:master or I@galera:slave', 'file.touch', ["/var/lib/mysql/.galera_bootstrap"], null, true)
- } catch (Exception er) {
- common.warningMsg('File is already present')
- }
- }
- }
-}