Updates for stacklight upgrade pipeline
- Add upgrade for td-agent-additional-plugins package
Related-PROD: PROD-25007 (PROD:25007)
- Fix global definitions
Related-PROD: PROD-25170 (PROD:25170)
- Use service.status salt command instead of output in
console
Related-PROD: PROD-25142 (PROD:25142)
Change-Id: Id98f8aef6e60aa01e9513c8d7e6999b88377ef12
diff --git a/stacklight-upgrade.groovy b/stacklight-upgrade.groovy
index 7b79f4c..443d56b 100644
--- a/stacklight-upgrade.groovy
+++ b/stacklight-upgrade.groovy
@@ -12,59 +12,42 @@
*
*/
-def common = new com.mirantis.mk.Common()
-def salt = new com.mirantis.mk.Salt()
-def python = new com.mirantis.mk.Python()
-
-def pepperEnv = "pepperEnv"
-def targetLiveSubset
-def targetLiveAll
-def minions
-def result
-def args
-def commandKwargs
-def probe = 1
-def errorOccured = false
+common = new com.mirantis.mk.Common()
+salt = new com.mirantis.mk.Salt()
+python = new com.mirantis.mk.Python()
+command = 'cmd.run'
+pepperEnv = "pepperEnv"
+errorOccured = false
def upgrade(master, target, service, pckg, state) {
- def common = new com.mirantis.mk.Common()
- def salt = new com.mirantis.mk.Salt()
- def command = 'cmd.run'
- stage("Change ${target} repos") {
+ stage("Upgrade ${service}") {
salt.runSaltProcessStep(master, "${target}", 'saltutil.refresh_pillar', [], null, true, 5)
salt.enforceState(master, "${target}", 'linux.system.repo', true)
- }
- stage("Update ${pckg} package") {
common.infoMsg("Upgrade ${service} package")
try {
salt.runSaltProcessStep(master, "${target}", command, ["apt-get install --only-upgrade ${pckg}"], null, true)
} catch (Exception er) {
errorOccured = true
- common.errorMsg("${pckg} package is not upgraded.")
+ common.errorMsg("[ERROR] ${pckg} package was not upgraded.")
return
}
- }
- stage("Run ${state} state on ${target} nodes") {
+ common.infoMsg("Run ${state} state on ${target} nodes")
try {
salt.enforceState(master, "${target}", ["${state}"], true)
} catch (Exception er) {
errorOccured = true
- common.errorMsg("${state} state was executed and failed. Please fix it manually.")
+ common.errorMsg("[ERROR] ${state} state was executed and failed. Please fix it manually.")
}
+ common.infoMsg("Check ${service} service status on the target nodes")
+ salt.runSaltProcessStep(master, "${target}", "service.status", ["${service}"], null, true)
+ return
}
- out = salt.runSaltCommand(master, 'local', ['expression': "${target}", 'type': 'compound'], command, null, "systemctl status ${service}.service", null)
- salt.printSaltCommandResult(out)
-
- common.warningMsg("Please check \'systemctl status ${service}.service\' on ${target} nodes if ${service} is running.")
- return
}
def upgrade_es_kibana(master) {
- def common = new com.mirantis.mk.Common()
- def salt = new com.mirantis.mk.Salt()
- def command = 'cmd.run'
- stage('Elasticsearch upgrade') {
+ stage('Upgrade elasticsearch') {
try {
+ common.infoMsg('Upgrade the Elasticsearch package')
salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["apt-get --only-upgrade install elasticsearch"], null, true)
salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl daemon-reload"], null, true)
@@ -72,11 +55,10 @@
salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
} catch (Exception er) {
errorOccured = true
- common.errorMsg("Elasticsearch upgrade failed. Please fix it manually.")
+ common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
return
}
- }
- stage('Verify that the Elasticsearch cluster status is green') {
+ common.infoMsg('Verify that the Elasticsearch cluster status is green')
try {
def retries_wait = 20
def retries = 15
@@ -102,24 +84,24 @@
}
} catch (Exception er) {
errorOccured = true
- common.errorMsg("Elasticsearch cluster status is not \'green\'. Please fix it manually.")
+ common.errorMsg("[ERROR] Elasticsearch cluster status is not \'green\'. Please fix it manually.")
return
}
}
- stage('Kibana upgrade') {
+ stage('Upgrade kibana') {
try {
+ common.infoMsg('Upgrade the Kibana package')
salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get --only-upgrade install kibana"], null, true)
salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl start kibana"], null, true)
} catch (Exception er) {
errorOccured = true
- common.errorMsg("Kibana upgrade failed. Please fix it manually.")
+ common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
return
}
- out = salt.runSaltCommand(master, 'local', ['expression': 'I@kibana:server', 'type': 'compound'], command, null, 'systemctl status kibana.service', null)
- salt.printSaltCommandResult(out)
- common.warningMsg('Please check if kibana service is running.')
+ common.infoMsg("Check kibana status on the target nodes")
+ salt.runSaltProcessStep(master, "I@kibana:server", "service.status", ["kibana"], null, true)
return
}
}
@@ -132,7 +114,7 @@
if (STAGE_UPGRADE_SYSTEM_PART.toBoolean() == true && !errorOccured) {
upgrade(pepperEnv, "I@telegraf:agent or I@telegraf:remote_agent", "telegraf", "telegraf", "telegraf")
- upgrade(pepperEnv, "I@fluentd:agent", "td-agent", "td-agent", "fluentd")
+ upgrade(pepperEnv, "I@fluentd:agent", "td-agent", "td-agent td-agent-additional-plugins", "fluentd")
if (salt.testTarget(pepperEnv, "I@prometheus:relay")) {
upgrade(pepperEnv, "I@prometheus:relay", "prometheus-relay", "prometheus-relay", "prometheus")
}
@@ -150,18 +132,23 @@
if (STAGE_UPGRADE_DOCKER_COMPONENTS.toBoolean() == true && !errorOccured) {
- stage('Docker components upgrade') {
+ stage('Upgrade docker components') {
try {
- salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', 'cmd.run', ["docker stack rm monitoring"], null, true)
+ common.infoMsg('Disable and remove the previous versions of monitoring services')
+ salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', command, ["docker stack rm monitoring"], null, true)
+ common.infoMsg('Rebuild the Prometheus configuration')
salt.enforceState(pepperEnv, 'I@docker:swarm and I@prometheus:server', 'prometheus')
- salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', 'cmd.run', ["docker stack rm dashboard"], null, true)
+ common.infoMsg('Disable and remove the previous version of Grafana')
+ salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', command, ["docker stack rm dashboard"], null, true)
+ common.infoMsg('Start the monitoring services')
salt.enforceState(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', 'docker')
salt.runSaltProcessStep(pepperEnv, '*', 'saltutil.sync_all', [], null, true)
+ common.infoMsg('Refresh the Grafana dashboards')
salt.enforceState(pepperEnv, 'I@grafana:client', 'grafana.client')
} catch (Exception er) {
errorOccured = true
- common.errorMsg("Upgrade of docker components failed. Please fix it manually.")
+ common.errorMsg("[ERROR] Upgrade of docker components failed. Please fix it manually.")
return
}
}