stacklight-upgrade.groovy

/**
 *
 * Upgrade Stacklight packages and components
 *
 * Requred parameters:
 *  SALT_MASTER_URL                 URL of Salt master
 *  SALT_MASTER_CREDENTIALS         Credentials to the Salt API
 *
 *  STAGE_UPGRADE_SYSTEM_PART           Set to True if upgrade of system part (telegraf, fluentd, prometheus-relay) is desired
 *  STAGE_UPGRADE_ES_KIBANA             Set to True if Elasticsearch and Kibana upgrade is desired
 *  STAGE_UPGRADE_DOCKER_COMPONENTS     Set to True if upgrade for components running in Docker Swarm is desired
 *
 */

common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
python = new com.mirantis.mk.Python()
command = 'cmd.run'
pepperEnv = "pepperEnv"
errorOccured = false

def upgrade(master, target, service, pckg, state) {
    stage("Upgrade ${service}") {
        salt.runSaltProcessStep(master, "${target}", 'saltutil.refresh_pillar', [], null, true, 5)
        salt.enforceState(master, "${target}", 'linux.system.repo', true)
        common.infoMsg("Upgrade ${service} package")
        try {
            salt.runSaltProcessStep(master, "${target}", command, ["apt-get install --only-upgrade ${pckg}"], null, true)
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] ${pckg} package was not upgraded.")
            return
        }
        common.infoMsg("Run ${state} state on ${target} nodes")
        try {
            salt.enforceState(master, "${target}", ["${state}"], true)
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] ${state} state was executed and failed. Please fix it manually.")
        }
        common.infoMsg("Check ${service} service status on the target nodes")
        salt.runSaltProcessStep(master, "${target}", "service.status", ["${service}"], null, true)
        return
    }
}

def upgrade_es_kibana(master) {
    stage('Upgrade elasticsearch') {
        try {
            common.infoMsg('Upgrade the Elasticsearch package')
            salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
            salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["apt-get --only-upgrade install elasticsearch"], null, true)
            salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl daemon-reload"], null, true)
            salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl start elasticsearch"], null, true)
            salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
            return
        }
        common.infoMsg('Verify that the Elasticsearch cluster status is green')
        try {
            def retries_wait = 20
            def retries = 15
            def elasticsearch_vip
            def pillar = salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:host')
            if(!pillar['return'].isEmpty()) {
                elasticsearch_vip = pillar['return'][0].values()[0]
            } else {
                errorOccured = true
                common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
            }
            pillar = salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:port')
            def elasticsearch_port
            if(!pillar['return'].isEmpty()) {
                elasticsearch_port = pillar['return'][0].values()[0]
            } else {
                errorOccured = true
                common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
            }
            common.retry(retries,retries_wait) {
                common.infoMsg('Waiting for Elasticsearch to become green..')
                salt.cmdRun(master, "I@elasticsearch:client", "curl -sf ${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
            }
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] Elasticsearch cluster status is not \'green\'. Please fix it manually.")
            return
        }
    }
    stage('Upgrade kibana') {
        try {
            common.infoMsg('Upgrade the Kibana package')
            salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
            salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get --only-upgrade install kibana"], null, true)
            salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl start kibana"], null, true)
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
            return
        }

        common.infoMsg("Check kibana status on the target nodes")
        salt.runSaltProcessStep(master, "I@kibana:server", "service.status", ["kibana"], null, true)
        return
    }
}
timeout(time: 12, unit: 'HOURS') {
    node("python") {

        stage('Setup virtualenv for Pepper') {
            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
        }

        if (STAGE_UPGRADE_SYSTEM_PART.toBoolean() == true && !errorOccured) {
            upgrade(pepperEnv, "I@telegraf:agent or I@telegraf:remote_agent", "telegraf", "telegraf", "telegraf")
            upgrade(pepperEnv, "I@fluentd:agent", "td-agent", "td-agent td-agent-additional-plugins", "fluentd")
            if (salt.testTarget(pepperEnv, "I@prometheus:relay")) {
                upgrade(pepperEnv, "I@prometheus:relay", "prometheus-relay", "prometheus-relay", "prometheus")
            }
            if (salt.testTarget(pepperEnv, "I@prometheus:exporters:libvirt")) {
                upgrade(pepperEnv, "I@prometheus:exporters:libvirt", "libvirt-exporter", "libvirt-exporter", "prometheus")
            }
            if (salt.testTarget(pepperEnv, "I@prometheus:exporters:jmx")) {
                upgrade(pepperEnv, "I@prometheus:exporters:jmx", "jmx-exporter", "jmx-exporter", "prometheus")
            }
        }

        if (STAGE_UPGRADE_ES_KIBANA.toBoolean() == true && !errorOccured) {
            upgrade_es_kibana(pepperEnv)
        }

        if (STAGE_UPGRADE_DOCKER_COMPONENTS.toBoolean() == true && !errorOccured) {

            stage('Upgrade docker components') {

                try {
                    common.infoMsg('Disable and remove the previous versions of monitoring services')
                    salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', command, ["docker stack rm monitoring"], null, true)
                    common.infoMsg('Rebuild the Prometheus configuration')
                    salt.enforceState(pepperEnv, 'I@docker:swarm and I@prometheus:server', 'prometheus')
                    common.infoMsg('Disable and remove the previous version of Grafana')
                    salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', command, ["docker stack rm dashboard"], null, true)
                    common.infoMsg('Start the monitoring services')
                    salt.enforceState(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', 'docker')
                    salt.runSaltProcessStep(pepperEnv, '*', 'saltutil.sync_all', [], null, true)
                    common.infoMsg('Refresh the Grafana dashboards')
                    salt.enforceState(pepperEnv, 'I@grafana:client', 'grafana.client')
                } catch (Exception er) {
                    errorOccured = true
                    common.errorMsg("[ERROR] Upgrade of docker components failed. Please fix it manually.")
                    return
                }
            }
        }
    }
}