stacklight-upgrade.groovy

/**
 *
 * Upgrade Stacklight packages and components
 *
 * Requred parameters:
 *  SALT_MASTER_URL                 URL of Salt master
 *  SALT_MASTER_CREDENTIALS         Credentials to the Salt API
 *
 *  STAGE_UPGRADE_SYSTEM_PART           Set to True if upgrade of system part (telegraf, fluentd, prometheus-relay) is desired
 *  STAGE_UPGRADE_ES_KIBANA             Set to True if Elasticsearch and Kibana upgrade is desired
 *  STAGE_UPGRADE_DOCKER_COMPONENTS     Set to True if upgrade for components running in Docker Swarm is desired
 *
 */

common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
python = new com.mirantis.mk.Python()
command = 'cmd.run'
pepperEnv = "pepperEnv"
errorOccured = false
def packageUpgradeMode = ''
def forceUpgradeComonents = false

def upgrade(master, target, service, pkg, state) {
    stage("Upgrade ${service}") {
        salt.runSaltProcessStep(master, "${target}", 'saltutil.refresh_pillar', [], null, true)
        salt.enforceState([saltId: master, target: "${target}", state: 'linux.system.repo', output: true, failOnError: true])
        common.infoMsg("Upgrade ${service} package(s)")
        try {
            salt.runSaltProcessStep(master, "${target}", command, ["apt-get install -y -o Dpkg::Options::=\"--force-confold\" ${pkg}"], null, true)
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] ${pkg} package(s) was not upgraded.")
            throw er
        }
        common.infoMsg("Run ${state} state on ${target} nodes")
        try {
            salt.enforceState([saltId: master, target: "${target}", state: ["${state}"], output: true, failOnError: true])
        } catch (Exception er) {
            errorOccured = true
            common.errorMsg("[ERROR] ${state} state was executed and failed. Please fix it manually.")
            throw er
        }
        common.infoMsg("Check ${service} service(s) status on the target nodes")
        for (s in service.split(" ")){
            salt.runSaltProcessStep(master, "${target}", "service.status", "${s}", null, true)
        }
    }
}

def verify_es_is_green(master) {
    common.infoMsg('Verify that the Elasticsearch cluster status is green')
    try {
        def retries_wait = 120
        def retries = 60

        def elasticsearch_vip
        def pillar = salt.getReturnValues(salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:host'))
        if(pillar) {
            elasticsearch_vip = pillar
        } else {
            errorOccured = true
            common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
        }

        pillar = salt.getReturnValues(salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:port'))
        def elasticsearch_port
        if(pillar) {
            elasticsearch_port = pillar
        } else {
            errorOccured = true
            common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
        }

        pillar = salt.getReturnValues(salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:scheme'))
        def elasticsearch_scheme
        if(pillar) {
            elasticsearch_scheme = pillar
            common.infoMsg("[INFO] Using elasticsearch scheme: ${elasticsearch_scheme}")
        } else {
            common.infoMsg('[INFO] No pillar with Elasticsearch server scheme, using scheme: http')
            elasticsearch_scheme = "http"
        }

        common.retry(retries,retries_wait) {
            common.infoMsg('Waiting for Elasticsearch to become green..')
            salt.cmdRun(master, "I@elasticsearch:client", "curl -sfk ${elasticsearch_scheme}://${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
        }
    } catch (Exception er) {
        errorOccured = true
        common.errorMsg("[ERROR] Elasticsearch cluster status is not \'green\'. Please fix it manually.")
        throw er
    }
}

def upgrade_es_kibana(master) {
    def elasticsearch_version
    def es_pillar = salt.getPillar(master, "I@elasticsearch:client", '_param:elasticsearch_version')
    if(!es_pillar['return'].isEmpty()) {
        elasticsearch_version = es_pillar['return'][0].values()[0]
    }
    stage('Upgrade elasticsearch') {
        if (elasticsearch_version == '5') {
            try {
                common.infoMsg('Upgrade the Elasticsearch package')
                salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
                salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["apt-get --only-upgrade install elasticsearch"], null, true)
                salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl daemon-reload"], null, true)
                salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl start elasticsearch"], null, true)
                salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
                verify_es_is_green(master)
            } catch (Exception er) {
                errorOccured = true
                common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
                throw er
            }
        } else {
            try {
                salt.runSaltProcessStep(master, "*", 'saltutil.refresh_pillar', [], null, true)
                salt.enforceState([saltId: master, target: "I@elasticsearch:server", state: 'linux.system.repo', output: true, failOnError: true])
                salt.runSaltProcessStep(master, 'I@elasticsearch:client', command, ["apt-get install -y -o Dpkg::Options::=\"--force-confold\" python-elasticsearch"], null, true)
                salt.enforceState([saltId: master, target: "I@elasticsearch:server", state: 'salt.minion', output: true, failOnError: true])
                salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
                salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["export ES_PATH_CONF=/etc/elasticsearch; apt-get install -y -o Dpkg::Options::=\"--force-confold\" elasticsearch"], null, true)
                salt.enforceState([saltId: master, target: "I@elasticsearch:server", state: 'elasticsearch.server', output: true, failOnError: true])
                verify_es_is_green(master)
                salt.enforceState([saltId: master, target: "I@elasticsearch:client", state: 'elasticsearch.client.update_index_templates', output: true, failOnError: true])
                salt.enforceState([saltId: master, target: "I@elasticsearch:client", state: 'elasticsearch.client', output: true, failOnError: true])
            } catch (Exception er) {
                errorOccured = true
                common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
                throw er
            }
        }
    }
    stage('Upgrade kibana') {
        def kibana_version
        def kibana_pillar = salt.getPillar(master, "I@kibana:client", '_param:kibana_version')
        if(!kibana_pillar['return'].isEmpty()) {
            kibana_version = kibana_pillar['return'][0].values()[0]
        }
        if (kibana_version == '5') {
            try {
                common.infoMsg('Upgrade the Kibana package')
                salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
                salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get --only-upgrade install kibana"], null, true)
                salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl start kibana"], null, true)
            } catch (Exception er) {
                errorOccured = true
                common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
                throw er
            }
        } else {
            try {
                salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
                salt.enforceStateWithExclude([saltId: master, target: "I@kibana:server", state: "kibana.server", excludedStates: "[{'id': 'kibana_service'}]"])
                salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get install -y -o Dpkg::Options::=\"--force-confold\" kibana"], null, true)
                salt.enforceState([saltId: master, target: "I@kibana:server", state: 'kibana.server', output: true, failOnError: true])
                salt.enforceState([saltId: master, target: "I@kibana:client", state: 'kibana.client', output: true, failOnError: true])
            } catch (Exception er) {
                errorOccured = true
                common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
                throw er
            }
        }

        common.infoMsg("Check kibana status on the target nodes")
        salt.runSaltProcessStep(master, "I@kibana:server", "service.status", ["kibana"], null, true)
    }
}
timeout(time: 12, unit: 'HOURS') {
    node("python") {

        if ((env.getProperty('OS_DIST_UPGRADE') ?: false).toBoolean()) {
            packageUpgradeMode = 'dist-upgrade'
            forceUpgradeComonents = true
        } else if ((env.getProperty('OS_UPGRADE') ?: false).toBoolean()) {
            packageUpgradeMode = 'upgrade'
            forceUpgradeComonents = true
        }

        if(forceUpgradeComonents) {
            common.infoMsg('Forcing to upgrade all Stacklight components because OS_DIST_UPGRADE or OS_UPGRADE is selected')
        }

        stage('Setup virtualenv for Pepper') {
            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
        }

        stage('Update grains and mine') {
            salt.enforceState([saltId: pepperEnv, target: '*', state: 'salt.minion.grains'])
            salt.runSaltProcessStep(pepperEnv, '*', 'saltutil.refresh_modules')
            salt.runSaltProcessStep(pepperEnv, '*', 'mine.update')
            sleep(30)
        }

        if (salt.testTarget(pepperEnv, "I@ceph:mon")) {
            stage('Enable Ceph prometheus plugin') {
                salt.enforceState([saltId: pepperEnv, target: 'I@ceph:mon', state: "ceph.mgr", output: true, failOnError: true])
            }
        }

        if (forceUpgradeComonents || (STAGE_UPGRADE_DOCKER_COMPONENTS.toBoolean() == true && !errorOccured)) {
            stage('Upgrade docker components') {
                try {
                    common.infoMsg('Disable and remove the previous versions of monitoring services')
                    salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', command, ["docker stack rm monitoring"], null, true)
                    common.infoMsg('Rebuild the Prometheus configuration')
                    salt.enforceState([saltId: pepperEnv, target: 'I@docker:swarm and I@prometheus:server', state: 'prometheus'])
                    common.infoMsg('Disable and remove the previous version of Grafana')
                    salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master and I@prometheus:server', command, ["docker stack rm dashboard"], null, true)
                    common.infoMsg('Start the monitoring services')
                    salt.enforceState([saltId: pepperEnv, target: 'I@docker:swarm:role:master and I@prometheus:server', state: 'docker'])
                    salt.runSaltProcessStep(pepperEnv, '*', 'saltutil.sync_all', [], null, true)
                    common.infoMsg("Waiting grafana service to start")
                    sleep(120)

                    common.infoMsg('Refresh the Grafana dashboards')
                    salt.enforceState([saltId: pepperEnv, target: 'I@grafana:client', state: 'grafana.client', retries: 10, retries_wait: 30])
                } catch (Exception er) {
                    errorOccured = true
                    common.errorMsg("[ERROR] Upgrade of docker components failed. Please fix it manually.")
                    throw er
                }
            }
        }

        if (forceUpgradeComonents || (STAGE_UPGRADE_SYSTEM_PART.toBoolean() == true && !errorOccured)) {
            upgrade(pepperEnv, "I@telegraf:agent or I@telegraf:remote_agent", "telegraf", "telegraf", "telegraf")
            upgrade(pepperEnv, "I@fluentd:agent", "td-agent", "td-agent td-agent-additional-plugins", "fluentd")
            if (salt.testTarget(pepperEnv, "I@prometheus:relay")) {
                upgrade(pepperEnv, "I@prometheus:relay", "prometheus prometheus-relay", "prometheus-bin prometheus-relay", "prometheus")
                salt.runSaltProcessStep(pepperEnv, "I@prometheus:relay", "service.restart", "prometheus", null, true)
            }
            if (salt.testTarget(pepperEnv, "I@prometheus:exporters:libvirt")) {
                upgrade(pepperEnv, "I@prometheus:exporters:libvirt", "libvirt-exporter", "libvirt-exporter", "prometheus")
            }
            if (salt.testTarget(pepperEnv, "I@prometheus:exporters:jmx")) {
                upgrade(pepperEnv, "I@prometheus:exporters:jmx", "jmx-exporter", "jmx-exporter", "prometheus")
            }
        }

        if (forceUpgradeComonents || (STAGE_UPGRADE_ES_KIBANA.toBoolean() == true && !errorOccured)) {
            upgrade_es_kibana(pepperEnv)
        }

        stage('Upgrade OS') {
            if (packageUpgradeMode) {
                def stacklightNodes = salt.getMinions(pepperEnv, 'I@elasticsearch:server or I@prometheus:server')
                def stacklightNodesWithDocker = salt.getMinions(pepperEnv, 'I@docker:swarm and I@prometheus:server')
                def elasticSearchNodes = salt.getMinions(pepperEnv, 'I@elasticsearch:server')
                def debian = new com.mirantis.mk.Debian()
                for (stacklightNode in stacklightNodes) {
                    salt.runSaltProcessStep(pepperEnv, stacklightNode, 'saltutil.refresh_pillar', [], null, true)
                    salt.enforceState([saltId: pepperEnv, target: stacklightNode, state: 'linux.system.repo', output: true, failOnError: true])
                    debian.osUpgradeNode(pepperEnv, stacklightNode, packageUpgradeMode, false, 60)
                    salt.checkTargetMinionsReady(['saltId': pepperEnv, 'target': stacklightNode, wait: 60, timeout: 10])
                    if (packageUpgradeMode == 'dist-upgrade' && stacklightNode in stacklightNodesWithDocker) {
                        common.infoMsg('Perform: Checking if Docker containers are up after reboot')
                        try {
                            common.retry(10, 30) {
                                salt.cmdRun(pepperEnv, stacklightNode, "! docker service ls | tail -n +2 | grep -v -E '\\s([0-9])/\\1\\s'")
                            }
                        }
                        catch (Exception ex) {
                            error("Docker containers for Stacklight services are having troubles with starting.")
                        }
                    }
                    if(stacklightNode in elasticSearchNodes) {
                        verify_es_is_green(pepperEnv)
                    }
                }
            } else {
                common.infoMsg('Upgrade OS skipped...')
            }
        }

        stage('Post upgrade steps') {
            common.infoMsg('Apply workaround for PROD-33878')
            salt.runSaltProcessStep(pepperEnv, "I@fluentd:agent and I@rabbitmq:server", "service.restart", "td-agent", null, true)
        }
    }
}