Merge "Revert "Only release-engineering can promote release images""
diff --git a/gating-pipeline.groovy b/gating-pipeline.groovy
index e1c80f0..eeb9f71 100644
--- a/gating-pipeline.groovy
+++ b/gating-pipeline.groovy
@@ -39,6 +39,7 @@
ssh.ensureKnownHosts(GERRIT_HOST)
def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID, true)
def doSubmit = false
+ def skipProjectsVerify = ['mk/docker-jnlp-slave']
stage("test") {
if (gerritChange.status != "MERGED" && !SKIP_TEST.equals("true")) {
// test max CodeReview
@@ -46,30 +47,35 @@
doSubmit = true
def gerritProjectArray = GERRIT_PROJECT.tokenize("/")
def gerritProject = gerritProjectArray[gerritProjectArray.size() - 1]
- def jobsNamespace = JOBS_NAMESPACE
- def plural_namespaces = ['salt-formulas', 'salt-models']
- // remove plural s on the end of job namespace
- if (JOBS_NAMESPACE in plural_namespaces) {
- jobsNamespace = JOBS_NAMESPACE.substring(0, JOBS_NAMESPACE.length() - 1)
- }
- // salt-formulas tests have -latest on end of the name
- if (JOBS_NAMESPACE.equals("salt-formulas")) {
- gerritProject = gerritProject + "-latest"
- }
- def testJob = String.format("test-%s-%s", jobsNamespace, gerritProject)
- if (env.GERRIT_PROJECT == 'mk/cookiecutter-templates' || env.GERRIT_PROJECT == 'salt-models/reclass-system') {
- callJobWithExtraVars('test-salt-model-ci-wrapper')
+ if (gerritProject in skipProjectsVerify) {
+ common.successMsg("Project ${gerritProject} doesn't require verify, skipping...")
+ giveVerify = true
} else {
- if (isJobExists(testJob)) {
- common.infoMsg("Test job ${testJob} found, running")
- def patchsetVerified = gerrit.patchsetHasApproval(gerritChange.currentPatchSet, "Verified", "+")
- build job: testJob, parameters: [
- [$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: "${GERRIT_SCHEME}://${GERRIT_NAME}@${GERRIT_HOST}:${GERRIT_PORT}/${GERRIT_PROJECT}"],
- [$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: GERRIT_REFSPEC]
- ]
- giveVerify = true
+ def jobsNamespace = JOBS_NAMESPACE
+ def plural_namespaces = ['salt-formulas', 'salt-models']
+ // remove plural s on the end of job namespace
+ if (JOBS_NAMESPACE in plural_namespaces) {
+ jobsNamespace = JOBS_NAMESPACE.substring(0, JOBS_NAMESPACE.length() - 1)
+ }
+ // salt-formulas tests have -latest on end of the name
+ if (JOBS_NAMESPACE.equals("salt-formulas")) {
+ gerritProject = gerritProject + "-latest"
+ }
+ def testJob = String.format("test-%s-%s", jobsNamespace, gerritProject)
+ if (env.GERRIT_PROJECT == 'mk/cookiecutter-templates' || env.GERRIT_PROJECT == 'salt-models/reclass-system') {
+ callJobWithExtraVars('test-salt-model-ci-wrapper')
} else {
- common.infoMsg("Test job ${testJob} not found")
+ if (isJobExists(testJob)) {
+ common.infoMsg("Test job ${testJob} found, running")
+ def patchsetVerified = gerrit.patchsetHasApproval(gerritChange.currentPatchSet, "Verified", "+")
+ build job: testJob, parameters: [
+ [$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: "${GERRIT_SCHEME}://${GERRIT_NAME}@${GERRIT_HOST}:${GERRIT_PORT}/${GERRIT_PROJECT}"],
+ [$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: GERRIT_REFSPEC]
+ ]
+ giveVerify = true
+ } else {
+ common.infoMsg("Test job ${testJob} not found")
+ }
}
}
} else {
diff --git a/stacklight-upgrade.groovy b/stacklight-upgrade.groovy
index 443d56b..0ce3e26 100644
--- a/stacklight-upgrade.groovy
+++ b/stacklight-upgrade.groovy
@@ -21,15 +21,15 @@
def upgrade(master, target, service, pckg, state) {
stage("Upgrade ${service}") {
- salt.runSaltProcessStep(master, "${target}", 'saltutil.refresh_pillar', [], null, true, 5)
+ salt.runSaltProcessStep(master, "${target}", 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(master, "${target}", 'linux.system.repo', true)
- common.infoMsg("Upgrade ${service} package")
+ common.infoMsg("Upgrade ${service} package(s)")
try {
- salt.runSaltProcessStep(master, "${target}", command, ["apt-get install --only-upgrade ${pckg}"], null, true)
+ salt.runSaltProcessStep(master, "${target}", command, ["apt-get install -y -o Dpkg::Options::=\"--force-confold\" ${pckg}"], null, true)
} catch (Exception er) {
errorOccured = true
- common.errorMsg("[ERROR] ${pckg} package was not upgraded.")
- return
+ common.errorMsg("[ERROR] ${pckg} package(s) was not upgraded.")
+ throw er
}
common.infoMsg("Run ${state} state on ${target} nodes")
try {
@@ -37,72 +37,120 @@
} catch (Exception er) {
errorOccured = true
common.errorMsg("[ERROR] ${state} state was executed and failed. Please fix it manually.")
+ throw er
}
- common.infoMsg("Check ${service} service status on the target nodes")
- salt.runSaltProcessStep(master, "${target}", "service.status", ["${service}"], null, true)
- return
+ common.infoMsg("Check ${service} service(s) status on the target nodes")
+ for (s in service.split(" ")){
+ salt.runSaltProcessStep(master, "${target}", "service.status", "${s}", null, true)
+ }
+ }
+}
+
+def verify_es_is_green(master) {
+ common.infoMsg('Verify that the Elasticsearch cluster status is green')
+ try {
+ def retries_wait = 20
+ def retries = 15
+ def elasticsearch_vip
+ def pillar = salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:host')
+ if(!pillar['return'].isEmpty()) {
+ elasticsearch_vip = pillar['return'][0].values()[0]
+ } else {
+ errorOccured = true
+ common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
+ }
+ pillar = salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:port')
+ def elasticsearch_port
+ if(!pillar['return'].isEmpty()) {
+ elasticsearch_port = pillar['return'][0].values()[0]
+ } else {
+ errorOccured = true
+ common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
+ }
+ common.retry(retries,retries_wait) {
+ common.infoMsg('Waiting for Elasticsearch to become green..')
+ salt.cmdRun(master, "I@elasticsearch:client", "curl -sf ${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
+ }
+ } catch (Exception er) {
+ errorOccured = true
+ common.errorMsg("[ERROR] Elasticsearch cluster status is not \'green\'. Please fix it manually.")
+ throw er
}
}
def upgrade_es_kibana(master) {
+ def elasticsearch_version
+ def es_pillar = salt.getPillar(master, "I@elasticsearch:client", '_param:elasticsearch_version')
+ if(!es_pillar['return'].isEmpty()) {
+ elasticsearch_version = es_pillar['return'][0].values()[0]
+ }
stage('Upgrade elasticsearch') {
- try {
- common.infoMsg('Upgrade the Elasticsearch package')
- salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
- salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["apt-get --only-upgrade install elasticsearch"], null, true)
- salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl daemon-reload"], null, true)
- salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl start elasticsearch"], null, true)
- salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
- } catch (Exception er) {
- errorOccured = true
- common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
- return
- }
- common.infoMsg('Verify that the Elasticsearch cluster status is green')
- try {
- def retries_wait = 20
- def retries = 15
- def elasticsearch_vip
- def pillar = salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:host')
- if(!pillar['return'].isEmpty()) {
- elasticsearch_vip = pillar['return'][0].values()[0]
- } else {
+ if (elasticsearch_version == '5') {
+ try {
+ common.infoMsg('Upgrade the Elasticsearch package')
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["apt-get --only-upgrade install elasticsearch"], null, true)
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl daemon-reload"], null, true)
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl start elasticsearch"], null, true)
+ salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
+ verify_es_is_green(master)
+ } catch (Exception er) {
errorOccured = true
- common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
+ common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
+ throw er
}
- pillar = salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:port')
- def elasticsearch_port
- if(!pillar['return'].isEmpty()) {
- elasticsearch_port = pillar['return'][0].values()[0]
- } else {
+ } else {
+ try {
+ salt.runSaltProcessStep(master, "*", 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(master, "I@elasticsearch:server", 'linux.system.repo', true)
+ salt.runSaltProcessStep(master, 'I@elasticsearch:client', command, ["apt-get install -y -o Dpkg::Options::=\"--force-confold\" python-elasticsearch"], null, true)
+ salt.enforceState(master, "I@elasticsearch:server", 'salt.minion', true)
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["systemctl stop elasticsearch"], null, true)
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', command, ["export ES_PATH_CONF=/etc/elasticsearch; apt-get install -y -o Dpkg::Options::=\"--force-confold\" elasticsearch"], null, true)
+ salt.enforceState(master, "I@elasticsearch:server", 'elasticsearch.server', true)
+ verify_es_is_green(master)
+ salt.enforceState(master, "I@elasticsearch:client", 'elasticsearch.client.update_index_templates', true)
+ salt.enforceState(master, "I@elasticsearch:client", 'elasticsearch.client', true)
+ } catch (Exception er) {
errorOccured = true
- common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
+ common.errorMsg("[ERROR] Elasticsearch upgrade failed. Please fix it manually.")
+ throw er
}
- common.retry(retries,retries_wait) {
- common.infoMsg('Waiting for Elasticsearch to become green..')
- salt.cmdRun(master, "I@elasticsearch:client", "curl -sf ${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
- }
- } catch (Exception er) {
- errorOccured = true
- common.errorMsg("[ERROR] Elasticsearch cluster status is not \'green\'. Please fix it manually.")
- return
}
}
stage('Upgrade kibana') {
- try {
- common.infoMsg('Upgrade the Kibana package')
- salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
- salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get --only-upgrade install kibana"], null, true)
- salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl start kibana"], null, true)
- } catch (Exception er) {
- errorOccured = true
- common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
- return
+ def kibana_version
+ def kibana_pillar = salt.getPillar(master, "I@kibana:client", '_param:kibana_version')
+ if(!kibana_pillar['return'].isEmpty()) {
+ kibana_version = kibana_pillar['return'][0].values()[0]
+ }
+ if (kibana_version == '5') {
+ try {
+ common.infoMsg('Upgrade the Kibana package')
+ salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
+ salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get --only-upgrade install kibana"], null, true)
+ salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl start kibana"], null, true)
+ } catch (Exception er) {
+ errorOccured = true
+ common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
+ throw er
+ }
+ } else {
+ try {
+ salt.runSaltProcessStep(master, 'I@kibana:server', command, ["systemctl stop kibana"], null, true)
+ salt.enforceStateWithExclude(pepperEnv, "I@kibana:server", "kibana.server", "[{'id': 'kibana_service'}]")
+ salt.runSaltProcessStep(master, 'I@kibana:server', command, ["apt-get install -y -o Dpkg::Options::=\"--force-confold\" kibana"], null, true)
+ salt.enforceState(master, "I@kibana:server", 'kibana.server', true)
+ salt.enforceState(master, "I@kibana:client", 'kibana.client', true)
+ } catch (Exception er) {
+ errorOccured = true
+ common.errorMsg("[ERROR] Kibana upgrade failed. Please fix it manually.")
+ throw er
+ }
}
common.infoMsg("Check kibana status on the target nodes")
salt.runSaltProcessStep(master, "I@kibana:server", "service.status", ["kibana"], null, true)
- return
}
}
timeout(time: 12, unit: 'HOURS') {
@@ -112,11 +160,18 @@
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+ salt.enforceState(pepperEnv, '*', 'salt.minion.grains')
+ salt.runSaltProcessStep(pepperEnv, '*', 'saltutil.refresh_modules')
+ salt.runSaltProcessStep(pepperEnv, '*', 'mine.update')
+ sleep(30)
+
+
if (STAGE_UPGRADE_SYSTEM_PART.toBoolean() == true && !errorOccured) {
upgrade(pepperEnv, "I@telegraf:agent or I@telegraf:remote_agent", "telegraf", "telegraf", "telegraf")
upgrade(pepperEnv, "I@fluentd:agent", "td-agent", "td-agent td-agent-additional-plugins", "fluentd")
if (salt.testTarget(pepperEnv, "I@prometheus:relay")) {
- upgrade(pepperEnv, "I@prometheus:relay", "prometheus-relay", "prometheus-relay", "prometheus")
+ upgrade(pepperEnv, "I@prometheus:relay", "prometheus prometheus-relay", "prometheus-bin prometheus-relay", "prometheus")
+ salt.runSaltProcessStep(pepperEnv, "I@prometheus:relay", "service.restart", "prometheus", null, true)
}
if (salt.testTarget(pepperEnv, "I@prometheus:exporters:libvirt")) {
upgrade(pepperEnv, "I@prometheus:exporters:libvirt", "libvirt-exporter", "libvirt-exporter", "prometheus")
@@ -149,7 +204,7 @@
} catch (Exception er) {
errorOccured = true
common.errorMsg("[ERROR] Upgrade of docker components failed. Please fix it manually.")
- return
+ throw er
}
}
}
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 52c7d79..404c0d0 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -106,8 +106,8 @@
img.inside("-v ${env.WORKSPACE}/:/formula/ -u root:root --cpus=4 --ulimit nofile=4096:8192") {
sh('''#!/bin/bash -xe
cd /etc/apt/
- echo "deb [arch=amd64] http://cz.archive.ubuntu.com/ubuntu xenial main restricted universe" > sources.list
- echo "deb [arch=amd64] http://cz.archive.ubuntu.com/ubuntu xenial-updates main restricted universe" >> sources.list
+ echo "deb [arch=amd64] http://mirror.mirantis.com/nightly/ubuntu xenial main restricted universe" > sources.list
+ echo "deb [arch=amd64] http://mirror.mirantis.com/nightly/ubuntu xenial-updates main restricted universe" >> sources.list
echo 'Acquire::Languages "none";' > apt.conf.d/docker-no-languages
echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > apt.conf.d/docker-gzip-indexes
echo 'APT::Get::Install-Recommends "false"; APT::Get::Install-Suggests "false";' > apt.conf.d/docker-recommends