Merge "Add INTERACTIVE flag to upgrade pipelines"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index e4847ee..7f7337a 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -342,9 +342,8 @@
}
if (common.checkContains('STACK_INSTALL', 'contrail')) {
- stage('Install Contrail for Kubernetes') {
+ stage('Install Contrail control') {
orchestrate.installContrailNetwork(venvPepper)
- orchestrate.installContrailCompute(venvPepper)
}
}
@@ -356,6 +355,12 @@
archiveArtifacts(artifacts: 'kubeconfig')
}
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
+ stage('Install Contrail compute') {
+ orchestrate.installContrailCompute(venvPepper)
+ }
+ }
+
stage('Install Kubernetes computes') {
if (common.validInputParam('STACK_COMPUTE_COUNT')) {
if (STACK_COMPUTE_COUNT > 0) {
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 8c8b8b6..b6a44b6 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -425,6 +425,22 @@
}
}
+def periodicCheck(pepperEnv, target, maxRetries=50) {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def count = 0
+ while(count < maxRetries) {
+ try {
+ sleep(10)
+ salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+ break
+ } catch (Exception e) {
+ common.warningMsg("${target} not ready yet. Waiting ...")
+ }
+ count++
+ }
+}
+
def highstate(pepperEnv, target, type) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
@@ -457,9 +473,17 @@
// optionally reboot
if (reboots.contains(type)) {
stage("Reboot ${target} nodes") {
- salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
- sleep(10)
- salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+ if (type == 'cfg') {
+ try {
+ salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
+ } catch (Exception e) {
+ periodicCheck(pepperEnv, target)
+ }
+ } else {
+ salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
+ sleep 10
+ salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+ }
}
}
}
@@ -683,13 +707,26 @@
def verifyAPIs(pepperEnv, target) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
- def out = salt.cmdRun(pepperEnv, target, '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
- if (out.toString().toLowerCase().contains('error')) {
- common.errorMsg(out)
- if (INTERACTIVE.toBoolean()) {
- input message: "APIs are not working as expected. Please fix it manually."
- } else {
- throw new Exception("APIs are not working as expected")
+ def cmds = ["openstack service list",
+ "openstack image list",
+ "openstack flavor list",
+ "openstack compute service list",
+ "openstack server list",
+ "openstack network list",
+ "openstack volume list",
+ "openstack orchestration service list"]
+ def sourcerc = ". /root/keystonercv3;"
+ def cmdOut = ">/dev/null 2>&1;echo \$?"
+ for (c in cmds) {
+ def command = sourcerc + c + cmdOut
+ def out = salt.cmdRun(pepperEnv, target, "${command}")
+ if (!out.toString().toLowerCase().contains('0')) {
+ common.errorMsg(out)
+ if (INTERACTIVE.toBoolean()) {
+ input message: "APIs are not working as expected. Please fix it manually."
+ } else {
+ throw new Exception("APIs are not working as expected")
+ }
}
}
}
@@ -700,7 +737,7 @@
def out
while(count < maxRetries) {
try {
- out = salt.getReturnValues(salt.cmdRun(pepperEnv, target, 'salt-call mysql.status | grep -A1 wsrep_cluster_size'))
+ out = salt.getReturnValues(salt.cmdRun(pepperEnv, target, 'salt-call -l quiet mysql.status | grep -A1 wsrep_cluster_size'))
} catch (Exception er) {
common.infoMsg(er)
}
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index fb8b0e6..0c5c971 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -6,25 +6,58 @@
* SALT_MASTER_URL Salt API server location
* SALT_MASTER_CREDENTIALS Credentials to the Salt API
* MCP_VERSION Version of MCP to upgrade to
+ * UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
+ * UPDATE_PIPELINES Update pipeline repositories on Gerrit
* UPDATE_LOCAL_REPOS Update local repositories
*/
+import hudson.model.*
+
// Load shared libs
salt = new com.mirantis.mk.Salt()
common = new com.mirantis.mk.Common()
python = new com.mirantis.mk.Python()
+jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
venvPepper = "venvPepper"
+def triggerMirrorJob(jobName){
+ params = jenkinsUtils.getJobParameters(jobName)
+ build job: jobName, parameters: [
+ [$class: 'StringParameterValue', name: 'BRANCHES', value: params.get("BRANCHES")],
+ [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get("CREDENTIALS_ID")],
+ [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get("SOURCE_URL")],
+ [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get("TARGET_URL")]
+ ]
+}
+
timeout(time: 12, unit: 'HOURS') {
node("python") {
try {
python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ if(MCP_VERSION == ""){
+ error("You must specify MCP version")
+ }
+
stage("Update Reclass"){
- common.infoMsg("Updating reclass model")
- salt.cmdRun(venvPepper, "I@salt:master", 'cd /srv/salt/reclass && git pull -r && git submodule update', false)
- salt.cmdRun(venvPepper, 'I@salt:master', 'reclass-salt --top')
- salt.enforceState(venvPepper, "I@salt:master", 'reclass', true)
+ def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
+ if(UPDATE_CLUSTER_MODEL.toBoolean()){
+ try{
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/ && git diff-index --quiet HEAD --")
+ }
+ catch(Exception ex){
+ error("You have uncommited changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.")
+ }
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -r --exclude-dir=aptly -l 'apt_mk_version: .*' * | xargs sed -i 's/apt_mk_version: .*/apt_mk_version: \"$MCP_VERSION\"/g'")
+ }
+
+ try{
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git diff-index --quiet HEAD --")
+ }
+ catch(Exception ex){
+ error("You have unstaged changes in your Reclass system model repository. Please reset them and rerun the pipeline.")
+ }
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout $MCP_VERSION")
}
if(UPDATE_LOCAL_REPOS.toBoolean()){
@@ -41,6 +74,8 @@
common.infoMsg("Aptly isn't running as Docker container. Going to use aptly user for executing aptly commands")
}
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/cicd/aptly && git checkout $MCP_VERSION")
+
if(runningOnDocker){
salt.cmdRun(venvPepper, 'I@aptly:server', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true)
}
@@ -69,17 +104,49 @@
}
}
- stage("Update APT repos"){
- common.infoMsg("Updating APT repositories")
- salt.enforceState(venvPepper, "I@linux:system", 'linux.system.repo', true)
- }
+ stage("Update Drivetrain"){
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_salt.list")
+ salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_salt.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
+ salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades salt-formula-*")
+ salt.fullRefresh(venvPepper, 'I@salt:master')
- stage("Update formulas"){
- common.infoMsg("Updating salt formulas")
- salt.cmdRun(venvPepper, "I@salt:master", 'apt-get clean && apt-get update && apt-get install -y salt-formula-*')
+ try{
+ salt.enforceState(venvPepper, "I@salt:master", 'reclass', true)
+ }
+ catch(Exception ex){
+ error("Reclass fails rendering. Pay attention to your cluster model.")
+ }
- common.infoMsg("Running salt sync-all")
- salt.runSaltProcessStep(venvPepper, '*', 'saltutil.sync_all', [], null, true)
+ salt.fullRefresh(venvPepper, '*')
+
+ try{
+ salt.cmdRun(venvPepper, 'I@salt:master', "reclass-salt --top")
+ }
+ catch(Exception ex){
+ error("Reclass fails rendering. Pay attention to your cluster model.")
+ }
+
+ if(UPDATE_PIPELINES.toBoolean()){
+ triggerMirrorJob("git-mirror-downstream-mk-pipelines")
+ triggerMirrorJob("git-mirror-downstream-pipeline-library")
+ }
+
+ salt.enforceState(venvPepper, "I@jenkins:client", 'jenkins.client', true)
+
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client' state.sls docker.client --async")
+
+ sleep(180)
+
+ common.infoMsg("Checking if Docker containers are up")
+
+ try{
+ common.retry(10, 30){
+ salt.cmdRun(venvPepper, 'I@jenkins:client and I@docker:client', "! docker service ls | tail -n +2 | grep -v -E '\\s([0-9])/\\1\\s'")
+ }
+ }
+ catch(Exception ex){
+ error("Docker containers for CI/CD services are having troubles with starting.")
+ }
}
}
catch (Throwable e) {
diff --git a/validate-cloud.groovy b/validate-cloud.groovy
index 4a2d893..7d3b2e2 100644
--- a/validate-cloud.groovy
+++ b/validate-cloud.groovy
@@ -15,7 +15,8 @@
* TEMPEST_VERSION Version of Tempest (tag, branch or commit)
* RUN_TEMPEST_TESTS If not false, run Tempest tests
* RUN_RALLY_TESTS If not false, run Rally tests
- * RUN_K8S_TESTS If not false, run Kubernetes tests
+ * K8S_RALLY If not false, run Kubernetes Rally tests
+ * RUN_K8S_TESTS If not false, run Kubernetes e2e/conformance tests
* RUN_SPT_TESTS If not false, run SPT tests
* SPT_SSH_USER The name of the user which should be used for ssh to nodes
* SPT_IMAGE The name of the image for SPT tests
@@ -25,6 +26,8 @@
* FLOATING_NETWORK The name of the external(floating) network
* RALLY_IMAGE The name of the image for Rally tests
* RALLY_FLAVOR The name of the flavor for Rally image
+ * RALLY_PLUGINS_REPO Git repository with Rally plugins
+ * RALLY_PLUGINS_BRANCH Git branch which will be used during the checkout
* RALLY_CONFIG_REPO Git repository with files for Rally
* RALLY_CONFIG_BRANCH Git branch which will be used during the checkout
* RALLY_SCENARIOS Path to file or directory with rally scenarios
@@ -36,6 +39,7 @@
* GENERATE_REPORT If not false, run report generation command
* ACCUMULATE_RESULTS If true, results from the previous build will be used
* JOB_TIMEOUT Job timeout in hours
+ *
*/
common = new com.mirantis.mk.Common()
@@ -76,11 +80,20 @@
stage('Run Rally tests') {
if (RUN_RALLY_TESTS.toBoolean() == true) {
def report_dir = env.REPORT_DIR ?: '/root/qa_results'
- def rally_variables = ["floating_network=${FLOATING_NETWORK}",
- "rally_image=${RALLY_IMAGE}",
- "rally_flavor=${RALLY_FLAVOR}",
- "availability_zone=${AVAILABILITY_ZONE}"]
- validate.runRallyTests(pepperEnv, TARGET_NODE, TEST_IMAGE, artifacts_dir, RALLY_CONFIG_REPO, RALLY_CONFIG_BRANCH, RALLY_SCENARIOS, RALLY_TASK_ARGS_FILE, rally_variables, report_dir)
+ def platform
+ def rally_variables
+ if (K8S_RALLY.toBoolean() == false) {
+ platform = 'openstack'
+ rally_variables = ["floating_network=${FLOATING_NETWORK}",
+ "rally_image=${RALLY_IMAGE}",
+ "rally_flavor=${RALLY_FLAVOR}",
+ "availability_zone=${AVAILABILITY_ZONE}"]
+ } else {
+ platform = 'k8s'
+ rally_variables = ["plugins_repo":"${RALLY_PLUGINS_REPO}",
+ "plugins_branch":"${RALLY_PLUGINS_BRANCH}"]
+ }
+ validate.runRallyTests(pepperEnv, TARGET_NODE, TEST_IMAGE, platform, artifacts_dir, RALLY_CONFIG_REPO, RALLY_CONFIG_BRANCH, RALLY_SCENARIOS, RALLY_TASK_ARGS_FILE, rally_variables, report_dir)
} else {
common.infoMsg("Skipping Rally tests")
}
@@ -100,7 +113,7 @@
}
}
- stage('Run k8s bootstrap tests') {
+ stage('Run K8S bootstrap tests') {
if (RUN_K8S_TESTS.toBoolean() == true) {
def image = 'tomkukral/k8s-scripts'
def output_file = 'k8s-bootstrap-tests.txt'
@@ -114,7 +127,7 @@
}
}
- stage('Run k8s conformance e2e tests') {
+ stage('Run K8S conformance e2e tests') {
if (RUN_K8S_TESTS.toBoolean() == true) {
def image = TEST_K8S_CONFORMANCE_IMAGE
def output_file = 'report-k8s-e2e-tests.txt'
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
index 60b8caf..da3d463 100644
--- a/xtrabackup-restore-mysql-db.groovy
+++ b/xtrabackup-restore-mysql-db.groovy
@@ -81,6 +81,12 @@
}
sleep(5)
salt.cmdRun(pepperEnv, 'I@galera:master', "su root -c 'salt-call mysql.status | grep -A1 wsrep_cluster_size'")
+
+ try {
+ salt.runSaltProcessStep(pepperEnv, 'I@galera:master or I@galera:slave', 'file.touch', ["/var/lib/mysql/.galera_bootstrap"], null, true)
+ } catch (Exception er) {
+ common.warningMsg('File is already present')
+ }
}
}
}