Merge "Improved error handling in test pipelines."
diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index 972ab81..a9f4edd 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -120,7 +120,9 @@
stage("Deploy GlusterFS") {
salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
- salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
+ retry(2) {
+ salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
+ }
sleep(5)
salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
print common.prettyPrint(salt.cmdRun(saltMaster, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"'))
@@ -140,33 +142,37 @@
print common.prettyPrint(salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'docker node ls'))
}
+ stage("Configure OSS services") {
+ salt.enforceState(saltMaster, 'I@devops_portal:config', 'devops_portal.config')
+ }
+
stage("Deploy Docker services") {
salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
-
- // XXX: Hack to fix dependency of gerrit on mysql
- print common.prettyPrint(salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', "docker service rm gerrit; sleep 5; rm -rf /srv/volumes/gerrit/*"))
-
- timeout(10) {
- salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'apt-get install -y mysql-client')
- println "Waiting for MySQL to come up.."
- salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'while true; do mysql -h172.16.10.254 -ppassword -e"show status;" >/dev/null && break; done')
- }
- salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
- // ---- cut here (end of hack) ----
}
stage("Configure CI/CD services") {
salt.syncAll(saltMaster, '*')
// Aptly
+ timeout(10) {
+ println "Waiting for Aptly to come up.."
+ salt.cmdRun(saltMaster, 'I@aptly:server', 'while true; do curl -svf http://172.16.10.254:8084/api/version >/dev/null && break; done')
+ }
salt.enforceState(saltMaster, 'I@aptly:server', 'aptly', true)
+ // OpenLDAP
+ timeout(10) {
+ println "Waiting for OpenLDAP to come up.."
+ salt.cmdRun(saltMaster, 'I@openldap:client', 'while true; do curl -svf ldap://172.16.10.254 >/dev/null && break; done')
+ }
+ salt.enforceState(saltMaster, 'I@openldap:client', 'openldap', true)
+
// Gerrit
timeout(10) {
println "Waiting for Gerrit to come up.."
salt.cmdRun(saltMaster, 'I@gerrit:client', 'while true; do curl -svf 172.16.10.254:8080 >/dev/null && break; done')
}
- retry(2) {
+ retry(3) {
// Needs to run twice to pass __virtual__ method of gerrit module
// after installation of dependencies
try {
@@ -259,9 +265,12 @@
And visit services running at 172.16.10.254 (vip address):
- 9600 haproxy stats
- 8080 gerrit
- 8081 jenkins
+ 9600 HAProxy statistics
+ 8080 Gerrit
+ 8081 Jenkins
+ 8089 LDAP administration
+ 4440 Rundeck
+ 8084 DevOps Portal
8091 Docker swarm visualizer
8090 Reclass-generated documentation
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
new file mode 100644
index 0000000..d836afb
--- /dev/null
+++ b/generate-cookiecutter-products.groovy
@@ -0,0 +1,157 @@
+/**
+ * Generate cookiecutter cluster by individual products
+ *
+ * Expected parameters:
+ * COOKIECUTTER_TEMPLATE_CREDENTIALS Credentials to the Cookiecutter template repo.
+ * COOKIECUTTER_TEMPLATE_URL Cookiecutter template repo address.
+ * COOKIECUTTER_TEMPLATE_BRANCH Branch for the template.
+ * COOKIECUTTER_TEMPLATE_CONTEXT Context parameters for the template generation.
+ * COOKIECUTTER_INSTALL_CICD Whether to install CI/CD stack.
+ * COOKIECUTTER_INSTALL_CONTRAIL Whether to install OpenContrail SDN.
+ * COOKIECUTTER_INSTALL_KUBERNETES Whether to install Kubernetes.
+ * COOKIECUTTER_INSTALL_OPENSTACK Whether to install OpenStack cloud.
+ * COOKIECUTTER_INSTALL_STACKLIGHT Whether to install StackLight monitoring.
+ * RECLASS_MODEL_URL Reclass model repo address
+ * RECLASS_MODEL_CREDENTIALS Credentials to the Reclass model repo.
+ * RECLASS_MODEL_BRANCH Branch for the template to push to model.
+ * COMMIT_CHANGES Commit model to repo
+ *
+**/
+
+common = new com.mirantis.mk.Common()
+git = new com.mirantis.mk.Git()
+python = new com.mirantis.mk.Python()
+
+timestamps {
+ node() {
+ def templateEnv = "${env.WORKSPACE}/template"
+ def modelEnv = "${env.WORKSPACE}/model"
+
+ try {
+ def templateContext = python.loadJson(COOKIECUTTER_TEMPLATE_CONTEXT)
+ def templateDir = "${templateEnv}/template/dir"
+ def templateOutputDir = "${env.WORKSPACE}/template"
+ def cutterEnv = "${env.WORKSPACE}/cutter"
+ def jinjaEnv = "${env.WORKSPACE}/jinja"
+ def clusterName = templateContext.cluster_name
+ def clusterDomain = templateContext.cluster_domain
+ def targetBranch = "feature/${clusterName}"
+ def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
+
+ stage ('Download Cookiecutter template') {
+ git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, COOKIECUTTER_TEMPLATE_CREDENTIALS)
+ }
+
+ stage ('Download full Reclass model') {
+ git.checkoutGitRepository(modelEnv, RECLASS_MODEL_URL, RECLASS_MODEL_BRANCH, RECLASS_MODEL_CREDENTIALS)
+ }
+
+ stage('Generate base infrastructure') {
+ templateDir = "${templateEnv}/cluster_product/infra"
+ templateOutputDir = "${env.WORKSPACE}/template/output/infra"
+ sh "mkdir -p ${templateOutputDir}"
+ sh "mkdir -p ${outputDestination}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, templateContext, templateOutputDir, cutterEnv)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+
+ stage('Generate product CI/CD') {
+ if (COOKIECUTTER_INSTALL_CICD.toBoolean()) {
+ templateDir = "${templateEnv}/cluster_product/cicd"
+ templateOutputDir = "${env.WORKSPACE}/template/output/cicd"
+ sh "mkdir -p ${templateOutputDir}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, templateContext, templateOutputDir, cutterEnv)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+ }
+
+ stage('Generate product OpenContrail') {
+ if (COOKIECUTTER_INSTALL_CONTRAIL.toBoolean()) {
+ templateDir = "${templateEnv}/cluster_product/opencontrail"
+ templateOutputDir = "${env.WORKSPACE}/template/output/opencontrail"
+ sh "mkdir -p ${templateOutputDir}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, templateContext, templateOutputDir, cutterEnv)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+ }
+
+ stage('Generate product Kubernetes') {
+ if (COOKIECUTTER_INSTALL_KUBERNETES.toBoolean()) {
+ templateDir = "${templateEnv}/cluster_product/kubernetes"
+ templateOutputDir = "${env.WORKSPACE}/template/output/kubernetes"
+ sh "mkdir -p ${templateOutputDir}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, templateContext, templateOutputDir, cutterEnv)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+ }
+
+ stage('Generate product OpenStack') {
+ if (COOKIECUTTER_INSTALL_OPENSTACK.toBoolean()) {
+ templateDir = "${templateEnv}/cluster_product/openstack"
+ templateOutputDir = "${env.WORKSPACE}/template/output/openstack"
+ sh "mkdir -p ${templateOutputDir}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, templateContext, templateOutputDir, cutterEnv)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+ }
+
+ stage('Generate product StackLight') {
+ if (COOKIECUTTER_INSTALL_STACKLIGHT.toBoolean()) {
+ templateDir = "${templateEnv}/cluster_product/stacklight"
+ templateOutputDir = "${env.WORKSPACE}/template/output/stacklight"
+ sh "mkdir -p ${templateOutputDir}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, templateContext, templateOutputDir, cutterEnv)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+ }
+
+ stage('Generate new SaltMaster node') {
+ def nodeFile = "${modelEnv}/nodes/cfg01.${clusterDomain}.yml"
+ def nodeString = """classes:
+- cluster.${clusterName}.infra.config
+parameters:
+ _param:
+ linux_system_codename: xenial
+ reclass_data_revision: master
+ linux:
+ system:
+ name: cfg01
+ domain: ${clusterDomain}
+"""
+ writeFile(file: nodeFile, text: nodeString)
+ }
+
+ stage('Inject changes to Reclass model') {
+ git.changeGitBranch(modelEnv, targetBranch)
+ def outputSource = "${env.WORKSPACE}/template/output/"
+ sh(returnStdout: true, script: "cp -vr ${outputSource} ${outputDestination}")
+ }
+
+ stage ('Save changes to Reclass model') {
+ if (COMMIT_CHANGES.toBoolean()) {
+ git.commitGitChanges(modelEnv, "Added new cluster ${clusterName}")
+ git.pushGitChanges(modelEnv, targetBranch, 'origin', RECLASS_MODEL_CREDENTIALS)
+ }
+ sh(returnStatus: true, script: "tar -zcvf ${clusterName}.tar.gz -C ${modelEnv} .")
+ archiveArtifacts artifacts: "${clusterName}.tar.gz"
+ }
+
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ } finally {
+ stage ('Clean workspace directories') {
+ sh(returnStatus: true, script: "rm -rfv ${templateEnv}")
+ sh(returnStatus: true, script: "rm -rfv ${modelEnv}")
+ }
+ // common.sendNotification(currentBuild.result,"",["slack"])
+ }
+ }
+}
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index a6efaca..5b7cc11 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -41,7 +41,7 @@
openstack = new com.mirantis.mk.Openstack()
salt = new com.mirantis.mk.Salt()
common = new com.mirantis.mk.Common()
-
+test = new com.mirantis.mk.Test()
timestamps {
node {
@@ -50,6 +50,7 @@
// Prepare machines
//
stage ('Create infrastructure') {
+
if (STACK_TYPE == 'heat') {
// value defaults
def openstackCloud
@@ -99,7 +100,7 @@
saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
- if (INSTALL.toLowerCase().contains('kvm')) {
+ if (common.checkContains('INSTALL', 'kvm')) {
saltPort = 6969
} else {
saltPort = 6969
@@ -122,7 +123,7 @@
// Install
//
- if (INSTALL.toLowerCase().contains('core')) {
+ if (common.checkContains('INSTALL', 'core')) {
stage('Install core infrastructure') {
// salt.master, reclass
// refresh_pillar
@@ -131,17 +132,19 @@
//orchestrate.installFoundationInfra(saltMaster)
salt.enforceState(saltMaster, 'I@salt:master', ['salt.master', 'reclass'], true)
+ salt.enforceState(saltMaster, '*', ['linux.system'], true)
+ salt.enforceState(saltMaster, '*', ['salt.minion'], true)
salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
- if (INSTALL.toLowerCase().contains('kvm')) {
+ if (common.checkContains('INSTALL', 'kvm')) {
//orchestrate.installInfraKvm(saltMaster)
- salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
+ //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
+ //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
- salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
+ //salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
@@ -167,7 +170,7 @@
}
// install k8s
- if (INSTALL.toLowerCase().contains('k8s')) {
+ if (common.checkContains('INSTALL', 'k8s')) {
stage('Install Kubernetes infra') {
//orchestrate.installOpenstackMcpInfra(saltMaster)
@@ -235,7 +238,7 @@
}
// install openstack
- if (INSTALL.toLowerCase().contains('openstack')) {
+ if (common.checkContains('INSTALL', 'openstack')) {
// install Infra and control, tests, ...
stage('Install OpenStack infra') {
@@ -258,14 +261,19 @@
salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'], null, true)
// Install rabbitmq
- salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true, false)
-
+ withEnv(['ASK_ON_ERROR=false']){
+ retry(2) {
+ salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
+ }
+ }
// Check the rabbitmq status
salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
// Install galera
- retry(2) {
- salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
+ withEnv(['ASK_ON_ERROR=false']){
+ retry(2) {
+ salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
+ }
}
salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
@@ -348,7 +356,7 @@
stage('Install OpenStack network') {
//orchestrate.installOpenstackMkNetwork(saltMaster, physical)
- if (INSTALL.toLowerCase().contains('contrail')) {
+ if (common.checkContains('INSTALL', 'contrail')) {
// Install opencontrail database services
//runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
try {
@@ -371,7 +379,7 @@
// Test opencontrail
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'], null, true)
- } else if (INSTALL.toLowerCase().contains('ovs')) {
+ } else if (common.checkContains('INSTALL', 'ovs')) {
// Apply gateway
salt.runSaltProcessStep(saltMaster, 'I@neutron:gateway', 'state.apply', [], null, true)
}
@@ -388,7 +396,7 @@
salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply', [], null, true)
}
- if (INSTALL.toLowerCase().contains('contrail')) {
+ if (common.checkContains('INSTALL', 'contrail')) {
// Provision opencontrail control services
salt.enforceState(saltMaster, 'I@opencontrail:database:id:1', 'opencontrail.client', true)
// Provision opencontrail virtual routers
@@ -402,7 +410,7 @@
}
- if (INSTALL.toLowerCase().contains('stacklight')) {
+ if (common.checkContains('INSTALL', 'stacklight')) {
stage('Install StackLight') {
// infra install
// Install the StackLight backends
@@ -506,7 +514,7 @@
// Test
//
- if (TEST.toLowerCase().contains('k8s')) {
+ if (common.checkContains('TEST', 'k8s')) {
stage('Run k8s bootstrap tests') {
orchestrate.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
}
@@ -516,7 +524,7 @@
}
}
- if (TEST.toLowerCase().contains('openstack')) {
+ if (common.checkContains('TEST', 'openstack')) {
stage('Run OpenStack tests') {
test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
}
@@ -538,14 +546,15 @@
throw e
} finally {
- // send notification
- common.sendNotification(currentBuild.result,HEAT_STACK_NAME,["slack"])
//
// Clean
//
if (STACK_TYPE == 'heat') {
+ // send notification
+ common.sendNotification(currentBuild.result, HEAT_STACK_NAME, ["slack"])
+
if (HEAT_STACK_DELETE.toBoolean() == true) {
common.errorMsg('Heat job cleanup triggered')
stage('Trigger cleanup job') {
diff --git a/mk-k8s-simple-deploy-pipeline.groovy b/mk-k8s-simple-deploy-pipeline.groovy
index c82771f..02f7709 100644
--- a/mk-k8s-simple-deploy-pipeline.groovy
+++ b/mk-k8s-simple-deploy-pipeline.groovy
@@ -86,7 +86,7 @@
}
if (RUN_TESTS == "1") {
- sleep(30000)
+ sleep(30)
stage('Run k8s bootstrap tests') {
test.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
}
diff --git a/test-nodejs-pipeline.groovy b/test-nodejs-pipeline.groovy
new file mode 100644
index 0000000..87b78d3
--- /dev/null
+++ b/test-nodejs-pipeline.groovy
@@ -0,0 +1,61 @@
+/**
+* JS testing pipeline
+* CREDENTIALS_ID - gerrit credentials id
+* NODE_IMAGE - NodeJS with NPM Docker image name
+* COMMANDS - a list of command(s) to run
+**/
+
+gerrit = new com.mirantis.mk.Gerrit()
+common = new com.mirantis.mk.Common()
+
+def executeCmd(containerId, cmd) {
+ stage(cmd) {
+ assert containerId != null
+ common.infoMsg("Starting command: ${cmd}")
+ def output = sh(
+ script: "docker exec ${containerId} ${cmd}",
+ returnStdout: true,
+ )
+ common.infoMsg(output)
+ common.successMsg("Successfully completed: ${cmd}")
+ }
+}
+
+node("docker") {
+ def containerId
+ try {
+ stage('Checkout source code') {
+ gerrit.gerritPatchsetCheckout ([
+ credentialsId : CREDENTIALS_ID,
+ withWipeOut : true,
+ ])
+ }
+ stage('Start container') {
+ def workspace = common.getWorkspace()
+ containerId = sh(
+ script: "docker run -d ${NODE_IMAGE}",
+ returnStdout: true,
+ ).trim()
+ common.successMsg("Container with id ${containerId} started.")
+ sh("docker cp ${workspace}/ ${containerId}:/opt/workspace/")
+ }
+ executeCmd(containerId, "npm install")
+ def cmds = COMMANDS.tokenize('\n')
+ for (int i = 0; i < cmds.size(); i++) {
+ executeCmd(containerId, cmds[i])
+ }
+ } catch (err) {
+ currentBuild.result = 'FAILURE'
+ common.errorMsg("Build failed due to error: ${err}")
+ throw err
+ } finally {
+ common.sendNotification(currentBuild.result, "" ,["slack"])
+ stage('Cleanup') {
+ if (containerId != null) {
+ sh("docker stop -t 0 ${containerId}")
+ sh("docker rm ${containerId}")
+ common.infoMsg("Container with id ${containerId} was removed.")
+ }
+ }
+ }
+}
diff --git a/update-package.groovy b/update-package.groovy
index 6c31c95..b37fe22 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -24,6 +24,7 @@
def result
def packages
def command
+def commandKwargs
node() {
try {
@@ -70,15 +71,17 @@
}
if (TARGET_PACKAGES != "") {
- command = "pkg.install";
+ command = "pkg.install"
packages = TARGET_PACKAGES.tokenize(' ')
+ commandKwargs = ['only_upgrade': 'true']
}else {
command = "pkg.upgrade"
packages = null
}
stage('Apply package upgrades on sample') {
- salt.runSaltProcessStep(saltMaster, targetLiveSubset, command, packages, null, true)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, packages, commandKwargs)
+ salt.printSaltCommandResult(out)
}
stage('Confirm package upgrades on all nodes') {
@@ -88,7 +91,8 @@
}
stage('Apply package upgrades on all nodes') {
- salt.runSaltProcessStep(saltMaster, targetLiveAll, command, packages, null, true)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, packages, commandKwargs)
+ salt.printSaltCommandResult(out)
}
} catch (Throwable e) {