Merge "Use tcpcloud/salt-models-testing and remove debug prints"
diff --git a/build.gradle b/build.gradle
index 4eb83d0..c1a431a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -3,6 +3,10 @@
def jcenterRepo = System.getenv('ARTIFACTORY_URL') ?: 'https://artifactory.mcp.mirantis.net/jcenter'
+dependencies{
+ compile 'org.codehaus.groovy:groovy-all:2.3.6'
+}
+
sourceSets {
main {
groovy {
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
new file mode 100644
index 0000000..e4550e5
--- /dev/null
+++ b/cloud-deploy-pipeline.groovy
@@ -0,0 +1,315 @@
+/**
+ *
+ * Launch heat/cloudformation stack
+ *
+ * Expected parameters:
+ * STACK_NAME Infrastructure stack name
+ * STACK_TEMPLATE Stack HOT/CFN template
+ * STACK_TYPE Deploy OpenStack/AWS [heat/aws]
+ *
+ * STACK_TEMPLATE_URL URL to git repo with stack templates
+ * STACK_TEMPLATE_CREDENTIALS Credentials to the templates repo
+ * STACK_TEMPLATE_BRANCH Stack templates repo branch
+ *
+ * STACK_DELETE Delete stack when finished (bool)
+ * STACK_REUSE Reuse existing stack (don't create one)
+ * STACK_INSTALL What should be installed (k8s, openstack, ...)
+ * STACK_TEST Run tests (bool)
+ * STACK_CLEANUP_JOB Name of job for deleting stack
+ *
+ * AWS_STACK_REGION CloudFormation AWS region
+ * AWS_API_CREDENTIALS AWS Access key ID with AWS secret access key
+ *
+ * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
+ * HEAT_STACK_ZONE Heat stack availability zone
+ * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
+ * OPENSTACK_API_URL OpenStack API address
+ * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
+ * OPENSTACK_API_PROJECT OpenStack project to connect to
+ * OPENSTACK_API_CLIENT Versions of OpenStack python clients
+ * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
+ *
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ * SALT_MASTER_URL URL of Salt master
+ *
+ * K8S_API_SERVER Kubernetes API address
+ * K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
+ *
+ * TEMPEST_IMAGE_LINK Tempest image link
+ *
+ */
+common = new com.mirantis.mk.Common()
+git = new com.mirantis.mk.Git()
+openstack = new com.mirantis.mk.Openstack()
+orchestrate = new com.mirantis.mk.Orchestrate()
+salt = new com.mirantis.mk.Salt()
+test = new com.mirantis.mk.Test()
+
+_MAX_PERMITTED_STACKS = 2
+overwriteFile = "/srv/salt/reclass/classes/cluster/override.yml"
+
+timestamps {
+ node {
+ try {
+ //
+ // Prepare machines
+ //
+ stage ('Create infrastructure') {
+
+ if (STACK_TYPE == 'heat') {
+ // value defaults
+ def openstackCloud
+ def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
+ def openstackEnv = "${env.WORKSPACE}/venv"
+
+ if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
+ error("If you want to reuse existing stack you need to provide it's name")
+ }
+
+ if (STACK_REUSE.toBoolean() == false) {
+ // Don't allow to set custom heat stack name
+ wrap([$class: 'BuildUser']) {
+ if (env.BUILD_USER_ID) {
+ STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
+ } else {
+ STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
+ }
+ currentBuild.description = STACK_NAME
+ }
+ }
+
+ // set description
+ currentBuild.description = "${STACK_NAME}"
+
+ // get templates
+ git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
+
+ // create openstack env
+ openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
+ openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
+ openstack.getKeystoneToken(openstackCloud, openstackEnv)
+ //
+ // Verify possibility of create stack for given user and stack type
+ //
+ wrap([$class: 'BuildUser']) {
+ if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !STACK_REUSE.toBoolean()) {
+ def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
+ if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
+ STACK_DELETE = "false"
+ throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
+ }
+ }
+ }
+ // launch stack
+ if (STACK_REUSE.toBoolean() == false) {
+ stage('Launch new Heat stack') {
+ // create stack
+ envParams = [
+ 'instance_zone': HEAT_STACK_ZONE,
+ 'public_net': HEAT_STACK_PUBLIC_NET
+ ]
+ openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+ }
+ }
+
+ // get SALT_MASTER_URL
+ saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, STACK_NAME, 'salt_master_ip', openstackEnv)
+ currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
+
+ SALT_MASTER_URL = "http://${saltMasterHost}:6969"
+ }
+
+ if (STACK_TYPE == 'aws') {
+ saltMasterHost = ''
+ currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
+ SALT_MASTER_URL = "http://${saltMasterHost}:6969"
+ }
+
+ }
+
+ //
+ // Connect to Salt master
+ //
+
+ def saltMaster
+ stage('Connect to Salt API') {
+ saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ //
+ // Install
+ //
+
+ if (common.checkContains('STACK_INSTALL', 'core')) {
+ stage('Install core infrastructure') {
+ orchestrate.installFoundationInfra(saltMaster)
+
+ if (common.checkContains('STACK_INSTALL', 'kvm')) {
+ orchestrate.installInfraKvm(saltMaster)
+ orchestrate.installFoundationInfra(saltMaster)
+ }
+
+ orchestrate.validateFoundationInfra(saltMaster)
+ }
+ }
+
+ // install k8s
+ if (common.checkContains('STACK_INSTALL', 'k8s')) {
+ stage('Install Kubernetes infra') {
+ orchestrate.installKubernetesInfra(saltMaster)
+ }
+
+ stage('Install Kubernetes control') {
+
+ // Overwrite Kubernetes vars if specified
+ if (env.getEnvironment().containsKey("KUBERNETES_HYPERKUBE_IMAGE")) {
+ salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_hyperkube_image: ${KUBERNETES_HYPERKUBE_IMAGE}")
+ }
+
+ orchestrate.installKubernetesControl(saltMaster)
+ }
+
+
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
+ state('Install Contrail for Kubernetes') {
+ orchestrate.installContrailNetwork(saltMaster)
+ orchestrate.installContrailCompute(saltMaster)
+ }
+ }
+ }
+
+ // install openstack
+ if (common.checkContains('STACK_INSTALL', 'openstack')) {
+ // install Infra and control, tests, ...
+
+ stage('Install OpenStack infra') {
+ orchestrate.installOpenstackInfra(saltMaster)
+ }
+
+ stage('Install OpenStack control') {
+ orchestrate.installOpenstackControl(saltMaster)
+ }
+
+ stage('Install OpenStack network') {
+
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
+ orchestrate.installContrailNetwork(saltMaster)
+ } else if (common.checkContains('STACK_INSTALL', 'ovs')) {
+ orchestrate.installOpenstackNetwork(saltMaster)
+ }
+
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
+ }
+
+ stage('Install OpenStack compute') {
+ orchestrate.installOpenstackCompute(saltMaster)
+
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
+ orchestrate.installContrailCompute(saltMaster)
+ }
+ }
+
+ }
+
+
+ if (common.checkContains('STACK_INSTALL', 'stacklight')) {
+ stage('Install StackLight') {
+ orchestrate.installStacklightControl(saltMaster)
+ orchestrate.installStacklightClient(saltMaster)
+ }
+ }
+
+ //
+ // Test
+ //
+ def artifacts_dir = '_artifacts/'
+
+ if (common.checkContains('STACK_TEST', 'k8s')) {
+ stage('Run k8s bootstrap tests') {
+ def image = 'tomkukral/k8s-scripts'
+ def output_file = image.replaceAll('/', '-') + '.output'
+
+ // run image
+ test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
+
+ // collect output
+ sh "mkdir -p ${artifacts_dir}"
+ file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
+ writeFile file: "${artifacts_dir}${output_file}", text: file_content
+ sh "cat ${artifacts_dir}${output_file}"
+
+ // collect artifacts
+ archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
+ }
+
+ stage('Run k8s conformance e2e tests') {
+ //test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
+
+ def image = K8S_CONFORMANCE_IMAGE
+ def output_file = image.replaceAll('/', '-') + '.output'
+
+ // run image
+ test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
+
+ // collect output
+ sh "mkdir -p ${artifacts_dir}"
+ file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
+ writeFile file: "${artifacts_dir}${output_file}", text: file_content
+ sh "cat ${artifacts_dir}${output_file}"
+
+ // collect artifacts
+ archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
+ }
+ }
+
+ if (common.checkContains('STACK_TEST', 'openstack')) {
+ stage('Run deployment tests') {
+ test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
+ }
+
+ stage('Copy test results to config node') {
+ test.copyTempestResults(saltMaster)
+ }
+ }
+
+ stage('Finalize') {
+ if (STACK_INSTALL != '') {
+ try {
+ salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
+ } catch (Exception e) {
+ common.warningMsg('State apply failed but we should continue to run')
+ }
+ }
+ }
+ } catch (Throwable e) {
+ currentBuild.result = 'FAILURE'
+ throw e
+ } finally {
+
+
+ //
+ // Clean
+ //
+
+ if (STACK_TYPE == 'heat') {
+ // send notification
+ common.sendNotification(currentBuild.result, STACK_NAME, ["slack"])
+
+ if (STACK_DELETE.toBoolean() == true) {
+ common.errorMsg('Heat job cleanup triggered')
+ stage('Trigger cleanup job') {
+ build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'STACK_NAME', value: STACK_NAME]]
+ }
+ } else {
+ if (currentBuild.result == 'FAILURE') {
+ common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
+ if (SALT_MASTER_URL) {
+ common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index da04f62..11ddc84 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -2,23 +2,24 @@
*
* Launch heat stack with basic k8s
* Flow parameters:
- * STACK_TYPE Orchestration engine: heat, ''
- * INSTALL What should be installed (k8s, openstack, ...)
- * TEST What should be tested (k8s, openstack, ...)
+ * STACK_NAME Heat stack name
+ * STACK_TYPE Orchestration engine: heat, ''
+ * STACK_INSTALL What should be installed (k8s, openstack, ...)
+ * STACK_TEST What should be tested (k8s, openstack, ...)
+ *
+ * STACK_TEMPLATE_URL URL to git repo with stack templates
+ * STACK_TEMPLATE_BRANCH Stack templates repo branch
+ * STACK_TEMPLATE_CREDENTIALS Credentials to the stack templates repo
+ * STACK_TEMPLATE Heat stack HOT template
+ * STACK_DELETE Delete stack when finished (bool)
+ * STACK_REUSE Reuse stack (don't create one)
+ * STACK_CLEANUP_JOB Name of job for deleting Heat stack
*
* Expected parameters:
- *
* required for STACK_TYPE=heat
- * HEAT_TEMPLATE_URL URL to git repo with Heat templates
- * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo
- * HEAT_TEMPLATE_BRANCH Heat templates repo branch
- * HEAT_STACK_TEMPLATE Heat stack HOT template
* HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
* HEAT_STACK_ZONE Heat stack availability zone
* HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
- * HEAT_STACK_DELETE Delete Heat stack when finished (bool)
- * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack
- * HEAT_STACK_REUSE Reuse Heat stack (don't create one)
* OPENSTACK_API_URL OpenStack API address
* OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
* OPENSTACK_API_PROJECT OpenStack project to connect to
@@ -37,6 +38,11 @@
*
* optional parameters for overwriting soft params
* KUBERNETES_HYPERKUBE_IMAGE Docker repository and tag for hyperkube image
+ * CALICO_CNI_IMAGE Docker repository and tag for calico CNI image
+ * CALICO_NODE_IMAGE Docker repository and tag for calico node image
+ * CALICOCTL_IMAGE Docker repository and tag for calicoctl image
+ * NETCHECKER_AGENT_IMAGE Docker repository and tag for netchecker agent image
+ * NETCHECKER_SERVER_IMAGE Docker repository and tag for netchecker server image
*
*/
@@ -64,27 +70,27 @@
def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
def openstackEnv = "${env.WORKSPACE}/venv"
- if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
+ if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
error("If you want to reuse existing stack you need to provide it's name")
}
- if (HEAT_STACK_REUSE.toBoolean() == false) {
+ if (STACK_REUSE.toBoolean() == false) {
// Don't allow to set custom heat stack name
wrap([$class: 'BuildUser']) {
if (env.BUILD_USER_ID) {
- HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
+ STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
} else {
- HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
+ STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
}
- currentBuild.description = HEAT_STACK_NAME
+ currentBuild.description = STACK_NAME
}
}
// set description
- currentBuild.description = "${HEAT_STACK_NAME}"
+ currentBuild.description = "${STACK_NAME}"
// get templates
- git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
+ git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
// create openstack env
openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
@@ -94,29 +100,29 @@
// Verify possibility of create stack for given user and stack type
//
wrap([$class: 'BuildUser']) {
- if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) {
+ if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !STACK_REUSE.toBoolean()) {
def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
- HEAT_STACK_DELETE = "false"
+ STACK_DELETE = "false"
throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
}
}
}
// launch stack
- if (HEAT_STACK_REUSE.toBoolean() == false) {
+ if (STACK_REUSE.toBoolean() == false) {
stage('Launch new Heat stack') {
// create stack
envParams = [
'instance_zone': HEAT_STACK_ZONE,
'public_net': HEAT_STACK_PUBLIC_NET
]
- openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+ openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
}
}
// get SALT_MASTER_URL
- saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
- currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
+ saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, STACK_NAME, 'salt_master_ip', openstackEnv)
+ currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
SALT_MASTER_URL = "http://${saltMasterHost}:6969"
}
@@ -135,11 +141,11 @@
// Install
//
- if (common.checkContains('INSTALL', 'core')) {
+ if (common.checkContains('STACK_INSTALL', 'core')) {
stage('Install core infrastructure') {
orchestrate.installFoundationInfra(saltMaster)
- if (common.checkContains('INSTALL', 'kvm')) {
+ if (common.checkContains('STACK_INSTALL', 'kvm')) {
orchestrate.installInfraKvm(saltMaster)
orchestrate.installFoundationInfra(saltMaster)
}
@@ -149,7 +155,7 @@
}
// install k8s
- if (common.checkContains('INSTALL', 'k8s')) {
+ if (common.checkContains('STACK_INSTALL', 'k8s')) {
stage('Install Kubernetes infra') {
orchestrate.installKubernetesInfra(saltMaster)
}
@@ -160,12 +166,31 @@
if (env.getEnvironment().containsKey("KUBERNETES_HYPERKUBE_IMAGE")) {
salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_hyperkube_image: ${KUBERNETES_HYPERKUBE_IMAGE}")
}
+ // Overwrite Calico vars if specified
+ if (env.getEnvironment().containsKey("CALICO_CNI_IMAGE")) {
+ salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_calico_cni_image: ${CALICO_CNI_IMAGE}")
+ }
+ if (env.getEnvironment().containsKey("CALICO_NODE_IMAGE")) {
+ salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_calico_node_image: ${CALICO_NODE_IMAGE}")
+ }
+ if (env.getEnvironment().containsKey("CALICOCTL_IMAGE")) {
+ salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_calicoctl_image: ${CALICOCTL_IMAGE}")
+ }
+
+ // Overwrite netchecker vars if specified
+ if (env.getEnvironment().containsKey("NETCHECKER_AGENT_IMAGE")) {
+ salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_netchecker_agent_image: ${NETCHECKER_AGENT_IMAGE}")
+ }
+ if (env.getEnvironment().containsKey("NETCHECKER_SERVER_IMAGE")) {
+ salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_netchecker_server_image: ${NETCHECKER_SERVER_IMAGE}")
+ }
+
orchestrate.installKubernetesControl(saltMaster)
}
- if (common.checkContains('INSTALL', 'contrail')) {
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
state('Install Contrail for Kubernetes') {
orchestrate.installContrailNetwork(saltMaster)
orchestrate.installContrailCompute(saltMaster)
@@ -174,7 +199,7 @@
}
// install openstack
- if (common.checkContains('INSTALL', 'openstack')) {
+ if (common.checkContains('STACK_INSTALL', 'openstack')) {
// install Infra and control, tests, ...
stage('Install OpenStack infra') {
@@ -187,9 +212,9 @@
stage('Install OpenStack network') {
- if (common.checkContains('INSTALL', 'contrail')) {
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
orchestrate.installContrailNetwork(saltMaster)
- } else if (common.checkContains('INSTALL', 'ovs')) {
+ } else if (common.checkContains('STACK_INSTALL', 'ovs')) {
orchestrate.installOpenstackNetwork(saltMaster)
}
@@ -200,7 +225,7 @@
stage('Install OpenStack compute') {
orchestrate.installOpenstackCompute(saltMaster)
- if (common.checkContains('INSTALL', 'contrail')) {
+ if (common.checkContains('STACK_INSTALL', 'contrail')) {
orchestrate.installContrailCompute(saltMaster)
}
}
@@ -208,7 +233,7 @@
}
- if (common.checkContains('INSTALL', 'stacklight')) {
+ if (common.checkContains('STACK_INSTALL', 'stacklight')) {
stage('Install StackLight') {
orchestrate.installStacklightControl(saltMaster)
orchestrate.installStacklightClient(saltMaster)
@@ -220,7 +245,7 @@
//
def artifacts_dir = '_artifacts/'
- if (common.checkContains('TEST', 'k8s')) {
+ if (common.checkContains('STACK_TEST', 'k8s')) {
stage('Run k8s bootstrap tests') {
def image = 'tomkukral/k8s-scripts'
def output_file = image.replaceAll('/', '-') + '.output'
@@ -258,7 +283,7 @@
}
}
- if (common.checkContains('TEST', 'openstack')) {
+ if (common.checkContains('STACK_TEST', 'openstack')) {
stage('Run OpenStack tests') {
test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
}
@@ -269,11 +294,12 @@
}
stage('Finalize') {
- if (INSTALL != '') {
+ if (STACK_INSTALL != '') {
try {
salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
} catch (Exception e) {
common.warningMsg('State apply failed but we should continue to run')
+ throw e
}
}
}
@@ -289,12 +315,12 @@
if (STACK_TYPE == 'heat') {
// send notification
- common.sendNotification(currentBuild.result, HEAT_STACK_NAME, ["slack"])
+ common.sendNotification(currentBuild.result, STACK_NAME, ["slack"])
- if (HEAT_STACK_DELETE.toBoolean() == true) {
+ if (STACK_DELETE.toBoolean() == true) {
common.errorMsg('Heat job cleanup triggered')
stage('Trigger cleanup job') {
- build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
+ build job: STACK_CLEANUP_JOB, parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: STACK_NAME]]
}
} else {
if (currentBuild.result == 'FAILURE') {
diff --git a/mk-k8s-simple-deploy-pipeline.groovy b/mk-k8s-simple-deploy-pipeline.groovy
index 48bd064..4aae816 100644
--- a/mk-k8s-simple-deploy-pipeline.groovy
+++ b/mk-k8s-simple-deploy-pipeline.groovy
@@ -1,5 +1,5 @@
/**
- * DO NOT USE THIS OUTDATED PIPELINE - add your steps to mk-lab-pipeline
+ * DO NOT USE THIS OUTDATED PIPELINE - add your steps to lab-pipeline
*
* Launch heat stack with basic k8s
*
diff --git a/mk-lab-pipeline.groovy b/mk-lab-pipeline.groovy
deleted file mode 100644
index 9e9a28d..0000000
--- a/mk-lab-pipeline.groovy
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- *
- * Launch heat stack with basic k8s
- *
- * Expected parameters:
- * HEAT_TEMPLATE_URL URL to git repo with Heat templates
- * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo
- * HEAT_TEMPLATE_BRANCH Heat templates repo branch
- * HEAT_STACK_NAME Heat stack name
- * HEAT_STACK_TEMPLATE Heat stack HOT template
- * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
- * HEAT_STACK_ZONE Heat stack availability zone
- * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
- * HEAT_STACK_DELETE Delete Heat stack when finished (bool)
- * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack
- * HEAT_STACK_REUSE Reuse Heat stack (don't create one)
- *
- * SALT_MASTER_CREDENTIALS Credentials to the Salt API
- *
- * OPENSTACK_API_URL OpenStack API address
- * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
- * OPENSTACK_API_PROJECT OpenStack project to connect to
- * OPENSTACK_API_CLIENT Versions of OpenStack python clients
- * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
- *
- * K8S_API_SERVER Kubernetes API address
- * K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
- *
- * TEMPEST_IMAGE_LINK Tempest image link
- *
- * INSTALL What should be installed (k8s, openstack, ...)
- * TESTS Run tests (bool)
- */
-
-git = new com.mirantis.mk.Git()
-openstack = new com.mirantis.mk.Openstack()
-salt = new com.mirantis.mk.Salt()
-orchestrate = new com.mirantis.mk.Orchestrate()
-test = new com.mirantis.mk.Test()
-
-node {
-
- // connection objects
- def openstackCloud
- def saltMaster
-
- // value defaults
- def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
- def openstackEnv = "${env.WORKSPACE}/venv"
-
- if (HEAT_STACK_NAME == '') {
- HEAT_STACK_NAME = BUILD_TAG
- }
-
- //
- // Bootstrap
- //
-
- stage ('Download Heat templates') {
- git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
- }
-
- stage('Install OpenStack CLI') {
- openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
- }
-
- stage('Connect to OpenStack cloud') {
- openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
- openstack.getKeystoneToken(openstackCloud, openstackEnv)
- }
-
- if (HEAT_STACK_REUSE == 'false') {
- stage('Launch new Heat stack') {
- envParams = [
- 'instance_zone': HEAT_STACK_ZONE,
- 'public_net': HEAT_STACK_PUBLIC_NET
- ]
- openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
- }
- }
-
- stage('Connect to Salt master') {
- saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
- saltMasterUrl = "http://${saltMasterHost}:8088"
- saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
- }
-
- //
- // Install
- //
-
- stage('Install core infra') {
- // salt.master, reclass
- // refresh_pillar
- // sync_all
- // linux,openssh,salt.minion.ntp
-
- orchestrate.installFoundationInfra(saltMaster)
- orchestrate.validateFoundationInfra(saltMaster)
- }
-
-
- if (INSTALL.toLowerCase().contains('k8s')) {
- stage('Install Kubernetes infra') {
- orchestrate.installOpenstackMcpInfra(saltMaster)
- }
-
- stage('Install Kubernetes control') {
- orchestrate.installOpenstackMcpControl(saltMaster)
- }
-
- if (TESTS.toLowerCase().contains('k8s')) {
- stage('Run k8s bootstrap tests') {
- test.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
- }
-
- stage('Run k8s conformance e2e tests') {
- test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
- }
-
- stage("Copy k8s e2e test output to config node ") {
- test.copyTestsOutput(saltMaster,K8S_CONFORMANCE_IMAGE)
- }
- }
- }
-
- if (INSTALL.toLowerCase().contains('openstack')) {
- // install Infra and control, tests, ...
-
- stage('Install OpenStack infra') {
- orchestrate.installOpenstackMkInfra(saltMaster)
- }
-
- stage('Install OpenStack control') {
- orchestrate.installOpenstackMkControl(saltMaster)
- }
-
- stage('Install OpenStack network') {
- orchestrate.installOpenstackMkNetwork(saltMaster)
- }
-
- stage('Install OpenStack compute') {
- orchestrate.installOpenstackMkCompute(saltMaster)
- }
-
- if (TESTS.toLowerCase().contains('openstack')) {
- stage('Run OpenStack tests') {
- test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
- }
-
- stage('Copy Tempest results to config node') {
- test.copyTempestResults(saltMaster)
- }
- }
- }
-
- //
- // Cleanup
- //
-
- if (HEAT_STACK_DELETE == 'true') {
- stage('Trigger cleanup job') {
- build job: 'deploy_heat_cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
- }
- }
-}
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 9e6fd01..53264be 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -26,11 +26,15 @@
if (STAGE_TEST_UPGRADE.toBoolean() == true) {
stage('Test upgrade') {
- //salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
- // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
- // salt '*' saltutil.sync_all
- // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
+ try {
+ salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
+ } catch (Exception e) {
+ common.warningMsg(" Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
+ }
+
+ salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
+ salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
@@ -46,12 +50,8 @@
_pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
def kvm01 = _pillar['return'][0].values()[0].values()[0]
- def kvm03 = _pillar['return'][0].values()[2].values()[0]
- def kvm02 = _pillar['return'][0].values()[1].values()[0]
print(_pillar)
print(kvm01)
- print(kvm02)
- print(kvm03)
_pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
def upgNodeProvider = _pillar['return'][0].values()[0]
@@ -110,7 +110,7 @@
salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
- salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
+ salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
@@ -131,7 +131,7 @@
try {
salt.enforceState(saltMaster, 'upg*', 'keystone.server')
} catch (Exception e) {
- common.warningMsg('Reloading Apache2 and enforcing keystone.server state again')
+ common.warningMsg('Restarting Apache2')
salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
}
try {
@@ -153,6 +153,13 @@
common.warningMsg('running nova state again')
salt.enforceState(saltMaster, 'upg*', 'nova')
}
+ // run nova state again as sometimes nova does not enforce itself for some reason
+ try {
+ salt.enforceState(saltMaster, 'upg*', 'nova')
+ } catch (Exception e) {
+ common.warningMsg('running nova state again')
+ salt.enforceState(saltMaster, 'upg*', 'nova')
+ }
try {
salt.enforceState(saltMaster, 'upg*', 'cinder')
} catch (Exception e) {
@@ -172,12 +179,12 @@
salt.enforceState(saltMaster, 'upg*', 'heat')
}
salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
- }
- }
- if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
- stage('Ask for manual confirmation') {
- input message: "Do you want to continue with upgrade?"
+ if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
+ stage('Ask for manual confirmation') {
+ input message: "Do you want to continue with upgrade?"
+ }
+ }
}
}
@@ -192,12 +199,10 @@
_pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
kvm01 = _pillar['return'][0].values()[0].values()[0]
- kvm03 = _pillar['return'][0].values()[2].values()[0]
- kvm02 = _pillar['return'][0].values()[1].values()[0]
print(_pillar)
print(kvm01)
- print(kvm02)
- print(kvm03)
+
+ def errorOccured = false
_pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
def ctl01NodeProvider = _pillar['return'][0].values()[0]
@@ -257,7 +262,7 @@
salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
- salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
+ salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
try {
salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
@@ -293,7 +298,7 @@
try {
salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
} catch (Exception e) {
- common.warningMsg('Reloading Apache2 and enforcing keystone.server state again')
+ common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
}
@@ -342,6 +347,7 @@
}
} catch (Exception e) {
+ errorOccured = true
common.warningMsg('Some states that require syncdb failed. Restoring production databases')
databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
if(databases && databases != ""){
@@ -355,41 +361,41 @@
}
salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
}else{
- common.errorMsg("No none _upgrade databases were returned. You have to restore production databases before running the real control upgrade again. This is because database schema for some services already happened. To do that delete the production databases and run salt 'I@mysql:client' state.sls mysql.client on the salt-master node")
+ common.errorMsg("No none _upgrade databases were returned. You have to restore production databases before running the real control upgrade again. This is because database schema for some services already happened. To do that delete the production databases, remove none upgrade database files from /root/mysql/flags/ and run salt 'I@mysql:client' state.sls mysql.client on the salt-master node")
}
common.errorMsg("Stage Real control upgrade failed")
}
-
- // salt 'cmp*' cmd.run 'service nova-compute restart'
- salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
+ if(!errorOccured){
+ // salt 'cmp*' cmd.run 'service nova-compute restart'
+ salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
- // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
- // salt 'ctl*' state.sls keepalived
- // salt 'prx*' state.sls keepalived
- salt.enforceState(saltMaster, 'prx*', 'keepalived')
- // salt 'prx*' state.sls horizon
- salt.enforceState(saltMaster, 'prx*', 'horizon')
- // salt 'prx*' state.sls nginx
- salt.enforceState(saltMaster, 'prx*', 'nginx')
+ // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
+ // salt 'ctl*' state.sls keepalived
+ // salt 'prx*' state.sls keepalived
+ salt.enforceState(saltMaster, 'prx*', 'keepalived')
+ // salt 'prx*' state.sls horizon
+ salt.enforceState(saltMaster, 'prx*', 'horizon')
+ // salt 'prx*' state.sls nginx
+ salt.enforceState(saltMaster, 'prx*', 'nginx')
- salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
+ salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
+ }
}
- }
-
-
- if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
- stage('Ask for manual confirmation') {
- input message: "Please verify that control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
- }
- stage('Ask for manual confirmation') {
- input message: "Do you really want to continue with the rollback?"
+ if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
+ stage('Ask for manual confirmation') {
+ input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
+ }
}
}
if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
stage('Rollback upgrade') {
+ stage('Ask for manual confirmation') {
+ input message: "Do you really want to continue with the rollback?"
+ }
+
_pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
domain = _pillar['return'][0].values()[0].values()[0]
print(_pillar)
@@ -397,12 +403,8 @@
_pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
kvm01 = _pillar['return'][0].values()[0].values()[0]
- kvm03 = _pillar['return'][0].values()[2].values()[0]
- kvm02 = _pillar['return'][0].values()[1].values()[0]
print(_pillar)
print(kvm01)
- print(kvm02)
- print(kvm03)
_pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
def ctl01NodeProvider = _pillar['return'][0].values()[0]
diff --git a/test-groovy-pipeline.groovy b/test-groovy-pipeline.groovy
index 918fc98..61dfae7 100644
--- a/test-groovy-pipeline.groovy
+++ b/test-groovy-pipeline.groovy
@@ -28,6 +28,15 @@
node("docker"){
try {
+ stage("stop old tests"){
+ if (gerritRef) {
+ def runningTestBuildNums = _getRunningTriggeredTestsBuildNumbers(env["JOB_NAME"], GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER)
+ for(int i=0; i<runningTestBuildNums.size(); i++){
+ common.infoMsg("Old test with run number ${runningTestBuildNums[i]} found, stopping")
+ Jenkins.instance.getItemByFullName(env["JOB_NAME"]).getBuildByNumber(runningTestBuildNums[i]).finish(hudson.model.Result.ABORTED, new java.io.IOException("Aborting build"));
+ }
+ }
+ }
stage ('Checkout source code'){
if (gerritRef) {
// job is triggered by Gerrit
@@ -72,3 +81,14 @@
common.sendNotification(currentBuild.result, "" ,["slack"])
}
}
+@NonCPS
+def _getRunningTriggeredTestsBuildNumbers(jobName, gerritChangeNumber, excludePatchsetNumber){
+ def gerrit = new com.mirantis.mk.Gerrit()
+ def jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
+ def triggeredBuilds= gerrit.getGerritTriggeredBuilds(jenkinsUtils.getJobRunningBuilds(jobName), gerritChangeNumber, excludePatchsetNumber)
+ def buildNums =[]
+ for(int i=0;i<triggeredBuilds.size();i++){
+ buildNums.add(triggeredBuilds[i].number)
+ }
+ return buildNums
+}
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 87c4f51..6baa25e 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -3,6 +3,7 @@
* DEFAULT_GIT_REF
* DEFAULT_GIT_URL
* CREDENTIALS_ID
+ * KITCHEN_TESTS_PARALLEL
*/
def common = new com.mirantis.mk.Common()
def gerrit = new com.mirantis.mk.Gerrit()
@@ -62,9 +63,39 @@
if (fileExists(".kitchen.yml")) {
common.infoMsg(".kitchen.yml found, running kitchen tests")
ruby.ensureRubyEnv()
- ruby.installKitchen()
+ def kitchenEnvs = []
+ if(fileExists(".travis.yml")){
+ common.infoMsg(".travis.yml found, running custom kitchen init")
+ def kitchenConfigYML = readYaml(file: ".travis.yml")
+ kitchenEnvs=kitchenConfigYML["env"]
+ def kitchenInit = kitchenConfigYML["install"]
+ def kitchenInstalled = false
+ if(kitchenInit && !kitchenInit.isEmpty()){
+ for(int i=0;i<kitchenInit.size();i++){
+ if(kitchenInit[i].trim().startsWith("test -e Gemfile")){ //found Gemfile config
+ common.infoMsg("Custom Gemfile configuration found, using them")
+ ruby.installKitchen(kitchenInit[i].trim())
+ kitchenInstalled = true
+ }
+ }
+ }
+ if(!kitchenInstalled){
+ ruby.installKitchen()
+ }
+ }else{
+ common.infoMsg(".travis.yml not found, running default kitchen init")
+ ruby.installKitchen()
+ }
wrap([$class: 'AnsiColorBuildWrapper']) {
- ruby.runKitchenTests()
+ common.infoMsg("Running kitchen testing, parallel mode: " + KITCHEN_TESTS_PARALLEL.toBoolean())
+ if(!kitchenEnvs.isEmpty()){
+ for(int i=0;i<kitchenEnvs.size();i++){
+ common.infoMsg("Found multiple environment, kitchen running with env: " + kitchenEnvs[i])
+ ruby.runKitchenTests(kitchenEnvs[i], KITCHEN_TESTS_PARALLEL.toBoolean())
+ }
+ }else{
+ ruby.runKitchenTests("", KITCHEN_TESTS_PARALLEL.toBoolean())
+ }
}
} else {
common.infoMsg(".kitchen.yml not found")
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index 2c894cd..0936113 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -4,6 +4,7 @@
* DEFAULT_GIT_REF
* DEFAULT_GIT_URL
* CREDENTIALS_ID
+ * EXTRA_FORMULAS
*/
def common = new com.mirantis.mk.Common()
@@ -30,6 +31,15 @@
def merged = false
node("python&&docker") {
try{
+ stage("stop old tests"){
+ if (gerritRef) {
+ def runningTestBuildNums = _getRunningTriggeredTestsBuildNumbers(env["JOB_NAME"], GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER)
+ for(int i=0; i<runningTestBuildNums.size(); i++){
+ common.infoMsg("Old test with run number ${runningTestBuildNums[i]} found, stopping")
+ Jenkins.instance.getItemByFullName(env["JOB_NAME"]).getBuildByNumber(runningTestBuildNums[i]).finish(hudson.model.Result.ABORTED, new java.io.IOException("Aborting build"));
+ }
+ }
+ }
stage("checkout") {
if (gerritRef) {
// job is triggered by Gerrit
@@ -127,7 +137,7 @@
sh("git config --global user.name || git config --global user.name 'CI'")
// setup iniot and verify salt master and minions
- withEnv(["FORMULAS_SOURCE=pkg", "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "MINION_ID=${masterName}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
+ withEnv(["FORMULAS_SOURCE=pkg", "EXTRA_FORMULAS=${EXTRA_FORMULAS}", "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "MINION_ID=${masterName}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
sh("bash -c 'echo $MASTER_HOSTNAME'")
sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && system_config'")
sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_bootstrap'")
@@ -153,4 +163,16 @@
def testMinion(minionName)
{
sh("service salt-master restart && service salt-minion restart && sleep 5 && bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'")
+}
+
+@NonCPS
+def _getRunningTriggeredTestsBuildNumbers(jobName, gerritChangeNumber, excludePatchsetNumber){
+ def gerrit = new com.mirantis.mk.Gerrit()
+ def jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
+ def triggeredBuilds= gerrit.getGerritTriggeredBuilds(jenkinsUtils.getJobRunningBuilds(jobName), gerritChangeNumber, excludePatchsetNumber)
+ def buildNums =[]
+ for(int i=0;i<triggeredBuilds.size();i++){
+ buildNums.add(triggeredBuilds[i].number)
+ }
+ return buildNums
}
\ No newline at end of file