Merge "Add more deps in salt model testing"
diff --git a/build-debian-packages-pipeline.groovy b/build-debian-packages-pipeline.groovy
index 91ddb16..bc4ed38 100644
--- a/build-debian-packages-pipeline.groovy
+++ b/build-debian-packages-pipeline.groovy
@@ -63,7 +63,7 @@
stage("build-binary") {
dsc = sh script: "ls build-area/*.dsc", returnStdout: true
if(common.validInputParam("PRE_BUILD_SCRIPT")) {
- writeFile([file:"pre-build-script.sh", text: env['PRE_BUILD_SCRIPT']])
+ writeFile([file:"pre_build_script.sh", text: env['PRE_BUILD_SCRIPT']])
}
debian.buildBinary(
dsc.trim(),
diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index 9077df7..4c45d6e 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -31,7 +31,7 @@
openstack = new com.mirantis.mk.Openstack()
salt = new com.mirantis.mk.Salt()
orchestrate = new com.mirantis.mk.Orchestrate()
-
+_MAX_PERMITTED_STACKS = 2
timestamps {
node {
try {
@@ -80,6 +80,15 @@
stage('Connect to OpenStack cloud') {
openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
openstack.getKeystoneToken(openstackCloud, openstackEnv)
+ wrap([$class: 'BuildUser']) {
+ if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) {
+ def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
+ if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
+ HEAT_STACK_DELETE = "false"
+ throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
+ }
+ }
+ }
}
if (HEAT_STACK_REUSE.toBoolean() == false) {
@@ -189,7 +198,7 @@
salt.enforceState(saltMaster, 'I@gerrit:client', 'gerrit', true)
} catch (Exception e) {
common.infoMsg("Restarting Salt minion")
- salt.cmdRun(saltMaster, 'I@gerrit:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
+ salt.cmdRun(saltMaster, 'I@gerrit:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &", false)
sleep(5)
throw e
}
diff --git a/heavy-lab-pipeline.groovy b/heavy-lab-pipeline.groovy
deleted file mode 100644
index fa42445..0000000
--- a/heavy-lab-pipeline.groovy
+++ /dev/null
@@ -1,598 +0,0 @@
-/**
- *
- * Launch heat stack with basic k8s
- * Flow parameters:
- * STACK_TYPE Orchestration engine: heat, ''
- * INSTALL What should be installed (k8s, openstack, ...)
- * TEST What should be tested (k8s, openstack, ...)
- *
- * Expected parameters:
- *
- * required for STACK_TYPE=heat
- * HEAT_TEMPLATE_URL URL to git repo with Heat templates
- * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo
- * HEAT_TEMPLATE_BRANCH Heat templates repo branch
- * HEAT_STACK_TEMPLATE Heat stack HOT template
- * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
- * HEAT_STACK_ZONE Heat stack availability zone
- * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
- * HEAT_STACK_DELETE Delete Heat stack when finished (bool)
- * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack
- * HEAT_STACK_REUSE Reuse Heat stack (don't create one)
- * OPENSTACK_API_URL OpenStack API address
- * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
- * OPENSTACK_API_PROJECT OpenStack project to connect to
- * OPENSTACK_API_CLIENT Versions of OpenStack python clients
- * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
- *
- * SALT_MASTER_CREDENTIALS Credentials to the Salt API
- *
- * required for STACK_TYPE=NONE or empty string
- * SALT_MASTER_URL URL of Salt-API
-
- * K8S_API_SERVER Kubernetes API address
- * K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
- *
- * TEMPEST_IMAGE_LINK Tempest image link
- *
- */
-
-git = new com.mirantis.mk.Git()
-openstack = new com.mirantis.mk.Openstack()
-salt = new com.mirantis.mk.Salt()
-common = new com.mirantis.mk.Common()
-test = new com.mirantis.mk.Test()
-
-timestamps {
- node {
- try {
- //
- // Prepare machines
- //
- stage ('Create infrastructure') {
-
- if (STACK_TYPE == 'heat') {
- // value defaults
- def openstackCloud
- def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
- def openstackEnv = "${env.WORKSPACE}/venv"
-
- if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
- error("If you want to reuse existing stack you need to provide it's name")
- }
-
- if (HEAT_STACK_REUSE.toBoolean() == false) {
- // Don't allow to set custom heat stack name
- wrap([$class: 'BuildUser']) {
- if (env.BUILD_USER_ID) {
- HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
- } else {
- HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
- }
- currentBuild.description = HEAT_STACK_NAME
- }
- }
-
- // set description
- currentBuild.description = "${HEAT_STACK_NAME}"
-
- // get templates
- git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
-
- // create openstack env
- openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
- openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
- openstack.getKeystoneToken(openstackCloud, openstackEnv)
-
- // launch stack
- if (HEAT_STACK_REUSE.toBoolean() == false) {
- stage('Launch new Heat stack') {
- // create stack
- envParams = [
- 'instance_zone': HEAT_STACK_ZONE,
- 'public_net': HEAT_STACK_PUBLIC_NET
- ]
- openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
- }
- }
-
- // get SALT_MASTER_URL
- saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
- currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
-
- if (common.checkContains('INSTALL', 'kvm')) {
- saltPort = 6969
- } else {
- saltPort = 6969
- }
-
- SALT_MASTER_URL = "http://${saltMasterHost}:${saltPort}"
- }
- }
-
- //
- // Connect to Salt master
- //
-
- def saltMaster
- stage('Connect to Salt API') {
- saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
- }
-
- //
- // Install
- //
-
- if (common.checkContains('INSTALL', 'core')) {
- stage('Install core infrastructure') {
- // orchestrate.installFoundationInfra(saltMaster)
- salt.enforceState(saltMaster, 'I@salt:master', ['salt.master', 'reclass'], true)
- salt.enforceState(saltMaster, '*', ['linux.system'], true)
- salt.enforceState(saltMaster, '*', ['salt.minion'], true)
- salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
- salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
-
-
- if (common.checkContains('INSTALL', 'kvm')) {
- // orchestrate.installInfraKvm(saltMaster)
- // orchestrate.installFoundationInfra(saltMaster)
- //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
- //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
-
- //salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
- salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
- salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
-
- sleep(600)
-
- salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.refresh_pillar', [], null, true)
- salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.sync_all', [], null, true)
-
- // workaround - install apt-transport-https
- //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'cmd.run', ['apt-get update -y && apt-get install -y apt-transport-https'], null, true)
- //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'pkg.install', ['apt-transport-https', 'refresh=True'], null, true)
- salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'salt.minion'], true, false)
- salt.enforceState(saltMaster, 'I@linux:system', ['openssh', 'salt.minion', 'ntp'], true, false)
- }
-
- // orchestrate.validateFoundationInfra(saltMaster)
- salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['salt-key'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'test.version', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['reclass-salt --top'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@reclass:storage', 'reclass.inventory', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'state.show_top', [], null, true)
- }
- }
-
- // install k8s
- if (common.checkContains('INSTALL', 'k8s')) {
- stage('Install Kubernetes infra') {
- // orchestrate.installKubernetesInfra(saltMaster)
-
- // Install glusterfs
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
-
- // Install keepalived
- salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster and *01*', 'state.sls', ['keepalived'])
- salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
-
- // Setup glusterfs
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server and *01*', 'state.sls', ['glusterfs.server.setup'])
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
-
- // Install haproxy
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
-
- // Install docker
- salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'state.sls', ['docker.host'])
- salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'cmd.run', ['docker ps'])
-
- // Install etcd
- salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
- salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'cmd.run', ['./var/lib/etcd/configenv && etcdctl cluster-health'])
-
- }
-
- stage('Install Kubernetes control') {
- // orchestrate.installKubernetesControl(saltMaster)
-
- // Install Kubernetes pool and Calico
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes.master.kube-addons'])
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
-
- // Setup etcd server
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['etcd.server.setup'])
-
- // Run k8s without master.setup
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
-
- // Run k8s master setup
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['kubernetes.master.setup'])
-
- // Restart kubelet
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
- }
-
- }
-
- // install openstack
- if (common.checkContains('INSTALL', 'openstack')) {
- // install Infra and control, tests, ...
-
- stage('Install OpenStack infra') {
- // orchestrate.installOpenstackInfra(saltMaster, physical)
-
- // Install glusterfs
- salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
-
- // Install keepaliveds
- //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
- salt.enforceState(saltMaster, 'I@keepalived:cluster and *01*', 'keepalived', true)
- salt.enforceState(saltMaster, 'I@keepalived:cluster', 'keepalived', true)
-
- // Check the keepalived VIPs
- salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
-
- salt.enforceState(saltMaster, 'I@glusterfs:server and *01*', 'glusterfs.server.setup', true)
-
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'], null, true)
-
- // Install rabbitmq
- withEnv(['ASK_ON_ERROR=false']){
- retry(2) {
- salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
- }
- }
- // Check the rabbitmq status
- salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
-
- // Install galera
- withEnv(['ASK_ON_ERROR=false']){
- retry(2) {
- salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
- }
- }
- salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
-
- // Check galera status
- salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'mysql.status')
- salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'mysql.status')
-
- // // Setup mysql client
- // salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client', true)
-
- // Install haproxy
- salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy', true)
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
-
- // Install memcached
- salt.enforceState(saltMaster, 'I@memcached:server', 'memcached', true)
-
- }
-
- stage('Install OpenStack control') {
- // orchestrate.installOpenstackControl(saltMaster)
-
- // Install horizon dashboard
- salt.enforceState(saltMaster, 'I@horizon:server', 'horizon', true)
- salt.enforceState(saltMaster, 'I@nginx:server', 'nginx', true)
-
- // setup keystone service
- //runSaltProcessStep(saltMaster, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
- salt.enforceState(saltMaster, 'I@keystone:server and *01*', 'keystone.server', true)
- salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
- // populate keystone services/tenants/roles/users
-
- // keystone:client must be called locally
- //salt.runSaltProcessStep(saltMaster, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'service.restart', ['apache2'])
- sleep(30)
- salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
- salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonercv3; openstack service list'], null, true)
-
- // Install glance and ensure glusterfs clusters
- //runSaltProcessStep(saltMaster, 'I@glance:server', 'state.sls', ['glance.server'], 1)
- salt.enforceState(saltMaster, 'I@glance:server and *01*', 'glance.server', true)
- salt.enforceState(saltMaster, 'I@glance:server', 'glance.server', true)
- salt.enforceState(saltMaster, 'I@glance:server', 'glusterfs.client', true)
-
- // Update fernet tokens before doing request on keystone server
- salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
-
- // Check glance service
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'], null, true)
-
- // Install and check nova service
- //runSaltProcessStep(saltMaster, 'I@nova:controller', 'state.sls', ['nova'], 1)
- salt.enforceState(saltMaster, 'I@nova:controller and *01*', 'nova.controller', true)
- salt.enforceState(saltMaster, 'I@nova:controller', 'nova.controller', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'], null, true)
-
- // Install and check cinder service
- //runSaltProcessStep(saltMaster, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
- salt.enforceState(saltMaster, 'I@cinder:controller and *01*', 'cinder', true)
- salt.enforceState(saltMaster, 'I@cinder:controller', 'cinder', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'], null, true)
-
- // Install neutron service
- //runSaltProcessStep(saltMaster, 'I@neutron:server', 'state.sls', ['neutron'], 1)
-
- salt.enforceState(saltMaster, 'I@neutron:server and *01*', 'neutron.server', true)
- salt.enforceState(saltMaster, 'I@neutron:server', 'neutron.server', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'], null, true)
-
- // Install heat service
- //runSaltProcessStep(saltMaster, 'I@heat:server', 'state.sls', ['heat'], 1)
- salt.enforceState(saltMaster, 'I@heat:server and *01*', 'heat', true)
- salt.enforceState(saltMaster, 'I@heat:server', 'heat', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'], null, true)
-
- // Restart nova api
- salt.runSaltProcessStep(saltMaster, 'I@nova:controller', 'service.restart', ['nova-api'])
-
- }
-
- stage('Install OpenStack network') {
-
- if (common.checkContains('INSTALL', 'contrail')) {
- // orchestrate.installContrailNetwork(saltMaster)
- // Install opencontrail database services
- //runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
- try {
- salt.enforceState(saltMaster, 'I@opencontrail:database and *01*', 'opencontrail.database', true)
- } catch (Exception e) {
- common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database and *01*')
- }
-
- try {
- salt.enforceState(saltMaster, 'I@opencontrail:database', 'opencontrail.database', true)
- } catch (Exception e) {
- common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database')
- }
-
- // Install opencontrail control services
- //runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
-
- // Test opencontrail
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'], null, true)
- } else if (common.checkContains('INSTALL', 'ovs')) {
- // orchestrate.installOpenstackNetwork(saltMaster)
- // Apply gateway
- salt.runSaltProcessStep(saltMaster, 'I@neutron:gateway', 'state.apply', [], null, true)
- }
-
- // Pring information
- // where is this
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'], null, true)
- }
-
- stage('Install OpenStack compute') {
- // orchestrate.installOpenstackCompute(saltMaster)
- // Configure compute nodes
- retry(2) {
- salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply', [], null, true)
- }
-
- if (common.checkContains('INSTALL', 'contrail')) {
- // orchestrate.installContrailCompute(saltMaster)
- // Provision opencontrail control services
- salt.enforceState(saltMaster, 'I@opencontrail:database:id:1', 'opencontrail.client', true)
- // Provision opencontrail virtual routers
- salt.enforceState(saltMaster, 'I@opencontrail:compute', 'opencontrail.client', true)
-
- salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'cmd.run', ['exec 0>&-; exec 1>&-; exec 2>&-; nohup bash -c "ip link | grep vhost && echo no_reboot || sleep 5 && reboot & "'], null, true)
- sleep(300)
- }
- }
-
- }
-
-
- if (common.checkContains('INSTALL', 'stacklight')) {
- stage('Install StackLight') {
- //TODO: orchestrate.installStacklightControl(saltMaster)
- //TODO: orchestrate.installStacklightClient(saltMaster)
-
- // infra install
- // Install the StackLight backends
- salt.enforceState(saltMaster, '*01* and I@elasticsearch:server', 'elasticsearch.server', true)
- salt.enforceState(saltMaster, 'I@elasticsearch:server', 'elasticsearch.server', true)
-
- salt.enforceState(saltMaster, '*01* and I@influxdb:server', 'influxdb', true)
- salt.enforceState(saltMaster, 'I@influxdb:server', 'influxdb', true)
-
- salt.enforceState(saltMaster, '*01* and I@kibana:server', 'kibana.server', true)
- salt.enforceState(saltMaster, 'I@kibana:server', 'kibana.server', true)
-
- salt.enforceState(saltMaster, '*01* and I@grafana:server','grafana.server', true)
- salt.enforceState(saltMaster, 'I@grafana:server','grafana.server', true)
-
- salt.enforceState(saltMaster, 'I@nagios:server', 'nagios.server', true)
- salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client.service', true)
- salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client.service', true)
- // nw salt.enforceState('I@kibana:client or I@elasticsearch:client' --async service.restart salt-minion
-
- sleep(10)
-
- salt.runSaltProcessStep(saltMaster, 'I@elasticsearch.client', 'cmd.run', ['salt-call state.sls elasticsearch.client'], null, true)
- // salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client', true)
- salt.runSaltProcessStep(saltMaster, 'I@kibana.client', 'cmd.run', ['salt-call state.sls kibana.client'], null, true)
- // salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client', true)
-
- // install monitor
- // Restart salt-minion to make sure that it uses the latest Jinja library
- // no way: salt '*' --async service.restart salt-minion; sleep 15
-
- // Start by flusing Salt Mine to make sure it is clean
- // Also clean-up the grains files to make sure that we start from a clean state
- // nw salt "*" mine.flush
- // nw salt "*" file.remove /etc/salt/grains.d/collectd
- // nw salt "*" file.remove /etc/salt/grains.d/grafana
- // nw salt "*" file.remove /etc/salt/grains.d/heka
- // nw salt "*" file.remove /etc/salt/grains.d/sensu
- // nw salt "*" file.remove /etc/salt/grains
-
- // Install collectd, heka and sensu services on the nodes, this will also
- // generate the metadata that goes into the grains and eventually into Salt Mine
- salt.enforceState(saltMaster, '*', 'collectd', true)
- salt.enforceState(saltMaster, '*', 'salt.minion', true)
- salt.enforceState(saltMaster, '*', 'heka', true)
-
- // Gather the Grafana metadata as grains
- salt.enforceState(saltMaster, 'I@grafana:collector', 'grafana.collector', true)
-
- // Update Salt Mine
- salt.enforceState(saltMaster, '*', 'salt.minion.grains', true)
- salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_modules', [], null, true)
- salt.runSaltProcessStep(saltMaster, '*', 'mine.update', [], null, true)
-
- sleep(5)
-
- // Update Heka
- salt.enforceState(saltMaster, 'I@heka:aggregator:enabled:True or I@heka:remote_collector:enabled:True', 'heka', true)
-
- // Update collectd
- salt.enforceState(saltMaster, 'I@collectd:remote_client:enabled:True', 'collectd', true)
-
- // Update Nagios
- salt.enforceState(saltMaster, 'I@nagios:server', 'nagios', true)
- // Stop the Nagios service because the package starts it by default and it will
- // started later only on the node holding the VIP address
- salt.runSaltProcessStep(saltMaster, 'I@nagios:server', 'service.stop', ['nagios3'], null, true)
-
- // Update Sensu
- // TODO for stacklight team, should be fixed in model
- //salt.enforceState(saltMaster, 'I@sensu:server', 'sensu', true)
-
- // Finalize the configuration of Grafana (add the dashboards...)
- salt.enforceState(saltMaster, 'I@grafana:client', 'grafana.client.service', true)
- // nw salt -C 'I@grafana:client' --async service.restart salt-minion; sleep 10
-
- salt.runSaltProcessStep(saltMaster, 'I@grafana.client and *01*', 'cmd.run', ['salt-call state.sls grafana.client'], null, true)
- // salt.enforceState(saltMaster, 'I@grafana:client and *01*', 'grafana.client', true)
-
- // Get the StackLight monitoring VIP addres
- //vip=$(salt-call pillar.data _param:stacklight_monitor_address --out key|grep _param: |awk '{print $2}')
- //vip=${vip:=172.16.10.253}
- def pillar = salt.getPillar(saltMaster, 'ctl01*', '_param:stacklight_monitor_address')
- print(common.prettyPrint(pillar))
- def stacklight_vip = pillar['return'][0].values()[0]
-
- if (stacklight_vip) {
- // (re)Start manually the services that are bound to the monitoring VIP
- common.infoMsg("restart services on node with IP: ${stacklight_vip}")
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collectd'], null, true)
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collector'], null, true)
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['aggregator'], null, true)
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['nagios3'], null, true)
- } else {
- throw new Exception("Missing stacklight_vip")
- }
- }
- }
-
- //
- // Test
- //
- def artifacts_dir = '_artifacts/'
-
- if (common.checkContains('TEST', 'k8s')) {
- stage('Run k8s bootstrap tests') {
- def image = 'tomkukral/k8s-scripts'
- def output_file = image.replaceAll('/', '-') + '.output'
-
- // run image
- test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
-
- // collect output
- sh "mkdir -p ${artifacts_dir}"
- file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
- writeFile file: "${artifacts_dir}${output_file}", text: file_content
- sh "cat ${artifacts_dir}${output_file}"
-
- // collect artifacts
- archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
- }
-
- stage('Run k8s conformance e2e tests') {
- //test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
-
- def image = K8S_CONFORMANCE_IMAGE
- def output_file = image.replaceAll('/', '-') + '.output'
-
- // run image
- test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
-
- // collect output
- sh "mkdir -p ${artifacts_dir}"
- file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
- writeFile file: "${artifacts_dir}${output_file}", text: file_content
- sh "cat ${artifacts_dir}${output_file}"
-
- // collect artifacts
- archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
- }
- }
-
- if (common.checkContains('TEST', 'openstack')) {
- stage('Run OpenStack tests') {
- test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
- }
-
- stage('Copy Tempest results to config node') {
- test.copyTempestResults(saltMaster)
- }
- }
-
- stage('Finalize') {
- if (INSTALL != '') {
- try {
- salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
- } catch (Exception e) {
- common.warningMsg('State apply failed but we should continue to run')
- }
- }
- }
- } catch (Throwable e) {
- currentBuild.result = 'FAILURE'
- throw e
- } finally {
-
-
- //
- // Clean
- //
-
- if (STACK_TYPE == 'heat') {
- // send notification
- common.sendNotification(currentBuild.result, HEAT_STACK_NAME, ["slack"])
-
- if (HEAT_STACK_DELETE.toBoolean() == true) {
- common.errorMsg('Heat job cleanup triggered')
- stage('Trigger cleanup job') {
- build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
- }
- } else {
- if (currentBuild.result == 'FAILURE') {
- common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
-
- if (SALT_MASTER_URL) {
- common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
- }
- }
-
- }
- }
- }
- }
-}
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index 5a26ba5..ddfe37a 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -47,6 +47,7 @@
salt = new com.mirantis.mk.Salt()
test = new com.mirantis.mk.Test()
+_MAX_PERMITTED_STACKS = 2
overwriteFile = "/srv/salt/reclass/classes/cluster/overwrite.yml"
timestamps {
@@ -89,7 +90,18 @@
openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
openstack.getKeystoneToken(openstackCloud, openstackEnv)
-
+ //
+ // Verify possibility of create stack for given user and stack type
+ //
+ wrap([$class: 'BuildUser']) {
+ if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) {
+ def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
+ if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
+ HEAT_STACK_DELETE = "false"
+ throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
+ }
+ }
+ }
// launch stack
if (HEAT_STACK_REUSE.toBoolean() == false) {
stage('Launch new Heat stack') {
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 920e791..6d12ec6 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -7,6 +7,7 @@
def common = new com.mirantis.mk.Common()
def gerrit = new com.mirantis.mk.Gerrit()
def ruby = new com.mirantis.mk.Ruby()
+def jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
def gerritRef
try {
@@ -28,6 +29,15 @@
node("python&&docker") {
try{
+ //stage("stop old tests"){
+ // if (gerritRef) {
+ // def runningTestBuildNums = _getRunningTriggeredTestsBuildNumbers(env["JOB_NAME"], GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER)
+ // for(int i=0; i<runningTestBuildNums.size(); i++){
+ // common.infoMsg("Old test with run number ${runningTestBuildNums[i]} found, stopping")
+ // Jenkins.instance.getItemByFullName(env["JOB_NAME"]).getBuildByNumber(runningTestBuildNums[i]).finish(hudson.model.Result.ABORTED, new java.io.IOException("Aborting build"));
+ // }
+ // }
+ //}
stage("checkout") {
if (gerritRef) {
// job is triggered by Gerrit
@@ -72,4 +82,10 @@
}
common.sendNotification(currentBuild.result,"",["slack"])
}
-}
\ No newline at end of file
+}
+
+@NonCPS
+def _getRunningTriggeredTestsBuildNumbers(jobName, gerritChangeNumber, excludePatchsetNumber){
+ return gerrit.getGerritTriggeredBuilds(jenkinsUtils.getJobRunningBuilds(jobName), gerritChangeNumber, excludePatchsetNumber)
+ .stream().map{it -> it.number}.collect(java.util.stream.Collectors.toList())
+}