Merge "Added pipelines previously defined in mk-ci-salt-model"
diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index 2ce94a4..af8e5cb 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -31,212 +31,215 @@
salt = new com.mirantis.mk.Salt()
orchestrate = new com.mirantis.mk.Orchestrate()
-node {
- try {
- // connection objects
- def openstackCloud
- def saltMaster
-
- // value defaults
- def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
- def openstackEnv = "${env.WORKSPACE}/venv"
-
- if (HEAT_STACK_NAME == '') {
- HEAT_STACK_NAME = BUILD_TAG
- }
-
- //
- // Bootstrap
- //
-
- stage ('Download Heat templates') {
- git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
- }
-
- stage('Install OpenStack CLI') {
- openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
- }
-
- stage('Connect to OpenStack cloud') {
- openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
- openstack.getKeystoneToken(openstackCloud, openstackEnv)
- }
-
- if (HEAT_STACK_REUSE == 'false') {
- stage('Launch new Heat stack') {
- envParams = [
- 'instance_zone': HEAT_STACK_ZONE,
- 'public_net': HEAT_STACK_PUBLIC_NET
- ]
- openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
- }
- }
-
- stage('Connect to Salt master') {
- def saltMasterPort
- try {
- saltMasterPort = SALT_MASTER_PORT
- } catch (MissingPropertyException e) {
- saltMasterPort = 8000
- }
- saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
- saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
- saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
- }
-
- //
- // Install
- //
-
- stage('Install core infra') {
- // salt.master, reclass
- // refresh_pillar
- // sync_all
- // linux,openssh,salt.minion.ntp
-
- orchestrate.installFoundationInfra(saltMaster)
- orchestrate.validateFoundationInfra(saltMaster)
- }
-
- stage("Deploy GlusterFS") {
- salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
- salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
- sleep(5)
- salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
- print salt.cmdRun(saltMaster, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"')
- }
-
- stage("Deploy GlusterFS") {
- salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy,keepalived')
- }
-
- stage("Setup Docker Swarm") {
- salt.enforceState(saltMaster, 'I@docker:host', 'docker.host', true)
- salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.swarm', true)
- salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'salt', true)
- salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.flush')
- salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.update')
- salt.enforceState(saltMaster, 'I@docker:swarm', 'docker.swarm', true)
- print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'docker node ls')
- }
-
- stage("Deploy Docker services") {
- salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
-
- // XXX: Hack to fix dependency of gerrit on mysql
- print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', "docker service rm gerrit; sleep 5; rm -rf /srv/volumes/gerrit/*")
- sleep(10)
- print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', "apt-get install -y mysql-client; mysql -ppassword -h172.16.10.11 -P13306 -e'drop database gerrit;create database gerrit;'")
- salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
- // ---- cut here (end of hack) ----
-
- retry(30) {
- out = salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', """/bin/bash -c 'docker service ls | grep -E "0/[0-9]+"' && echo 'Some services are not running'""")
- for (int a = 0; a < out['return'].size(); a++) {
- def entry = out['return'].get(a)
- for (int i = 0; i < entry.size(); i++) {
- def node = entry.get(i)
- if (node) {
- if (node.value =~ /Some services are not running/) {
- sleep(10)
- throw new Exception("$node.key: $node.value")
- } else {
- print out
- }
- }
+def waitForServices(saltMaster) {
+ retry(30) {
+ out = salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', """/bin/bash -c 'docker service ls | grep -E "0/[0-9]+"' && echo 'Some services are not running'""")
+ for (int a = 0; a < out['return'].size(); a++) {
+ def entry = out['return'].get(a)
+ for (int i = 0; i < entry.size(); i++) {
+ def node = entry.get(i)
+ if (node) {
+ if (node.value =~ /Some services are not running/) {
+ sleep(10)
+ throw new Exception("$node.key: $node.value")
+ } else {
+ print out
}
}
}
- // Give services some time to settle
- sleep(30)
}
+ }
+}
- stage("Configure CI/CD services") {
- // Aptly
- salt.enforceState(saltMaster, 'I@aptly:server', 'aptly', true)
+timestamps {
+ node {
+ try {
+ // connection objects
+ def openstackCloud
+ def saltMaster
- // Gerrit
- timeout(10) {
- println "Waiting for Gerrit to come up.."
- salt.cmdRun(saltMaster, 'I@gerrit:client', 'while true; do curl -svf 172.16.10.254:8080 >/dev/null && break; done')
- }
- retry(2) {
- // Needs to run twice to pass __virtual__ method of gerrit module
- // after installation of dependencies
- try {
- salt.enforceState(saltMaster, 'I@gerrit:client', 'gerrit', true)
- } catch (Exception e) {
- print "Restarting Salt minion"
- salt.cmdRun(saltMaster, 'I@gerrit:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
- sleep(5)
- // XXX: Restart of salt-minion will cause glusterfs to
- // receive SIGTERM and got unmounted which will break
- // everything
- salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
- throw e
- }
+ // value defaults
+ def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
+ def openstackEnv = "${env.WORKSPACE}/venv"
+
+ if (HEAT_STACK_NAME == '') {
+ HEAT_STACK_NAME = BUILD_TAG
}
- // Jenkins
- timeout(10) {
- println "Waiting for Jenkins to come up.."
- salt.cmdRun(saltMaster, 'I@jenkins:client', 'while true; do curl -svf 172.16.10.254:8081 >/dev/null && break; done')
- }
- retry(2) {
- // Same for jenkins
- try {
- salt.enforceState(saltMaster, 'I@jenkins:client', 'jenkins', true)
- } catch (Exception e) {
- print "Restarting Salt minion"
- salt.cmdRun(saltMaster, 'I@jenkins:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
- sleep(5)
- // XXX: Restart of salt-minion will cause glusterfs to
- // receive SIGTERM and got unmounted which will break
- // everything
- salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
- throw e
- }
- }
- }
-
- stage("Finalize") {
//
- // Generate docs
+ // Bootstrap
//
- try {
- retry(3) {
- print salt.orchestrateSystem(saltMaster, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc')
- }
- } catch (Throwable e) {
- // We don't want sphinx docs to ruin whole build, so possible
- // errors are just ignored here
- true
+
+ stage ('Download Heat templates') {
+ git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
}
- salt.enforceState(saltMaster, 'I@nginx:server', 'nginx')
- print """============================================================
-Your CI/CD lab has been deployed and you can enjoy it:
-Use sshuttle -r ubuntu@${saltMasterHost} 172.16.10.0/24
-to connect to your private subnet and visit services
-running at 172.16.10.254 (vip address):
- 9600 haproxy stats
- 8080 gerrit
- 8081 jenkins
- 8091 Docker swarm visualizer
- 8090 Reclass-generated documentation
+ stage('Install OpenStack CLI') {
+ openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
+ }
-Don't forget to terminate your stack when you don't needed!
-============================================================"""
- }
- } catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
- currentBuild.result = "FAILURE"
- throw e
- } finally {
- // Cleanup
- if (HEAT_STACK_DELETE == 'true') {
- stage('Trigger cleanup job') {
- build job: 'deploy_heat_cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
+ stage('Connect to OpenStack cloud') {
+ openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
+ openstack.getKeystoneToken(openstackCloud, openstackEnv)
+ }
+
+ if (HEAT_STACK_REUSE == 'false') {
+ stage('Launch new Heat stack') {
+ envParams = [
+ 'instance_zone': HEAT_STACK_ZONE,
+ 'public_net': HEAT_STACK_PUBLIC_NET
+ ]
+ openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+ }
+ }
+
+ stage('Connect to Salt master') {
+ def saltMasterPort
+ try {
+ saltMasterPort = SALT_MASTER_PORT
+ } catch (MissingPropertyException e) {
+ saltMasterPort = 8000
+ }
+ saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
+ saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
+ saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
+ }
+
+ //
+ // Install
+ //
+
+ stage('Install core infra') {
+ // salt.master, reclass
+ // refresh_pillar
+ // sync_all
+ // linux,openssh,salt.minion.ntp
+
+ orchestrate.installFoundationInfra(saltMaster)
+ orchestrate.validateFoundationInfra(saltMaster)
+ }
+
+ stage("Deploy GlusterFS") {
+ salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
+ salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
+ sleep(5)
+ salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
+ print salt.cmdRun(saltMaster, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"')
+ }
+
+ stage("Deploy GlusterFS") {
+ salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy,keepalived')
+ }
+
+ stage("Setup Docker Swarm") {
+ salt.enforceState(saltMaster, 'I@docker:host', 'docker.host', true)
+ salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.swarm', true)
+ salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'salt', true)
+ salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.flush')
+ salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.update')
+ salt.enforceState(saltMaster, 'I@docker:swarm', 'docker.swarm', true)
+ print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'docker node ls')
+ }
+
+ stage("Deploy Docker services") {
+ salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
+
+ // XXX: Hack to fix dependency of gerrit on mysql
+ print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', "docker service rm gerrit; sleep 5; rm -rf /srv/volumes/gerrit/*")
+ waitForServices(saltMaster)
+
+ timeout(10) {
+ salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'apt-get install -y mysql-client')
+ println "Waiting for MySQL to come up.."
+ salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'while true; do mysql -h172.16.10.254 -ppassword -e"show status;" >/dev/null && break; done')
+ }
+ salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
+ // ---- cut here (end of hack) ----
+
+ waitForServices(saltMaster)
+ }
+
+ stage("Configure CI/CD services") {
+ // Aptly
+ salt.enforceState(saltMaster, 'I@aptly:server', 'aptly', true)
+
+ // Gerrit
+ timeout(10) {
+ println "Waiting for Gerrit to come up.."
+ salt.cmdRun(saltMaster, 'I@gerrit:client', 'while true; do curl -svf 172.16.10.254:8080 >/dev/null && break; done')
+ }
+ retry(2) {
+ // Needs to run twice to pass __virtual__ method of gerrit module
+ // after installation of dependencies
+ try {
+ salt.enforceState(saltMaster, 'I@gerrit:client', 'gerrit', true)
+ } catch (Exception e) {
+ print "Restarting Salt minion"
+ salt.cmdRun(saltMaster, 'I@gerrit:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
+ sleep(5)
+ throw e
+ }
+ }
+
+ // Jenkins
+ timeout(10) {
+ println "Waiting for Jenkins to come up.."
+ salt.cmdRun(saltMaster, 'I@jenkins:client', 'while true; do curl -svf 172.16.10.254:8081 >/dev/null && break; done')
+ }
+ retry(2) {
+ // Same for jenkins
+ try {
+ salt.enforceState(saltMaster, 'I@jenkins:client', 'jenkins', true)
+ } catch (Exception e) {
+ print "Restarting Salt minion"
+ salt.cmdRun(saltMaster, 'I@jenkins:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
+ sleep(5)
+ throw e
+ }
+ }
+ }
+
+ stage("Finalize") {
+ //
+ // Generate docs
+ //
+ try {
+ retry(3) {
+ // TODO: fix salt.orchestrateSystem
+ // print salt.orchestrateSystem(saltMaster, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc')
+ print salt.cmdRun(saltMaster, 'I@salt:master', 'salt-run state.orchestrate sphinx.orch.generate_doc')
+ }
+ } catch (Throwable e) {
+ // We don't want sphinx docs to ruin whole build, so possible
+ // errors are just ignored here
+ true
+ }
+ salt.enforceState(saltMaster, 'I@nginx:server', 'nginx')
+
+ print """============================================================
+ Your CI/CD lab has been deployed and you can enjoy it:
+ Use sshuttle -r ubuntu@${saltMasterHost} 172.16.10.0/24
+ to connect to your private subnet and visit services
+ running at 172.16.10.254 (vip address):
+ 9600 haproxy stats
+ 8080 gerrit
+ 8081 jenkins
+ 8091 Docker swarm visualizer
+ 8090 Reclass-generated documentation
+
+ Don't forget to terminate your stack when you don't need it!
+ ============================================================"""
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ } finally {
+ // Cleanup
+ if (HEAT_STACK_DELETE == 'true') {
+ stage('Trigger cleanup job') {
+ build job: 'deploy_heat_cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
+ }
}
}
}
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index 29f55d6..d426d3e 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -38,138 +38,350 @@
git = new com.mirantis.mk.Git()
openstack = new com.mirantis.mk.Openstack()
salt = new com.mirantis.mk.Salt()
-orchestrate = new com.mirantis.mk.Orchestrate()
-node {
+timestamps {
+ node {
- //
- // Prepare machines
- //
+ //
+ // Prepare machines
+ //
- stage ('Create infrastructure') {
- if (STACK_TYPE == 'heat') {
- // value defaults
- def openstackCloud
- def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
- def openstackEnv = "${env.WORKSPACE}/venv"
+ stage ('Create infrastructure') {
+ if (STACK_TYPE == 'heat') {
+ // value defaults
+ def openstackCloud
+ def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
+ def openstackEnv = "${env.WORKSPACE}/venv"
-
- if (HEAT_STACK_NAME == '') {
- HEAT_STACK_NAME = BUILD_TAG
- }
-
- // get templates
- git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
-
- // create openstack env
- openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
- openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
- openstack.getKeystoneToken(openstackCloud, openstackEnv)
-
-
- // launch stack
- if (HEAT_STACK_REUSE == 'false') {
- stage('Launch new Heat stack') {
- // create stack
- envParams = [
- 'instance_zone': HEAT_STACK_ZONE,
- 'public_net': HEAT_STACK_PUBLIC_NET
- ]
- openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+ if (HEAT_STACK_NAME == '') {
+ HEAT_STACK_NAME = BUILD_TAG
}
+
+ // get templates
+ git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
+
+ // create openstack env
+ openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
+ openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
+ openstack.getKeystoneToken(openstackCloud, openstackEnv)
+
+ // launch stack
+ if (HEAT_STACK_REUSE == 'false') {
+ stage('Launch new Heat stack') {
+ // create stack
+ envParams = [
+ 'instance_zone': HEAT_STACK_ZONE,
+ 'public_net': HEAT_STACK_PUBLIC_NET
+ ]
+ openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+ }
+ }
+
+ // get SALT_MASTER_URL
+ saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
+ SALT_MASTER_URL = "http://${saltMasterHost}:8088"
+ }
+ }
+
+ //
+ // Connect to Salt master
+ //
+
+ def saltMaster
+ stage('Connect to Salt API') {
+ saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ //
+ // Install
+ //
+
+ if (INSTALL.toLowerCase().contains('core')) {
+ stage('Install core infrastructure') {
+ // salt.master, reclass
+ // refresh_pillar
+ // sync_all
+ // linux,openssh,salt.minion.ntp
+
+ //orchestrate.installFoundationInfra(saltMaster)
+ salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'state.sls', ['salt.master,reclass'])
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar')
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all')
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'state.sls', ['linux,openssh,salt.minion,ntp'])
+
+
+ if (INSTALL.toLowerCase().contains('kvm')) {
+ //orchestrate.installInfraKvm(saltMaster)
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar')
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all')
+
+ salt.runSaltProcessStep(saltMaster, 'I@salt:control', 'state.sls', ['salt.minion,linux.system,linux.network,ntp'])
+ salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
+ salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
+
+ sleep(300)
+
+ salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.refresh_pillar')
+ salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.sync_all')
+
+ // workaround - install apt-transport-https
+ salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'pkg.install', ['apt-transport-https'])
+ salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'state.sls', ['linux,openssh,salt.minion,ntp'])
+ }
+
+ //orchestrate.validateFoundationInfra(saltMaster)
+ salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['salt-key'])
+ salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'test.version')
+ salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['reclass-salt --top'])
+ salt.runSaltProcessStep(saltMaster, 'I@reclass:storage', 'reclass.inventory')
+ salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'state.show_top')
+ }
+ }
+
+ // install k8s
+ if (INSTALL.toLowerCase().contains('k8s')) {
+ stage('Install Kubernetes infra') {
+ //orchestrate.installOpenstackMcpInfra(saltMaster)
+
+ // Comment nameserver
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
+
+ // Install glusterfs
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
+
+ // Install keepalived
+ salt.runSaltProcessStep(saltMaster, 'ctl01*', 'state.sls', ['keepalived'])
+ salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
+
+ // Check the keepalived VIPs
+ salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
+
+ // Setup glusterfs
+ salt.runSaltProcessStep(saltMaster, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(saltMaster, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(saltMaster, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
+
+ // Install haproxy
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
+
+ // Install docker
+ salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'state.sls', ['docker.host'])
+ salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'cmd.run', ['docker ps'])
+
+ // Install bird
+ salt.runSaltProcessStep(saltMaster, 'I@bird:server', 'state.sls', ['bird'])
+
+ // Install etcd
+ salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
+ salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'cmd.run', ['etcdctl cluster-health'])
+
}
- // get SALT_MASTER_URL
- saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
- SALT_MASTER_URL = "http://${saltMasterHost}:8088"
- }
- }
+ stage('Install Kubernetes control') {
+ //orchestrate.installOpenstackMcpControl(saltMaster)
- //
- // Connect to Salt master
- //
+ // Install Kubernetes pool and Calico
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'cmd.run', ['calicoctl node status'])
- def saltMaster
- stage('Connect to Salt API') {
- saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
- }
+ // Setup etcd server
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['etcd.server.setup'])
- //
- // Install
- //
+ // Run k8s without master.setup
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
- stage('Install core infrastructure') {
- // salt.master, reclass
- // refresh_pillar
- // sync_all
- // linux,openssh,salt.minion.ntp
+ // Run k8s master setup
+ salt.runSaltProcessStep(saltMaster, 'ctl01*', 'state.sls', ['kubernetes.master.setup'])
- orchestrate.installFoundationInfra(saltMaster)
+ // Revert comment nameserver
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
- if (INSTALL.toLowerCase().contains('kvm')) {
- orchestrate.installInfraKvm(saltMaster)
+ // Set route
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'cmd.run', ['ip r a 10.254.0.0/16 dev ens4'])
+
+ // Restart kubelet
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
+ }
+
}
- orchestrate.validateFoundationInfra(saltMaster)
- }
+ // install openstack
+ if (INSTALL.toLowerCase().contains('openstack')) {
+ // install Infra and control, tests, ...
+ stage('Install OpenStack infra') {
+ //orchestrate.installOpenstackMkInfra(saltMaster, physical)
- // install k8s
- if (INSTALL.toLowerCase().contains('k8s')) {
- stage('Install Kubernetes infra') {
- orchestrate.installOpenstackMcpInfra(saltMaster)
+ // Install keepaliveds
+ //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'keepalived', true)
+ salt.enforceState(saltMaster, 'I@keepalived:cluster', 'keepalived', true)
+
+ // Check the keepalived VIPs
+ salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
+
+ // Install glusterfs
+ salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
+
+ //runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.setup'], 1)
+ if (INSTALL.toLowerCase().contains('kvm')) {
+ salt.enforceState(saltMaster, 'ctl01*', 'glusterfs.server.setup', true)
+ salt.enforceState(saltMaster, 'ctl02*', 'glusterfs.server.setup', true)
+ salt.enforceState(saltMaster, 'ctl03*', 'glusterfs.server.setup', true)
+ } else {
+ salt.enforceState(saltMaster, 'kvm01*', 'glusterfs.server.setup', true)
+ salt.enforceState(saltMaster, 'kvm02*', 'glusterfs.server.setup', true)
+ salt.enforceState(saltMaster, 'kvm03*', 'glusterfs.server.setup', true)
+ }
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
+
+ // Install rabbitmq
+ salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
+
+ // Check the rabbitmq status
+ salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
+
+ // Install galera
+ salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
+ salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
+
+ // Check galera status
+ salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'mysql.status')
+ salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'mysql.status')
+
+ // Install haproxy
+ salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy', true)
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
+
+ // Install memcached
+ salt.enforceState(saltMaster, 'I@memcached:server', 'memcached', true)
+
+ }
+
+ stage('Install OpenStack control') {
+ //orchestrate.installOpenstackMkControl(saltMaster)
+
+ // setup keystone service
+ //runSaltProcessStep(saltMaster, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'keystone.server', true)
+ salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
+ // populate keystone services/tenants/roles/users
+
+ // keystone:client must be called locally
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'])
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; keystone service-list'])
+
+ // Install glance and ensure glusterfs clusters
+ //runSaltProcessStep(saltMaster, 'I@glance:server', 'state.sls', ['glance.server'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'glance.server', true)
+ salt.enforceState(saltMaster, 'I@glance:server', 'glance.server', true)
+ salt.enforceState(saltMaster, 'I@glance:server', 'glusterfs.client', true)
+
+ // Update fernet tokens before doing request on keystone server
+ salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
+
+ // Check glance service
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'])
+
+ // Install and check nova service
+ //runSaltProcessStep(saltMaster, 'I@nova:controller', 'state.sls', ['nova'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'nova', true)
+ salt.enforceState(saltMaster, 'I@nova:controller', 'nova', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'])
+
+ // Install and check cinder service
+ //runSaltProcessStep(saltMaster, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'cinder', true)
+ salt.enforceState(saltMaster, 'I@cinder:controller', 'cinder', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'])
+
+ // Install neutron service
+ //runSaltProcessStep(saltMaster, 'I@neutron:server', 'state.sls', ['neutron'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'neutron', true)
+ salt.enforceState(saltMaster, 'I@neutron:server', 'neutron', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'])
+
+ // Install heat service
+ //runSaltProcessStep(saltMaster, 'I@heat:server', 'state.sls', ['heat'], 1)
+ salt.enforceState(saltMaster, 'ctl01*', 'heat', true)
+ salt.enforceState(saltMaster, 'I@heat:server', 'heat', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'])
+
+ // Install horizon dashboard
+ salt.enforceState(saltMaster, 'I@horizon:server', 'horizon', true)
+ salt.enforceState(saltMaster, 'I@nginx:server', 'nginx', true)
+
+ }
+
+ stage('Install OpenStack network') {
+ //orchestrate.installOpenstackMkNetwork(saltMaster, physical)
+
+ // Install opencontrail database services
+ //runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
+ salt.enforceState(saltMaster, 'ntw01*', 'opencontrail.database', true)
+ salt.enforceState(saltMaster, 'I@opencontrail:database', 'opencontrail.database', true)
+
+ // Install opencontrail control services
+ //runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
+ salt.enforceState(saltMaster, 'ntw01*', 'opencontrail', true)
+ salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail', true)
+
+ // Provision opencontrail control services
+ if (INSTALL.toLowerCase().contains('kvm')) {
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl01 --host_ip 172.16.10.101 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl02 --host_ip 172.16.10.102 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl03 --host_ip 172.16.10.103 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+ }
+
+ // Test opencontrail
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'])
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
+ }
+
+ stage('Install OpenStack compute') {
+ //orchestrate.installOpenstackMkCompute(saltMaster, physical)
+ // Configure compute nodes
+ salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply')
+ salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply')
+
+ // Provision opencontrail virtual routers
+ if (INSTALL.toLowerCase().contains('kvm')) {
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_vrouter.py --host_name cmp01 --host_ip 172.16.10.105 --api_server_ip 172.16.10.254 --oper add --admin_user admin --admin_password workshop --admin_tenant_name admin'])
+ }
+
+ salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'system.reboot')
+ }
}
- stage('Install Kubernetes control') {
- orchestrate.installOpenstackMcpControl(saltMaster)
+ //
+ // Test
+ //
+
+ if (TEST.toLowerCase().contains('k8s')) {
+ stage('Run k8s bootstrap tests') {
+ orchestrate.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
+ }
+
+ stage('Run k8s conformance e2e tests') {
+ orchestrate.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
+ }
}
- }
- // install openstack
- if (INSTALL.toLowerCase().contains('openstack')) {
- // install Infra and control, tests, ...
+ //
+ // Clean
+ //
- stage('Install OpenStack infra') {
- orchestrate.installOpenstackMkInfra(saltMaster)
- }
-
- stage('Install OpenStack control') {
- orchestrate.installOpenstackMkControl(saltMaster)
- }
-
- stage('Install OpenStack network') {
- orchestrate.installOpenstackMkNetwork(saltMaster)
- }
-
- stage('Install OpenStack compute') {
- orchestrate.installOpenstackMkCompute(saltMaster)
- }
-
- }
-
- //
- // Test
- //
-
- if (TEST.toLowerCase().contains('k8s')) {
- stage('Run k8s bootstrap tests') {
- orchestrate.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
- }
-
- stage('Run k8s conformance e2e tests') {
- orchestrate.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
- }
- }
-
-
- //
- // Clean
- //
-
- if (HEAT_STACK_DELETE == 'true' && STACK_TYPE == 'heat') {
- stage('Trigger cleanup job') {
- build job: 'deploy_heat_cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
+ if (HEAT_STACK_DELETE == 'true' && STACK_TYPE == 'heat') {
+ stage('Trigger cleanup job') {
+ build job: 'deploy_heat_cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
+ }
}
}
}