use Orchestrate class in lab pipeline
Previous pipeline temporarily copied to heavey-lab-pipeline.groovy and
will be delete in future.
Change-Id: I5c825e2cc6e67bea829ba5d2e2df7a971da4b0d7
diff --git a/heavy-lab-pipeline.groovy b/heavy-lab-pipeline.groovy
new file mode 100644
index 0000000..fa42445
--- /dev/null
+++ b/heavy-lab-pipeline.groovy
@@ -0,0 +1,598 @@
+/**
+ *
+ * Launch heat stack with basic k8s
+ * Flow parameters:
+ * STACK_TYPE Orchestration engine: heat, ''
+ * INSTALL What should be installed (k8s, openstack, ...)
+ * TEST What should be tested (k8s, openstack, ...)
+ *
+ * Expected parameters:
+ *
+ * required for STACK_TYPE=heat
+ * HEAT_TEMPLATE_URL URL to git repo with Heat templates
+ * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo
+ * HEAT_TEMPLATE_BRANCH Heat templates repo branch
+ * HEAT_STACK_TEMPLATE Heat stack HOT template
+ * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
+ * HEAT_STACK_ZONE Heat stack availability zone
+ * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
+ * HEAT_STACK_DELETE Delete Heat stack when finished (bool)
+ * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack
+ * HEAT_STACK_REUSE Reuse Heat stack (don't create one)
+ * OPENSTACK_API_URL OpenStack API address
+ * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
+ * OPENSTACK_API_PROJECT OpenStack project to connect to
+ * OPENSTACK_API_CLIENT Versions of OpenStack python clients
+ * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
+ *
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ *
+ * required for STACK_TYPE=NONE or empty string
+ * SALT_MASTER_URL URL of Salt-API
+
+ * K8S_API_SERVER Kubernetes API address
+ * K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
+ *
+ * TEMPEST_IMAGE_LINK Tempest image link
+ *
+ */
+
+git = new com.mirantis.mk.Git()
+openstack = new com.mirantis.mk.Openstack()
+salt = new com.mirantis.mk.Salt()
+common = new com.mirantis.mk.Common()
+test = new com.mirantis.mk.Test()
+
+timestamps {
+ node {
+ try {
+ //
+ // Prepare machines
+ //
+ stage ('Create infrastructure') {
+
+ if (STACK_TYPE == 'heat') {
+ // value defaults
+ def openstackCloud
+ def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
+ def openstackEnv = "${env.WORKSPACE}/venv"
+
+ if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
+ error("If you want to reuse existing stack you need to provide it's name")
+ }
+
+ if (HEAT_STACK_REUSE.toBoolean() == false) {
+ // Don't allow to set custom heat stack name
+ wrap([$class: 'BuildUser']) {
+ if (env.BUILD_USER_ID) {
+ HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
+ } else {
+ HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
+ }
+ currentBuild.description = HEAT_STACK_NAME
+ }
+ }
+
+ // set description
+ currentBuild.description = "${HEAT_STACK_NAME}"
+
+ // get templates
+ git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
+
+ // create openstack env
+ openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
+ openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
+ openstack.getKeystoneToken(openstackCloud, openstackEnv)
+
+ // launch stack
+ if (HEAT_STACK_REUSE.toBoolean() == false) {
+ stage('Launch new Heat stack') {
+ // create stack
+ envParams = [
+ 'instance_zone': HEAT_STACK_ZONE,
+ 'public_net': HEAT_STACK_PUBLIC_NET
+ ]
+ openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+ }
+ }
+
+ // get SALT_MASTER_URL
+ saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
+ currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
+
+ if (common.checkContains('INSTALL', 'kvm')) {
+ saltPort = 6969
+ } else {
+ saltPort = 6969
+ }
+
+ SALT_MASTER_URL = "http://${saltMasterHost}:${saltPort}"
+ }
+ }
+
+ //
+ // Connect to Salt master
+ //
+
+ def saltMaster
+ stage('Connect to Salt API') {
+ saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ //
+ // Install
+ //
+
+ if (common.checkContains('INSTALL', 'core')) {
+ stage('Install core infrastructure') {
+ // orchestrate.installFoundationInfra(saltMaster)
+ salt.enforceState(saltMaster, 'I@salt:master', ['salt.master', 'reclass'], true)
+ salt.enforceState(saltMaster, '*', ['linux.system'], true)
+ salt.enforceState(saltMaster, '*', ['salt.minion'], true)
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
+ salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
+
+
+ if (common.checkContains('INSTALL', 'kvm')) {
+ // orchestrate.installInfraKvm(saltMaster)
+ // orchestrate.installFoundationInfra(saltMaster)
+ //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
+ //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
+
+ //salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
+ salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
+ salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
+
+ sleep(600)
+
+ salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.refresh_pillar', [], null, true)
+ salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.sync_all', [], null, true)
+
+ // workaround - install apt-transport-https
+ //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'cmd.run', ['apt-get update -y && apt-get install -y apt-transport-https'], null, true)
+ //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'pkg.install', ['apt-transport-https', 'refresh=True'], null, true)
+ salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'salt.minion'], true, false)
+ salt.enforceState(saltMaster, 'I@linux:system', ['openssh', 'salt.minion', 'ntp'], true, false)
+ }
+
+ // orchestrate.validateFoundationInfra(saltMaster)
+ salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['salt-key'], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'test.version', [], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['reclass-salt --top'], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@reclass:storage', 'reclass.inventory', [], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'state.show_top', [], null, true)
+ }
+ }
+
+ // install k8s
+ if (common.checkContains('INSTALL', 'k8s')) {
+ stage('Install Kubernetes infra') {
+ // orchestrate.installKubernetesInfra(saltMaster)
+
+ // Install glusterfs
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
+
+ // Install keepalived
+ salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster and *01*', 'state.sls', ['keepalived'])
+ salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
+
+ // Setup glusterfs
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server and *01*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
+
+ // Install haproxy
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
+
+ // Install docker
+ salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'state.sls', ['docker.host'])
+ salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'cmd.run', ['docker ps'])
+
+ // Install etcd
+ salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
+ salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'cmd.run', ['./var/lib/etcd/configenv && etcdctl cluster-health'])
+
+ }
+
+ stage('Install Kubernetes control') {
+ // orchestrate.installKubernetesControl(saltMaster)
+
+ // Install Kubernetes pool and Calico
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes.master.kube-addons'])
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
+
+ // Setup etcd server
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['etcd.server.setup'])
+
+ // Run k8s without master.setup
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
+
+ // Run k8s master setup
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['kubernetes.master.setup'])
+
+ // Restart kubelet
+ salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
+ }
+
+ }
+
+ // install openstack
+ if (common.checkContains('INSTALL', 'openstack')) {
+ // install Infra and control, tests, ...
+
+ stage('Install OpenStack infra') {
+ // orchestrate.installOpenstackInfra(saltMaster, physical)
+
+ // Install glusterfs
+ salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
+
+ // Install keepaliveds
+ //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
+ salt.enforceState(saltMaster, 'I@keepalived:cluster and *01*', 'keepalived', true)
+ salt.enforceState(saltMaster, 'I@keepalived:cluster', 'keepalived', true)
+
+ // Check the keepalived VIPs
+ salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
+
+ salt.enforceState(saltMaster, 'I@glusterfs:server and *01*', 'glusterfs.server.setup', true)
+
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'], null, true)
+
+ // Install rabbitmq
+ withEnv(['ASK_ON_ERROR=false']){
+ retry(2) {
+ salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
+ }
+ }
+ // Check the rabbitmq status
+ salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
+
+ // Install galera
+ withEnv(['ASK_ON_ERROR=false']){
+ retry(2) {
+ salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
+ }
+ }
+ salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
+
+ // Check galera status
+ salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'mysql.status')
+ salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'mysql.status')
+
+ // // Setup mysql client
+ // salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client', true)
+
+ // Install haproxy
+ salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy', true)
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
+ salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
+
+ // Install memcached
+ salt.enforceState(saltMaster, 'I@memcached:server', 'memcached', true)
+
+ }
+
+ stage('Install OpenStack control') {
+ // orchestrate.installOpenstackControl(saltMaster)
+
+ // Install horizon dashboard
+ salt.enforceState(saltMaster, 'I@horizon:server', 'horizon', true)
+ salt.enforceState(saltMaster, 'I@nginx:server', 'nginx', true)
+
+ // setup keystone service
+ //runSaltProcessStep(saltMaster, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
+ salt.enforceState(saltMaster, 'I@keystone:server and *01*', 'keystone.server', true)
+ salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
+ // populate keystone services/tenants/roles/users
+
+ // keystone:client must be called locally
+ //salt.runSaltProcessStep(saltMaster, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'service.restart', ['apache2'])
+ sleep(30)
+ salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
+ salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonercv3; openstack service list'], null, true)
+
+ // Install glance and ensure glusterfs clusters
+ //runSaltProcessStep(saltMaster, 'I@glance:server', 'state.sls', ['glance.server'], 1)
+ salt.enforceState(saltMaster, 'I@glance:server and *01*', 'glance.server', true)
+ salt.enforceState(saltMaster, 'I@glance:server', 'glance.server', true)
+ salt.enforceState(saltMaster, 'I@glance:server', 'glusterfs.client', true)
+
+ // Update fernet tokens before doing request on keystone server
+ salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
+
+ // Check glance service
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'], null, true)
+
+ // Install and check nova service
+ //runSaltProcessStep(saltMaster, 'I@nova:controller', 'state.sls', ['nova'], 1)
+ salt.enforceState(saltMaster, 'I@nova:controller and *01*', 'nova.controller', true)
+ salt.enforceState(saltMaster, 'I@nova:controller', 'nova.controller', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'], null, true)
+
+ // Install and check cinder service
+ //runSaltProcessStep(saltMaster, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
+ salt.enforceState(saltMaster, 'I@cinder:controller and *01*', 'cinder', true)
+ salt.enforceState(saltMaster, 'I@cinder:controller', 'cinder', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'], null, true)
+
+ // Install neutron service
+ //runSaltProcessStep(saltMaster, 'I@neutron:server', 'state.sls', ['neutron'], 1)
+
+ salt.enforceState(saltMaster, 'I@neutron:server and *01*', 'neutron.server', true)
+ salt.enforceState(saltMaster, 'I@neutron:server', 'neutron.server', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'], null, true)
+
+ // Install heat service
+ //runSaltProcessStep(saltMaster, 'I@heat:server', 'state.sls', ['heat'], 1)
+ salt.enforceState(saltMaster, 'I@heat:server and *01*', 'heat', true)
+ salt.enforceState(saltMaster, 'I@heat:server', 'heat', true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'], null, true)
+
+ // Restart nova api
+ salt.runSaltProcessStep(saltMaster, 'I@nova:controller', 'service.restart', ['nova-api'])
+
+ }
+
+ stage('Install OpenStack network') {
+
+ if (common.checkContains('INSTALL', 'contrail')) {
+ // orchestrate.installContrailNetwork(saltMaster)
+ // Install opencontrail database services
+ //runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
+ try {
+ salt.enforceState(saltMaster, 'I@opencontrail:database and *01*', 'opencontrail.database', true)
+ } catch (Exception e) {
+ common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database and *01*')
+ }
+
+ try {
+ salt.enforceState(saltMaster, 'I@opencontrail:database', 'opencontrail.database', true)
+ } catch (Exception e) {
+ common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database')
+ }
+
+ // Install opencontrail control services
+ //runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
+
+ // Test opencontrail
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'], null, true)
+ } else if (common.checkContains('INSTALL', 'ovs')) {
+ // orchestrate.installOpenstackNetwork(saltMaster)
+ // Apply gateway
+ salt.runSaltProcessStep(saltMaster, 'I@neutron:gateway', 'state.apply', [], null, true)
+ }
+
+ // Pring information
+ // where is this
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'], null, true)
+ }
+
+ stage('Install OpenStack compute') {
+ // orchestrate.installOpenstackCompute(saltMaster)
+ // Configure compute nodes
+ retry(2) {
+ salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply', [], null, true)
+ }
+
+ if (common.checkContains('INSTALL', 'contrail')) {
+ // orchestrate.installContrailCompute(saltMaster)
+ // Provision opencontrail control services
+ salt.enforceState(saltMaster, 'I@opencontrail:database:id:1', 'opencontrail.client', true)
+ // Provision opencontrail virtual routers
+ salt.enforceState(saltMaster, 'I@opencontrail:compute', 'opencontrail.client', true)
+
+ salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'cmd.run', ['exec 0>&-; exec 1>&-; exec 2>&-; nohup bash -c "ip link | grep vhost && echo no_reboot || sleep 5 && reboot & "'], null, true)
+ sleep(300)
+ }
+ }
+
+ }
+
+
+ if (common.checkContains('INSTALL', 'stacklight')) {
+ stage('Install StackLight') {
+ //TODO: orchestrate.installStacklightControl(saltMaster)
+ //TODO: orchestrate.installStacklightClient(saltMaster)
+
+ // infra install
+ // Install the StackLight backends
+ salt.enforceState(saltMaster, '*01* and I@elasticsearch:server', 'elasticsearch.server', true)
+ salt.enforceState(saltMaster, 'I@elasticsearch:server', 'elasticsearch.server', true)
+
+ salt.enforceState(saltMaster, '*01* and I@influxdb:server', 'influxdb', true)
+ salt.enforceState(saltMaster, 'I@influxdb:server', 'influxdb', true)
+
+ salt.enforceState(saltMaster, '*01* and I@kibana:server', 'kibana.server', true)
+ salt.enforceState(saltMaster, 'I@kibana:server', 'kibana.server', true)
+
+ salt.enforceState(saltMaster, '*01* and I@grafana:server','grafana.server', true)
+ salt.enforceState(saltMaster, 'I@grafana:server','grafana.server', true)
+
+ salt.enforceState(saltMaster, 'I@nagios:server', 'nagios.server', true)
+ salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client.service', true)
+ salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client.service', true)
+ // nw salt.enforceState('I@kibana:client or I@elasticsearch:client' --async service.restart salt-minion
+
+ sleep(10)
+
+ salt.runSaltProcessStep(saltMaster, 'I@elasticsearch.client', 'cmd.run', ['salt-call state.sls elasticsearch.client'], null, true)
+ // salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client', true)
+ salt.runSaltProcessStep(saltMaster, 'I@kibana.client', 'cmd.run', ['salt-call state.sls kibana.client'], null, true)
+ // salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client', true)
+
+ // install monitor
+ // Restart salt-minion to make sure that it uses the latest Jinja library
+ // no way: salt '*' --async service.restart salt-minion; sleep 15
+
+ // Start by flusing Salt Mine to make sure it is clean
+ // Also clean-up the grains files to make sure that we start from a clean state
+ // nw salt "*" mine.flush
+ // nw salt "*" file.remove /etc/salt/grains.d/collectd
+ // nw salt "*" file.remove /etc/salt/grains.d/grafana
+ // nw salt "*" file.remove /etc/salt/grains.d/heka
+ // nw salt "*" file.remove /etc/salt/grains.d/sensu
+ // nw salt "*" file.remove /etc/salt/grains
+
+ // Install collectd, heka and sensu services on the nodes, this will also
+ // generate the metadata that goes into the grains and eventually into Salt Mine
+ salt.enforceState(saltMaster, '*', 'collectd', true)
+ salt.enforceState(saltMaster, '*', 'salt.minion', true)
+ salt.enforceState(saltMaster, '*', 'heka', true)
+
+ // Gather the Grafana metadata as grains
+ salt.enforceState(saltMaster, 'I@grafana:collector', 'grafana.collector', true)
+
+ // Update Salt Mine
+ salt.enforceState(saltMaster, '*', 'salt.minion.grains', true)
+ salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_modules', [], null, true)
+ salt.runSaltProcessStep(saltMaster, '*', 'mine.update', [], null, true)
+
+ sleep(5)
+
+ // Update Heka
+ salt.enforceState(saltMaster, 'I@heka:aggregator:enabled:True or I@heka:remote_collector:enabled:True', 'heka', true)
+
+ // Update collectd
+ salt.enforceState(saltMaster, 'I@collectd:remote_client:enabled:True', 'collectd', true)
+
+ // Update Nagios
+ salt.enforceState(saltMaster, 'I@nagios:server', 'nagios', true)
+ // Stop the Nagios service because the package starts it by default and it will
+ // started later only on the node holding the VIP address
+ salt.runSaltProcessStep(saltMaster, 'I@nagios:server', 'service.stop', ['nagios3'], null, true)
+
+ // Update Sensu
+ // TODO for stacklight team, should be fixed in model
+ //salt.enforceState(saltMaster, 'I@sensu:server', 'sensu', true)
+
+ // Finalize the configuration of Grafana (add the dashboards...)
+ salt.enforceState(saltMaster, 'I@grafana:client', 'grafana.client.service', true)
+ // nw salt -C 'I@grafana:client' --async service.restart salt-minion; sleep 10
+
+ salt.runSaltProcessStep(saltMaster, 'I@grafana.client and *01*', 'cmd.run', ['salt-call state.sls grafana.client'], null, true)
+ // salt.enforceState(saltMaster, 'I@grafana:client and *01*', 'grafana.client', true)
+
+ // Get the StackLight monitoring VIP addres
+ //vip=$(salt-call pillar.data _param:stacklight_monitor_address --out key|grep _param: |awk '{print $2}')
+ //vip=${vip:=172.16.10.253}
+ def pillar = salt.getPillar(saltMaster, 'ctl01*', '_param:stacklight_monitor_address')
+ print(common.prettyPrint(pillar))
+ def stacklight_vip = pillar['return'][0].values()[0]
+
+ if (stacklight_vip) {
+ // (re)Start manually the services that are bound to the monitoring VIP
+ common.infoMsg("restart services on node with IP: ${stacklight_vip}")
+ salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collectd'], null, true)
+ salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collector'], null, true)
+ salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['aggregator'], null, true)
+ salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['nagios3'], null, true)
+ } else {
+ throw new Exception("Missing stacklight_vip")
+ }
+ }
+ }
+
+ //
+ // Test
+ //
+ def artifacts_dir = '_artifacts/'
+
+ if (common.checkContains('TEST', 'k8s')) {
+ stage('Run k8s bootstrap tests') {
+ def image = 'tomkukral/k8s-scripts'
+ def output_file = image.replaceAll('/', '-') + '.output'
+
+ // run image
+ test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
+
+ // collect output
+ sh "mkdir -p ${artifacts_dir}"
+ file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
+ writeFile file: "${artifacts_dir}${output_file}", text: file_content
+ sh "cat ${artifacts_dir}${output_file}"
+
+ // collect artifacts
+ archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
+ }
+
+ stage('Run k8s conformance e2e tests') {
+ //test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
+
+ def image = K8S_CONFORMANCE_IMAGE
+ def output_file = image.replaceAll('/', '-') + '.output'
+
+ // run image
+ test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
+
+ // collect output
+ sh "mkdir -p ${artifacts_dir}"
+ file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
+ writeFile file: "${artifacts_dir}${output_file}", text: file_content
+ sh "cat ${artifacts_dir}${output_file}"
+
+ // collect artifacts
+ archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
+ }
+ }
+
+ if (common.checkContains('TEST', 'openstack')) {
+ stage('Run OpenStack tests') {
+ test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
+ }
+
+ stage('Copy Tempest results to config node') {
+ test.copyTempestResults(saltMaster)
+ }
+ }
+
+ stage('Finalize') {
+ if (INSTALL != '') {
+ try {
+ salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
+ } catch (Exception e) {
+ common.warningMsg('State apply failed but we should continue to run')
+ }
+ }
+ }
+ } catch (Throwable e) {
+ currentBuild.result = 'FAILURE'
+ throw e
+ } finally {
+
+
+ //
+ // Clean
+ //
+
+ if (STACK_TYPE == 'heat') {
+ // send notification
+ common.sendNotification(currentBuild.result, HEAT_STACK_NAME, ["slack"])
+
+ if (HEAT_STACK_DELETE.toBoolean() == true) {
+ common.errorMsg('Heat job cleanup triggered')
+ stage('Trigger cleanup job') {
+ build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
+ }
+ } else {
+ if (currentBuild.result == 'FAILURE') {
+ common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
+
+ if (SALT_MASTER_URL) {
+ common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
+ }
+ }
+
+ }
+ }
+ }
+ }
+}
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index 23d3443..73ef34e 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -104,13 +104,7 @@
saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
- if (common.checkContains('INSTALL', 'kvm')) {
- saltPort = 6969
- } else {
- saltPort = 6969
- }
-
- SALT_MASTER_URL = "http://${saltMasterHost}:${saltPort}"
+ SALT_MASTER_URL = "http://${saltMasterHost}:6969"
}
}
@@ -129,106 +123,32 @@
if (common.checkContains('INSTALL', 'core')) {
stage('Install core infrastructure') {
- // salt.master, reclass
- // refresh_pillar
- // sync_all
- // linux,openssh,salt.minion.ntp
-
- //orchestrate.installFoundationInfra(saltMaster)
- salt.enforceState(saltMaster, 'I@salt:master', ['salt.master', 'reclass'], true)
- salt.enforceState(saltMaster, '*', ['linux.system'], true)
- salt.enforceState(saltMaster, '*', ['salt.minion'], true)
- salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
- salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
-
+ orchestrate.installFoundationInfra(saltMaster)
if (common.checkContains('INSTALL', 'kvm')) {
- //orchestrate.installInfraKvm(saltMaster)
- //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
- //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
-
- //salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
- salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
- salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
-
- sleep(600)
-
- salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.refresh_pillar', [], null, true)
- salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.sync_all', [], null, true)
-
- // workaround - install apt-transport-https
- //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'cmd.run', ['apt-get update -y && apt-get install -y apt-transport-https'], null, true)
- //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'pkg.install', ['apt-transport-https', 'refresh=True'], null, true)
- salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'salt.minion'], true, false)
- salt.enforceState(saltMaster, 'I@linux:system', ['openssh', 'salt.minion', 'ntp'], true, false)
+ orchestrate.installInfraKvm(saltMaster)
+ orchestrate.installFoundationInfra(saltMaster)
}
- //orchestrate.validateFoundationInfra(saltMaster)
- salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['salt-key'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'test.version', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['reclass-salt --top'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@reclass:storage', 'reclass.inventory', [], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'state.show_top', [], null, true)
+ orchestrate.validateFoundationInfra(saltMaster)
}
}
// install k8s
if (common.checkContains('INSTALL', 'k8s')) {
stage('Install Kubernetes infra') {
- //orchestrate.installOpenstackMcpInfra(saltMaster)
-
- // Overwrite Kubernetes vars if specified
- if (env.getEnvironment().containsKey("KUBERNETES_HYPERKUBE_IMAGE")) {
- salt.runSaltProcessStep(master, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_hyperkube_image: ${KUBERNETES_HYPERKUBE_IMAGE}")
- }
-
- // Install glusterfs
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
-
- // Install keepalived
- salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster and *01*', 'state.sls', ['keepalived'])
- salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
-
- // Setup glusterfs
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server and *01*', 'state.sls', ['glusterfs.server.setup'])
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
-
- // Install haproxy
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
-
- // Install docker
- salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'state.sls', ['docker.host'])
- salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'cmd.run', ['docker ps'])
-
- // Install etcd
- salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
- salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'cmd.run', ['./var/lib/etcd/configenv && etcdctl cluster-health'])
-
+ orchestrate.installKubernetesInfra(saltMaster)
}
stage('Install Kubernetes control') {
- //orchestrate.installOpenstackMcpControl(saltMaster)
- // Install Kubernetes pool and Calico
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes.master.kube-addons'])
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
+ // Overwrite Kubernetes vars if specified
+ if (env.getEnvironment().containsKey("KUBERNETES_HYPERKUBE_IMAGE")) {
+ salt.runSaltProcessStep(master, 'I@salt:master', 'file.append', overwriteFile, " kubernetes_hyperkube_image: ${KUBERNETES_HYPERKUBE_IMAGE}")
+ }
- // Setup etcd server
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['etcd.server.setup'])
-
- // Run k8s without master.setup
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
-
- // Run k8s master setup
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['kubernetes.master.setup'])
-
- // Restart kubelet
- salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
+ orchestrate.installKubernetesControl(saltMaster)
}
-
}
// install openstack
@@ -236,172 +156,30 @@
// install Infra and control, tests, ...
stage('Install OpenStack infra') {
- //orchestrate.installOpenstackMkInfra(saltMaster, physical)
-
- // Install glusterfs
- salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
-
- // Install keepaliveds
- //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
- salt.enforceState(saltMaster, 'I@keepalived:cluster and *01*', 'keepalived', true)
- salt.enforceState(saltMaster, 'I@keepalived:cluster', 'keepalived', true)
-
- // Check the keepalived VIPs
- salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
-
- salt.enforceState(saltMaster, 'I@glusterfs:server and *01*', 'glusterfs.server.setup', true)
-
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'], null, true)
-
- // Install rabbitmq
- withEnv(['ASK_ON_ERROR=false']){
- retry(2) {
- salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
- }
- }
- // Check the rabbitmq status
- salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
-
- // Install galera
- withEnv(['ASK_ON_ERROR=false']){
- retry(2) {
- salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
- }
- }
- salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
-
- // Check galera status
- salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'mysql.status')
- salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'mysql.status')
-
- // // Setup mysql client
- // salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client', true)
-
- // Install haproxy
- salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy', true)
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
- salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
-
- // Install memcached
- salt.enforceState(saltMaster, 'I@memcached:server', 'memcached', true)
-
+ orchestrate.installOpenstackInfra(saltMaster)
}
stage('Install OpenStack control') {
- //orchestrate.installOpenstackMkControl(saltMaster)
-
- // Install horizon dashboard
- salt.enforceState(saltMaster, 'I@horizon:server', 'horizon', true)
- salt.enforceState(saltMaster, 'I@nginx:server', 'nginx', true)
-
- // setup keystone service
- //runSaltProcessStep(saltMaster, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
- salt.enforceState(saltMaster, 'I@keystone:server and *01*', 'keystone.server', true)
- salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
- // populate keystone services/tenants/roles/users
-
- // keystone:client must be called locally
- //salt.runSaltProcessStep(saltMaster, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'service.restart', ['apache2'])
- sleep(30)
- salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
- salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonercv3; openstack service list'], null, true)
-
- // Install glance and ensure glusterfs clusters
- //runSaltProcessStep(saltMaster, 'I@glance:server', 'state.sls', ['glance.server'], 1)
- salt.enforceState(saltMaster, 'I@glance:server and *01*', 'glance.server', true)
- salt.enforceState(saltMaster, 'I@glance:server', 'glance.server', true)
- salt.enforceState(saltMaster, 'I@glance:server', 'glusterfs.client', true)
-
- // Update fernet tokens before doing request on keystone server
- salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
-
- // Check glance service
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'], null, true)
-
- // Install and check nova service
- //runSaltProcessStep(saltMaster, 'I@nova:controller', 'state.sls', ['nova'], 1)
- salt.enforceState(saltMaster, 'I@nova:controller and *01*', 'nova.controller', true)
- salt.enforceState(saltMaster, 'I@nova:controller', 'nova.controller', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'], null, true)
-
- // Install and check cinder service
- //runSaltProcessStep(saltMaster, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
- salt.enforceState(saltMaster, 'I@cinder:controller and *01*', 'cinder', true)
- salt.enforceState(saltMaster, 'I@cinder:controller', 'cinder', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'], null, true)
-
- // Install neutron service
- //runSaltProcessStep(saltMaster, 'I@neutron:server', 'state.sls', ['neutron'], 1)
-
- salt.enforceState(saltMaster, 'I@neutron:server and *01*', 'neutron.server', true)
- salt.enforceState(saltMaster, 'I@neutron:server', 'neutron.server', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'], null, true)
-
- // Install heat service
- //runSaltProcessStep(saltMaster, 'I@heat:server', 'state.sls', ['heat'], 1)
- salt.enforceState(saltMaster, 'I@heat:server and *01*', 'heat', true)
- salt.enforceState(saltMaster, 'I@heat:server', 'heat', true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'], null, true)
-
- // Restart nova api
- salt.runSaltProcessStep(saltMaster, 'I@nova:controller', 'service.restart', ['nova-api'])
-
+ orchestrate.installOpenstackControl(saltMaster)
}
stage('Install OpenStack network') {
- //orchestrate.installOpenstackMkNetwork(saltMaster, physical)
if (common.checkContains('INSTALL', 'contrail')) {
- // Install opencontrail database services
- //runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
- try {
- salt.enforceState(saltMaster, 'I@opencontrail:database and *01*', 'opencontrail.database', true)
- } catch (Exception e) {
- common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database and *01*')
- }
-
- try {
- salt.enforceState(saltMaster, 'I@opencontrail:database', 'opencontrail.database', true)
- } catch (Exception e) {
- common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database')
- }
-
- // Install opencontrail control services
- //runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
-
- // Test opencontrail
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'], null, true)
+ orchestrate.installContrailNetwork(saltMaster)
} else if (common.checkContains('INSTALL', 'ovs')) {
- // Apply gateway
- salt.runSaltProcessStep(saltMaster, 'I@neutron:gateway', 'state.apply', [], null, true)
+ orchestrate.installOpenstackNetwork(saltMaster)
}
- // Pring information
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'], null, true)
- salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'], null, true)
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
}
stage('Install OpenStack compute') {
- //orchestrate.installOpenstackMkCompute(saltMaster, physical)
- // Configure compute nodes
- retry(2) {
- salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply', [], null, true)
- }
+ orchestrate.installOpenstackCompute(saltMaster)
if (common.checkContains('INSTALL', 'contrail')) {
- // Provision opencontrail control services
- salt.enforceState(saltMaster, 'I@opencontrail:database:id:1', 'opencontrail.client', true)
- // Provision opencontrail virtual routers
- salt.enforceState(saltMaster, 'I@opencontrail:compute', 'opencontrail.client', true)
-
- salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'cmd.run', ['exec 0>&-; exec 1>&-; exec 2>&-; nohup bash -c "ip link | grep vhost && echo no_reboot || sleep 5 && reboot & "'], null, true)
- sleep(300)
+ orchestrate.installContrailCompute(saltMaster)
}
}
@@ -410,101 +188,8 @@
if (common.checkContains('INSTALL', 'stacklight')) {
stage('Install StackLight') {
- // infra install
- // Install the StackLight backends
- salt.enforceState(saltMaster, '*01* and I@elasticsearch:server', 'elasticsearch.server', true)
- salt.enforceState(saltMaster, 'I@elasticsearch:server', 'elasticsearch.server', true)
-
- salt.enforceState(saltMaster, '*01* and I@influxdb:server', 'influxdb', true)
- salt.enforceState(saltMaster, 'I@influxdb:server', 'influxdb', true)
-
- salt.enforceState(saltMaster, '*01* and I@kibana:server', 'kibana.server', true)
- salt.enforceState(saltMaster, 'I@kibana:server', 'kibana.server', true)
-
- salt.enforceState(saltMaster, '*01* and I@grafana:server','grafana.server', true)
- salt.enforceState(saltMaster, 'I@grafana:server','grafana.server', true)
-
- salt.enforceState(saltMaster, 'I@nagios:server', 'nagios.server', true)
- salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client.service', true)
- salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client.service', true)
- // nw salt.enforceState('I@kibana:client or I@elasticsearch:client' --async service.restart salt-minion
-
- sleep(10)
-
- salt.runSaltProcessStep(saltMaster, 'I@elasticsearch.client', 'cmd.run', ['salt-call state.sls elasticsearch.client'], null, true)
- // salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client', true)
- salt.runSaltProcessStep(saltMaster, 'I@kibana.client', 'cmd.run', ['salt-call state.sls kibana.client'], null, true)
- // salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client', true)
-
- // install monitor
- // Restart salt-minion to make sure that it uses the latest Jinja library
- // no way: salt '*' --async service.restart salt-minion; sleep 15
-
- // Start by flusing Salt Mine to make sure it is clean
- // Also clean-up the grains files to make sure that we start from a clean state
- // nw salt "*" mine.flush
- // nw salt "*" file.remove /etc/salt/grains.d/collectd
- // nw salt "*" file.remove /etc/salt/grains.d/grafana
- // nw salt "*" file.remove /etc/salt/grains.d/heka
- // nw salt "*" file.remove /etc/salt/grains.d/sensu
- // nw salt "*" file.remove /etc/salt/grains
-
- // Install collectd, heka and sensu services on the nodes, this will also
- // generate the metadata that goes into the grains and eventually into Salt Mine
- salt.enforceState(saltMaster, '*', 'collectd', true)
- salt.enforceState(saltMaster, '*', 'salt.minion', true)
- salt.enforceState(saltMaster, '*', 'heka', true)
-
- // Gather the Grafana metadata as grains
- salt.enforceState(saltMaster, 'I@grafana:collector', 'grafana.collector', true)
-
- // Update Salt Mine
- salt.enforceState(saltMaster, '*', 'salt.minion.grains', true)
- salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_modules', [], null, true)
- salt.runSaltProcessStep(saltMaster, '*', 'mine.update', [], null, true)
-
- sleep(5)
-
- // Update Heka
- salt.enforceState(saltMaster, 'I@heka:aggregator:enabled:True or I@heka:remote_collector:enabled:True', 'heka', true)
-
- // Update collectd
- salt.enforceState(saltMaster, 'I@collectd:remote_client:enabled:True', 'collectd', true)
-
- // Update Nagios
- salt.enforceState(saltMaster, 'I@nagios:server', 'nagios', true)
- // Stop the Nagios service because the package starts it by default and it will
- // started later only on the node holding the VIP address
- salt.runSaltProcessStep(saltMaster, 'I@nagios:server', 'service.stop', ['nagios3'], null, true)
-
- // Update Sensu
- // TODO for stacklight team, should be fixed in model
- //salt.enforceState(saltMaster, 'I@sensu:server', 'sensu', true)
-
- // Finalize the configuration of Grafana (add the dashboards...)
- salt.enforceState(saltMaster, 'I@grafana:client', 'grafana.client.service', true)
- // nw salt -C 'I@grafana:client' --async service.restart salt-minion; sleep 10
-
- salt.runSaltProcessStep(saltMaster, 'I@grafana.client and *01*', 'cmd.run', ['salt-call state.sls grafana.client'], null, true)
- // salt.enforceState(saltMaster, 'I@grafana:client and *01*', 'grafana.client', true)
-
- // Get the StackLight monitoring VIP addres
- //vip=$(salt-call pillar.data _param:stacklight_monitor_address --out key|grep _param: |awk '{print $2}')
- //vip=${vip:=172.16.10.253}
- def pillar = salt.getPillar(saltMaster, 'ctl01*', '_param:stacklight_monitor_address')
- print(common.prettyPrint(pillar))
- def stacklight_vip = pillar['return'][0].values()[0]
-
- if (stacklight_vip) {
- // (re)Start manually the services that are bound to the monitoring VIP
- common.infoMsg("restart services on node with IP: ${stacklight_vip}")
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collectd'], null, true)
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collector'], null, true)
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['aggregator'], null, true)
- salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['nagios3'], null, true)
- } else {
- throw new Exception("Missing stacklight_vip")
- }
+ orchestrate.installStacklightControl(saltMaster)
+ orchestrate.installStacklightClient(saltMaster)
}
}