blob: 5aca1242ee4b8790e10b40f6288c5ff01ad571e7 [file] [log] [blame]
Jakub Josef79ecec32017-02-17 14:36:28 +01001package com.mirantis.mk
2/**
3 * Orchestration functions
4 *
5*/
6
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03007/**
8 * Function runs Salt states to check infra
9 * @param master Salt Connection object or pepperEnv
10 * @param extra_tgt Extra target - adds ability to address commands using extra targeting to different clouds, e.g.: salt -C 'I@keystone:server and *ogrudev-deploy-heat-os-ha-ovs-82*' ...
11 */
12def validateFoundationInfra(master, extra_tgt = '') {
Jakub Josef79ecec32017-02-17 14:36:28 +010013 def salt = new com.mirantis.mk.Salt()
Ales Komarekef2bab62017-04-26 11:48:28 +020014
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030015 salt.cmdRun(master, "I@salt:master ${extra_tgt}" ,'salt-key')
16 salt.runSaltProcessStep(master, "I@salt:minion ${extra_tgt}", 'test.version')
17 salt.cmdRun(master, "I@salt:master ${extra_tgt}" ,'reclass-salt --top')
18 salt.runSaltProcessStep(master, "I@reclass:storage ${extra_tgt}", 'reclass.inventory')
19 salt.runSaltProcessStep(master, "I@salt:minion ${extra_tgt}", 'state.show_top')
Jakub Josef79ecec32017-02-17 14:36:28 +010020}
21
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030022def installFoundationInfra(master, staticMgmtNet=false, extra_tgt = '') {
Jakub Josef79ecec32017-02-17 14:36:28 +010023 def salt = new com.mirantis.mk.Salt()
Jiri Broulik037f8982018-04-23 17:07:16 +020024 def common = new com.mirantis.mk.Common()
Ales Komarekef2bab62017-04-26 11:48:28 +020025
Vasyl Saienko0869b3a2017-11-22 14:37:57 +020026 // NOTE(vsaienko) Apply reclass first, it may update cluster model
27 // apply linux and salt.master salt.minion states afterwards to make sure
28 // correct cluster model is used.
Martin Polreich80fa3182018-10-22 15:16:53 +020029 salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['reclass']])
Tomáš Kukrálb4d75222017-06-06 10:36:02 +020030
Martin Polreich80fa3182018-10-22 15:16:53 +020031 salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['linux.system']])
32 salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.master'], failOnError: false, read_timeout: 120, retries: 2])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030033 salt.fullRefresh(master, "* ${extra_tgt}")
Tomáš Kukrálb4d75222017-06-06 10:36:02 +020034
Martin Polreich80fa3182018-10-22 15:16:53 +020035 salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.minion'], failOnError: false, read_timeout: 60, retries: 2])
36 salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.minion']])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030037 salt.fullRefresh(master, "* ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +020038 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.network.proxy'], failOnError: false, read_timeout: 60, retries: 2])
iberezovskiy6394a8a2018-10-12 13:16:47 +040039 // Make sure all repositories are in place before proceeding with package installation from other states
Martin Polreich80fa3182018-10-22 15:16:53 +020040 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.system.repo'], failOnError: false, read_timeout: 60, retries: 2])
Jiri Broulik94fc2592018-04-20 13:47:51 +020041 try {
Martin Polreich80fa3182018-10-22 15:16:53 +020042 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['salt.minion.base'], failOnError: false, read_timeout: 60, retries: 2])
Jiri Broulik94fc2592018-04-20 13:47:51 +020043 sleep(5)
44 } catch (Throwable e) {
45 common.warningMsg('Salt state salt.minion.base is not present in the Salt-formula yet.')
46 }
Martin Polreich331f2b62019-02-08 10:16:52 +010047 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.system'], retries: 2])
Jiri Broulik8e1f1fe2017-12-13 11:54:06 +010048 if (staticMgmtNet) {
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030049 salt.runSaltProcessStep(master, "* ${extra_tgt}", 'cmd.shell', ["salt-call state.sls linux.network; salt-call service.restart salt-minion"], null, true, 60)
Jiri Broulik8e1f1fe2017-12-13 11:54:06 +010050 }
Martin Polreich331f2b62019-02-08 10:16:52 +010051 salt.enforceState([saltId: master, target: "I@linux:network:interface ${extra_tgt}", state: ['linux.network.interface'], retries: 2])
Jiri Broulik94538462018-05-11 15:45:13 +020052 sleep(5)
Martin Polreich80fa3182018-10-22 15:16:53 +020053 salt.enforceState([saltId: master, target: "I@linux:system ${extra_tgt}", state: ['linux', 'openssh', 'ntp', 'rsyslog']])
Ann Taraday49059762018-10-25 17:04:58 +040054
Ann Taradayfb2d4372018-10-30 17:17:09 +040055
Martin Polreich80fa3182018-10-22 15:16:53 +020056 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['salt.minion'], failOnError: false, read_timeout: 60, retries: 2])
Ann Taradayfb2d4372018-10-30 17:17:09 +040057
Jakub Josef4f8f4df2017-09-01 14:32:42 +020058 sleep(5)
Victor Ryzhenkinf610c572018-05-15 15:22:52 +040059
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030060 salt.fullRefresh(master, "* ${extra_tgt}")
61 salt.runSaltProcessStep(master, "* ${extra_tgt}", 'mine.update', [], null, true)
Martin Polreich80fa3182018-10-22 15:16:53 +020062 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.network.host']])
Ivan Suzdal9a6bc392018-04-26 18:29:56 +040063
64 // Install and configure iptables
Martin Polreich80fa3182018-10-22 15:16:53 +020065 salt.enforceStateWithTest([saltId: master, target: "I@iptables:service ${extra_tgt}", state: 'iptables'])
sgarbuzbb61e812018-05-03 16:58:40 +030066
67 // Install and configure logrotate
Martin Polreich80fa3182018-10-22 15:16:53 +020068 salt.enforceStateWithTest([saltId: master, target: "I@logrotate:server ${extra_tgt}", state: 'logrotate'])
Ivan Suzdal9f8f0e12018-06-21 13:16:21 +040069
70 // Install and configure auditd
Martin Polreich80fa3182018-10-22 15:16:53 +020071 salt.enforceStateWithTest([saltId: master, target: "I@auditd:service ${extra_tgt}", state: 'auditd'])
Ivan Suzdalca980912018-08-02 09:35:07 +040072
73 // Install and configure openscap
Martin Polreich80fa3182018-10-22 15:16:53 +020074 salt.enforceStateWithTest([saltId: master, target: "I@openscap:service ${extra_tgt}", state: 'openscap'])
Jakub Josef79ecec32017-02-17 14:36:28 +010075}
76
Oleh Hryhorov5f96e092018-06-22 18:37:08 +030077def installFoundationInfraOnTarget(master, target, staticMgmtNet=false, extra_tgt = '') {
Jiri Broulik976bf6c2017-10-20 14:43:25 +020078 def salt = new com.mirantis.mk.Salt()
Jiri Broulik9fb138a2018-04-23 17:13:00 +020079 def common = new com.mirantis.mk.Common()
Jiri Broulik976bf6c2017-10-20 14:43:25 +020080
Martin Polreich80fa3182018-10-22 15:16:53 +020081 salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['reclass'], failOnError: false, read_timeout: 120, retries: 2])
Jakub Josef432e9d92018-02-06 18:28:37 +010082 salt.fullRefresh(master, target)
Martin Polreich80fa3182018-10-22 15:16:53 +020083 salt.enforceState([saltId: master, target: target, state: ['linux.network.proxy'], failOnError: false, read_timeout: 60, retries: 2])
Jiri Broulik94fc2592018-04-20 13:47:51 +020084 try {
Martin Polreich80fa3182018-10-22 15:16:53 +020085 salt.enforceState([saltId: master, target: target, state: ['salt.minion.base'], failOnError: false, read_timeout: 60, retries: 2])
Jiri Broulik94fc2592018-04-20 13:47:51 +020086 sleep(5)
87 } catch (Throwable e) {
88 common.warningMsg('Salt state salt.minion.base is not present in the Salt-formula yet.')
89 }
Martin Polreich331f2b62019-02-08 10:16:52 +010090 salt.enforceState([saltId: master, target: target, state: ['linux.system'], retries: 2])
Jiri Broulik8e1f1fe2017-12-13 11:54:06 +010091 if (staticMgmtNet) {
92 salt.runSaltProcessStep(master, target, 'cmd.shell', ["salt-call state.sls linux.network; salt-call service.restart salt-minion"], null, true, 60)
93 }
Martin Polreich80fa3182018-10-22 15:16:53 +020094 salt.enforceState([saltId: master, target: target, state: ['salt.minion'], failOnError: false, read_timeout: 60, retries: 2])
95 salt.enforceState([saltId: master, target: target, state: ['salt.minion']])
96 salt.enforceState([saltId: master, target: target, state: ['linux.network.interface']])
Jiri Broulik94538462018-05-11 15:45:13 +020097 sleep(5)
Martin Polreich80fa3182018-10-22 15:16:53 +020098 salt.enforceState([saltId: master, target: target, state: ['linux', 'openssh', 'ntp', 'rsyslog']])
Jiri Broulik976bf6c2017-10-20 14:43:25 +020099 sleep(5)
Victor Ryzhenkinf610c572018-05-15 15:22:52 +0400100
101 salt.fullRefresh(master, target)
vrovachevc4424a32018-03-15 17:35:57 +0400102 salt.runSaltProcessStep(master, target, 'mine.update', [], null, true)
Martin Polreich80fa3182018-10-22 15:16:53 +0200103 salt.enforceState([saltId: master, target: target, state: ['linux.network.host']])
Jiri Broulik976bf6c2017-10-20 14:43:25 +0200104}
105
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300106def installInfraKvm(master, extra_tgt = '') {
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200107 def common = new com.mirantis.mk.Common()
Tomáš Kukrál8d2be5a2017-02-28 11:22:29 +0100108 def salt = new com.mirantis.mk.Salt()
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300109 def infra_compound = "I@salt:control ${extra_tgt}"
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200110 def minions = []
111 def wait_timeout = 10
112 def retries = wait_timeout * 30
Ales Komarekef2bab62017-04-26 11:48:28 +0200113
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300114 salt.fullRefresh(master, "I@linux:system ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200115 salt.enforceState([saltId: master, target: "I@salt:control ${extra_tgt}", state: ['salt.minion'], failOnError: false, read_timeout: 60, retries: 2])
116 salt.enforceState([saltId: master, target: "I@salt:control ${extra_tgt}", state: ['linux.system', 'linux.network', 'ntp', 'rsyslog']])
117 salt.enforceState([saltId: master, target: "I@salt:control ${extra_tgt}", state: 'libvirt'])
118 salt.enforceState([saltId: master, target: "I@salt:control ${extra_tgt}", state: 'salt.control'])
Tomáš Kukrálb8e97112017-03-01 16:03:16 +0100119
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200120 common.infoMsg("Building minions list...")
121 if (salt.testTarget(master, infra_compound)) {
122 // Gathering minions
123 for ( infra_node in salt.getMinionsSorted(master, infra_compound) ) {
124 def pillar = salt.getPillar(master, infra_node, 'salt:control:cluster')
125 if ( !pillar['return'].isEmpty() ) {
126 for ( cluster in pillar['return'][0].values() ) {
127 def engine = cluster.values()[0]['engine']
128 def domain = cluster.values()[0]['domain']
129 def node = cluster.values()[0]['node']
130 if ( engine == "virt" ) {
131 def nodes = node.values()
132 if ( !nodes.isEmpty() ) {
133 for ( vm in nodes ) {
134 if ( vm['name'] != null ) {
135 def vm_fqdn = vm['name'] + '.' + domain
136 if ( !minions.contains(vm_fqdn) ) {
137 minions.add(vm_fqdn)
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200138 }
139 }
140 }
141 }
142 }
143 }
144 }
145 }
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200146 }
Tomáš Kukrálb8e97112017-03-01 16:03:16 +0100147
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200148 def minions_compound = minions.join(' or ')
149
150 common.infoMsg("Waiting for next minions to register within ${wait_timeout} minutes: " + minions_compound)
151 timeout(time: wait_timeout, unit: 'MINUTES') {
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300152 salt.minionsPresentFromList(master, "I@salt:master ${extra_tgt}", minions, true, null, true, retries, 1)
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200153 }
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200154
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200155 common.infoMsg('Waiting for minions to respond')
156 timeout(time: wait_timeout, unit: 'MINUTES') {
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300157 salt.minionsReachable(master, "I@salt:master ${extra_tgt}", minions_compound)
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200158 }
159
160 common.infoMsg("All minions are up.")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300161 salt.fullRefresh(master, "* and not kvm* ${extra_tgt}")
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200162
Tomáš Kukrál8d2be5a2017-02-28 11:22:29 +0100163}
Jakub Josef79ecec32017-02-17 14:36:28 +0100164
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300165def installInfra(master, extra_tgt = '') {
Jakub Josef61463c72018-02-13 16:10:56 +0100166 def common = new com.mirantis.mk.Common()
Jakub Josef79ecec32017-02-17 14:36:28 +0100167 def salt = new com.mirantis.mk.Salt()
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400168 def first_target
Ales Komarekef2bab62017-04-26 11:48:28 +0200169
Jakub Josef79ecec32017-02-17 14:36:28 +0100170 // Install glusterfs
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300171 if (salt.testTarget(master, "I@glusterfs:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200172 salt.enforceState([saltId: master, target: "I@glusterfs:server ${extra_tgt}", state: 'glusterfs.server.service'])
Tomáš Kukrálb8e97112017-03-01 16:03:16 +0100173
Martin Polreich80fa3182018-10-22 15:16:53 +0200174 salt.enforceState([saltId: master, target: "I@glusterfs:server:role:primary ${extra_tgt}", state: 'glusterfs.server.setup', retries: 5])
Jakub Josef61463c72018-02-13 16:10:56 +0100175 sleep(10)
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300176 salt.cmdRun(master, "I@glusterfs:server ${extra_tgt}", "gluster peer status; gluster volume status")
Alexander Noskov93acac42017-06-20 17:55:55 +0200177 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200178
Oleg Grigorov26c8a6c2017-08-17 18:54:02 +0300179 // Ensure glusterfs clusters is ready
Martin Polreich80fa3182018-10-22 15:16:53 +0200180 salt.enforceStateWithTest([saltId: master, target: "I@glusterfs:client ${extra_tgt}", state: 'glusterfs.client', retries: 2])
Oleg Grigorov26c8a6c2017-08-17 18:54:02 +0300181
Jakub Josef79ecec32017-02-17 14:36:28 +0100182 // Install galera
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300183 if (salt.testTarget(master, "I@galera:master ${extra_tgt}") || salt.testTarget(master, "I@galera:slave ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200184 salt.enforceState([saltId: master, target: "I@galera:master ${extra_tgt}", state: 'galera', retries: 2])
185 salt.enforceStateWithTest([saltId: master, target: "I@galera:slave ${extra_tgt}", state: 'galera', retries: 2])
vrovachev1c4770b2017-07-05 13:25:21 +0400186
187 // Check galera status
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300188 salt.runSaltProcessStep(master, "I@galera:master ${extra_tgt}", 'mysql.status')
Dzmitry Stremkouskidf727b22018-10-29 17:19:55 +0100189 if (salt.testTarget(master, "I@galera:slave ${extra_tgt}")) {
190 salt.runSaltProcessStep(master, "I@galera:slave ${extra_tgt}", 'mysql.status')
191 }
192
Vasyl Saienkoc1fb9892017-07-13 14:54:13 +0300193 // If galera is not enabled check if we need to install mysql:server
Martin Polreich675dc942018-08-15 09:37:32 +0200194 } else {
Dzmitry Stremkouskidf727b22018-10-29 17:19:55 +0100195
Martin Polreich80fa3182018-10-22 15:16:53 +0200196 salt.enforceStateWithTest([saltId: master, target: "I@mysql:server ${extra_tgt}", state: 'mysql.server'])
197 salt.enforceStateWithTest([saltId: master, target: "I@mysql:client ${extra_tgt}", state: 'mysql.client'])
Dzmitry Stremkouskidf727b22018-10-29 17:19:55 +0100198
Ales Komarekef2bab62017-04-26 11:48:28 +0200199 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300200 installBackup(master, 'mysql', extra_tgt)
Ales Komarekef2bab62017-04-26 11:48:28 +0200201
vrovachev1c4770b2017-07-05 13:25:21 +0400202 // Install docker
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300203 if (salt.testTarget(master, "I@docker:host ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200204 salt.enforceState([saltId: master, target: "I@docker:host ${extra_tgt}", state: 'docker.host', retries: 3])
Pavel Cizinskyee34e312018-08-06 15:11:58 +0200205 salt.cmdRun(master, "I@docker:host and I@docker:host:enabled:true ${extra_tgt}", 'docker ps')
vrovachev1c4770b2017-07-05 13:25:21 +0400206 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200207
vrovachev1c4770b2017-07-05 13:25:21 +0400208 // Install keepalived
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300209 if (salt.testTarget(master, "I@keepalived:cluster ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400210 first_target = salt.getFirstMinion(master, "I@keepalived:cluster ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200211 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'keepalived'])
212 salt.enforceState([saltId: master, target: "I@keepalived:cluster ${extra_tgt}", state: 'keepalived'])
vrovachev1c4770b2017-07-05 13:25:21 +0400213 }
214
215 // Install rabbitmq
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300216 if (salt.testTarget(master, "I@rabbitmq:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200217 salt.enforceState([saltId: master, target: "I@rabbitmq:server ${extra_tgt}", state: 'rabbitmq', retries: 2])
vrovachev1c4770b2017-07-05 13:25:21 +0400218
219 // Check the rabbitmq status
Jakub Josef61463c72018-02-13 16:10:56 +0100220 common.retry(3,5){
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300221 salt.cmdRun(master, "I@rabbitmq:server ${extra_tgt}", 'rabbitmqctl cluster_status')
Jakub Josef61463c72018-02-13 16:10:56 +0100222 }
vrovachev1c4770b2017-07-05 13:25:21 +0400223 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200224
Jakub Josef79ecec32017-02-17 14:36:28 +0100225 // Install haproxy
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300226 if (salt.testTarget(master, "I@haproxy:proxy ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200227 salt.enforceState([saltId: master, target: "I@haproxy:proxy ${extra_tgt}", state: 'haproxy'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300228 salt.runSaltProcessStep(master, "I@haproxy:proxy ${extra_tgt}", 'service.status', ['haproxy'])
229 salt.runSaltProcessStep(master, "I@haproxy:proxy ${extra_tgt}", 'service.restart', ['rsyslog'])
vrovachev1c4770b2017-07-05 13:25:21 +0400230 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200231
Jakub Josef79ecec32017-02-17 14:36:28 +0100232 // Install memcached
Martin Polreich80fa3182018-10-22 15:16:53 +0200233 salt.enforceStateWithTest([saltId: master, target: "I@memcached:server ${extra_tgt}", state: 'memcached'])
vrovachev1c4770b2017-07-05 13:25:21 +0400234
235 // Install etcd
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300236 if (salt.testTarget(master, "I@etcd:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200237 salt.enforceState([saltId: master, target: "I@etcd:server ${extra_tgt}", state: 'etcd.server.service'])
Jakub Josef61463c72018-02-13 16:10:56 +0100238 common.retry(3,5){
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300239 salt.cmdRun(master, "I@etcd:server ${extra_tgt}", '. /var/lib/etcd/configenv && etcdctl cluster-health')
Jakub Josef61463c72018-02-13 16:10:56 +0100240 }
vrovachev1c4770b2017-07-05 13:25:21 +0400241 }
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200242
243 // Install redis
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300244 if (salt.testTarget(master, "I@redis:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200245 salt.enforceStateWithTest([saltId: master, target: "I@redis:cluster:role:master ${extra_tgt}", state: 'redis'])
246 salt.enforceState([saltId: master, target: "I@redis:server ${extra_tgt}", state: 'redis'])
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200247 }
Mykyta Karpin0a3dc8b2018-09-05 10:09:18 +0300248
249 // Install DNS services
250 if (salt.testTarget(master, "I@bind:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200251 salt.enforceState([saltId: master, target: "I@bind:server ${extra_tgt}", state: 'bind.server'])
Mykyta Karpin0a3dc8b2018-09-05 10:09:18 +0300252 }
253 if (salt.testTarget(master, "I@powerdns:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200254 salt.enforceState([saltId: master, target: "I@powerdns:server ${extra_tgt}", state: 'powerdns.server'])
Mykyta Karpin0a3dc8b2018-09-05 10:09:18 +0300255 }
256
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300257 installBackup(master, 'common', extra_tgt)
vrovachev1c4770b2017-07-05 13:25:21 +0400258}
259
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300260def installOpenstackInfra(master, extra_tgt = '') {
Jakub Josef432e9d92018-02-06 18:28:37 +0100261 def common = new com.mirantis.mk.Common()
262 common.warningMsg("You calling orchestrate.installOpenstackInfra(). This function is deprecated please use orchestrate.installInfra() directly")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300263 installInfra(master, extra_tgt)
Jakub Josef79ecec32017-02-17 14:36:28 +0100264}
265
266
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300267def installOpenstackControl(master, extra_tgt = '') {
Jakub Josef79ecec32017-02-17 14:36:28 +0100268 def salt = new com.mirantis.mk.Salt()
Jakub Josef61463c72018-02-13 16:10:56 +0100269 def common = new com.mirantis.mk.Common()
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400270 def first_target
Ales Komarekef2bab62017-04-26 11:48:28 +0200271
Jiri Broulike3e7bc32018-02-26 10:50:07 +0100272 // Install sphinx server
Martin Polreich80fa3182018-10-22 15:16:53 +0200273 salt.enforceStateWithTest([saltId: master, target: "I@sphinx:server ${extra_tgt}", state: 'sphinx'])
Mykyta Karpin014131a2018-12-18 17:35:38 +0200274 // Running minion states in a batch to avoid races related to certificates which are placed on glusterfs
275 // Details on races: https://mirantis.jira.com/browse/PROD-25796
276 // TODO: Run in parallel when glusterfs for certificates is dropped in cookiecutter
Martin Polreich80fa3182018-10-22 15:16:53 +0200277 salt.enforceStateWithTest([saltId: master, target: "I@nginx:server ${extra_tgt}", state: 'salt.minion', batch: 1])
278 salt.enforceStateWithTest([saltId: master, target: "I@nginx:server ${extra_tgt}", state: 'nginx'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200279
Jakub Josef79ecec32017-02-17 14:36:28 +0100280 // setup keystone service
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300281 if (salt.testTarget(master, "I@keystone:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200282 salt.enforceState([saltId: master, target: "I@keystone:server:role:primary ${extra_tgt}", state: 'keystone.server'])
283 salt.enforceState([saltId: master, target: "I@keystone:server ${extra_tgt}", state: 'keystone.server'])
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300284 // populate keystone services/tenants/roles/users
Tomáš Kukrál926e6332017-03-02 09:23:03 +0100285
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300286 // keystone:client must be called locally
287 //salt.runSaltProcessStep(master, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'], null, true)
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300288 salt.runSaltProcessStep(master, "I@keystone:server ${extra_tgt}", 'service.restart', ['apache2'])
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300289 sleep(30)
290 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300291 if (salt.testTarget(master, "I@keystone:client ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400292 first_target = salt.getFirstMinion(master, "I@keystone:client ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200293 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'keystone.client'])
294 salt.enforceState([saltId: master, target: "I@keystone:client ${extra_tgt}", state: 'keystone.client'])
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300295 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300296 if (salt.testTarget(master, "I@keystone:server ${extra_tgt}")) {
Jakub Josef61463c72018-02-13 16:10:56 +0100297 common.retry(3,5){
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300298 salt.cmdRun(master, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; openstack service list')
Jakub Josef61463c72018-02-13 16:10:56 +0100299 }
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300300 }
Tomáš Kukrál926e6332017-03-02 09:23:03 +0100301
Oleg Grigorov26c8a6c2017-08-17 18:54:02 +0300302 // Install glance
Martin Polreich80fa3182018-10-22 15:16:53 +0200303 salt.enforceStateWithTest([saltId: master, target: "I@glance:server:role:primary ${extra_tgt}", state: 'glance.server', testTargetMatcher: "I@glance:server ${extra_tgt}"])
304 salt.enforceStateWithTest([saltId: master, target: "I@glance:server ${extra_tgt}", state: 'glance.server'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200305
Oleg Grigorov26c8a6c2017-08-17 18:54:02 +0300306 // Check glance service
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300307 if (salt.testTarget(master, "I@glance:server ${extra_tgt}")) {
Mykyta Karpinfc169b72018-11-19 09:57:07 +0200308 common.retry(10,5){
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300309 salt.cmdRun(master, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; glance image-list')
Jakub Josef61463c72018-02-13 16:10:56 +0100310 }
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300311 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200312
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400313 // Create glance resources
Martin Polreich80fa3182018-10-22 15:16:53 +0200314 salt.enforceStateWithTest([saltId: master, target: "I@glance:client ${extra_tgt}", state: 'glance.client'])
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400315
Jakub Josef79ecec32017-02-17 14:36:28 +0100316 // Install and check nova service
Martin Polreich675dc942018-08-15 09:37:32 +0200317 // run on first node first
Martin Polreich80fa3182018-10-22 15:16:53 +0200318 salt.enforceStateWithTest([saltId: master, target: "I@nova:controller:role:primary ${extra_tgt}", state: 'nova.controller', testTargetMatcher: "I@nova:controller ${extra_tgt}"])
319 salt.enforceStateWithTest([saltId: master, target: "I@nova:controller ${extra_tgt}", state: 'nova.controller'])
Martin Polreich675dc942018-08-15 09:37:32 +0200320 if (salt.testTarget(master, "I@keystone:server and I@nova:controller ${extra_tgt}")) {
321 common.retry(3,5){
322 salt.cmdRun(master, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; nova service-list')
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300323 }
324 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200325
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400326 // Create nova resources
Martin Polreich80fa3182018-10-22 15:16:53 +0200327 salt.enforceStateWithTest([saltId: master, target: "I@nova:client ${extra_tgt}", state: 'nova.client'])
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400328
Jakub Josef79ecec32017-02-17 14:36:28 +0100329 // Install and check cinder service
Martin Polreich675dc942018-08-15 09:37:32 +0200330 // run on first node first
Martin Polreich80fa3182018-10-22 15:16:53 +0200331 salt.enforceStateWithTest([saltId: master, target: "I@cinder:controller:role:primary ${extra_tgt}", state: 'cinder', testTargetMatcher: "I@cinder:controller ${extra_tgt}"])
332 salt.enforceStateWithTest([saltId: master, target: "I@cinder:controller ${extra_tgt}", state: 'cinder'])
Martin Polreich675dc942018-08-15 09:37:32 +0200333 if (salt.testTarget(master, "I@keystone:server and I@cinder:controller ${extra_tgt}")) {
334 common.retry(3,5){
335 salt.cmdRun(master, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; cinder list')
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300336 }
337 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200338
Jakub Josef79ecec32017-02-17 14:36:28 +0100339 // Install neutron service
Martin Polreich675dc942018-08-15 09:37:32 +0200340 // run on first node first
Martin Polreich80fa3182018-10-22 15:16:53 +0200341 salt.enforceStateWithTest([saltId: master, target: "I@neutron:server:role:primary ${extra_tgt}", state: 'neutron.server', testTargetMatcher: "I@neutron:server ${extra_tgt}"])
342 salt.enforceStateWithTest([saltId: master, target: "I@neutron:server ${extra_tgt}", state: 'neutron.server'])
Martin Polreich675dc942018-08-15 09:37:32 +0200343 if (salt.testTarget(master, "I@keystone:server and I@neutron:server ${extra_tgt}")) {
Vasyl Saienko0b86a4f2018-11-16 14:02:34 +0200344 common.retry(10,5){
Martin Polreich675dc942018-08-15 09:37:32 +0200345 salt.cmdRun(master, "I@keystone:server ${extra_tgt}",'. /root/keystonercv3; neutron agent-list')
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300346 }
347 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200348
Jakub Josef79ecec32017-02-17 14:36:28 +0100349 // Install heat service
Martin Polreich80fa3182018-10-22 15:16:53 +0200350 salt.enforceStateWithTest([saltId: master, target: "I@heat:server:role:primary ${extra_tgt}", state: 'heat', testTargetMatcher: "I@heat:server ${extra_tgt}"])
351 salt.enforceStateWithTest([saltId: master, target: "I@heat:server ${extra_tgt}", state: 'heat'])
Martin Polreich675dc942018-08-15 09:37:32 +0200352 if (salt.testTarget(master, "I@keystone:server and I@heat:server ${extra_tgt}")) {
Mykyta Karpina678da62018-12-20 17:26:51 +0200353 common.retry(10,5){
Martin Polreich675dc942018-08-15 09:37:32 +0200354 salt.cmdRun(master, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; openstack orchestration resource type list')
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300355 }
356 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200357
358 // Restart nova api
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300359 if (salt.testTarget(master, "I@nova:controller ${extra_tgt}")) {
360 salt.runSaltProcessStep(master, "I@nova:controller ${extra_tgt}", 'service.restart', ['nova-api'])
Vasyl Saienko962e0b52017-07-11 10:49:19 +0300361 }
Vasyl Saienkoa7c10c02017-07-13 10:43:43 +0300362
363 // Install ironic service
Martin Polreich80fa3182018-10-22 15:16:53 +0200364 salt.enforceStateWithTest([saltId: master, target: "I@ironic:api:role:primary ${extra_tgt}", state: 'ironic.api', testTargetMatcher: "I@ironic:api ${extra_tgt}"])
365 salt.enforceStateWithTest([saltId: master, target: "I@ironic:api ${extra_tgt}", state: 'ironic.api'])
Andrey Shestakov82c0b892017-07-24 18:46:21 +0300366
Vasyl Saienkob7e4cdf2018-02-21 18:33:17 +0200367 // Install manila service
Martin Polreich80fa3182018-10-22 15:16:53 +0200368 salt.enforceStateWithTest([saltId: master, target: "I@manila:api:role:primary ${extra_tgt}", state: 'manila.api', testTargetMatcher: "I@manila:api ${extra_tgt}"])
369 salt.enforceStateWithTest([saltId: master, target: "I@manila:api ${extra_tgt}", state: 'manila.api'])
370 salt.enforceStateWithTest([saltId: master, target: "I@manila:scheduler ${extra_tgt}", state: 'manila.scheduler'])
Vasyl Saienkob7e4cdf2018-02-21 18:33:17 +0200371
Oleh Hryhorova070fd02018-03-15 14:47:52 +0200372 // Install designate services
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300373 if (salt.testTarget(master, "I@designate:server:enabled ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200374 salt.enforceState([saltId: master, target: "I@designate:server:role:primary ${extra_tgt}", state: 'designate.server'])
375 salt.enforceState([saltId: master, target: "I@designate:server ${extra_tgt}", state: 'designate'])
Andrey Shestakov82c0b892017-07-24 18:46:21 +0300376 }
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400377
378 // Install octavia api service
Martin Polreich80fa3182018-10-22 15:16:53 +0200379 salt.enforceStateWithTest([saltId: master, target: "I@octavia:api:role:primary ${extra_tgt}", state: 'octavia.api', testTargetMatcher: "I@octavia:api ${extra_tgt}"])
380 salt.enforceStateWithTest([saltId: master, target: "I@octavia:api ${extra_tgt}", state: 'octavia.api'])
Marek Celoudc9dd50f2017-08-28 13:54:40 +0200381
Oleg Iurchenkod0898c92017-12-14 14:28:44 +0200382 // Install DogTag server service
Martin Polreich80fa3182018-10-22 15:16:53 +0200383 salt.enforceStateWithTest([saltId: master, target: "I@dogtag:server:role:master ${extra_tgt}", state: 'dogtag.server', testTargetMatcher: "I@dogtag:server ${extra_tgt}"])
Vasyl Saienko6f83b6b2019-02-01 11:55:52 +0200384 // Run dogtag state on slaves in serial to avoid races during replications PROD-26810
385 salt.enforceStateWithTest([saltId: master, target: "I@dogtag:server ${extra_tgt}", state: 'dogtag.server', batch: 1])
Oleg Iurchenkod0898c92017-12-14 14:28:44 +0200386
Marek Celoudc9dd50f2017-08-28 13:54:40 +0200387 // Install barbican server service
Martin Polreich80fa3182018-10-22 15:16:53 +0200388 salt.enforceStateWithTest([saltId: master, target: "I@barbican:server:role:primary ${extra_tgt}", state: 'barbican.server', testTargetMatcher: "I@barbican:server ${extra_tgt}"])
389 salt.enforceStateWithTest([saltId: master, target: "I@barbican:server ${extra_tgt}", state: 'barbican.server'])
Martin Polreich675dc942018-08-15 09:37:32 +0200390
Vasyl Saienkoa96a7662019-02-04 14:29:19 +0200391 if (salt.testTarget(master, "I@barbican:server ${extra_tgt}")) {
392 // Restart apache to make sure we don't have races between barbican-api and barbican-worker on db init.
393 // For more info please see PROD-26988
394 // The permanent fix is prepared to barbican formula https://gerrit.mcp.mirantis.com/#/c/35097/ but due to rush in release
395 // add this workaround here as well.
396 // TODO(vsaienko): cleanup once release passed in favor of permanent fix.
397 salt.runSaltProcessStep(master, "I@barbican:server ${extra_tgt}", 'service.restart', ['apache2'])
398 sleep(30)
399 }
400
Marek Celoudc9dd50f2017-08-28 13:54:40 +0200401 // Install barbican client
Martin Polreich80fa3182018-10-22 15:16:53 +0200402 salt.enforceStateWithTest([saltId: master, target: "I@barbican:client ${extra_tgt}", state: 'barbican.client'])
Ondrej Smola5908ae62018-01-08 13:42:08 +0100403
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200404 // Install gnocchi server
Martin Polreich80fa3182018-10-22 15:16:53 +0200405 salt.enforceStateWithTest([saltId: master, target: "I@gnocchi:server:role:primary ${extra_tgt}", state: 'gnocchi.server', testTargetMatcher: "I@gnocchi:server ${extra_tgt}"])
406 salt.enforceStateWithTest([saltId: master, target: "I@gnocchi:server ${extra_tgt}", state: 'gnocchi.server'])
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200407
Mykyta Karpin4816e522018-06-15 16:00:03 +0300408 // Apply gnocchi client state to create gnocchi archive policies, due to possible
409 // races, apply on the first node initially
410 if (salt.testTarget(master, "I@gnocchi:client ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400411 first_target = salt.getFirstMinion(master, "I@gnocchi:client ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200412 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'gnocchi.client'])
413 salt.enforceState([saltId: master, target: "I@gnocchi:client ${extra_tgt}", state: 'gnocchi.client'])
Mykyta Karpin4816e522018-06-15 16:00:03 +0300414 }
415
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200416 // Install gnocchi statsd
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300417 if (salt.testTarget(master, "I@gnocchi:statsd ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400418 first_target = salt.getFirstMinion(master, "I@gnocchi:statsd ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200419 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'gnocchi.statsd'])
420 salt.enforceState([saltId: master, target: "I@gnocchi:statsd ${extra_tgt}", state: 'gnocchi.statsd'])
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200421 }
422
423 // Install panko server
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300424 if (salt.testTarget(master, "I@panko:server ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400425 first_target = salt.getFirstMinion(master, "I@panko:server ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200426 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'panko'])
427 salt.enforceState([saltId: master, target: "I@panko:server ${extra_tgt}", state: 'panko'])
Mykyta Karpin8185e4a2018-02-23 19:15:48 +0200428 }
429
Ondrej Smola5908ae62018-01-08 13:42:08 +0100430 // Install ceilometer server
Martin Polreich80fa3182018-10-22 15:16:53 +0200431 salt.enforceStateWithTest([saltId: master, target: "I@ceilometer:server:role:primary ${extra_tgt}", state: 'ceilometer', testTargetMatcher: "I@ceilometer:server ${extra_tgt}"])
432 salt.enforceStateWithTest([saltId: master, target: "I@ceilometer:server ${extra_tgt}", state: 'ceilometer'])
Ondrej Smola5908ae62018-01-08 13:42:08 +0100433
434 // Install aodh server
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300435 if (salt.testTarget(master, "I@aodh:server ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400436 first_target = salt.getFirstMinion(master, "I@aodh:server ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200437 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'aodh'])
438 salt.enforceState([saltId: master, target: "I@aodh:server ${extra_tgt}", state: 'aodh'])
Ondrej Smola5908ae62018-01-08 13:42:08 +0100439 }
Oleksandr Bryndzii642cb252018-11-08 12:24:42 +0200440
441 // Install horizon dashboard
Martin Polreich80fa3182018-10-22 15:16:53 +0200442 salt.enforceStateWithTest([saltId: master, target: "I@horizon:server ${extra_tgt}", state: 'horizon'])
Jakub Josef79ecec32017-02-17 14:36:28 +0100443}
444
445
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300446def installIronicConductor(master, extra_tgt = ''){
Vasyl Saienkoa7c10c02017-07-13 10:43:43 +0300447 def salt = new com.mirantis.mk.Salt()
448
Martin Polreich80fa3182018-10-22 15:16:53 +0200449 salt.enforceStateWithTest([saltId: master, target: "I@ironic:conductor ${extra_tgt}", state: 'ironic.conductor'])
450 salt.enforceStateWithTest([saltId: master, target: "I@ironic:conductor ${extra_tgt}", state: 'apache'])
451 salt.enforceStateWithTest([saltId: master, target: "I@tftpd_hpa:server ${extra_tgt}", state: 'tftpd_hpa'])
Vasyl Saienkoa7c10c02017-07-13 10:43:43 +0300452
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300453 if (salt.testTarget(master, "I@nova:compute ${extra_tgt}")) {
454 salt.runSaltProcessStep(master, "I@nova:compute ${extra_tgt}", 'service.restart', ['nova-compute'])
Vasyl Saienkoa7c10c02017-07-13 10:43:43 +0300455 }
456
Martin Polreich80fa3182018-10-22 15:16:53 +0200457 salt.enforceStateWithTest([saltId: master, target: "I@baremetal_simulator:enabled ${extra_tgt}", state: 'baremetal_simulator'])
458 salt.enforceStateWithTest([saltId: master, target: "I@ironic:client ${extra_tgt}", state: 'ironic.client'])
Vasyl Saienkoa7c10c02017-07-13 10:43:43 +0300459}
460
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300461def installManilaShare(master, extra_tgt = ''){
Vasyl Saienkob7e4cdf2018-02-21 18:33:17 +0200462 def salt = new com.mirantis.mk.Salt()
463
Martin Polreich80fa3182018-10-22 15:16:53 +0200464 salt.enforceStateWithTest([saltId: master, target: "I@manila:share ${extra_tgt}", state: 'manila.share'])
465 salt.enforceStateWithTest([saltId: master, target: "I@manila:data ${extra_tgt}", state: 'manila.data'])
466 salt.enforceStateWithTest([saltId: master, target: "I@manila:client ${extra_tgt}", state: 'manila.client'])
Vasyl Saienkob7e4cdf2018-02-21 18:33:17 +0200467}
Vasyl Saienkoa7c10c02017-07-13 10:43:43 +0300468
469
Oleh Hryhorov37fbd0c2018-07-11 16:58:54 +0300470def installOpenstackNetwork(master, extra_tgt = '') {
Jakub Josef79ecec32017-02-17 14:36:28 +0100471 def salt = new com.mirantis.mk.Salt()
Ann Taradayea9848c2018-10-25 11:15:58 +0400472 def common = new com.mirantis.mk.Common()
Dzmitry Stremkouskidf727b22018-10-29 17:19:55 +0100473
Oleksii Grudevc4a485e2018-04-20 14:25:03 +0300474 //run full neutron state on neutron.gateway - this will install
475 //neutron agents in addition to neutron server. Once neutron agents
476 //are up neutron resources can be created without hitting the situation when neutron resources are created
477 //prior to neutron agents which results in creating ports in non-usable state
Martin Polreich80fa3182018-10-22 15:16:53 +0200478 salt.enforceStateWithTest([saltId: master, target: "I@neutron:gateway ${extra_tgt}", state: 'neutron'])
Oleksii Grudevc4a485e2018-04-20 14:25:03 +0300479
480 // Create neutron resources - this step was moved here to ensure that
481 //neutron resources are created after neutron agens are up. In this case neutron ports will be in
482 //usable state. More information: https://bugs.launchpad.net/neutron/+bug/1399249
Martin Polreich80fa3182018-10-22 15:16:53 +0200483 salt.enforceStateWithTest([saltId: master, target: "I@neutron:client ${extra_tgt}", state: 'neutron.client'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200484
Dzmitry Stremkouskidf727b22018-10-29 17:19:55 +0100485 if (salt.testTarget(master, "I@neutron:gateway ${extra_tgt}")) {
486 salt.enforceHighstate(master, "I@neutron:gateway ${extra_tgt}")
487 }
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400488
489 // install octavia manager services
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300490 if (salt.testTarget(master, "I@octavia:manager ${extra_tgt}")) {
Anna Taraday82224102018-10-09 07:44:24 +0000491 salt.runSaltProcessStep(master, "I@neutron:client ${extra_tgt}", 'mine.update')
Martin Polreich80fa3182018-10-22 15:16:53 +0200492 salt.enforceState([saltId: master, target: "I@octavia:manager ${extra_tgt}", state: 'octavia.manager'])
493 salt.enforceState([saltId: master, target: "I@octavia:client ${extra_tgt}", state: 'octavia.client'])
Elena Ezhova8d53bff2017-08-01 17:43:30 +0400494 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200495}
496
497
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300498def installOpenstackCompute(master, extra_tgt = '') {
Ales Komarekef2bab62017-04-26 11:48:28 +0200499 def salt = new com.mirantis.mk.Salt()
Mykyta Karpin3ec78a22018-07-10 12:27:09 +0300500 def common = new com.mirantis.mk.Common()
Ales Komarekef2bab62017-04-26 11:48:28 +0200501 // Configure compute nodes
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300502 def compute_compound = "I@nova:compute ${extra_tgt}"
Dzmitry Stremkouskifeb8ce92018-03-21 18:03:28 +0100503 if (salt.testTarget(master, compute_compound)) {
504 // In case if infrastructure nodes are used as nova computes too
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300505 def gluster_compound = "I@glusterfs:server ${extra_tgt}"
506 def salt_ca_compound = "I@salt:minion:ca:salt_master_ca ${extra_tgt}"
Dzmitry Stremkouski7eca99d2018-04-25 18:44:26 +0200507 // Enforce highstate asynchronous only on compute nodes which are not glusterfs and not salt ca servers
Mykyta Karpinb1eaba92018-07-10 10:09:37 +0300508 def hightstateTarget = "${compute_compound} and not ${gluster_compound} and not ${salt_ca_compound}"
Oleh Hryhorov73f9fa22018-05-02 17:36:58 +0300509 if (salt.testTarget(master, hightstateTarget)) {
510 retry(2) {
Vasyl Saienkobe599d12018-11-10 06:42:50 +0000511 salt.enforceHighstate(master, hightstateTarget)
Oleh Hryhorov73f9fa22018-05-02 17:36:58 +0300512 }
Mykyta Karpinb1eaba92018-07-10 10:09:37 +0300513 } else {
514 common.infoMsg("No minions matching highstate target found for target ${hightstateTarget}")
Dzmitry Stremkouski7eca99d2018-04-25 18:44:26 +0200515 }
516 // Iterate through salt ca servers and check if they have compute role
517 // TODO: switch to batch once salt 2017.7+ would be used
Mykyta Karpinb1eaba92018-07-10 10:09:37 +0300518 common.infoMsg("Checking whether ${salt_ca_compound} minions have ${compute_compound} compound")
Dzmitry Stremkouski7eca99d2018-04-25 18:44:26 +0200519 for ( target in salt.getMinionsSorted(master, salt_ca_compound) ) {
520 for ( cmp_target in salt.getMinionsSorted(master, compute_compound) ) {
521 if ( target == cmp_target ) {
522 // Enforce highstate one by one on salt ca servers which are compute nodes
523 retry(2) {
Vasyl Saienkobe599d12018-11-10 06:42:50 +0000524 salt.enforceHighstate(master, target)
Dzmitry Stremkouski7eca99d2018-04-25 18:44:26 +0200525 }
526 }
527 }
Dzmitry Stremkouskifeb8ce92018-03-21 18:03:28 +0100528 }
529 // Iterate through glusterfs servers and check if they have compute role
530 // TODO: switch to batch once salt 2017.7+ would be used
Mykyta Karpinb1eaba92018-07-10 10:09:37 +0300531 common.infoMsg("Checking whether ${gluster_compound} minions have ${compute_compound} compound")
Dzmitry Stremkouskifeb8ce92018-03-21 18:03:28 +0100532 for ( target in salt.getMinionsSorted(master, gluster_compound) ) {
533 for ( cmp_target in salt.getMinionsSorted(master, compute_compound) ) {
534 if ( target == cmp_target ) {
535 // Enforce highstate one by one on glusterfs servers which are compute nodes
536 retry(2) {
Vasyl Saienkobe599d12018-11-10 06:42:50 +0000537 salt.enforceHighstate(master, target)
Dzmitry Stremkouskifeb8ce92018-03-21 18:03:28 +0100538 }
539 }
540 }
Sergio Lystopadcd5b2772018-02-09 14:37:10 +0200541 }
Ales Komarekef2bab62017-04-26 11:48:28 +0200542 }
Vasyl Saienko51144e42018-06-19 09:30:30 +0300543
544 // Run nova:controller to map cmp with cells
Martin Polreich80fa3182018-10-22 15:16:53 +0200545 salt.enforceStateWithTest([saltId: master, target: "I@nova:controller:role:primary ${extra_tgt}", state: 'nova.controller', testTargetMatcher: "I@nova:controller ${extra_tgt}"])
Ales Komarekef2bab62017-04-26 11:48:28 +0200546}
547
548
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300549def installContrailNetwork(master, extra_tgt = '') {
Martin Polreichfc9854c2017-06-30 11:17:36 +0200550 def common = new com.mirantis.mk.Common()
Ales Komarekef2bab62017-04-26 11:48:28 +0200551 def salt = new com.mirantis.mk.Salt()
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400552 def first_target
Ales Komarekef2bab62017-04-26 11:48:28 +0200553
Jakub Josef79ecec32017-02-17 14:36:28 +0100554 // Install opencontrail database services
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400555 first_target = salt.getFirstMinion(master, "I@opencontrail:database ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200556 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'opencontrail.database'])
557 salt.enforceState([saltId: master, target: "I@opencontrail:database ${extra_tgt}", state: 'opencontrail.database'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200558
Jakub Josef79ecec32017-02-17 14:36:28 +0100559 // Install opencontrail control services
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400560 first_target = salt.getFirstMinion(master, "I@opencontrail:control ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200561 salt.enforceStateWithExclude([saltId: master, target: "${first_target} ${extra_tgt}", state: "opencontrail", excludedStates: "opencontrail.client"])
562 salt.enforceStateWithExclude([saltId: master, target: "I@opencontrail:control ${extra_tgt}", state: "opencontrail", excludedStates: "opencontrail.client"])
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400563 first_target = salt.getFirstMinion(master, "I@opencontrail:collector ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200564 salt.enforceStateWithExclude([saltId: master, target: "${first_target} ${extra_tgt}", state: "opencontrail", excludedStates: "opencontrail.client"])
565 salt.enforceStateWithExclude([saltId: master, target: "I@opencontrail:collector ${extra_tgt}", state: "opencontrail", excludedStates: "opencontrail.client"])
Tomáš Kukrálb8e97112017-03-01 16:03:16 +0100566
Martin Polreich80fa3182018-10-22 15:16:53 +0200567 salt.enforceStateWithTest([saltId: master, target: "( I@opencontrail:control or I@opencontrail:collector ) ${extra_tgt}", state: 'docker.client', testTargetMatcher: "I@docker:client and I@opencontrail:control ${extra_tgt}"])
Anton Samoylov328a5722019-01-23 17:37:33 +0400568}
Svimbafb242db2019-01-09 17:27:20 +0100569
Anton Samoylov328a5722019-01-23 17:37:33 +0400570
571def checkContrailApiReadiness(master, extra_tgt = '') {
572 def common = new com.mirantis.mk.Common()
573 def salt = new com.mirantis.mk.Salt()
574
Anton Samoylovaf569b52019-01-25 14:49:52 +0400575 def apiCheckResult = salt.getReturnValues(salt.runSaltProcessStep(master, "I@opencontrail:control:role:primary ${extra_tgt}", 'contrail_health.get_api_status', ['wait_for=900', 'tries=50']))
Anton Samoylov37b58292019-01-22 15:28:31 +0400576 if (!apiCheckResult){
577 throw new Exception("Contrail is not working after deployment: contrail-api service is not in healthy state")
Anton Samoylov328a5722019-01-23 17:37:33 +0400578 } else {
579 common.infoMsg('Contrail API is ready to service requests')
Anton Samoylov37b58292019-01-22 15:28:31 +0400580 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100581}
582
583
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300584def installContrailCompute(master, extra_tgt = '') {
Tomáš Kukrál104a07a2017-07-13 12:19:11 +0200585 def salt = new com.mirantis.mk.Salt()
vrovachev32f7f542017-08-25 16:17:25 +0400586 def common = new com.mirantis.mk.Common()
Jakub Josef79ecec32017-02-17 14:36:28 +0100587 // Configure compute nodes
Ales Komarekef2bab62017-04-26 11:48:28 +0200588 // Provision opencontrail control services
Martin Polreich80fa3182018-10-22 15:16:53 +0200589 salt.enforceState([saltId: master, target: "I@opencontrail:database:id:1 ${extra_tgt}", state: 'opencontrail.client'])
Jakub Josef79ecec32017-02-17 14:36:28 +0100590 // Provision opencontrail virtual routers
Tomáš Kukrálb8e97112017-03-01 16:03:16 +0100591
vrovachev4ff77302017-08-09 20:39:57 +0400592 // Generate script /usr/lib/contrail/if-vhost0 for up vhost0
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300593 if (salt.testTarget(master, "I@opencontrail:compute ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200594 salt.enforceStateWithExclude([saltId: master, target: "I@opencontrail:compute ${extra_tgt}", state: "opencontrail", excludedStates: "opencontrail.client"])
Sergio Lystopadcd5b2772018-02-09 14:37:10 +0200595 }
vrovachev4ff77302017-08-09 20:39:57 +0400596
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300597 if (salt.testTarget(master, "I@nova:compute ${extra_tgt}")) {
Anton Samoylov817b0df2018-12-11 19:00:28 +0400598 salt.cmdRun(master, "I@nova:compute ${extra_tgt}", 'exec 0>&-; exec 1>&-; exec 2>&-; nohup bash -c "ip link | grep vhost && echo no_reboot || reboot & "', false)
Sergio Lystopadcd5b2772018-02-09 14:37:10 +0200599 }
vrovachev4ff77302017-08-09 20:39:57 +0400600
Jakub Josef61463c72018-02-13 16:10:56 +0100601 sleep(300)
Martin Polreich80fa3182018-10-22 15:16:53 +0200602 salt.enforceStateWithTest([saltId: master, target: "I@opencontrail:compute ${extra_tgt}", state: 'opencontrail.client'])
603 salt.enforceStateWithTest([saltId: master, target: "I@opencontrail:compute ${extra_tgt}", state: 'opencontrail'])
Jakub Josef79ecec32017-02-17 14:36:28 +0100604}
605
606
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300607def installKubernetesInfra(master, extra_tgt = '') {
Jakub Josef432e9d92018-02-06 18:28:37 +0100608 def common = new com.mirantis.mk.Common()
609 common.warningMsg("You calling orchestrate.installKubernetesInfra(). This function is deprecated please use orchestrate.installInfra() directly")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300610 installInfra(master, extra_tgt)
Jakub Josef79ecec32017-02-17 14:36:28 +0100611}
612
613
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300614def installKubernetesControl(master, extra_tgt = '') {
Jakub Josef79ecec32017-02-17 14:36:28 +0100615 def salt = new com.mirantis.mk.Salt()
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400616 def first_target
Victor Ryzhenkineb617412018-08-20 18:54:23 +0400617 salt.fullRefresh(master, "* ${extra_tgt}")
618
619 // Bootstrap all nodes
Martin Polreich80fa3182018-10-22 15:16:53 +0200620 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'linux'])
621 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'salt.minion'])
622 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: ['openssh', 'ntp']])
Victor Ryzhenkineb617412018-08-20 18:54:23 +0400623
624 // Create and distribute SSL certificates for services using salt state
Martin Polreich80fa3182018-10-22 15:16:53 +0200625 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'salt.minion.cert'])
Victor Ryzhenkineb617412018-08-20 18:54:23 +0400626
627 // Install docker
Victor Ryzhenkindbb39862019-01-09 18:24:16 +0400628 salt.enforceStateWithTest([saltId: master, target: "I@docker:host ${extra_tgt}", state: 'docker.host'])
Tomáš Kukrálb05cf002017-05-02 11:05:43 +0200629
Svimba470b3312018-09-26 11:27:20 +0200630 // If network engine is not opencontrail, run addons state for kubernetes
631 if (!salt.getPillar(master, "I@kubernetes:master ${extra_tgt}", 'kubernetes:master:network:opencontrail:enabled')) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200632 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'kubernetes.master.kube-addons'])
Svimba470b3312018-09-26 11:27:20 +0200633 }
634
Jakub Josef79ecec32017-02-17 14:36:28 +0100635 // Install Kubernetes pool and Calico
Martin Polreich80fa3182018-10-22 15:16:53 +0200636 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'kubernetes.pool'])
Jakub Josef79ecec32017-02-17 14:36:28 +0100637
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300638 if (salt.testTarget(master, "I@etcd:server:setup ${extra_tgt}")) {
Marek Celoud029391c2017-11-14 10:06:18 +0100639 // Setup etcd server
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400640 first_target = salt.getFirstMinion(master, "I@kubernetes:master ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200641 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'etcd.server.setup'])
Marek Celoud029391c2017-11-14 10:06:18 +0100642 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100643
Victor Ryzhenkinfb5d1b52018-05-22 18:49:26 +0400644 // Run k8s master at *01* to simplify namespaces creation
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400645 first_target = salt.getFirstMinion(master, "I@kubernetes:master ${extra_tgt}")
Victor Ryzhenkinfb5d1b52018-05-22 18:49:26 +0400646
Svimba470b3312018-09-26 11:27:20 +0200647 // If network engine is opencontrail, run master state for kubernetes without kube-addons
648 // The kube-addons state will be called later only in case of opencontrail
649 if (salt.getPillar(master, "I@kubernetes:master ${extra_tgt}", 'kubernetes:master:network:opencontrail:enabled')) {
650 // Run k8s on first node without master.setup and master.kube-addons
Martin Polreich80fa3182018-10-22 15:16:53 +0200651 salt.enforceStateWithExclude([saltId: master, target: "${first_target} ${extra_tgt}", state: "kubernetes.master", excludedStates: "kubernetes.master.setup,kubernetes.master.kube-addons"])
Svimba470b3312018-09-26 11:27:20 +0200652 // Run k8s without master.setup and master.kube-addons
Martin Polreich80fa3182018-10-22 15:16:53 +0200653 salt.enforceStateWithExclude([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: "kubernetes", excludedStates: "kubernetes.master.setup,kubernetes.master.kube-addons"])
Svimba470b3312018-09-26 11:27:20 +0200654 } else {
655 // Run k8s on first node without master.setup and master.kube-addons
Martin Polreich80fa3182018-10-22 15:16:53 +0200656 salt.enforceStateWithExclude([saltId: master, target: "${first_target} ${extra_tgt}", state: "kubernetes.master", excludedStates: "kubernetes.master.setup"])
Svimba470b3312018-09-26 11:27:20 +0200657 // Run k8s without master.setup
Martin Polreich80fa3182018-10-22 15:16:53 +0200658 salt.enforceStateWithExclude([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: "kubernetes", excludedStates: "kubernetes.master.setup"])
Svimba470b3312018-09-26 11:27:20 +0200659 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100660
661 // Run k8s master setup
Martin Polreich80fa3182018-10-22 15:16:53 +0200662 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'kubernetes.master.setup'])
Jakub Josef79ecec32017-02-17 14:36:28 +0100663
664 // Restart kubelet
Victor Ryzhenkinc8d5f8f2018-09-10 16:55:13 +0400665 salt.runSaltProcessStep(master, "I@kubernetes:master ${extra_tgt}", 'service.restart', ['kubelet'])
Jakub Josef79ecec32017-02-17 14:36:28 +0100666}
667
668
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300669def installKubernetesCompute(master, extra_tgt = '') {
Tomáš Kukrála53b5352017-05-04 11:45:53 +0200670 def salt = new com.mirantis.mk.Salt()
Jakub Josef432e9d92018-02-06 18:28:37 +0100671 salt.fullRefresh(master, "*")
Tomáš Kukrále7e28962017-06-06 11:11:16 +0200672
673 // Bootstrap all nodes
Martin Polreich80fa3182018-10-22 15:16:53 +0200674 salt.enforceState([saltId: master, target: "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", state: 'linux'])
675 salt.enforceState([saltId: master, target: "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", state: 'salt.minion'])
676 salt.enforceState([saltId: master, target: "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", state: ['openssh', 'ntp']])
Tomáš Kukrále7e28962017-06-06 11:11:16 +0200677
678 // Create and distribute SSL certificates for services using salt state
Martin Polreich80fa3182018-10-22 15:16:53 +0200679 salt.enforceState([saltId: master, target: "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", state: 'salt.minion.cert'])
Tomáš Kukrále7e28962017-06-06 11:11:16 +0200680
681 // Install docker
Victor Ryzhenkindbb39862019-01-09 18:24:16 +0400682 salt.enforceStateWithTest([saltId: master, target: "I@docker:host ${extra_tgt}", state: 'docker.host'])
Tomáš Kukrále7e28962017-06-06 11:11:16 +0200683
684 // Install Kubernetes and Calico
Martin Polreich80fa3182018-10-22 15:16:53 +0200685 salt.enforceState([saltId: master, target: "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", state: 'kubernetes.pool'])
Tomáš Kukrále7e28962017-06-06 11:11:16 +0200686
Yuriy Taradayab564d72017-10-11 15:14:35 +0400687 // Install Tiller and all configured releases
Martin Polreich80fa3182018-10-22 15:16:53 +0200688 salt.enforceStateWithTest([saltId: master, target: "I@helm:client ${extra_tgt}", state: 'helm'])
Victor Ryzhenkinc8d5f8f2018-09-10 16:55:13 +0400689 salt.runSaltProcessStep(master, "I@kubernetes:pool and not I@kubernetes:master ${extra_tgt}", 'service.restart', ['kubelet'])
Tomáš Kukrále7e28962017-06-06 11:11:16 +0200690}
691
692
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300693def installDockerSwarm(master, extra_tgt = '') {
Martin Polreich8663a342017-06-30 17:33:08 +0200694 def salt = new com.mirantis.mk.Salt()
Jakub Josef79ecec32017-02-17 14:36:28 +0100695
vrovachev1c4770b2017-07-05 13:25:21 +0400696 //Install and Configure Docker
Martin Polreichc0bb4fd2018-08-06 11:34:43 +0200697 if (salt.testTarget(master, "I@docker:swarm ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200698 salt.enforceState([saltId: master, target: "I@docker:swarm ${extra_tgt}", state: 'docker.host'])
699 salt.enforceState([saltId: master, target: "I@docker:swarm:role:master ${extra_tgt}", state: 'docker.swarm'])
700 salt.enforceState([saltId: master, target: "I@docker:swarm ${extra_tgt}", state: 'salt.minion.grains'])
Martin Polreichc0bb4fd2018-08-06 11:34:43 +0200701 salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'mine.update')
702 salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'saltutil.refresh_modules')
703 sleep(5)
Martin Polreich80fa3182018-10-22 15:16:53 +0200704 salt.enforceState([saltId: master, target: "I@docker:swarm:role:master ${extra_tgt}", state: 'docker.swarm'])
705 salt.enforceStateWithTest([saltId: master, target: "I@docker:swarm:role:manager ${extra_tgt}", state: 'docker.swarm'])
Martin Polreichc0bb4fd2018-08-06 11:34:43 +0200706 sleep(10)
707 salt.cmdRun(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker node ls')
Machi Hoshino87aa39d2018-07-26 20:41:53 +0900708 }
Martin Polreich8663a342017-06-30 17:33:08 +0200709}
710
Svimba470b3312018-09-26 11:27:20 +0200711// Setup addons for kubernetes - For OpenContrail network engine
712// Use after compute nodes are ready, because K8s addons like DNS should be placed on cmp nodes
713def setupKubeAddonForContrail(master, extra_tgt = '') {
714 def salt = new com.mirantis.mk.Salt()
715
716 if (salt.getPillar(master, "I@kubernetes:master ${extra_tgt}", 'kubernetes:master:network:opencontrail:enabled')){
717 // Setup Addons for Kubernetes only in case of OpenContrail is used as neteork engine
Martin Polreich80fa3182018-10-22 15:16:53 +0200718 salt.enforceState([saltId: master, target: "I@kubernetes:master ${extra_tgt}", state: 'kubernetes.master.kube-addons'])
Svimba470b3312018-09-26 11:27:20 +0200719 }
720}
Jiri Broulikb9a0c592017-09-05 16:41:48 +0200721
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300722def installCicd(master, extra_tgt = '') {
Jiri Broulikb9a0c592017-09-05 16:41:48 +0200723 def salt = new com.mirantis.mk.Salt()
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200724 def common = new com.mirantis.mk.Common()
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400725 def gerrit_compound = "I@gerrit:client ${extra_tgt}"
726 def jenkins_compound = "I@jenkins:client ${extra_tgt}"
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200727
728 salt.fullRefresh(master, gerrit_compound)
729 salt.fullRefresh(master, jenkins_compound)
Richard Felkl9d2cd922017-11-03 11:49:32 +0100730
Ivan Berezovskiy3aa91f52018-11-28 18:47:34 +0400731 // Temporary exclude cfg node from docker.client state (PROD-24934)
732 def dockerClientExclude = !salt.getPillar(master, 'I@salt:master', 'docker:client:stack:jenkins').isEmpty() ? 'and not I@salt:master' : ''
Ivan Berezovskiy2d6b3d52018-12-24 18:31:39 +0400733 // Pull images first if any
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400734 def listCIMinions = salt.getMinions(master, "* ${dockerClientExclude} ${extra_tgt}")
Ivan Berezovskiy2d6b3d52018-12-24 18:31:39 +0400735 for (int i = 0; i < listCIMinions.size(); i++) {
736 if (!salt.getReturnValues(salt.getPillar(master, listCIMinions[i], 'docker:client:images')).isEmpty()) {
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400737 salt.enforceStateWithTest([saltId: master, target: listCIMinions[i], state: 'docker.client.images', retries: 2])
Ivan Berezovskiy2d6b3d52018-12-24 18:31:39 +0400738 }
739 }
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400740 salt.enforceStateWithTest([saltId: master, target: "I@docker:swarm:role:master and I@jenkins:client ${dockerClientExclude} ${extra_tgt}", state: 'docker.client', retries: 2])
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200741
742 // API timeout in minutes
743 def wait_timeout = 10
744
745 // Gerrit
746 def gerrit_master_url = salt.getPillar(master, gerrit_compound, '_param:gerrit_master_url')
747
748 if(!gerrit_master_url['return'].isEmpty()) {
749 gerrit_master_url = gerrit_master_url['return'][0].values()[0]
750 } else {
751 gerrit_master_url = ''
752 }
753
754 if (gerrit_master_url != '') {
755 common.infoMsg('Gerrit master url "' + gerrit_master_url + '" retrieved at _param:gerrit_master_url')
756 } else {
757
758 common.infoMsg('Gerrit master url could not be retrieved at _param:gerrit_master_url. Falling back to gerrit pillar')
759
760 def gerrit_host
761 def gerrit_http_port
762 def gerrit_http_scheme
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400763 def gerrit_http_prefix
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200764
765 def host_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:host')
766 gerrit_host = salt.getReturnValues(host_pillar)
767
768 def port_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:http_port')
769 gerrit_http_port = salt.getReturnValues(port_pillar)
770
771 def scheme_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:protocol')
772 gerrit_http_scheme = salt.getReturnValues(scheme_pillar)
773
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400774 def prefix_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:url_prefix')
775 gerrit_http_prefix = salt.getReturnValues(prefix_pillar)
776
777 gerrit_master_url = gerrit_http_scheme + '://' + gerrit_host + ':' + gerrit_http_port + gerrit_http_prefix
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200778
779 }
780
781 timeout(wait_timeout) {
782 common.infoMsg('Waiting for Gerrit to come up..')
Dzmitry Stremkouski5440ffa2018-05-31 12:00:02 +0200783 def check_gerrit_cmd = 'while true; do curl -sI -m 3 -o /dev/null -w' + " '" + '%{http_code}' + "' " + gerrit_master_url + '/ | grep 200 && break || sleep 1; done'
784 salt.cmdRun(master, gerrit_compound, 'timeout ' + (wait_timeout*60+3) + ' /bin/sh -c -- ' + '"' + check_gerrit_cmd + '"')
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200785 }
786
787 // Jenkins
Ivan Berezovskiyb672db72019-02-12 20:51:27 +0400788 def jenkins_master_host_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_host')
789 def jenkins_master_port_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_port')
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400790 def jenkins_master_url_prefix_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_url_prefix')
791
792 jenkins_master_url = "http://${salt.getReturnValues(jenkins_master_host_pillar)}:${salt.getReturnValues(jenkins_master_port_pillar)}${salt.getReturnValues(jenkins_master_url_prefix_pillar)}"
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200793
794 timeout(wait_timeout) {
795 common.infoMsg('Waiting for Jenkins to come up..')
Dzmitry Stremkouski5440ffa2018-05-31 12:00:02 +0200796 def check_jenkins_cmd = 'while true; do curl -sI -m 3 -o /dev/null -w' + " '" + '%{http_code}' + "' " + jenkins_master_url + '/whoAmI/ | grep 200 && break || sleep 1; done'
797 salt.cmdRun(master, jenkins_compound, 'timeout ' + (wait_timeout*60+3) + ' /bin/sh -c -- ' + '"' + check_jenkins_cmd + '"')
Dzmitry Stremkouskif8a873e2018-05-05 22:15:27 +0200798 }
Richard Felklc1aba9b2018-01-02 09:25:20 +0100799
Martin Polreich80fa3182018-10-22 15:16:53 +0200800 salt.enforceStateWithTest([saltId: master, target: "I@openldap:client ${extra_tgt}", state: 'openldap', retries: 2])
Richard Felklb6fdec92017-09-26 10:52:36 +0200801
Martin Polreich80fa3182018-10-22 15:16:53 +0200802 salt.enforceStateWithTest([saltId: master, target: "I@python:environment ${extra_tgt}", state: 'python'])
Richard Felklc1aba9b2018-01-02 09:25:20 +0100803
Richard Felkl6e208fc2017-12-08 10:18:44 +0100804 withEnv(['ASK_ON_ERROR=false']){
805 retry(2){
806 try{
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400807 salt.enforceStateWithTest([saltId: master, target: "I@gerrit:client ${extra_tgt}", state: 'gerrit'])
Richard Felkl6e208fc2017-12-08 10:18:44 +0100808 }catch(e){
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300809 salt.fullRefresh(master, "I@gerrit:client ${extra_tgt}")
Richard Felkl6e208fc2017-12-08 10:18:44 +0100810 throw e //rethrow for retry handler
811 }
812 }
813 retry(2){
814 try{
Stanislav Riazanovcfc4ac52019-03-04 17:44:06 +0400815 salt.enforceStateWithTest([saltId: master, target: "I@jenkins:client ${extra_tgt}", state: 'jenkins'])
Richard Felkl6e208fc2017-12-08 10:18:44 +0100816 }catch(e){
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300817 salt.fullRefresh(master, "I@jenkins:client ${extra_tgt}")
Richard Felkl6e208fc2017-12-08 10:18:44 +0100818 throw e //rethrow for retry handler
819 }
820 }
821 }
Jiri Broulikb9a0c592017-09-05 16:41:48 +0200822}
823
824
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300825def installStacklight(master, extra_tgt = '') {
Martin Polreich8663a342017-06-30 17:33:08 +0200826 def common = new com.mirantis.mk.Common()
827 def salt = new com.mirantis.mk.Salt()
Martin Polreich331f2b62019-02-08 10:16:52 +0100828 def step_retries_wait = 20
829 def step_retries = 15
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400830 def first_target
Martin Polreich8663a342017-06-30 17:33:08 +0200831
Simon Pasquieref4122c2017-08-25 15:52:07 +0200832 // Install core services for K8S environments:
azvyagintsev833b4772018-11-21 22:30:59 +0200833 // HAProxy, Nginx and glusterFS clients.
834 // glusterFS clients must be first one, since nginx should store certs on it.
Simon Pasquieref4122c2017-08-25 15:52:07 +0200835 // In case of OpenStack, those are already installed
Martin Polreich792f2662017-07-11 13:36:32 +0200836 if (common.checkContains('STACK_INSTALL', 'k8s')) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200837 salt.enforceStateWithTest([saltId: master, target: "I@glusterfs:client ${extra_tgt}", state: 'glusterfs.client', retries: 2])
Martin Polreich331f2b62019-02-08 10:16:52 +0100838 salt.enforceState([saltId: master, target: "I@nginx:server ${extra_tgt}", state: 'salt.minion.cert', retries: 3])
azvyagintsev833b4772018-11-21 22:30:59 +0200839
Martin Polreich80fa3182018-10-22 15:16:53 +0200840 salt.enforceState([saltId: master, target: "I@haproxy:proxy ${extra_tgt}", state: 'haproxy'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300841 salt.runSaltProcessStep(master, "I@haproxy:proxy ${extra_tgt}", 'service.status', ['haproxy'])
Simon Pasquieref4122c2017-08-25 15:52:07 +0200842
Martin Polreich80fa3182018-10-22 15:16:53 +0200843 salt.enforceStateWithTest([saltId: master, target: "I@nginx:server ${extra_tgt}", state: 'nginx'])
Martin Polreich792f2662017-07-11 13:36:32 +0200844 }
Jiri Broulike57a0242017-10-06 13:38:47 +0200845
Dmitry Kalashnik69ecb782018-05-21 17:36:50 +0400846 // Install MongoDB for Alerta
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300847 if (salt.testTarget(master, "I@mongodb:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200848 salt.enforceState([saltId: master, target: "I@mongodb:server ${extra_tgt}", state: 'mongodb.server'])
Ildar Svetlov829f94a2018-06-05 17:57:08 +0400849
850 // Initialize mongodb replica set
Martin Polreich331f2b62019-02-08 10:16:52 +0100851 salt.enforceState([saltId: master, target: "I@mongodb:server ${extra_tgt}", state: 'mongodb.cluster', retries: 5, retries_wait: 20])
Dmitry Kalashnik69ecb782018-05-21 17:36:50 +0400852 }
853
Martin Polreich8663a342017-06-30 17:33:08 +0200854 //Install Telegraf
Martin Polreich80fa3182018-10-22 15:16:53 +0200855 salt.enforceState([saltId: master, target: "( I@telegraf:agent or I@telegraf:remote_agent ) ${extra_tgt}", state: 'telegraf'])
Martin Polreich8663a342017-06-30 17:33:08 +0200856
Dmitry Kalashnikebc1e1e2017-07-13 14:20:08 +0400857 // Install Prometheus exporters
Martin Polreich80fa3182018-10-22 15:16:53 +0200858 salt.enforceStateWithTest([saltId: master, target: "I@prometheus:exporters ${extra_tgt}", state: 'prometheus'])
Dmitry Kalashnikebc1e1e2017-07-13 14:20:08 +0400859
Martin Polreich8663a342017-06-30 17:33:08 +0200860 //Install Elasticsearch and Kibana
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400861 if (salt.testTarget(master, "I@elasticsearch:server:enabled:true ${extra_tgt}")) {
862 first_target = salt.getFirstMinion(master, "I@elasticsearch:server:enabled:true ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200863 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'elasticsearch.server'])
Ramon Meleroae6e16a2018-08-07 13:42:16 -0500864 }
Martin Polreich80fa3182018-10-22 15:16:53 +0200865 salt.enforceStateWithTest([saltId: master, target: "I@elasticsearch:server:enabled:true ${extra_tgt}", state: 'elasticsearch.server'])
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400866 if (salt.testTarget(master, "I@kibana:server:enabled:true ${extra_tgt}")) {
867 first_target = salt.getFirstMinion(master, "I@kibana:server:enabled:true ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200868 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'kibana.server'])
Ramon Meleroae6e16a2018-08-07 13:42:16 -0500869 }
Martin Polreich80fa3182018-10-22 15:16:53 +0200870 salt.enforceStateWithTest([saltId: master, target: "I@kibana:server:enabled:true ${extra_tgt}", state: 'kibana.server'])
Martin Polreich675dc942018-08-15 09:37:32 +0200871
Mateusz Matuszkowiak018a3902018-06-06 11:54:15 +0200872 // Check ES health cluster status
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300873 def pillar = salt.getPillar(master, "I@elasticsearch:client ${extra_tgt}", 'elasticsearch:client:server:host')
Mateusz Matuszkowiak018a3902018-06-06 11:54:15 +0200874 def elasticsearch_vip
875 if(!pillar['return'].isEmpty()) {
876 elasticsearch_vip = pillar['return'][0].values()[0]
877 } else {
878 common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
879 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300880 pillar = salt.getPillar(master, "I@elasticsearch:client ${extra_tgt}", 'elasticsearch:client:server:port')
Mateusz Matuszkowiak018a3902018-06-06 11:54:15 +0200881 def elasticsearch_port
882 if(!pillar['return'].isEmpty()) {
883 elasticsearch_port = pillar['return'][0].values()[0]
884 } else {
885 common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
886 }
Martin Polreich331f2b62019-02-08 10:16:52 +0100887 common.retry(step_retries,step_retries_wait) {
Mateusz Matuszkowiak018a3902018-06-06 11:54:15 +0200888 common.infoMsg('Waiting for Elasticsearch to become green..')
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300889 salt.cmdRun(master, "I@elasticsearch:client ${extra_tgt}", "curl -sf ${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
Mateusz Matuszkowiak018a3902018-06-06 11:54:15 +0200890 }
891
Martin Polreich331f2b62019-02-08 10:16:52 +0100892 salt.enforceState([saltId: master, target: "I@elasticsearch:client ${extra_tgt}", state: 'elasticsearch.client', retries: step_retries, retries_wait: step_retries_wait])
Mateusz Matuszkowiak018a3902018-06-06 11:54:15 +0200893
Martin Polreich331f2b62019-02-08 10:16:52 +0100894 salt.enforceState([saltId: master, target: "I@kibana:client ${extra_tgt}", state: 'kibana.client', retries: step_retries, retries_wait: step_retries_wait])
Dmitry Kalashnik53feda72018-03-16 16:06:10 +0400895
896 //Install InfluxDB
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300897 if (salt.testTarget(master, "I@influxdb:server ${extra_tgt}")) {
Dmitry Ukov8d50e182018-08-27 17:25:34 +0400898 first_target = salt.getFirstMinion(master, "I@influxdb:server ${extra_tgt}")
Martin Polreich80fa3182018-10-22 15:16:53 +0200899 salt.enforceState([saltId: master, target: "${first_target} ${extra_tgt}", state: 'influxdb'])
900 salt.enforceState([saltId: master, target: "I@influxdb:server ${extra_tgt}", state: 'influxdb'])
Dmitry Kalashnik53feda72018-03-16 16:06:10 +0400901 }
902
Ildar Svetlov794f89e2018-02-14 16:28:53 +0400903 // Install service for the log collection
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300904 if (salt.testTarget(master, "I@fluentd:agent ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200905 salt.enforceState([saltId: master, target: "I@fluentd:agent ${extra_tgt}", state: 'fluentd'])
Ildar Svetlov794f89e2018-02-14 16:28:53 +0400906 } else {
Martin Polreich80fa3182018-10-22 15:16:53 +0200907 salt.enforceState([saltId: master, target: "I@heka:log_collector ${extra_tgt}", state: 'heka.log_collector'])
Ildar Svetlov794f89e2018-02-14 16:28:53 +0400908 }
Simon Pasquier6301ddd2017-09-05 11:56:58 +0200909
Ondrej Smola5908ae62018-01-08 13:42:08 +0100910 // Install heka ceilometer collector
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300911 if (salt.testTarget(master, "I@heka:ceilometer_collector:enabled ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200912 salt.enforceState([saltId: master, target: "I@heka:ceilometer_collector:enabled ${extra_tgt}", state: 'heka.ceilometer_collector'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300913 salt.runSaltProcessStep(master, "I@heka:ceilometer_collector:enabled ${extra_tgt}", 'service.restart', ['ceilometer_collector'], null, true)
Ondrej Smola5908ae62018-01-08 13:42:08 +0100914 }
915
Martin Polreich470925a2017-07-06 17:12:59 +0200916 // Install galera
Martin Polreich792f2662017-07-11 13:36:32 +0200917 if (common.checkContains('STACK_INSTALL', 'k8s')) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200918 salt.enforceState([saltId: master, target: "I@galera:master ${extra_tgt}", state: 'galera', retries: 2])
919 salt.enforceState([saltId: master, target: "I@galera:slave ${extra_tgt}", state: 'galera', retries: 2])
Martin Polreich406f6462017-07-11 11:33:42 +0000920
Martin Polreich792f2662017-07-11 13:36:32 +0200921 // Check galera status
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300922 salt.runSaltProcessStep(master, "I@galera:master ${extra_tgt}", 'mysql.status')
923 salt.runSaltProcessStep(master, "I@galera:slave ${extra_tgt}", 'mysql.status')
Martin Polreich792f2662017-07-11 13:36:32 +0200924 }
Martin Polreich470925a2017-07-06 17:12:59 +0200925
Martin Polreich8663a342017-06-30 17:33:08 +0200926 //Collect Grains
Martin Polreich80fa3182018-10-22 15:16:53 +0200927 salt.enforceState([saltId: master, target: "I@salt:minion ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300928 salt.runSaltProcessStep(master, "I@salt:minion ${extra_tgt}", 'saltutil.refresh_modules')
929 salt.runSaltProcessStep(master, "I@salt:minion ${extra_tgt}", 'mine.update')
Martin Polreich8663a342017-06-30 17:33:08 +0200930 sleep(5)
931
Ildar Svetlov794f89e2018-02-14 16:28:53 +0400932 // Configure Prometheus in Docker Swarm
Martin Polreich80fa3182018-10-22 15:16:53 +0200933 salt.enforceState([saltId: master, target: "I@docker:swarm and I@prometheus:server ${extra_tgt}", state: 'prometheus'])
Ildar Svetlov794f89e2018-02-14 16:28:53 +0400934
Ildar Svetlov0eec97d2018-02-15 10:06:40 +0400935 //Configure Remote Collector in Docker Swarm for Openstack deployments
Ildar Svetlov794f89e2018-02-14 16:28:53 +0400936 if (!common.checkContains('STACK_INSTALL', 'k8s')) {
Martin Polreich80fa3182018-10-22 15:16:53 +0200937 salt.enforceState([saltId: master, target: "I@docker:swarm and I@prometheus:server ${extra_tgt}", state: 'heka.remote_collector', failOnError: false])
Martin Polreich8017f452017-08-21 16:01:42 +0200938 }
Martin Polreich8663a342017-06-30 17:33:08 +0200939
Ildar Svetlov225fb6c2018-06-06 14:59:31 +0400940 // Launch containers
Ivan Berezovskiye59448c2018-12-25 18:47:29 +0400941 // Pull images first if any
942 def listMinions = salt.getMinions(master, "I@docker:swarm and I@prometheus:server ${extra_tgt}")
943 for (int i = 0; i < listMinions.size(); i++) {
944 if (!salt.getReturnValues(salt.getPillar(master, listMinions[i], 'docker:client:images')).isEmpty()) {
945 salt.enforceState([saltId: master, target: listMinions[i], state: 'docker.client.images', retries: 2])
946 }
947 }
Martin Polreich80fa3182018-10-22 15:16:53 +0200948 salt.enforceState([saltId: master, target: "I@docker:swarm:role:master and I@prometheus:server ${extra_tgt}", state: 'docker.client'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300949 salt.runSaltProcessStep(master, "I@docker:swarm and I@prometheus:server ${extra_tgt}", 'dockerng.ps')
Ildar Svetlov225fb6c2018-06-06 14:59:31 +0400950
951 //Install Prometheus LTS
Martin Polreich80fa3182018-10-22 15:16:53 +0200952 salt.enforceStateWithTest([saltId: master, target: "I@prometheus:relay ${extra_tgt}", state: 'prometheus'])
Ildar Svetlov225fb6c2018-06-06 14:59:31 +0400953
Jiri Broulike3e7bc32018-02-26 10:50:07 +0100954 // Install sphinx server
Martin Polreich80fa3182018-10-22 15:16:53 +0200955 salt.enforceStateWithTest([saltId: master, target: "I@sphinx:server ${extra_tgt}", state: 'sphinx'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +0100956
Martin Polreich8663a342017-06-30 17:33:08 +0200957 //Configure Grafana
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300958 pillar = salt.getPillar(master, "ctl01* ${extra_tgt}", '_param:stacklight_monitor_address')
Martin Polreich8663a342017-06-30 17:33:08 +0200959 common.prettyPrint(pillar)
Martin Polreich595baef2017-07-04 16:11:04 +0200960
Martin Polreich8663a342017-06-30 17:33:08 +0200961 def stacklight_vip
962 if(!pillar['return'].isEmpty()) {
963 stacklight_vip = pillar['return'][0].values()[0]
964 } else {
965 common.errorMsg('[ERROR] Stacklight VIP address could not be retrieved')
966 }
vrovachev1c4770b2017-07-05 13:25:21 +0400967
Martin Polreich305e2222017-07-06 11:33:00 +0200968 common.infoMsg("Waiting for service on http://${stacklight_vip}:15013/ to start")
Martin Polreichbd7ad222017-07-06 12:53:51 +0200969 sleep(120)
Martin Polreich80fa3182018-10-22 15:16:53 +0200970 salt.enforceState([saltId: master, target: "I@grafana:client ${extra_tgt}", state: 'grafana.client'])
Martin Polreich8663a342017-06-30 17:33:08 +0200971}
972
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300973def installStacklightv1Control(master, extra_tgt = '') {
Tomáš Kukrála53b5352017-05-04 11:45:53 +0200974 def salt = new com.mirantis.mk.Salt()
Ales Komarekef2bab62017-04-26 11:48:28 +0200975
976 // infra install
977 // Install the StackLight backends
Martin Polreich80fa3182018-10-22 15:16:53 +0200978 salt.enforceState([saltId: master, target: "*01* and I@elasticsearch:server ${extra_tgt}", state: 'elasticsearch.server'])
979 salt.enforceState([saltId: master, target: "I@elasticsearch:server ${extra_tgt}", state: 'elasticsearch.server'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200980
Martin Polreich80fa3182018-10-22 15:16:53 +0200981 salt.enforceState([saltId: master, target: "*01* and I@influxdb:server ${extra_tgt}", state: 'influxdb'])
982 salt.enforceState([saltId: master, target: "I@influxdb:server ${extra_tgt}", state: 'influxdb'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200983
Martin Polreich80fa3182018-10-22 15:16:53 +0200984 salt.enforceState([saltId: master, target: "*01* and I@kibana:server ${extra_tgt}", state: 'kibana.server'])
985 salt.enforceState([saltId: master, target: "I@kibana:server ${extra_tgt}", state: 'kibana.server'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200986
Martin Polreich80fa3182018-10-22 15:16:53 +0200987 salt.enforceState([saltId: master, target: "*01* and I@grafana:server ${extra_tgt}",state: 'grafana.server'])
988 salt.enforceState([saltId: master, target: "I@grafana:server ${extra_tgt}", state: 'grafana.server'])
Ales Komarekef2bab62017-04-26 11:48:28 +0200989
Oleh Hryhorov5f96e092018-06-22 18:37:08 +0300990 def alarming_service_pillar = salt.getPillar(master, "mon*01* ${extra_tgt}", '_param:alarming_service')
Kalynovskyi3e370112017-06-26 12:48:23 +0300991 def alarming_service = alarming_service_pillar['return'][0].values()[0]
992
993 switch (alarming_service) {
994 case 'sensu':
995 // Update Sensu
Martin Polreich80fa3182018-10-22 15:16:53 +0200996 salt.enforceState([saltId: master, target: "I@sensu:server and I@rabbitmq:server ${extra_tgt}", state: 'rabbitmq'])
997 salt.enforceState([saltId: master, target: "I@redis:cluster:role:master ${extra_tgt}", state: 'redis'])
998 salt.enforceState([saltId: master, target: "I@redis:server ${extra_tgt}", state: 'redis'])
999 salt.enforceState([saltId: master, target: "I@sensu:server ${extra_tgt}", state: 'sensu'])
Kalynovskyi3e370112017-06-26 12:48:23 +03001000 default:
1001 // Update Nagios
Martin Polreich80fa3182018-10-22 15:16:53 +02001002 salt.enforceState([saltId: master, target: "I@nagios:server ${extra_tgt}", state: 'nagios.server'])
Kalynovskyi3e370112017-06-26 12:48:23 +03001003 // Stop the Nagios service because the package starts it by default and it will
1004 // started later only on the node holding the VIP address
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001005 salt.runSaltProcessStep(master, "I@nagios:server ${extra_tgt}", 'service.stop', ['nagios3'], null, true)
Kalynovskyi3e370112017-06-26 12:48:23 +03001006 }
1007
Martin Polreich80fa3182018-10-22 15:16:53 +02001008 salt.enforceState([saltId: master, target: "I@elasticsearch:client ${extra_tgt}", state: 'elasticsearch.client.service'])
1009 salt.enforceState([saltId: master, target: "I@kibana:client ${extra_tgt}", state: 'kibana.client'])
Ales Komarekef2bab62017-04-26 11:48:28 +02001010
1011 sleep(10)
1012}
1013
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001014def installStacklightv1Client(master, extra_tgt = '') {
Tomáš Kukrála53b5352017-05-04 11:45:53 +02001015 def salt = new com.mirantis.mk.Salt()
1016 def common = new com.mirantis.mk.Common()
Ales Komarekef2bab62017-04-26 11:48:28 +02001017
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001018 salt.cmdRun(master, "I@elasticsearch:client ${extra_tgt}", 'salt-call state.sls elasticsearch.client')
Martin Polreich80fa3182018-10-22 15:16:53 +02001019 // salt.enforceState([saltId: master, target: "I@elasticsearch:client", state: 'elasticsearch.client", true])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001020 salt.cmdRun(master, "I@kibana:client ${extra_tgt}", 'salt-call state.sls kibana.client')
Martin Polreich80fa3182018-10-22 15:16:53 +02001021 // salt.enforceState([saltId: master, target: "I@kibana:client", state: 'kibana.client", true])
Ales Komarekef2bab62017-04-26 11:48:28 +02001022
1023 // Install collectd, heka and sensu services on the nodes, this will also
1024 // generate the metadata that goes into the grains and eventually into Salt Mine
Martin Polreich80fa3182018-10-22 15:16:53 +02001025 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'collectd'])
1026 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'salt.minion'])
1027 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'heka'])
Ales Komarekef2bab62017-04-26 11:48:28 +02001028
1029 // Gather the Grafana metadata as grains
Martin Polreich80fa3182018-10-22 15:16:53 +02001030 salt.enforceState([saltId: master, target: "I@grafana:collector ${extra_tgt}", state: 'grafana.collector'])
Ales Komarekef2bab62017-04-26 11:48:28 +02001031
1032 // Update Salt Mine
Martin Polreich80fa3182018-10-22 15:16:53 +02001033 salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001034 salt.runSaltProcessStep(master, "* ${extra_tgt}", 'saltutil.refresh_modules')
1035 salt.runSaltProcessStep(master, "* ${extra_tgt}", 'mine.update')
Ales Komarekef2bab62017-04-26 11:48:28 +02001036
1037 sleep(5)
1038
1039 // Update Heka
Martin Polreich80fa3182018-10-22 15:16:53 +02001040 salt.enforceState([saltId: master, target: "( I@heka:aggregator:enabled:True or I@heka:remote_collector:enabled:True ) ${extra_tgt}", state: 'heka'])
Ales Komarekef2bab62017-04-26 11:48:28 +02001041
1042 // Update collectd
Martin Polreich80fa3182018-10-22 15:16:53 +02001043 salt.enforceState([saltId: master, target: "I@collectd:remote_client:enabled:True ${extra_tgt}", state: 'collectd'])
Ales Komarekef2bab62017-04-26 11:48:28 +02001044
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001045 def alarming_service_pillar = salt.getPillar(master, "mon*01* ${extra_tgt}", '_param:alarming_service')
Kalynovskyi3e370112017-06-26 12:48:23 +03001046 def alarming_service = alarming_service_pillar['return'][0].values()[0]
Ales Komarekef2bab62017-04-26 11:48:28 +02001047
Kalynovskyi3e370112017-06-26 12:48:23 +03001048 switch (alarming_service) {
1049 case 'sensu':
1050 // Update Sensu
1051 // TODO for stacklight team, should be fixed in model
Martin Polreich80fa3182018-10-22 15:16:53 +02001052 salt.enforceState([saltId: master, target: "I@sensu:client ${extra_tgt}", state: 'sensu'])
Kalynovskyi3e370112017-06-26 12:48:23 +03001053 default:
1054 break
1055 // Default is nagios, and was enforced in installStacklightControl()
1056 }
Ales Komarekef2bab62017-04-26 11:48:28 +02001057
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001058 salt.cmdRun(master, "I@grafana:client and *01* ${extra_tgt}", 'salt-call state.sls grafana.client')
Martin Polreich80fa3182018-10-22 15:16:53 +02001059 // salt.enforceState([saltId: master, target: "I@grafana:client and *01*", state: 'grafana.client"])
Ales Komarekef2bab62017-04-26 11:48:28 +02001060
Marek Celoudd200cad2017-06-07 10:34:46 +02001061 // Finalize the configuration of Grafana (add the dashboards...)
Martin Polreich80fa3182018-10-22 15:16:53 +02001062 salt.enforceState([saltId: master, target: "I@grafana:client and *01* ${extra_tgt}", state: 'grafana.client'])
1063 salt.enforceState([saltId: master, target: "I@grafana:client and *02* ${extra_tgt}", state: 'grafana.client'])
1064 salt.enforceState([saltId: master, target: "I@grafana:client and *03* ${extra_tgt}", state: 'grafana.client'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001065 // nw salt -C "I@grafana:client' --async service.restart salt-minion; sleep 10
Marek Celoudd200cad2017-06-07 10:34:46 +02001066
Ales Komarekef2bab62017-04-26 11:48:28 +02001067 // Get the StackLight monitoring VIP addres
1068 //vip=$(salt-call pillar.data _param:stacklight_monitor_address --out key|grep _param: |awk '{print $2}')
1069 //vip=${vip:=172.16.10.253}
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001070 def pillar = salt.getPillar(master, "ctl01* ${extra_tgt}", '_param:stacklight_monitor_address')
Jakub Josefbceaa322017-06-13 18:28:27 +02001071 common.prettyPrint(pillar)
Ales Komarekef2bab62017-04-26 11:48:28 +02001072 def stacklight_vip = pillar['return'][0].values()[0]
1073
1074 if (stacklight_vip) {
1075 // (re)Start manually the services that are bound to the monitoring VIP
1076 common.infoMsg("restart services on node with IP: ${stacklight_vip}")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001077 salt.runSaltProcessStep(master, "G@ipv4:${stacklight_vip} ${extra_tgt}", 'service.restart', ['remote_collectd'])
1078 salt.runSaltProcessStep(master, "G@ipv4:${stacklight_vip} ${extra_tgt}", 'service.restart', ['remote_collector'])
1079 salt.runSaltProcessStep(master, "G@ipv4:${stacklight_vip} ${extra_tgt}", 'service.restart', ['aggregator'])
1080 salt.runSaltProcessStep(master, "G@ipv4:${stacklight_vip} ${extra_tgt}", 'service.restart', ['nagios3'])
Ales Komarekef2bab62017-04-26 11:48:28 +02001081 } else {
1082 throw new Exception("Missing stacklight_vip")
1083 }
Tomáš Kukrálf06ca0f2017-02-27 15:18:50 +01001084}
Tomáš Kukrál64f8e1c2017-08-11 10:32:20 +02001085
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001086//
1087// backups
1088//
1089
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001090def installBackup(master, component='common', extra_tgt = '') {
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001091 def salt = new com.mirantis.mk.Salt()
1092 if (component == 'common') {
1093 // Install Backupninja
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001094 if (salt.testTarget(master, "I@backupninja:client ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001095 salt.enforceState([saltId: master, target: "I@backupninja:client ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001096 salt.runSaltProcessStep(master, "I@backupninja:client ${extra_tgt}", 'saltutil.sync_grains')
1097 salt.runSaltProcessStep(master, "I@backupninja:client ${extra_tgt}", 'mine.flush')
1098 salt.runSaltProcessStep(master, "I@backupninja:client ${extra_tgt}", 'mine.update')
Martin Polreich80fa3182018-10-22 15:16:53 +02001099 salt.enforceState([saltId: master, target: "I@backupninja:client ${extra_tgt}", state: 'backupninja'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001100 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001101 salt.enforceStateWithTest([saltId: master, target: "I@backupninja:server ${extra_tgt}", state: 'salt.minion.grains'])
1102 salt.enforceStateWithTest([saltId: master, target: "I@backupninja:server ${extra_tgt}", state: 'backupninja'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001103 } else if (component == 'mysql') {
1104 // Install Xtrabackup
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001105 if (salt.testTarget(master, "I@xtrabackup:client ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001106 salt.enforceState([saltId: master, target: "I@xtrabackup:client ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001107 salt.runSaltProcessStep(master, "I@xtrabackup:client ${extra_tgt}", 'saltutil.sync_grains')
1108 salt.runSaltProcessStep(master, "I@xtrabackup:client ${extra_tgt}", 'mine.flush')
1109 salt.runSaltProcessStep(master, "I@xtrabackup:client ${extra_tgt}", 'mine.update')
Martin Polreich80fa3182018-10-22 15:16:53 +02001110 salt.enforceState([saltId: master, target: "I@xtrabackup:client ${extra_tgt}", state: 'xtrabackup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001111 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001112 salt.enforceStateWithTest([saltId: master, target: "I@xtrabackup:server ${extra_tgt}", state: 'xtrabackup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001113 } else if (component == 'contrail') {
1114
1115 // Install Cassandra backup
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001116 if (salt.testTarget(master, "I@cassandra:backup:client ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001117 salt.enforceState([saltId: master, target: "I@cassandra:backup:client ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001118 salt.runSaltProcessStep(master, "I@cassandra:backup:client ${extra_tgt}", 'saltutil.sync_grains')
1119 salt.runSaltProcessStep(master, "I@cassandra:backup:client ${extra_tgt}", 'mine.flush')
1120 salt.runSaltProcessStep(master, "I@cassandra:backup:client ${extra_tgt}", 'mine.update')
Martin Polreich80fa3182018-10-22 15:16:53 +02001121 salt.enforceState([saltId: master, target: "I@cassandra:backup:client ${extra_tgt}", state: 'cassandra.backup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001122 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001123 salt.enforceStateWithTest([saltId: master, target: "I@cassandra:backup:server ${extra_tgt}", state: 'cassandra.backup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001124 // Install Zookeeper backup
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001125 if (salt.testTarget(master, "I@zookeeper:backup:client ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001126 salt.enforceState([saltId: master, target: "I@zookeeper:backup:client ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001127 salt.runSaltProcessStep(master, "I@zookeeper:backup:client ${extra_tgt}", 'saltutil.sync_grains')
1128 salt.runSaltProcessStep(master, "I@zookeeper:backup:client ${extra_tgt}", 'mine.flush')
1129 salt.runSaltProcessStep(master, "I@zookeeper:backup:client ${extra_tgt}", 'mine.update')
Martin Polreich80fa3182018-10-22 15:16:53 +02001130 salt.enforceState([saltId: master, target: "I@zookeeper:backup:client ${extra_tgt}", state: 'zookeeper.backup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001131 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001132 salt.enforceStateWithTest([saltId: master, target: "I@zookeeper:backup:server ${extra_tgt}", state: 'zookeeper.backup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001133 } else if (component == 'ceph') {
1134 // Install Ceph backup
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001135 if (salt.testTarget(master, "I@ceph:backup:client ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001136 salt.enforceState([saltId: master, target: "I@ceph:backup:client ${extra_tgt}", state: 'salt.minion.grains'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001137 salt.runSaltProcessStep(master, "I@ceph:backup:client ${extra_tgt}", 'saltutil.sync_grains')
1138 salt.runSaltProcessStep(master, "I@ceph:backup:client ${extra_tgt}", 'mine.flush')
1139 salt.runSaltProcessStep(master, "I@ceph:backup:client ${extra_tgt}", 'mine.update')
Martin Polreich80fa3182018-10-22 15:16:53 +02001140 salt.enforceState([saltId: master, target: "I@ceph:backup:client ${extra_tgt}", state: 'ceph.backup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001141 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001142 salt.enforceStateWithTest([saltId: master, target: "I@ceph:backup:server ${extra_tgt}", state: 'ceph.backup'])
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001143 }
1144
1145}
Tomáš Kukrál64f8e1c2017-08-11 10:32:20 +02001146
1147//
1148// Ceph
1149//
1150
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001151def installCephMon(master, target="I@ceph:mon", extra_tgt = '') {
Tomáš Kukrál64f8e1c2017-08-11 10:32:20 +02001152 def salt = new com.mirantis.mk.Salt()
1153
Martin Polreich80fa3182018-10-22 15:16:53 +02001154 salt.enforceState([saltId: master, target: "I@ceph:common ${extra_tgt}", state: 'salt.minion.grains'])
Jiri Broulikbfc234f2017-10-17 15:45:38 +02001155
Jiri Broulike57a0242017-10-06 13:38:47 +02001156 // generate keyrings
Martin Polreich44331002018-08-13 11:27:06 +02001157 if (salt.testTarget(master, "( I@ceph:mon:keyring:mon or I@ceph:common:keyring:admin ) ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001158 salt.enforceState([saltId: master, target: "( I@ceph:mon:keyring:mon or I@ceph:common:keyring:admin ) ${extra_tgt}", state: 'ceph.mon'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001159 salt.runSaltProcessStep(master, "I@ceph:mon ${extra_tgt}", 'saltutil.sync_grains')
Martin Polreich44331002018-08-13 11:27:06 +02001160 salt.runSaltProcessStep(master, "( I@ceph:mon:keyring:mon or I@ceph:common:keyring:admin ) ${extra_tgt}", 'mine.update')
Vasyl Saienko03ed4e72018-11-14 11:35:29 +02001161
1162 // on target nodes mine is used to get pillar from 'ceph:common:keyring:admin' via grain.items
1163 // we need to refresh all pillar/grains to make data sharing work correctly
1164 salt.fullRefresh(master, "( I@ceph:mon:keyring:mon or I@ceph:common:keyring:admin ) ${extra_tgt}")
1165
Jiri Broulik3c75a6d2017-10-06 18:44:37 +02001166 sleep(5)
Jiri Broulike57a0242017-10-06 13:38:47 +02001167 }
Ondrej Smolafd5c2352017-08-11 14:25:39 +02001168 // install Ceph Mons
Martin Polreich80fa3182018-10-22 15:16:53 +02001169 salt.enforceState([saltId: master, target: target, state: 'ceph.mon'])
1170 salt.enforceStateWithTest([saltId: master, target: "I@ceph:mgr ${extra_tgt}", state: 'ceph.mgr'])
Tomáš Kukrál64f8e1c2017-08-11 10:32:20 +02001171}
1172
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001173def installCephOsd(master, target="I@ceph:osd", setup=true, extra_tgt = '') {
Tomáš Kukrál64f8e1c2017-08-11 10:32:20 +02001174 def salt = new com.mirantis.mk.Salt()
1175
1176 // install Ceph OSDs
Martin Polreich80fa3182018-10-22 15:16:53 +02001177 salt.enforceState([saltId: master, target: target, state: 'ceph.osd'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001178 salt.runSaltProcessStep(master, "I@ceph:osd ${extra_tgt}", 'saltutil.sync_grains')
Martin Polreich80fa3182018-10-22 15:16:53 +02001179 salt.enforceState([saltId: master, target: target, state: 'ceph.osd.custom'])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001180 salt.runSaltProcessStep(master, "I@ceph:osd ${extra_tgt}", 'saltutil.sync_grains')
1181 salt.runSaltProcessStep(master, "I@ceph:osd ${extra_tgt}", 'mine.update')
Jiri Broulike3e7bc32018-02-26 10:50:07 +01001182 installBackup(master, 'ceph')
Tomáš Kukráled27a472017-08-29 10:26:25 +02001183
Jiri Broulike57a0242017-10-06 13:38:47 +02001184 // setup pools, keyrings and maybe crush
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001185 if (salt.testTarget(master, "I@ceph:setup ${extra_tgt}") && setup) {
Jiri Broulik07f8de72017-10-09 14:12:17 +02001186 sleep(5)
Martin Polreich80fa3182018-10-22 15:16:53 +02001187 salt.enforceState([saltId: master, target: "I@ceph:setup ${extra_tgt}", state: 'ceph.setup'])
Tomáš Kukráled27a472017-08-29 10:26:25 +02001188 }
Tomáš Kukrál64f8e1c2017-08-11 10:32:20 +02001189}
Tomáš Kukrál1e5b2192017-08-15 16:58:18 +02001190
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001191def installCephClient(master, extra_tgt = '') {
Tomáš Kukrál1e5b2192017-08-15 16:58:18 +02001192 def salt = new com.mirantis.mk.Salt()
1193
1194 // install Ceph Radosgw
Mykyta Karpinf7a714c2018-11-06 14:22:36 +02001195 if (salt.testTarget(master, "I@ceph:radosgw ${extra_tgt}")) {
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001196 salt.runSaltProcessStep(master, "I@ceph:radosgw ${extra_tgt}", 'saltutil.sync_grains')
Martin Polreich80fa3182018-10-22 15:16:53 +02001197 salt.enforceState([saltId: master, target: "I@ceph:radosgw ${extra_tgt}", state: 'ceph.radosgw'])
Alexander Noskov229035f2017-12-06 00:10:20 +04001198 }
Mykyta Karpin9b8beab2018-09-11 17:52:35 +03001199
1200 // setup keyring for Openstack services
Martin Polreich80fa3182018-10-22 15:16:53 +02001201 salt.enforceStateWithTest([saltId: master, target: "I@ceph:common and I@glance:server ${extra_tgt}", state: ['ceph.common', 'ceph.setup.keyring']])
Mykyta Karpin9b8beab2018-09-11 17:52:35 +03001202
Martin Polreich80fa3182018-10-22 15:16:53 +02001203 salt.enforceStateWithTest([saltId: master, target: "I@ceph:common and I@cinder:controller ${extra_tgt}", state: ['ceph.common', 'ceph.setup.keyring']])
Mykyta Karpin9b8beab2018-09-11 17:52:35 +03001204
1205 if (salt.testTarget(master, "I@ceph:common and I@nova:compute ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001206 salt.enforceState([saltId: master, target: "I@ceph:common and I@nova:compute ${extra_tgt}", state: ['ceph.common', 'ceph.setup.keyring']])
Mykyta Karpin9b8beab2018-09-11 17:52:35 +03001207 salt.runSaltProcessStep(master, "I@ceph:common and I@nova:compute ${extra_tgt}", 'saltutil.sync_grains')
1208 }
1209
Martin Polreich80fa3182018-10-22 15:16:53 +02001210 salt.enforceStateWithTest([saltId: master, target: "I@ceph:common and I@gnocchi:server ${extra_tgt}", state: ['ceph.common', 'ceph.setup.keyring']])
Tomáš Kukrál1e5b2192017-08-15 16:58:18 +02001211}
Jiri Broulike57a0242017-10-06 13:38:47 +02001212
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001213def connectCeph(master, extra_tgt = '') {
Jiri Broulike57a0242017-10-06 13:38:47 +02001214 def salt = new com.mirantis.mk.Salt()
1215
Mykyta Karpin9b8beab2018-09-11 17:52:35 +03001216 // setup Keystone service and endpoints for swift or / and S3
Martin Polreich80fa3182018-10-22 15:16:53 +02001217 salt.enforceStateWithTest([saltId: master, target: "I@keystone:client ${extra_tgt}", state: 'keystone.client'])
Mykyta Karpin9b8beab2018-09-11 17:52:35 +03001218
Jiri Broulike57a0242017-10-06 13:38:47 +02001219 // connect Ceph to the env
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001220 if (salt.testTarget(master, "I@ceph:common and I@glance:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001221 salt.enforceState([saltId: master, target: "I@ceph:common and I@glance:server ${extra_tgt}", state: ['glance']])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001222 salt.runSaltProcessStep(master, "I@ceph:common and I@glance:server ${extra_tgt}", 'service.restart', ['glance-api'])
Jiri Broulike57a0242017-10-06 13:38:47 +02001223 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001224 if (salt.testTarget(master, "I@ceph:common and I@cinder:controller ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001225 salt.enforceState([saltId: master, target: "I@ceph:common and I@cinder:controller ${extra_tgt}", state: ['cinder']])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001226 salt.runSaltProcessStep(master, "I@ceph:common and I@cinder:controller ${extra_tgt}", 'service.restart', ['cinder-volume'])
Jiri Broulike57a0242017-10-06 13:38:47 +02001227 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001228 if (salt.testTarget(master, "I@ceph:common and I@nova:compute ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001229 salt.enforceState([saltId: master, target: "I@ceph:common and I@nova:compute ${extra_tgt}", state: ['nova']])
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001230 salt.runSaltProcessStep(master, "I@ceph:common and I@nova:compute ${extra_tgt}", 'service.restart', ['nova-compute'])
Jiri Broulike57a0242017-10-06 13:38:47 +02001231 }
Mykyta Karpin0dec8b32018-09-19 09:37:45 +03001232 if (salt.testTarget(master, "I@ceph:common and I@gnocchi:server ${extra_tgt}")) {
Martin Polreich80fa3182018-10-22 15:16:53 +02001233 salt.enforceState([saltId: master, target: "I@ceph:common and I@gnocchi:server:role:primary ${extra_tgt}", state: 'gnocchi.server'])
1234 salt.enforceState([saltId: master, target: "I@ceph:common and I@gnocchi:server ${extra_tgt}", state: 'gnocchi.server'])
Mykyta Karpin0dec8b32018-09-19 09:37:45 +03001235 }
Jiri Broulike57a0242017-10-06 13:38:47 +02001236}
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001237
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001238def installOssInfra(master, extra_tgt = '') {
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001239 def common = new com.mirantis.mk.Common()
1240 def salt = new com.mirantis.mk.Salt()
1241
Martin Polreich80fa3182018-10-22 15:16:53 +02001242 salt.enforceStateWithTest([saltId: master, target: "I@devops_portal:config ${extra_tgt}", state: 'devops_portal.config'])
1243 salt.enforceStateWithTest([saltId: master, target: "I@rundeck:client ${extra_tgt}", state: ['linux.system.user', 'openssh'], testTargetMatcher: "I@devops_portal:config ${extra_tgt}"])
1244 salt.enforceStateWithTest([saltId: master, target: "I@rundeck:server ${extra_tgt}", state: 'rundeck.server', testTargetMatcher: "I@devops_portal:config ${extra_tgt}"])
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001245}
1246
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001247def installOss(master, extra_tgt = '') {
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001248 def common = new com.mirantis.mk.Common()
1249 def salt = new com.mirantis.mk.Salt()
1250
1251 //Get oss VIP address
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001252 def pillar = salt.getPillar(master, "cfg01* ${extra_tgt}", '_param:stacklight_monitor_address')
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001253 common.prettyPrint(pillar)
1254
1255 def oss_vip
1256 if(!pillar['return'].isEmpty()) {
1257 oss_vip = pillar['return'][0].values()[0]
1258 } else {
1259 common.errorMsg('[ERROR] Oss VIP address could not be retrieved')
1260 }
1261
1262 // Postgres client - initialize OSS services databases
1263 timeout(120){
1264 common.infoMsg("Waiting for postgresql database to come up..")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001265 salt.cmdRun(master, "I@postgresql:client ${extra_tgt}", 'while true; do if docker service logs postgresql_postgresql-db 2>&1 | grep "ready to accept"; then break; else sleep 5; fi; done')
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001266 }
1267 // XXX: first run usually fails on some inserts, but we need to create databases at first
Martin Polreich80fa3182018-10-22 15:16:53 +02001268 salt.enforceState([saltId: master, target: "I@postgresql:client ${extra_tgt}", state: 'postgresql.client', failOnError: false])
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001269
1270 // Setup postgres database with integration between
1271 // Pushkin notification service and Security Monkey security audit service
1272 timeout(10) {
1273 common.infoMsg("Waiting for Pushkin to come up..")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001274 salt.cmdRun(master, "I@postgresql:client ${extra_tgt}", "while true; do curl -sf ${oss_vip}:8887/apps >/dev/null && break; done")
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001275 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001276 salt.enforceState([saltId: master, target: "I@postgresql:client ${extra_tgt}", state: 'postgresql.client'])
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001277
1278 // Rundeck
1279 timeout(10) {
1280 common.infoMsg("Waiting for Rundeck to come up..")
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001281 salt.cmdRun(master, "I@rundeck:client ${extra_tgt}", "while true; do curl -sf ${oss_vip}:4440 >/dev/null && break; done")
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001282 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001283 salt.enforceState([saltId: master, target: "I@rundeck:client ${extra_tgt}", state: 'rundeck.client'])
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001284
1285 // Elasticsearch
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001286 pillar = salt.getPillar(master, "I@elasticsearch:client ${extra_tgt}", 'elasticsearch:client:server:host')
Mikhail Ivanovfcd55d62017-12-19 16:51:35 +04001287 def elasticsearch_vip
1288 if(!pillar['return'].isEmpty()) {
1289 elasticsearch_vip = pillar['return'][0].values()[0]
1290 } else {
1291 common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
1292 }
1293
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001294 timeout(10) {
1295 common.infoMsg('Waiting for Elasticsearch to come up..')
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001296 salt.cmdRun(master, "I@elasticsearch:client ${extra_tgt}", "while true; do curl -sf ${elasticsearch_vip}:9200 >/dev/null && break; done")
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001297 }
Martin Polreich80fa3182018-10-22 15:16:53 +02001298 salt.enforceState([saltId: master, target: "I@elasticsearch:client ${extra_tgt}", state: 'elasticsearch.client'])
Mikhail Ivanov5312d1f2017-12-06 17:36:36 +04001299}
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +03001300
1301/**
1302 * Function receives connection string, target and configuration yaml pattern
1303 * and retrieves config fom salt minion according to pattern. After that it
1304 * sorts applications according to priorities and runs orchestration states
1305 * @param master Salt Connection object or pepperEnv
1306 * @param tgt Target
1307 * @param conf Configuration pattern
1308 */
1309def OrchestrateApplications(master, tgt, conf) {
1310 def salt = new com.mirantis.mk.Salt()
1311 def common = new com.mirantis.mk.Common()
1312 def _orch = salt.getConfig(master, tgt, conf)
1313 if ( !_orch['return'][0].values()[0].isEmpty() ) {
1314 Map<String,Integer> _orch_app = [:]
1315 for (k in _orch['return'][0].values()[0].keySet()) {
1316 _orch_app[k] = _orch['return'][0].values()[0][k].values()[0].toInteger()
1317 }
1318 def _orch_app_sorted = common.SortMapByValueAsc(_orch_app)
1319 common.infoMsg("Applications will be deployed in following order:"+_orch_app_sorted.keySet())
1320 for (app in _orch_app_sorted.keySet()) {
1321 salt.orchestrateSystem(master, ['expression': tgt, 'type': 'compound'], "${app}.orchestrate.deploy")
1322 }
1323 }
1324 else {
1325 common.infoMsg("No applications found for orchestration")
1326 }
Inessa Vasilevskaya85ad1a92018-10-08 12:19:47 +02001327}