blob: 2c58e9a5d4bcd550c5932772d72c077b5c393510 [file] [log] [blame]
Jakub Josef79ecec32017-02-17 14:36:28 +01001package com.mirantis.mk
2
Jakub Josefb77c0812017-03-27 14:11:01 +02003import java.util.stream.Collectors
Jakub Josef79ecec32017-02-17 14:36:28 +01004/**
5 * Salt functions
6 *
7*/
8
9/**
10 * Salt connection and context parameters
11 *
12 * @param url Salt API server URL
13 * @param credentialsID ID of credentials store entry
14 */
15def connection(url, credentialsId = "salt") {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +010016 def common = new com.mirantis.mk.Common()
Jakub Josef79ecec32017-02-17 14:36:28 +010017 params = [
18 "url": url,
19 "credentialsId": credentialsId,
20 "authToken": null,
21 "creds": common.getCredentials(credentialsId)
22 ]
23 params["authToken"] = saltLogin(params)
Jakub Josef79ecec32017-02-17 14:36:28 +010024 return params
25}
26
27/**
28 * Login to Salt API, return auth token
29 *
30 * @param master Salt connection object
31 */
32def saltLogin(master) {
Tomáš Kukrál7bec0532017-02-20 15:39:31 +010033 def http = new com.mirantis.mk.Http()
Jakub Josef79ecec32017-02-17 14:36:28 +010034 data = [
35 'username': master.creds.username,
36 'password': master.creds.password.toString(),
37 'eauth': 'pam'
38 ]
Tomáš Kukrál7bec0532017-02-20 15:39:31 +010039 authToken = http.restGet(master, '/login', data)['return'][0]['token']
Jakub Josef79ecec32017-02-17 14:36:28 +010040 return authToken
41}
42
43/**
chnydaa0dbb252017-10-05 10:46:09 +020044 * Run action using Salt API (using plain HTTP request from Jenkins master) or Pepper (from slave shell)
Jakub Josef79ecec32017-02-17 14:36:28 +010045 *
chnydaa0dbb252017-10-05 10:46:09 +020046 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method) (determines if command will be sent with Pepper of Salt API )
Jakub Josef79ecec32017-02-17 14:36:28 +010047 * @param client Client type
48 * @param target Target specification, eg. for compound matches by Pillar
49 * data: ['expression': 'I@openssh:server', 'type': 'compound'])
50 * @param function Function to execute (eg. "state.sls")
Jakub Josef2f25cf22017-03-28 13:34:57 +020051 * @param batch Batch param to salt (integer or string with percents)
Martin Polreichf7976952019-03-04 10:06:11 +010052 * - null - automatic decision (based on number of worker threads env var or not use batch at all)
53 * - int - fixed size of batch
54 * - 'str%' - percantage of the requests in one batch
Jakub Josef79ecec32017-02-17 14:36:28 +010055 * @param args Additional arguments to function
56 * @param kwargs Additional key-value arguments to function
Jiri Broulik48544be2017-06-14 18:33:54 +020057 * @param timeout Additional argument salt api timeout
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +030058 * @param read_timeout http session read timeout
Jakub Josef79ecec32017-02-17 14:36:28 +010059 */
Jakub Josef79ecec32017-02-17 14:36:28 +010060
Martin Polreichf7976952019-03-04 10:06:11 +010061def runSaltCommand(saltId, client, target, function, batch = null, args = null, kwargs = null, timeout = -1, read_timeout = -1) {
Jakub Josef79ecec32017-02-17 14:36:28 +010062 data = [
63 'tgt': target.expression,
64 'fun': function,
65 'client': client,
66 'expr_form': target.type,
67 ]
Richard Felkld9476ac2018-07-12 19:01:33 +020068
Martin Polreichf7976952019-03-04 10:06:11 +010069 if (batch) {
Richard Felkld9476ac2018-07-12 19:01:33 +020070 batch = batch.toString()
Martin Polreichf7976952019-03-04 10:06:11 +010071 } else if (env.getEnvironment().containsKey('SALT_MASTER_OPT_WORKER_THREADS')) {
72 batch = env['SALT_MASTER_OPT_WORKER_THREADS'].toString()
73 }
74
75 if (batch instanceof String) {
76 if ((batch.isInteger() && batch.toInteger() > 0) || (batch.matches(/(\d){1,2}%/))){
Richard Felkld9476ac2018-07-12 19:01:33 +020077 data['client']= "local_batch"
78 data['batch'] = batch
79 }
Jakub Josef79ecec32017-02-17 14:36:28 +010080 }
81
82 if (args) {
83 data['arg'] = args
84 }
85
86 if (kwargs) {
87 data['kwarg'] = kwargs
88 }
89
Jiri Broulik48544be2017-06-14 18:33:54 +020090 if (timeout != -1) {
91 data['timeout'] = timeout
92 }
93
Martin Polreichf7976952019-03-04 10:06:11 +010094 def result = [:]
chnydaa0dbb252017-10-05 10:46:09 +020095 // Command will be sent using HttpRequest
96 if (saltId instanceof HashMap && saltId.containsKey("authToken") ) {
Jakub Josef79ecec32017-02-17 14:36:28 +010097
chnydaa0dbb252017-10-05 10:46:09 +020098 def headers = [
99 'X-Auth-Token': "${saltId.authToken}"
100 ]
101
102 def http = new com.mirantis.mk.Http()
Martin Polreichf7976952019-03-04 10:06:11 +0100103 result = http.sendHttpPostRequest("${saltId.url}/", data, headers, read_timeout)
chnydaa0dbb252017-10-05 10:46:09 +0200104 } else if (saltId instanceof HashMap) {
105 throw new Exception("Invalid saltId")
Martin Polreichf7976952019-03-04 10:06:11 +0100106 } else {
107 // Command will be sent using Pepper
108 result = runPepperCommand(data, saltId)
chnydaa0dbb252017-10-05 10:46:09 +0200109 }
110
Martin Polreichf7976952019-03-04 10:06:11 +0100111 // Convert returned Object to the same structure as from 'local' client to keep compatibility
112 if (data['client'].equals('local_batch')) {
113 def resultMap = ['return': [[:]]]
114 result['return'].each { it -> resultMap['return'][0] = it + resultMap['return'][0] }
115 return resultMap
116 } else {
117 return result
118 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100119}
120
Jakub Josef5ade54c2017-03-10 16:14:01 +0100121/**
chnydaa0dbb252017-10-05 10:46:09 +0200122 * Return pillar for given saltId and target
123 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100124 * @param target Get pillar target
125 * @param pillar pillar name (optional)
Martin Polreichf7976952019-03-04 10:06:11 +0100126 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100127 * @return output of salt command
128 */
Martin Polreichf7976952019-03-04 10:06:11 +0100129def getPillar(saltId, target, pillar = null, batch = null) {
Tomáš Kukráld2589702017-03-10 16:30:46 +0100130 if (pillar != null) {
Martin Polreichf7976952019-03-04 10:06:11 +0100131 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'pillar.get', batch, [pillar.replace('.', ':')])
Tomáš Kukráld2589702017-03-10 16:30:46 +0100132 } else {
Martin Polreichf7976952019-03-04 10:06:11 +0100133 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'pillar.data', batch)
Ales Komareka3c7e502017-03-13 11:20:44 +0100134 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100135}
136
Jakub Josef5ade54c2017-03-10 16:14:01 +0100137/**
chnydaa0dbb252017-10-05 10:46:09 +0200138 * Return grain for given saltId and target
139 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100140 * @param target Get grain target
141 * @param grain grain name (optional)
Martin Polreichf7976952019-03-04 10:06:11 +0100142 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100143 * @return output of salt command
144 */
Martin Polreichf7976952019-03-04 10:06:11 +0100145def getGrain(saltId, target, grain = null, batch = null) {
Ales Komarekcec24d42017-03-08 10:25:45 +0100146 if(grain != null) {
Martin Polreichf7976952019-03-04 10:06:11 +0100147 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'grains.item', batch, [grain])
Jakub Josef5ade54c2017-03-10 16:14:01 +0100148 } else {
Martin Polreichf7976952019-03-04 10:06:11 +0100149 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'grains.items', batch)
Ales Komarekcec24d42017-03-08 10:25:45 +0100150 }
Ales Komarekcec24d42017-03-08 10:25:45 +0100151}
152
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +0300153/**
154 * Return config items for given saltId and target
155 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
156 * @param target Get grain target
157 * @param config grain name (optional)
Martin Polreichf7976952019-03-04 10:06:11 +0100158 * @param batch Batch param to salt (integer or string with percents)
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +0300159 * @return output of salt command
160 */
Martin Polreichf7976952019-03-04 10:06:11 +0100161def getConfig(saltId, target, config, batch = null) {
162 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'config.get', batch, [config.replace('.', ':')], '--out=json')
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +0300163}
Jakub Josef432e9d92018-02-06 18:28:37 +0100164
Jakub Josef5ade54c2017-03-10 16:14:01 +0100165/**
chnydaa0dbb252017-10-05 10:46:09 +0200166 * Enforces state on given saltId and target
167 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100168 * @param target State enforcing target
169 * @param state Salt state
Jakub Josef432e9d92018-02-06 18:28:37 +0100170 * @param excludedStates states which will be excluded from main state (default empty string)
171 * @param output print output (optional, default true)
172 * @param failOnError throw exception on salt state result:false (optional, default true)
173 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
174 * @param optional Optional flag (if true pipeline will continue even if no minions for target found)
175 * @param read_timeout http session read timeout (optional, default -1 - disabled)
176 * @param retries Retry count for salt state. (optional, default -1 - no retries)
177 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
178 * @param saltArgs additional salt args eq. ["runas=aptly"]
179 * @return output of salt command
180 */
Martin Polreichc19e66b2018-10-22 12:22:03 +0200181def enforceStateWithExclude(Map params) {
182 //Set defaults
183 defaults = ["excludedStates": "", "output": true, "failOnError": true, "batch": null, "optional": false,
Martin Polreichf67b39a2019-02-08 10:06:34 +0100184 "read_timeout": -1, "retries": -1, "retries_wait": 5, "queue": true, "saltArgs": []]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200185 params = defaults + params
186 params.saltArgs << "exclude=${params.excludedStates}"
187 params.remove('excludedStates')
188 return enforceState(params)
189}
190
191
Martin Polreichf67b39a2019-02-08 10:06:34 +0100192def enforceStateWithExclude(saltId, target, state, excludedStates = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[], retries_wait=5) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200193// Deprecated, convert state to use Map as input parameter
194 def common = new com.mirantis.mk.Common()
Martin Polreich49288692018-12-20 15:41:04 +0100195 common.infoMsg("This method will be deprecated. Convert you method call to use Map as input parameter")
Martin Polreichc19e66b2018-10-22 12:22:03 +0200196 // Convert to Map
197 params = ['saltId': saltId, 'target': target, 'state': state, 'excludedStates': excludedStates, 'output': output,
198 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
Martin Polreichf67b39a2019-02-08 10:06:34 +0100199 'retries': retries, 'retries_wait': retries_wait, 'queue': queue, 'saltArgs': saltArgs]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200200 // Call new method with Map as parameter
201 return enforceStateWithExclude(params)
Jakub Josef432e9d92018-02-06 18:28:37 +0100202}
203
Martin Polreich75e40642018-08-13 16:05:08 +0200204/**
205 * Allows to test the given target for reachability and if reachable enforces the state
206* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
207 * @param target State enforcing target
208 * @param state Salt state
209 * @param testTargetMatcher Salt compound matcher to be tested (default is empty string). If empty string, param `target` will be used for tests
210 * @param output print output (optional, default true)
211 * @param failOnError throw exception on salt state result:false (optional, default true)
212 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
213 * @param optional Optional flag (if true pipeline will continue even if no minions for target found)
214 * @param read_timeout http session read timeout (optional, default -1 - disabled)
215 * @param retries Retry count for salt state. (optional, default -1 - no retries)
216 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
217 * @param saltArgs additional salt args eq. ["runas=aptly"]
218 * @return output of salt command
219 */
Martin Polreichc19e66b2018-10-22 12:22:03 +0200220def enforceStateWithTest(Map params) {
Martin Polreich75e40642018-08-13 16:05:08 +0200221 def common = new com.mirantis.mk.Common()
Martin Polreichc19e66b2018-10-22 12:22:03 +0200222 //Set defaults
223 defaults = ["testTargetMatcher": "", "output": true, "failOnError": true, "batch": null, "optional": false,
Martin Polreichf67b39a2019-02-08 10:06:34 +0100224 "read_timeout": -1, "retries": -1, "retries_wait": 5, "queue": true, "saltArgs":[]]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200225 params = defaults + params
226 if (!params.testTargetMatcher) {
227 params.testTargetMatcher = params.target
Martin Polreich75e40642018-08-13 16:05:08 +0200228 }
Martin Polreichf7976952019-03-04 10:06:11 +0100229 if (testTarget(params.saltId, params.testTargetMatcher, params.batch)) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200230 return enforceState(params)
Martin Polreich75e40642018-08-13 16:05:08 +0200231 } else {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200232 if (!params.optional) {
233 common.infoMsg("No Minions matched the target matcher: ${params.testTargetMatcher}, and 'optional' param was set to false. - This may signify missing pillar definition!!")
Martin Polreich92db3e12018-09-27 15:29:23 +0200234// throw new Exception("No Minions matched the target matcher: ${testTargetMatcher}.") TODO: Change the infoMsg to Error once the methods are changed to Use named params and optional param will be set globally
Martin Polreich75e40642018-08-13 16:05:08 +0200235 } else {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200236 common.infoMsg("No Minions matched the target matcher: ${params.testTargetMatcher}, but 'optional' param was set to true - Pipeline continues. ")
Martin Polreich75e40642018-08-13 16:05:08 +0200237 }
238 }
239}
240
Martin Polreichc19e66b2018-10-22 12:22:03 +0200241
Martin Polreichf67b39a2019-02-08 10:06:34 +0100242def enforceStateWithTest(saltId, target, state, testTargetMatcher = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[], retries_wait=5) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200243// Deprecated, convert state to use Map as input parameter
244 def common = new com.mirantis.mk.Common()
Martin Polreich49288692018-12-20 15:41:04 +0100245 common.infoMsg("This method will be deprecated. Convert you method call to use Map as input parameter")
Martin Polreichc19e66b2018-10-22 12:22:03 +0200246 // Convert to Map
247 params = ['saltId': saltId, 'target': target, 'state': state, 'testTargetMatcher': testTargetMatcher, 'output': output,
248 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
Martin Polreichf67b39a2019-02-08 10:06:34 +0100249 'retries': retries, 'retries_wait': retries_wait, 'queue': queue, 'saltArgs': saltArgs]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200250 // Call new method with Map as parameter
251 return enforceStateWithTest(params)
252}
253
Jakub Josef432e9d92018-02-06 18:28:37 +0100254/* Enforces state on given saltId and target
255 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
256 * @param target State enforcing target
257 * @param state Salt state
Jakub Josef5ade54c2017-03-10 16:14:01 +0100258 * @param output print output (optional, default true)
259 * @param failOnError throw exception on salt state result:false (optional, default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200260 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef432e9d92018-02-06 18:28:37 +0100261 * @param optional Optional flag (if true pipeline will continue even if no minions for target found)
Petr Michalecde0ff322017-10-04 09:32:14 +0200262 * @param read_timeout http session read timeout (optional, default -1 - disabled)
263 * @param retries Retry count for salt state. (optional, default -1 - no retries)
264 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
Jakub Josef432e9d92018-02-06 18:28:37 +0100265 * @param saltArgs additional salt args eq. ["runas=aptly", exclude="opencontrail.database"]
Vasyl Saienko6a396212018-06-08 09:20:08 +0300266 * @param minionRestartWaitTimeout specifies timeout that we should wait after minion restart.
Jakub Josef5ade54c2017-03-10 16:14:01 +0100267 * @return output of salt command
268 */
Martin Polreichc19e66b2018-10-22 12:22:03 +0200269def enforceState(Map params) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100270 def common = new com.mirantis.mk.Common()
Martin Polreichc19e66b2018-10-22 12:22:03 +0200271 //Set defaults
Martin Polreichf67b39a2019-02-08 10:06:34 +0100272 defaults = ["output": true, "failOnError": true, "batch": null, "optional": false, "read_timeout": -1,
273 "retries": -1, "retries_wait": 5, "queue": true, "saltArgs": [], "minionRestartWaitTimeout": 10]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200274 params = defaults + params
Jakub Josef432e9d92018-02-06 18:28:37 +0100275 // add state to salt args
Martin Polreichc19e66b2018-10-22 12:22:03 +0200276 if (params.state instanceof String) {
277 params.saltArgs << params.state
Jakub Josef79ecec32017-02-17 14:36:28 +0100278 } else {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200279 params.saltArgs << params.state.join(',')
Jakub Josef79ecec32017-02-17 14:36:28 +0100280 }
281
Martin Polreichc19e66b2018-10-22 12:22:03 +0200282 common.infoMsg("Running state ${params.state} on ${params.target}")
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300283 def out
Petr Michalecde0ff322017-10-04 09:32:14 +0200284 def kwargs = [:]
285
Martin Polreichc19e66b2018-10-22 12:22:03 +0200286 if (params.queue && params.batch == null) {
Petr Michalecde0ff322017-10-04 09:32:14 +0200287 kwargs["queue"] = true
288 }
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300289
Martin Polreichf7976952019-03-04 10:06:11 +0100290 if (params.optional == false || testTarget(params.saltId, params.target, params.batch)){
Martin Polreichc19e66b2018-10-22 12:22:03 +0200291 if (params.retries > 0){
Jakub Josef962ba912018-04-04 17:39:19 +0200292 def retriesCounter = 0
Martin Polreichc19e66b2018-10-22 12:22:03 +0200293 retry(params.retries){
Jakub Josef962ba912018-04-04 17:39:19 +0200294 retriesCounter++
Jakub Josef432e9d92018-02-06 18:28:37 +0100295 // we have to reverse order in saltArgs because salt state have to be first
Martin Polreichc19e66b2018-10-22 12:22:03 +0200296 out = runSaltCommand(params.saltId, 'local', ['expression': params.target, 'type': 'compound'], 'state.sls', params.batch, params.saltArgs.reverse(), kwargs, -1, params.read_timeout)
Jakub Josef432e9d92018-02-06 18:28:37 +0100297 // failOnError should be passed as true because we need to throw exception for retry block handler
Martin Polreichc19e66b2018-10-22 12:22:03 +0200298 checkResult(out, true, params.output, true, retriesCounter < params.retries) //disable ask on error for every interation except last one
Ivan Berezovskiy3e7656b2019-02-11 20:28:40 +0400299 sleep(params['retries_wait'])
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300300 }
Petr Michalecde0ff322017-10-04 09:32:14 +0200301 } else {
Jakub Josef432e9d92018-02-06 18:28:37 +0100302 // we have to reverse order in saltArgs because salt state have to be first
Martin Polreichc19e66b2018-10-22 12:22:03 +0200303 out = runSaltCommand(params.saltId, 'local', ['expression': params.target, 'type': 'compound'], 'state.sls', params.batch, params.saltArgs.reverse(), kwargs, -1, params.read_timeout)
304 checkResult(out, params.failOnError, params.output)
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300305 }
Martin Polreichc19e66b2018-10-22 12:22:03 +0200306 waitForMinion(out, params.minionRestartWaitTimeout)
Martin Polreich1c77afa2017-07-18 11:27:02 +0200307 return out
Martin Polreich1c77afa2017-07-18 11:27:02 +0200308 } else {
309 common.infoMsg("No Minions matched the target given, but 'optional' param was set to true - Pipeline continues. ")
310 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100311}
312
Martin Polreichf67b39a2019-02-08 10:06:34 +0100313def enforceState(saltId, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs = [], minionRestartWaitTimeout=10, retries_wait=5) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200314// Deprecated, convert state to use Map as input parameter
315 def common = new com.mirantis.mk.Common()
Martin Polreich49288692018-12-20 15:41:04 +0100316 common.infoMsg("This method will be deprecated. Convert you method call to use Map as input parameter")
Martin Polreichc19e66b2018-10-22 12:22:03 +0200317 // Convert to Map
Martin Polreichf67b39a2019-02-08 10:06:34 +0100318 params = ['saltId': saltId, 'target': target, 'state': state, 'output': output, 'failOnError': failOnError,
319 'batch': batch, 'optional': optional, 'read_timeout': read_timeout, 'retries': retries,
320 'retries_wait': retries_wait, 'queue': queue, 'saltArgs': saltArgs, 'minionRestartWaitTimeout': minionRestartWaitTimeout]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200321 // Call new method with Map as parameter
322 return enforceState(params)
323}
324
Jakub Josef5ade54c2017-03-10 16:14:01 +0100325/**
326 * Run command on salt minion (salt cmd.run wrapper)
chnydaa0dbb252017-10-05 10:46:09 +0200327 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100328 * @param target Get pillar target
329 * @param cmd command
Jakub Josef053df392017-05-03 15:51:05 +0200330 * @param checkResponse test command success execution (default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200331 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200332 * @param output do you want to print output
chnyda205a92b2018-01-11 17:07:32 +0100333 * @param saltArgs additional salt args eq. ["runas=aptly"]
Sergey Galkin7137ad02019-11-07 14:52:13 +0400334 * @param replacing list with maps for deletion in info message (passwords, logins, etc)
Denis V. Meltsaykin9f997b22020-05-17 17:24:07 +0200335 * @param async run commands with async client (default false)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100336 * @return output of salt command
337 */
Denis V. Meltsaykin9f997b22020-05-17 17:24:07 +0200338def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = [], replacing = [], async = false) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100339 def common = new com.mirantis.mk.Common()
Jakub Josef053df392017-05-03 15:51:05 +0200340 def originalCmd = cmd
Sergey Galkin7137ad02019-11-07 14:52:13 +0400341 common.infoSensitivityMsg("Running command ${cmd} on ${target}", true, replacing)
Jakub Josef053df392017-05-03 15:51:05 +0200342 if (checkResponse) {
343 cmd = cmd + " && echo Salt command execution success"
344 }
chnyda205a92b2018-01-11 17:07:32 +0100345
Jakub Josef432e9d92018-02-06 18:28:37 +0100346 // add cmd name to salt args list
chnyda205a92b2018-01-11 17:07:32 +0100347 saltArgs << cmd
348
Denis V. Meltsaykin9f997b22020-05-17 17:24:07 +0200349 def client = async ? 'local_async' : 'local'
350 def out = runSaltCommand(saltId, client, ['expression': target, 'type': 'compound'], 'cmd.run', batch, saltArgs.reverse())
Jakub Josef053df392017-05-03 15:51:05 +0200351 if (checkResponse) {
352 // iterate over all affected nodes and check success return code
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200353 if (out["return"]){
354 for(int i=0;i<out["return"].size();i++){
355 def node = out["return"][i];
Jakub Josef053df392017-05-03 15:51:05 +0200356 for(int j=0;j<node.size();j++){
357 def nodeKey = node.keySet()[j]
Martin Polreicha2effb82018-08-01 11:35:11 +0200358 if (node[nodeKey] instanceof String) {
359 if (!node[nodeKey].contains("Salt command execution success")) {
360 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
361 }
362 } else if (node[nodeKey] instanceof Boolean) {
363 if (!node[nodeKey]) {
364 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
365 }
366 } else {
367 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
Jakub Josef053df392017-05-03 15:51:05 +0200368 }
369 }
370 }
Martin Polreicha2effb82018-08-01 11:35:11 +0200371 } else {
Jakub Josef053df392017-05-03 15:51:05 +0200372 throw new Exception("Salt Api response doesn't have return param!")
373 }
374 }
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200375 if (output == true) {
376 printSaltCommandResult(out)
377 }
378 return out
Jakub Josef79ecec32017-02-17 14:36:28 +0100379}
380
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200381/**
Jiri Broulik71512bc2017-08-04 10:00:18 +0200382 * Checks if salt minion is in a list of salt master's accepted keys
chnydaa0dbb252017-10-05 10:46:09 +0200383 * @usage minionPresent(saltId, 'I@salt:master', 'ntw', true, null, true, 200, 3)
384 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200385 * @param target Get pillar target
386 * @param minion_name unique identification of a minion in salt-key command output
387 * @param waitUntilPresent return after the minion becomes present (default true)
388 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
389 * @param output print salt command (default true)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200390 * @param maxRetries finite number of iterations to check status of a command (default 200)
391 * @param answers how many minions should return (optional, default 1)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200392 * @return output of salt command
393 */
lmercl94189272018-06-01 11:03:46 +0200394def minionPresent(saltId, target, minion_name, waitUntilPresent = true, batch=null, output = true, maxRetries = 180, answers = 1) {
Jiri Broulik71512bc2017-08-04 10:00:18 +0200395 minion_name = minion_name.replace("*", "")
396 def common = new com.mirantis.mk.Common()
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200397 common.infoMsg("Looking for minion: " + minion_name)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200398 def cmd = 'salt-key | grep ' + minion_name
399 if (waitUntilPresent){
400 def count = 0
401 while(count < maxRetries) {
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200402 try {
403 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
404 if (output) {
405 printSaltCommandResult(out)
406 }
407 def valueMap = out["return"][0]
408 def result = valueMap.get(valueMap.keySet()[0])
409 def resultsArray = result.tokenize("\n")
410 def size = resultsArray.size()
411 if (size >= answers) {
412 return out
413 }
414 count++
415 sleep(time: 1000, unit: 'MILLISECONDS')
416 common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
417 } catch (Exception er) {
418 common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
419 }
420 }
421 } else {
422 try {
chnydaa0dbb252017-10-05 10:46:09 +0200423 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200424 if (output) {
425 printSaltCommandResult(out)
426 }
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200427 return out
428 } catch (Exception er) {
429 common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
Jiri Broulik71512bc2017-08-04 10:00:18 +0200430 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200431 }
432 // otherwise throw exception
433 common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
434 throw new Exception("${cmd} signals failure of status check!")
435}
436
437/**
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200438 * Checks if salt minions are in a list of salt master's accepted keys by matching compound
439 * @usage minionsPresent(saltId, 'I@salt:master', 'I@salt:minion', true, null, true, 200, 3)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100440 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
441 * @param target Performs tests on this target node
442 * @param target_minions all targeted minions to test (for ex. I@salt:minion)
443 * @param waitUntilPresent return after the minion becomes present (default true)
444 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
445 * @param output print salt command (default true)
446 * @param maxRetries finite number of iterations to check status of a command (default 200)
447 * @param answers how many minions should return (optional, default 1)
448 * @return output of salt command
449 */
450def minionsPresent(saltId, target = 'I@salt:master', target_minions = '', waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
Martin Polreichf7976952019-03-04 10:06:11 +0100451 def target_hosts = getMinionsSorted(saltId, target_minions, batch)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100452 for (t in target_hosts) {
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200453 def tgt = stripDomainName(t)
454 minionPresent(saltId, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
455 }
456}
457
458/**
459 * Checks if salt minions are in a list of salt master's accepted keys by matching a list
460 * @usage minionsPresentFromList(saltId, 'I@salt:master', ["cfg01.example.com", "bmk01.example.com"], true, null, true, 200, 3)
461 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
462 * @param target Performs tests on this target node
463 * @param target_minions list to test (for ex. ["cfg01.example.com", "bmk01.example.com"])
464 * @param waitUntilPresent return after the minion becomes present (default true)
465 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
466 * @param output print salt command (default true)
467 * @param maxRetries finite number of iterations to check status of a command (default 200)
468 * @param answers how many minions should return (optional, default 1)
469 * @return output of salt command
470 */
471def minionsPresentFromList(saltId, target = 'I@salt:master', target_minions = [], waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
472 def common = new com.mirantis.mk.Common()
473 for (tgt in target_minions) {
474 common.infoMsg("Checking if minion " + tgt + " is present")
475 minionPresent(saltId, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100476 }
477}
478
479/**
Jiri Broulik71512bc2017-08-04 10:00:18 +0200480 * You can call this function when salt-master already contains salt keys of the target_nodes
chnydaa0dbb252017-10-05 10:46:09 +0200481 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200482 * @param target Should always be salt-master
Martin Polreich712d85c2019-07-01 17:21:12 +0200483 * @param targetNodes unique identification of a minion or group of salt minions
Jiri Broulik71512bc2017-08-04 10:00:18 +0200484 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Martin Polreich712d85c2019-07-01 17:21:12 +0200485 * @param cmdTimeout timeout for the salt command if minions do not return (default 10)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200486 * @param maxRetries finite number of iterations to check status of a command (default 200)
487 * @return output of salt command
488 */
Martin Polreich712d85c2019-07-01 17:21:12 +0200489
490def minionsReachable(saltId, target, targetNodes, batch=null, cmdTimeout = 10, maxRetries = 200) {
Jiri Broulik71512bc2017-08-04 10:00:18 +0200491 def common = new com.mirantis.mk.Common()
Martin Polreich712d85c2019-07-01 17:21:12 +0200492 def cmd = "salt -t${cmdTimeout} -C '${targetNodes}' test.ping"
493 common.infoMsg("Checking if all ${targetNodes} minions are reachable")
494 def retriesCount = 0
495 while(retriesCount < maxRetries) {
Jiri Broulik71512bc2017-08-04 10:00:18 +0200496 Calendar timeout = Calendar.getInstance();
Martin Polreich712d85c2019-07-01 17:21:12 +0200497 timeout.add(Calendar.SECOND, cmdTimeout);
498 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, cmdTimeout)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200499 Calendar current = Calendar.getInstance();
500 if (current.getTime().before(timeout.getTime())) {
Martin Polreich712d85c2019-07-01 17:21:12 +0200501 common.infoMsg("Successful response received from all targeted nodes.")
502 printSaltCommandResult(out)
503 return out
Jiri Broulik71512bc2017-08-04 10:00:18 +0200504 }
Martin Polreich712d85c2019-07-01 17:21:12 +0200505 def outYaml = readYaml text: getReturnValues(out)
506 def successfulNodes = []
507 def failedNodes = []
508 for (node in outYaml.keySet()) {
509 if (outYaml[node] == true || outYaml[node].toString().toLowerCase() == 'true') {
510 successfulNodes.add(node)
511 } else {
512 failedNodes.add(node)
513 }
514 }
515 common.infoMsg("Not all of the targeted minions returned yet. Successful response from ${successfulNodes}. Still waiting for ${failedNodes}.")
516 retriesCount++
Jiri Broulik71512bc2017-08-04 10:00:18 +0200517 sleep(time: 500, unit: 'MILLISECONDS')
518 }
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200519}
520
Martin Polreich712d85c2019-07-01 17:21:12 +0200521
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200522/**
Denis Egorenkoa1edaba2019-02-27 19:19:12 +0400523 * You can call this function when need to check that all minions are available, free and ready for command execution
524 * @param config LinkedHashMap config parameter, which contains next:
525 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
526 * @param target unique identification of a minion or group of salt minions
527 * @param target_reachable unique identification of a minion or group of salt minions to check availability
528 * @param wait timeout between retries to check target minions (default 5)
529 * @param retries finite number of iterations to check minions (default 10)
530 * @param timeout timeout for the salt command if minions do not return (default 5)
531 * @param availability check that minions also are available before checking readiness (default true)
532 */
533def checkTargetMinionsReady(LinkedHashMap config) {
534 def common = new com.mirantis.mk.Common()
535 def saltId = config.get('saltId')
536 def target = config.get('target')
537 def target_reachable = config.get('target_reachable', target)
538 def wait = config.get('wait', 30)
539 def retries = config.get('retries', 10)
540 def timeout = config.get('timeout', 5)
541 def checkAvailability = config.get('availability', true)
Martin Polreichf7976952019-03-04 10:06:11 +0100542 def batch = config.get('batch', null)
Denis Egorenkoa1edaba2019-02-27 19:19:12 +0400543 common.retry(retries, wait) {
544 if (checkAvailability) {
Martin Polreichf7976952019-03-04 10:06:11 +0100545 minionsReachable(saltId, 'I@salt:master', target_reachable, batch)
Denis Egorenkoa1edaba2019-02-27 19:19:12 +0400546 }
Martin Polreichf7976952019-03-04 10:06:11 +0100547 def running = runSaltProcessStep(saltId, target, 'saltutil.running', [], batch, true, timeout)
Denis Egorenkoa1edaba2019-02-27 19:19:12 +0400548 for (value in running.get("return")[0].values()) {
549 if (value != []) {
550 throw new Exception("Not all salt-minions are ready for execution")
551 }
552 }
553 }
554}
555
556/**
Vasyl Saienko121f34c2019-06-24 23:39:32 +0300557 * Restart and wait for salt-minions on target nodes.
558 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
559 * @param target unique identification of a minion or group of salt minions
Ivan Berezovskiy768dabd2019-08-01 15:52:44 +0400560 * @param wait timeout for the salt command if minions do not return (default 10)
561 * @param maxRetries finite number of iterations to check status of a command (default 15)
562 * @param async Run salt minion restart and do not wait for response
Vasyl Saienko121f34c2019-06-24 23:39:32 +0300563 * @return output of salt command
564 */
Ivan Berezovskiy768dabd2019-08-01 15:52:44 +0400565def restartSaltMinion(saltId, target, wait = 10, maxRetries = 15, async = true) {
Vasyl Saienko121f34c2019-06-24 23:39:32 +0300566 def common = new com.mirantis.mk.Common()
567 common.infoMsg("Restarting salt-minion on ${target} and waiting for they are reachable.")
Ivan Berezovskiy768dabd2019-08-01 15:52:44 +0400568 runSaltProcessStep(saltId, target, 'cmd.shell', ['salt-call service.restart salt-minion'], null, true, 60, null, async)
569 checkTargetMinionsReady(['saltId': saltId, 'target': target, timeout: wait, retries: maxRetries])
Vasyl Saienko121f34c2019-06-24 23:39:32 +0300570 common.infoMsg("All ${target} minions are alive...")
571}
572
573/**
574 * Upgrade package and restart salt minion.
575 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
576 * @param target unique identification of a minion or group of salt minions
577 * @param the name of pkg_name to upgrade
578 * @param wait timeout for the salt command if minions do not return (default 5)
579 * @param maxRetries finite number of iterations to check status of a command (default 10)
580 * @return output of salt command
581 */
582def upgradePackageAndRestartSaltMinion(saltId, target, pkg_name, wait = 5, maxRetries = 10) {
583 def common = new com.mirantis.mk.Common()
584 def latest_version = getReturnValues(runSaltProcessStep(saltId, target, 'pkg.latest_version', [pkg_name, 'show_installed=True'])).split('\n')[0]
585 def current_version = getReturnValues(runSaltProcessStep(saltId, target, 'pkg.version', [pkg_name])).split('\n')[0]
586 if (current_version && latest_version != current_version) {
587 common.infoMsg("Upgrading current ${pkg_name}: ${current_version} to ${latest_version}")
588 runSaltProcessStep(saltId, target, 'pkg.install', [pkg_name], 'only_upgrade=True')
589 restartSaltMinion(saltId, target, wait, maxRetries)
590 }
591}
592
593/**
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200594 * Run command on salt minion (salt cmd.run wrapper)
chnydaa0dbb252017-10-05 10:46:09 +0200595 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200596 * @param target Get pillar target
597 * @param cmd name of a service
598 * @param correct_state string that command must contain if status is in correct state (optional, default 'running')
Jiri Broulikcf1f2332017-07-25 11:30:03 +0200599 * @param find bool value if it is suppose to find some string in the output or the cmd should return empty string (optional, default true)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200600 * @param waitUntilOk return after the minion becomes present (optional, default true)
601 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
602 * @param output print salt command (default true)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200603 * @param maxRetries finite number of iterations to check status of a command (default 200)
604 * @param answers how many minions should return (optional, default 0)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200605 * @return output of salt command
606 */
chnydaa0dbb252017-10-05 10:46:09 +0200607def commandStatus(saltId, target, cmd, correct_state='running', find = true, waitUntilOk = true, batch=null, output = true, maxRetries = 200, answers = 0) {
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200608 def common = new com.mirantis.mk.Common()
609 common.infoMsg("Checking if status of verification command ${cmd} on ${target} is in correct state")
610 if (waitUntilOk){
611 def count = 0
612 while(count < maxRetries) {
chnydaa0dbb252017-10-05 10:46:09 +0200613 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200614 if (output) {
615 printSaltCommandResult(out)
616 }
Jakub Josef115a78f2017-07-18 15:04:00 +0200617 def resultMap = out["return"][0]
Jiri Broulik71512bc2017-08-04 10:00:18 +0200618 def success = 0
619 if (answers == 0){
620 answers = resultMap.size()
621 }
622 for (int i=0;i<answers;i++) {
623 result = resultMap.get(resultMap.keySet()[i])
624 // if the goal is to find some string in output of the command
625 if (find) {
626 if(result == null || result instanceof Boolean || result.isEmpty()) { result='' }
627 if (result.toLowerCase().contains(correct_state.toLowerCase())) {
628 success++
629 if (success == answers) {
630 return out
631 }
Jiri Broulikd0c27572017-07-24 20:01:10 +0200632 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200633 // else the goal is to not find any string in output of the command
634 } else {
635 if(result instanceof String && result.isEmpty()) {
636 success++
637 if (success == answers) {
638 return out
chnydaa0dbb252017-10-05 10:46:09 +0200639 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200640 }
641 }
642 }
643 count++
644 sleep(time: 500, unit: 'MILLISECONDS')
645 common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
646 }
647 } else {
chnydaa0dbb252017-10-05 10:46:09 +0200648 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200649 def resultMap = out["return"][0]
650 if (output) {
651 printSaltCommandResult(out)
652 }
653 for (int i=0;i<resultMap.size();i++) {
654 result = resultMap.get(resultMap.keySet()[i])
655 // if the goal is to find some string in output of the command
656 if (find) {
657 if(result == null || result instanceof Boolean || result.isEmpty()) { result='' }
658 if (result.toLowerCase().contains(correct_state.toLowerCase())) {
Jiri Broulikd0c27572017-07-24 20:01:10 +0200659 return out
660 }
661
662 // else the goal is to not find any string in output of the command
663 } else {
664 if(result instanceof String && result.isEmpty()) {
665 return out
666 }
667 }
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200668 }
669 }
670 // otherwise throw exception
Jiri Broulikd0c27572017-07-24 20:01:10 +0200671 common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200672 throw new Exception("${cmd} signals failure of status check!")
673}
674
Jakub Josef5ade54c2017-03-10 16:14:01 +0100675/**
676 * Perform complete salt sync between master and target
chnydaa0dbb252017-10-05 10:46:09 +0200677 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100678 * @param target Get pillar target
Martin Polreichf7976952019-03-04 10:06:11 +0100679 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100680 * @return output of salt command
681 */
Martin Polreichf7976952019-03-04 10:06:11 +0100682def syncAll(saltId, target, batch = null) {
683 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'saltutil.sync_all', batch)
Jakub Josef79ecec32017-02-17 14:36:28 +0100684}
685
Jakub Josef5ade54c2017-03-10 16:14:01 +0100686/**
Jakub Josef432e9d92018-02-06 18:28:37 +0100687 * Perform complete salt refresh between master and target
688 * Method will call saltutil.refresh_pillar, saltutil.refresh_grains and saltutil.sync_all
689 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
690 * @param target Get pillar target
Martin Polreichf7976952019-03-04 10:06:11 +0100691 * @param batch Batch param to salt (integer or string with percents)
Ivan Berezovskiy19063902020-03-19 13:25:30 +0400692 * @param oneByOne Refresh each node separately
Jakub Josef432e9d92018-02-06 18:28:37 +0100693 * @return output of salt command
694 */
Ivan Berezovskiy19063902020-03-19 13:25:30 +0400695def fullRefresh(saltId, target, batch=20, oneByOne=false) {
696 if (oneByOne) {
697 def minions = getMinions(saltId, target)
698 for (minion in minions) {
699 runSaltProcessStep(saltId, minion, 'saltutil.refresh_pillar', [], null, true, 60)
700 runSaltProcessStep(saltId, minion, 'saltutil.refresh_grains', [], null, true, 60)
701 runSaltProcessStep(saltId, minion, 'saltutil.sync_all', [], null, true, 180)
702 }
703 } else {
704 runSaltProcessStep(saltId, target, 'saltutil.refresh_pillar', [], batch, true)
705 runSaltProcessStep(saltId, target, 'saltutil.refresh_grains', [], batch, true)
706 runSaltProcessStep(saltId, target, 'saltutil.sync_all', [], batch, true)
707 }
Jakub Josef432e9d92018-02-06 18:28:37 +0100708}
709
710/**
711 * Enforce highstate on given targets
712 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
713 * @param target Highstate enforcing target
714 * @param excludedStates states which will be excluded from main state (default empty string)
715 * @param output print output (optional, default true)
716 * @param failOnError throw exception on salt state result:false (optional, default true)
717 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
718 * @param saltArgs additional salt args eq. ["runas=aptly", exclude="opencontrail.database"]
719 * @return output of salt command
720 */
721def enforceHighstateWithExclude(saltId, target, excludedStates = "", output = false, failOnError = true, batch = null, saltArgs = []) {
722 saltArgs << "exclude=${excludedStates}"
723 return enforceHighstate(saltId, target, output, failOnError, batch, saltArgs)
724}
725/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100726 * Enforce highstate on given targets
chnydaa0dbb252017-10-05 10:46:09 +0200727 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100728 * @param target Highstate enforcing target
729 * @param output print output (optional, default true)
730 * @param failOnError throw exception on salt state result:false (optional, default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200731 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100732 * @return output of salt command
733 */
Jakub Josef432e9d92018-02-06 18:28:37 +0100734def enforceHighstate(saltId, target, output = false, failOnError = true, batch = null, saltArgs = []) {
Petr Jediný30be7032018-05-29 18:22:46 +0200735 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'state.highstate', batch, saltArgs)
Alexander Noskov657ccfc2017-07-14 11:35:52 +0000736 def common = new com.mirantis.mk.Common()
737
Marek Celoud63366112017-07-25 17:27:24 +0200738 common.infoMsg("Running state highstate on ${target}")
Alexander Noskov657ccfc2017-07-14 11:35:52 +0000739
Jakub Josef374beb72017-04-27 15:45:09 +0200740 checkResult(out, failOnError, output)
Jakub Josef79ecec32017-02-17 14:36:28 +0100741 return out
742}
743
Jakub Josef5ade54c2017-03-10 16:14:01 +0100744/**
Ales Komarek5276ebe2017-03-16 08:46:34 +0100745 * Get running minions IDs according to the target
chnydaa0dbb252017-10-05 10:46:09 +0200746 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Ales Komarek5276ebe2017-03-16 08:46:34 +0100747 * @param target Get minions target
Martin Polreichf7976952019-03-04 10:06:11 +0100748 * @param batch Batch param to salt (integer or string with percents)
Ales Komarek5276ebe2017-03-16 08:46:34 +0100749 * @return list of active minions fitin
750 */
Martin Polreichf7976952019-03-04 10:06:11 +0100751def getMinions(saltId, target, batch = null) {
752 def minionsRaw = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'test.ping', batch)
Ales Komarek5276ebe2017-03-16 08:46:34 +0100753 return new ArrayList<String>(minionsRaw['return'][0].keySet())
754}
755
Jiri Broulikf8f96942018-02-15 10:03:42 +0100756/**
757 * Get sorted running minions IDs according to the target
758 * @param saltId Salt Connection object or pepperEnv
759 * @param target Get minions target
Martin Polreichf7976952019-03-04 10:06:11 +0100760 * @param batch Batch param to salt (integer or string with percents)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100761 * @return list of sorted active minions fitin
762 */
Martin Polreichf7976952019-03-04 10:06:11 +0100763def getMinionsSorted(saltId, target, batch = null) {
764 return getMinions(saltId, target, batch).sort()
Jiri Broulikf8f96942018-02-15 10:03:42 +0100765}
766
767/**
768 * Get first out of running minions IDs according to the target
769 * @param saltId Salt Connection object or pepperEnv
770 * @param target Get minions target
Martin Polreichf7976952019-03-04 10:06:11 +0100771 * @param batch Batch param to salt (integer or string with percents)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100772 * @return first of active minions fitin
773 */
Martin Polreichf7976952019-03-04 10:06:11 +0100774def getFirstMinion(saltId, target, batch = null) {
775 def minionsSorted = getMinionsSorted(saltId, target, batch)
Dmitry Ukovd72cd2a2018-09-04 17:31:46 +0400776 return minionsSorted[0]
Jiri Broulikf8f96942018-02-15 10:03:42 +0100777}
778
779/**
780 * Get running salt minions IDs without it's domain name part and its numbering identifications
781 * @param saltId Salt Connection object or pepperEnv
782 * @param target Get minions target
Martin Polreichf7976952019-03-04 10:06:11 +0100783 * @param batch Batch param to salt (integer or string with percents)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100784 * @return list of active minions fitin without it's domain name part name numbering
785 */
Martin Polreichf7976952019-03-04 10:06:11 +0100786def getMinionsGeneralName(saltId, target, batch = null) {
787 def minionsSorted = getMinionsSorted(saltId, target, batch)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100788 return stripDomainName(minionsSorted[0]).replaceAll('\\d+$', "")
789}
790
791/**
792 * Get domain name of the env
793 * @param saltId Salt Connection object or pepperEnv
794 * @return domain name
795 */
796def getDomainName(saltId) {
797 return getReturnValues(getPillar(saltId, 'I@salt:master', '_param:cluster_domain'))
798}
799
800/**
801 * Remove domain name from Salt minion ID
802 * @param name String of Salt minion ID
803 * @return Salt minion ID without its domain name
804 */
805def stripDomainName(name) {
806 return name.split("\\.")[0]
807}
808
809/**
810 * Gets return values of a salt command
811 * @param output String of Salt minion ID
812 * @return Return values of a salt command
813 */
814def getReturnValues(output) {
Martin Polreichf7976952019-03-04 10:06:11 +0100815 if(output && output.containsKey("return") && !output.get("return").isEmpty()) {
Jiri Broulikf8f96942018-02-15 10:03:42 +0100816 return output['return'][0].values()[0]
817 }
818 def common = new com.mirantis.mk.Common()
819 common.errorMsg('output does not contain return key')
820 return ''
821}
822
823/**
824 * Get minion ID of one of KVM nodes
825 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
826 * @return Salt minion ID of one of KVM nodes in env
827 */
828def getKvmMinionId(saltId) {
829 return getReturnValues(getGrain(saltId, 'I@salt:control', 'id')).values()[0]
830}
831
832/**
833 * Get Salt minion ID of KVM node hosting 'name' VM
834 * @param saltId Salt Connection object or pepperEnv
835 * @param name Name of the VM (for ex. ctl01)
836 * @return Salt minion ID of KVM node hosting 'name' VM
837 */
Jiri Broulikd2a50552018-04-25 17:17:59 +0200838def getNodeProvider(saltId, nodeName) {
839 def salt = new com.mirantis.mk.Salt()
840 def common = new com.mirantis.mk.Common()
841 def kvms = salt.getMinions(saltId, 'I@salt:control')
842 for (kvm in kvms) {
843 try {
844 vms = salt.getReturnValues(salt.runSaltProcessStep(saltId, kvm, 'virt.list_domains', [], null, true))
845 if (vms.toString().contains(nodeName)) {
846 return kvm
847 }
848 } catch (Exception er) {
849 common.infoMsg("${nodeName} not present on ${kvm}")
850 }
851 }
Jiri Broulikf8f96942018-02-15 10:03:42 +0100852}
853
Ales Komarek5276ebe2017-03-16 08:46:34 +0100854/**
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200855 * Test if there are any minions to target
chnydaa0dbb252017-10-05 10:46:09 +0200856 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200857 * @param target Target to test
Martin Polreichf7976952019-03-04 10:06:11 +0100858 * @param batch Batch param to salt (integer or string with percents)
vrovachev1c4770b2017-07-05 13:25:21 +0400859 * @return bool indicating if target was succesful
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200860 */
861
Martin Polreichf7976952019-03-04 10:06:11 +0100862def testTarget(saltId, target, batch = null) {
863 return getMinions(saltId, target, batch).size() > 0
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200864}
865
866/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100867 * Generates node key using key.gen_accept call
chnydaa0dbb252017-10-05 10:46:09 +0200868 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100869 * @param target Key generating target
870 * @param host Key generating host
871 * @param keysize generated key size (optional, default 4096)
Martin Polreichf7976952019-03-04 10:06:11 +0100872 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100873 * @return output of salt command
874 */
Martin Polreichf7976952019-03-04 10:06:11 +0100875def generateNodeKey(saltId, target, host, keysize = 4096, batch = null) {
876 return runSaltCommand(saltId, 'wheel', target, 'key.gen_accept', batch, [host], ['keysize': keysize])
Jakub Josef79ecec32017-02-17 14:36:28 +0100877}
878
Jakub Josef5ade54c2017-03-10 16:14:01 +0100879/**
Jakub Josef2f25cf22017-03-28 13:34:57 +0200880 * Generates node reclass metadata
chnydaa0dbb252017-10-05 10:46:09 +0200881 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100882 * @param target Metadata generating target
883 * @param host Metadata generating host
884 * @param classes Reclass classes
885 * @param parameters Reclass parameters
Martin Polreichf7976952019-03-04 10:06:11 +0100886 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100887 * @return output of salt command
888 */
Martin Polreichf7976952019-03-04 10:06:11 +0100889def generateNodeMetadata(saltId, target, host, classes, parameters, batch = null) {
890 return runSaltCommand(saltId, 'local', target, 'reclass.node_create', batch, [host, '_generated'], ['classes': classes, 'parameters': parameters])
Jakub Josef79ecec32017-02-17 14:36:28 +0100891}
892
Jakub Josef5ade54c2017-03-10 16:14:01 +0100893/**
894 * Run salt orchestrate on given targets
chnydaa0dbb252017-10-05 10:46:09 +0200895 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100896 * @param target Orchestration target
897 * @param orchestrate Salt orchestrate params
Dzmitry Stremkouskidd020d92018-07-22 12:01:07 +0200898 * @param kwargs Salt orchestrate params
Martin Polreichf7976952019-03-04 10:06:11 +0100899 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100900 * @return output of salt command
901 */
Martin Polreichf7976952019-03-04 10:06:11 +0100902def orchestrateSystem(saltId, target, orchestrate=[], kwargs = null, batch = null) {
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +0300903 //Since the runSaltCommand uses "arg" (singular) for "runner" client this won`t work correctly on old salt 2016
904 //cause this version of salt used "args" (plural) for "runner" client, see following link for reference:
905 //https://github.com/saltstack/salt/pull/32938
Oleksii Grudev0c098382018-08-08 16:16:38 +0300906 def common = new com.mirantis.mk.Common()
Martin Polreichf7976952019-03-04 10:06:11 +0100907 def result = runSaltCommand(saltId, 'runner', target, 'state.orchestrate', batch, orchestrate, kwargs, 7200, 7200)
Oleksii Grudev0c098382018-08-08 16:16:38 +0300908 if(result != null){
909 if(result['return']){
910 def retcode = result['return'][0].get('retcode')
911 if (retcode != 0) {
912 throw new Exception("Orchestration state failed while running: "+orchestrate)
913 }else{
914 common.infoMsg("Orchestrate state "+orchestrate+" succeeded")
915 }
916 }else{
917 common.errorMsg("Salt result has no return attribute! Result: ${result}")
918 }
919 }else{
920 common.errorMsg("Cannot check salt result, given result is null")
921 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100922}
923
Jakub Josef5ade54c2017-03-10 16:14:01 +0100924/**
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200925 * Run salt pre or post orchestrate tasks
926 *
927 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
928 * @param pillar_tree Reclass pillar that has orchestrate pillar for desired stage
929 * @param extra_tgt Extra targets for compound
Martin Polreichf7976952019-03-04 10:06:11 +0100930 * @param batch Batch param to salt (integer or string with percents)
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200931 * @return output of salt command
932 */
Martin Polreichf7976952019-03-04 10:06:11 +0100933def orchestratePrePost(saltId, pillar_tree, extra_tgt = '', batch = null) {
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200934
935 def common = new com.mirantis.mk.Common()
936 def salt = new com.mirantis.mk.Salt()
937 def compound = 'I@' + pillar_tree + " " + extra_tgt
938
939 common.infoMsg("Refreshing pillars")
Martin Polreichf7976952019-03-04 10:06:11 +0100940 runSaltProcessStep(saltId, '*', 'saltutil.refresh_pillar', [], batch, true)
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200941
942 common.infoMsg("Looking for orchestrate pillars")
Martin Polreichf7976952019-03-04 10:06:11 +0100943 if (salt.testTarget(saltId, compound, batch)) {
944 for ( node in salt.getMinionsSorted(saltId, compound, batch) ) {
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200945 def pillar = salt.getPillar(saltId, node, pillar_tree)
946 if ( !pillar['return'].isEmpty() ) {
947 for ( orch_id in pillar['return'][0].values() ) {
948 def orchestrator = orch_id.values()['orchestrator']
949 def orch_enabled = orch_id.values()['enabled']
950 if ( orch_enabled ) {
951 common.infoMsg("Orchestrating: ${orchestrator}")
Martin Polreichf7976952019-03-04 10:06:11 +0100952 salt.printSaltCommandResult(salt.orchestrateSystem(saltId, ['expression': node], [orchestrator], null, batch))
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200953 }
954 }
955 }
956 }
957 }
958}
959
960/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100961 * Run salt process step
chnydaa0dbb252017-10-05 10:46:09 +0200962 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100963 * @param tgt Salt process step target
964 * @param fun Salt process step function
965 * @param arg process step arguments (optional, default [])
Ivan Berezovskiy768dabd2019-08-01 15:52:44 +0400966 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch). Can't be used with async
Jakub Josef432e9d92018-02-06 18:28:37 +0100967 * @param output print output (optional, default true)
Jiri Broulik48544be2017-06-14 18:33:54 +0200968 * @param timeout Additional argument salt api timeout
Ivan Berezovskiy768dabd2019-08-01 15:52:44 +0400969 * @param async Run the salt command but don't wait for a reply. Can't be used with batch
Jakub Josef5ade54c2017-03-10 16:14:01 +0100970 * @return output of salt command
971 */
Ivan Berezovskiy768dabd2019-08-01 15:52:44 +0400972def runSaltProcessStep(saltId, tgt, fun, arg = [], batch = null, output = true, timeout = -1, kwargs = null, async = false) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100973 def common = new com.mirantis.mk.Common()
Jiri Broulik48544be2017-06-14 18:33:54 +0200974 def salt = new com.mirantis.mk.Salt()
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100975 def out
976
Marek Celoud63366112017-07-25 17:27:24 +0200977 common.infoMsg("Running step ${fun} ${arg} on ${tgt}")
Martin Polreichf7976952019-03-04 10:06:11 +0100978 if (async == true) {
979 out = runSaltCommand(saltId, 'local_async', ['expression': tgt, 'type': 'compound'], fun, null, arg, kwargs, timeout)
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100980 } else {
chnydaa0dbb252017-10-05 10:46:09 +0200981 out = runSaltCommand(saltId, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg, kwargs, timeout)
Jakub Josef79ecec32017-02-17 14:36:28 +0100982 }
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100983
Tomáš Kukrálf5dda642017-03-02 14:22:59 +0100984 if (output == true) {
Jiri Broulik48544be2017-06-14 18:33:54 +0200985 salt.printSaltCommandResult(out)
Jakub Josef79ecec32017-02-17 14:36:28 +0100986 }
Jiri Broulikae19c262017-05-16 19:06:52 +0200987 return out
Jakub Josef79ecec32017-02-17 14:36:28 +0100988}
989
990/**
991 * Check result for errors and throw exception if any found
992 *
993 * @param result Parsed response of Salt API
Jakub Josef8021c002017-03-27 15:41:28 +0200994 * @param failOnError Do you want to throw exception if salt-call fails (optional, default true)
Jakub Josefa87941c2017-04-20 17:14:58 +0200995 * @param printResults Do you want to print salt results (optional, default true)
Jakub Josefa87941c2017-04-20 17:14:58 +0200996 * @param printOnlyChanges If true (default), print only changed resources
Jakub Josef432e9d92018-02-06 18:28:37 +0100997 * @param disableAskOnError Flag for disabling ASK_ON_ERROR feature (optional, default false)
Jakub Josef79ecec32017-02-17 14:36:28 +0100998 */
Jakub Josef432e9d92018-02-06 18:28:37 +0100999def checkResult(result, failOnError = true, printResults = true, printOnlyChanges = true, disableAskOnError = false) {
Jakub Josef5ade54c2017-03-10 16:14:01 +01001000 def common = new com.mirantis.mk.Common()
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001001 if(result != null){
1002 if(result['return']){
1003 for (int i=0;i<result['return'].size();i++) {
1004 def entry = result['return'][i]
1005 if (!entry) {
1006 if (failOnError) {
1007 throw new Exception("Salt API returned empty response: ${result}")
1008 } else {
1009 common.errorMsg("Salt API returned empty response: ${result}")
Jakub Josefece32af2017-03-14 19:20:08 +01001010 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001011 }
1012 for (int j=0;j<entry.size();j++) {
1013 def nodeKey = entry.keySet()[j]
1014 def node=entry[nodeKey]
Jakub Josefa87941c2017-04-20 17:14:58 +02001015 def outputResources = []
Jakub Josef47145942018-04-04 17:30:38 +02001016 def errorResources = []
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001017 common.infoMsg("Node ${nodeKey} changes:")
1018 if(node instanceof Map || node instanceof List){
1019 for (int k=0;k<node.size();k++) {
1020 def resource;
1021 def resKey;
1022 if(node instanceof Map){
1023 resKey = node.keySet()[k]
Victor Ryzhenkin49d67812019-01-09 15:28:21 +04001024 if (resKey == "retcode") {
Richard Felkld9476ac2018-07-12 19:01:33 +02001025 continue
Victor Ryzhenkin49d67812019-01-09 15:28:21 +04001026 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001027 }else if(node instanceof List){
1028 resKey = k
1029 }
1030 resource = node[resKey]
Jakub Josefc4c40202017-04-28 12:04:24 +02001031 // print
Jakub Josefa87941c2017-04-20 17:14:58 +02001032 if(printResults){
1033 if(resource instanceof Map && resource.keySet().contains("result")){
1034 //clean unnesaccary fields
1035 if(resource.keySet().contains("__run_num__")){
1036 resource.remove("__run_num__")
1037 }
1038 if(resource.keySet().contains("__id__")){
1039 resource.remove("__id__")
1040 }
1041 if(resource.keySet().contains("pchanges")){
1042 resource.remove("pchanges")
1043 }
1044 if(!resource["result"] || (resource["result"] instanceof String && resource["result"] != "true")){
1045 if(resource["result"] != null){
Jakub Josefbceaa322017-06-13 18:28:27 +02001046 outputResources.add(String.format("Resource: %s\n\u001B[31m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001047 }else{
Jakub Josefbceaa322017-06-13 18:28:27 +02001048 outputResources.add(String.format("Resource: %s\n\u001B[33m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001049 }
1050 }else{
Ivan Berezovskiy65fb6372019-08-09 19:49:55 +04001051 if(!printOnlyChanges || (resource.changes && resource.changes.size() > 0)) {
Jakub Josefbceaa322017-06-13 18:28:27 +02001052 outputResources.add(String.format("Resource: %s\n\u001B[32m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001053 }
1054 }
1055 }else{
Jakub Josefbceaa322017-06-13 18:28:27 +02001056 outputResources.add(String.format("Resource: %s\n\u001B[36m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001057 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001058 }
Jakub Josefc4c40202017-04-28 12:04:24 +02001059 common.debugMsg("checkResult: checking resource: ${resource}")
1060 if(resource instanceof String || (resource["result"] != null && !resource["result"]) || (resource["result"] instanceof String && resource["result"] == "false")){
Jakub Josef47145942018-04-04 17:30:38 +02001061 errorResources.add(resource)
Jakub Josefc4c40202017-04-28 12:04:24 +02001062 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001063 }
Jakub Josefa87941c2017-04-20 17:14:58 +02001064 }else if(node!=null && node!=""){
Jakub Josef62f6c842017-08-04 16:36:35 +02001065 outputResources.add(String.format("Resource: %s\n\u001B[36m%s\u001B[0m", nodeKey, common.prettify(node)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001066 }
1067 if(printResults && !outputResources.isEmpty()){
Jakub Josef47145942018-04-04 17:30:38 +02001068 println outputResources.stream().collect(Collectors.joining("\n"))
1069 }
1070 if(!errorResources.isEmpty()){
1071 for(resource in errorResources){
1072 def prettyResource = common.prettify(resource)
1073 if (!disableAskOnError && env["ASK_ON_ERROR"] && env["ASK_ON_ERROR"] == "true") {
1074 timeout(time:1, unit:'HOURS') {
1075 input message: "False result on ${nodeKey} found, resource ${prettyResource}. \nDo you want to continue?"
1076 }
1077 } else {
1078 def errorMsg = "Salt state on node ${nodeKey} failed. Resource: ${prettyResource}"
1079 if (failOnError) {
1080 throw new Exception(errorMsg)
1081 } else {
1082 common.errorMsg(errorMsg)
1083 }
1084 }
1085 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001086 }
1087 }
Jakub Josef52f69f72017-03-14 15:18:08 +01001088 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001089 }else{
1090 common.errorMsg("Salt result hasn't return attribute! Result: ${result}")
Jakub Josef79ecec32017-02-17 14:36:28 +01001091 }
Jakub Josef52f69f72017-03-14 15:18:08 +01001092 }else{
Jakub Josefa87941c2017-04-20 17:14:58 +02001093 common.errorMsg("Cannot check salt result, given result is null")
Jakub Josef79ecec32017-02-17 14:36:28 +01001094 }
1095}
1096
1097/**
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001098* Parse salt API output to check minion restart and wait some time to be sure minion is up.
1099* See https://mirantis.jira.com/browse/PROD-16258 for more details
1100* TODO: change sleep to more tricky procedure.
1101*
1102* @param result Parsed response of Salt API
1103*/
Vasyl Saienko6a396212018-06-08 09:20:08 +03001104def waitForMinion(result, minionRestartWaitTimeout=10) {
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001105 def common = new com.mirantis.mk.Common()
Oleg Iurchenko3eedc782017-12-12 11:49:29 +02001106 //In order to prevent multiple sleeps use bool variable to catch restart for any minion.
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001107 def isMinionRestarted = false
Oleg Iurchenko3eedc782017-12-12 11:49:29 +02001108 if(result != null){
1109 if(result['return']){
1110 for (int i=0;i<result['return'].size();i++) {
1111 def entry = result['return'][i]
1112 // exit in case of empty response.
1113 if (!entry) {
1114 return
1115 }
1116 // Loop for nodes
1117 for (int j=0;j<entry.size();j++) {
1118 def nodeKey = entry.keySet()[j]
1119 def node=entry[nodeKey]
1120 if(node instanceof Map || node instanceof List){
1121 // Loop for node resources
1122 for (int k=0;k<node.size();k++) {
1123 def resource;
1124 def resKey;
1125 if(node instanceof Map){
1126 resKey = node.keySet()[k]
1127 }else if(node instanceof List){
1128 resKey = k
1129 }
1130 resource = node[resKey]
Jakub Joseffb9996d2018-04-10 14:05:31 +02001131 // try to find if salt_minion service was restarted
1132 if(resKey instanceof String && resKey.contains("salt_minion_service_restart") && resource instanceof Map && resource.keySet().contains("result")){
Oleg Iurchenko3eedc782017-12-12 11:49:29 +02001133 if((resource["result"] instanceof Boolean && resource["result"]) || (resource["result"] instanceof String && resource["result"] == "true")){
1134 if(resource.changes.size() > 0){
1135 isMinionRestarted=true
1136 }
1137 }
1138 }
1139 }
1140 }
1141 }
1142 }
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001143 }
1144 }
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001145 if (isMinionRestarted){
Vasyl Saienko6a396212018-06-08 09:20:08 +03001146 common.infoMsg("Salt minion service restart detected. Sleep ${minionRestartWaitTimeout} seconds to wait minion restart")
1147 sleep(minionRestartWaitTimeout)
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001148 }
1149}
1150
1151/**
Jakub Josef7852fe12017-03-15 16:02:41 +01001152 * Print salt command run results in human-friendly form
Jakub Josef79ecec32017-02-17 14:36:28 +01001153 *
1154 * @param result Parsed response of Salt API
Jakub Josef79ecec32017-02-17 14:36:28 +01001155 */
Filip Pytlound2f1bbe2017-02-27 19:03:51 +01001156def printSaltCommandResult(result) {
Jakub Josef871bf152017-03-14 20:13:41 +01001157 def common = new com.mirantis.mk.Common()
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001158 if(result != null){
1159 if(result['return']){
1160 for (int i=0; i<result['return'].size(); i++) {
1161 def entry = result['return'][i]
1162 for (int j=0; j<entry.size(); j++) {
1163 common.debugMsg("printSaltCommandResult: printing salt command entry: ${entry}")
1164 def nodeKey = entry.keySet()[j]
1165 def node=entry[nodeKey]
Jakub Josefbceaa322017-06-13 18:28:27 +02001166 common.infoMsg(String.format("Node %s changes:\n%s",nodeKey, common.prettify(node)))
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001167 }
Jakub Josef8a715bf2017-03-14 21:39:01 +01001168 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001169 }else{
1170 common.errorMsg("Salt result hasn't return attribute! Result: ${result}")
Jakub Josef79ecec32017-02-17 14:36:28 +01001171 }
Jakub Josef8a715bf2017-03-14 21:39:01 +01001172 }else{
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001173 common.errorMsg("Cannot print salt command result, given result is null")
Jakub Josef52f69f72017-03-14 15:18:08 +01001174 }
Jakub Josef79ecec32017-02-17 14:36:28 +01001175}
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +02001176
1177
1178/**
1179 * Return content of file target
1180 *
chnydaa0dbb252017-10-05 10:46:09 +02001181 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +02001182 * @param target Compound target (should target only one host)
1183 * @param file File path to read (/etc/hosts for example)
1184 */
1185
Pavlo Shchelokovskyy97100072018-10-04 11:53:44 +03001186def getFileContent(saltId, target, file, checkResponse = true, batch=null, output = true, saltArgs = []) {
1187 result = cmdRun(saltId, target, "cat ${file}", checkResponse, batch, output, saltArgs)
Tomáš Kukrálf1a692a2017-08-11 13:29:28 +02001188 return result['return'][0].values()[0].replaceAll('Salt command execution success','')
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +02001189}
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001190
1191/**
1192 * Set override parameters in Salt cluster metadata
1193 *
chnydaa0dbb252017-10-05 10:46:09 +02001194 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001195 * @param salt_overrides YAML formatted string containing key: value, one per line
Matthew Mosesohne5646842017-07-19 16:54:57 +03001196 * @param reclass_dir Directory where Reclass git repo is located
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +02001197 * @param extra_tgt Extra targets for compound
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001198 */
1199
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001200def setSaltOverrides(saltId, salt_overrides, reclass_dir="/srv/salt/reclass", extra_tgt = '') {
Tomáš Kukrálf178f052017-07-11 11:31:00 +02001201 def common = new com.mirantis.mk.Common()
Mykyta Karpin1c165e22017-08-22 18:27:01 +03001202 def salt_overrides_map = readYaml text: salt_overrides
Tomáš Kukrál243cf842017-07-11 13:11:56 +02001203 for (entry in common.entries(salt_overrides_map)) {
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001204 def key = entry[0]
1205 def value = entry[1]
1206
1207 common.debugMsg("Set salt override ${key}=${value}")
Victor Ryzhenkin2f997302019-01-10 15:12:55 +04001208 runSaltProcessStep(saltId, "I@salt:master ${extra_tgt}", 'reclass.cluster_meta_set', ["name=${key}", "value=${value}"], false)
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001209 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001210 runSaltProcessStep(saltId, "I@salt:master ${extra_tgt}", 'cmd.run', ["git -C ${reclass_dir} update-index --skip-worktree classes/cluster/overrides.yml"])
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001211}
Oleg Grigorovbec45582017-09-12 20:29:24 +03001212
1213/**
1214* Execute salt commands via salt-api with
1215* CLI client salt-pepper
1216*
1217* @param data Salt command map
1218* @param venv Path to virtualenv with
1219*/
1220
azvyagintsev386e94e2019-06-13 13:39:04 +03001221def runPepperCommand(data, venv) {
Jakub Josef03d4d5a2017-12-20 16:35:09 +01001222 def common = new com.mirantis.mk.Common()
Oleg Grigorovbec45582017-09-12 20:29:24 +03001223 def python = new com.mirantis.mk.Python()
1224 def dataStr = new groovy.json.JsonBuilder(data).toString()
azvyagintsev386e94e2019-06-13 13:39:04 +03001225 // TODO(alexz): parametrize?
1226 int retry = 10
chnyda4901a042017-11-16 12:14:56 +01001227
Jakub Josefa2491ad2018-01-15 16:26:27 +01001228 def pepperCmdFile = "${venv}/pepper-cmd.json"
1229 writeFile file: pepperCmdFile, text: dataStr
1230 def pepperCmd = "pepper -c ${venv}/pepperrc --make-token -x ${venv}/.peppercache --json-file ${pepperCmdFile}"
Oleg Grigorovbec45582017-09-12 20:29:24 +03001231
azvyagintsev386e94e2019-06-13 13:39:04 +03001232 int tries = 0
1233 def FullOutput = ['status': 1]
Jakub Josef37cd4972018-02-01 16:25:25 +01001234 def outputObj
azvyagintsev386e94e2019-06-13 13:39:04 +03001235 while (tries++ < retry) {
1236 try {
1237 if (venv) {
1238 FullOutput = python.runVirtualenvCommand(venv, pepperCmd, true, true)
1239 } else {
1240 FullOutput = common.shCmdStatus(pepperCmd)
1241 }
1242 if (FullOutput['status'] != 0) {
1243 error()
1244 }
1245 break
1246 } catch (e) {
1247 // Check , if we get failed pepper HTTP call, and retry
1248 common.errorMsg("Command: ${pepperCmd} failed to execute with error:\n${FullOutput['stderr']}")
1249 if (FullOutput['stderr'].contains('Error with request: HTTP Error 50') || FullOutput['stderr'].contains('Pepper error: Server error')) {
1250 common.errorMsg("Pepper HTTP Error detected. Most probably, " +
1251 "master SaltReqTimeoutError in master zmq thread issue...lets retry ${tries}/${retry}")
1252 sleep(5)
1253 continue
1254 }
1255 }
1256 }
1257 // Try to parse json output. No sense to check exit code, since we always expect json answer only.
Jakub Josef37cd4972018-02-01 16:25:25 +01001258 try {
azvyagintsev386e94e2019-06-13 13:39:04 +03001259 outputObj = new groovy.json.JsonSlurperClassic().parseText(FullOutput['stdout'])
1260 } catch (Exception jsonE) {
1261 common.errorMsg('Parsing Salt API JSON response failed! Response: ' + FullOutput)
1262 throw jsonE
Jakub Josef37cd4972018-02-01 16:25:25 +01001263 }
1264 return outputObj
Oleg Grigorovbec45582017-09-12 20:29:24 +03001265}
Martin Polreich232ad902019-01-21 14:31:00 +01001266
azvyagintsev386e94e2019-06-13 13:39:04 +03001267
Martin Polreich232ad902019-01-21 14:31:00 +01001268/**
1269* Check time settings on defined nodes, compares them
1270* and evaluates the results
1271*
1272* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1273* @param target Targeted nodes to be checked
1274* @param diff Maximum time difference (in seconds) to be accepted during time sync check
Martin Polreichf7976952019-03-04 10:06:11 +01001275* @param batch Batch param to salt (integer or string with percents)
Martin Polreich232ad902019-01-21 14:31:00 +01001276* @return bool Return true if time difference is <= diff and returns false if time difference is > diff
1277*/
1278
Martin Polreichf7976952019-03-04 10:06:11 +01001279def checkClusterTimeSync(saltId, target, batch = null) {
Martin Polreich232ad902019-01-21 14:31:00 +01001280 def common = new com.mirantis.mk.Common()
1281 def salt = new com.mirantis.mk.Salt()
1282
1283 times = []
1284 try {
1285 diff = salt.getReturnValues(salt.getPillar(saltId, 'I@salt:master', 'linux:system:time_diff'))
1286 if (diff != null && diff != "" && diff.isInteger()) {
1287 diff = diff.toInteger()
1288 } else {
1289 diff = 5
1290 }
Martin Polreichf7976952019-03-04 10:06:11 +01001291 out = salt.runSaltProcessStep(saltId, target, 'status.time', '%s', batch)
Martin Polreich232ad902019-01-21 14:31:00 +01001292 outParsed = out['return'][0]
1293 def outKeySet = outParsed.keySet()
1294 for (key in outKeySet) {
1295 def time = outParsed[key].readLines().get(0)
1296 common.infoMsg(time)
1297 if (time.isInteger()) {
1298 times.add(time.toInteger())
1299 }
1300 }
1301 if ((times.max() - times.min()) <= diff) {
1302 return true
1303 } else {
1304 return false
1305 }
1306 } catch(Exception e) {
1307 common.errorMsg("Could not check cluster time sync.")
1308 return false
1309 }
1310}
Martin Polreichb577f2d2019-02-27 09:28:35 +01001311
1312/**
1313* Finds out IP address of the given node or a list of nodes
1314*
1315* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1316* @param nodes Targeted node hostnames to be checked (String or List of strings)
1317* @param useGrains If the, the value will be taken from grains. If false, it will be taken from 'hostname' command.
1318* @return Map Return result Map in format ['nodeName1': 'ipAdress1', 'nodeName2': 'ipAdress2', ...]
1319*/
1320
1321def getIPAddressesForNodenames(saltId, nodes = [], useGrains = true) {
1322 result = [:]
1323
1324 if (nodes instanceof String) {
1325 nodes = [nodes]
1326 }
1327
1328 if (useGrains) {
1329 for (String node in nodes) {
1330 ip = getReturnValues(getGrain(saltId, node, "fqdn_ip4"))["fqdn_ip4"][0]
1331 result[node] = ip
1332 }
1333 } else {
1334 for (String node in nodes) {
1335 ip = getReturnValues(cmdRun(saltId, node, "hostname -i")).readLines()[0]
1336 result[node] = ip
1337 }
1338 }
1339 return result
Martin Polreich4512e2e2019-03-29 12:10:00 +01001340}
1341
1342/**
1343* Checks if required package is installed and returns averaged IO stats for selected disks.
1344* Allows getting averaged values of specific parameter for all disks or a specified disk.
1345* Interval between checks and its number is parametrized and configurable.
1346*
1347* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1348* @param target Node to be targeted (Should only match 1 node)
1349* @param parameterName Name of parameter from 'iostat' output (default = '' -- returns all variables)
1350* @param interval Interval between checks (default = 1)
1351* @param count Number of checks (default = 5)
1352* @param disks Disks to be checked (default = '' -- returns all disks)
1353* @param output Print Salt command return (default = true)
1354* @return Map Map containing desired values in format ['disk':'value']
1355*/
1356
1357def getIostatValues(Map params) {
1358 def common = new com.mirantis.mk.Common()
1359 def ret = [:]
Martin Polreicha7744202019-04-04 16:58:28 +02001360 if (isPackageInstalled(['saltId': params.saltId, 'target': params.target, 'packageName': 'sysstat', 'output': false])) {
Martin Polreich4512e2e2019-03-29 12:10:00 +01001361 def arg = [params.get('interval', 1), params.get('count', 5), params.get('disks', '')]
1362 def res = getReturnValues(runSaltProcessStep(params.saltId, params.target, 'disk.iostat', arg, null, params.output))
1363 if (res instanceof Map) {
1364 for (int i = 0; i < res.size(); i++) {
1365 def key = res.keySet()[i]
1366 if (params.containsKey('parameterName')) {
1367 if (res[key].containsKey(params.parameterName)){
1368 ret[key] = res[key][params.parameterName]
1369 } else {
1370 common.errorMsg("Parameter '${params.parameterName}' not found for disk '${key}'. Valid parameter for this disk are: '${res[key].keySet()}'")
1371 }
1372 } else {
1373 return res // If no parameterName is defined, return all of them.
1374 }
1375 }
1376 }
1377 } else {
1378 common.errorMsg("Package 'sysstat' seems not to be installed on at least one of tageted nodes: ${params.target}. Please fix this to be able to check 'iostat' values. Find more in the docs TODO:<Add docs link>")
1379 }
1380 return ret
1381}
1382
1383/**
1384* Checks if defined package is installed on all nodes defined by target parameter.
1385*
1386* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1387* @param target Node or nodes to be targeted
1388* @param packageName Name of package to be checked
1389* @param output Print Salt command return (default = true)
1390* @return boolean True if package is installed on all defined nodes. False if not found on at least one of defined nodes.
1391*/
1392
1393def isPackageInstalled(Map params) {
1394 def output = params.get('output', true)
Ivan Berezovskiy61a493c2019-08-06 18:50:39 +04001395 def res = runSaltProcessStep(params.saltId, params.target, "pkg.list_pkgs", [], null, output)['return'][0]
Ivan Berezovskiy775a5532019-07-26 16:44:22 +04001396 if (res) {
1397 for (int i = 0; i < res.size(); i++) {
1398 def key = res.keySet()[i]
Denis Egorenkofcc4aef2019-10-30 14:07:12 +04001399 if (!(res[key] instanceof Map && res[key].get(params.packageName.toString(), false))) {
Ivan Berezovskiy775a5532019-07-26 16:44:22 +04001400 return false
1401 }
Martin Polreich4512e2e2019-03-29 12:10:00 +01001402 }
Ivan Berezovskiy775a5532019-07-26 16:44:22 +04001403 return true
1404 } else {
1405 return false
Martin Polreich4512e2e2019-03-29 12:10:00 +01001406 }
azvyagintsev386e94e2019-06-13 13:39:04 +03001407}
Denis Egorenko3621b962019-09-23 16:13:43 +04001408
1409/**
1410* Returns nubmer of worker_threads set for Salt Master
1411*
1412* @param saltId Salt Connection object or pepperEnv
1413*
1414*/
1415def getWorkerThreads(saltId) {
1416 if (env.getEnvironment().containsKey('SALT_MASTER_OPT_WORKER_THREADS')) {
1417 return env['SALT_MASTER_OPT_WORKER_THREADS'].toString()
1418 }
1419 def threads = cmdRun(saltId, "I@salt:master", "cat /etc/salt/master.d/master.conf | grep worker_threads | cut -f 2 -d ':'", true, null, true)
1420 return threads['return'][0].values()[0].replaceAll('Salt command execution success','').trim()
1421}
1422