blob: 7347205c5aa2329466a9f9439960131b484e24d5 [file] [log] [blame]
Jakub Josef79ecec32017-02-17 14:36:28 +01001package com.mirantis.mk
2
Jakub Josefbceaa322017-06-13 18:28:27 +02003import com.cloudbees.groovy.cps.NonCPS
Jakub Josefb77c0812017-03-27 14:11:01 +02004import java.util.stream.Collectors
Jakub Josef79ecec32017-02-17 14:36:28 +01005/**
6 * Salt functions
7 *
8*/
9
10/**
11 * Salt connection and context parameters
12 *
13 * @param url Salt API server URL
14 * @param credentialsID ID of credentials store entry
15 */
16def connection(url, credentialsId = "salt") {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +010017 def common = new com.mirantis.mk.Common()
Jakub Josef79ecec32017-02-17 14:36:28 +010018 params = [
19 "url": url,
20 "credentialsId": credentialsId,
21 "authToken": null,
22 "creds": common.getCredentials(credentialsId)
23 ]
24 params["authToken"] = saltLogin(params)
Jakub Josef79ecec32017-02-17 14:36:28 +010025 return params
26}
27
28/**
29 * Login to Salt API, return auth token
30 *
31 * @param master Salt connection object
32 */
33def saltLogin(master) {
Tomáš Kukrál7bec0532017-02-20 15:39:31 +010034 def http = new com.mirantis.mk.Http()
Jakub Josef79ecec32017-02-17 14:36:28 +010035 data = [
36 'username': master.creds.username,
37 'password': master.creds.password.toString(),
38 'eauth': 'pam'
39 ]
Tomáš Kukrál7bec0532017-02-20 15:39:31 +010040 authToken = http.restGet(master, '/login', data)['return'][0]['token']
Jakub Josef79ecec32017-02-17 14:36:28 +010041 return authToken
42}
43
44/**
chnydaa0dbb252017-10-05 10:46:09 +020045 * Run action using Salt API (using plain HTTP request from Jenkins master) or Pepper (from slave shell)
Jakub Josef79ecec32017-02-17 14:36:28 +010046 *
chnydaa0dbb252017-10-05 10:46:09 +020047 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method) (determines if command will be sent with Pepper of Salt API )
Jakub Josef79ecec32017-02-17 14:36:28 +010048 * @param client Client type
49 * @param target Target specification, eg. for compound matches by Pillar
50 * data: ['expression': 'I@openssh:server', 'type': 'compound'])
51 * @param function Function to execute (eg. "state.sls")
Jakub Josef2f25cf22017-03-28 13:34:57 +020052 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef79ecec32017-02-17 14:36:28 +010053 * @param args Additional arguments to function
54 * @param kwargs Additional key-value arguments to function
Jiri Broulik48544be2017-06-14 18:33:54 +020055 * @param timeout Additional argument salt api timeout
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +030056 * @param read_timeout http session read timeout
Jakub Josef79ecec32017-02-17 14:36:28 +010057 */
58@NonCPS
chnydaa0dbb252017-10-05 10:46:09 +020059def runSaltCommand(saltId, client, target, function, batch = null, args = null, kwargs = null, timeout = -1, read_timeout = -1) {
Jakub Josef79ecec32017-02-17 14:36:28 +010060
61 data = [
62 'tgt': target.expression,
63 'fun': function,
64 'client': client,
65 'expr_form': target.type,
66 ]
Richard Felkld9476ac2018-07-12 19:01:33 +020067
68 if(batch != null){
69 batch = batch.toString()
70 if( (batch.isInteger() && batch.toInteger() > 0) || (batch.contains("%"))){
71 data['client']= "local_batch"
72 data['batch'] = batch
73 }
Jakub Josef79ecec32017-02-17 14:36:28 +010074 }
75
76 if (args) {
77 data['arg'] = args
78 }
79
80 if (kwargs) {
81 data['kwarg'] = kwargs
82 }
83
Jiri Broulik48544be2017-06-14 18:33:54 +020084 if (timeout != -1) {
85 data['timeout'] = timeout
86 }
87
chnydaa0dbb252017-10-05 10:46:09 +020088 // Command will be sent using HttpRequest
89 if (saltId instanceof HashMap && saltId.containsKey("authToken") ) {
Jakub Josef79ecec32017-02-17 14:36:28 +010090
chnydaa0dbb252017-10-05 10:46:09 +020091 def headers = [
92 'X-Auth-Token': "${saltId.authToken}"
93 ]
94
95 def http = new com.mirantis.mk.Http()
96 return http.sendHttpPostRequest("${saltId.url}/", data, headers, read_timeout)
97 } else if (saltId instanceof HashMap) {
98 throw new Exception("Invalid saltId")
99 }
100
101 // Command will be sent using Pepper
102 return runPepperCommand(data, saltId)
Jakub Josef79ecec32017-02-17 14:36:28 +0100103}
104
Jakub Josef5ade54c2017-03-10 16:14:01 +0100105/**
chnydaa0dbb252017-10-05 10:46:09 +0200106 * Return pillar for given saltId and target
107 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100108 * @param target Get pillar target
109 * @param pillar pillar name (optional)
110 * @return output of salt command
111 */
chnydaa0dbb252017-10-05 10:46:09 +0200112def getPillar(saltId, target, pillar = null) {
Tomáš Kukráld2589702017-03-10 16:30:46 +0100113 if (pillar != null) {
chnydaa0dbb252017-10-05 10:46:09 +0200114 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'pillar.get', null, [pillar.replace('.', ':')])
Tomáš Kukráld2589702017-03-10 16:30:46 +0100115 } else {
chnydaa0dbb252017-10-05 10:46:09 +0200116 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'pillar.data')
Ales Komareka3c7e502017-03-13 11:20:44 +0100117 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100118}
119
Jakub Josef5ade54c2017-03-10 16:14:01 +0100120/**
chnydaa0dbb252017-10-05 10:46:09 +0200121 * Return grain for given saltId and target
122 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100123 * @param target Get grain target
124 * @param grain grain name (optional)
125 * @return output of salt command
126 */
chnydaa0dbb252017-10-05 10:46:09 +0200127def getGrain(saltId, target, grain = null) {
Ales Komarekcec24d42017-03-08 10:25:45 +0100128 if(grain != null) {
chnydaa0dbb252017-10-05 10:46:09 +0200129 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'grains.item', null, [grain])
Jakub Josef5ade54c2017-03-10 16:14:01 +0100130 } else {
chnydaa0dbb252017-10-05 10:46:09 +0200131 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'grains.items')
Ales Komarekcec24d42017-03-08 10:25:45 +0100132 }
Ales Komarekcec24d42017-03-08 10:25:45 +0100133}
134
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +0300135/**
136 * Return config items for given saltId and target
137 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
138 * @param target Get grain target
139 * @param config grain name (optional)
140 * @return output of salt command
141 */
142def getConfig(saltId, target, config) {
143 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'config.get', null, [config.replace('.', ':')], '--out=json')
144}
Jakub Josef432e9d92018-02-06 18:28:37 +0100145
Jakub Josef5ade54c2017-03-10 16:14:01 +0100146/**
chnydaa0dbb252017-10-05 10:46:09 +0200147 * Enforces state on given saltId and target
148 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100149 * @param target State enforcing target
150 * @param state Salt state
Jakub Josef432e9d92018-02-06 18:28:37 +0100151 * @param excludedStates states which will be excluded from main state (default empty string)
152 * @param output print output (optional, default true)
153 * @param failOnError throw exception on salt state result:false (optional, default true)
154 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
155 * @param optional Optional flag (if true pipeline will continue even if no minions for target found)
156 * @param read_timeout http session read timeout (optional, default -1 - disabled)
157 * @param retries Retry count for salt state. (optional, default -1 - no retries)
158 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
159 * @param saltArgs additional salt args eq. ["runas=aptly"]
160 * @return output of salt command
161 */
Martin Polreichc19e66b2018-10-22 12:22:03 +0200162def enforceStateWithExclude(Map params) {
163 //Set defaults
164 defaults = ["excludedStates": "", "output": true, "failOnError": true, "batch": null, "optional": false,
Martin Polreichb1a369f2019-03-07 11:21:05 +0100165 "read_timeout": -1, "retries": -1, "retries_wait": 5, "queue": true, "saltArgs": []]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200166 params = defaults + params
167 params.saltArgs << "exclude=${params.excludedStates}"
168 params.remove('excludedStates')
169 return enforceState(params)
170}
171
172
Martin Polreichb1a369f2019-03-07 11:21:05 +0100173def enforceStateWithExclude(saltId, target, state, excludedStates = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[], retries_wait=5) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200174// Deprecated, convert state to use Map as input parameter
175 def common = new com.mirantis.mk.Common()
Martin Polreich49288692018-12-20 15:41:04 +0100176 common.infoMsg("This method will be deprecated. Convert you method call to use Map as input parameter")
Martin Polreichc19e66b2018-10-22 12:22:03 +0200177 // Convert to Map
178 params = ['saltId': saltId, 'target': target, 'state': state, 'excludedStates': excludedStates, 'output': output,
179 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
Martin Polreichb1a369f2019-03-07 11:21:05 +0100180 'retries': retries, 'retries_wait': retries_wait, 'queue': queue, 'saltArgs': saltArgs]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200181 // Call new method with Map as parameter
182 return enforceStateWithExclude(params)
Jakub Josef432e9d92018-02-06 18:28:37 +0100183}
184
Martin Polreich75e40642018-08-13 16:05:08 +0200185/**
186 * Allows to test the given target for reachability and if reachable enforces the state
187* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
188 * @param target State enforcing target
189 * @param state Salt state
190 * @param testTargetMatcher Salt compound matcher to be tested (default is empty string). If empty string, param `target` will be used for tests
191 * @param output print output (optional, default true)
192 * @param failOnError throw exception on salt state result:false (optional, default true)
193 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
194 * @param optional Optional flag (if true pipeline will continue even if no minions for target found)
195 * @param read_timeout http session read timeout (optional, default -1 - disabled)
196 * @param retries Retry count for salt state. (optional, default -1 - no retries)
197 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
198 * @param saltArgs additional salt args eq. ["runas=aptly"]
199 * @return output of salt command
200 */
Martin Polreichc19e66b2018-10-22 12:22:03 +0200201def enforceStateWithTest(Map params) {
Martin Polreich75e40642018-08-13 16:05:08 +0200202 def common = new com.mirantis.mk.Common()
Martin Polreichc19e66b2018-10-22 12:22:03 +0200203 //Set defaults
204 defaults = ["testTargetMatcher": "", "output": true, "failOnError": true, "batch": null, "optional": false,
Martin Polreichb1a369f2019-03-07 11:21:05 +0100205 "read_timeout": -1, "retries": -1, "retries_wait": 5, "queue": true, "saltArgs":[]]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200206 params = defaults + params
207 if (!params.testTargetMatcher) {
208 params.testTargetMatcher = params.target
Martin Polreich75e40642018-08-13 16:05:08 +0200209 }
Martin Polreichc19e66b2018-10-22 12:22:03 +0200210 if (testTarget(params.saltId, params.testTargetMatcher)) {
211 return enforceState(params)
Martin Polreich75e40642018-08-13 16:05:08 +0200212 } else {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200213 if (!params.optional) {
214 common.infoMsg("No Minions matched the target matcher: ${params.testTargetMatcher}, and 'optional' param was set to false. - This may signify missing pillar definition!!")
Martin Polreich92db3e12018-09-27 15:29:23 +0200215// throw new Exception("No Minions matched the target matcher: ${testTargetMatcher}.") TODO: Change the infoMsg to Error once the methods are changed to Use named params and optional param will be set globally
Martin Polreich75e40642018-08-13 16:05:08 +0200216 } else {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200217 common.infoMsg("No Minions matched the target matcher: ${params.testTargetMatcher}, but 'optional' param was set to true - Pipeline continues. ")
Martin Polreich75e40642018-08-13 16:05:08 +0200218 }
219 }
220}
221
Martin Polreichc19e66b2018-10-22 12:22:03 +0200222
Martin Polreichb1a369f2019-03-07 11:21:05 +0100223def enforceStateWithTest(saltId, target, state, testTargetMatcher = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[], retries_wait=5) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200224// Deprecated, convert state to use Map as input parameter
225 def common = new com.mirantis.mk.Common()
Martin Polreich49288692018-12-20 15:41:04 +0100226 common.infoMsg("This method will be deprecated. Convert you method call to use Map as input parameter")
Martin Polreichc19e66b2018-10-22 12:22:03 +0200227 // Convert to Map
228 params = ['saltId': saltId, 'target': target, 'state': state, 'testTargetMatcher': testTargetMatcher, 'output': output,
229 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
Martin Polreichb1a369f2019-03-07 11:21:05 +0100230 'retries': retries, 'retries_wait': retries_wait, 'queue': queue, 'saltArgs': saltArgs]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200231 // Call new method with Map as parameter
232 return enforceStateWithTest(params)
233}
234
Jakub Josef432e9d92018-02-06 18:28:37 +0100235/* Enforces state on given saltId and target
236 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
237 * @param target State enforcing target
238 * @param state Salt state
Jakub Josef5ade54c2017-03-10 16:14:01 +0100239 * @param output print output (optional, default true)
240 * @param failOnError throw exception on salt state result:false (optional, default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200241 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef432e9d92018-02-06 18:28:37 +0100242 * @param optional Optional flag (if true pipeline will continue even if no minions for target found)
Petr Michalecde0ff322017-10-04 09:32:14 +0200243 * @param read_timeout http session read timeout (optional, default -1 - disabled)
244 * @param retries Retry count for salt state. (optional, default -1 - no retries)
245 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
Jakub Josef432e9d92018-02-06 18:28:37 +0100246 * @param saltArgs additional salt args eq. ["runas=aptly", exclude="opencontrail.database"]
Vasyl Saienko6a396212018-06-08 09:20:08 +0300247 * @param minionRestartWaitTimeout specifies timeout that we should wait after minion restart.
Jakub Josef5ade54c2017-03-10 16:14:01 +0100248 * @return output of salt command
249 */
Martin Polreichc19e66b2018-10-22 12:22:03 +0200250def enforceState(Map params) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100251 def common = new com.mirantis.mk.Common()
Martin Polreichc19e66b2018-10-22 12:22:03 +0200252 //Set defaults
Martin Polreichb1a369f2019-03-07 11:21:05 +0100253 defaults = ["output": true, "failOnError": true, "batch": null, "optional": false, "read_timeout": -1,
254 "retries": -1, "retries_wait": 5, "queue": true, "saltArgs": [], "minionRestartWaitTimeout": 10]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200255 params = defaults + params
Jakub Josef432e9d92018-02-06 18:28:37 +0100256 // add state to salt args
Martin Polreichc19e66b2018-10-22 12:22:03 +0200257 if (params.state instanceof String) {
258 params.saltArgs << params.state
Jakub Josef79ecec32017-02-17 14:36:28 +0100259 } else {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200260 params.saltArgs << params.state.join(',')
Jakub Josef79ecec32017-02-17 14:36:28 +0100261 }
262
Martin Polreichc19e66b2018-10-22 12:22:03 +0200263 common.infoMsg("Running state ${params.state} on ${params.target}")
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300264 def out
Petr Michalecde0ff322017-10-04 09:32:14 +0200265 def kwargs = [:]
266
Martin Polreichc19e66b2018-10-22 12:22:03 +0200267 if (params.queue && params.batch == null) {
Petr Michalecde0ff322017-10-04 09:32:14 +0200268 kwargs["queue"] = true
269 }
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300270
Victor Ryzhenkin7e4d18f2019-01-09 17:46:17 +0400271 if (params.optional == false || testTarget(params.saltId, params.target)){
Martin Polreichc19e66b2018-10-22 12:22:03 +0200272 if (params.retries > 0){
Jakub Josef962ba912018-04-04 17:39:19 +0200273 def retriesCounter = 0
Martin Polreichc19e66b2018-10-22 12:22:03 +0200274 retry(params.retries){
Jakub Josef962ba912018-04-04 17:39:19 +0200275 retriesCounter++
Jakub Josef432e9d92018-02-06 18:28:37 +0100276 // we have to reverse order in saltArgs because salt state have to be first
Martin Polreichc19e66b2018-10-22 12:22:03 +0200277 out = runSaltCommand(params.saltId, 'local', ['expression': params.target, 'type': 'compound'], 'state.sls', params.batch, params.saltArgs.reverse(), kwargs, -1, params.read_timeout)
Jakub Josef432e9d92018-02-06 18:28:37 +0100278 // failOnError should be passed as true because we need to throw exception for retry block handler
Martin Polreichc19e66b2018-10-22 12:22:03 +0200279 checkResult(out, true, params.output, true, retriesCounter < params.retries) //disable ask on error for every interation except last one
Martin Polreichb1a369f2019-03-07 11:21:05 +0100280 sleep(params['retries_wait'])
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300281 }
Petr Michalecde0ff322017-10-04 09:32:14 +0200282 } else {
Jakub Josef432e9d92018-02-06 18:28:37 +0100283 // we have to reverse order in saltArgs because salt state have to be first
Martin Polreichc19e66b2018-10-22 12:22:03 +0200284 out = runSaltCommand(params.saltId, 'local', ['expression': params.target, 'type': 'compound'], 'state.sls', params.batch, params.saltArgs.reverse(), kwargs, -1, params.read_timeout)
285 checkResult(out, params.failOnError, params.output)
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300286 }
Martin Polreichc19e66b2018-10-22 12:22:03 +0200287 waitForMinion(out, params.minionRestartWaitTimeout)
Martin Polreich1c77afa2017-07-18 11:27:02 +0200288 return out
Martin Polreich1c77afa2017-07-18 11:27:02 +0200289 } else {
290 common.infoMsg("No Minions matched the target given, but 'optional' param was set to true - Pipeline continues. ")
291 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100292}
293
Martin Polreichb1a369f2019-03-07 11:21:05 +0100294def enforceState(saltId, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs = [], minionRestartWaitTimeout=10, retries_wait=5) {
Martin Polreichc19e66b2018-10-22 12:22:03 +0200295// Deprecated, convert state to use Map as input parameter
296 def common = new com.mirantis.mk.Common()
Martin Polreich49288692018-12-20 15:41:04 +0100297 common.infoMsg("This method will be deprecated. Convert you method call to use Map as input parameter")
Martin Polreichc19e66b2018-10-22 12:22:03 +0200298 // Convert to Map
Martin Polreichb1a369f2019-03-07 11:21:05 +0100299 params = ['saltId': saltId, 'target': target, 'state': state, 'output': output, 'failOnError': failOnError,
300 'batch': batch, 'optional': optional, 'read_timeout': read_timeout, 'retries': retries,
301 'retries_wait': retries_wait, 'queue': queue, 'saltArgs': saltArgs, 'minionRestartWaitTimeout': minionRestartWaitTimeout]
Martin Polreichc19e66b2018-10-22 12:22:03 +0200302 // Call new method with Map as parameter
303 return enforceState(params)
304}
305
Jakub Josef5ade54c2017-03-10 16:14:01 +0100306/**
307 * Run command on salt minion (salt cmd.run wrapper)
chnydaa0dbb252017-10-05 10:46:09 +0200308 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100309 * @param target Get pillar target
310 * @param cmd command
Jakub Josef053df392017-05-03 15:51:05 +0200311 * @param checkResponse test command success execution (default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200312 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200313 * @param output do you want to print output
chnyda205a92b2018-01-11 17:07:32 +0100314 * @param saltArgs additional salt args eq. ["runas=aptly"]
Jakub Josef5ade54c2017-03-10 16:14:01 +0100315 * @return output of salt command
316 */
chnyda205a92b2018-01-11 17:07:32 +0100317def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = []) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100318 def common = new com.mirantis.mk.Common()
Jakub Josef053df392017-05-03 15:51:05 +0200319 def originalCmd = cmd
Tomáš Kukráldfd4b492017-03-02 12:08:50 +0100320 common.infoMsg("Running command ${cmd} on ${target}")
Jakub Josef053df392017-05-03 15:51:05 +0200321 if (checkResponse) {
322 cmd = cmd + " && echo Salt command execution success"
323 }
chnyda205a92b2018-01-11 17:07:32 +0100324
Jakub Josef432e9d92018-02-06 18:28:37 +0100325 // add cmd name to salt args list
chnyda205a92b2018-01-11 17:07:32 +0100326 saltArgs << cmd
327
328 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.run', batch, saltArgs.reverse())
Jakub Josef053df392017-05-03 15:51:05 +0200329 if (checkResponse) {
330 // iterate over all affected nodes and check success return code
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200331 if (out["return"]){
332 for(int i=0;i<out["return"].size();i++){
333 def node = out["return"][i];
Jakub Josef053df392017-05-03 15:51:05 +0200334 for(int j=0;j<node.size();j++){
335 def nodeKey = node.keySet()[j]
Martin Polreicha2effb82018-08-01 11:35:11 +0200336 if (node[nodeKey] instanceof String) {
337 if (!node[nodeKey].contains("Salt command execution success")) {
338 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
339 }
340 } else if (node[nodeKey] instanceof Boolean) {
341 if (!node[nodeKey]) {
342 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
343 }
344 } else {
345 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
Jakub Josef053df392017-05-03 15:51:05 +0200346 }
347 }
348 }
Martin Polreicha2effb82018-08-01 11:35:11 +0200349 } else {
Jakub Josef053df392017-05-03 15:51:05 +0200350 throw new Exception("Salt Api response doesn't have return param!")
351 }
352 }
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200353 if (output == true) {
354 printSaltCommandResult(out)
355 }
356 return out
Jakub Josef79ecec32017-02-17 14:36:28 +0100357}
358
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200359/**
Jiri Broulik71512bc2017-08-04 10:00:18 +0200360 * Checks if salt minion is in a list of salt master's accepted keys
chnydaa0dbb252017-10-05 10:46:09 +0200361 * @usage minionPresent(saltId, 'I@salt:master', 'ntw', true, null, true, 200, 3)
362 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200363 * @param target Get pillar target
364 * @param minion_name unique identification of a minion in salt-key command output
365 * @param waitUntilPresent return after the minion becomes present (default true)
366 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
367 * @param output print salt command (default true)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200368 * @param maxRetries finite number of iterations to check status of a command (default 200)
369 * @param answers how many minions should return (optional, default 1)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200370 * @return output of salt command
371 */
lmercl94189272018-06-01 11:03:46 +0200372def minionPresent(saltId, target, minion_name, waitUntilPresent = true, batch=null, output = true, maxRetries = 180, answers = 1) {
Jiri Broulik71512bc2017-08-04 10:00:18 +0200373 minion_name = minion_name.replace("*", "")
374 def common = new com.mirantis.mk.Common()
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200375 common.infoMsg("Looking for minion: " + minion_name)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200376 def cmd = 'salt-key | grep ' + minion_name
377 if (waitUntilPresent){
378 def count = 0
379 while(count < maxRetries) {
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200380 try {
381 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
382 if (output) {
383 printSaltCommandResult(out)
384 }
385 def valueMap = out["return"][0]
386 def result = valueMap.get(valueMap.keySet()[0])
387 def resultsArray = result.tokenize("\n")
388 def size = resultsArray.size()
389 if (size >= answers) {
390 return out
391 }
392 count++
393 sleep(time: 1000, unit: 'MILLISECONDS')
394 common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
395 } catch (Exception er) {
396 common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
397 }
398 }
399 } else {
400 try {
chnydaa0dbb252017-10-05 10:46:09 +0200401 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200402 if (output) {
403 printSaltCommandResult(out)
404 }
Dzmitry Stremkouski5425d232018-06-07 00:46:00 +0200405 return out
406 } catch (Exception er) {
407 common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
Jiri Broulik71512bc2017-08-04 10:00:18 +0200408 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200409 }
410 // otherwise throw exception
411 common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
412 throw new Exception("${cmd} signals failure of status check!")
413}
414
415/**
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200416 * Checks if salt minions are in a list of salt master's accepted keys by matching compound
417 * @usage minionsPresent(saltId, 'I@salt:master', 'I@salt:minion', true, null, true, 200, 3)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100418 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
419 * @param target Performs tests on this target node
420 * @param target_minions all targeted minions to test (for ex. I@salt:minion)
421 * @param waitUntilPresent return after the minion becomes present (default true)
422 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
423 * @param output print salt command (default true)
424 * @param maxRetries finite number of iterations to check status of a command (default 200)
425 * @param answers how many minions should return (optional, default 1)
426 * @return output of salt command
427 */
428def minionsPresent(saltId, target = 'I@salt:master', target_minions = '', waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200429 def target_hosts = getMinionsSorted(saltId, target_minions)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100430 for (t in target_hosts) {
Dzmitry Stremkouski8a8d56b2018-05-01 12:20:04 +0200431 def tgt = stripDomainName(t)
432 minionPresent(saltId, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
433 }
434}
435
436/**
437 * Checks if salt minions are in a list of salt master's accepted keys by matching a list
438 * @usage minionsPresentFromList(saltId, 'I@salt:master', ["cfg01.example.com", "bmk01.example.com"], true, null, true, 200, 3)
439 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
440 * @param target Performs tests on this target node
441 * @param target_minions list to test (for ex. ["cfg01.example.com", "bmk01.example.com"])
442 * @param waitUntilPresent return after the minion becomes present (default true)
443 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
444 * @param output print salt command (default true)
445 * @param maxRetries finite number of iterations to check status of a command (default 200)
446 * @param answers how many minions should return (optional, default 1)
447 * @return output of salt command
448 */
449def minionsPresentFromList(saltId, target = 'I@salt:master', target_minions = [], waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
450 def common = new com.mirantis.mk.Common()
451 for (tgt in target_minions) {
452 common.infoMsg("Checking if minion " + tgt + " is present")
453 minionPresent(saltId, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
Jiri Broulikf8f96942018-02-15 10:03:42 +0100454 }
455}
456
457/**
Jiri Broulik71512bc2017-08-04 10:00:18 +0200458 * You can call this function when salt-master already contains salt keys of the target_nodes
chnydaa0dbb252017-10-05 10:46:09 +0200459 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200460 * @param target Should always be salt-master
Martin Polreich0c1e2782019-07-01 17:21:12 +0200461 * @param targetNodes unique identification of a minion or group of salt minions
Jiri Broulik71512bc2017-08-04 10:00:18 +0200462 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Martin Polreich0c1e2782019-07-01 17:21:12 +0200463 * @param cmdTimeout timeout for the salt command if minions do not return (default 10)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200464 * @param maxRetries finite number of iterations to check status of a command (default 200)
465 * @return output of salt command
466 */
Martin Polreich0c1e2782019-07-01 17:21:12 +0200467
468def minionsReachable(saltId, target, targetNodes, batch=null, cmdTimeout = 10, maxRetries = 200) {
Jiri Broulik71512bc2017-08-04 10:00:18 +0200469 def common = new com.mirantis.mk.Common()
Martin Polreich0c1e2782019-07-01 17:21:12 +0200470 def cmd = "salt -t${cmdTimeout} -C '${targetNodes}' test.ping"
471 common.infoMsg("Checking if all ${targetNodes} minions are reachable")
472 def retriesCount = 0
473 while(retriesCount < maxRetries) {
Jiri Broulik71512bc2017-08-04 10:00:18 +0200474 Calendar timeout = Calendar.getInstance();
Martin Polreich0c1e2782019-07-01 17:21:12 +0200475 timeout.add(Calendar.SECOND, cmdTimeout);
476 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, cmdTimeout)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200477 Calendar current = Calendar.getInstance();
478 if (current.getTime().before(timeout.getTime())) {
Martin Polreich0c1e2782019-07-01 17:21:12 +0200479 common.infoMsg("Successful response received from all targeted nodes.")
480 printSaltCommandResult(out)
481 return out
Jiri Broulik71512bc2017-08-04 10:00:18 +0200482 }
Martin Polreich0c1e2782019-07-01 17:21:12 +0200483 def outYaml = readYaml text: getReturnValues(out)
484 def successfulNodes = []
485 def failedNodes = []
486 for (node in outYaml.keySet()) {
487 if (outYaml[node] == true || outYaml[node].toString().toLowerCase() == 'true') {
488 successfulNodes.add(node)
489 } else {
490 failedNodes.add(node)
491 }
492 }
493 common.infoMsg("Not all of the targeted minions returned yet. Successful response from ${successfulNodes}. Still waiting for ${failedNodes}.")
494 retriesCount++
Jiri Broulik71512bc2017-08-04 10:00:18 +0200495 sleep(time: 500, unit: 'MILLISECONDS')
496 }
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200497}
498
Martin Polreich0c1e2782019-07-01 17:21:12 +0200499
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200500/**
Denis Egorenkod20a4182019-02-27 19:19:12 +0400501 * You can call this function when need to check that all minions are available, free and ready for command execution
502 * @param config LinkedHashMap config parameter, which contains next:
503 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
504 * @param target unique identification of a minion or group of salt minions
505 * @param target_reachable unique identification of a minion or group of salt minions to check availability
506 * @param wait timeout between retries to check target minions (default 5)
507 * @param retries finite number of iterations to check minions (default 10)
508 * @param timeout timeout for the salt command if minions do not return (default 5)
509 * @param availability check that minions also are available before checking readiness (default true)
510 */
511def checkTargetMinionsReady(LinkedHashMap config) {
512 def common = new com.mirantis.mk.Common()
513 def saltId = config.get('saltId')
514 def target = config.get('target')
515 def target_reachable = config.get('target_reachable', target)
516 def wait = config.get('wait', 30)
517 def retries = config.get('retries', 10)
518 def timeout = config.get('timeout', 5)
519 def checkAvailability = config.get('availability', true)
520 common.retry(retries, wait) {
521 if (checkAvailability) {
522 minionsReachable(saltId, 'I@salt:master', target_reachable)
523 }
524 def running = runSaltProcessStep(saltId, target, 'saltutil.running', [], null, true, timeout)
525 for (value in running.get("return")[0].values()) {
526 if (value != []) {
527 throw new Exception("Not all salt-minions are ready for execution")
528 }
529 }
530 }
531}
532
533/**
Vasyl Saienkod8dd2c92019-06-24 23:39:32 +0300534 * Restart and wait for salt-minions on target nodes.
535 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
536 * @param target unique identification of a minion or group of salt minions
537 * @param wait timeout for the salt command if minions do not return (default 5)
538 * @param maxRetries finite number of iterations to check status of a command (default 10)
539 * @return output of salt command
540 */
541def restartSaltMinion(saltId, target, wait = 5, maxRetries = 10) {
542 def common = new com.mirantis.mk.Common()
543 common.infoMsg("Restarting salt-minion on ${target} and waiting for they are reachable.")
544 runSaltProcessStep(saltId, target, 'cmd.shell', ['salt-call service.restart salt-minion'], null, true, 60)
545 checkTargetMinionsReady(['saltId': saltId, 'target_reachable': target, timeout: wait, retries: maxRetries])
546 common.infoMsg("All ${target} minions are alive...")
547}
548
549/**
550 * Upgrade package and restart salt minion.
551 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
552 * @param target unique identification of a minion or group of salt minions
553 * @param the name of pkg_name to upgrade
554 * @param wait timeout for the salt command if minions do not return (default 5)
555 * @param maxRetries finite number of iterations to check status of a command (default 10)
556 * @return output of salt command
557 */
558def upgradePackageAndRestartSaltMinion(saltId, target, pkg_name, wait = 5, maxRetries = 10) {
559 def common = new com.mirantis.mk.Common()
560 def latest_version = getReturnValues(runSaltProcessStep(saltId, target, 'pkg.latest_version', [pkg_name, 'show_installed=True'])).split('\n')[0]
561 def current_version = getReturnValues(runSaltProcessStep(saltId, target, 'pkg.version', [pkg_name])).split('\n')[0]
562 if (current_version && latest_version != current_version) {
563 common.infoMsg("Upgrading current ${pkg_name}: ${current_version} to ${latest_version}")
564 runSaltProcessStep(saltId, target, 'pkg.install', [pkg_name], 'only_upgrade=True')
565 restartSaltMinion(saltId, target, wait, maxRetries)
566 }
567}
568
569/**
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200570 * Run command on salt minion (salt cmd.run wrapper)
chnydaa0dbb252017-10-05 10:46:09 +0200571 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200572 * @param target Get pillar target
573 * @param cmd name of a service
574 * @param correct_state string that command must contain if status is in correct state (optional, default 'running')
Jiri Broulikcf1f2332017-07-25 11:30:03 +0200575 * @param find bool value if it is suppose to find some string in the output or the cmd should return empty string (optional, default true)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200576 * @param waitUntilOk return after the minion becomes present (optional, default true)
577 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
578 * @param output print salt command (default true)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200579 * @param maxRetries finite number of iterations to check status of a command (default 200)
580 * @param answers how many minions should return (optional, default 0)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200581 * @return output of salt command
582 */
chnydaa0dbb252017-10-05 10:46:09 +0200583def commandStatus(saltId, target, cmd, correct_state='running', find = true, waitUntilOk = true, batch=null, output = true, maxRetries = 200, answers = 0) {
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200584 def common = new com.mirantis.mk.Common()
585 common.infoMsg("Checking if status of verification command ${cmd} on ${target} is in correct state")
586 if (waitUntilOk){
587 def count = 0
588 while(count < maxRetries) {
chnydaa0dbb252017-10-05 10:46:09 +0200589 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200590 if (output) {
591 printSaltCommandResult(out)
592 }
Jakub Josef115a78f2017-07-18 15:04:00 +0200593 def resultMap = out["return"][0]
Jiri Broulik71512bc2017-08-04 10:00:18 +0200594 def success = 0
595 if (answers == 0){
596 answers = resultMap.size()
597 }
598 for (int i=0;i<answers;i++) {
599 result = resultMap.get(resultMap.keySet()[i])
600 // if the goal is to find some string in output of the command
601 if (find) {
602 if(result == null || result instanceof Boolean || result.isEmpty()) { result='' }
603 if (result.toLowerCase().contains(correct_state.toLowerCase())) {
604 success++
605 if (success == answers) {
606 return out
607 }
Jiri Broulikd0c27572017-07-24 20:01:10 +0200608 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200609 // else the goal is to not find any string in output of the command
610 } else {
611 if(result instanceof String && result.isEmpty()) {
612 success++
613 if (success == answers) {
614 return out
chnydaa0dbb252017-10-05 10:46:09 +0200615 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200616 }
617 }
618 }
619 count++
620 sleep(time: 500, unit: 'MILLISECONDS')
621 common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
622 }
623 } else {
chnydaa0dbb252017-10-05 10:46:09 +0200624 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200625 def resultMap = out["return"][0]
626 if (output) {
627 printSaltCommandResult(out)
628 }
629 for (int i=0;i<resultMap.size();i++) {
630 result = resultMap.get(resultMap.keySet()[i])
631 // if the goal is to find some string in output of the command
632 if (find) {
633 if(result == null || result instanceof Boolean || result.isEmpty()) { result='' }
634 if (result.toLowerCase().contains(correct_state.toLowerCase())) {
Jiri Broulikd0c27572017-07-24 20:01:10 +0200635 return out
636 }
637
638 // else the goal is to not find any string in output of the command
639 } else {
640 if(result instanceof String && result.isEmpty()) {
641 return out
642 }
643 }
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200644 }
645 }
646 // otherwise throw exception
Jiri Broulikd0c27572017-07-24 20:01:10 +0200647 common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200648 throw new Exception("${cmd} signals failure of status check!")
649}
650
Jakub Josef5ade54c2017-03-10 16:14:01 +0100651/**
652 * Perform complete salt sync between master and target
chnydaa0dbb252017-10-05 10:46:09 +0200653 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100654 * @param target Get pillar target
655 * @return output of salt command
656 */
chnydaa0dbb252017-10-05 10:46:09 +0200657def syncAll(saltId, target) {
658 return runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'saltutil.sync_all')
Jakub Josef79ecec32017-02-17 14:36:28 +0100659}
660
Jakub Josef5ade54c2017-03-10 16:14:01 +0100661/**
Jakub Josef432e9d92018-02-06 18:28:37 +0100662 * Perform complete salt refresh between master and target
663 * Method will call saltutil.refresh_pillar, saltutil.refresh_grains and saltutil.sync_all
664 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
665 * @param target Get pillar target
666 * @return output of salt command
667 */
668def fullRefresh(saltId, target){
669 runSaltProcessStep(saltId, target, 'saltutil.refresh_pillar', [], null, true)
670 runSaltProcessStep(saltId, target, 'saltutil.refresh_grains', [], null, true)
671 runSaltProcessStep(saltId, target, 'saltutil.sync_all', [], null, true)
672}
673
674/**
675 * Enforce highstate on given targets
676 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
677 * @param target Highstate enforcing target
678 * @param excludedStates states which will be excluded from main state (default empty string)
679 * @param output print output (optional, default true)
680 * @param failOnError throw exception on salt state result:false (optional, default true)
681 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
682 * @param saltArgs additional salt args eq. ["runas=aptly", exclude="opencontrail.database"]
683 * @return output of salt command
684 */
685def enforceHighstateWithExclude(saltId, target, excludedStates = "", output = false, failOnError = true, batch = null, saltArgs = []) {
686 saltArgs << "exclude=${excludedStates}"
687 return enforceHighstate(saltId, target, output, failOnError, batch, saltArgs)
688}
689/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100690 * Enforce highstate on given targets
chnydaa0dbb252017-10-05 10:46:09 +0200691 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100692 * @param target Highstate enforcing target
693 * @param output print output (optional, default true)
694 * @param failOnError throw exception on salt state result:false (optional, default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200695 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100696 * @return output of salt command
697 */
Jakub Josef432e9d92018-02-06 18:28:37 +0100698def enforceHighstate(saltId, target, output = false, failOnError = true, batch = null, saltArgs = []) {
Petr Jediný30be7032018-05-29 18:22:46 +0200699 def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'state.highstate', batch, saltArgs)
Alexander Noskov657ccfc2017-07-14 11:35:52 +0000700 def common = new com.mirantis.mk.Common()
701
Marek Celoud63366112017-07-25 17:27:24 +0200702 common.infoMsg("Running state highstate on ${target}")
Alexander Noskov657ccfc2017-07-14 11:35:52 +0000703
Jakub Josef374beb72017-04-27 15:45:09 +0200704 checkResult(out, failOnError, output)
Jakub Josef79ecec32017-02-17 14:36:28 +0100705 return out
706}
707
Jakub Josef5ade54c2017-03-10 16:14:01 +0100708/**
Ales Komarek5276ebe2017-03-16 08:46:34 +0100709 * Get running minions IDs according to the target
chnydaa0dbb252017-10-05 10:46:09 +0200710 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Ales Komarek5276ebe2017-03-16 08:46:34 +0100711 * @param target Get minions target
712 * @return list of active minions fitin
713 */
chnydaa0dbb252017-10-05 10:46:09 +0200714def getMinions(saltId, target) {
715 def minionsRaw = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'test.ping')
Ales Komarek5276ebe2017-03-16 08:46:34 +0100716 return new ArrayList<String>(minionsRaw['return'][0].keySet())
717}
718
Jiri Broulikf8f96942018-02-15 10:03:42 +0100719/**
720 * Get sorted running minions IDs according to the target
721 * @param saltId Salt Connection object or pepperEnv
722 * @param target Get minions target
723 * @return list of sorted active minions fitin
724 */
725def getMinionsSorted(saltId, target) {
726 return getMinions(saltId, target).sort()
727}
728
729/**
730 * Get first out of running minions IDs according to the target
731 * @param saltId Salt Connection object or pepperEnv
732 * @param target Get minions target
733 * @return first of active minions fitin
734 */
735def getFirstMinion(saltId, target) {
736 def minionsSorted = getMinionsSorted(saltId, target)
Dmitry Ukovd72cd2a2018-09-04 17:31:46 +0400737 return minionsSorted[0]
Jiri Broulikf8f96942018-02-15 10:03:42 +0100738}
739
740/**
741 * Get running salt minions IDs without it's domain name part and its numbering identifications
742 * @param saltId Salt Connection object or pepperEnv
743 * @param target Get minions target
744 * @return list of active minions fitin without it's domain name part name numbering
745 */
746def getMinionsGeneralName(saltId, target) {
747 def minionsSorted = getMinionsSorted(saltId, target)
748 return stripDomainName(minionsSorted[0]).replaceAll('\\d+$', "")
749}
750
751/**
752 * Get domain name of the env
753 * @param saltId Salt Connection object or pepperEnv
754 * @return domain name
755 */
756def getDomainName(saltId) {
757 return getReturnValues(getPillar(saltId, 'I@salt:master', '_param:cluster_domain'))
758}
759
760/**
761 * Remove domain name from Salt minion ID
762 * @param name String of Salt minion ID
763 * @return Salt minion ID without its domain name
764 */
765def stripDomainName(name) {
766 return name.split("\\.")[0]
767}
768
769/**
770 * Gets return values of a salt command
771 * @param output String of Salt minion ID
772 * @return Return values of a salt command
773 */
774def getReturnValues(output) {
775 if(output.containsKey("return") && !output.get("return").isEmpty()) {
776 return output['return'][0].values()[0]
777 }
778 def common = new com.mirantis.mk.Common()
779 common.errorMsg('output does not contain return key')
780 return ''
781}
782
783/**
784 * Get minion ID of one of KVM nodes
785 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
786 * @return Salt minion ID of one of KVM nodes in env
787 */
788def getKvmMinionId(saltId) {
789 return getReturnValues(getGrain(saltId, 'I@salt:control', 'id')).values()[0]
790}
791
792/**
793 * Get Salt minion ID of KVM node hosting 'name' VM
794 * @param saltId Salt Connection object or pepperEnv
795 * @param name Name of the VM (for ex. ctl01)
796 * @return Salt minion ID of KVM node hosting 'name' VM
797 */
Jiri Broulikd2a50552018-04-25 17:17:59 +0200798def getNodeProvider(saltId, nodeName) {
799 def salt = new com.mirantis.mk.Salt()
800 def common = new com.mirantis.mk.Common()
801 def kvms = salt.getMinions(saltId, 'I@salt:control')
802 for (kvm in kvms) {
803 try {
804 vms = salt.getReturnValues(salt.runSaltProcessStep(saltId, kvm, 'virt.list_domains', [], null, true))
805 if (vms.toString().contains(nodeName)) {
806 return kvm
807 }
808 } catch (Exception er) {
809 common.infoMsg("${nodeName} not present on ${kvm}")
810 }
811 }
Jiri Broulikf8f96942018-02-15 10:03:42 +0100812}
813
Ales Komarek5276ebe2017-03-16 08:46:34 +0100814/**
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200815 * Test if there are any minions to target
chnydaa0dbb252017-10-05 10:46:09 +0200816 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200817 * @param target Target to test
vrovachev1c4770b2017-07-05 13:25:21 +0400818 * @return bool indicating if target was succesful
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200819 */
820
chnydaa0dbb252017-10-05 10:46:09 +0200821def testTarget(saltId, target) {
822 return getMinions(saltId, target).size() > 0
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200823}
824
825/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100826 * Generates node key using key.gen_accept call
chnydaa0dbb252017-10-05 10:46:09 +0200827 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100828 * @param target Key generating target
829 * @param host Key generating host
830 * @param keysize generated key size (optional, default 4096)
831 * @return output of salt command
832 */
chnydaa0dbb252017-10-05 10:46:09 +0200833def generateNodeKey(saltId, target, host, keysize = 4096) {
834 return runSaltCommand(saltId, 'wheel', target, 'key.gen_accept', [host], ['keysize': keysize])
Jakub Josef79ecec32017-02-17 14:36:28 +0100835}
836
Jakub Josef5ade54c2017-03-10 16:14:01 +0100837/**
Jakub Josef2f25cf22017-03-28 13:34:57 +0200838 * Generates node reclass metadata
chnydaa0dbb252017-10-05 10:46:09 +0200839 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100840 * @param target Metadata generating target
841 * @param host Metadata generating host
842 * @param classes Reclass classes
843 * @param parameters Reclass parameters
844 * @return output of salt command
845 */
chnydaa0dbb252017-10-05 10:46:09 +0200846def generateNodeMetadata(saltId, target, host, classes, parameters) {
847 return runSaltCommand(saltId, 'local', target, 'reclass.node_create', [host, '_generated'], ['classes': classes, 'parameters': parameters])
Jakub Josef79ecec32017-02-17 14:36:28 +0100848}
849
Jakub Josef5ade54c2017-03-10 16:14:01 +0100850/**
851 * Run salt orchestrate on given targets
chnydaa0dbb252017-10-05 10:46:09 +0200852 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100853 * @param target Orchestration target
854 * @param orchestrate Salt orchestrate params
Dzmitry Stremkouskidd020d92018-07-22 12:01:07 +0200855 * @param kwargs Salt orchestrate params
Jakub Josef5ade54c2017-03-10 16:14:01 +0100856 * @return output of salt command
857 */
Dzmitry Stremkouskidd020d92018-07-22 12:01:07 +0200858def orchestrateSystem(saltId, target, orchestrate=[], kwargs = null) {
Oleksii Grudev9e1d97a2018-06-29 16:04:30 +0300859 //Since the runSaltCommand uses "arg" (singular) for "runner" client this won`t work correctly on old salt 2016
860 //cause this version of salt used "args" (plural) for "runner" client, see following link for reference:
861 //https://github.com/saltstack/salt/pull/32938
Oleksii Grudev0c098382018-08-08 16:16:38 +0300862 def common = new com.mirantis.mk.Common()
863 def result = runSaltCommand(saltId, 'runner', target, 'state.orchestrate', true, orchestrate, kwargs, 7200, 7200)
864 if(result != null){
865 if(result['return']){
866 def retcode = result['return'][0].get('retcode')
867 if (retcode != 0) {
868 throw new Exception("Orchestration state failed while running: "+orchestrate)
869 }else{
870 common.infoMsg("Orchestrate state "+orchestrate+" succeeded")
871 }
872 }else{
873 common.errorMsg("Salt result has no return attribute! Result: ${result}")
874 }
875 }else{
876 common.errorMsg("Cannot check salt result, given result is null")
877 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100878}
879
Jakub Josef5ade54c2017-03-10 16:14:01 +0100880/**
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +0200881 * Run salt pre or post orchestrate tasks
882 *
883 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
884 * @param pillar_tree Reclass pillar that has orchestrate pillar for desired stage
885 * @param extra_tgt Extra targets for compound
886 *
887 * @return output of salt command
888 */
889def orchestratePrePost(saltId, pillar_tree, extra_tgt = '') {
890
891 def common = new com.mirantis.mk.Common()
892 def salt = new com.mirantis.mk.Salt()
893 def compound = 'I@' + pillar_tree + " " + extra_tgt
894
895 common.infoMsg("Refreshing pillars")
896 runSaltProcessStep(saltId, '*', 'saltutil.refresh_pillar', [], null, true)
897
898 common.infoMsg("Looking for orchestrate pillars")
899 if (salt.testTarget(saltId, compound)) {
900 for ( node in salt.getMinionsSorted(saltId, compound) ) {
901 def pillar = salt.getPillar(saltId, node, pillar_tree)
902 if ( !pillar['return'].isEmpty() ) {
903 for ( orch_id in pillar['return'][0].values() ) {
904 def orchestrator = orch_id.values()['orchestrator']
905 def orch_enabled = orch_id.values()['enabled']
906 if ( orch_enabled ) {
907 common.infoMsg("Orchestrating: ${orchestrator}")
908 salt.printSaltCommandResult(salt.orchestrateSystem(saltId, ['expression': node], [orchestrator]))
909 }
910 }
911 }
912 }
913 }
914}
915
916/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100917 * Run salt process step
chnydaa0dbb252017-10-05 10:46:09 +0200918 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100919 * @param tgt Salt process step target
920 * @param fun Salt process step function
921 * @param arg process step arguments (optional, default [])
Jakub Josef2f25cf22017-03-28 13:34:57 +0200922 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef432e9d92018-02-06 18:28:37 +0100923 * @param output print output (optional, default true)
Jiri Broulik48544be2017-06-14 18:33:54 +0200924 * @param timeout Additional argument salt api timeout
Jakub Josef5ade54c2017-03-10 16:14:01 +0100925 * @return output of salt command
926 */
Jakub Josef432e9d92018-02-06 18:28:37 +0100927def runSaltProcessStep(saltId, tgt, fun, arg = [], batch = null, output = true, timeout = -1, kwargs = null) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100928 def common = new com.mirantis.mk.Common()
Jiri Broulik48544be2017-06-14 18:33:54 +0200929 def salt = new com.mirantis.mk.Salt()
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100930 def out
931
Marek Celoud63366112017-07-25 17:27:24 +0200932 common.infoMsg("Running step ${fun} ${arg} on ${tgt}")
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100933
Filip Pytlounf0435c02017-03-02 17:48:54 +0100934 if (batch == true) {
chnydaa0dbb252017-10-05 10:46:09 +0200935 out = runSaltCommand(saltId, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg, kwargs, timeout)
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100936 } else {
chnydaa0dbb252017-10-05 10:46:09 +0200937 out = runSaltCommand(saltId, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg, kwargs, timeout)
Jakub Josef79ecec32017-02-17 14:36:28 +0100938 }
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100939
Tomáš Kukrálf5dda642017-03-02 14:22:59 +0100940 if (output == true) {
Jiri Broulik48544be2017-06-14 18:33:54 +0200941 salt.printSaltCommandResult(out)
Jakub Josef79ecec32017-02-17 14:36:28 +0100942 }
Jiri Broulikae19c262017-05-16 19:06:52 +0200943 return out
Jakub Josef79ecec32017-02-17 14:36:28 +0100944}
945
946/**
947 * Check result for errors and throw exception if any found
948 *
949 * @param result Parsed response of Salt API
Jakub Josef8021c002017-03-27 15:41:28 +0200950 * @param failOnError Do you want to throw exception if salt-call fails (optional, default true)
Jakub Josefa87941c2017-04-20 17:14:58 +0200951 * @param printResults Do you want to print salt results (optional, default true)
Jakub Josefa87941c2017-04-20 17:14:58 +0200952 * @param printOnlyChanges If true (default), print only changed resources
Jakub Josef432e9d92018-02-06 18:28:37 +0100953 * @param disableAskOnError Flag for disabling ASK_ON_ERROR feature (optional, default false)
Jakub Josef79ecec32017-02-17 14:36:28 +0100954 */
Jakub Josef432e9d92018-02-06 18:28:37 +0100955def checkResult(result, failOnError = true, printResults = true, printOnlyChanges = true, disableAskOnError = false) {
Jakub Josef5ade54c2017-03-10 16:14:01 +0100956 def common = new com.mirantis.mk.Common()
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100957 if(result != null){
958 if(result['return']){
959 for (int i=0;i<result['return'].size();i++) {
960 def entry = result['return'][i]
961 if (!entry) {
962 if (failOnError) {
963 throw new Exception("Salt API returned empty response: ${result}")
964 } else {
965 common.errorMsg("Salt API returned empty response: ${result}")
Jakub Josefece32af2017-03-14 19:20:08 +0100966 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100967 }
968 for (int j=0;j<entry.size();j++) {
969 def nodeKey = entry.keySet()[j]
970 def node=entry[nodeKey]
Jakub Josefa87941c2017-04-20 17:14:58 +0200971 def outputResources = []
Jakub Josef47145942018-04-04 17:30:38 +0200972 def errorResources = []
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100973 common.infoMsg("Node ${nodeKey} changes:")
974 if(node instanceof Map || node instanceof List){
975 for (int k=0;k<node.size();k++) {
976 def resource;
977 def resKey;
978 if(node instanceof Map){
979 resKey = node.keySet()[k]
Victor Ryzhenkin4a45d732019-01-09 15:28:21 +0400980 if (resKey == "retcode") {
Richard Felkld9476ac2018-07-12 19:01:33 +0200981 continue
Victor Ryzhenkin4a45d732019-01-09 15:28:21 +0400982 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100983 }else if(node instanceof List){
984 resKey = k
985 }
986 resource = node[resKey]
Jakub Josefc4c40202017-04-28 12:04:24 +0200987 // print
Jakub Josefa87941c2017-04-20 17:14:58 +0200988 if(printResults){
989 if(resource instanceof Map && resource.keySet().contains("result")){
990 //clean unnesaccary fields
991 if(resource.keySet().contains("__run_num__")){
992 resource.remove("__run_num__")
993 }
994 if(resource.keySet().contains("__id__")){
995 resource.remove("__id__")
996 }
997 if(resource.keySet().contains("pchanges")){
998 resource.remove("pchanges")
999 }
1000 if(!resource["result"] || (resource["result"] instanceof String && resource["result"] != "true")){
1001 if(resource["result"] != null){
Jakub Josefbceaa322017-06-13 18:28:27 +02001002 outputResources.add(String.format("Resource: %s\n\u001B[31m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001003 }else{
Jakub Josefbceaa322017-06-13 18:28:27 +02001004 outputResources.add(String.format("Resource: %s\n\u001B[33m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001005 }
1006 }else{
1007 if(!printOnlyChanges || resource.changes.size() > 0){
Jakub Josefbceaa322017-06-13 18:28:27 +02001008 outputResources.add(String.format("Resource: %s\n\u001B[32m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001009 }
1010 }
1011 }else{
Jakub Josefbceaa322017-06-13 18:28:27 +02001012 outputResources.add(String.format("Resource: %s\n\u001B[36m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001013 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001014 }
Jakub Josefc4c40202017-04-28 12:04:24 +02001015 common.debugMsg("checkResult: checking resource: ${resource}")
1016 if(resource instanceof String || (resource["result"] != null && !resource["result"]) || (resource["result"] instanceof String && resource["result"] == "false")){
Jakub Josef47145942018-04-04 17:30:38 +02001017 errorResources.add(resource)
Jakub Josefc4c40202017-04-28 12:04:24 +02001018 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001019 }
Jakub Josefa87941c2017-04-20 17:14:58 +02001020 }else if(node!=null && node!=""){
Jakub Josef62f6c842017-08-04 16:36:35 +02001021 outputResources.add(String.format("Resource: %s\n\u001B[36m%s\u001B[0m", nodeKey, common.prettify(node)))
Jakub Josefa87941c2017-04-20 17:14:58 +02001022 }
1023 if(printResults && !outputResources.isEmpty()){
Jakub Josef47145942018-04-04 17:30:38 +02001024 println outputResources.stream().collect(Collectors.joining("\n"))
1025 }
1026 if(!errorResources.isEmpty()){
1027 for(resource in errorResources){
1028 def prettyResource = common.prettify(resource)
1029 if (!disableAskOnError && env["ASK_ON_ERROR"] && env["ASK_ON_ERROR"] == "true") {
1030 timeout(time:1, unit:'HOURS') {
1031 input message: "False result on ${nodeKey} found, resource ${prettyResource}. \nDo you want to continue?"
1032 }
1033 } else {
1034 def errorMsg = "Salt state on node ${nodeKey} failed. Resource: ${prettyResource}"
1035 if (failOnError) {
1036 throw new Exception(errorMsg)
1037 } else {
1038 common.errorMsg(errorMsg)
1039 }
1040 }
1041 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001042 }
1043 }
Jakub Josef52f69f72017-03-14 15:18:08 +01001044 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001045 }else{
1046 common.errorMsg("Salt result hasn't return attribute! Result: ${result}")
Jakub Josef79ecec32017-02-17 14:36:28 +01001047 }
Jakub Josef52f69f72017-03-14 15:18:08 +01001048 }else{
Jakub Josefa87941c2017-04-20 17:14:58 +02001049 common.errorMsg("Cannot check salt result, given result is null")
Jakub Josef79ecec32017-02-17 14:36:28 +01001050 }
1051}
1052
1053/**
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001054* Parse salt API output to check minion restart and wait some time to be sure minion is up.
1055* See https://mirantis.jira.com/browse/PROD-16258 for more details
1056* TODO: change sleep to more tricky procedure.
1057*
1058* @param result Parsed response of Salt API
1059*/
Vasyl Saienko6a396212018-06-08 09:20:08 +03001060def waitForMinion(result, minionRestartWaitTimeout=10) {
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001061 def common = new com.mirantis.mk.Common()
Oleg Iurchenko3eedc782017-12-12 11:49:29 +02001062 //In order to prevent multiple sleeps use bool variable to catch restart for any minion.
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001063 def isMinionRestarted = false
Oleg Iurchenko3eedc782017-12-12 11:49:29 +02001064 if(result != null){
1065 if(result['return']){
1066 for (int i=0;i<result['return'].size();i++) {
1067 def entry = result['return'][i]
1068 // exit in case of empty response.
1069 if (!entry) {
1070 return
1071 }
1072 // Loop for nodes
1073 for (int j=0;j<entry.size();j++) {
1074 def nodeKey = entry.keySet()[j]
1075 def node=entry[nodeKey]
1076 if(node instanceof Map || node instanceof List){
1077 // Loop for node resources
1078 for (int k=0;k<node.size();k++) {
1079 def resource;
1080 def resKey;
1081 if(node instanceof Map){
1082 resKey = node.keySet()[k]
1083 }else if(node instanceof List){
1084 resKey = k
1085 }
1086 resource = node[resKey]
Jakub Joseffb9996d2018-04-10 14:05:31 +02001087 // try to find if salt_minion service was restarted
1088 if(resKey instanceof String && resKey.contains("salt_minion_service_restart") && resource instanceof Map && resource.keySet().contains("result")){
Oleg Iurchenko3eedc782017-12-12 11:49:29 +02001089 if((resource["result"] instanceof Boolean && resource["result"]) || (resource["result"] instanceof String && resource["result"] == "true")){
1090 if(resource.changes.size() > 0){
1091 isMinionRestarted=true
1092 }
1093 }
1094 }
1095 }
1096 }
1097 }
1098 }
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001099 }
1100 }
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001101 if (isMinionRestarted){
Vasyl Saienko6a396212018-06-08 09:20:08 +03001102 common.infoMsg("Salt minion service restart detected. Sleep ${minionRestartWaitTimeout} seconds to wait minion restart")
1103 sleep(minionRestartWaitTimeout)
Oleg Iurchenko7eb21502017-11-28 18:53:43 +02001104 }
1105}
1106
1107/**
Jakub Josef7852fe12017-03-15 16:02:41 +01001108 * Print salt command run results in human-friendly form
Jakub Josef79ecec32017-02-17 14:36:28 +01001109 *
1110 * @param result Parsed response of Salt API
Jakub Josef79ecec32017-02-17 14:36:28 +01001111 */
Filip Pytlound2f1bbe2017-02-27 19:03:51 +01001112def printSaltCommandResult(result) {
Jakub Josef871bf152017-03-14 20:13:41 +01001113 def common = new com.mirantis.mk.Common()
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001114 if(result != null){
1115 if(result['return']){
1116 for (int i=0; i<result['return'].size(); i++) {
1117 def entry = result['return'][i]
1118 for (int j=0; j<entry.size(); j++) {
1119 common.debugMsg("printSaltCommandResult: printing salt command entry: ${entry}")
1120 def nodeKey = entry.keySet()[j]
1121 def node=entry[nodeKey]
Jakub Josefbceaa322017-06-13 18:28:27 +02001122 common.infoMsg(String.format("Node %s changes:\n%s",nodeKey, common.prettify(node)))
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001123 }
Jakub Josef8a715bf2017-03-14 21:39:01 +01001124 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001125 }else{
1126 common.errorMsg("Salt result hasn't return attribute! Result: ${result}")
Jakub Josef79ecec32017-02-17 14:36:28 +01001127 }
Jakub Josef8a715bf2017-03-14 21:39:01 +01001128 }else{
Jakub Josefd9afd0e2017-03-15 19:19:23 +01001129 common.errorMsg("Cannot print salt command result, given result is null")
Jakub Josef52f69f72017-03-14 15:18:08 +01001130 }
Jakub Josef79ecec32017-02-17 14:36:28 +01001131}
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +02001132
1133
1134/**
1135 * Return content of file target
1136 *
chnydaa0dbb252017-10-05 10:46:09 +02001137 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +02001138 * @param target Compound target (should target only one host)
1139 * @param file File path to read (/etc/hosts for example)
1140 */
1141
Pavlo Shchelokovskyy97100072018-10-04 11:53:44 +03001142def getFileContent(saltId, target, file, checkResponse = true, batch=null, output = true, saltArgs = []) {
1143 result = cmdRun(saltId, target, "cat ${file}", checkResponse, batch, output, saltArgs)
Tomáš Kukrálf1a692a2017-08-11 13:29:28 +02001144 return result['return'][0].values()[0].replaceAll('Salt command execution success','')
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +02001145}
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001146
1147/**
1148 * Set override parameters in Salt cluster metadata
1149 *
chnydaa0dbb252017-10-05 10:46:09 +02001150 * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001151 * @param salt_overrides YAML formatted string containing key: value, one per line
Matthew Mosesohne5646842017-07-19 16:54:57 +03001152 * @param reclass_dir Directory where Reclass git repo is located
Dzmitry Stremkouskib5440702018-07-22 13:00:05 +02001153 * @param extra_tgt Extra targets for compound
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001154 */
1155
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001156def setSaltOverrides(saltId, salt_overrides, reclass_dir="/srv/salt/reclass", extra_tgt = '') {
Tomáš Kukrálf178f052017-07-11 11:31:00 +02001157 def common = new com.mirantis.mk.Common()
Mykyta Karpin1c165e22017-08-22 18:27:01 +03001158 def salt_overrides_map = readYaml text: salt_overrides
Tomáš Kukrál243cf842017-07-11 13:11:56 +02001159 for (entry in common.entries(salt_overrides_map)) {
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001160 def key = entry[0]
1161 def value = entry[1]
1162
1163 common.debugMsg("Set salt override ${key}=${value}")
Victor Ryzhenkinb742f4b2019-01-10 15:12:55 +04001164 runSaltProcessStep(saltId, "I@salt:master ${extra_tgt}", 'reclass.cluster_meta_set', ["name=${key}", "value=${value}"], false)
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001165 }
Oleh Hryhorov5f96e092018-06-22 18:37:08 +03001166 runSaltProcessStep(saltId, "I@salt:master ${extra_tgt}", 'cmd.run', ["git -C ${reclass_dir} update-index --skip-worktree classes/cluster/overrides.yml"])
Matthew Mosesohn9e880852017-07-04 21:17:53 +03001167}
Oleg Grigorovbec45582017-09-12 20:29:24 +03001168
1169/**
1170* Execute salt commands via salt-api with
1171* CLI client salt-pepper
1172*
1173* @param data Salt command map
1174* @param venv Path to virtualenv with
1175*/
1176
azvyagintsevb8b7f922019-06-13 13:39:04 +03001177def runPepperCommand(data, venv) {
Jakub Josef03d4d5a2017-12-20 16:35:09 +01001178 def common = new com.mirantis.mk.Common()
Oleg Grigorovbec45582017-09-12 20:29:24 +03001179 def python = new com.mirantis.mk.Python()
1180 def dataStr = new groovy.json.JsonBuilder(data).toString()
azvyagintsevb8b7f922019-06-13 13:39:04 +03001181 // TODO(alexz): parametrize?
1182 int retry = 10
chnyda4901a042017-11-16 12:14:56 +01001183
Jakub Josefa2491ad2018-01-15 16:26:27 +01001184 def pepperCmdFile = "${venv}/pepper-cmd.json"
1185 writeFile file: pepperCmdFile, text: dataStr
1186 def pepperCmd = "pepper -c ${venv}/pepperrc --make-token -x ${venv}/.peppercache --json-file ${pepperCmdFile}"
Oleg Grigorovbec45582017-09-12 20:29:24 +03001187
azvyagintsevb8b7f922019-06-13 13:39:04 +03001188 int tries = 0
1189 def FullOutput = ['status': 1]
Jakub Josef37cd4972018-02-01 16:25:25 +01001190 def outputObj
azvyagintsevb8b7f922019-06-13 13:39:04 +03001191 while (tries++ < retry) {
1192 try {
1193 if (venv) {
1194 FullOutput = python.runVirtualenvCommand(venv, pepperCmd, true, true)
1195 } else {
1196 FullOutput = common.shCmdStatus(pepperCmd)
1197 }
1198 if (FullOutput['status'] != 0) {
1199 error()
1200 }
1201 break
1202 } catch (e) {
1203 // Check , if we get failed pepper HTTP call, and retry
1204 common.errorMsg("Command: ${pepperCmd} failed to execute with error:\n${FullOutput['stderr']}")
1205 if (FullOutput['stderr'].contains('Error with request: HTTP Error 50') || FullOutput['stderr'].contains('Pepper error: Server error')) {
1206 common.errorMsg("Pepper HTTP Error detected. Most probably, " +
1207 "master SaltReqTimeoutError in master zmq thread issue...lets retry ${tries}/${retry}")
1208 sleep(5)
1209 continue
1210 }
1211 }
1212 }
1213 // Try to parse json output. No sense to check exit code, since we always expect json answer only.
Jakub Josef37cd4972018-02-01 16:25:25 +01001214 try {
azvyagintsevb8b7f922019-06-13 13:39:04 +03001215 outputObj = new groovy.json.JsonSlurperClassic().parseText(FullOutput['stdout'])
1216 } catch (Exception jsonE) {
1217 common.errorMsg('Parsing Salt API JSON response failed! Response: ' + FullOutput)
1218 throw jsonE
Jakub Josef37cd4972018-02-01 16:25:25 +01001219 }
1220 return outputObj
Oleg Grigorovbec45582017-09-12 20:29:24 +03001221}
Martin Polreichddfe51c2019-01-21 14:31:00 +01001222
azvyagintsevb8b7f922019-06-13 13:39:04 +03001223
Martin Polreichddfe51c2019-01-21 14:31:00 +01001224/**
1225* Check time settings on defined nodes, compares them
1226* and evaluates the results
1227*
1228* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1229* @param target Targeted nodes to be checked
1230* @param diff Maximum time difference (in seconds) to be accepted during time sync check
1231* @return bool Return true if time difference is <= diff and returns false if time difference is > diff
1232*/
1233
1234def checkClusterTimeSync(saltId, target) {
1235 def common = new com.mirantis.mk.Common()
1236 def salt = new com.mirantis.mk.Salt()
1237
1238 times = []
1239 try {
1240 diff = salt.getReturnValues(salt.getPillar(saltId, 'I@salt:master', 'linux:system:time_diff'))
1241 if (diff != null && diff != "" && diff.isInteger()) {
1242 diff = diff.toInteger()
1243 } else {
1244 diff = 5
1245 }
1246 out = salt.runSaltProcessStep(saltId, target, 'status.time', '%s')
1247 outParsed = out['return'][0]
1248 def outKeySet = outParsed.keySet()
1249 for (key in outKeySet) {
1250 def time = outParsed[key].readLines().get(0)
1251 common.infoMsg(time)
1252 if (time.isInteger()) {
1253 times.add(time.toInteger())
1254 }
1255 }
1256 if ((times.max() - times.min()) <= diff) {
1257 return true
1258 } else {
1259 return false
1260 }
1261 } catch(Exception e) {
1262 common.errorMsg("Could not check cluster time sync.")
1263 return false
1264 }
1265}
Martin Polreichad8e95b2019-03-29 12:10:00 +01001266
1267/**
1268* Finds out IP address of the given node or a list of nodes
1269*
1270* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1271* @param nodes Targeted node hostnames to be checked (String or List of strings)
1272* @param useGrains If the, the value will be taken from grains. If false, it will be taken from 'hostname' command.
1273* @return Map Return result Map in format ['nodeName1': 'ipAdress1', 'nodeName2': 'ipAdress2', ...]
1274*/
1275
1276def getIPAddressesForNodenames(saltId, nodes = [], useGrains = true) {
1277 result = [:]
1278
1279 if (nodes instanceof String) {
1280 nodes = [nodes]
1281 }
1282
1283 if (useGrains) {
1284 for (String node in nodes) {
1285 ip = getReturnValues(getGrain(saltId, node, "fqdn_ip4"))["fqdn_ip4"][0]
1286 result[node] = ip
1287 }
1288 } else {
1289 for (String node in nodes) {
1290 ip = getReturnValues(cmdRun(saltId, node, "hostname -i")).readLines()[0]
1291 result[node] = ip
1292 }
1293 }
1294 return result
1295}
1296
1297/**
1298* Checks if required package is installed and returns averaged IO stats for selected disks.
1299* Allows getting averaged values of specific parameter for all disks or a specified disk.
1300* Interval between checks and its number is parametrized and configurable.
1301*
1302* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1303* @param target Node to be targeted (Should only match 1 node)
1304* @param parameterName Name of parameter from 'iostat' output (default = '' -- returns all variables)
1305* @param interval Interval between checks (default = 1)
1306* @param count Number of checks (default = 5)
1307* @param disks Disks to be checked (default = '' -- returns all disks)
1308* @param output Print Salt command return (default = true)
1309* @return Map Map containing desired values in format ['disk':'value']
1310*/
1311
1312def getIostatValues(Map params) {
1313 def common = new com.mirantis.mk.Common()
1314 def ret = [:]
1315 if (isPackageInstalled(['saltId': params.saltId, 'target': params.target, 'packageName': 'sysstat', 'output': false])) {
1316 def arg = [params.get('interval', 1), params.get('count', 5), params.get('disks', '')]
1317 def res = getReturnValues(runSaltProcessStep(params.saltId, params.target, 'disk.iostat', arg, null, params.output))
1318 if (res instanceof Map) {
1319 for (int i = 0; i < res.size(); i++) {
1320 def key = res.keySet()[i]
1321 if (params.containsKey('parameterName')) {
1322 if (res[key].containsKey(params.parameterName)){
1323 ret[key] = res[key][params.parameterName]
1324 } else {
1325 common.errorMsg("Parameter '${params.parameterName}' not found for disk '${key}'. Valid parameter for this disk are: '${res[key].keySet()}'")
1326 }
1327 } else {
1328 return res // If no parameterName is defined, return all of them.
1329 }
1330 }
1331 }
1332 } else {
1333 common.errorMsg("Package 'sysstat' seems not to be installed on at least one of tageted nodes: ${params.target}. Please fix this to be able to check 'iostat' values. Find more in the docs TODO:<Add docs link>")
1334 }
1335 return ret
1336}
1337
1338/**
1339* Checks if defined package is installed on all nodes defined by target parameter.
1340*
1341* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
1342* @param target Node or nodes to be targeted
1343* @param packageName Name of package to be checked
1344* @param output Print Salt command return (default = true)
1345* @return boolean True if package is installed on all defined nodes. False if not found on at least one of defined nodes.
1346*/
1347
1348def isPackageInstalled(Map params) {
1349 def output = params.get('output', true)
1350 def res = runSaltProcessStep(params.saltId, params.target, "pkg.info_installed", params.packageName, null, output)['return'][0]
Ivan Berezovskiy1480c7e2019-07-26 16:44:22 +04001351 if (res) {
1352 for (int i = 0; i < res.size(); i++) {
1353 def key = res.keySet()[i]
1354 if (!(res[key] instanceof Map && res[key].containsKey(params.packageName))) {
1355 return false
1356 }
Martin Polreichad8e95b2019-03-29 12:10:00 +01001357 }
Ivan Berezovskiy1480c7e2019-07-26 16:44:22 +04001358 return true
1359 } else {
1360 return false
Martin Polreichad8e95b2019-03-29 12:10:00 +01001361 }
Martin Polreichad8e95b2019-03-29 12:10:00 +01001362}