blob: 248f8ebb4e85144d95f6092cfb03c82c4bc9644a [file] [log] [blame]
Jakub Josef79ecec32017-02-17 14:36:28 +01001package com.mirantis.mk
2
Jakub Josefbceaa322017-06-13 18:28:27 +02003import com.cloudbees.groovy.cps.NonCPS
Jakub Josefb77c0812017-03-27 14:11:01 +02004import java.util.stream.Collectors
Jakub Josef79ecec32017-02-17 14:36:28 +01005/**
6 * Salt functions
7 *
8*/
9
10/**
11 * Salt connection and context parameters
12 *
13 * @param url Salt API server URL
14 * @param credentialsID ID of credentials store entry
15 */
16def connection(url, credentialsId = "salt") {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +010017 def common = new com.mirantis.mk.Common()
Jakub Josef79ecec32017-02-17 14:36:28 +010018 params = [
19 "url": url,
20 "credentialsId": credentialsId,
21 "authToken": null,
22 "creds": common.getCredentials(credentialsId)
23 ]
24 params["authToken"] = saltLogin(params)
Jakub Josef79ecec32017-02-17 14:36:28 +010025 return params
26}
27
28/**
29 * Login to Salt API, return auth token
30 *
31 * @param master Salt connection object
32 */
33def saltLogin(master) {
Tomáš Kukrál7bec0532017-02-20 15:39:31 +010034 def http = new com.mirantis.mk.Http()
Jakub Josef79ecec32017-02-17 14:36:28 +010035 data = [
36 'username': master.creds.username,
37 'password': master.creds.password.toString(),
38 'eauth': 'pam'
39 ]
Tomáš Kukrál7bec0532017-02-20 15:39:31 +010040 authToken = http.restGet(master, '/login', data)['return'][0]['token']
Jakub Josef79ecec32017-02-17 14:36:28 +010041 return authToken
42}
43
44/**
Oleg Grigorovbec45582017-09-12 20:29:24 +030045 * Run action using Salt API (using plain HTTP request from Jenkins master)
Jakub Josef79ecec32017-02-17 14:36:28 +010046 *
47 * @param master Salt connection object
48 * @param client Client type
49 * @param target Target specification, eg. for compound matches by Pillar
50 * data: ['expression': 'I@openssh:server', 'type': 'compound'])
51 * @param function Function to execute (eg. "state.sls")
Jakub Josef2f25cf22017-03-28 13:34:57 +020052 * @param batch Batch param to salt (integer or string with percents)
Jakub Josef79ecec32017-02-17 14:36:28 +010053 * @param args Additional arguments to function
54 * @param kwargs Additional key-value arguments to function
Jiri Broulik48544be2017-06-14 18:33:54 +020055 * @param timeout Additional argument salt api timeout
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +030056 * @param read_timeout http session read timeout
Jakub Josef79ecec32017-02-17 14:36:28 +010057 */
58@NonCPS
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +030059def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null, timeout = -1, read_timeout = -1) {
iberezovskiyd4240b52017-02-20 17:18:28 +040060 def http = new com.mirantis.mk.Http()
Jakub Josef79ecec32017-02-17 14:36:28 +010061
62 data = [
63 'tgt': target.expression,
64 'fun': function,
65 'client': client,
66 'expr_form': target.type,
67 ]
Jakub Josef5f838212017-04-06 12:43:58 +020068 if(batch != null && ( (batch instanceof Integer && batch > 0) || (batch instanceof String && batch.contains("%")))){
Jakub Josef2f25cf22017-03-28 13:34:57 +020069 data['client']= "local_batch"
70 data['batch'] = batch
Jakub Josef79ecec32017-02-17 14:36:28 +010071 }
72
73 if (args) {
74 data['arg'] = args
75 }
76
77 if (kwargs) {
78 data['kwarg'] = kwargs
79 }
80
Jiri Broulik48544be2017-06-14 18:33:54 +020081 if (timeout != -1) {
82 data['timeout'] = timeout
83 }
84
Jakub Josef79ecec32017-02-17 14:36:28 +010085 headers = [
86 'X-Auth-Token': "${master.authToken}"
87 ]
88
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +030089 return http.sendHttpPostRequest("${master.url}/", data, headers, read_timeout)
Jakub Josef79ecec32017-02-17 14:36:28 +010090}
91
Jakub Josef5ade54c2017-03-10 16:14:01 +010092/**
Oleg Grigorovbec45582017-09-12 20:29:24 +030093 * Run action using Salt API (using salt pepper from slave shell)
94 *
95 * @param master Salt connection object
96 * @param client Client type
97 * @param target Target specification, eg. for compound matches by Pillar
98 * data: ['expression': 'I@openssh:server', 'type': 'compound'])
99 * @param function Function to execute (eg. "state.sls")
100 * @param batch Batch param to salt (integer or string with percents)
101 * @param args Additional arguments to function
102 * @param kwargs Additional key-value arguments to function
103 * @param timeout Additional argument salt api timeout
104 * @param read_timeout http session read timeout
105 */
106@NonCPS
107def runSaltCommandPepper(pepperVenv, client, target, function, batch = null, args = null, kwargs = null, timeout = -1, read_timeout = -1) {
108 def http = new com.mirantis.mk.Http()
109
110 data = [
111 'tgt': target.expression,
112 'fun': function,
113 'client': client,
114 'expr_form': target.type,
115 ]
116 if(batch != null && ( (batch instanceof Integer && batch > 0) || (batch instanceof String && batch.contains("%")))){
117 data['client']= "local_batch"
118 data['batch'] = batch
119 }
120
121 if (args) {
122 data['arg'] = args
123 }
124
125 if (kwargs) {
126 data['kwarg'] = kwargs
127 }
128
129 if (timeout != -1) {
130 data['timeout'] = timeout
131 }
132
133 headers = [
134 'X-Auth-Token': "${master.authToken}"
135 ]
136
137 return runPepperCommand(data, pepperVenv)
138}
139
140/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100141 * Return pillar for given master and target
142 * @param master Salt connection object
143 * @param target Get pillar target
144 * @param pillar pillar name (optional)
145 * @return output of salt command
146 */
Ales Komarekcec24d42017-03-08 10:25:45 +0100147def getPillar(master, target, pillar = null) {
Tomáš Kukráld2589702017-03-10 16:30:46 +0100148 if (pillar != null) {
Jakub Josef5ade54c2017-03-10 16:14:01 +0100149 return runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'pillar.get', null, [pillar.replace('.', ':')])
Tomáš Kukráld2589702017-03-10 16:30:46 +0100150 } else {
Jakub Josef5ade54c2017-03-10 16:14:01 +0100151 return runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'pillar.data')
Ales Komareka3c7e502017-03-13 11:20:44 +0100152 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100153}
154
Jakub Josef5ade54c2017-03-10 16:14:01 +0100155/**
156 * Return grain for given master and target
157 * @param master Salt connection object
158 * @param target Get grain target
159 * @param grain grain name (optional)
160 * @return output of salt command
161 */
Ales Komarekcec24d42017-03-08 10:25:45 +0100162def getGrain(master, target, grain = null) {
163 if(grain != null) {
Jiri Broulik852dfd52017-05-16 13:38:55 +0200164 return runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'grains.item', null, [grain])
Jakub Josef5ade54c2017-03-10 16:14:01 +0100165 } else {
Jiri Broulik852dfd52017-05-16 13:38:55 +0200166 return runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'grains.items')
Ales Komarekcec24d42017-03-08 10:25:45 +0100167 }
Ales Komarekcec24d42017-03-08 10:25:45 +0100168}
169
Jakub Josef5ade54c2017-03-10 16:14:01 +0100170/**
171 * Enforces state on given master and target
172 * @param master Salt connection object
173 * @param target State enforcing target
174 * @param state Salt state
175 * @param output print output (optional, default true)
176 * @param failOnError throw exception on salt state result:false (optional, default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200177 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Petr Michalecde0ff322017-10-04 09:32:14 +0200178 * @param read_timeout http session read timeout (optional, default -1 - disabled)
179 * @param retries Retry count for salt state. (optional, default -1 - no retries)
180 * @param queue salt queue parameter for state.sls calls (optional, default true) - CANNOT BE USED WITH BATCH
Jakub Josef5ade54c2017-03-10 16:14:01 +0100181 * @return output of salt command
182 */
Petr Michalecde0ff322017-10-04 09:32:14 +0200183def enforceState(master, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100184 def common = new com.mirantis.mk.Common()
Jakub Josef79ecec32017-02-17 14:36:28 +0100185 def run_states
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100186
Jakub Josef79ecec32017-02-17 14:36:28 +0100187 if (state instanceof String) {
188 run_states = state
189 } else {
190 run_states = state.join(',')
191 }
192
Marek Celoud63366112017-07-25 17:27:24 +0200193 common.infoMsg("Running state ${run_states} on ${target}")
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300194 def out
Petr Michalecde0ff322017-10-04 09:32:14 +0200195 def kwargs = [:]
196
197 if (queue && batch == null) {
198 kwargs["queue"] = true
199 }
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300200
201 if (optional == false || testTarget(master, target)){
202 if (retries != -1){
203 retry(retries){
Petr Michalecde0ff322017-10-04 09:32:14 +0200204 out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states], kwargs, -1, read_timeout)
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300205 }
Petr Michalecde0ff322017-10-04 09:32:14 +0200206 } else {
207 out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states], kwargs, -1, read_timeout)
Vasyl Saienkoe36ab7c2017-07-17 14:35:48 +0300208 }
Martin Polreich1c77afa2017-07-18 11:27:02 +0200209 checkResult(out, failOnError, output)
210 return out
Martin Polreich1c77afa2017-07-18 11:27:02 +0200211 } else {
212 common.infoMsg("No Minions matched the target given, but 'optional' param was set to true - Pipeline continues. ")
213 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100214}
215
Jakub Josef5ade54c2017-03-10 16:14:01 +0100216/**
217 * Run command on salt minion (salt cmd.run wrapper)
218 * @param master Salt connection object
219 * @param target Get pillar target
220 * @param cmd command
Jakub Josef053df392017-05-03 15:51:05 +0200221 * @param checkResponse test command success execution (default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200222 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200223 * @param output do you want to print output
Jakub Josef5ade54c2017-03-10 16:14:01 +0100224 * @return output of salt command
225 */
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200226def cmdRun(master, target, cmd, checkResponse = true, batch=null, output = true) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100227 def common = new com.mirantis.mk.Common()
Jakub Josef053df392017-05-03 15:51:05 +0200228 def originalCmd = cmd
Tomáš Kukráldfd4b492017-03-02 12:08:50 +0100229 common.infoMsg("Running command ${cmd} on ${target}")
Jakub Josef053df392017-05-03 15:51:05 +0200230 if (checkResponse) {
231 cmd = cmd + " && echo Salt command execution success"
232 }
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200233 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.run', batch, [cmd])
Jakub Josef053df392017-05-03 15:51:05 +0200234 if (checkResponse) {
235 // iterate over all affected nodes and check success return code
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200236 if (out["return"]){
237 for(int i=0;i<out["return"].size();i++){
238 def node = out["return"][i];
Jakub Josef053df392017-05-03 15:51:05 +0200239 for(int j=0;j<node.size();j++){
240 def nodeKey = node.keySet()[j]
241 if (!node[nodeKey].contains("Salt command execution success")) {
242 throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
243 }
244 }
245 }
246 }else{
247 throw new Exception("Salt Api response doesn't have return param!")
248 }
249 }
Jiri Broulik16e9ce72017-05-17 13:28:31 +0200250 if (output == true) {
251 printSaltCommandResult(out)
252 }
253 return out
Jakub Josef79ecec32017-02-17 14:36:28 +0100254}
255
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200256/**
Jiri Broulik71512bc2017-08-04 10:00:18 +0200257 * Checks if salt minion is in a list of salt master's accepted keys
258 * @usage minionPresent(saltMaster, 'I@salt:master', 'ntw', true, null, true, 200, 3)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200259 * @param master Salt connection object
260 * @param target Get pillar target
261 * @param minion_name unique identification of a minion in salt-key command output
262 * @param waitUntilPresent return after the minion becomes present (default true)
263 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
264 * @param output print salt command (default true)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200265 * @param maxRetries finite number of iterations to check status of a command (default 200)
266 * @param answers how many minions should return (optional, default 1)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200267 * @return output of salt command
268 */
Jiri Broulik71512bc2017-08-04 10:00:18 +0200269def minionPresent(master, target, minion_name, waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
270 minion_name = minion_name.replace("*", "")
271 def common = new com.mirantis.mk.Common()
272 def cmd = 'salt-key | grep ' + minion_name
273 if (waitUntilPresent){
274 def count = 0
275 while(count < maxRetries) {
276 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
277 if (output) {
278 printSaltCommandResult(out)
279 }
280 def valueMap = out["return"][0]
281 def result = valueMap.get(valueMap.keySet()[0])
282 def resultsArray = result.tokenize("\n")
283 def size = resultsArray.size()
284 if (size >= answers) {
285 return out
286 }
287 count++
288 sleep(time: 500, unit: 'MILLISECONDS')
289 common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
290 }
291 } else {
292 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
293 if (output) {
294 printSaltCommandResult(out)
295 }
296 return out
297 }
298 // otherwise throw exception
299 common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
300 throw new Exception("${cmd} signals failure of status check!")
301}
302
303/**
304 * You can call this function when salt-master already contains salt keys of the target_nodes
305 * @param master Salt connection object
306 * @param target Should always be salt-master
307 * @param target_nodes unique identification of a minion or group of salt minions
308 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
309 * @param wait timeout for the salt command if minions do not return (default 10)
310 * @param maxRetries finite number of iterations to check status of a command (default 200)
311 * @return output of salt command
312 */
313def minionsReachable(master, target, target_nodes, batch=null, wait = 10, maxRetries = 200) {
314 def common = new com.mirantis.mk.Common()
315 def cmd = "salt -t${wait} -C '${target_nodes}' test.ping"
316 common.infoMsg("Checking if all ${target_nodes} minions are reachable")
317 def count = 0
318 while(count < maxRetries) {
319 Calendar timeout = Calendar.getInstance();
320 timeout.add(Calendar.SECOND, wait);
321 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, wait)
322 Calendar current = Calendar.getInstance();
323 if (current.getTime().before(timeout.getTime())) {
324 printSaltCommandResult(out)
325 return out
326 }
327 common.infoMsg("Not all of the targeted '${target_nodes}' minions returned yet. Waiting ...")
328 count++
329 sleep(time: 500, unit: 'MILLISECONDS')
330 }
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200331}
332
333/**
334 * Run command on salt minion (salt cmd.run wrapper)
335 * @param master Salt connection object
336 * @param target Get pillar target
337 * @param cmd name of a service
338 * @param correct_state string that command must contain if status is in correct state (optional, default 'running')
Jiri Broulikcf1f2332017-07-25 11:30:03 +0200339 * @param find bool value if it is suppose to find some string in the output or the cmd should return empty string (optional, default true)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200340 * @param waitUntilOk return after the minion becomes present (optional, default true)
341 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
342 * @param output print salt command (default true)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200343 * @param maxRetries finite number of iterations to check status of a command (default 200)
344 * @param answers how many minions should return (optional, default 0)
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200345 * @return output of salt command
346 */
Jiri Broulik71512bc2017-08-04 10:00:18 +0200347def commandStatus(master, target, cmd, correct_state='running', find = true, waitUntilOk = true, batch=null, output = true, maxRetries = 200, answers = 0) {
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200348 def common = new com.mirantis.mk.Common()
349 common.infoMsg("Checking if status of verification command ${cmd} on ${target} is in correct state")
350 if (waitUntilOk){
351 def count = 0
352 while(count < maxRetries) {
Jiri Broulikd0c27572017-07-24 20:01:10 +0200353 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
Jiri Broulik71512bc2017-08-04 10:00:18 +0200354 if (output) {
355 printSaltCommandResult(out)
356 }
Jakub Josef115a78f2017-07-18 15:04:00 +0200357 def resultMap = out["return"][0]
Jiri Broulik71512bc2017-08-04 10:00:18 +0200358 def success = 0
359 if (answers == 0){
360 answers = resultMap.size()
361 }
362 for (int i=0;i<answers;i++) {
363 result = resultMap.get(resultMap.keySet()[i])
364 // if the goal is to find some string in output of the command
365 if (find) {
366 if(result == null || result instanceof Boolean || result.isEmpty()) { result='' }
367 if (result.toLowerCase().contains(correct_state.toLowerCase())) {
368 success++
369 if (success == answers) {
370 return out
371 }
Jiri Broulikd0c27572017-07-24 20:01:10 +0200372 }
Jiri Broulik71512bc2017-08-04 10:00:18 +0200373 // else the goal is to not find any string in output of the command
374 } else {
375 if(result instanceof String && result.isEmpty()) {
376 success++
377 if (success == answers) {
378 return out
379 }
380 }
381 }
382 }
383 count++
384 sleep(time: 500, unit: 'MILLISECONDS')
385 common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
386 }
387 } else {
388 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
389 def resultMap = out["return"][0]
390 if (output) {
391 printSaltCommandResult(out)
392 }
393 for (int i=0;i<resultMap.size();i++) {
394 result = resultMap.get(resultMap.keySet()[i])
395 // if the goal is to find some string in output of the command
396 if (find) {
397 if(result == null || result instanceof Boolean || result.isEmpty()) { result='' }
398 if (result.toLowerCase().contains(correct_state.toLowerCase())) {
Jiri Broulikd0c27572017-07-24 20:01:10 +0200399 return out
400 }
401
402 // else the goal is to not find any string in output of the command
403 } else {
404 if(result instanceof String && result.isEmpty()) {
405 return out
406 }
407 }
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200408 }
409 }
410 // otherwise throw exception
Jiri Broulikd0c27572017-07-24 20:01:10 +0200411 common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
Jiri Broulik2c69f3d2017-07-18 14:23:58 +0200412 throw new Exception("${cmd} signals failure of status check!")
413}
414
Jakub Josef5ade54c2017-03-10 16:14:01 +0100415/**
416 * Perform complete salt sync between master and target
417 * @param master Salt connection object
418 * @param target Get pillar target
419 * @return output of salt command
420 */
Jakub Josef79ecec32017-02-17 14:36:28 +0100421def syncAll(master, target) {
Filip Pytloun5a7f7fd2017-02-27 18:50:25 +0100422 return runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'saltutil.sync_all')
Jakub Josef79ecec32017-02-17 14:36:28 +0100423}
424
Jakub Josef5ade54c2017-03-10 16:14:01 +0100425/**
426 * Enforce highstate on given targets
427 * @param master Salt connection object
428 * @param target Highstate enforcing target
429 * @param output print output (optional, default true)
430 * @param failOnError throw exception on salt state result:false (optional, default true)
Jakub Josef2f25cf22017-03-28 13:34:57 +0200431 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100432 * @return output of salt command
433 */
Jakub Josef2f25cf22017-03-28 13:34:57 +0200434def enforceHighstate(master, target, output = false, failOnError = true, batch = null) {
435 def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.highstate', batch)
Alexander Noskov657ccfc2017-07-14 11:35:52 +0000436 def common = new com.mirantis.mk.Common()
437
Marek Celoud63366112017-07-25 17:27:24 +0200438 common.infoMsg("Running state highstate on ${target}")
Alexander Noskov657ccfc2017-07-14 11:35:52 +0000439
Jakub Josef374beb72017-04-27 15:45:09 +0200440 checkResult(out, failOnError, output)
Jakub Josef79ecec32017-02-17 14:36:28 +0100441 return out
442}
443
Jakub Josef5ade54c2017-03-10 16:14:01 +0100444/**
Ales Komarek5276ebe2017-03-16 08:46:34 +0100445 * Get running minions IDs according to the target
446 * @param master Salt connection object
447 * @param target Get minions target
448 * @return list of active minions fitin
449 */
450def getMinions(master, target) {
Tomáš Kukrál18ac50f2017-07-13 10:22:09 +0200451 def minionsRaw = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'test.ping')
Ales Komarek5276ebe2017-03-16 08:46:34 +0100452 return new ArrayList<String>(minionsRaw['return'][0].keySet())
453}
454
455
456/**
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200457 * Test if there are any minions to target
458 * @param master Salt connection object
459 * @param target Target to test
vrovachev1c4770b2017-07-05 13:25:21 +0400460 * @return bool indicating if target was succesful
Tomáš Kukrálb12ff9f2017-07-12 12:32:34 +0200461 */
462
463def testTarget(master, target) {
464 return getMinions(master, target).size() > 0
465}
466
467/**
Jakub Josef5ade54c2017-03-10 16:14:01 +0100468 * Generates node key using key.gen_accept call
469 * @param master Salt connection object
470 * @param target Key generating target
471 * @param host Key generating host
472 * @param keysize generated key size (optional, default 4096)
473 * @return output of salt command
474 */
Jakub Josef79ecec32017-02-17 14:36:28 +0100475def generateNodeKey(master, target, host, keysize = 4096) {
Jakub Josef5ade54c2017-03-10 16:14:01 +0100476 return runSaltCommand(master, 'wheel', target, 'key.gen_accept', [host], ['keysize': keysize])
Jakub Josef79ecec32017-02-17 14:36:28 +0100477}
478
Jakub Josef5ade54c2017-03-10 16:14:01 +0100479/**
Jakub Josef2f25cf22017-03-28 13:34:57 +0200480 * Generates node reclass metadata
Jakub Josef5ade54c2017-03-10 16:14:01 +0100481 * @param master Salt connection object
482 * @param target Metadata generating target
483 * @param host Metadata generating host
484 * @param classes Reclass classes
485 * @param parameters Reclass parameters
486 * @return output of salt command
487 */
Jakub Josef79ecec32017-02-17 14:36:28 +0100488def generateNodeMetadata(master, target, host, classes, parameters) {
Jakub Josef5ade54c2017-03-10 16:14:01 +0100489 return runSaltCommand(master, 'local', target, 'reclass.node_create', [host, '_generated'], ['classes': classes, 'parameters': parameters])
Jakub Josef79ecec32017-02-17 14:36:28 +0100490}
491
Jakub Josef5ade54c2017-03-10 16:14:01 +0100492/**
493 * Run salt orchestrate on given targets
494 * @param master Salt connection object
495 * @param target Orchestration target
496 * @param orchestrate Salt orchestrate params
497 * @return output of salt command
498 */
Jakub Josef79ecec32017-02-17 14:36:28 +0100499def orchestrateSystem(master, target, orchestrate) {
500 return runSaltCommand(master, 'runner', target, 'state.orchestrate', [orchestrate])
501}
502
Jakub Josef5ade54c2017-03-10 16:14:01 +0100503/**
504 * Run salt process step
505 * @param master Salt connection object
506 * @param tgt Salt process step target
507 * @param fun Salt process step function
508 * @param arg process step arguments (optional, default [])
Jakub Josef2f25cf22017-03-28 13:34:57 +0200509 * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
Jakub Josef5ade54c2017-03-10 16:14:01 +0100510 * @param output print output (optional, default false)
Jiri Broulik48544be2017-06-14 18:33:54 +0200511 * @param timeout Additional argument salt api timeout
Jakub Josef5ade54c2017-03-10 16:14:01 +0100512 * @return output of salt command
513 */
Kalynovskyi19282122017-08-31 10:45:33 +0300514def runSaltProcessStep(master, tgt, fun, arg = [], batch = null, output = false, timeout = -1, kwargs = null) {
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100515 def common = new com.mirantis.mk.Common()
Jiri Broulik48544be2017-06-14 18:33:54 +0200516 def salt = new com.mirantis.mk.Salt()
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100517 def out
518
Marek Celoud63366112017-07-25 17:27:24 +0200519 common.infoMsg("Running step ${fun} ${arg} on ${tgt}")
Tomáš Kukrál6c04bd02017-03-01 22:18:52 +0100520
Filip Pytlounf0435c02017-03-02 17:48:54 +0100521 if (batch == true) {
Kalynovskyi19282122017-08-31 10:45:33 +0300522 out = runSaltCommand(master, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg, kwargs, timeout)
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100523 } else {
Kalynovskyi19282122017-08-31 10:45:33 +0300524 out = runSaltCommand(master, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg, kwargs, timeout)
Jakub Josef79ecec32017-02-17 14:36:28 +0100525 }
Tomáš Kukráladb4ecd2017-03-02 10:06:36 +0100526
Tomáš Kukrálf5dda642017-03-02 14:22:59 +0100527 if (output == true) {
Jiri Broulik48544be2017-06-14 18:33:54 +0200528 salt.printSaltCommandResult(out)
Jakub Josef79ecec32017-02-17 14:36:28 +0100529 }
Jiri Broulikae19c262017-05-16 19:06:52 +0200530 return out
Jakub Josef79ecec32017-02-17 14:36:28 +0100531}
532
533/**
534 * Check result for errors and throw exception if any found
535 *
536 * @param result Parsed response of Salt API
Jakub Josef8021c002017-03-27 15:41:28 +0200537 * @param failOnError Do you want to throw exception if salt-call fails (optional, default true)
Jakub Josefa87941c2017-04-20 17:14:58 +0200538 * @param printResults Do you want to print salt results (optional, default true)
Jakub Josefa87941c2017-04-20 17:14:58 +0200539 * @param printOnlyChanges If true (default), print only changed resources
Jakub Josef79ecec32017-02-17 14:36:28 +0100540 */
Jakub Josef374beb72017-04-27 15:45:09 +0200541def checkResult(result, failOnError = true, printResults = true, printOnlyChanges = true) {
Jakub Josef5ade54c2017-03-10 16:14:01 +0100542 def common = new com.mirantis.mk.Common()
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100543 if(result != null){
544 if(result['return']){
545 for (int i=0;i<result['return'].size();i++) {
546 def entry = result['return'][i]
547 if (!entry) {
548 if (failOnError) {
549 throw new Exception("Salt API returned empty response: ${result}")
550 } else {
551 common.errorMsg("Salt API returned empty response: ${result}")
Jakub Josefece32af2017-03-14 19:20:08 +0100552 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100553 }
554 for (int j=0;j<entry.size();j++) {
555 def nodeKey = entry.keySet()[j]
556 def node=entry[nodeKey]
Jakub Josefa87941c2017-04-20 17:14:58 +0200557 def outputResources = []
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100558 common.infoMsg("Node ${nodeKey} changes:")
559 if(node instanceof Map || node instanceof List){
560 for (int k=0;k<node.size();k++) {
561 def resource;
562 def resKey;
563 if(node instanceof Map){
564 resKey = node.keySet()[k]
565 }else if(node instanceof List){
566 resKey = k
567 }
568 resource = node[resKey]
Jakub Josefc4c40202017-04-28 12:04:24 +0200569 // print
Jakub Josefa87941c2017-04-20 17:14:58 +0200570 if(printResults){
571 if(resource instanceof Map && resource.keySet().contains("result")){
572 //clean unnesaccary fields
573 if(resource.keySet().contains("__run_num__")){
574 resource.remove("__run_num__")
575 }
576 if(resource.keySet().contains("__id__")){
577 resource.remove("__id__")
578 }
579 if(resource.keySet().contains("pchanges")){
580 resource.remove("pchanges")
581 }
582 if(!resource["result"] || (resource["result"] instanceof String && resource["result"] != "true")){
583 if(resource["result"] != null){
Jakub Josefbceaa322017-06-13 18:28:27 +0200584 outputResources.add(String.format("Resource: %s\n\u001B[31m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +0200585 }else{
Jakub Josefbceaa322017-06-13 18:28:27 +0200586 outputResources.add(String.format("Resource: %s\n\u001B[33m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +0200587 }
588 }else{
589 if(!printOnlyChanges || resource.changes.size() > 0){
Jakub Josefbceaa322017-06-13 18:28:27 +0200590 outputResources.add(String.format("Resource: %s\n\u001B[32m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +0200591 }
592 }
593 }else{
Jakub Josefbceaa322017-06-13 18:28:27 +0200594 outputResources.add(String.format("Resource: %s\n\u001B[36m%s\u001B[0m", resKey, common.prettify(resource)))
Jakub Josefa87941c2017-04-20 17:14:58 +0200595 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100596 }
Jakub Josefc4c40202017-04-28 12:04:24 +0200597 common.debugMsg("checkResult: checking resource: ${resource}")
598 if(resource instanceof String || (resource["result"] != null && !resource["result"]) || (resource["result"] instanceof String && resource["result"] == "false")){
Jakub Josefbceaa322017-06-13 18:28:27 +0200599 def prettyResource = common.prettify(resource)
Jakub Josefc4c40202017-04-28 12:04:24 +0200600 if(env["ASK_ON_ERROR"] && env["ASK_ON_ERROR"] == "true"){
601 timeout(time:1, unit:'HOURS') {
602 input message: "False result on ${nodeKey} found, resource ${prettyResource}. \nDo you want to continue?"
603 }
604 }else{
Jakub Josefd97d7db2017-05-11 19:11:53 +0200605 common.errorMsg(String.format("Resource: %s\n%s", resKey, prettyResource))
Jakub Josefd9001df2017-05-11 16:45:28 +0200606 def errorMsg = "Salt state on node ${nodeKey} failed: ${prettyResource}."
Jakub Josefc4c40202017-04-28 12:04:24 +0200607 if (failOnError) {
608 throw new Exception(errorMsg)
609 } else {
610 common.errorMsg(errorMsg)
611 }
612 }
613 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100614 }
Jakub Josefa87941c2017-04-20 17:14:58 +0200615 }else if(node!=null && node!=""){
Jakub Josef62f6c842017-08-04 16:36:35 +0200616 outputResources.add(String.format("Resource: %s\n\u001B[36m%s\u001B[0m", nodeKey, common.prettify(node)))
Jakub Josefa87941c2017-04-20 17:14:58 +0200617 }
618 if(printResults && !outputResources.isEmpty()){
Jakub Josefe6c562e2017-08-09 14:41:03 +0200619 print outputResources.stream().collect(Collectors.joining("\n"))
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100620 }
621 }
Jakub Josef52f69f72017-03-14 15:18:08 +0100622 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100623 }else{
624 common.errorMsg("Salt result hasn't return attribute! Result: ${result}")
Jakub Josef79ecec32017-02-17 14:36:28 +0100625 }
Jakub Josef52f69f72017-03-14 15:18:08 +0100626 }else{
Jakub Josefa87941c2017-04-20 17:14:58 +0200627 common.errorMsg("Cannot check salt result, given result is null")
Jakub Josef79ecec32017-02-17 14:36:28 +0100628 }
629}
630
631/**
Jakub Josef7852fe12017-03-15 16:02:41 +0100632 * Print salt command run results in human-friendly form
Jakub Josef79ecec32017-02-17 14:36:28 +0100633 *
634 * @param result Parsed response of Salt API
Jakub Josef79ecec32017-02-17 14:36:28 +0100635 */
Filip Pytlound2f1bbe2017-02-27 19:03:51 +0100636def printSaltCommandResult(result) {
Jakub Josef871bf152017-03-14 20:13:41 +0100637 def common = new com.mirantis.mk.Common()
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100638 if(result != null){
639 if(result['return']){
640 for (int i=0; i<result['return'].size(); i++) {
641 def entry = result['return'][i]
642 for (int j=0; j<entry.size(); j++) {
643 common.debugMsg("printSaltCommandResult: printing salt command entry: ${entry}")
644 def nodeKey = entry.keySet()[j]
645 def node=entry[nodeKey]
Jakub Josefbceaa322017-06-13 18:28:27 +0200646 common.infoMsg(String.format("Node %s changes:\n%s",nodeKey, common.prettify(node)))
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100647 }
Jakub Josef8a715bf2017-03-14 21:39:01 +0100648 }
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100649 }else{
650 common.errorMsg("Salt result hasn't return attribute! Result: ${result}")
Jakub Josef79ecec32017-02-17 14:36:28 +0100651 }
Jakub Josef8a715bf2017-03-14 21:39:01 +0100652 }else{
Jakub Josefd9afd0e2017-03-15 19:19:23 +0100653 common.errorMsg("Cannot print salt command result, given result is null")
Jakub Josef52f69f72017-03-14 15:18:08 +0100654 }
Jakub Josef79ecec32017-02-17 14:36:28 +0100655}
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +0200656
657
658/**
659 * Return content of file target
660 *
661 * @param master Salt master object
662 * @param target Compound target (should target only one host)
663 * @param file File path to read (/etc/hosts for example)
664 */
665
666def getFileContent(master, target, file) {
667 result = cmdRun(master, target, "cat ${file}")
Tomáš Kukrálf1a692a2017-08-11 13:29:28 +0200668 return result['return'][0].values()[0].replaceAll('Salt command execution success','')
Tomáš Kukrálb12eedd2017-04-21 10:45:13 +0200669}
Matthew Mosesohn9e880852017-07-04 21:17:53 +0300670
671/**
672 * Set override parameters in Salt cluster metadata
673 *
674 * @param master Salt master object
675 * @param salt_overrides YAML formatted string containing key: value, one per line
Matthew Mosesohne5646842017-07-19 16:54:57 +0300676 * @param reclass_dir Directory where Reclass git repo is located
Matthew Mosesohn9e880852017-07-04 21:17:53 +0300677 */
678
Matthew Mosesohne5646842017-07-19 16:54:57 +0300679def setSaltOverrides(master, salt_overrides, reclass_dir="/srv/salt/reclass") {
Tomáš Kukrálf178f052017-07-11 11:31:00 +0200680 def common = new com.mirantis.mk.Common()
Mykyta Karpin1c165e22017-08-22 18:27:01 +0300681 def salt_overrides_map = readYaml text: salt_overrides
Tomáš Kukrál243cf842017-07-11 13:11:56 +0200682 for (entry in common.entries(salt_overrides_map)) {
Matthew Mosesohn9e880852017-07-04 21:17:53 +0300683 def key = entry[0]
684 def value = entry[1]
685
686 common.debugMsg("Set salt override ${key}=${value}")
Matthew Mosesohn29c9e332017-07-25 14:00:17 +0300687 runSaltProcessStep(master, 'I@salt:master', 'reclass.cluster_meta_set', ["${key}", "${value}"], false)
Matthew Mosesohn9e880852017-07-04 21:17:53 +0300688 }
Matthew Mosesohne5646842017-07-19 16:54:57 +0300689 runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ["git -C ${reclass_dir} update-index --skip-worktree classes/cluster/overrides.yml"])
Matthew Mosesohn9e880852017-07-04 21:17:53 +0300690}
Oleg Grigorovbec45582017-09-12 20:29:24 +0300691
692/**
693* Execute salt commands via salt-api with
694* CLI client salt-pepper
695*
696* @param data Salt command map
697* @param venv Path to virtualenv with
698*/
699
700def runPepperCommand(data, venv) {
701 def python = new com.mirantis.mk.Python()
702 def dataStr = new groovy.json.JsonBuilder(data).toString()
703
704 def pepperCmd = "pepper -c ${venv}/pepperrc --make-token --json \'${dataStr}\'"
705
706 if (venv) {
707 output = python.runVirtualenvCommand(venv, pepperCmd)
708 } else {
709 echo("[Command]: ${pepperCmd}")
710 output = sh (
711 script: pepperCmd,
712 returnStdout: true
713 ).trim()
714 }
715
716 return new groovy.json.JsonSlurperClassic().parseText(output)
717}
718
719/**
720* Create config file for pepper
721*
722* @param url SALT_MASTER URL
723* @param credentialsId credentials to SALT_API
724* @param path path to pepper env
725*/
726def createPepperEnv(url, credentialsId, path) {
727 def common = new com.mirantis.mk.Common()
728 rcFile = "${path}/pepperrc"
729 creds = common.getPasswordCredentials(credentialsId)
730 rc = """\
731[main]
732SALTAPI_EAUTH=pam
733SALTAPI_URL=${url}
734SALTAPI_USER=${creds.username}
735SALTAPI_PASS=${creds.password.toString()}
736"""
737 writeFile file: rcFile, text: rc
738 return rcFile
739}