Coverage for cfg_checker/common/salt_utils.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Module to handle interaction with salt
3"""
4import json
5import os
6import time
8from cfg_checker.common import config, logger, logger_cli
9from cfg_checker.common.exception import InvalidReturnException, SaltException
10from cfg_checker.common.other import shell
12import requests
15def _extract_password(_raw):
16 if not isinstance(_raw, str):
17 raise InvalidReturnException(_raw)
18 else:
19 try:
20 _json = json.loads(_raw)
21 except ValueError:
22 raise SaltException(
23 "# Return value is not a json: '{}'".format(_raw)
24 )
26 return _json["local"]
29def get_remote_env_password():
30 """Uses ssh call with configured options to get password from salt master
32 :return: password string
33 """
34 _salt_cmd = "salt-call --out=json pillar.get _param:salt_api_password"
35 _ssh_cmd = ["ssh"]
36 # Build SSH cmd
37 if config.ssh_key:
38 _ssh_cmd.append("-i " + config.ssh_key)
39 if config.ssh_user:
40 _ssh_cmd.append(config.ssh_user+'@'+config.ssh_host)
41 else:
42 _ssh_cmd.append(config.ssh_host)
43 if config.ssh_uses_sudo:
44 _ssh_cmd.append("sudo")
46 _ssh_cmd.append(_salt_cmd)
47 _ssh_cmd = " ".join(_ssh_cmd)
48 logger_cli.debug("... calling salt: '{}'".format(_ssh_cmd))
49 try:
50 _result = shell(_ssh_cmd)
51 if len(_result) < 1:
52 raise InvalidReturnException(
53 "# Empty value returned for '{}".format(
54 _ssh_cmd
55 )
56 )
57 else:
58 return _extract_password(_result)
59 except OSError as e:
60 raise SaltException(
61 "Salt error calling '{}': '{}'\n"
62 "\nConsider checking 'SALT_ENV' "
63 "and '<pkg>/etc/<env>.env' files".format(_ssh_cmd, e.strerror)
64 )
67def get_local_password():
68 """Calls salt locally to get password from the pillar
70 :return: password string
71 """
72 _cmd = "salt-call"
73 _args = "--out=json pillar.get _param:salt_api_password"
74 try:
75 _result = shell(" ".join([_cmd, _args]))
76 except OSError as e:
77 raise SaltException(
78 "Salt error calling '{}': '{}'\n"
79 "\nConsider checking 'SALT_ENV' "
80 "and '<pkg>/etc/<env>.env' files".format(_cmd, e.strerror)
81 )
82 return _extract_password(_result)
85def list_to_target_string(node_list, separator):
86 result = ''
87 for node in node_list:
88 result += node + ' ' + separator + ' '
89 return result[:-(len(separator)+2)]
92class SaltRest(object):
93 _host = config.salt_host
94 _port = config.salt_port
95 uri = "http://" + config.salt_host + ":" + config.salt_port
96 _auth = {}
98 default_headers = {
99 'Accept': 'application/json',
100 'Content-Type': 'application/json',
101 'X-Auth-Token': None
102 }
104 def __init__(self):
105 self._token = self._login()
106 self.last_response = None
108 def get(
109 self,
110 path='',
111 headers=default_headers,
112 cookies=None,
113 timeout=None
114 ):
115 _path = os.path.join(self.uri, path)
116 logger.debug("# GET '{}'\nHeaders: '{}'\nCookies: {}".format(
117 _path,
118 headers,
119 cookies
120 ))
121 return requests.get(
122 _path,
123 headers=headers,
124 cookies=cookies,
125 timeout=timeout
126 )
128 def post(self, data, path='', headers=default_headers, cookies=None):
129 if data is None:
130 data = {}
131 _path = os.path.join(self.uri, path)
132 if path == 'login':
133 _data = str(data).replace(self._pass, "*****")
134 else:
135 _data = data
136 logger.debug(
137 "# POST '{}'\nHeaders: '{}'\nCookies: {}\nBody: {}".format(
138 _path,
139 headers,
140 cookies,
141 _data
142 )
143 )
144 return requests.post(
145 os.path.join(self.uri, path),
146 headers=headers,
147 json=data,
148 cookies=cookies
149 )
151 def _login(self):
152 # if there is no password - try to get local, if this available
153 if config.salt_env == "local":
154 _pass = get_local_password()
155 else:
156 _pass = get_remote_env_password()
157 login_payload = {
158 'username': config.salt_user,
159 'password': _pass,
160 'eauth': 'pam'
161 }
162 self._pass = _pass
163 logger.debug("# Logging in to salt master...")
164 _response = self.post(login_payload, path='login')
166 if _response.ok:
167 self._auth['response'] = _response.json()['return'][0]
168 self._auth['cookies'] = _response.cookies
169 self.default_headers['X-Auth-Token'] = \
170 self._auth['response']['token']
171 return self._auth['response']['token']
172 else:
173 raise EnvironmentError(
174 "# HTTP:{}, Not authorized?".format(_response.status_code)
175 )
177 def salt_request(self, fn, *args, **kwargs):
178 # if token will expire in 5 min, re-login
179 if self._auth['response']['expire'] < time.time() + 300:
180 self._auth['response']['X-Auth-Token'] = self._login()
182 _method = getattr(self, fn)
183 _response = _method(*args, **kwargs)
184 self.last_response = _response
185 _content = "..."
186 _len = len(_response.content)
187 if _len < 1024:
188 _content = _response.content
189 logger.debug(
190 "# Response (HTTP {}/{}), {}: {}".format(
191 _response.status_code,
192 _response.reason,
193 _len,
194 _content
195 )
196 )
197 if _response.ok:
198 return _response.json()['return']
199 else:
200 raise EnvironmentError(
201 "# Salt Error: HTTP:{}, '{}'".format(
202 _response.status_code,
203 _response.reason
204 )
205 )
208class SaltRemote(SaltRest):
209 master_node = ""
211 def __init__(self):
212 super(SaltRemote, self).__init__()
214 def cmd(
215 self,
216 tgt,
217 fun,
218 param=None,
219 client='local',
220 kwarg=None,
221 expr_form=None,
222 tgt_type=None,
223 timeout=None
224 ):
225 _timeout = timeout if timeout is not None else config.salt_timeout
226 _payload = {
227 'fun': fun,
228 'tgt': tgt,
229 'client': client,
230 'timeout': _timeout
231 }
233 if expr_form:
234 _payload['expr_form'] = expr_form
235 if tgt_type:
236 _payload['tgt_type'] = tgt_type
237 if param:
238 _payload['arg'] = param
239 if kwarg:
240 _payload['kwarg'] = kwarg
242 _response = self.salt_request('post', [_payload])
243 if isinstance(_response, list):
244 return _response[0]
245 else:
246 raise EnvironmentError(
247 "# Unexpected response from from salt-api/LocalClient: "
248 "{}".format(_response)
249 )
251 def run(self, fun, kwarg=None):
252 _payload = {
253 'client': 'runner',
254 'fun': fun,
255 'timeout': config.salt_timeout
256 }
258 if kwarg:
259 _payload['kwarg'] = kwarg
261 _response = self.salt_request('post', [_payload])
262 if isinstance(_response, list):
263 return _response[0]
264 else:
265 raise EnvironmentError(
266 "# Unexpected response from from salt-api/RunnerClient: "
267 "{}".format(_response)
268 )
270 def wheel(self, fun, arg=None, kwarg=None):
271 _payload = {
272 'client': 'wheel',
273 'fun': fun,
274 'timeout': config.salt_timeout
275 }
277 if arg:
278 _payload['arg'] = arg
279 if kwarg:
280 _payload['kwarg'] = kwarg
282 _response = self.salt_request('post', _payload)['data']
283 if _response['success']:
284 return _response
285 else:
286 raise EnvironmentError(
287 "# Salt Error: '{}'".format(_response['return']))
289 def pillar_request(self, node_target, pillar_submodule, argument):
290 # example cli: 'salt "ctl01*" pillar.keys rsyslog'
291 _type = "compound"
292 if isinstance(node_target, list):
293 _type = "list"
294 return self.cmd(
295 node_target,
296 "pillar." + pillar_submodule,
297 argument,
298 expr_form=_type
299 )
301 def pillar_keys(self, node_target, argument):
302 return self.pillar_request(node_target, 'keys', argument)
304 def pillar_get(self, node_target, argument):
305 return self.pillar_request(node_target, 'get', argument)
307 def pillar_data(self, node_target, argument):
308 return self.pillar_request(node_target, 'data', argument)
310 def pillar_raw(self, node_target, argument):
311 return self.pillar_request(node_target, 'raw', argument)
313 def list_minions(self):
314 """
315 Fails in salt version 2016.3.8
316 Works starting from 2017.7.7
317 api returns dict of minions with grains
318 """
319 try:
320 _r = self.salt_request('get', 'minions', timeout=10)
321 except requests.exceptions.ReadTimeout:
322 logger_cli.debug("... timeout waiting list minions from Salt API")
323 _r = None
324 return _r[0] if _r else None
326 def list_keys(self):
327 """
328 Fails in salt version 2016.3.8
329 Works starting from 2017.7.7
330 api should return dict:
331 {
332 'local': [],
333 'minions': [],
334 'minions_denied': [],
335 'minions_pre': [],
336 'minions_rejected': [],
337 }
338 """
339 return self.salt_request('get', path='keys')
341 def get_status(self):
342 """
343 Fails in salt version 2017.7.7
344 'runner' client is the equivalent of 'salt-run'
345 Returns the
346 """
347 return self.run(
348 'manage.status',
349 kwarg={'timeout': 10}
350 )
352 def get_active_nodes(self):
353 """Used when other minion list metods fail
355 :return: json result from salt test.ping
356 """
357 if config.skip_nodes:
358 logger.info("# Nodes to be skipped: {0}".format(config.skip_nodes))
359 _r = self.cmd(
360 '* and not ' + list_to_target_string(
361 config.skip_nodes,
362 'and not'
363 ),
364 'test.ping',
365 expr_form='compound')
366 else:
367 _r = self.cmd('*', 'test.ping')
368 # Return all nodes that responded
369 return [node for node in _r.keys() if _r[node]]
371 def get_monitoring_ip(self, param_name):
372 salt_output = self.cmd(
373 'docker:client:stack:monitoring',
374 'pillar.get',
375 param=param_name,
376 expr_form='pillar')
377 return salt_output[salt_output.keys()[0]]
379 def f_touch_master(self, path, makedirs=True):
380 _kwarg = {
381 "makedirs": makedirs
382 }
383 salt_output = self.cmd(
384 self.master_node,
385 "file.touch",
386 param=path,
387 kwarg=_kwarg
388 )
389 return [*salt_output.values()][0]
391 def f_append_master(self, path, strings_list, makedirs=True):
392 _kwarg = {
393 "makedirs": makedirs
394 }
395 _args = [path]
396 _args.extend(strings_list)
397 salt_output = self.cmd(
398 self.master_node,
399 "file.write",
400 param=_args,
401 kwarg=_kwarg
402 )
403 return [*salt_output.values()][0]
405 def mkdir(self, target, path, tgt_type=None):
406 salt_output = self.cmd(
407 target,
408 "file.mkdir",
409 param=path,
410 expr_form=tgt_type
411 )
412 return salt_output
414 def f_manage_file(self, target_path, source,
415 sfn='', ret='{}',
416 source_hash={},
417 user='root', group='root', backup_mode='755',
418 show_diff='base',
419 contents='', makedirs=True):
420 """
421 REST variation of file.get_managed
422 CLI execution goes like this (10 agrs):
423 salt cfg01\\* file.manage_file /root/test_scripts/pkg_versions.py
424 '' '{}' /root/diff_pkg_version.py
425 '{hash_type: 'md5', 'hsum': <md5sum>}' root root '755' base ''
426 makedirs=True
427 param: name - target file placement when managed
428 param: source - source for the file
429 """
430 _source_hash = {
431 "hash_type": "md5",
432 "hsum": 000
433 }
434 _arg = [
435 target_path,
436 sfn,
437 ret,
438 source,
439 _source_hash,
440 user,
441 group,
442 backup_mode,
443 show_diff,
444 contents
445 ]
446 _kwarg = {
447 "makedirs": makedirs
448 }
449 salt_output = self.cmd(
450 self.master_node,
451 "file.manage_file",
452 param=_arg,
453 kwarg=_kwarg
454 )
455 return [*salt_output.values()][0]
457 def cache_file(self, target, source_path):
458 salt_output = self.cmd(
459 target,
460 "cp.cache_file",
461 param=source_path
462 )
463 return [*salt_output.values()][0]
465 def get_file(self, target, source_path, target_path, tgt_type=None):
466 return self.cmd(
467 target,
468 "cp.get_file",
469 param=[source_path, target_path],
470 expr_form=tgt_type
471 )
473 @staticmethod
474 def compound_string_from_list(nodes_list):
475 return " or ".join(nodes_list)