Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2Module to handle interaction with salt 

3""" 

4import json 

5import os 

6import time 

7 

8from cfg_checker.common import config, logger, logger_cli 

9from cfg_checker.common.exception import InvalidReturnException, SaltException 

10from cfg_checker.common.other import shell 

11 

12import requests 

13 

14 

15def _extract_password(_raw): 

16 if not isinstance(_raw, str): 

17 raise InvalidReturnException(_raw) 

18 else: 

19 try: 

20 _json = json.loads(_raw) 

21 except ValueError: 

22 raise SaltException( 

23 "# Return value is not a json: '{}'".format(_raw) 

24 ) 

25 

26 return _json["local"] 

27 

28 

29def get_remote_env_password(): 

30 """Uses ssh call with configured options to get password from salt master 

31 

32 :return: password string 

33 """ 

34 _salt_cmd = "salt-call --out=json pillar.get _param:salt_api_password" 

35 _ssh_cmd = ["ssh"] 

36 # Build SSH cmd 

37 if config.ssh_key: 

38 _ssh_cmd.append("-i " + config.ssh_key) 

39 if config.ssh_user: 

40 _ssh_cmd.append(config.ssh_user+'@'+config.ssh_host) 

41 else: 

42 _ssh_cmd.append(config.ssh_host) 

43 if config.ssh_uses_sudo: 

44 _ssh_cmd.append("sudo") 

45 

46 _ssh_cmd.append(_salt_cmd) 

47 _ssh_cmd = " ".join(_ssh_cmd) 

48 logger_cli.debug("... calling salt: '{}'".format(_ssh_cmd)) 

49 try: 

50 _result = shell(_ssh_cmd) 

51 if len(_result) < 1: 

52 raise InvalidReturnException( 

53 "# Empty value returned for '{}".format( 

54 _ssh_cmd 

55 ) 

56 ) 

57 else: 

58 return _extract_password(_result) 

59 except OSError as e: 

60 raise SaltException( 

61 "Salt error calling '{}': '{}'\n" 

62 "\nConsider checking 'SALT_ENV' " 

63 "and '<pkg>/etc/<env>.env' files".format(_ssh_cmd, e.strerror) 

64 ) 

65 

66 

67def get_local_password(): 

68 """Calls salt locally to get password from the pillar 

69 

70 :return: password string 

71 """ 

72 _cmd = "salt-call" 

73 _args = "--out=json pillar.get _param:salt_api_password" 

74 try: 

75 _result = shell(" ".join([_cmd, _args])) 

76 except OSError as e: 

77 raise SaltException( 

78 "Salt error calling '{}': '{}'\n" 

79 "\nConsider checking 'SALT_ENV' " 

80 "and '<pkg>/etc/<env>.env' files".format(_cmd, e.strerror) 

81 ) 

82 return _extract_password(_result) 

83 

84 

85def list_to_target_string(node_list, separator): 

86 result = '' 

87 for node in node_list: 

88 result += node + ' ' + separator + ' ' 

89 return result[:-(len(separator)+2)] 

90 

91 

92class SaltRest(object): 

93 _host = config.salt_host 

94 _port = config.salt_port 

95 uri = "http://" + config.salt_host + ":" + config.salt_port 

96 _auth = {} 

97 

98 default_headers = { 

99 'Accept': 'application/json', 

100 'Content-Type': 'application/json', 

101 'X-Auth-Token': None 

102 } 

103 

104 def __init__(self): 

105 self._token = self._login() 

106 self.last_response = None 

107 

108 def get( 

109 self, 

110 path='', 

111 headers=default_headers, 

112 cookies=None, 

113 timeout=None 

114 ): 

115 _path = os.path.join(self.uri, path) 

116 logger.debug("# GET '{}'\nHeaders: '{}'\nCookies: {}".format( 

117 _path, 

118 headers, 

119 cookies 

120 )) 

121 return requests.get( 

122 _path, 

123 headers=headers, 

124 cookies=cookies, 

125 timeout=timeout 

126 ) 

127 

128 def post(self, data, path='', headers=default_headers, cookies=None): 

129 if data is None: 

130 data = {} 

131 _path = os.path.join(self.uri, path) 

132 if path == 'login': 

133 _data = str(data).replace(self._pass, "*****") 

134 else: 

135 _data = data 

136 logger.debug( 

137 "# POST '{}'\nHeaders: '{}'\nCookies: {}\nBody: {}".format( 

138 _path, 

139 headers, 

140 cookies, 

141 _data 

142 ) 

143 ) 

144 return requests.post( 

145 os.path.join(self.uri, path), 

146 headers=headers, 

147 json=data, 

148 cookies=cookies 

149 ) 

150 

151 def _login(self): 

152 # if there is no password - try to get local, if this available 

153 if config.salt_env == "local": 

154 _pass = get_local_password() 

155 else: 

156 _pass = get_remote_env_password() 

157 login_payload = { 

158 'username': config.salt_user, 

159 'password': _pass, 

160 'eauth': 'pam' 

161 } 

162 self._pass = _pass 

163 logger.debug("# Logging in to salt master...") 

164 _response = self.post(login_payload, path='login') 

165 

166 if _response.ok: 

167 self._auth['response'] = _response.json()['return'][0] 

168 self._auth['cookies'] = _response.cookies 

169 self.default_headers['X-Auth-Token'] = \ 

170 self._auth['response']['token'] 

171 return self._auth['response']['token'] 

172 else: 

173 raise EnvironmentError( 

174 "# HTTP:{}, Not authorized?".format(_response.status_code) 

175 ) 

176 

177 def salt_request(self, fn, *args, **kwargs): 

178 # if token will expire in 5 min, re-login 

179 if self._auth['response']['expire'] < time.time() + 300: 

180 self._auth['response']['X-Auth-Token'] = self._login() 

181 

182 _method = getattr(self, fn) 

183 _response = _method(*args, **kwargs) 

184 self.last_response = _response 

185 _content = "..." 

186 _len = len(_response.content) 

187 if _len < 1024: 

188 _content = _response.content 

189 logger.debug( 

190 "# Response (HTTP {}/{}), {}: {}".format( 

191 _response.status_code, 

192 _response.reason, 

193 _len, 

194 _content 

195 ) 

196 ) 

197 if _response.ok: 

198 return _response.json()['return'] 

199 else: 

200 raise EnvironmentError( 

201 "# Salt Error: HTTP:{}, '{}'".format( 

202 _response.status_code, 

203 _response.reason 

204 ) 

205 ) 

206 

207 

208class SaltRemote(SaltRest): 

209 master_node = "" 

210 

211 def __init__(self): 

212 super(SaltRemote, self).__init__() 

213 

214 def cmd( 

215 self, 

216 tgt, 

217 fun, 

218 param=None, 

219 client='local', 

220 kwarg=None, 

221 expr_form=None, 

222 tgt_type=None, 

223 timeout=None 

224 ): 

225 _timeout = timeout if timeout is not None else config.salt_timeout 

226 _payload = { 

227 'fun': fun, 

228 'tgt': tgt, 

229 'client': client, 

230 'timeout': _timeout 

231 } 

232 

233 if expr_form: 

234 _payload['expr_form'] = expr_form 

235 if tgt_type: 

236 _payload['tgt_type'] = tgt_type 

237 if param: 

238 _payload['arg'] = param 

239 if kwarg: 

240 _payload['kwarg'] = kwarg 

241 

242 _response = self.salt_request('post', [_payload]) 

243 if isinstance(_response, list): 

244 return _response[0] 

245 else: 

246 raise EnvironmentError( 

247 "# Unexpected response from from salt-api/LocalClient: " 

248 "{}".format(_response) 

249 ) 

250 

251 def run(self, fun, kwarg=None): 

252 _payload = { 

253 'client': 'runner', 

254 'fun': fun, 

255 'timeout': config.salt_timeout 

256 } 

257 

258 if kwarg: 

259 _payload['kwarg'] = kwarg 

260 

261 _response = self.salt_request('post', [_payload]) 

262 if isinstance(_response, list): 

263 return _response[0] 

264 else: 

265 raise EnvironmentError( 

266 "# Unexpected response from from salt-api/RunnerClient: " 

267 "{}".format(_response) 

268 ) 

269 

270 def wheel(self, fun, arg=None, kwarg=None): 

271 _payload = { 

272 'client': 'wheel', 

273 'fun': fun, 

274 'timeout': config.salt_timeout 

275 } 

276 

277 if arg: 

278 _payload['arg'] = arg 

279 if kwarg: 

280 _payload['kwarg'] = kwarg 

281 

282 _response = self.salt_request('post', _payload)['data'] 

283 if _response['success']: 

284 return _response 

285 else: 

286 raise EnvironmentError( 

287 "# Salt Error: '{}'".format(_response['return'])) 

288 

289 def pillar_request(self, node_target, pillar_submodule, argument): 

290 # example cli: 'salt "ctl01*" pillar.keys rsyslog' 

291 _type = "compound" 

292 if isinstance(node_target, list): 

293 _type = "list" 

294 return self.cmd( 

295 node_target, 

296 "pillar." + pillar_submodule, 

297 argument, 

298 expr_form=_type 

299 ) 

300 

301 def pillar_keys(self, node_target, argument): 

302 return self.pillar_request(node_target, 'keys', argument) 

303 

304 def pillar_get(self, node_target, argument): 

305 return self.pillar_request(node_target, 'get', argument) 

306 

307 def pillar_data(self, node_target, argument): 

308 return self.pillar_request(node_target, 'data', argument) 

309 

310 def pillar_raw(self, node_target, argument): 

311 return self.pillar_request(node_target, 'raw', argument) 

312 

313 def list_minions(self): 

314 """ 

315 Fails in salt version 2016.3.8 

316 Works starting from 2017.7.7 

317 api returns dict of minions with grains 

318 """ 

319 try: 

320 _r = self.salt_request('get', 'minions', timeout=10) 

321 except requests.exceptions.ReadTimeout: 

322 logger_cli.debug("... timeout waiting list minions from Salt API") 

323 _r = None 

324 return _r[0] if _r else None 

325 

326 def list_keys(self): 

327 """ 

328 Fails in salt version 2016.3.8 

329 Works starting from 2017.7.7 

330 api should return dict: 

331 { 

332 'local': [], 

333 'minions': [], 

334 'minions_denied': [], 

335 'minions_pre': [], 

336 'minions_rejected': [], 

337 } 

338 """ 

339 return self.salt_request('get', path='keys') 

340 

341 def get_status(self): 

342 """ 

343 Fails in salt version 2017.7.7 

344 'runner' client is the equivalent of 'salt-run' 

345 Returns the 

346 """ 

347 return self.run( 

348 'manage.status', 

349 kwarg={'timeout': 10} 

350 ) 

351 

352 def get_active_nodes(self): 

353 """Used when other minion list metods fail 

354 

355 :return: json result from salt test.ping 

356 """ 

357 if config.skip_nodes: 

358 logger.info("# Nodes to be skipped: {0}".format(config.skip_nodes)) 

359 _r = self.cmd( 

360 '* and not ' + list_to_target_string( 

361 config.skip_nodes, 

362 'and not' 

363 ), 

364 'test.ping', 

365 expr_form='compound') 

366 else: 

367 _r = self.cmd('*', 'test.ping') 

368 # Return all nodes that responded 

369 return [node for node in _r.keys() if _r[node]] 

370 

371 def get_monitoring_ip(self, param_name): 

372 salt_output = self.cmd( 

373 'docker:client:stack:monitoring', 

374 'pillar.get', 

375 param=param_name, 

376 expr_form='pillar') 

377 return salt_output[salt_output.keys()[0]] 

378 

379 def f_touch_master(self, path, makedirs=True): 

380 _kwarg = { 

381 "makedirs": makedirs 

382 } 

383 salt_output = self.cmd( 

384 self.master_node, 

385 "file.touch", 

386 param=path, 

387 kwarg=_kwarg 

388 ) 

389 return [*salt_output.values()][0] 

390 

391 def f_append_master(self, path, strings_list, makedirs=True): 

392 _kwarg = { 

393 "makedirs": makedirs 

394 } 

395 _args = [path] 

396 _args.extend(strings_list) 

397 salt_output = self.cmd( 

398 self.master_node, 

399 "file.write", 

400 param=_args, 

401 kwarg=_kwarg 

402 ) 

403 return [*salt_output.values()][0] 

404 

405 def mkdir(self, target, path, tgt_type=None): 

406 salt_output = self.cmd( 

407 target, 

408 "file.mkdir", 

409 param=path, 

410 expr_form=tgt_type 

411 ) 

412 return salt_output 

413 

414 def f_manage_file(self, target_path, source, 

415 sfn='', ret='{}', 

416 source_hash={}, 

417 user='root', group='root', backup_mode='755', 

418 show_diff='base', 

419 contents='', makedirs=True): 

420 """ 

421 REST variation of file.get_managed 

422 CLI execution goes like this (10 agrs): 

423 salt cfg01\\* file.manage_file /root/test_scripts/pkg_versions.py 

424 '' '{}' /root/diff_pkg_version.py 

425 '{hash_type: 'md5', 'hsum': <md5sum>}' root root '755' base '' 

426 makedirs=True 

427 param: name - target file placement when managed 

428 param: source - source for the file 

429 """ 

430 _source_hash = { 

431 "hash_type": "md5", 

432 "hsum": 000 

433 } 

434 _arg = [ 

435 target_path, 

436 sfn, 

437 ret, 

438 source, 

439 _source_hash, 

440 user, 

441 group, 

442 backup_mode, 

443 show_diff, 

444 contents 

445 ] 

446 _kwarg = { 

447 "makedirs": makedirs 

448 } 

449 salt_output = self.cmd( 

450 self.master_node, 

451 "file.manage_file", 

452 param=_arg, 

453 kwarg=_kwarg 

454 ) 

455 return [*salt_output.values()][0] 

456 

457 def cache_file(self, target, source_path): 

458 salt_output = self.cmd( 

459 target, 

460 "cp.cache_file", 

461 param=source_path 

462 ) 

463 return [*salt_output.values()][0] 

464 

465 def get_file(self, target, source_path, target_path, tgt_type=None): 

466 return self.cmd( 

467 target, 

468 "cp.get_file", 

469 param=[source_path, target_path], 

470 expr_form=tgt_type 

471 ) 

472 

473 @staticmethod 

474 def compound_string_from_list(nodes_list): 

475 return " or ".join(nodes_list)