Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import json 

2 

3from cfg_checker.common import const, logger_cli 

4from cfg_checker.common.exception import ConfigException 

5from cfg_checker.common.other import merge_dict 

6from cfg_checker.helpers.console_utils import Progress 

7from cfg_checker.modules.packages.repos import RepoManager 

8from cfg_checker.nodes import salt_master 

9from cfg_checker.reports import reporter 

10 

11from .versions import DebianVersion, PkgVersions, VersionCmpResult 

12 

13 

14class CloudPackageChecker(object): 

15 def __init__(self, force_tag=None, exclude_keywords=[]): 

16 # Init salt master info 

17 if not salt_master.nodes: 

18 salt_master.nodes = salt_master.get_nodes() 

19 

20 # check that this env tag is present in Manager 

21 self.rm = RepoManager() 

22 _tags = self.rm.get_available_tags(tag=salt_master.mcp_release) 

23 if not _tags: 

24 logger_cli.warning( 

25 "\n# hWARNING: '{0}' is not listed in repo index. " 

26 "Consider running:\n\t{1}\nto add info on this tag's " 

27 "release package versions".format( 

28 salt_master.mcp_release, 

29 "mcp-checker packages versions --tag {0}" 

30 ) 

31 ) 

32 

33 self.force_tag = force_tag 

34 self.exclude_keywords = exclude_keywords 

35 

36 @staticmethod 

37 def presort_packages(all_packages, full=None): 

38 logger_cli.info("-> Presorting packages") 

39 # labels 

40 _data = {} 

41 _data = { 

42 "cs": { 

43 "ok": const.VERSION_OK, 

44 "up": const.VERSION_UP, 

45 "down": const.VERSION_DOWN, 

46 "warn": const.VERSION_WARN, 

47 "err": const.VERSION_ERR 

48 }, 

49 "ca": { 

50 "na": const.ACT_NA, 

51 "up": const.ACT_UPGRADE, 

52 "need_up": const.ACT_NEED_UP, 

53 "need_down": const.ACT_NEED_DOWN, 

54 "repo": const.ACT_REPO 

55 } 

56 } 

57 _data['status_err'] = const.VERSION_ERR 

58 _data['status_warn'] = const.VERSION_WARN 

59 _data['status_down'] = const.VERSION_DOWN 

60 

61 # Presort packages 

62 _data['critical'] = {} 

63 _data['system'] = {} 

64 _data['other'] = {} 

65 _data['unlisted'] = {} 

66 

67 _l = len(all_packages) 

68 _progress = Progress(_l) 

69 _progress_index = 0 

70 # counters 

71 _ec = _es = _eo = _eu = 0 

72 _wc = _ws = _wo = _wu = 0 

73 _dc = _ds = _do = _du = 0 

74 while _progress_index < _l: 

75 # progress bar 

76 _progress_index += 1 

77 _progress.write_progress(_progress_index) 

78 # sort packages 

79 _pn, _val = all_packages.popitem() 

80 _c = _val['desc']['section'] 

81 

82 if not full: 

83 # Check if this packet has errors 

84 # if all is ok -> just skip it 

85 _max_status = max(_val['results'].keys()) 

86 if _max_status <= const.VERSION_OK: 

87 _max_action = max(_val['results'][_max_status].keys()) 

88 if _max_action == const.ACT_NA: 

89 # this package does not have any comments 

90 # ...just skip it from report 

91 continue 

92 

93 _differ = len(set(_val['results'].keys())) > 1 

94 if _differ: 

95 # in case package has different status across nodes 

96 # Warning becomes Error. 

97 if const.VERSION_WARN in _val['results']: 

98 if const.VERSION_ERR in _val['results']: 

99 # add warns to err 

100 # should never happen, though 

101 merge_dict( 

102 _val['results'].pop(const.VERSION_WARN), 

103 _val['results'][const.VERSION_ERR] 

104 ) 

105 else: 

106 _val['results'][const.VERSION_ERR] = \ 

107 _val['results'].pop(const.VERSION_WARN) 

108 else: 

109 # in case package has same status on all nodes 

110 # Error becomes Warning 

111 if const.VERSION_ERR in _val['results']: 

112 if const.VERSION_WARN in _val['results']: 

113 # add warns to err 

114 # should never happen, though 

115 merge_dict( 

116 _val['results'].pop(const.VERSION_ERR), 

117 _val['results'][const.VERSION_WARN] 

118 ) 

119 else: 

120 _val['results'][const.VERSION_WARN] = \ 

121 _val['results'].pop(const.VERSION_ERR) 

122 

123 if len(_c) > 0 and _val['is_mirantis'] is None: 

124 # not listed package in version lib 

125 _data['unlisted'].update({ 

126 _pn: _val 

127 }) 

128 _eu += _val['results'].keys().count(const.VERSION_ERR) 

129 _wu += _val['results'].keys().count(const.VERSION_WARN) 

130 _du += _val['results'].keys().count(const.VERSION_DOWN) 

131 # mirantis/critical 

132 # elif len(_c) > 0 and _c != 'System': 

133 elif _val['is_mirantis']: 

134 # not blank and not system 

135 _data['critical'].update({ 

136 _pn: _val 

137 }) 

138 _ec += _val['results'].keys().count(const.VERSION_ERR) 

139 _wc += _val['results'].keys().count(const.VERSION_WARN) 

140 _dc += _val['results'].keys().count(const.VERSION_DOWN) 

141 # system 

142 elif _c == 'System': 

143 _data['system'].update({ 

144 _pn: _val 

145 }) 

146 _es += _val['results'].keys().count(const.VERSION_ERR) 

147 _ws += _val['results'].keys().count(const.VERSION_WARN) 

148 _ds += _val['results'].keys().count(const.VERSION_DOWN) 

149 # rest 

150 else: 

151 _data['other'].update({ 

152 _pn: _val 

153 }) 

154 _eo += _val['results'].keys().count(const.VERSION_ERR) 

155 _wo += _val['results'].keys().count(const.VERSION_WARN) 

156 _do += _val['results'].keys().count(const.VERSION_DOWN) 

157 

158 _progress.end() 

159 

160 _data['errors'] = { 

161 'mirantis': _ec, 

162 'system': _es, 

163 'other': _eo, 

164 'unlisted': _eu 

165 } 

166 _data['warnings'] = { 

167 'mirantis': _wc, 

168 'system': _ws, 

169 'other': _wo, 

170 'unlisted': _wu 

171 } 

172 _data['downgrades'] = { 

173 'mirantis': _dc, 

174 'system': _ds, 

175 'other': _do, 

176 'unlisted': _du 

177 } 

178 

179 return _data 

180 

181 def collect_installed_packages(self): 

182 """ 

183 Collect installed packages on each node 

184 sets 'installed' dict property in the class 

185 

186 :return: none 

187 """ 

188 logger_cli.info("# Collecting installed packages") 

189 salt_master.prepare_script_on_active_nodes("pkg_versions.py") 

190 _result = salt_master.execute_script_on_active_nodes("pkg_versions.py") 

191 

192 for key in salt_master.nodes.keys(): 

193 # due to much data to be passed from salt, it is happening in order 

194 if key in _result and _result[key]: 

195 _text = _result[key] 

196 try: 

197 _dict = json.loads(_text[_text.find('{'):]) 

198 except ValueError: 

199 logger_cli.info("... no JSON for '{}'".format( 

200 key 

201 )) 

202 logger_cli.debug( 

203 "ERROR:\n{}\n".format(_text[:_text.find('{')]) 

204 ) 

205 _dict = {} 

206 

207 salt_master.nodes[key]['packages'] = _dict 

208 else: 

209 salt_master.nodes[key]['packages'] = {} 

210 logger_cli.debug("... {} has {} packages installed".format( 

211 key, 

212 len(salt_master.nodes[key]['packages'].keys()) 

213 )) 

214 logger_cli.info("-> Done") 

215 

216 def collect_packages(self): 

217 """ 

218 Check package versions in repos vs installed 

219 

220 :return: no return values, all date put to dict in place 

221 """ 

222 # Preload OpenStack release versions 

223 _desc = PkgVersions() 

224 logger_cli.info( 

225 "# Cross-comparing: Installed vs Candidates vs Release" 

226 ) 

227 # shortcuts for this cloud values 

228 _os = salt_master.openstack_release 

229 _mcp = salt_master.mcp_release 

230 _t = [self.force_tag] if self.force_tag else [] 

231 _t.append(_mcp) 

232 

233 logger_cli.info("# Tag search list: {}".format(", ".join(_t))) 

234 logger_cli.info("# Openstack version: {}".format(_os)) 

235 logger_cli.info( 

236 "# Release versions repos keyword exclude list: {}".format( 

237 ", ".join(self.exclude_keywords) 

238 ) 

239 ) 

240 

241 # Progress class 

242 _progress = Progress(len(salt_master.nodes.keys())) 

243 _progress_index = 0 

244 _total_processed = 0 

245 # Collect packages from all of the nodes in flat dict 

246 _all_packages = {} 

247 for node_name, node_value in salt_master.nodes.items(): 

248 _uniq_len = len(_all_packages.keys()) 

249 _progress_index += 1 

250 # progress updates shown before next node only 

251 # it is costly operation to do it for each of the 150k packages 

252 _progress.write_progress( 

253 _progress_index, 

254 note="/ {} uniq out of {} packages found".format( 

255 _uniq_len, 

256 _total_processed 

257 ) 

258 ) 

259 for _name, _value in node_value['packages'].items(): 

260 _total_processed += 1 

261 # Parse versions from nodes 

262 _ver_ins = DebianVersion(_value['installed']) 

263 _ver_can = DebianVersion(_value['candidate']) 

264 

265 # Process package description and release version 

266 # at a first sight 

267 if _name not in _all_packages: 

268 # get node attributes 

269 _linux = salt_master.nodes[node_name]['linux_codename'] 

270 _arch = salt_master.nodes[node_name]['linux_arch'] 

271 # get versions for tag, Openstack release and repo headers 

272 # excluding 'nightly' repos by default 

273 _r = {} 

274 # if there is a forced tag = use it 

275 if self.force_tag: 

276 _r = self.rm.get_filtered_versions( 

277 _name, 

278 tag=self.force_tag, 

279 include=[_os, _linux, _arch], 

280 exclude=self.exclude_keywords 

281 ) 

282 # if nothing found, look everywhere 

283 # but with no word 'openstack' 

284 if not _r: 

285 _r = self.rm.get_filtered_versions( 

286 _name, 

287 tag=self.force_tag, 

288 include=[_linux, _arch], 

289 exclude=self.exclude_keywords + ['openstack'] 

290 ) 

291 # if nothing is found at this point, 

292 # repeat search using normal tags 

293 if not _r: 

294 _r = self.rm.get_filtered_versions( 

295 _name, 

296 tag=_mcp, 

297 include=[_os, _linux, _arch], 

298 exclude=self.exclude_keywords 

299 ) 

300 # Once again, if nothing found, look everywhere 

301 if not _r: 

302 _r = self.rm.get_filtered_versions( 

303 _name, 

304 tag=_mcp, 

305 include=[_linux, _arch], 

306 exclude=self.exclude_keywords + ['openstack'] 

307 ) 

308 # repack versions in flat format 

309 _vs = {} 

310 _sections = {} 

311 _apps = {} 

312 for s, apps in _r.items(): 

313 for a, versions in apps.items(): 

314 for v, repos in versions.items(): 

315 for repo in repos: 

316 if v not in _vs: 

317 _vs[v] = [] 

318 _vs[v].append(repo) 

319 if v not in _sections: 

320 _sections[v] = [] 

321 _sections[v].append(s) 

322 if v not in _apps: 

323 _apps[v] = [] 

324 _apps[v].append(a) 

325 # search for the newest version among filtered 

326 _r_desc = [] 

327 _vs_keys = _vs.keys() 

328 if _vs_keys: 

329 _newest = _newest = DebianVersion(_vs_keys.pop()) 

330 else: 

331 _newest = DebianVersion('') 

332 for v in _vs_keys: 

333 _this = DebianVersion(v) 

334 if _this > _newest: 

335 _newest = _this 

336 _release = _newest 

337 # Get best description for the package 

338 if _release.version != 'n/a': 

339 _r_desc = _vs[_release.version] 

340 # preload special description 

341 if _desc[_name]: 

342 _pkg_desc = _desc[_name] 

343 else: 

344 _pkg_desc = _desc.dummy_desc 

345 # Save repos list and desc for this version 

346 # Check if we can provide better from the package 

347 if _release.version != 'n/a': 

348 if not _pkg_desc['section']: 

349 _pkg_desc['section'] = \ 

350 "/".join(_sections[_release.version]) 

351 if not _pkg_desc['app']: 

352 _pkg_desc['app'] = \ 

353 "/".join(_apps[_release.version]) 

354 

355 # Populate package info, once for package 

356 _m = _r_desc[0]["maintainer"] if _r_desc else 'n/a' 

357 _all_packages[_name] = { 

358 "desc": _pkg_desc, 

359 "repos": _r_desc, 

360 "maintainer": _m, 

361 "is_mirantis": self.rm.is_mirantis( 

362 _name, 

363 tag=_mcp 

364 ), 

365 "results": {}, 

366 "r": _release, 

367 } 

368 # Cross-compare versions 

369 _cmp = VersionCmpResult( 

370 _ver_ins, 

371 _ver_can, 

372 _all_packages[_name]['r'] 

373 ) 

374 # Update results structure 

375 # shortcut to results 

376 _res = _all_packages[_name]['results'] 

377 # update status 

378 if _cmp.status not in _res: 

379 _res[_cmp.status] = {} 

380 # update action 

381 if _cmp.action not in _res[_cmp.status]: 

382 _res[_cmp.status][_cmp.action] = {} 

383 # update node 

384 if node_name not in _res[_cmp.status][_cmp.action]: 

385 _res[_cmp.status][_cmp.action][node_name] = {} 

386 # put result 

387 _res[_cmp.status][_cmp.action][node_name] = { 

388 'i': _ver_ins, 

389 'c': _ver_can, 

390 'res': _cmp, 

391 'raw': _value['raw'] 

392 } 

393 

394 self._packages = _all_packages 

395 _progress.end() 

396 

397 def create_report(self, filename, rtype, full=None): 

398 """ 

399 Create static html showing packages diff per node 

400 

401 :return: buff with html 

402 """ 

403 logger_cli.info("# Generating report to '{}'".format(filename)) 

404 if rtype == 'html': 

405 _type = reporter.HTMLPackageCandidates() 

406 elif rtype == 'csv': 

407 _type = reporter.CSVAllPackages() 

408 else: 

409 raise ConfigException("Report type not set") 

410 _report = reporter.ReportToFile( 

411 _type, 

412 filename 

413 ) 

414 payload = { 

415 "nodes": salt_master.nodes, 

416 "mcp_release": salt_master.mcp_release, 

417 "openstack_release": salt_master.openstack_release 

418 } 

419 payload.update(self.presort_packages(self._packages, full)) 

420 _report(payload) 

421 logger_cli.info("-> Done")