Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1"""Model Comparer: 

2- yaml parser 

3- class tree comparison 

4""" 

5import itertools 

6import os 

7 

8from cfg_checker.common import logger, logger_cli 

9from cfg_checker.reports import reporter 

10 

11import yaml 

12 

13 

14def get_element(element_path, input_data): 

15 paths = element_path.split(":") 

16 data = input_data 

17 for i in range(0, len(paths)): 

18 data = data[paths[i]] 

19 return data 

20 

21 

22def pop_element(element_path, input_data): 

23 paths = element_path.split(":") 

24 data = input_data 

25 # Search for last dict 

26 for i in range(0, len(paths)-1): 

27 data = data[paths[i]] 

28 # pop the actual element 

29 return data.pop(paths[-1]) 

30 

31 

32class ModelComparer(object): 

33 """Collection of functions to compare model data. 

34 """ 

35 # key order is important 

36 _model_parts = { 

37 "01_nodes": "nodes", 

38 "02_system": "classes:system", 

39 "03_cluster": "classes:cluster", 

40 "04_other": "classes" 

41 } 

42 

43 models = {} 

44 models_path = "/srv/salt/reclass" 

45 model_name_1 = "source" 

46 model_path_1 = os.path.join(models_path, model_name_1) 

47 model_name_2 = "target" 

48 model_path_2 = os.path.join(models_path, model_name_1) 

49 

50 @staticmethod 

51 def load_yaml_class(fname): 

52 """Loads a yaml from the file and forms a tree item 

53 

54 Arguments: 

55 fname {string} -- full path to the yaml file 

56 """ 

57 _yaml = {} 

58 try: 

59 _size = 0 

60 with open(fname, 'r') as f: 

61 _yaml = yaml.load(f, Loader=yaml.FullLoader) 

62 _size = f.tell() 

63 # TODO: do smth with the data 

64 if not _yaml: 

65 # logger.warning("WARN: empty file '{}'".format(fname)) 

66 _yaml = {} 

67 else: 

68 logger.debug("...loaded YAML '{}' ({}b)".format(fname, _size)) 

69 return _yaml 

70 except yaml.YAMLError as exc: 

71 logger_cli.error(exc) 

72 except IOError as e: 

73 logger_cli.error( 

74 "Error loading file '{}': {}".format(fname, e.message) 

75 ) 

76 raise Exception("CRITICAL: Failed to load YAML data: {}".format( 

77 e.message + e.strerror 

78 )) 

79 

80 def load_model_tree(self, name, root_path="/srv/salt/reclass"): 

81 """Walks supplied path for the YAML filed and loads the tree 

82 

83 Arguments: 

84 root_folder_path {string} -- Path to Model's root folder. Optional 

85 """ 

86 logger_cli.info("# Loading reclass tree from '{}'".format(root_path)) 

87 # prepare the file tree to walk 

88 raw_tree = {} 

89 # Credits to Andrew Clark@MIT. Original code is here: 

90 # http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/ 

91 root_path = root_path.rstrip(os.sep) 

92 start = root_path.rfind(os.sep) + 1 

93 root_key = root_path.rsplit(os.sep, 1)[1] 

94 # Look Ma! I am walking the file tree with no recursion! 

95 for path, dirs, files in os.walk(root_path): 

96 # if this is a hidden folder, ignore it 

97 _folders_list = path[start:].split(os.sep) 

98 if any(item.startswith(".") for item in _folders_list): 

99 continue 

100 # cut absolute part of the path and split folder names 

101 folders = path[start:].split(os.sep) 

102 subdir = {} 

103 # create generator of files that are not hidden 

104 _exts = ('.yml', '.yaml') 

105 _subfiles = (_fl for _fl in files 

106 if _fl.endswith(_exts) and not _fl.startswith('.')) 

107 for _file in _subfiles: 

108 # cut file extension. All reclass files are '.yml' 

109 _subnode = _file 

110 # load all YAML class data into the tree 

111 subdir[_subnode] = self.load_yaml_class( 

112 os.path.join(path, _file) 

113 ) 

114 try: 

115 # Save original filepath, just in case 

116 subdir[_subnode]["_source"] = os.path.join( 

117 path[start:], 

118 _file 

119 ) 

120 except Exception: 

121 logger.warning( 

122 "Non-yaml file detected: {}".format(_file) 

123 ) 

124 # creating dict structure out of folder list. Pure python magic 

125 parent = reduce(dict.get, folders[:-1], raw_tree) 

126 parent[folders[-1]] = subdir 

127 

128 self.models[name] = {} 

129 # Brake in according to pathes 

130 _parts = self._model_parts.keys() 

131 _parts = sorted(_parts) 

132 for ii in range(0, len(_parts)): 

133 self.models[name][_parts[ii]] = pop_element( 

134 self._model_parts[_parts[ii]], 

135 raw_tree[root_key] 

136 ) 

137 

138 # save it as a single data object 

139 self.models[name]["rc_diffs"] = raw_tree[root_key] 

140 return True 

141 

142 def find_changes(self, dict1, dict2, path=""): 

143 _report = {} 

144 for k in dict1.keys(): 

145 # yamls might load values as non-str types 

146 if not isinstance(k, str): 

147 _new_path = path + ":" + str(k) 

148 else: 

149 _new_path = path + ":" + k 

150 # ignore _source key 

151 if k == "_source": 

152 continue 

153 # ignore secrets 

154 if isinstance(k, str) and k == "secrets.yml": 

155 continue 

156 if isinstance(k, str) and k.find("_password") > 0: 

157 continue 

158 # check if this is an env name cluster entry 

159 if dict2 is not None and \ 

160 k == self.model_name_1 and \ 

161 self.model_name_2 in dict2.keys(): 

162 k1 = self.model_name_1 

163 k2 = self.model_name_2 

164 if type(dict1[k1]) is dict: 

165 if path == "": 

166 _new_path = k1 

167 _child_report = self.find_changes( 

168 dict1[k1], 

169 dict2[k2], 

170 _new_path 

171 ) 

172 _report.update(_child_report) 

173 elif dict2 is None or k not in dict2: 

174 # no key in dict2 

175 _report[_new_path] = { 

176 "type": "value", 

177 "raw_values": [dict1[k], "N/A"], 

178 "str_values": [ 

179 "{}".format(dict1[k]), 

180 "n/a" 

181 ] 

182 } 

183 logger.info( 

184 "{}: {}, {}".format(_new_path, dict1[k], "N/A") 

185 ) 

186 else: 

187 if type(dict1[k]) is dict: 

188 if path == "": 

189 _new_path = k 

190 _child_report = self.find_changes( 

191 dict1[k], 

192 dict2[k], 

193 _new_path 

194 ) 

195 _report.update(_child_report) 

196 elif type(dict1[k]) is list and type(dict2[k]) is list: 

197 # use ifilterfalse to compare lists of dicts 

198 try: 

199 _removed = list( 

200 itertools.ifilterfalse( 

201 lambda x: x in dict2[k], 

202 dict1[k] 

203 ) 

204 ) 

205 _added = list( 

206 itertools.ifilterfalse( 

207 lambda x: x in dict1[k], 

208 dict2[k] 

209 ) 

210 ) 

211 except TypeError as e: 

212 # debug routine, 

213 # should not happen, due to list check above 

214 logger.error( 

215 "Caught lambda type mismatch: {}".format( 

216 e.message 

217 ) 

218 ) 

219 logger_cli.warning( 

220 "Types mismatch for correct compare: " 

221 "{}, {}".format( 

222 type(dict1[k]), 

223 type(dict2[k]) 

224 ) 

225 ) 

226 _removed = None 

227 _added = None 

228 _original = ["= {}".format(item) for item in dict1[k]] 

229 if _removed or _added: 

230 _removed_str_lst = ["- {}".format(item) 

231 for item in _removed] 

232 _added_str_lst = ["+ {}".format(i) for i in _added] 

233 _report[_new_path] = { 

234 "type": "list", 

235 "raw_values": [ 

236 dict1[k], 

237 _removed_str_lst + _added_str_lst 

238 ], 

239 "str_values": [ 

240 "{}".format('\n'.join(_original)), 

241 "{}\n{}".format( 

242 '\n'.join(_removed_str_lst), 

243 '\n'.join(_added_str_lst) 

244 ) 

245 ] 

246 } 

247 logger.info( 

248 "{}:\n" 

249 "{} original items total".format( 

250 _new_path, 

251 len(dict1[k]) 

252 ) 

253 ) 

254 if _removed: 

255 logger.info( 

256 "{}".format('\n'.join(_removed_str_lst)) 

257 ) 

258 if _added: 

259 logger.info( 

260 "{}".format('\n'.join(_added_str_lst)) 

261 ) 

262 else: 

263 # in case of type mismatch 

264 # considering it as not equal 

265 d1 = dict1 

266 d2 = dict2 

267 val1 = d1[k] if isinstance(d1, dict) else d1 

268 val2 = d2[k] if isinstance(d2, dict) else d2 

269 try: 

270 match = val1 == val2 

271 except TypeError as e: 

272 logger.warning( 

273 "One of the values is not a dict: " 

274 "{}, {}".format( 

275 str(dict1), 

276 str(dict2) 

277 )) 

278 match = False 

279 if not match: 

280 _report[_new_path] = { 

281 "type": "value", 

282 "raw_values": [val1, val2], 

283 "str_values": [ 

284 "{}".format(val1), 

285 "{}".format(val2) 

286 ] 

287 } 

288 logger.info("{}: {}, {}".format( 

289 _new_path, 

290 val1, 

291 val2 

292 )) 

293 return _report 

294 

295 def generate_model_report_tree(self): 

296 """Use two loaded models to generate comparison table with 

297 values are groupped by YAML files 

298 """ 

299 # We are to cut both models into logical pieces 

300 # nodes, will not be equal most of the time 

301 # system, must be pretty much the same or we in trouble 

302 # cluster, will be the most curious part for comparison 

303 # other, all of the rest 

304 

305 _diff_report = {} 

306 for _key in self._model_parts.keys(): 

307 # tmp report for keys 

308 _tmp_diffs = self.find_changes( 

309 self.models[self.model_name_1][_key], 

310 self.models[self.model_name_2][_key] 

311 ) 

312 # prettify the report 

313 for key in _tmp_diffs.keys(): 

314 # break the key in two parts 

315 _ext = ".yml" 

316 if ".yaml" in key: 

317 _ext = ".yaml" 

318 _split = key.split(_ext) 

319 _file_path = _split[0] 

320 _param_path = "none" 

321 if len(_split) > 1: 

322 _param_path = _split[1] 

323 _tmp_diffs[key].update({ 

324 "class_file": _file_path + _ext, 

325 "param": _param_path, 

326 }) 

327 _diff_report[_key[3:]] = { 

328 "path": self._model_parts[_key], 

329 "diffs": _tmp_diffs 

330 } 

331 

332 _diff_report["diff_names"] = [self.model_name_1, self.model_name_2] 

333 return _diff_report 

334 

335 def compare_models(self): 

336 # Do actual compare using model names from the class 

337 self.load_model_tree( 

338 self.model_name_1, 

339 self.model_path_1 

340 ) 

341 self.load_model_tree( 

342 self.model_name_2, 

343 self.model_path_2 

344 ) 

345 # Models should have similar structure to be compared 

346 # classes/system 

347 # classes/cluster 

348 # nodes 

349 

350 diffs = self.generate_model_report_tree() 

351 

352 report_file = \ 

353 self.model_name_1 + "-vs-" + self.model_name_2 + ".html" 

354 # HTML report class is post-callable 

355 report = reporter.ReportToFile( 

356 reporter.HTMLModelCompare(), 

357 report_file 

358 ) 

359 logger_cli.info("...generating report to {}".format(report_file)) 

360 # report will have tabs for each of the comparable entities in diffs 

361 report({ 

362 "nodes": {}, 

363 "rc_diffs": diffs, 

364 }) 

365 # with open("./gen_tree.json", "w+") as _out: 

366 # _out.write(json.dumps(mComparer.generate_model_report_tree)) 

367 

368 return