Module refactoring and dynamic loading
diff --git a/cfg_checker/modules/__init__.py b/cfg_checker/modules/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cfg_checker/modules/__init__.py
diff --git a/cfg_checker/modules/network/__init__.py b/cfg_checker/modules/network/__init__.py
new file mode 100644
index 0000000..627c7d5
--- /dev/null
+++ b/cfg_checker/modules/network/__init__.py
@@ -0,0 +1,23 @@
+import checker
+
+from cfg_checker.helpers import args_utils
+from cfg_checker.common import logger_cli
+
+def do_check(args):
+ logger_cli.info("# Network check (CLI output)")
+ netChecker = checker.NetworkChecker()
+ netChecker.collect_network_info()
+ netChecker.print_network_report()
+
+ return
+
+
+def do_report(args):
+ logger_cli.info("# Network check (HTML report: '{}')".format(args.file))
+ _filename = args_utils.get_file_arg(args)
+
+ netChecker = checker.NetworkChecker()
+ netChecker.collect_network_info()
+ netChecker.create_html_report(_filename)
+
+ return
diff --git a/cfg_checker/modules/network/checker.py b/cfg_checker/modules/network/checker.py
new file mode 100644
index 0000000..03a48de
--- /dev/null
+++ b/cfg_checker/modules/network/checker.py
@@ -0,0 +1,129 @@
+import json
+import os
+import sys
+import ipaddress
+
+from copy import deepcopy
+
+from cfg_checker import reporter
+from cfg_checker.common import utils, const
+from cfg_checker.common import config, logger, logger_cli, pkg_dir
+from cfg_checker.common import salt_utils
+from cfg_checker.nodes import SaltNodes, node_tmpl
+
+
+class NetworkChecker(SaltNodes):
+ def collect_network_info(self):
+ """
+ Collects info on the network using ifs_data.py script
+
+ :return: none
+ """
+ logger_cli.info("### Collecting network data")
+ _result = self.execute_script("ifs_data.py", args=["json"])
+
+ for key in self.nodes.keys():
+ # due to much data to be passed from salt, it is happening in order
+ if key in _result:
+ _text = _result[key]
+ _dict = json.loads(_text[_text.find('{'):])
+ self.nodes[key]['networks'] = _dict
+ else:
+ self.nodes[key]['networks'] = {}
+ logger_cli.debug("# {} has {} networks".format(
+ key,
+ len(self.nodes[key]['networks'].keys())
+ ))
+ logger_cli.info("-> done collecting networks data")
+
+ # dump collected data to speed up coding
+ # with open('dump.json', 'w+') as ff:
+ # ff.write(json.dumps(self.nodes))
+
+ # load dump data
+ # with open('dump.json', 'r') as ff:
+ # _nodes = json.loads(ff.read())
+
+ logger_cli.info("### Building network tree")
+ # match physical interfaces by MAC addresses
+ _all_nets = {}
+ for host, node_data in self.nodes.iteritems():
+ for net_name, net_data in node_data['networks'].iteritems():
+ # get ips and calculate subnets
+ if net_name == 'lo':
+ continue
+ _ip4s = net_data['ipv4']
+ for _ip_str in _ip4s.keys():
+ _if = ipaddress.IPv4Interface(_ip_str)
+ if not any(_if.ip in net for net in _all_nets.keys()):
+ # IP not fits into existing networks
+ if _if.network not in _all_nets.keys():
+ _all_nets[_if.network] = {}
+
+ _all_nets[_if.network][host] = {}
+ _all_nets[_if.network][host]['text'] = \
+ "{0:30}: {1:19} {2:5} {3:4}".format(
+ net_name,
+ str(_if.ip),
+ net_data['mtu'],
+ net_data['state']
+ )
+ _all_nets[_if.network][host]['if_data'] = net_data
+ else:
+ # There is a network that ip fits into
+ for _net in _all_nets.keys():
+ if _if.ip in _net:
+ if host not in _all_nets[_net]:
+ _all_nets[_net][host] = {}
+ _all_nets[_net][host]['text'] = \
+ "{0:30}: {1:19} {2:5} {3:4}".format(
+ net_name,
+ str(_if.ip),
+ net_data['mtu'],
+ net_data['state']
+ )
+ _all_nets[_net][host]['if_data'] = \
+ net_data
+
+ # save collected info
+ self.all_networks = _all_nets
+
+ # Get networks from reclass
+ # TODO:
+
+ return
+
+ def print_network_report(self):
+ """
+ Create text report for CLI
+
+ :return: none
+ """
+ for network, nodes in self.all_networks.iteritems():
+ logger_cli.info("-> {}".format(str(network)))
+ names = sorted(nodes.keys())
+
+ for hostname in names:
+ logger_cli.info(
+ "\t{0:10} {1}".format(
+ hostname.split('.')[0],
+ nodes[hostname]['text']
+ )
+ )
+
+ def create_html_report(self, filename):
+ """
+ Create static html showing network schema-like report
+
+ :return: none
+ """
+ logger_cli.info("### Generating report to '{}'".format(filename))
+ _report = reporter.ReportToFile(
+ reporter.HTMLNetworkReport(),
+ filename
+ )
+ _report({
+ "nodes": self.nodes,
+ "diffs": {}
+ })
+ logger_cli.info("-> Done")
diff --git a/cfg_checker/modules/packages/__init__.py b/cfg_checker/modules/packages/__init__.py
new file mode 100644
index 0000000..4f1a809
--- /dev/null
+++ b/cfg_checker/modules/packages/__init__.py
@@ -0,0 +1,21 @@
+import checker
+
+from cfg_checker.helpers import args_utils
+
+
+def do_report(args):
+ """Create package versions report
+
+ :args: - parser arguments
+ :return: - no return value
+ """
+ _filename = args_utils.get_file_arg(args)
+
+ # init connection to salt and collect minion data
+ pChecker = checker.CloudPackageChecker()
+ # collect data on installed packages
+ pChecker.collect_installed_packages()
+ # diff installed and candidates
+ # pChecker.collect_packages()
+ # report it
+ pChecker.create_html_report(_filename)
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
new file mode 100644
index 0000000..5f48dcd
--- /dev/null
+++ b/cfg_checker/modules/packages/checker.py
@@ -0,0 +1,71 @@
+import json
+import os
+#import sys
+
+from copy import deepcopy
+
+from cfg_checker import reporter
+from cfg_checker.common import utils, const
+from cfg_checker.common import config, logger, logger_cli, pkg_dir
+from cfg_checker.common import salt_utils
+from cfg_checker.nodes import SaltNodes, node_tmpl
+
+
+class CloudPackageChecker(SaltNodes):
+ def collect_installed_packages(self):
+ """
+ Collect installed packages on each node
+ sets 'installed' dict property in the class
+
+ :return: none
+ """
+ logger_cli.info("### Collecting installed packages")
+ _result = self.execute_script("pkg_versions.py")
+
+ for key in self.nodes.keys():
+ # due to much data to be passed from salt, it is happening in order
+ if key in _result:
+ _text = _result[key]
+ _dict = json.loads(_text[_text.find('{'):])
+ self.nodes[key]['packages'] = _dict
+ else:
+ self.nodes[key]['packages'] = {}
+ logger_cli.debug("# {} has {} packages installed".format(
+ key,
+ len(self.nodes[key]['packages'].keys())
+ ))
+ logger_cli.info("-> Done")
+
+ def collect_packages(self):
+ """
+ Check package versions in repos vs installed
+
+ :return: no return values, all date put to dict in place
+ """
+ _all_packages = {}
+ for node_name, node_value in self.nodes.iteritems():
+ for package_name in node_value['packages']:
+ if package_name not in _all_packages:
+ _all_packages[package_name] = {}
+ _all_packages[package_name][node_name] = node_value
+
+ # TODO: process data for per-package basis
+
+ self.all_packages = _all_packages
+
+ def create_html_report(self, filename):
+ """
+ Create static html showing packages diff per node
+
+ :return: buff with html
+ """
+ logger_cli.info("### Generating report to '{}'".format(filename))
+ _report = reporter.ReportToFile(
+ reporter.HTMLPackageCandidates(),
+ filename
+ )
+ _report({
+ "nodes": self.nodes,
+ "diffs": {}
+ })
+ logger_cli.info("-> Done")
diff --git a/cfg_checker/modules/reclass/__init__.py b/cfg_checker/modules/reclass/__init__.py
new file mode 100644
index 0000000..a695482
--- /dev/null
+++ b/cfg_checker/modules/reclass/__init__.py
@@ -0,0 +1,74 @@
+import os
+
+import comparer
+import validator
+
+from cfg_checker.common import logger_cli
+from cfg_checker.helpers import args_utils
+from cfg_checker.reports import reporter
+
+
+def do_list(args):
+ logger_cli.info("# Reclass list")
+ _path = args_utils.get_path_arg(args.models_path)
+
+ logger_cli.info("# ...models path is '{}'".format(args.models_path))
+
+ models = {}
+ for _folder in os.listdir(args.models_path):
+ # validate item as a model
+ _model_path = os.path.join(
+ args.models_path,
+ _folder
+ )
+ _validated = validator.basic_model_validation_by_path(_model_path)
+
+ if not _validated:
+ logger_cli.info("-> '{}' not a valid model".format(_folder))
+ continue
+ else:
+ models[_folder] = _model_path
+
+ logger_cli.info("-> '{}' at '{}'".format(_folder, _model_path))
+
+ # TODO: collect info about the model
+
+ return
+
+
+def do_diff(args):
+ logger_cli.info("Reclass comparer (HTML report: '{}'".format(args.file))
+ _filename = args_utils.get_file_arg(args)
+
+ # checking folder params
+ _model1 = args_utils.get_path_arg(args.model1)
+ _model2 = args_utils.get_path_arg(args.model2)
+
+ # Do actual compare using hardcoded model names
+ mComparer = comparer.ModelComparer()
+
+ mComparer.model_name_1 = os.path.split(_model1)[1]
+ mComparer.model_path_1 = _model1
+ mComparer.model_name_2 = os.path.split(_model2)[1]
+ mComparer.model_path_2 = _model2
+
+ mComparer.load_model_tree(
+ mComparer.model_name_1,
+ mComparer.model_path_1
+ )
+ mComparer.load_model_tree(
+ mComparer.model_name_2,
+ mComparer.model_path_2
+ )
+
+ diffs = mComparer.generate_model_report_tree()
+
+ report = reporter.ReportToFile(
+ reporter.HTMLModelCompare(),
+ _filename
+ )
+ logger_cli.info("# Generating report to {}".format(_filename))
+ report({
+ "nodes": {},
+ "diffs": diffs
+ })
diff --git a/cfg_checker/modules/reclass/comparer.py b/cfg_checker/modules/reclass/comparer.py
new file mode 100644
index 0000000..5a26f29
--- /dev/null
+++ b/cfg_checker/modules/reclass/comparer.py
@@ -0,0 +1,364 @@
+"""Model Comparer:
+- yaml parser
+- class tree comparison
+"""
+import itertools
+import os
+import yaml
+
+from cfg_checker import reporter
+from cfg_checker.common import logger, logger_cli
+
+
+def get_element(element_path, input_data):
+ paths = element_path.split(":")
+ data = input_data
+ for i in range(0, len(paths)):
+ data = data[paths[i]]
+ return data
+
+
+def pop_element(element_path, input_data):
+ paths = element_path.split(":")
+ data = input_data
+ # Search for last dict
+ for i in range(0, len(paths)-1):
+ data = data[paths[i]]
+ # pop the actual element
+ return data.pop(paths[-1])
+
+
+class ModelComparer(object):
+ """Collection of functions to compare model data.
+ """
+ # key order is important
+ _model_parts = {
+ "01_nodes": "nodes",
+ "02_system": "classes:system",
+ "03_cluster": "classes:cluster",
+ "04_other": "classes"
+ }
+
+ models = {}
+ models_path = "/srv/salt/reclass"
+ model_name_1 = "source"
+ model_path_1 = os.path.join(models_path, model_name_1)
+ model_name_2 = "target"
+ model_path_2 = os.path.join(models_path, model_name_1)
+
+ @staticmethod
+ def load_yaml_class(fname):
+ """Loads a yaml from the file and forms a tree item
+
+ Arguments:
+ fname {string} -- full path to the yaml file
+ """
+ _yaml = {}
+ try:
+ _size = 0
+ with open(fname, 'r') as f:
+ _yaml = yaml.load(f)
+ _size = f.tell()
+ # TODO: do smth with the data
+ if not _yaml:
+ logger_cli.warning("WARN: empty file '{}'".format(fname))
+ _yaml = {}
+ else:
+ logger.debug("...loaded YAML '{}' ({}b)".format(fname, _size))
+ return _yaml
+ except yaml.YAMLError as exc:
+ logger_cli.error(exc)
+ except IOError as e:
+ logger_cli.error(
+ "Error loading file '{}': {}".format(fname, e.message)
+ )
+ raise Exception("CRITICAL: Failed to load YAML data: {}".format(
+ e.message + e.strerror
+ ))
+
+ def load_model_tree(self, name, root_path="/srv/salt/reclass"):
+ """Walks supplied path for the YAML filed and loads the tree
+
+ Arguments:
+ root_folder_path {string} -- Path to Model's root folder. Optional
+ """
+ logger_cli.info("Loading reclass tree from '{}'".format(root_path))
+ # prepare the file tree to walk
+ raw_tree = {}
+ # Credits to Andrew Clark@MIT. Original code is here:
+ # http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/
+ root_path = root_path.rstrip(os.sep)
+ start = root_path.rfind(os.sep) + 1
+ root_key = root_path.rsplit(os.sep, 1)[1]
+ # Look Ma! I am walking the file tree with no recursion!
+ for path, dirs, files in os.walk(root_path):
+ # if this is a hidden folder, ignore it
+ _folders_list = path[start:].split(os.sep)
+ if any(item.startswith(".") for item in _folders_list):
+ continue
+ # cut absolute part of the path and split folder names
+ folders = path[start:].split(os.sep)
+ subdir = {}
+ # create generator of files that are not hidden
+ _exts = ('.yml', '.yaml')
+ _subfiles = (_fl for _fl in files
+ if _fl.endswith(_exts) and not _fl.startswith('.'))
+ for _file in _subfiles:
+ # cut file extension. All reclass files are '.yml'
+ _subnode = _file
+ # load all YAML class data into the tree
+ subdir[_subnode] = self.load_yaml_class(
+ os.path.join(path, _file)
+ )
+ try:
+ # Save original filepath, just in case
+ subdir[_subnode]["_source"] = os.path.join(
+ path[start:],
+ _file
+ )
+ except Exception:
+ logger.warning(
+ "Non-yaml file detected: {}".format(_file)
+ )
+ # creating dict structure out of folder list. Pure python magic
+ parent = reduce(dict.get, folders[:-1], raw_tree)
+ parent[folders[-1]] = subdir
+
+ self.models[name] = {}
+ # Brake in according to pathes
+ _parts = self._model_parts.keys()
+ _parts = sorted(_parts)
+ for ii in range(0, len(_parts)):
+ self.models[name][_parts[ii]] = pop_element(
+ self._model_parts[_parts[ii]],
+ raw_tree[root_key]
+ )
+
+ # save it as a single data object
+ self.models[name]["all_diffs"] = raw_tree[root_key]
+ return True
+
+ def find_changes(self, dict1, dict2, path=""):
+ _report = {}
+ for k in dict1.keys():
+ # yamls might load values as non-str types
+ if not isinstance(k, str):
+ _new_path = path + ":" + str(k)
+ else:
+ _new_path = path + ":" + k
+ # ignore _source key
+ if k == "_source":
+ continue
+ # check if this is an env name cluster entry
+ if dict2 is not None and \
+ k == self.model_name_1 and \
+ self.model_name_2 in dict2.keys():
+ k1 = self.model_name_1
+ k2 = self.model_name_2
+ if type(dict1[k1]) is dict:
+ if path == "":
+ _new_path = k1
+ _child_report = self.find_changes(
+ dict1[k1],
+ dict2[k2],
+ _new_path
+ )
+ _report.update(_child_report)
+ elif dict2 is None or k not in dict2:
+ # no key in dict2
+ _report[_new_path] = {
+ "type": "value",
+ "raw_values": [dict1[k], "N/A"],
+ "str_values": [
+ "{}".format(dict1[k]),
+ "n/a"
+ ]
+ }
+ logger.info(
+ "{}: {}, {}".format(_new_path, dict1[k], "N/A")
+ )
+ else:
+ if type(dict1[k]) is dict:
+ if path == "":
+ _new_path = k
+ _child_report = self.find_changes(
+ dict1[k],
+ dict2[k],
+ _new_path
+ )
+ _report.update(_child_report)
+ elif type(dict1[k]) is list and type(dict2[k]) is list:
+ # use ifilterfalse to compare lists of dicts
+ try:
+ _removed = list(
+ itertools.ifilterfalse(
+ lambda x: x in dict2[k],
+ dict1[k]
+ )
+ )
+ _added = list(
+ itertools.ifilterfalse(
+ lambda x: x in dict1[k],
+ dict2[k]
+ )
+ )
+ except TypeError as e:
+ # debug routine,
+ # should not happen, due to list check above
+ logger.error(
+ "Caught lambda type mismatch: {}".format(
+ e.message
+ )
+ )
+ logger_cli.warning(
+ "Types mismatch for correct compare: "
+ "{}, {}".format(
+ type(dict1[k]),
+ type(dict2[k])
+ )
+ )
+ _removed = None
+ _added = None
+ _original = ["= {}".format(item) for item in dict1[k]]
+ if _removed or _added:
+ _removed_str_lst = ["- {}".format(item)
+ for item in _removed]
+ _added_str_lst = ["+ {}".format(item)
+ for item in _added]
+ _report[_new_path] = {
+ "type": "list",
+ "raw_values": [
+ dict1[k],
+ _removed_str_lst + _added_str_lst
+ ],
+ "str_values": [
+ "{}".format('\n'.join(_original)),
+ "{}\n{}".format(
+ '\n'.join(_removed_str_lst),
+ '\n'.join(_added_str_lst)
+ )
+ ]
+ }
+ logger.info(
+ "{}:\n"
+ "{} original items total".format(
+ _new_path,
+ len(dict1[k])
+ )
+ )
+ if _removed:
+ logger.info(
+ "{}".format('\n'.join(_removed_str_lst))
+ )
+ if _added:
+ logger.info(
+ "{}".format('\n'.join(_added_str_lst))
+ )
+ else:
+ # in case of type mismatch
+ # considering it as not equal
+ d1 = dict1
+ d2 = dict2
+ val1 = d1[k] if isinstance(d1, dict) else d1
+ val2 = d2[k] if isinstance(d2, dict) else d2
+ try:
+ match = val1 == val2
+ except TypeError as e:
+ logger.warning(
+ "One of the values is not a dict: "
+ "{}, {}".format(
+ str(dict1),
+ str(dict2)
+ ))
+ match = False
+ if not match:
+ _report[_new_path] = {
+ "type": "value",
+ "raw_values": [val1, val2],
+ "str_values": [
+ "{}".format(val1),
+ "{}".format(val2)
+ ]
+ }
+ logger.info("{}: {}, {}".format(
+ _new_path,
+ val1,
+ val2
+ ))
+ return _report
+
+
+ def generate_model_report_tree(self):
+ """Use two loaded models to generate comparison table with
+ values are groupped by YAML files
+ """
+ # We are to cut both models into logical pieces
+ # nodes, will not be equal most of the time
+ # system, must be pretty much the same or we in trouble
+ # cluster, will be the most curious part for comparison
+ # other, all of the rest
+
+ _diff_report = {}
+ for _key in self._model_parts.keys():
+ # tmp report for keys
+ _tmp_diffs = self.find_changes(
+ self.models[self.model_name_1][_key],
+ self.models[self.model_name_2][_key]
+ )
+ # prettify the report
+ for key in _tmp_diffs.keys():
+ # break the key in two parts
+ _ext = ".yml"
+ if ".yaml" in key:
+ _ext = ".yaml"
+ _split = key.split(_ext)
+ _file_path = _split[0]
+ _param_path = "none"
+ if len(_split) > 1:
+ _param_path = _split[1]
+ _tmp_diffs[key].update({
+ "class_file": _file_path + _ext,
+ "param": _param_path,
+ })
+ _diff_report[_key[3:]] = {
+ "path": self._model_parts[_key],
+ "diffs": _tmp_diffs
+ }
+
+ _diff_report["diff_names"] = [self.model_name_1, self.model_name_2]
+ return _diff_report
+
+ def compare_models(self):
+ # Do actual compare using model names from the class
+ self.load_model_tree(
+ self.model_name_1,
+ self.model_path_1
+ )
+ self.load_model_tree(
+ self.model_name_2,
+ self.model_path_2
+ )
+ # Models should have similar structure to be compared
+ # classes/system
+ # classes/cluster
+ # nodes
+
+ diffs = self.generate_model_report_tree()
+
+ report_file = \
+ self.model_name_1 + "-vs-" + self.model_name_2 + ".html"
+ # HTML report class is post-callable
+ report = reporter.ReportToFile(
+ reporter.HTMLModelCompare(),
+ report_file
+ )
+ logger_cli.info("...generating report to {}".format(report_file))
+ # report will have tabs for each of the comparable entities in diffs
+ report({
+ "nodes": {},
+ "all_diffs": diffs,
+ })
+ # with open("./gen_tree.json", "w+") as _out:
+ # _out.write(json.dumps(mComparer.generate_model_report_tree))
+
+ return
diff --git a/cfg_checker/modules/reclass/validator.py b/cfg_checker/modules/reclass/validator.py
new file mode 100644
index 0000000..e7d7f06
--- /dev/null
+++ b/cfg_checker/modules/reclass/validator.py
@@ -0,0 +1,28 @@
+import os
+
+from cfg_checker.common import logger_cli
+
+def basic_model_validation_by_path(path):
+ logger_cli.debug("\t...validating '{}' as a model".format(path))
+ _checks = []
+ _is_folder = os.path.isdir(path)
+ logger_cli.debug("\t- folder? -> {}".format(_is_folder))
+ _checks.append(_is_folder)
+ _has_classes = os.path.isdir(os.path.join(path, "classes"))
+ logger_cli.debug("\t- has classes? -> {}".format(_has_classes))
+ _checks.append(_has_classes)
+ _has_cluster = os.path.isdir(os.path.join(path, "classes", "cluster"))
+ logger_cli.debug("\t- has classes/cluster? -> {}".format(_has_cluster))
+ _checks.append(_has_cluster)
+ _has_system = os.path.isdir(os.path.join(path, "classes", "system"))
+ logger_cli.debug("\t- has classes/system? -> {}".format(_has_system))
+ _checks.append(_has_system)
+ _has_nodes = os.path.isdir(os.path.join(path, "nodes"))
+ logger_cli.debug("\t- has nodes? -> {}".format(_has_nodes))
+ _checks.append(_has_nodes)
+
+ logger_cli.debug("\t-> {}".format(
+ all(_checks)
+ ))
+
+ return all(_checks)