cfg_checker/modules/reclass/comparer.py

"""Model Comparer:
- yaml parser
- class tree comparison
"""
import itertools
import os

from cfg_checker.common import logger, logger_cli
from cfg_checker.reports import reporter

import yaml


def get_element(element_path, input_data):
    paths = element_path.split(":")
    data = input_data
    for i in range(0, len(paths)):
        data = data[paths[i]]
    return data


def pop_element(element_path, input_data):
    paths = element_path.split(":")
    data = input_data
    # Search for last dict
    for i in range(0, len(paths)-1):
        data = data[paths[i]]
    # pop the actual element
    return data.pop(paths[-1])


class ModelComparer(object):
    """Collection of functions to compare model data.
    """
    # key order is important
    _model_parts = {
        "01_nodes": "nodes",
        "02_system": "classes:system",
        "03_cluster": "classes:cluster",
        "04_other": "classes"
    }

    models = {}
    models_path = "/srv/salt/reclass"
    model_name_1 = "source"
    model_path_1 = os.path.join(models_path, model_name_1)
    model_name_2 = "target"
    model_path_2 = os.path.join(models_path, model_name_1)

    @staticmethod
    def load_yaml_class(fname):
        """Loads a yaml from the file and forms a tree item

        Arguments:
            fname {string} -- full path to the yaml file
        """
        _yaml = {}
        try:
            _size = 0
            with open(fname, 'r') as f:
                _yaml = yaml.load(f, Loader=yaml.FullLoader)
                _size = f.tell()
            # TODO: do smth with the data
            if not _yaml:
                # logger.warning("WARN: empty file '{}'".format(fname))
                _yaml = {}
            else:
                logger.debug("...loaded YAML '{}' ({}b)".format(fname, _size))
            return _yaml
        except yaml.YAMLError as exc:
            logger_cli.error(exc)
        except IOError as e:
            logger_cli.error(
                "Error loading file '{}': {}".format(fname, e.message)
            )
            raise Exception("CRITICAL: Failed to load YAML data: {}".format(
                e.message + e.strerror
            ))

    def load_model_tree(self, name, root_path="/srv/salt/reclass"):
        """Walks supplied path for the YAML filed and loads the tree

        Arguments:
            root_folder_path {string} -- Path to Model's root folder. Optional
        """
        logger_cli.info("# Loading reclass tree from '{}'".format(root_path))
        # prepare the file tree to walk
        raw_tree = {}
        # Credits to Andrew Clark@MIT. Original code is here:
        # http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/
        root_path = root_path.rstrip(os.sep)
        start = root_path.rfind(os.sep) + 1
        root_key = root_path.rsplit(os.sep, 1)[1]
        # Look Ma! I am walking the file tree with no recursion!
        for path, dirs, files in os.walk(root_path):
            # if this is a hidden folder, ignore it
            _folders_list = path[start:].split(os.sep)
            if any(item.startswith(".") for item in _folders_list):
                continue
            # cut absolute part of the path and split folder names
            folders = path[start:].split(os.sep)
            subdir = {}
            # create generator of files that are not hidden
            _exts = ('.yml', '.yaml')
            _subfiles = (_fl for _fl in files
                         if _fl.endswith(_exts) and not _fl.startswith('.'))
            for _file in _subfiles:
                # cut file extension. All reclass files are '.yml'
                _subnode = _file
                # load all YAML class data into the tree
                subdir[_subnode] = self.load_yaml_class(
                    os.path.join(path, _file)
                )
                try:
                    # Save original filepath, just in case
                    subdir[_subnode]["_source"] = os.path.join(
                        path[start:],
                        _file
                    )
                except Exception:
                    logger.warning(
                        "Non-yaml file detected: {}".format(_file)
                    )
            # creating dict structure out of folder list. Pure python magic
            parent = reduce(dict.get, folders[:-1], raw_tree)
            parent[folders[-1]] = subdir

        self.models[name] = {}
        # Brake in according to pathes
        _parts = self._model_parts.keys()
        _parts = sorted(_parts)
        for ii in range(0, len(_parts)):
            self.models[name][_parts[ii]] = pop_element(
                self._model_parts[_parts[ii]],
                raw_tree[root_key]
            )

        # save it as a single data object
        self.models[name]["rc_diffs"] = raw_tree[root_key]
        return True

    def find_changes(self, dict1, dict2, path=""):
        _report = {}
        for k in dict1.keys():
            # yamls might load values as non-str types
            if not isinstance(k, str):
                _new_path = path + ":" + str(k)
            else:
                _new_path = path + ":" + k
            # ignore _source key
            if k == "_source":
                continue
            # ignore secrets
            if isinstance(k, str) and k == "secrets.yml":
                continue
            if isinstance(k, str) and k.find("_password") > 0:
                continue
            # check if this is an env name cluster entry
            if dict2 is not None and \
                    k == self.model_name_1 and \
                    self.model_name_2 in dict2.keys():
                k1 = self.model_name_1
                k2 = self.model_name_2
                if type(dict1[k1]) is dict:
                    if path == "":
                        _new_path = k1
                    _child_report = self.find_changes(
                        dict1[k1],
                        dict2[k2],
                        _new_path
                    )
                    _report.update(_child_report)
            elif dict2 is None or k not in dict2:
                # no key in dict2
                _report[_new_path] = {
                    "type": "value",
                    "raw_values": [dict1[k], "N/A"],
                    "str_values": [
                        "{}".format(dict1[k]),
                        "n/a"
                    ]
                }
                logger.info(
                    "{}: {}, {}".format(_new_path, dict1[k], "N/A")
                )
            else:
                if type(dict1[k]) is dict:
                    if path == "":
                        _new_path = k
                    _child_report = self.find_changes(
                        dict1[k],
                        dict2[k],
                        _new_path
                    )
                    _report.update(_child_report)
                elif type(dict1[k]) is list and type(dict2[k]) is list:
                    # use ifilterfalse to compare lists of dicts
                    try:
                        _removed = list(
                            itertools.ifilterfalse(
                                lambda x: x in dict2[k],
                                dict1[k]
                            )
                        )
                        _added = list(
                            itertools.ifilterfalse(
                                lambda x: x in dict1[k],
                                dict2[k]
                            )
                        )
                    except TypeError as e:
                        # debug routine,
                        # should not happen, due to list check above
                        logger.error(
                            "Caught lambda type mismatch: {}".format(
                                e.message
                            )
                        )
                        logger_cli.warning(
                            "Types mismatch for correct compare: "
                            "{}, {}".format(
                                type(dict1[k]),
                                type(dict2[k])
                            )
                        )
                        _removed = None
                        _added = None
                    _original = ["= {}".format(item) for item in dict1[k]]
                    if _removed or _added:
                        _removed_str_lst = ["- {}".format(item)
                                            for item in _removed]
                        _added_str_lst = ["+ {}".format(i) for i in _added]
                        _report[_new_path] = {
                            "type": "list",
                            "raw_values": [
                                dict1[k],
                                _removed_str_lst + _added_str_lst
                            ],
                            "str_values": [
                                "{}".format('\n'.join(_original)),
                                "{}\n{}".format(
                                    '\n'.join(_removed_str_lst),
                                    '\n'.join(_added_str_lst)
                                )
                            ]
                        }
                        logger.info(
                            "{}:\n"
                            "{} original items total".format(
                                _new_path,
                                len(dict1[k])
                            )
                        )
                        if _removed:
                            logger.info(
                                "{}".format('\n'.join(_removed_str_lst))
                            )
                        if _added:
                            logger.info(
                                "{}".format('\n'.join(_added_str_lst))
                            )
                else:
                    # in case of type mismatch
                    # considering it as not equal
                    d1 = dict1
                    d2 = dict2
                    val1 = d1[k] if isinstance(d1, dict) else d1
                    val2 = d2[k] if isinstance(d2, dict) else d2
                    try:
                        match = val1 == val2
                    except TypeError as e:
                        logger.warning(
                            "One of the values is not a dict: "
                            "{}, {}".format(
                                str(dict1),
                                str(dict2)
                            ))
                        match = False
                    if not match:
                        _report[_new_path] = {
                            "type": "value",
                            "raw_values": [val1, val2],
                            "str_values": [
                                "{}".format(val1),
                                "{}".format(val2)
                            ]
                        }
                        logger.info("{}: {}, {}".format(
                            _new_path,
                            val1,
                            val2
                        ))
        return _report

    def generate_model_report_tree(self):
        """Use two loaded models to generate comparison table with
        values are groupped by YAML files
        """
        # We are to cut both models into logical pieces
        # nodes, will not be equal most of the time
        # system, must be pretty much the same or we in trouble
        # cluster, will be the most curious part for comparison
        # other, all of the rest

        _diff_report = {}
        for _key in self._model_parts.keys():
            # tmp report for keys
            _tmp_diffs = self.find_changes(
                self.models[self.model_name_1][_key],
                self.models[self.model_name_2][_key]
            )
            # prettify the report
            for key in _tmp_diffs.keys():
                # break the key in two parts
                _ext = ".yml"
                if ".yaml" in key:
                    _ext = ".yaml"
                _split = key.split(_ext)
                _file_path = _split[0]
                _param_path = "none"
                if len(_split) > 1:
                    _param_path = _split[1]
                _tmp_diffs[key].update({
                    "class_file": _file_path + _ext,
                    "param": _param_path,
                })
            _diff_report[_key[3:]] = {
                "path": self._model_parts[_key],
                "diffs": _tmp_diffs
            }

        _diff_report["diff_names"] = [self.model_name_1, self.model_name_2]
        return _diff_report

    def compare_models(self):
        # Do actual compare using model names from the class
        self.load_model_tree(
            self.model_name_1,
            self.model_path_1
        )
        self.load_model_tree(
            self.model_name_2,
            self.model_path_2
        )
        # Models should have similar structure to be compared
        # classes/system
        # classes/cluster
        # nodes

        diffs = self.generate_model_report_tree()

        report_file = \
            self.model_name_1 + "-vs-" + self.model_name_2 + ".html"
        # HTML report class is post-callable
        report = reporter.ReportToFile(
            reporter.HTMLModelCompare(),
            report_file
        )
        logger_cli.info("...generating report to {}".format(report_file))
        # report will have tabs for each of the comparable entities in diffs
        report({
            "nodes": {},
            "rc_diffs": diffs,
        })
        # with open("./gen_tree.json", "w+") as _out:
        #     _out.write(json.dumps(mComparer.generate_model_report_tree))

        return