ci_checker/reclass_cmp.py

"""Model Comparer:
- yaml parser
- class tree comparison
"""
import itertools
# import json
import os
import yaml

import reporter
from ci_checker.common import utils
from ci_checker.common import base_config, logger, logger_cli, PKG_DIR


global prefix_name
global model_name_1, model_path_1
global model_name_2, model_path_2

prefix_name = "emk"
model_name_1 = "dev"
model_path_1 = "/Users/savex/proj/mediakind/reclass-dev"
model_name_2 = "stg"
model_path_2 = "/Users/savex/proj/mediakind/reclass-stg"


class ModelComparer(object):
    """Collection of functions to compare model data.
    """
    models = {}

    @staticmethod
    def load_yaml_class(fname):
        """Loads a yaml from the file and forms a tree item

        Arguments:
            fname {string} -- full path to the yaml file
        """
        _yaml = {}
        try:
            _size = 0
            with open(fname, 'r') as f:
                _yaml = yaml.load(f)
                _size = f.tell()
            # TODO: do smth with the data
            if not _yaml:
                logger_cli.warning("WARN: empty file '{}'".format(fname))
                _yaml = {}
            else:
                logger.debug("...loaded YAML '{}' ({}b)".format(fname, _size))
            return _yaml
        except yaml.YAMLError as exc:
            logger_cli.error(exc)
        except IOError as e:
            logger_cli.error(
                "Error loading file '{}': {}".format(fname, e.message)
            )
            raise Exception("CRITICAL: Failed to load YAML data: {}".format(
                e.message + e.strerror
            ))

    def load_model_tree(self, name, root_path="/srv/salt/reclass"):
        """Walks supplied path for the YAML filed and loads the tree

        Arguments:
            root_folder_path {string} -- Path to Model's root folder. Optional
        """
        logger_cli.info("Loading reclass tree from '{}'".format(root_path))
        # prepare the file tree to walk
        raw_tree = {}
        # Credits to Andrew Clark@MIT. Original code is here:
        # http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/
        root_path = root_path.rstrip(os.sep)
        start = root_path.rfind(os.sep) + 1
        root_key = root_path.rsplit(os.sep, 1)[1]
        # Look Ma! I am walking the file tree with no recursion!
        for path, dirs, files in os.walk(root_path):
            # if this is a hidden folder, ignore it
            _filders_list = path[start:].split(os.sep)
            if any(item.startswith(".") for item in _filders_list):
                continue
            # cut absolute part of the path and split folder names
            folders = path[start:].split(os.sep)
            subdir = {}
            # create generator of files that are not hidden
            _exts = ('.yml', '.yaml')
            _subfiles = (file for file in files
                         if file.endswith(_exts) and not file.startswith('.'))
            for _file in _subfiles:
                # cut file extension. All reclass files are '.yml'
                _subnode = _file
                # load all YAML class data into the tree
                subdir[_subnode] = self.load_yaml_class(
                    os.path.join(path, _file)
                )
                try:
                    # Save original filepath, just in case
                    subdir[_subnode]["_source"] = os.path.join(
                        path[start:],
                        _file
                    )
                except Exception:
                    logger.warning(
                        "Non-yaml file detected: {}".format(_file)
                    )
            # creating dict structure out of folder list. Pure python magic
            parent = reduce(dict.get, folders[:-1], raw_tree)
            parent[folders[-1]] = subdir
        # save it as a single data object
        self.models[name] = raw_tree[root_key]
        return True

    def generate_model_report_tree(self):
        """Use all loaded models to generate comparison table with
        values are groupped by YAML files
        """
        def find_changes(dict1, dict2, path=""):
            _report = {}
            for k in dict1.keys():
                if not isinstance(k, str):
                    _new_path = path + ":" + str(k)
                else:
                    _new_path = path + ":" + k
                if k == "_source":
                    continue
                # check if this is an env name cluster entry
                if dict2 is not None and \
                        k == model_name_1 and \
                        model_name_2 in dict2.keys():
                    k1 = model_name_1
                    k2 = model_name_2
                    if type(dict1[k1]) is dict:
                        if path == "":
                            _new_path = k1
                        _child_report = find_changes(
                            dict1[k1],
                            dict2[k2],
                            _new_path
                        )
                        _report.update(_child_report)
                elif dict2 is None or k not in dict2:
                    # no key in dict2
                    _report[_new_path] = {
                        "type": "value",
                        "raw_values": [dict1[k], "N/A"],
                        "str_values": [
                            "{}".format(dict1[k]),
                            "n/a"
                        ]
                    }
                    logger.info(
                        "{}: {}, {}".format(_new_path, dict1[k], "N/A")
                    )
                else:
                    if type(dict1[k]) is dict:
                        if path == "":
                            _new_path = k
                        _child_report = find_changes(
                            dict1[k],
                            dict2[k],
                            _new_path
                        )
                        _report.update(_child_report)
                    elif type(dict1[k]) is list and type(dict2[k]) is list:
                        # use ifilterfalse to compare lists of dicts
                        try:
                            _removed = list(
                                itertools.ifilterfalse(
                                    lambda x: x in dict2[k],
                                    dict1[k]
                                )
                            )
                            _added = list(
                                itertools.ifilterfalse(
                                    lambda x: x in dict1[k],
                                    dict2[k]
                                )
                            )
                        except TypeError as e:
                            # debug routine,
                            # should not happen, due to list check above
                            logger.error(
                                "Caught lambda type mismatch: {}".format(
                                    e.message
                                )
                            )
                            logger_cli.warning(
                                "Types mismatch for correct compare: "
                                "{}, {}".format(
                                    type(dict1[k]),
                                    type(dict2[k])
                                )
                            )
                            _removed = None
                            _added = None
                        _original = ["= {}".format(item) for item in dict1[k]]
                        if _removed or _added:
                            _removed_str_lst = ["- {}".format(item)
                                                for item in _removed]
                            _added_str_lst = ["+ {}".format(item)
                                              for item in _added]
                            _report[_new_path] = {
                                "type": "list",
                                "raw_values": [
                                    dict1[k],
                                    _removed_str_lst + _added_str_lst
                                ],
                                "str_values": [
                                    "{}".format('\n'.join(_original)),
                                    "{}\n{}".format(
                                        '\n'.join(_removed_str_lst),
                                        '\n'.join(_added_str_lst)
                                    )
                                ]
                            }
                            logger.info(
                                "{}:\n"
                                "{} original items total".format(
                                    _new_path,
                                    len(dict1[k])
                                )
                            )
                            if _removed:
                                logger.info(
                                    "{}".format('\n'.join(_removed_str_lst))
                                )
                            if _added:
                                logger.info(
                                    "{}".format('\n'.join(_added_str_lst))
                                )
                    else:
                        # in case of type mismatch
                        # considering it as not equal
                        d1 = dict1
                        d2 = dict2
                        val1 = d1[k] if isinstance(d1, dict) else d1
                        val2 = d2[k] if isinstance(d2, dict) else d2
                        try:
                            match = val1 == val2
                        except TypeError as e:
                            logger.warning(
                                "One of the values is not a dict: "
                                "{}, {}".format(
                                    str(dict1),
                                    str(dict2)
                                ))
                            match = False
                        if not match:
                            _report[_new_path] = {
                                "type": "value",
                                "raw_values": [val1, val2],
                                "str_values": [
                                    "{}".format(val1),
                                    "{}".format(val2)
                                ]
                            }
                            logger.info("{}: {}, {}".format(
                                _new_path,
                                val1,
                                val2
                            ))
            return _report
        # tmp report for keys
        diff_report = find_changes(
            self.models[model_name_1],
            self.models[model_name_2]
        )
        # prettify the report
        for key in diff_report.keys():
            # break the key in two parts
            _ext = ".yml"
            if ".yaml" in key:
                _ext = ".yaml"
            _split = key.split(_ext)
            _file_path = _split[0]
            _param_path = "none"
            if len(_split) > 1:
                _param_path = _split[1]
            diff_report[key].update({
                "class_file": _file_path + _ext,
                "param": _param_path,
            })

        diff_report["diff_names"] = [model_name_1, model_name_2]
        return diff_report


# temporary executing the parser as a main prog
if __name__ == '__main__':
    mComparer = ModelComparer()
    mComparer.load_model_tree(
        model_name_1,
        model_path_1
    )
    mComparer.load_model_tree(
        model_name_2,
        model_path_2
    )
    diffs = mComparer.generate_model_report_tree()

    report_file = \
        prefix_name + "-" + model_name_1 + "-vs-" + model_name_2 + ".html"
    report = reporter.ReportToFile(
        reporter.HTMLModelCompare(),
        report_file
    )
    logger_cli.info("...generating report to {}".format(report_file))
    report({
        "nodes": {},
        "diffs": diffs
    })
    # with open("./gen_tree.json", "w+") as _out:
    #     _out.write(json.dumps(mComparer.generate_model_report_tree))