cfg_checker/modules/packages/checker.py

import json

from cfg_checker.common import const, logger_cli
from cfg_checker.common.exception import ConfigException
from cfg_checker.helpers.console_utils import Progress
from cfg_checker.modules.packages.repos import RepoManager
from cfg_checker.nodes import salt_master
from cfg_checker.reports import reporter

from versions import DebianVersion, PkgVersions, VersionCmpResult


class CloudPackageChecker(object):
    rm = RepoManager()

    def __init__(self):
        # Init salt master info
        if not salt_master.nodes:
            salt_master.nodes = salt_master.get_nodes()

        # check that this env tag is present in Manager
        _tags = self.rm.get_available_tags(tag=salt_master.mcp_release)
        if not _tags:
            logger_cli.warning(
                "\n# hWARNING: '{0}' is not listed in repo index. "
                "Consider running:\n\t{1}\nto add info on this tag's "
                "release package versions".format(
                    salt_master.mcp_release,
                    "mcp-checker packages versions --tag {0}"
                )
            )

    @staticmethod
    def presort_packages(all_packages, full=None):
        logger_cli.info("-> Presorting packages")
        # labels
        _data = {}
        _data = {
            "cs": {
                "ok": const.VERSION_OK,
                "up": const.VERSION_UP,
                "down": const.VERSION_DOWN,
                "err": const.VERSION_ERR
            },
            "ca": {
                "na": const.ACT_NA,
                "up": const.ACT_UPGRADE,
                "need_up": const.ACT_NEED_UP,
                "need_down": const.ACT_NEED_DOWN,
                "repo": const.ACT_REPO
            }
        }
        _data['status_err'] = const.VERSION_ERR
        _data['status_down'] = const.VERSION_DOWN

        # Presort packages
        _data['critical'] = {}
        _data['system'] = {}
        _data['other'] = {}
        _data['unlisted'] = {}

        _l = len(all_packages)
        _progress = Progress(_l)
        _progress_index = 0
        # counters
        _ec = _es = _eo = _eu = 0
        _dc = _ds = _do = _du = 0
        while _progress_index < _l:
            # progress bar
            _progress_index += 1
            _progress.write_progress(_progress_index)
            # sort packages
            _pn, _val = all_packages.popitem()
            _c = _val['desc']['section']

            if not full:
                # Check if this packet has errors
                # if all is ok -> just skip it
                _max_status = max(_val['results'].keys())
                if _max_status <= const.VERSION_OK:
                    _max_action = max(_val['results'][_max_status].keys())
                    if _max_action == const.ACT_NA:
                        # this package does not have any comments
                        # ...just skip it from report
                        continue

            if len(_c) > 0 and _val['is_mirantis'] is None:
                # not listed package in version lib
                _data['unlisted'].update({
                    _pn: _val
                })
                _eu += _val['results'].keys().count(const.VERSION_ERR)
                _du += _val['results'].keys().count(const.VERSION_DOWN)
            # mirantis/critical
            # elif len(_c) > 0 and _c != 'System':
            elif _val['is_mirantis']:
                # not blank and not system
                _data['critical'].update({
                    _pn: _val
                })
                _ec += _val['results'].keys().count(const.VERSION_ERR)
                _dc += _val['results'].keys().count(const.VERSION_DOWN)
            # system
            elif _c == 'System':
                _data['system'].update({
                    _pn: _val
                })
                _es += _val['results'].keys().count(const.VERSION_ERR)
                _ds += _val['results'].keys().count(const.VERSION_DOWN)
            # rest
            else:
                _data['other'].update({
                    _pn: _val
                })
                _eo += _val['results'].keys().count(const.VERSION_ERR)
                _do += _val['results'].keys().count(const.VERSION_DOWN)

        _progress.end()

        _data['errors'] = {
            'mirantis': _ec,
            'system': _es,
            'other': _eo,
            'unlisted': _eu
        }
        _data['downgrades'] = {
            'mirantis': _dc,
            'system': _ds,
            'other': _do,
            'unlisted': _du
        }

        return _data

    def collect_installed_packages(self):
        """
        Collect installed packages on each node
        sets 'installed' dict property in the class

        :return: none
        """
        logger_cli.info("# Collecting installed packages")
        salt_master.prepare_script_on_active_nodes("pkg_versions.py")
        _result = salt_master.execute_script_on_active_nodes("pkg_versions.py")

        for key in salt_master.nodes.keys():
            # due to much data to be passed from salt, it is happening in order
            if key in _result:
                _text = _result[key]
                try:
                    _dict = json.loads(_text[_text.find('{'):])
                except ValueError:
                    logger_cli.info("... no JSON for '{}'".format(
                        key
                    ))
                    logger_cli.debug(
                        "ERROR:\n{}\n".format(_text[:_text.find('{')])
                    )
                    _dict = {}

                salt_master.nodes[key]['packages'] = _dict
            else:
                salt_master.nodes[key]['packages'] = {}
            logger_cli.debug("... {} has {} packages installed".format(
                key,
                len(salt_master.nodes[key]['packages'].keys())
            ))
        logger_cli.info("-> Done")

    def collect_packages(self):
        """
        Check package versions in repos vs installed

        :return: no return values, all date put to dict in place
        """
        # Preload OpenStack release versions
        _desc = PkgVersions()

        logger_cli.info(
            "# Cross-comparing: Installed vs Candidates vs Release"
        )
        # shortcuts for this cloud values
        _os = salt_master.openstack_release
        _mcp = salt_master.mcp_release
        # Progress class
        _progress = Progress(len(salt_master.nodes.keys()))
        _progress_index = 0
        _total_processed = 0
        # Collect packages from all of the nodes in flat dict
        _all_packages = {}
        _all_tags = set([])
        # get env tag's year and major version
        _tag_major = _mcp[:_mcp.find('.', _mcp.find('.')+1)]
        for node_name, node_value in salt_master.nodes.iteritems():
            _uniq_len = len(_all_packages.keys())
            _progress_index += 1
            # progress updates shown before next node only
            # it is costly operation to do it for each of the 150k packages
            _progress.write_progress(
                _progress_index,
                note="/ {} uniq out of {} packages found".format(
                    _uniq_len,
                    _total_processed
                )
            )
            for _name, _value in node_value['packages'].iteritems():
                _total_processed += 1
                # Parse versions
                _ver_ins = DebianVersion(_value['installed'])
                _ver_can = DebianVersion(_value['candidate'])

                # All packages list with version and node list
                if _name not in _all_packages:
                    # get repo versions list,
                    _linux = salt_master.nodes[node_name]['linux_codename']
                    _arch = salt_master.nodes[node_name]['linux_arch']
                    if _name == u'qemu-block-extra':
                        a = 1
                    # omit tag as target versions might be of different tag
                    _r = self.rm.get_filtered_versions(
                        _name,
                        tag=_mcp,
                        include=[_os, _linux, _arch],
                        exclude=["nightly"]
                    )
                    # save versions for matching major tags
                    _vs = {}
                    _sections = {}
                    _apps = {}
                    # get all versions for this year
                    for s, apps in _r.iteritems():
                        for a, versions in apps.iteritems():
                            for v, repos in versions.iteritems():
                                for repo in repos:
                                    t = repo['tag']
                                    _major = t[:t.find('.', t.find('.')+1)]
                                    if _tag_major == _major:
                                        if v not in _vs:
                                            _vs[v] = []
                                        _vs[v].append(repo)
                                        if v not in _sections:
                                            _sections[v] = []
                                        _sections[v].append(s)
                                        if v not in _apps:
                                            _apps[v] = []
                                        _apps[v].append(a)

                    # check if we have candidate version among found
                    _r_desc = []
                    _vs_keys = _vs.keys()
                    if _vs_keys:
                        _newest = _newest = DebianVersion(_vs_keys.pop())
                    else:
                        _newest = DebianVersion('')
                    # if _ver_ins.version in _vs_keys:
                    #     # exact match, save it
                    #     _release = _ver_ins
                    # else:
                    # detect newest version among saved

                    for v in _vs_keys:
                        _this = DebianVersion(v)
                        if _this > _newest:
                            _newest = _this
                    # newest version for the YEAR.MAJOR will be the release
                    _release = _newest
                    # save repos list for this version
                    if _release.version != 'n/a':
                        _r_desc = _vs[_release.version]
                    # preload special description
                    if _desc[_name]:
                        _pkg_desc = _desc[_name]
                    else:
                        _pkg_desc = _desc.dummy_desc
                    # Check if we can provide better from the package
                    if _release.version != 'n/a':
                        if not _pkg_desc['section']:
                            _pkg_desc['section'] = \
                                "/".join(_sections[_release.version])
                        if not _pkg_desc['app']:
                            _pkg_desc['app'] = \
                                "/".join(_apps[_release.version])

                    # get specific set for this OS release if present
                    # if not, try search in all repos for this tag
                    # _r_desc = _r[_newest.version]
                    # if _r:
                    #     _vs = _r.keys()
                    #     if len(_vs) > 1:
                    #         # search best match
                    #         _release = None
                    #         for _v in _vs:
                    #             _deb = DebianVersion(_v)
                    #             if _ver_can == _deb:
                    #                 _release = _deb
                    #                 _r_desc = _r[_v]
                    #                 break
                    #         if not _release:
                    #             _progress.clearline()
                    #             logger_cli.error(
                    #                 "# ERROR: No release version found "
                    #                 "for '{}'".format(_name)
                    #             )
                    #             _release = DebianVersion('')
                    #     else:
                    #         _release = DebianVersion(_vs[0])
                    #         _r_desc = _r[_vs[0]]
                    # else:
                    #     # not found... 99% that it will not happen
                    #     _release = DebianVersion('')

                    # Old versions match routine
                    # ########
                    # if _os in _vers:
                    #     _v = _vers[_os]
                    # elif 'any' in _vers:
                    #     _v = _vers['any']
                    # else:
                    #     _v = {}

                    # Finally, get specific version
                    # _release = DebianVersion(_v[_mcp] if _mcp in _v else '')
                    for repo in _r_desc:
                        _all_tags.add(repo['tag'])

                    # Populate package info
                    _m = _r_desc[0]["maintainer"] if _r_desc else 'n/a'
                    _all_packages[_name] = {
                        "desc": _pkg_desc,
                        "repos": _r_desc,
                        "maintainer": _m,
                        "is_mirantis": self.rm.is_mirantis(
                            _name,
                            tag=_tag_major
                        ),
                        "results": {},
                        "r": _release,
                    }

                _cmp = VersionCmpResult(
                    _ver_ins,
                    _ver_can,
                    _all_packages[_name]['r']
                )

                # shortcut to results
                _res = _all_packages[_name]['results']
                # update status
                if _cmp.status not in _res:
                    _res[_cmp.status] = {}
                # update action
                if _cmp.action not in _res[_cmp.status]:
                    _res[_cmp.status][_cmp.action] = {}
                # update node
                if node_name not in _res[_cmp.status][_cmp.action]:
                    _res[_cmp.status][_cmp.action][node_name] = {}
                # put result
                _res[_cmp.status][_cmp.action][node_name] = {
                    'i': _ver_ins,
                    'c': _ver_can,
                    'res': _cmp,
                    'raw': _value['raw']
                }

        self._packages = _all_packages
        _progress.end()

    def create_report(self, filename, rtype, full=None):
        """
        Create static html showing packages diff per node

        :return: buff with html
        """
        logger_cli.info("# Generating report to '{}'".format(filename))
        if rtype == 'html':
            _type = reporter.HTMLPackageCandidates()
        elif rtype == 'csv':
            _type = reporter.CSVAllPackages()
        else:
            raise ConfigException("Report type not set")
        _report = reporter.ReportToFile(
            _type,
            filename
        )
        payload = {
            "nodes": salt_master.nodes,
            "mcp_release": salt_master.mcp_release,
            "openstack_release": salt_master.openstack_release
        }
        payload.update(self.presort_packages(self._packages, full))
        _report(payload)
        logger_cli.info("-> Done")