cfg_checker/modules/packages/checker.py

import json
import os
#import sys

from copy import deepcopy

from cfg_checker.common.exception import ConfigException
from cfg_checker.common import utils, const
from cfg_checker.common import config, logger, logger_cli, pkg_dir
from cfg_checker.common import salt_utils
from cfg_checker.helpers.console_utils import Progress
from cfg_checker.nodes import SaltNodes, node_tmpl
from cfg_checker.reports import reporter

from versions import PkgVersions, DebianVersion, VersionCmpResult


class CloudPackageChecker(SaltNodes):
    @staticmethod
    def presort_packages(all_packages, full=None):
        logger_cli.info("-> Presorting packages")
        # labels
        _data = {}
        _data = {
            "cs": {
                "ok": const.VERSION_OK,
                "up": const.VERSION_UP,
                "down": const.VERSION_DOWN,
                "err": const.VERSION_ERR
            },
            "ca": {
                "na": const.ACT_NA,
                "up": const.ACT_UPGRADE,
                "need_up": const.ACT_NEED_UP,
                "need_down": const.ACT_NEED_DOWN,
                "repo": const.ACT_REPO
            }
        }
        _data['status_err'] = const.VERSION_ERR
        _data['status_down'] = const.VERSION_DOWN

        # Presort packages
        _data['critical'] = {}
        _data['system'] = {}
        _data['other'] = {}
        _data['unlisted'] = {}

        _l = len(all_packages)
        _progress = Progress(_l)
        _progress_index = 0
        # counters
        _ec = _es = _eo = _eu = 0
        _dc = _ds = _do = _du = 0
        while _progress_index < _l:
            # progress bar
            _progress_index += 1
            _progress.write_progress(_progress_index)
            # sort packages
            _pn, _val = all_packages.popitem()
            _c = _val['desc']['component']
            if full:
                # Check if this packet has errors
                # if all is ok -> just skip it
                _max_status = max(_val['results'].keys())
                if _max_status <= const.VERSION_OK:
                    _max_action = max(_val['results'][_max_status].keys())
                    if _max_action == const.ACT_NA:
                        # this package do not ha any comments
                        # ...just skip it from report
                        continue

            if len(_c) > 0 and _c == 'unlisted':
                # not listed package in version lib
                _data['unlisted'].update({
                    _pn: _val
                })
                _eu += _val['results'].keys().count(const.VERSION_ERR)
                _du += _val['results'].keys().count(const.VERSION_DOWN)
            # mirantis/critical
            elif len(_c) > 0 and _c != 'System':
                # not blank and not system
                _data['critical'].update({
                    _pn: _val
                })
                _ec += _val['results'].keys().count(const.VERSION_ERR)
                _dc += _val['results'].keys().count(const.VERSION_DOWN)
            # system
            elif _c == 'System':
                _data['system'].update({
                    _pn: _val
                })
                _es += _val['results'].keys().count(const.VERSION_ERR)
                _ds += _val['results'].keys().count(const.VERSION_DOWN)
            # rest
            else:
                _data['other'].update({
                    _pn: _val
                })
                _eo += _val['results'].keys().count(const.VERSION_ERR)
                _do += _val['results'].keys().count(const.VERSION_DOWN)

        
        _progress.newline()

        _data['errors'] = {
            'mirantis': _ec,
            'system': _es,
            'other': _eo,
            'unlisted': _eu
        }
        _data['downgrades'] = {
            'mirantis': _dc,
            'system': _ds,
            'other': _do,
            'unlisted': _du
        }

        return _data

    def collect_installed_packages(self):
        """
        Collect installed packages on each node
        sets 'installed' dict property in the class

        :return: none
        """
        logger_cli.info("# Collecting installed packages")
        _result = self.execute_script_on_active_nodes("pkg_versions.py")

        for key in self.nodes.keys():
            # due to much data to be passed from salt, it is happening in order
            if key in _result:
                _text = _result[key]
                try:
                    _dict = json.loads(_text[_text.find('{'):])
                except ValueError as e:
                    logger_cli.info("... no JSON for '{}'".format(
                        key
                    ))
                    logger_cli.debug("ERROR:\n{}\n".format(_text[:_text.find('{')]))
                    _dict = {}
                
                self.nodes[key]['packages'] = _dict
            else:
                self.nodes[key]['packages'] = {}
            logger_cli.debug("... {} has {} packages installed".format(
                key,
                len(self.nodes[key]['packages'].keys())
            ))
        logger_cli.info("-> Done")

    def collect_packages(self):
        """
        Check package versions in repos vs installed

        :return: no return values, all date put to dict in place
        """
        # Preload OpenStack release versions
        _desc = PkgVersions()
        
        logger_cli.info("# Cross-comparing: Installed vs Candidates vs Release")
        _progress = Progress(len(self.nodes.keys()))
        _progress_index = 0
        _total_processed = 0
        # Collect packages from all of the nodes in flat dict
        _all_packages = {}
        for node_name, node_value in self.nodes.iteritems():
            _uniq_len = len(_all_packages.keys())
            _progress_index += 1
            # progress will jump from node to node
            # it is very costly operation to execute it for each pkg
            _progress.write_progress(
                _progress_index,
                note="/ {} uniq out of {} packages found".format(
                    _uniq_len,
                    _total_processed
                )
            )
            for _name, _value in node_value['packages'].iteritems():
                _total_processed += 1
                # Parse versions
                _ver_ins = DebianVersion(_value['installed'])
                _ver_can = DebianVersion(_value['candidate'])

                # All packages list with version and node list
                if _name not in _all_packages:
                    # shortcuts for this cloud values
                    _os = self.openstack_release
                    _mcp = self.mcp_release
                    _pkg_desc = {}
                    if _desc[_name]:
                        # shortcut to version library
                        _vers = _desc[_name]['versions']
                        _pkg_desc = _desc[_name]
                    else:
                        # no description - no library :)
                        _vers = {}
                        _pkg_desc = _desc.dummy_desc
                    
                    # get specific set for this OS release if present
                    if _os in _vers:
                        _v = _vers[_os] 
                    elif 'any' in _vers:
                        _v = _vers['any']
                    else:
                        _v = {}
                    # Finally, get specific version
                    _release = DebianVersion(_v[_mcp] if _mcp in _v else '')
                    # Populate package info
                    _all_packages[_name] = {
                        "desc": _pkg_desc,
                        "results": {},
                        "r": _release,
                    }
                
                _cmp = VersionCmpResult(
                    _ver_ins,
                    _ver_can,
                    _all_packages[_name]['r']
                )
                
                # shortcut to results
                _res = _all_packages[_name]['results']
                # update status
                if _cmp.status not in _res:
                    _res[_cmp.status] = {}
                # update action
                if _cmp.action not in _res[_cmp.status]:
                    _res[_cmp.status][_cmp.action] = {}
                # update node
                if node_name not in _res[_cmp.status][_cmp.action]:
                    _res[_cmp.status][_cmp.action][node_name] = {}
                # put result
                _res[_cmp.status][_cmp.action][node_name] = {
                    'i': _ver_ins,
                    'c': _ver_can,
                    'res': _cmp,
                    'raw': _value['raw']
                }

        self._packages = _all_packages
        _progress.newline()
    

    def create_report(self, filename, rtype, full=None):
        """
        Create static html showing packages diff per node

        :return: buff with html
        """
        logger_cli.info("# Generating report to '{}'".format(filename))
        if rtype == 'html':
            _type = reporter.HTMLPackageCandidates()
        elif rtype == 'csv':
            _type = reporter.CSVAllPackages()
        else:
            raise ConfigException("Report type not set")
        _report = reporter.ReportToFile(
            _type,
            filename
        )
        payload = {
            "nodes": self.nodes,
            "mcp_release": self.mcp_release,
            "openstack_release": self.openstack_release
        }
        payload.update(self.presort_packages(self._packages, full))
        _report(payload)
        logger_cli.info("-> Done")