| import json |
| |
| from cfg_checker.common import const, logger_cli |
| from cfg_checker.common.exception import ConfigException |
| from cfg_checker.helpers.console_utils import Progress |
| from cfg_checker.modules.packages.repos import RepoManager |
| from cfg_checker.nodes import salt_master |
| from cfg_checker.reports import reporter |
| |
| from versions import DebianVersion, PkgVersions, VersionCmpResult |
| |
| |
| class CloudPackageChecker(object): |
| def __init__(self, force_tag=None, exclude_keywords=[]): |
| # Init salt master info |
| if not salt_master.nodes: |
| salt_master.nodes = salt_master.get_nodes() |
| |
| # check that this env tag is present in Manager |
| self.rm = RepoManager() |
| _tags = self.rm.get_available_tags(tag=salt_master.mcp_release) |
| if not _tags: |
| logger_cli.warning( |
| "\n# hWARNING: '{0}' is not listed in repo index. " |
| "Consider running:\n\t{1}\nto add info on this tag's " |
| "release package versions".format( |
| salt_master.mcp_release, |
| "mcp-checker packages versions --tag {0}" |
| ) |
| ) |
| |
| self.force_tag = force_tag |
| self.exclude_repos_keywords = exclude_keywords |
| |
| @staticmethod |
| def presort_packages(all_packages, full=None): |
| logger_cli.info("-> Presorting packages") |
| # labels |
| _data = {} |
| _data = { |
| "cs": { |
| "ok": const.VERSION_OK, |
| "up": const.VERSION_UP, |
| "down": const.VERSION_DOWN, |
| "err": const.VERSION_ERR |
| }, |
| "ca": { |
| "na": const.ACT_NA, |
| "up": const.ACT_UPGRADE, |
| "need_up": const.ACT_NEED_UP, |
| "need_down": const.ACT_NEED_DOWN, |
| "repo": const.ACT_REPO |
| } |
| } |
| _data['status_err'] = const.VERSION_ERR |
| _data['status_down'] = const.VERSION_DOWN |
| |
| # Presort packages |
| _data['critical'] = {} |
| _data['system'] = {} |
| _data['other'] = {} |
| _data['unlisted'] = {} |
| |
| _l = len(all_packages) |
| _progress = Progress(_l) |
| _progress_index = 0 |
| # counters |
| _ec = _es = _eo = _eu = 0 |
| _dc = _ds = _do = _du = 0 |
| while _progress_index < _l: |
| # progress bar |
| _progress_index += 1 |
| _progress.write_progress(_progress_index) |
| # sort packages |
| _pn, _val = all_packages.popitem() |
| _c = _val['desc']['section'] |
| |
| if not full: |
| # Check if this packet has errors |
| # if all is ok -> just skip it |
| _max_status = max(_val['results'].keys()) |
| if _max_status <= const.VERSION_OK: |
| _max_action = max(_val['results'][_max_status].keys()) |
| if _max_action == const.ACT_NA: |
| # this package does not have any comments |
| # ...just skip it from report |
| continue |
| |
| if len(_c) > 0 and _val['is_mirantis'] is None: |
| # not listed package in version lib |
| _data['unlisted'].update({ |
| _pn: _val |
| }) |
| _eu += _val['results'].keys().count(const.VERSION_ERR) |
| _du += _val['results'].keys().count(const.VERSION_DOWN) |
| # mirantis/critical |
| # elif len(_c) > 0 and _c != 'System': |
| elif _val['is_mirantis']: |
| # not blank and not system |
| _data['critical'].update({ |
| _pn: _val |
| }) |
| _ec += _val['results'].keys().count(const.VERSION_ERR) |
| _dc += _val['results'].keys().count(const.VERSION_DOWN) |
| # system |
| elif _c == 'System': |
| _data['system'].update({ |
| _pn: _val |
| }) |
| _es += _val['results'].keys().count(const.VERSION_ERR) |
| _ds += _val['results'].keys().count(const.VERSION_DOWN) |
| # rest |
| else: |
| _data['other'].update({ |
| _pn: _val |
| }) |
| _eo += _val['results'].keys().count(const.VERSION_ERR) |
| _do += _val['results'].keys().count(const.VERSION_DOWN) |
| |
| _progress.end() |
| |
| _data['errors'] = { |
| 'mirantis': _ec, |
| 'system': _es, |
| 'other': _eo, |
| 'unlisted': _eu |
| } |
| _data['downgrades'] = { |
| 'mirantis': _dc, |
| 'system': _ds, |
| 'other': _do, |
| 'unlisted': _du |
| } |
| |
| return _data |
| |
| def collect_installed_packages(self): |
| """ |
| Collect installed packages on each node |
| sets 'installed' dict property in the class |
| |
| :return: none |
| """ |
| logger_cli.info("# Collecting installed packages") |
| salt_master.prepare_script_on_active_nodes("pkg_versions.py") |
| _result = salt_master.execute_script_on_active_nodes("pkg_versions.py") |
| |
| for key in salt_master.nodes.keys(): |
| # due to much data to be passed from salt, it is happening in order |
| if key in _result and _result[key]: |
| _text = _result[key] |
| try: |
| _dict = json.loads(_text[_text.find('{'):]) |
| except ValueError: |
| logger_cli.info("... no JSON for '{}'".format( |
| key |
| )) |
| logger_cli.debug( |
| "ERROR:\n{}\n".format(_text[:_text.find('{')]) |
| ) |
| _dict = {} |
| |
| salt_master.nodes[key]['packages'] = _dict |
| else: |
| salt_master.nodes[key]['packages'] = {} |
| logger_cli.debug("... {} has {} packages installed".format( |
| key, |
| len(salt_master.nodes[key]['packages'].keys()) |
| )) |
| logger_cli.info("-> Done") |
| |
| def collect_packages(self): |
| """ |
| Check package versions in repos vs installed |
| |
| :return: no return values, all date put to dict in place |
| """ |
| # Preload OpenStack release versions |
| _desc = PkgVersions() |
| logger_cli.info( |
| "# Cross-comparing: Installed vs Candidates vs Release" |
| ) |
| # shortcuts for this cloud values |
| _os = salt_master.openstack_release |
| _mcp = salt_master.mcp_release |
| _t = [self.force_tag] if self.force_tag else [] |
| _t.append(_mcp) |
| |
| logger_cli.info("# Tag search list: {}".format(", ".join(_t))) |
| logger_cli.info("# Openstack version: {}".format(_os)) |
| logger_cli.info( |
| "# Release versions repos keyword exclude list: {}".format( |
| ", ".join(self.exclude_repos_keywords) |
| ) |
| ) |
| |
| # Progress class |
| _progress = Progress(len(salt_master.nodes.keys())) |
| _progress_index = 0 |
| _total_processed = 0 |
| # Collect packages from all of the nodes in flat dict |
| _all_packages = {} |
| for node_name, node_value in salt_master.nodes.iteritems(): |
| _uniq_len = len(_all_packages.keys()) |
| _progress_index += 1 |
| # progress updates shown before next node only |
| # it is costly operation to do it for each of the 150k packages |
| _progress.write_progress( |
| _progress_index, |
| note="/ {} uniq out of {} packages found".format( |
| _uniq_len, |
| _total_processed |
| ) |
| ) |
| for _name, _value in node_value['packages'].iteritems(): |
| _total_processed += 1 |
| # Parse versions from nodes |
| _ver_ins = DebianVersion(_value['installed']) |
| _ver_can = DebianVersion(_value['candidate']) |
| |
| # Process package description and release version |
| # at a first sight |
| if _name not in _all_packages: |
| # get node attributes |
| _linux = salt_master.nodes[node_name]['linux_codename'] |
| _arch = salt_master.nodes[node_name]['linux_arch'] |
| # get versions for tag, Openstack release and repo headers |
| # excluding 'nightly' repos by default |
| _r = {} |
| # if there is a forced tag = use it |
| if self.force_tag: |
| _r = self.rm.get_filtered_versions( |
| _name, |
| tag=self.force_tag, |
| include=[_os, _linux, _arch], |
| exclude=self.exclude_repos_keywords |
| ) |
| # if nothing found, look everywhere |
| if not _r: |
| _r = self.rm.get_filtered_versions( |
| _name, |
| tag=self.force_tag, |
| include=[_linux, _arch], |
| exclude=self.exclude_repos_keywords |
| ) |
| # if nothing is found at this point, |
| # repeat search using normal tags |
| if not _r: |
| _r = self.rm.get_filtered_versions( |
| _name, |
| tag=_mcp, |
| include=[_os, _linux, _arch], |
| exclude=self.exclude_repos_keywords |
| ) |
| # Once again, if nothing found, look everywhere |
| if not _r: |
| _r = self.rm.get_filtered_versions( |
| _name, |
| tag=_mcp, |
| include=[_linux, _arch], |
| exclude=self.exclude_repos_keywords |
| ) |
| # repack versions in flat format |
| _vs = {} |
| _sections = {} |
| _apps = {} |
| for s, apps in _r.iteritems(): |
| for a, versions in apps.iteritems(): |
| for v, repos in versions.iteritems(): |
| for repo in repos: |
| if v not in _vs: |
| _vs[v] = [] |
| _vs[v].append(repo) |
| if v not in _sections: |
| _sections[v] = [] |
| _sections[v].append(s) |
| if v not in _apps: |
| _apps[v] = [] |
| _apps[v].append(a) |
| # search for the newest version among filtered |
| _r_desc = [] |
| _vs_keys = _vs.keys() |
| if _vs_keys: |
| _newest = _newest = DebianVersion(_vs_keys.pop()) |
| else: |
| _newest = DebianVersion('') |
| for v in _vs_keys: |
| _this = DebianVersion(v) |
| if _this > _newest: |
| _newest = _this |
| _release = _newest |
| # Get best description for the package |
| if _release.version != 'n/a': |
| _r_desc = _vs[_release.version] |
| # preload special description |
| if _desc[_name]: |
| _pkg_desc = _desc[_name] |
| else: |
| _pkg_desc = _desc.dummy_desc |
| # Save repos list and desc for this version |
| # Check if we can provide better from the package |
| if _release.version != 'n/a': |
| if not _pkg_desc['section']: |
| _pkg_desc['section'] = \ |
| "/".join(_sections[_release.version]) |
| if not _pkg_desc['app']: |
| _pkg_desc['app'] = \ |
| "/".join(_apps[_release.version]) |
| |
| # Populate package info, once for package |
| _m = _r_desc[0]["maintainer"] if _r_desc else 'n/a' |
| _all_packages[_name] = { |
| "desc": _pkg_desc, |
| "repos": _r_desc, |
| "maintainer": _m, |
| "is_mirantis": self.rm.is_mirantis( |
| _name, |
| tag=_mcp |
| ), |
| "results": {}, |
| "r": _release, |
| } |
| # Cross-compare versions |
| _cmp = VersionCmpResult( |
| _ver_ins, |
| _ver_can, |
| _all_packages[_name]['r'] |
| ) |
| # Update results structure |
| # shortcut to results |
| _res = _all_packages[_name]['results'] |
| # update status |
| if _cmp.status not in _res: |
| _res[_cmp.status] = {} |
| # update action |
| if _cmp.action not in _res[_cmp.status]: |
| _res[_cmp.status][_cmp.action] = {} |
| # update node |
| if node_name not in _res[_cmp.status][_cmp.action]: |
| _res[_cmp.status][_cmp.action][node_name] = {} |
| # put result |
| _res[_cmp.status][_cmp.action][node_name] = { |
| 'i': _ver_ins, |
| 'c': _ver_can, |
| 'res': _cmp, |
| 'raw': _value['raw'] |
| } |
| |
| self._packages = _all_packages |
| _progress.end() |
| |
| def create_report(self, filename, rtype, full=None): |
| """ |
| Create static html showing packages diff per node |
| |
| :return: buff with html |
| """ |
| logger_cli.info("# Generating report to '{}'".format(filename)) |
| if rtype == 'html': |
| _type = reporter.HTMLPackageCandidates() |
| elif rtype == 'csv': |
| _type = reporter.CSVAllPackages() |
| else: |
| raise ConfigException("Report type not set") |
| _report = reporter.ReportToFile( |
| _type, |
| filename |
| ) |
| payload = { |
| "nodes": salt_master.nodes, |
| "mcp_release": salt_master.mcp_release, |
| "openstack_release": salt_master.openstack_release |
| } |
| payload.update(self.presort_packages(self._packages, full)) |
| _report(payload) |
| logger_cli.info("-> Done") |