Merge upstream version
Related-PROD: PROD-28199
Change-Id: I5d9dbde1c3ac577fb30fa5d6b1ff18bcee28a0d7
diff --git a/cfg_checker/modules/packages/__init__.py b/cfg_checker/modules/packages/__init__.py
index 0e2d956..774e674 100644
--- a/cfg_checker/modules/packages/__init__.py
+++ b/cfg_checker/modules/packages/__init__.py
@@ -4,12 +4,12 @@
def do_report(args):
- """Create package versions report
+ """Create package versions report, HTML
:args: - parser arguments
:return: - no return value
"""
- _filename = args_utils.get_file_arg(args)
+ _type, _filename = args_utils.get_report_type_and_filename(args)
# init connection to salt and collect minion data
pChecker = checker.CloudPackageChecker()
@@ -18,4 +18,4 @@
# diff installed and candidates
pChecker.collect_packages()
# report it
- pChecker.create_html_report(_filename)
+ pChecker.create_report(_filename, rtype=_type, full=args.full)
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
index 4956a52..8a3456d 100644
--- a/cfg_checker/modules/packages/checker.py
+++ b/cfg_checker/modules/packages/checker.py
@@ -4,14 +4,119 @@
from copy import deepcopy
-from cfg_checker.reports import reporter
+from cfg_checker.common.exception import ConfigException
from cfg_checker.common import utils, const
from cfg_checker.common import config, logger, logger_cli, pkg_dir
from cfg_checker.common import salt_utils
+from cfg_checker.helpers.console_utils import Progress
from cfg_checker.nodes import SaltNodes, node_tmpl
+from cfg_checker.reports import reporter
+
+from versions import PkgVersions, DebianVersion, VersionCmpResult
class CloudPackageChecker(SaltNodes):
+ @staticmethod
+ def presort_packages(all_packages, full=None):
+ logger_cli.info("-> Presorting packages")
+ # labels
+ _data = {}
+ _data = {
+ "cs": {
+ "ok": const.VERSION_OK,
+ "up": const.VERSION_UP,
+ "down": const.VERSION_DOWN,
+ "err": const.VERSION_ERR
+ },
+ "ca": {
+ "na": const.ACT_NA,
+ "up": const.ACT_UPGRADE,
+ "need_up": const.ACT_NEED_UP,
+ "need_down": const.ACT_NEED_DOWN,
+ "repo": const.ACT_REPO
+ }
+ }
+ _data['status_err'] = const.VERSION_ERR
+ _data['status_down'] = const.VERSION_DOWN
+
+ # Presort packages
+ _data['critical'] = {}
+ _data['system'] = {}
+ _data['other'] = {}
+ _data['unlisted'] = {}
+
+ _l = len(all_packages)
+ _progress = Progress(_l)
+ _progress_index = 0
+ # counters
+ _ec = _es = _eo = _eu = 0
+ _dc = _ds = _do = _du = 0
+ while _progress_index < _l:
+ # progress bar
+ _progress_index += 1
+ _progress.write_progress(_progress_index)
+ # sort packages
+ _pn, _val = all_packages.popitem()
+ _c = _val['desc']['component']
+ if full:
+ # Check if this packet has errors
+ # if all is ok -> just skip it
+ _max_status = max(_val['results'].keys())
+ if _max_status <= const.VERSION_OK:
+ _max_action = max(_val['results'][_max_status].keys())
+ if _max_action == const.ACT_NA:
+ # this package do not ha any comments
+ # ...just skip it from report
+ continue
+
+ if len(_c) > 0 and _c == 'unlisted':
+ # not listed package in version lib
+ _data['unlisted'].update({
+ _pn: _val
+ })
+ _eu += _val['results'].keys().count(const.VERSION_ERR)
+ _du += _val['results'].keys().count(const.VERSION_DOWN)
+ # mirantis/critical
+ elif len(_c) > 0 and _c != 'System':
+ # not blank and not system
+ _data['critical'].update({
+ _pn: _val
+ })
+ _ec += _val['results'].keys().count(const.VERSION_ERR)
+ _dc += _val['results'].keys().count(const.VERSION_DOWN)
+ # system
+ elif _c == 'System':
+ _data['system'].update({
+ _pn: _val
+ })
+ _es += _val['results'].keys().count(const.VERSION_ERR)
+ _ds += _val['results'].keys().count(const.VERSION_DOWN)
+ # rest
+ else:
+ _data['other'].update({
+ _pn: _val
+ })
+ _eo += _val['results'].keys().count(const.VERSION_ERR)
+ _do += _val['results'].keys().count(const.VERSION_DOWN)
+
+
+ _progress.newline()
+
+ _data['errors'] = {
+ 'mirantis': _ec,
+ 'system': _es,
+ 'other': _eo,
+ 'unlisted': _eu
+ }
+ _data['downgrades'] = {
+ 'mirantis': _dc,
+ 'system': _ds,
+ 'other': _do,
+ 'unlisted': _du
+ }
+
+ return _data
+
def collect_installed_packages(self):
"""
Collect installed packages on each node
@@ -50,46 +155,115 @@
:return: no return values, all date put to dict in place
"""
+ # Preload OpenStack release versions
+ _desc = PkgVersions()
+
+ logger_cli.info("# Cross-comparing: Installed vs Candidates vs Release")
+ _progress = Progress(len(self.nodes.keys()))
+ _progress_index = 0
+ _total_processed = 0
# Collect packages from all of the nodes in flat dict
- _diff_packages = {}
+ _all_packages = {}
for node_name, node_value in self.nodes.iteritems():
+ _uniq_len = len(_all_packages.keys())
+ _progress_index += 1
+ # progress will jump from node to node
+ # it is very costly operation to execute it for each pkg
+ _progress.write_progress(
+ _progress_index,
+ note="/ {} uniq out of {} packages found".format(
+ _uniq_len,
+ _total_processed
+ )
+ )
for _name, _value in node_value['packages'].iteritems():
- if _name not in _diff_packages:
- _diff_packages[_name] = {}
- _diff_packages[_name]['df_nodes'] = {}
- _diff_packages[_name]['eq_nodes'] = []
-
- # compare packages, mark if equal
- if _value['installed'] != _value['candidate']:
- # Saving compare value so we not do str compare again
- _value['is_equal'] = False
- # add node name to list
- _diff_packages[_name]['df_nodes'][node_name] = {
- 'i': _value['installed'],
- 'c': _value['candidate'],
- 'raw': _value['raw']
+ _total_processed += 1
+ # Parse versions
+ _ver_ins = DebianVersion(_value['installed'])
+ _ver_can = DebianVersion(_value['candidate'])
+
+ # All packages list with version and node list
+ if _name not in _all_packages:
+ # shortcuts for this cloud values
+ _os = self.openstack_release
+ _mcp = self.mcp_release
+ _pkg_desc = {}
+ if _desc[_name]:
+ # shortcut to version library
+ _vers = _desc[_name]['versions']
+ _pkg_desc = _desc[_name]
+ else:
+ # no description - no library :)
+ _vers = {}
+ _pkg_desc = _desc.dummy_desc
+
+ # get specific set for this OS release if present
+ if _os in _vers:
+ _v = _vers[_os]
+ elif 'any' in _vers:
+ _v = _vers['any']
+ else:
+ _v = {}
+ # Finally, get specific version
+ _release = DebianVersion(_v[_mcp] if _mcp in _v else '')
+ # Populate package info
+ _all_packages[_name] = {
+ "desc": _pkg_desc,
+ "results": {},
+ "r": _release,
}
- else:
- # Saving compare value so we not do str compare again
- _value['is_equal'] = True
- _diff_packages[_name]['eq_nodes'].append(node_name)
+
+ _cmp = VersionCmpResult(
+ _ver_ins,
+ _ver_can,
+ _all_packages[_name]['r']
+ )
+
+ # shortcut to results
+ _res = _all_packages[_name]['results']
+ # update status
+ if _cmp.status not in _res:
+ _res[_cmp.status] = {}
+ # update action
+ if _cmp.action not in _res[_cmp.status]:
+ _res[_cmp.status][_cmp.action] = {}
+ # update node
+ if node_name not in _res[_cmp.status][_cmp.action]:
+ _res[_cmp.status][_cmp.action][node_name] = {}
+ # put result
+ _res[_cmp.status][_cmp.action][node_name] = {
+ 'i': _ver_ins,
+ 'c': _ver_can,
+ 'res': _cmp,
+ 'raw': _value['raw']
+ }
- self.diff_packages = _diff_packages
+ self._packages = _all_packages
+ _progress.newline()
+
- def create_html_report(self, filename):
+ def create_report(self, filename, rtype, full=None):
"""
Create static html showing packages diff per node
:return: buff with html
"""
logger_cli.info("# Generating report to '{}'".format(filename))
+ if rtype == 'html':
+ _type = reporter.HTMLPackageCandidates()
+ elif rtype == 'csv':
+ _type = reporter.CSVAllPackages()
+ else:
+ raise ConfigException("Report type not set")
_report = reporter.ReportToFile(
- reporter.HTMLPackageCandidates(),
+ _type,
filename
)
- _report({
+ payload = {
"nodes": self.nodes,
- "rc_diffs": {},
- "pkg_diffs": self.diff_packages
- })
+ "mcp_release": self.mcp_release,
+ "openstack_release": self.openstack_release
+ }
+ payload.update(self.presort_packages(self._packages, full))
+ _report(payload)
logger_cli.info("-> Done")
diff --git a/cfg_checker/modules/packages/versions.py b/cfg_checker/modules/packages/versions.py
new file mode 100644
index 0000000..10f65dc
--- /dev/null
+++ b/cfg_checker/modules/packages/versions.py
@@ -0,0 +1,345 @@
+import csv
+import os
+
+from cfg_checker.common import config, logger, logger_cli, pkg_dir, const
+
+
+class PkgVersions(object):
+ _labels = []
+ _list = {}
+
+ dummy_desc = {
+ "component": "unlisted",
+ "app": "-",
+ "repo": "-",
+ "versions": {}
+ }
+
+ def __init__(self):
+ # preload csv file
+ logger_cli.info("# Preloading MCP release versions")
+ with open(os.path.join(pkg_dir, 'etc', config.pkg_versions_map)) as f:
+ _reader = csv.reader(f, delimiter=',')
+ # load packages
+ for row in _reader:
+ # load release version labels
+ if _reader.line_num == 1:
+ self._labels = [v for v in row[5:]]
+ continue
+ # package_name,component,application_or_service,repo,openstack_release,2018.4.0,2018.11.0,2019.2.0,2019.2.1,2019.2.2
+ # reassign for code readability
+ _pkg = row[0]
+ _component = row[1]
+ _app = row[2]
+ _repo = row[3]
+ # if release cell empty - use keyword 'any'
+ _os_release = row[4] if len(row[4]) > 0 else 'any'
+
+ # prepare versions dict
+ _l = self._labels
+ _versions = {_l[i]:row[5+i] for i in range(0, len(row[5:]))}
+
+ if _pkg in self._list:
+ if _os_release in self._list[_pkg]["versions"]:
+ # all pkg/os_releases should be uniq. If found, latest one used
+ logger_cli.info(
+ "-> WARNING: Duplicate package info found "
+ "'{}' (line {})".format(
+ _pkg,
+ _reader.line_num
+ )
+ )
+ else:
+ # update pkg data in list
+ self._list.update({
+ _pkg: {
+ "component": _component,
+ "app": _app,
+ "repo": _repo,
+ "versions": {}
+ }
+ })
+
+ # and finally, update the versions for this release
+ self._list[_pkg]["versions"].update({
+ _os_release: _versions
+ })
+
+ def __getitem__(self, pkg_name):
+ if pkg_name in self._list:
+ return self._list[pkg_name]
+ else:
+ #return self._dummy_desc
+ return None
+
+
+class DebianVersion(object):
+ epoch = None
+ epoch_status = const.VERSION_NA
+ upstream = None
+ upstream_rev = None
+ upstream_status = const.VERSION_NA
+ debian = None
+ debian_rev = None
+ debian_status = const.VERSION_NA
+
+ status = ""
+ version = ""
+
+ @staticmethod
+ def split_revision(version_fragment):
+ # The symbols are -, +, ~
+ _symbols = ['-', '+', '~']
+ # nums, coz it is faster then regex
+ _chars = [46, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57]
+ _ord_map = [ord(ch) not in _chars for ch in version_fragment]
+ # if there is nothing to extract, return at once
+ if not any([_s in version_fragment for _s in _symbols]) \
+ and not any(_ord_map):
+ # no revisions
+ return version_fragment, ""
+ else:
+ _main = _rev = ""
+ # get indices
+ _indices = []
+ for _s in _symbols:
+ if _s in version_fragment:
+ _indices.append(version_fragment.index(_s))
+ for _s in version_fragment:
+ if ord(_s) not in _chars:
+ _indices.append(version_fragment.index(_s))
+ # sort indices
+ _indices.sort()
+ # extract starting from the lowest one
+ _main = version_fragment[:_indices[0]]
+ _rev = version_fragment[_indices[0]:]
+ return _main, _rev
+
+ def __init__(self, version_string):
+ # save
+ if len(version_string) < 1:
+ self.epoch = None
+ self.upstream = None
+ self.debian = None
+ self.version = 'n/a'
+ return
+ else:
+ # do parse the main versions
+ _v = version_string
+ # colon presence, means epoch present
+ _e = _v.split(':', 1)[0] if ':' in _v else ''
+ # if epoch was there, upstream should be cut
+ _m = _v if ':' not in _v else _v.split(':', 1)[1]
+ # dash presence, means debian present
+ _d = _m.rsplit('-', 1)[1] if '-' in _m else ''
+ # if debian was there, upstream version should be cut
+ _m = _m if '-' not in _m else _m.rsplit('-', 1)[0]
+
+ self.epoch = _e
+ self.upstream, self.upstream_rev = self.split_revision(_m)
+ self.debian, self.debian_rev = self.split_revision(_d)
+ self.version = version_string
+
+ # Following functions is a freestyle python mimic of apt's upstream, enjoy
+ # https://github.com/chaos/apt/blob/master/apt/apt-pkg/deb/debversion.cc#L42
+ # mimic produced in order not to pull any packages or call external code
+ @staticmethod
+ def _cmp_fragment(lhf, rhf):
+ # search for difference
+ # indices
+ _li = _ri = 0
+ # pre-calc len
+ _lL = len(lhf)
+ _rL = len(rhf)
+ # bool for compare found
+ _diff = False
+ while _li < _lL and _ri < _rL:
+ # iterate lists
+ _num = lhf[_li] - rhf[_ri]
+ if _num:
+ return _num
+ _li += 1
+ _ri += 1
+
+ # diff found? lens equal?
+ if not _diff and _lL != _rL:
+ # lens not equal? Longer - later
+ return _lL - _rL
+ else:
+ # equal
+ return 0
+
+ def _cmp_num(self, lf, rf):
+ # split fragments into lists
+ _lhf = lf.split('.') if '.' in lf else list(lf)
+ _rhf = rf.split('.') if '.' in rf else list(rf)
+ # cast them to ints, delete empty strs
+ _lhf = [int(n) for n in _lhf if len(n)]
+ _rhf = [int(n) for n in _rhf if len(n)]
+
+ return self._cmp_fragment(_lhf, _rhf)
+
+ def _cmp_lex(self, lf, rf):
+ # cast each item into its ORD value
+ _lhf = [ord(n) for n in lf]
+ _rhf = [ord(n) for n in rf]
+
+ return self._cmp_fragment(_lhf, _rhf)
+ # end of cmps
+
+ # main part compared using splitted numbers
+ # if equal, revision is compared using lexical comparizon
+ def __lt__(self, v):
+ if self._cmp_num(self.epoch, v.epoch) < 0:
+ return True
+ elif self._cmp_num(self.upstream, v.upstream) < 0:
+ return True
+ elif self._cmp_lex(self.upstream_rev, v.upstream_rev) < 0:
+ return True
+ else:
+ return False
+
+ def __eq__(self, v):
+ # compare all portions
+ _result = []
+ _result.append(self._cmp_num(self.epoch, v.epoch))
+ _result.append(self._cmp_num(self.upstream, v.upstream))
+ _result.append(self._cmp_lex(self.upstream_rev, v.upstream_rev))
+ # if there is any non-zero, its not equal
+ return not any(_result)
+
+ def __gt__(self, v):
+ if self._cmp_num(self.epoch, v.epoch) > 0:
+ return True
+ elif self._cmp_num(self.upstream, v.upstream) > 0:
+ return True
+ elif self._cmp_lex(self.upstream_rev, v.upstream_rev) > 0:
+ return True
+ else:
+ return False
+
+ def update_parts(self, target, status):
+ # updating parts of version statuses
+ if self._cmp_num(self.epoch, target.epoch) != 0:
+ self.epoch_status = status
+ else:
+ self.epoch_status = const.VERSION_OK
+
+ if self._cmp_num(self.upstream, target.upstream) != 0 \
+ or self._cmp_lex(self.upstream_rev, target.upstream_rev) != 0:
+ self.upstream_status = status
+ else:
+ self.upstream_status = const.VERSION_OK
+
+ if self._cmp_lex(self.debian, target.debian) != 0 \
+ or self._cmp_lex(self.debian_rev, target.debian_rev) != 0:
+ self.debian_status = status
+ else:
+ self.debian_status = const.VERSION_OK
+
+
+class VersionCmpResult(object):
+ status = ""
+ action = ""
+
+ source = None
+ target = None
+
+
+ def __init__(self, i, c, r):
+ # compare three versions and write a result
+ self.source = i
+ self.status = const.VERSION_NA
+ self.action = const.ACT_NA
+
+ # Check if there is a release version present
+ if r and len(r.version) > 0 and r.version != 'n/a':
+ # I < C, installed version is older
+ if i < c:
+ self.target = c
+ if i == r:
+ # installed version is equal vs release version
+ self.status = const.VERSION_OK
+ self.action = const.ACT_UPGRADE
+ elif i > r:
+ # installed version is newer vs release version
+ self.status = const.VERSION_UP
+ self.action = const.ACT_UPGRADE
+ elif i < r and r < c:
+ # installed version is older vs release version
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_NEED_UP
+ self.target = r
+ elif i < r and c == r:
+ # installed version is older vs release version
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_NEED_UP
+ self.target = c
+ elif c < r:
+ # installed and repo versions older vs release version
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_REPO
+ # I > C
+ # installed version is newer
+ elif i > c:
+ self.target = c
+ if c == r:
+ # some unknown version installed
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_NEED_DOWN
+ elif c > r:
+ # installed and repo versions newer than release
+ self.status = const.VERSION_UP
+ self.action = const.ACT_NEED_DOWN
+ elif c < r and r < i:
+ # repo is older vs release and both older vs installed
+ self.status = const.VERSION_UP
+ self.action = const.ACT_REPO
+ elif c < r and r == i:
+ # repo is older vs release, but release version installed
+ self.status = const.VERSION_OK
+ self.action = const.ACT_REPO
+ elif i < r:
+ # both repo and installed older vs release, new target
+ self.status = const.VERSION_DOWN
+ self.action = const.ACT_REPO
+ self.target = r
+ # I = C
+ # installed and linked repo is inline,
+ elif i == c:
+ self.target = c
+ if i < r:
+ # both are old, new target
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_REPO
+ self.target = r
+ elif i > r:
+ # both are newer, same target
+ self.status = const.VERSION_UP
+ self.action = const.ACT_NA
+ elif i == r:
+ # all is ok
+ self.status = const.VERSION_OK
+ self.action = const.ACT_NA
+ else:
+ # no release version present
+ self.target = c
+ if i < c:
+ self.status = const.VERSION_OK
+ self.action = const.ACT_UPGRADE
+ elif i > c:
+ self.status = const.VERSION_UP
+ self.action = const.ACT_NEED_DOWN
+ elif i == c:
+ self.status = const.VERSION_OK
+ self.action = const.ACT_NA
+
+ # and we need to update per-part status
+ self.source.update_parts(self.target, self.status)
+
+ @staticmethod
+ def deb_lower(_s, _t):
+ if _t.debian and _t.debian > _s.debian:
+ return True
+ else:
+ return false