Package report/repo parser integration
- parser able to filter package versions using keywords
- warning message on missing tag
- on the fly versions lookup (excluding '*.hotfix')
- updated versions compare routine
- lexical compare uses numbers, not ordinal values
- updated release version detection
- final report lists pkg section/app if no description given
- final report shows repo info for detected release version
Fixes:
- shorter alternate entrpoints: mcp-pkg, mcp-net, cmp-reclass
- flake8 syntax
- proper mirantis/non-mirantis versions getting
- exit on unexpected arguments
- salt-master class now gets linux codename by default and architecture
Change-Id: I0a2daadca8a1acaecafc8680226dc00d20cc24ce
Related-PROD: PROD-28199
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
index 514bd9c..bf192f9 100644
--- a/cfg_checker/modules/packages/checker.py
+++ b/cfg_checker/modules/packages/checker.py
@@ -3,6 +3,7 @@
from cfg_checker.common import const, logger_cli
from cfg_checker.common.exception import ConfigException
from cfg_checker.helpers.console_utils import Progress
+from cfg_checker.modules.packages.repos import RepoManager
from cfg_checker.nodes import salt_master
from cfg_checker.reports import reporter
@@ -10,6 +11,25 @@
class CloudPackageChecker(object):
+ rm = RepoManager()
+
+ def __init__(self):
+ # Init salt master info
+ if not salt_master.nodes:
+ salt_master.nodes = salt_master.get_nodes()
+
+ # check that this env tag is present in Manager
+ _tags = self.rm.get_available_tags(tag=salt_master.mcp_release)
+ if not _tags:
+ logger_cli.warning(
+ "\n# hWARNING: '{0}' is not listed in repo index. "
+ "Consider running:\n\t{1}\nto add info on this tag's "
+ "release package versions".format(
+ salt_master.mcp_release,
+ "mcp-checker packages versions --tag {0}"
+ )
+ )
+
@staticmethod
def presort_packages(all_packages, full=None):
logger_cli.info("-> Presorting packages")
@@ -51,7 +71,8 @@
_progress.write_progress(_progress_index)
# sort packages
_pn, _val = all_packages.popitem()
- _c = _val['desc']['component']
+ _c = _val['desc']['section']
+
if not full:
# Check if this packet has errors
# if all is ok -> just skip it
@@ -59,11 +80,11 @@
if _max_status <= const.VERSION_OK:
_max_action = max(_val['results'][_max_status].keys())
if _max_action == const.ACT_NA:
- # this package do not has any comments
+ # this package does not have any comments
# ...just skip it from report
continue
- if len(_c) > 0 and _c == 'unlisted':
+ if len(_c) > 0 and _val['is_mirantis'] is None:
# not listed package in version lib
_data['unlisted'].update({
_pn: _val
@@ -71,7 +92,8 @@
_eu += _val['results'].keys().count(const.VERSION_ERR)
_du += _val['results'].keys().count(const.VERSION_DOWN)
# mirantis/critical
- elif len(_c) > 0 and _c != 'System':
+ # elif len(_c) > 0 and _c != 'System':
+ elif _val['is_mirantis']:
# not blank and not system
_data['critical'].update({
_pn: _val
@@ -118,8 +140,6 @@
:return: none
"""
logger_cli.info("# Collecting installed packages")
- if not salt_master.nodes:
- salt_master.nodes = salt_master.get_nodes()
salt_master.prepare_script_on_active_nodes("pkg_versions.py")
_result = salt_master.execute_script_on_active_nodes("pkg_versions.py")
@@ -159,16 +179,23 @@
logger_cli.info(
"# Cross-comparing: Installed vs Candidates vs Release"
)
+ # shortcuts for this cloud values
+ _os = salt_master.openstack_release
+ _mcp = salt_master.mcp_release
+ # Progress class
_progress = Progress(len(salt_master.nodes.keys()))
_progress_index = 0
_total_processed = 0
# Collect packages from all of the nodes in flat dict
_all_packages = {}
+ _all_tags = set([])
+ # get env tag's year and major version
+ _tag_major = _mcp[:_mcp.find('.', _mcp.find('.')+1)]
for node_name, node_value in salt_master.nodes.iteritems():
_uniq_len = len(_all_packages.keys())
_progress_index += 1
- # progress will jump from node to node
- # it is very costly operation to execute it for each pkg
+ # progress updates shown before next node only
+ # it is costly operation to do it for each of the 150k packages
_progress.write_progress(
_progress_index,
note="/ {} uniq out of {} packages found".format(
@@ -184,31 +211,128 @@
# All packages list with version and node list
if _name not in _all_packages:
- # shortcuts for this cloud values
- _os = salt_master.openstack_release
- _mcp = salt_master.mcp_release
- _pkg_desc = {}
+ # get repo versions list,
+ _linux = salt_master.nodes[node_name]['linux_codename']
+ _arch = salt_master.nodes[node_name]['linux_arch']
+ if _name == u'qemu-block-extra':
+ a = 1
+ # omit tag as target versions might be of different tag
+ _r = self.rm.get_filtered_versions(
+ _name,
+ tag=_mcp,
+ include=[_os, _linux, _arch],
+ exclude=["nightly"]
+ )
+ # save versions for matching major tags
+ _vs = {}
+ _sections = {}
+ _apps = {}
+ # get all versions for this year
+ for s, apps in _r.iteritems():
+ for a, versions in apps.iteritems():
+ for v, repos in versions.iteritems():
+ for repo in repos:
+ t = repo['tag']
+ _major = t[:t.find('.', t.find('.')+1)]
+ if _tag_major == _major:
+ if v not in _vs:
+ _vs[v] = []
+ _vs[v].append(repo)
+ if v not in _sections:
+ _sections[v] = []
+ _sections[v].append(s)
+ if v not in _apps:
+ _apps[v] = []
+ _apps[v].append(a)
+
+ # check if we have candidate version among found
+ _r_desc = []
+ _vs_keys = _vs.keys()
+ if _vs_keys:
+ _newest = _newest = DebianVersion(_vs_keys.pop())
+ else:
+ _newest = DebianVersion('')
+ # if _ver_ins.version in _vs_keys:
+ # # exact match, save it
+ # _release = _ver_ins
+ # else:
+ # detect newest version among saved
+
+ for v in _vs_keys:
+ _this = DebianVersion(v)
+ if _this > _newest:
+ _newest = _this
+ # newest version for the YEAR.MAJOR will be the release
+ _release = _newest
+ # save repos list for this version
+ if _release.version != 'n/a':
+ _r_desc = _vs[_release.version]
+ # preload special description
if _desc[_name]:
- # shortcut to version library
- _vers = _desc[_name]['versions']
_pkg_desc = _desc[_name]
else:
- # no description - no library :)
- _vers = {}
_pkg_desc = _desc.dummy_desc
+ # Check if we can provide better from the package
+ if _release.version != 'n/a':
+ if not _pkg_desc['section']:
+ _pkg_desc['section'] = \
+ "/".join(_sections[_release.version])
+ if not _pkg_desc['app']:
+ _pkg_desc['app'] = \
+ "/".join(_apps[_release.version])
# get specific set for this OS release if present
- if _os in _vers:
- _v = _vers[_os]
- elif 'any' in _vers:
- _v = _vers['any']
- else:
- _v = {}
+ # if not, try search in all repos for this tag
+ # _r_desc = _r[_newest.version]
+ # if _r:
+ # _vs = _r.keys()
+ # if len(_vs) > 1:
+ # # search best match
+ # _release = None
+ # for _v in _vs:
+ # _deb = DebianVersion(_v)
+ # if _ver_can == _deb:
+ # _release = _deb
+ # _r_desc = _r[_v]
+ # break
+ # if not _release:
+ # _progress.clearline()
+ # logger_cli.error(
+ # "# ERROR: No release version found "
+ # "for '{}'".format(_name)
+ # )
+ # _release = DebianVersion('')
+ # else:
+ # _release = DebianVersion(_vs[0])
+ # _r_desc = _r[_vs[0]]
+ # else:
+ # # not found... 99% that it will not happen
+ # _release = DebianVersion('')
+
+ # Old versions match routine
+ # ########
+ # if _os in _vers:
+ # _v = _vers[_os]
+ # elif 'any' in _vers:
+ # _v = _vers['any']
+ # else:
+ # _v = {}
+
# Finally, get specific version
- _release = DebianVersion(_v[_mcp] if _mcp in _v else '')
+ # _release = DebianVersion(_v[_mcp] if _mcp in _v else '')
+ for repo in _r_desc:
+ _all_tags.add(repo['tag'])
+
# Populate package info
+ _m = _r_desc[0]["maintainer"] if _r_desc else 'n/a'
_all_packages[_name] = {
"desc": _pkg_desc,
+ "repos": _r_desc,
+ "maintainer": _m,
+ "is_mirantis": self.rm.is_mirantis(
+ _name,
+ tag=_tag_major
+ ),
"results": {},
"r": _release,
}