Merge upstream version
Related-PROD: PROD-28199
Change-Id: I5d9dbde1c3ac577fb30fa5d6b1ff18bcee28a0d7
diff --git a/cfg_checker/cfg_check.py b/cfg_checker/cfg_check.py
index a0df7e6..678724e 100644
--- a/cfg_checker/cfg_check.py
+++ b/cfg_checker/cfg_check.py
@@ -64,11 +64,6 @@
help="Set CLI logging level to DEBUG"
)
parser.add_argument(
- '-f',
- '--file',
- help="HTML filename to save report"
- )
- parser.add_argument(
'-s',
'--sudo',
action='store_true', default=True,
@@ -86,6 +81,21 @@
'report',
help="Report package versions to HTML file"
)
+ pkg_report_parser.add_argument(
+ '--full',
+ metavar='packages_html_filename',
+ help="HTML report will have all of the packages, not just errors"
+ )
+ pkg_report_parser.add_argument(
+ '--html',
+ metavar='packages_html_filename',
+ help="HTML filename to save report"
+ )
+ pkg_report_parser.add_argument(
+ '--csv',
+ metavar='packages_csv_filename',
+ help="CSV filename to save report"
+ )
# networking
net_parser = subparsers.add_parser(
@@ -103,6 +113,12 @@
'report',
help="Generate network check report"
)
+
+ net_report_parser.add_argument(
+ '--html',
+ metavar='network_html_filename',
+ help="HTML filename to save report"
+ )
# reclass
reclass_parser = subparsers.add_parser(
@@ -135,6 +151,12 @@
required=True,
help="Model B <path>. Model name is the folder name"
)
+ reclass_diff_parser.add_argument(
+ '--html',
+ metavar='reclass_html_filename',
+ help="HTML filename to save report"
+ )
+
#parse arguments
diff --git a/cfg_checker/common/const.py b/cfg_checker/common/const.py
index 4142ea7..8ca0d8c 100644
--- a/cfg_checker/common/const.py
+++ b/cfg_checker/common/const.py
@@ -9,8 +9,40 @@
NODE_DOWN = next(_cnt)
NODE_UP = next(_cnt)
+# version const order is important!
+# biggest get shown in report top row
+VERSION_NA = next(_cnt)
+VERSION_OK = next(_cnt)
+VERSION_UP = next(_cnt)
+VERSION_DOWN = next(_cnt)
+VERSION_ERR = next(_cnt)
+
+# action const order is important!
+# biggest get shown in report top row
+ACT_NA = next(_cnt)
+ACT_UPGRADE = next(_cnt)
+ACT_NEED_UP = next(_cnt)
+ACT_NEED_DOWN = next(_cnt)
+ACT_REPO = next(_cnt)
+
del _cnt
+all_actions = {
+ ACT_UPGRADE: "upgrade possible",
+ ACT_NEED_UP: "needs upgrade",
+ ACT_NEED_DOWN: "needs downgrade",
+ ACT_REPO: "needs repo update",
+ ACT_NA: ""
+}
+
+all_statuses = {
+ VERSION_OK: "ok",
+ VERSION_UP: "upgraded",
+ VERSION_DOWN: "downgraded",
+ VERSION_ERR: "error",
+ VERSION_NA: "no status"
+}
+
all_roles_map = {
"apt": "repository",
"bmk": "validation",
diff --git a/cfg_checker/common/settings.py b/cfg_checker/common/settings.py
index 30cc7f2..8d8111c 100644
--- a/cfg_checker/common/settings.py
+++ b/cfg_checker/common/settings.py
@@ -31,6 +31,8 @@
self.date_format = "%Y-%m-%d %H:%M:%S.%f%z"
self.default_tz = "UTC"
+ self.pkg_versions_map = 'versions_map.csv'
+
self.ssh_uses_sudo = False
self.ssh_key = os.environ.get('SSH_KEY', None)
self.ssh_user = os.environ.get('SSH_USER', None)
diff --git a/cfg_checker/helpers/args_utils.py b/cfg_checker/helpers/args_utils.py
index d7b694d..f8453e4 100644
--- a/cfg_checker/helpers/args_utils.py
+++ b/cfg_checker/helpers/args_utils.py
@@ -3,11 +3,20 @@
from cfg_checker.common.exception import ConfigException
-def get_file_arg(args):
- if args.file:
- return args.file
+def get_arg(args, str_arg):
+ _attr = getattr(args, str_arg)
+ if _attr:
+ return _attr
else:
- raise ConfigException("No report filename supplied")
+ _c = args.command if hasattr(args, 'command') else ''
+ _t = args.type if hasattr(args, 'type') else ''
+ raise ConfigException(
+ "Argument '{}' not found executing: mcp_check {} {}".format(
+ str_arg,
+ _c,
+ _t
+ )
+ )
def get_path_arg(path):
@@ -15,3 +24,15 @@
return path
else:
raise ConfigException("'{}' not exists".format(path))
+
+
+def get_report_type_and_filename(args):
+ if args.html or args.csv:
+ if args.html and args.csv:
+ raise ConfigException("Multuple report types not supported")
+ if args.html is not None:
+ return 'html', args.html
+ if args.csv is not None:
+ return 'csv', args.csv
+ else:
+ raise ConfigException("Report type and filename not set")
diff --git a/cfg_checker/helpers/console_utils.py b/cfg_checker/helpers/console_utils.py
new file mode 100644
index 0000000..33e1a39
--- /dev/null
+++ b/cfg_checker/helpers/console_utils.py
@@ -0,0 +1,30 @@
+from time import sleep
+import sys
+
+
+class Progress(object):
+ def __init__(self, max_index, bar_size=21):
+ self.total = max_index
+ # bar size in symbols
+ self.bar_size = bar_size
+
+ def write_progress(self, index, note=''):
+ #calc index and percent values
+ _percent = (100 * index) / self.total
+ _index = (self.bar_size * index) / self.total
+ # clear the line
+ sys.stdout.write('\r')
+ # print new progress
+ _format = "[{:"+str(self.bar_size-1)+"}] {}/{} ({}%) {}"
+ sys.stdout.write(_format.format(
+ '='*_index,
+ index,
+ self.total,
+ _percent,
+ note
+ ))
+ sys.stdout.flush()
+
+ @staticmethod
+ def newline():
+ sys.stdout.write('\n')
diff --git a/cfg_checker/modules/network/__init__.py b/cfg_checker/modules/network/__init__.py
index 8074a71..6217b1a 100644
--- a/cfg_checker/modules/network/__init__.py
+++ b/cfg_checker/modules/network/__init__.py
@@ -3,22 +3,26 @@
from cfg_checker.helpers import args_utils
from cfg_checker.common import logger_cli
+def _prepare_check():
+ _checker_class = checker.NetworkChecker()
+ _checker_class.collect_reclass_networks()
+ _checker_class.collect_network_info()
+ return _checker_class
+
def do_check(args):
- logger_cli.info("# Network check (CLI output)")
- netChecker = checker.NetworkChecker()
- netChecker.collect_reclass_networks()
- netChecker.collect_network_info()
+ logger_cli.info("# Network check to console")
+ netChecker = _prepare_check()
netChecker.print_network_report()
return
def do_report(args):
- logger_cli.info("# Network check (HTML report: '{}')".format(args.file))
- _filename = args_utils.get_file_arg(args)
+ logger_cli.info("# Network report")
- netChecker = checker.NetworkChecker()
- netChecker.collect_network_info()
+ _filename = args_utils.get_arg(args, 'html')
+
+ netChecker = _prepare_check()
netChecker.create_html_report(_filename)
return
diff --git a/cfg_checker/modules/network/checker.py b/cfg_checker/modules/network/checker.py
index 362b877..b0056c8 100644
--- a/cfg_checker/modules/network/checker.py
+++ b/cfg_checker/modules/network/checker.py
@@ -249,6 +249,9 @@
)
_report({
"nodes": self.nodes,
- "diffs": {}
+ "network": {},
+ "mcp_release": self.mcp_release,
+ "openstack_release": self.openstack_release
+
})
logger_cli.info("-> Done")
diff --git a/cfg_checker/modules/packages/__init__.py b/cfg_checker/modules/packages/__init__.py
index 0e2d956..774e674 100644
--- a/cfg_checker/modules/packages/__init__.py
+++ b/cfg_checker/modules/packages/__init__.py
@@ -4,12 +4,12 @@
def do_report(args):
- """Create package versions report
+ """Create package versions report, HTML
:args: - parser arguments
:return: - no return value
"""
- _filename = args_utils.get_file_arg(args)
+ _type, _filename = args_utils.get_report_type_and_filename(args)
# init connection to salt and collect minion data
pChecker = checker.CloudPackageChecker()
@@ -18,4 +18,4 @@
# diff installed and candidates
pChecker.collect_packages()
# report it
- pChecker.create_html_report(_filename)
+ pChecker.create_report(_filename, rtype=_type, full=args.full)
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
index 4956a52..8a3456d 100644
--- a/cfg_checker/modules/packages/checker.py
+++ b/cfg_checker/modules/packages/checker.py
@@ -4,14 +4,119 @@
from copy import deepcopy
-from cfg_checker.reports import reporter
+from cfg_checker.common.exception import ConfigException
from cfg_checker.common import utils, const
from cfg_checker.common import config, logger, logger_cli, pkg_dir
from cfg_checker.common import salt_utils
+from cfg_checker.helpers.console_utils import Progress
from cfg_checker.nodes import SaltNodes, node_tmpl
+from cfg_checker.reports import reporter
+
+from versions import PkgVersions, DebianVersion, VersionCmpResult
class CloudPackageChecker(SaltNodes):
+ @staticmethod
+ def presort_packages(all_packages, full=None):
+ logger_cli.info("-> Presorting packages")
+ # labels
+ _data = {}
+ _data = {
+ "cs": {
+ "ok": const.VERSION_OK,
+ "up": const.VERSION_UP,
+ "down": const.VERSION_DOWN,
+ "err": const.VERSION_ERR
+ },
+ "ca": {
+ "na": const.ACT_NA,
+ "up": const.ACT_UPGRADE,
+ "need_up": const.ACT_NEED_UP,
+ "need_down": const.ACT_NEED_DOWN,
+ "repo": const.ACT_REPO
+ }
+ }
+ _data['status_err'] = const.VERSION_ERR
+ _data['status_down'] = const.VERSION_DOWN
+
+ # Presort packages
+ _data['critical'] = {}
+ _data['system'] = {}
+ _data['other'] = {}
+ _data['unlisted'] = {}
+
+ _l = len(all_packages)
+ _progress = Progress(_l)
+ _progress_index = 0
+ # counters
+ _ec = _es = _eo = _eu = 0
+ _dc = _ds = _do = _du = 0
+ while _progress_index < _l:
+ # progress bar
+ _progress_index += 1
+ _progress.write_progress(_progress_index)
+ # sort packages
+ _pn, _val = all_packages.popitem()
+ _c = _val['desc']['component']
+ if full:
+ # Check if this packet has errors
+ # if all is ok -> just skip it
+ _max_status = max(_val['results'].keys())
+ if _max_status <= const.VERSION_OK:
+ _max_action = max(_val['results'][_max_status].keys())
+ if _max_action == const.ACT_NA:
+ # this package do not ha any comments
+ # ...just skip it from report
+ continue
+
+ if len(_c) > 0 and _c == 'unlisted':
+ # not listed package in version lib
+ _data['unlisted'].update({
+ _pn: _val
+ })
+ _eu += _val['results'].keys().count(const.VERSION_ERR)
+ _du += _val['results'].keys().count(const.VERSION_DOWN)
+ # mirantis/critical
+ elif len(_c) > 0 and _c != 'System':
+ # not blank and not system
+ _data['critical'].update({
+ _pn: _val
+ })
+ _ec += _val['results'].keys().count(const.VERSION_ERR)
+ _dc += _val['results'].keys().count(const.VERSION_DOWN)
+ # system
+ elif _c == 'System':
+ _data['system'].update({
+ _pn: _val
+ })
+ _es += _val['results'].keys().count(const.VERSION_ERR)
+ _ds += _val['results'].keys().count(const.VERSION_DOWN)
+ # rest
+ else:
+ _data['other'].update({
+ _pn: _val
+ })
+ _eo += _val['results'].keys().count(const.VERSION_ERR)
+ _do += _val['results'].keys().count(const.VERSION_DOWN)
+
+
+ _progress.newline()
+
+ _data['errors'] = {
+ 'mirantis': _ec,
+ 'system': _es,
+ 'other': _eo,
+ 'unlisted': _eu
+ }
+ _data['downgrades'] = {
+ 'mirantis': _dc,
+ 'system': _ds,
+ 'other': _do,
+ 'unlisted': _du
+ }
+
+ return _data
+
def collect_installed_packages(self):
"""
Collect installed packages on each node
@@ -50,46 +155,115 @@
:return: no return values, all date put to dict in place
"""
+ # Preload OpenStack release versions
+ _desc = PkgVersions()
+
+ logger_cli.info("# Cross-comparing: Installed vs Candidates vs Release")
+ _progress = Progress(len(self.nodes.keys()))
+ _progress_index = 0
+ _total_processed = 0
# Collect packages from all of the nodes in flat dict
- _diff_packages = {}
+ _all_packages = {}
for node_name, node_value in self.nodes.iteritems():
+ _uniq_len = len(_all_packages.keys())
+ _progress_index += 1
+ # progress will jump from node to node
+ # it is very costly operation to execute it for each pkg
+ _progress.write_progress(
+ _progress_index,
+ note="/ {} uniq out of {} packages found".format(
+ _uniq_len,
+ _total_processed
+ )
+ )
for _name, _value in node_value['packages'].iteritems():
- if _name not in _diff_packages:
- _diff_packages[_name] = {}
- _diff_packages[_name]['df_nodes'] = {}
- _diff_packages[_name]['eq_nodes'] = []
-
- # compare packages, mark if equal
- if _value['installed'] != _value['candidate']:
- # Saving compare value so we not do str compare again
- _value['is_equal'] = False
- # add node name to list
- _diff_packages[_name]['df_nodes'][node_name] = {
- 'i': _value['installed'],
- 'c': _value['candidate'],
- 'raw': _value['raw']
+ _total_processed += 1
+ # Parse versions
+ _ver_ins = DebianVersion(_value['installed'])
+ _ver_can = DebianVersion(_value['candidate'])
+
+ # All packages list with version and node list
+ if _name not in _all_packages:
+ # shortcuts for this cloud values
+ _os = self.openstack_release
+ _mcp = self.mcp_release
+ _pkg_desc = {}
+ if _desc[_name]:
+ # shortcut to version library
+ _vers = _desc[_name]['versions']
+ _pkg_desc = _desc[_name]
+ else:
+ # no description - no library :)
+ _vers = {}
+ _pkg_desc = _desc.dummy_desc
+
+ # get specific set for this OS release if present
+ if _os in _vers:
+ _v = _vers[_os]
+ elif 'any' in _vers:
+ _v = _vers['any']
+ else:
+ _v = {}
+ # Finally, get specific version
+ _release = DebianVersion(_v[_mcp] if _mcp in _v else '')
+ # Populate package info
+ _all_packages[_name] = {
+ "desc": _pkg_desc,
+ "results": {},
+ "r": _release,
}
- else:
- # Saving compare value so we not do str compare again
- _value['is_equal'] = True
- _diff_packages[_name]['eq_nodes'].append(node_name)
+
+ _cmp = VersionCmpResult(
+ _ver_ins,
+ _ver_can,
+ _all_packages[_name]['r']
+ )
+
+ # shortcut to results
+ _res = _all_packages[_name]['results']
+ # update status
+ if _cmp.status not in _res:
+ _res[_cmp.status] = {}
+ # update action
+ if _cmp.action not in _res[_cmp.status]:
+ _res[_cmp.status][_cmp.action] = {}
+ # update node
+ if node_name not in _res[_cmp.status][_cmp.action]:
+ _res[_cmp.status][_cmp.action][node_name] = {}
+ # put result
+ _res[_cmp.status][_cmp.action][node_name] = {
+ 'i': _ver_ins,
+ 'c': _ver_can,
+ 'res': _cmp,
+ 'raw': _value['raw']
+ }
- self.diff_packages = _diff_packages
+ self._packages = _all_packages
+ _progress.newline()
+
- def create_html_report(self, filename):
+ def create_report(self, filename, rtype, full=None):
"""
Create static html showing packages diff per node
:return: buff with html
"""
logger_cli.info("# Generating report to '{}'".format(filename))
+ if rtype == 'html':
+ _type = reporter.HTMLPackageCandidates()
+ elif rtype == 'csv':
+ _type = reporter.CSVAllPackages()
+ else:
+ raise ConfigException("Report type not set")
_report = reporter.ReportToFile(
- reporter.HTMLPackageCandidates(),
+ _type,
filename
)
- _report({
+ payload = {
"nodes": self.nodes,
- "rc_diffs": {},
- "pkg_diffs": self.diff_packages
- })
+ "mcp_release": self.mcp_release,
+ "openstack_release": self.openstack_release
+ }
+ payload.update(self.presort_packages(self._packages, full))
+ _report(payload)
logger_cli.info("-> Done")
diff --git a/cfg_checker/modules/packages/versions.py b/cfg_checker/modules/packages/versions.py
new file mode 100644
index 0000000..10f65dc
--- /dev/null
+++ b/cfg_checker/modules/packages/versions.py
@@ -0,0 +1,345 @@
+import csv
+import os
+
+from cfg_checker.common import config, logger, logger_cli, pkg_dir, const
+
+
+class PkgVersions(object):
+ _labels = []
+ _list = {}
+
+ dummy_desc = {
+ "component": "unlisted",
+ "app": "-",
+ "repo": "-",
+ "versions": {}
+ }
+
+ def __init__(self):
+ # preload csv file
+ logger_cli.info("# Preloading MCP release versions")
+ with open(os.path.join(pkg_dir, 'etc', config.pkg_versions_map)) as f:
+ _reader = csv.reader(f, delimiter=',')
+ # load packages
+ for row in _reader:
+ # load release version labels
+ if _reader.line_num == 1:
+ self._labels = [v for v in row[5:]]
+ continue
+ # package_name,component,application_or_service,repo,openstack_release,2018.4.0,2018.11.0,2019.2.0,2019.2.1,2019.2.2
+ # reassign for code readability
+ _pkg = row[0]
+ _component = row[1]
+ _app = row[2]
+ _repo = row[3]
+ # if release cell empty - use keyword 'any'
+ _os_release = row[4] if len(row[4]) > 0 else 'any'
+
+ # prepare versions dict
+ _l = self._labels
+ _versions = {_l[i]:row[5+i] for i in range(0, len(row[5:]))}
+
+ if _pkg in self._list:
+ if _os_release in self._list[_pkg]["versions"]:
+ # all pkg/os_releases should be uniq. If found, latest one used
+ logger_cli.info(
+ "-> WARNING: Duplicate package info found "
+ "'{}' (line {})".format(
+ _pkg,
+ _reader.line_num
+ )
+ )
+ else:
+ # update pkg data in list
+ self._list.update({
+ _pkg: {
+ "component": _component,
+ "app": _app,
+ "repo": _repo,
+ "versions": {}
+ }
+ })
+
+ # and finally, update the versions for this release
+ self._list[_pkg]["versions"].update({
+ _os_release: _versions
+ })
+
+ def __getitem__(self, pkg_name):
+ if pkg_name in self._list:
+ return self._list[pkg_name]
+ else:
+ #return self._dummy_desc
+ return None
+
+
+class DebianVersion(object):
+ epoch = None
+ epoch_status = const.VERSION_NA
+ upstream = None
+ upstream_rev = None
+ upstream_status = const.VERSION_NA
+ debian = None
+ debian_rev = None
+ debian_status = const.VERSION_NA
+
+ status = ""
+ version = ""
+
+ @staticmethod
+ def split_revision(version_fragment):
+ # The symbols are -, +, ~
+ _symbols = ['-', '+', '~']
+ # nums, coz it is faster then regex
+ _chars = [46, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57]
+ _ord_map = [ord(ch) not in _chars for ch in version_fragment]
+ # if there is nothing to extract, return at once
+ if not any([_s in version_fragment for _s in _symbols]) \
+ and not any(_ord_map):
+ # no revisions
+ return version_fragment, ""
+ else:
+ _main = _rev = ""
+ # get indices
+ _indices = []
+ for _s in _symbols:
+ if _s in version_fragment:
+ _indices.append(version_fragment.index(_s))
+ for _s in version_fragment:
+ if ord(_s) not in _chars:
+ _indices.append(version_fragment.index(_s))
+ # sort indices
+ _indices.sort()
+ # extract starting from the lowest one
+ _main = version_fragment[:_indices[0]]
+ _rev = version_fragment[_indices[0]:]
+ return _main, _rev
+
+ def __init__(self, version_string):
+ # save
+ if len(version_string) < 1:
+ self.epoch = None
+ self.upstream = None
+ self.debian = None
+ self.version = 'n/a'
+ return
+ else:
+ # do parse the main versions
+ _v = version_string
+ # colon presence, means epoch present
+ _e = _v.split(':', 1)[0] if ':' in _v else ''
+ # if epoch was there, upstream should be cut
+ _m = _v if ':' not in _v else _v.split(':', 1)[1]
+ # dash presence, means debian present
+ _d = _m.rsplit('-', 1)[1] if '-' in _m else ''
+ # if debian was there, upstream version should be cut
+ _m = _m if '-' not in _m else _m.rsplit('-', 1)[0]
+
+ self.epoch = _e
+ self.upstream, self.upstream_rev = self.split_revision(_m)
+ self.debian, self.debian_rev = self.split_revision(_d)
+ self.version = version_string
+
+ # Following functions is a freestyle python mimic of apt's upstream, enjoy
+ # https://github.com/chaos/apt/blob/master/apt/apt-pkg/deb/debversion.cc#L42
+ # mimic produced in order not to pull any packages or call external code
+ @staticmethod
+ def _cmp_fragment(lhf, rhf):
+ # search for difference
+ # indices
+ _li = _ri = 0
+ # pre-calc len
+ _lL = len(lhf)
+ _rL = len(rhf)
+ # bool for compare found
+ _diff = False
+ while _li < _lL and _ri < _rL:
+ # iterate lists
+ _num = lhf[_li] - rhf[_ri]
+ if _num:
+ return _num
+ _li += 1
+ _ri += 1
+
+ # diff found? lens equal?
+ if not _diff and _lL != _rL:
+ # lens not equal? Longer - later
+ return _lL - _rL
+ else:
+ # equal
+ return 0
+
+ def _cmp_num(self, lf, rf):
+ # split fragments into lists
+ _lhf = lf.split('.') if '.' in lf else list(lf)
+ _rhf = rf.split('.') if '.' in rf else list(rf)
+ # cast them to ints, delete empty strs
+ _lhf = [int(n) for n in _lhf if len(n)]
+ _rhf = [int(n) for n in _rhf if len(n)]
+
+ return self._cmp_fragment(_lhf, _rhf)
+
+ def _cmp_lex(self, lf, rf):
+ # cast each item into its ORD value
+ _lhf = [ord(n) for n in lf]
+ _rhf = [ord(n) for n in rf]
+
+ return self._cmp_fragment(_lhf, _rhf)
+ # end of cmps
+
+ # main part compared using splitted numbers
+ # if equal, revision is compared using lexical comparizon
+ def __lt__(self, v):
+ if self._cmp_num(self.epoch, v.epoch) < 0:
+ return True
+ elif self._cmp_num(self.upstream, v.upstream) < 0:
+ return True
+ elif self._cmp_lex(self.upstream_rev, v.upstream_rev) < 0:
+ return True
+ else:
+ return False
+
+ def __eq__(self, v):
+ # compare all portions
+ _result = []
+ _result.append(self._cmp_num(self.epoch, v.epoch))
+ _result.append(self._cmp_num(self.upstream, v.upstream))
+ _result.append(self._cmp_lex(self.upstream_rev, v.upstream_rev))
+ # if there is any non-zero, its not equal
+ return not any(_result)
+
+ def __gt__(self, v):
+ if self._cmp_num(self.epoch, v.epoch) > 0:
+ return True
+ elif self._cmp_num(self.upstream, v.upstream) > 0:
+ return True
+ elif self._cmp_lex(self.upstream_rev, v.upstream_rev) > 0:
+ return True
+ else:
+ return False
+
+ def update_parts(self, target, status):
+ # updating parts of version statuses
+ if self._cmp_num(self.epoch, target.epoch) != 0:
+ self.epoch_status = status
+ else:
+ self.epoch_status = const.VERSION_OK
+
+ if self._cmp_num(self.upstream, target.upstream) != 0 \
+ or self._cmp_lex(self.upstream_rev, target.upstream_rev) != 0:
+ self.upstream_status = status
+ else:
+ self.upstream_status = const.VERSION_OK
+
+ if self._cmp_lex(self.debian, target.debian) != 0 \
+ or self._cmp_lex(self.debian_rev, target.debian_rev) != 0:
+ self.debian_status = status
+ else:
+ self.debian_status = const.VERSION_OK
+
+
+class VersionCmpResult(object):
+ status = ""
+ action = ""
+
+ source = None
+ target = None
+
+
+ def __init__(self, i, c, r):
+ # compare three versions and write a result
+ self.source = i
+ self.status = const.VERSION_NA
+ self.action = const.ACT_NA
+
+ # Check if there is a release version present
+ if r and len(r.version) > 0 and r.version != 'n/a':
+ # I < C, installed version is older
+ if i < c:
+ self.target = c
+ if i == r:
+ # installed version is equal vs release version
+ self.status = const.VERSION_OK
+ self.action = const.ACT_UPGRADE
+ elif i > r:
+ # installed version is newer vs release version
+ self.status = const.VERSION_UP
+ self.action = const.ACT_UPGRADE
+ elif i < r and r < c:
+ # installed version is older vs release version
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_NEED_UP
+ self.target = r
+ elif i < r and c == r:
+ # installed version is older vs release version
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_NEED_UP
+ self.target = c
+ elif c < r:
+ # installed and repo versions older vs release version
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_REPO
+ # I > C
+ # installed version is newer
+ elif i > c:
+ self.target = c
+ if c == r:
+ # some unknown version installed
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_NEED_DOWN
+ elif c > r:
+ # installed and repo versions newer than release
+ self.status = const.VERSION_UP
+ self.action = const.ACT_NEED_DOWN
+ elif c < r and r < i:
+ # repo is older vs release and both older vs installed
+ self.status = const.VERSION_UP
+ self.action = const.ACT_REPO
+ elif c < r and r == i:
+ # repo is older vs release, but release version installed
+ self.status = const.VERSION_OK
+ self.action = const.ACT_REPO
+ elif i < r:
+ # both repo and installed older vs release, new target
+ self.status = const.VERSION_DOWN
+ self.action = const.ACT_REPO
+ self.target = r
+ # I = C
+ # installed and linked repo is inline,
+ elif i == c:
+ self.target = c
+ if i < r:
+ # both are old, new target
+ self.status = const.VERSION_ERR
+ self.action = const.ACT_REPO
+ self.target = r
+ elif i > r:
+ # both are newer, same target
+ self.status = const.VERSION_UP
+ self.action = const.ACT_NA
+ elif i == r:
+ # all is ok
+ self.status = const.VERSION_OK
+ self.action = const.ACT_NA
+ else:
+ # no release version present
+ self.target = c
+ if i < c:
+ self.status = const.VERSION_OK
+ self.action = const.ACT_UPGRADE
+ elif i > c:
+ self.status = const.VERSION_UP
+ self.action = const.ACT_NEED_DOWN
+ elif i == c:
+ self.status = const.VERSION_OK
+ self.action = const.ACT_NA
+
+ # and we need to update per-part status
+ self.source.update_parts(self.target, self.status)
+
+ @staticmethod
+ def deb_lower(_s, _t):
+ if _t.debian and _t.debian > _s.debian:
+ return True
+ else:
+ return false
diff --git a/cfg_checker/modules/reclass/__init__.py b/cfg_checker/modules/reclass/__init__.py
index 66576b6..0dcf426 100644
--- a/cfg_checker/modules/reclass/__init__.py
+++ b/cfg_checker/modules/reclass/__init__.py
@@ -10,7 +10,9 @@
def do_list(args):
logger_cli.info("# Reclass list")
- _path = args_utils.get_path_arg(args.models_path)
+ _arg_path = args_utils.get_arg(args, 'models_path')
+ logger_cli.info("-> Current path is: {}".format(_arg_path))
+ _path = args_utils.get_path_arg(_arg_path)
logger_cli.info("# ...models path is '{}'".format(args.models_path))
@@ -38,7 +40,7 @@
def do_diff(args):
logger_cli.info("# Reclass comparer (HTML report: '{}'".format(args.file))
- _filename = args_utils.get_file_arg(args)
+ _filename = args_utils.get_arg(args, 'html')
# checking folder params
_model1 = args_utils.get_path_arg(args.model1)
diff --git a/cfg_checker/nodes.py b/cfg_checker/nodes.py
index b2bfd88..776c8b2 100644
--- a/cfg_checker/nodes.py
+++ b/cfg_checker/nodes.py
@@ -22,7 +22,7 @@
logger_cli.info("# Collecting nodes")
# simple salt rest client
self.salt = salt_utils.SaltRemote()
-
+
# Keys for all nodes
# this is not working in scope of 2016.8.3, will overide with list
logger_cli.debug("...collecting node names existing in the cloud")
@@ -82,6 +82,22 @@
self.nodes
)
)
+ # get master node fqdn
+ self.master_node = filter(
+ lambda nd: self.nodes[nd]['role'] == const.all_roles_map['cfg'],
+ self.nodes
+ )[0]
+
+ # OpenStack versions
+ self.mcp_release = self.salt.pillar_get(
+ self.master_node,
+ "_param:apt_mk_version"
+ )[self.master_node]
+ self.openstack_release = self.salt.pillar_get(
+ self.master_node,
+ "_param:openstack_version"
+ )[self.master_node]
+
def skip_node(self, node):
# Add node to skip list
diff --git a/cfg_checker/reports/reporter.py b/cfg_checker/reports/reporter.py
index e1d6b6f..377c061 100644
--- a/cfg_checker/reports/reporter.py
+++ b/cfg_checker/reports/reporter.py
@@ -1,35 +1,82 @@
-import jinja2
-import six
import abc
+import jinja2
import os
+import six
+import time
from cfg_checker.common import const
+from cfg_checker.common import logger, logger_cli
+from cfg_checker.helpers.console_utils import Progress
pkg_dir = os.path.dirname(__file__)
pkg_dir = os.path.join(pkg_dir, os.pardir, os.pardir)
pkg_dir = os.path.normpath(pkg_dir)
-def shortname(node_fqdn):
- # form shortname out of node fqdn
- return node_fqdn.split(".", 1)[0]
-
-
-def is_equal(pkg_dict):
- # compare versions of given package
- return pkg_dict['installed'] == pkg_dict['candidate']
-
-
-def is_active(node_dict):
- # check node status in node dict
- return node_dict['status'] == const.NODE_UP
-
-
def line_breaks(text):
# replace python linebreaks with html breaks
return text.replace("\n", "<br />")
+def get_sorted_keys(td):
+ # detect if we can sort by desc
+ # Yes, this is slow, but bullet-proof from empty desc
+ _desc = all([bool(td[k]['desc']) for k in td.keys()])
+ # Get sorted list
+ if not _desc:
+ return sorted(td.keys())
+ else:
+ return sorted(
+ td.keys(),
+ key=lambda k: (
+ td[k]['desc']['component'],
+ td[k]['desc']['app'],
+ k
+ )
+ )
+
+
+def get_max(_list):
+ return sorted(_list)[-1]
+
+
+def make_action_label(act):
+ _act_labels = {
+ const.ACT_UPGRADE: "Upgrade possible",
+ const.ACT_NEED_UP: "Needs upgrade",
+ const.ACT_NEED_DOWN: "Needs downgrade",
+ const.ACT_REPO: "Needs repo update",
+ const.ACT_NA: ""
+ }
+ return _act_labels[act]
+
+
+def make_action_class(act):
+ _act_classes = {
+ const.ACT_UPGRADE: "possible",
+ const.ACT_NEED_UP: "needs_up",
+ const.ACT_NEED_DOWN: "needs_down",
+ const.ACT_REPO: "needs_repo",
+ const.ACT_NA: ""
+ }
+ return _act_classes[act]
+
+
+def make_status_label(sts):
+ _status_labels = {
+ const.VERSION_OK: "OK",
+ const.VERSION_UP: "Upgraded",
+ const.VERSION_DOWN: "Downgraded",
+ const.VERSION_ERR: "ERROR",
+ const.VERSION_NA: "N/A"
+ }
+ return _status_labels[sts]
+
+
+def make_status_class(sts):
+ return const.all_statuses[sts]
+
+
@six.add_metaclass(abc.ABCMeta)
class _Base(object):
def __init__(self):
@@ -59,14 +106,9 @@
def __call__(self, payload):
# init data structures
data = self.common_data()
- # payload should have pre-sorted structure
- # system, nodes, clusters, and the rest in other
- data.update({
- "nodes": payload['nodes'],
- "rc_diffs": payload['rc_diffs'],
- "pkg_diffs": payload['pkg_diffs'],
- "tabs": {}
- })
+ # payload should have pre-sorted structure according to report called
+ # nodes, openstack_release, mcp_release, etc...
+ data.update(payload)
# add template specific data
self._extend_data(data)
@@ -75,83 +117,40 @@
self._count_totals(data)
# specific filters
- self.jinja2_env.filters['shortname'] = shortname
- self.jinja2_env.filters['is_equal'] = is_equal
- self.jinja2_env.filters['is_active'] = is_active
self.jinja2_env.filters['linebreaks'] = line_breaks
+ self.jinja2_env.filters['get_max'] = get_max
+
+ self.jinja2_env.filters['get_sorted_keys'] = get_sorted_keys
+ self.jinja2_env.filters['make_status_label'] = make_status_label
+ self.jinja2_env.filters['make_status_class'] = make_status_class
+ self.jinja2_env.filters['make_action_label'] = make_action_label
+ self.jinja2_env.filters['make_action_class'] = make_action_class
# render!
+ logger_cli.info("-> Using template: {}".format(self.tmpl))
tmpl = self.jinja2_env.get_template(self.tmpl)
+ logger_cli.info("-> Rendering")
return tmpl.render(data)
def common_data(self):
return {
'counters': {},
- 'salt_info': {}
+ 'salt_info': {},
+ 'gen_date': time.strftime("%m/%d/%Y %H:%M:%S")
}
def _extend_data(self, data):
pass
-# Package versions report
+# HTML Package versions report
+class CSVAllPackages(_TMPLBase):
+ tmpl = "pkg_versions_csv.j2"
+
+
+# HTML Package versions report
class HTMLPackageCandidates(_TMPLBase):
- tmpl = "pkg_versions_tmpl.j2"
-
- @staticmethod
- def is_fail_uniq(p_dict, p_name, nodes, node_name):
- # look up package fail for nodes with similar role
- _tgroup = nodes[node_name]['node_group']
- # filter all nodes with the same role
- _nodes_list = filter(
- lambda nd: nodes[nd]['node_group'] == _tgroup and nd != node_name,
- nodes
- )
- # lookup same package
- _fail_uniq = False
- for _node_name in _nodes_list:
- # check if there is a package present on node
- _nd = nodes[_node_name]['packages']
- if p_name not in _nd:
- continue
- # if both backages has same version and differ from candidate
- if p_dict['candidate'] == _nd[p_name]['candidate'] \
- and _nd[p_name]['candidate'] == _nd[p_name]['installed']:
- # it is not uniq, mark and break
- _fail_uniq = True
- return _fail_uniq
-
- def _extend_data(self, data):
- # Count values on per-node basis
- for key, value in data['nodes'].iteritems():
- # count differences
- data['counters'][key] = {}
- data['counters'][key]['packages'] = len(value['packages'].keys())
- data['counters'][key]['package_diff'] = 0
- data['counters'][key]['package_eq'] = 0
-
- # Lookup if this fail is uniq for this node
- for pkg_name, pkg_value in value['packages'].iteritems():
- if pkg_value['is_equal']:
- pkg_value['fail_uniq'] = False
- data['counters'][key]['package_eq'] += 1
- else:
- pkg_value['fail_uniq'] = self.is_fail_uniq(
- pkg_value,
- pkg_name,
- data['nodes'],
- key
- )
- data['counters'][key]['package_diff'] += 1
-
- # Count values on all-diffs basis
- for key, value in data['pkg_diffs'].iteritems():
- data['counters'][key] = {}
- data['counters'][key]['df_nodes'] = len(value['df_nodes'].keys())
- data['counters'][key]['eq_nodes'] = len(value['eq_nodes'])
-
- # Save all packages counter
- data['counters']['total_packages'] = data['pkg_diffs'].keys()
+ tmpl = "pkg_versions_html.j2"
# Package versions report
@@ -171,10 +170,6 @@
class HTMLNetworkReport(_TMPLBase):
tmpl = "network_check_tmpl.j2"
- def _extend_data(self, data):
-
- return
-
class ReportToFile(object):
def __init__(self, report, target):