Fixes after live cloud run
- ping, proper handling of multiple IPs on one interface
- node skips accounted on gathering linux versions
- '--force-tag' option for package report
- '--exclude-keywords' option for package report
- 'versions' foldereincluded on 'setup.py install'
Change-Id: I5e1b84f187270789223d50887d9d5d5cb78ee5ba
Related-PROD: PROD-28199
diff --git a/cfg_checker/modules/network/pinger.py b/cfg_checker/modules/network/pinger.py
index 351c05a..e5fa614 100644
--- a/cfg_checker/modules/network/pinger.py
+++ b/cfg_checker/modules/network/pinger.py
@@ -70,13 +70,15 @@
}
for tgt_host, tgt_data in nodes.iteritems():
+ _t = _packets[src_host]["targets"]
for tgt_if in tgt_data:
tgt_if_name = tgt_if['name']
_ip_index = 0
for tgt_ip in tgt_if['ifs']:
_ip = str(tgt_ip.ip)
if _ip not in src_ips:
- _packets[src_host]["targets"][tgt_host] = []
+ if tgt_host not in _t:
+ _t[tgt_host] = []
_tgt = {
"ip": _ip,
"tgt_host": tgt_host,
@@ -85,7 +87,7 @@
"mtu": self.target_mtu,
"size": self.packet_size
}
- _packets[src_host]["targets"][tgt_host].append(
+ _t[tgt_host].append(
_tgt
)
_count += 1
@@ -135,7 +137,14 @@
)
# Parse salt output
_result = _results[src]
- _result = json.loads(_result)
+ try:
+ _result = json.loads(_result)
+ except ValueError:
+ _progress.clearline()
+ logger_cli.error(
+ "# ERROR: Unexpected salt return: '{}'\n".format(_result)
+ )
+ continue
# Handle return codes
for tgt_node, _tgt_ips in _result.iteritems():
for _params in _tgt_ips:
diff --git a/cfg_checker/modules/packages/__init__.py b/cfg_checker/modules/packages/__init__.py
index 45b1f77..9d55c05 100644
--- a/cfg_checker/modules/packages/__init__.py
+++ b/cfg_checker/modules/packages/__init__.py
@@ -29,7 +29,18 @@
metavar='packages_csv_filename',
help="CSV filename to save report"
)
-
+ pkg_report_parser.add_argument(
+ '--force-tag',
+ metavar='force_tag', default=None,
+ help="Tag to do a forced search of release versions in. "
+ "If nothing is found, 'mcp_version' tag will be used"
+ )
+ pkg_report_parser.add_argument(
+ '--exclude-keywords',
+ metavar='exclude_repos_keywords', default="nightly extra",
+ help="Keywords to exclude repos from searching "
+ "release versions. Space delimited: 'nightly extra'"
+ )
pkg_repos = pkg_subparsers.add_parser(
'versions',
help="Parse versions at given URL and create local map"
@@ -95,8 +106,16 @@
"""
_type, _filename = args_utils.get_package_report_type_and_filename(args)
+ if ' ' in args.exclude_keywords:
+ _kw = args.exclude_keywords.split(' ')
+ else:
+ _kw = [args.exclude_keywords]
+
# init connection to salt and collect minion data
- pChecker = checker.CloudPackageChecker()
+ pChecker = checker.CloudPackageChecker(
+ force_tag=args.force_tag,
+ exclude_keywords=_kw
+ )
# collect data on installed packages
pChecker.collect_installed_packages()
# diff installed and candidates
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
index c37ecee..972b12f 100644
--- a/cfg_checker/modules/packages/checker.py
+++ b/cfg_checker/modules/packages/checker.py
@@ -11,7 +11,7 @@
class CloudPackageChecker(object):
- def __init__(self):
+ def __init__(self, force_tag=None, exclude_keywords=[]):
# Init salt master info
if not salt_master.nodes:
salt_master.nodes = salt_master.get_nodes()
@@ -29,6 +29,9 @@
)
)
+ self.force_tag = force_tag
+ self.exclude_repos_keywords = exclude_keywords
+
@staticmethod
def presort_packages(all_packages, full=None):
logger_cli.info("-> Presorting packages")
@@ -144,7 +147,7 @@
for key in salt_master.nodes.keys():
# due to much data to be passed from salt, it is happening in order
- if key in _result:
+ if key in _result and _result[key]:
_text = _result[key]
try:
_dict = json.loads(_text[_text.find('{'):])
@@ -174,13 +177,23 @@
"""
# Preload OpenStack release versions
_desc = PkgVersions()
-
logger_cli.info(
"# Cross-comparing: Installed vs Candidates vs Release"
)
# shortcuts for this cloud values
_os = salt_master.openstack_release
_mcp = salt_master.mcp_release
+ _t = [self.force_tag] if self.force_tag else []
+ _t.append(_mcp)
+
+ logger_cli.info("# Tag search list: {}".format(", ".join(_t)))
+ logger_cli.info("# Openstack version: {}".format(_os))
+ logger_cli.info(
+ "# Release versions repos keyword exclude list: {}".format(
+ ", ".join(self.exclude_repos_keywords)
+ )
+ )
+
# Progress class
_progress = Progress(len(salt_master.nodes.keys()))
_progress_index = 0
@@ -211,14 +224,42 @@
# get node attributes
_linux = salt_master.nodes[node_name]['linux_codename']
_arch = salt_master.nodes[node_name]['linux_arch']
- # get versions for this tag and repo headers
- # excluding 'nightly' repos
- _r = self.rm.get_filtered_versions(
- _name,
- tag=_mcp,
- include=[_os, _linux, _arch],
- exclude=["nightly"]
- )
+ # get versions for tag, Openstack release and repo headers
+ # excluding 'nightly' repos by default
+ _r = {}
+ # if there is a forced tag = use it
+ if self.force_tag:
+ _r = self.rm.get_filtered_versions(
+ _name,
+ tag=self.force_tag,
+ include=[_os, _linux, _arch],
+ exclude=self.exclude_repos_keywords
+ )
+ # if nothing found, look everywhere
+ if not _r:
+ _r = self.rm.get_filtered_versions(
+ _name,
+ tag=self.force_tag,
+ include=[_linux, _arch],
+ exclude=self.exclude_repos_keywords
+ )
+ # if nothing is found at this point,
+ # repeat search using normal tags
+ if not _r:
+ _r = self.rm.get_filtered_versions(
+ _name,
+ tag=_mcp,
+ include=[_os, _linux, _arch],
+ exclude=self.exclude_repos_keywords
+ )
+ # Once again, if nothing found, look everywhere
+ if not _r:
+ _r = self.rm.get_filtered_versions(
+ _name,
+ tag=_mcp,
+ include=[_linux, _arch],
+ exclude=self.exclude_repos_keywords
+ )
# repack versions in flat format
_vs = {}
_sections = {}
diff --git a/cfg_checker/modules/packages/versions.py b/cfg_checker/modules/packages/versions.py
index f12d0b7..c5689d3 100644
--- a/cfg_checker/modules/packages/versions.py
+++ b/cfg_checker/modules/packages/versions.py
@@ -19,7 +19,7 @@
def __init__(self):
# preload csv file
- logger_cli.info("# Preloading MCP release versions")
+ logger_cli.info("# Preloading specific MCP release versions")
with open(os.path.join(
pkg_dir,
'versions',
diff --git a/cfg_checker/nodes.py b/cfg_checker/nodes.py
index d752655..5e535b4 100644
--- a/cfg_checker/nodes.py
+++ b/cfg_checker/nodes.py
@@ -117,9 +117,11 @@
self.get_specific_pillar_for_nodes("_param:linux_system_codename")
self.get_specific_pillar_for_nodes("_param:linux_system_architecture")
for _name in self.nodes.keys():
- _p = self.nodes[_name]['pillars']['_param']
- self.nodes[_name]['linux_codename'] = _p['linux_system_codename']
- self.nodes[_name]['linux_arch'] = _p['linux_system_architecture']
+ _n = self.nodes[_name]
+ if _name not in self.skip_list:
+ _p = _n['pillars']['_param']
+ _n['linux_codename'] = _p['linux_system_codename']
+ _n['linux_arch'] = _p['linux_system_architecture']
def skip_node(self, node):
# Add node to skip list