Skip nodes functionality for Package and Network modules
Added to main entrypoint
- Skip nodes using simple argument with '*' as a trailing wildcard
- Skip nodes using file list
Usability improovement
- Node list preview in status line
- Node stats alignment in net report
Minor fixes:
- Python version detection (3.5+)
- Node counter for each status
- Proper node skip handling
Change-Id: I086ef501bc06f0e739df25349257f1c63a2e2fcf
Related-PROD: PROD-35009
diff --git a/cfg_checker/nodes.py b/cfg_checker/nodes.py
index c261752..13ab3e7 100644
--- a/cfg_checker/nodes.py
+++ b/cfg_checker/nodes.py
@@ -3,7 +3,9 @@
from copy import deepcopy
from cfg_checker.clients import get_salt_remote, salt
-from cfg_checker.common import config, const
+from cfg_checker.common import config
+from cfg_checker.common.const import all_roles_map
+from cfg_checker.common.const import NODE_UP, NODE_DOWN, NODE_SKIP
from cfg_checker.common import logger, logger_cli
from cfg_checker.common import utils
from cfg_checker.common.exception import SaltException
@@ -12,7 +14,7 @@
node_tmpl = {
'role': '',
'node_group': '',
- 'status': const.NODE_DOWN,
+ 'status': NODE_DOWN,
'pillars': {},
'grains': {}
}
@@ -25,7 +27,7 @@
self.salt = salt
self.nodes = None
- def gather_node_info(self):
+ def gather_node_info(self, skip_list, skip_list_file):
# Keys for all nodes
# this is not working in scope of 2016.8.3, will overide with list
logger_cli.debug("... collecting node names existing in the cloud")
@@ -61,33 +63,86 @@
else:
_minions = self.node_keys['minions']
+ # Skip nodes if needed
+ _skipped_minions = []
+ # skip list file
+ if skip_list_file:
+ _valid, _invalid = utils.get_nodes_list(skip_list_file)
+ logger_cli.info(
+ "\n# WARNING: Detected invalid entries "
+ "in nodes skip list:\n".format(
+ "\n".join(_invalid)
+ )
+ )
+ _skipped_minions.extend(_valid)
+ # process wildcard, create node list out of mask
+ if skip_list:
+ _list = []
+ _invalid = []
+ for _item in skip_list:
+ if '*' in _item:
+ _str = _item[:_item.index('*')]
+ _nodes = [_m for _m in _minions if _m.startswith(_str)]
+ if not _nodes:
+ logger_cli.warn(
+ "# WARNING: No nodes found for {}".format(_item)
+ )
+ _list.extend(_nodes)
+ else:
+ if _item in _minions:
+ _list += _item
+ else:
+ logger_cli.warn(
+ "# WARNING: No node found for {}".format(_item)
+ )
+ # removing duplicates
+ _list = list(set(_list))
+ _skipped_minions.extend(_list)
+
# in case API not listed minions, we need all that answer ping
_active = self.salt.get_active_nodes()
logger_cli.info("-> nodes responded: {}".format(len(_active)))
# iterate through all accepted nodes and create a dict for it
self.nodes = {}
self.skip_list = []
+ _domains = set()
for _name in _minions:
_nc = utils.get_node_code(_name)
- _rmap = const.all_roles_map
+ _rmap = all_roles_map
_role = _rmap[_nc] if _nc in _rmap else 'unknown'
- _status = const.NODE_UP if _name in _active else const.NODE_DOWN
- if _status == const.NODE_DOWN:
+ if _name in _skipped_minions:
+ _status = NODE_SKIP
self.skip_list.append(_name)
- logger_cli.info("-> '{}' is down, marked to skip".format(
- _name
- ))
+ else:
+ _status = NODE_UP if _name in _active else NODE_DOWN
+ if _status == NODE_DOWN:
+ self.skip_list.append(_name)
+ logger_cli.info(
+ "-> '{}' is down, "
+ "added to skip list".format(
+ _name
+ )
+ )
self.nodes[_name] = deepcopy(node_tmpl)
+ self.nodes[_name]['shortname'] = _name.split(".", 1)[0]
+ _domains.add(_name.split(".", 1)[1])
self.nodes[_name]['node_group'] = _nc
self.nodes[_name]['role'] = _role
self.nodes[_name]['status'] = _status
+ _domains = list(_domains)
+ if len(_domains) > 1:
+ logger_cli.warning(
+ "Multiple domains detected: {}".format(",".join(_domains))
+ )
+ else:
+ self.domain = _domains[0]
logger_cli.info("-> {} nodes inactive".format(len(self.skip_list)))
logger_cli.info("-> {} nodes collected".format(len(self.nodes)))
# form an all nodes compound string to use in salt
self.active_nodes_compound = self.salt.compound_string_from_list(
filter(
- lambda nd: self.nodes[nd]['status'] == const.NODE_UP,
+ lambda nd: self.nodes[nd]['status'] == NODE_UP,
self.nodes
)
)
@@ -96,7 +151,7 @@
# lambda nd: self.nodes[nd]['role'] == const.all_roles_map['cfg'],
# self.nodes
# )
- _role = const.all_roles_map['cfg']
+ _role = all_roles_map['cfg']
_filtered = [n for n, v in self.nodes.items() if v['role'] == _role]
if len(_filtered) < 1:
raise SaltException(
@@ -137,9 +192,12 @@
else:
return False
- def get_nodes(self):
+ def get_nodes(self, skip_list=None, skip_list_file=None):
if not self.nodes:
- self.gather_node_info()
+ if not skip_list and config.skip_nodes:
+ self.gather_node_info(config.skip_nodes, skip_list_file)
+ else:
+ self.gather_node_info(skip_list, skip_list_file)
return self.nodes
def get_info(self):
@@ -177,7 +235,7 @@
if target_key not in data:
data[target_key] = None
# Save data
- if data['status'] == const.NODE_DOWN:
+ if data['status'] in [NODE_DOWN, NODE_SKIP]:
data[target_key] = None
elif node not in _result:
continue
@@ -219,7 +277,7 @@
if _key not in _data:
_data[_key] = {}
_data = _data[_key]
- if data['status'] == const.NODE_DOWN:
+ if data['status'] in [NODE_DOWN, NODE_SKIP]:
_data[_pillar_keys[-1]] = None
elif not _result[node]:
logger_cli.debug(