Refactor working with Networks and Pinger class
- Mapper moved to separate module
- Other modules can use Mapper to get desired networks
- salt_master is now a separate single instance
- Updated file handling on salt
- ping.py, an scripted flexible interface to ping command
multithreaded ping execution, 15 at once
- New commands in network: 'ping' and 'list'
- New error when runtime has no network listed in reclass
Fixes:
- Master node code handling
- Unknown node codes detection
- Proper node code search and handling
- File upload procedures updated
- Packages report fix
Change-Id: I5959210aed53b20b04b05ea880218e93239bb661
Related-PROD: PROD-28199
diff --git a/cfg_checker/modules/network/__init__.py b/cfg_checker/modules/network/__init__.py
index 075dccf..c81b63a 100644
--- a/cfg_checker/modules/network/__init__.py
+++ b/cfg_checker/modules/network/__init__.py
@@ -1,7 +1,7 @@
from cfg_checker.common import logger_cli
from cfg_checker.helpers import args_utils
+from cfg_checker.modules.network import checker, mapper, pinger
-import checker
command_help = "Network infrastructure checks and reports"
@@ -32,20 +32,47 @@
help="HTML filename to save report"
)
+ net_ping_parser = net_subparsers.add_parser(
+ 'ping',
+ help="Ping all nodes with each other using network CIDR"
+ )
+
+ net_ping_parser.add_argument(
+ '--cidr',
+ metavar='network_ping_cidr',
+ help="Subnet CIDR to ping nodes in"
+ )
+ net_ping_parser.add_argument(
+ '--mtu',
+ metavar='network_ping_mtu',
+ help="MTU size to use. Ping will be done for MTU - 20 - 8"
+ )
+ net_ping_parser.add_argument(
+ '--detailed',
+ action="store_true", default=False,
+ help="Print detailed report at the end"
+ )
+
+ net_subparsers.add_parser(
+ 'map',
+ help="Print network map"
+ )
+
+ net_subparsers.add_parser(
+ 'list',
+ help="List networks in reclass"
+ )
+
return _parser
-def _prepare_check():
- _checker_class = checker.NetworkChecker()
- _checker_class.collect_reclass_networks()
- _checker_class.collect_network_info()
- return _checker_class
-
-
def _prepare_map():
- _map_class = None
+ _mapper = mapper.NetworkMapper()
+ _mapper.map_network(_mapper.RECLASS)
+ _mapper.map_network(_mapper.RUNTIME)
+ _mapper.map_network(_mapper.CONFIG)
- return _map_class
+ return _mapper
def do_check(args):
@@ -53,8 +80,8 @@
# should not print map, etc...
# Just bare summary and errors
logger_cli.info("# Network check to console")
- netChecker = _prepare_check()
- netChecker.print_network_report()
+ netChecker = checker.NetworkChecker()
+ netChecker.check_networks()
# save what was collected
netChecker.errors.save_iteration_data()
@@ -76,7 +103,7 @@
_filename = args_utils.get_arg(args, 'html')
- netChecker = _prepare_check()
+ netChecker = checker.NetworkChecker()
netChecker.create_html_report(_filename)
return
@@ -88,9 +115,21 @@
logger_cli.info("# Network report")
_type, _filename = args_utils.get_network_map_type_and_filename(args)
- # networkMap = _prepare_map
+ networkMap = _prepare_map()
- # TODO: Create map class to generate network map
+ networkMap.print_map()
+
+ return
+
+
+def do_list(args):
+ # Network Map
+ # Should generate network map to console or HTML
+ _map = mapper.NetworkMapper()
+ reclass = _map.map_network(_map.RECLASS)
+ _s = [str(_n) for _n in reclass.keys()]
+ logger_cli.info("# Reclass networks list")
+ logger_cli.info("\n".join(_s))
return
@@ -99,8 +138,13 @@
# Network pinger
# Checks if selected nodes are pingable
# with a desireble parameters: MTU, Frame, etc
+ if not args.cidr:
+ logger_cli.error("\n# Use mcp-check network list to get list of CIDRs")
+ _cidr = args_utils.get_arg(args, "cidr")
+ _pinger = pinger.NetworkPinger(mtu=args.mtu, detailed=args.detailed)
# TODO: Simple ping based on parameters
+ _pinger.ping_nodes(_cidr)
return
@@ -108,6 +152,7 @@
def do_trace(args):
# Network packet tracer
# Check if packet is delivered to proper network host
+ logger_cli.info("# Packet Tracer not yet implemented")
# TODO: Packet tracer
diff --git a/cfg_checker/modules/network/checker.py b/cfg_checker/modules/network/checker.py
index 7e6a239..89db6ba 100644
--- a/cfg_checker/modules/network/checker.py
+++ b/cfg_checker/modules/network/checker.py
@@ -1,382 +1,20 @@
-import ipaddress
-import json
-
-
from cfg_checker.common import logger_cli
+from cfg_checker.modules.network.mapper import NetworkMapper
from cfg_checker.modules.network.network_errors import NetworkErrors
-from cfg_checker.nodes import SaltNodes
from cfg_checker.reports import reporter
-class NetworkChecker(SaltNodes):
+class NetworkChecker(object):
def __init__(self):
- logger_cli.info("# Gathering environment information")
- super(NetworkChecker, self).__init__()
- logger_cli.info("# Initializing error logs folder")
+ logger_cli.debug("... init error logs folder")
self.errors = NetworkErrors()
+ self.mapper = NetworkMapper(self.errors)
- # adding net data to tree
- def _add_data(self, _list, _n, _h, _d):
- if _n not in _list:
- _list[_n] = {}
- _list[_n][_h] = [_d]
- elif _h not in _list[_n]:
- # there is no such host, just create it
- _list[_n][_h] = [_d]
- else:
- # there is such host... this is an error
- self.errors.add_error(
- self.errors.NET_DUPLICATE_IF,
- host=_h,
- dup_if=_d['name']
- )
- _list[_n][_h].append(_d)
+ def check_networks(self):
+ self.mapper.map_network(self.mapper.RECLASS)
+ self.mapper.map_network(self.mapper.RUNTIME)
- # TODO: refactor map creation. Build one map instead of two separate
- def _map_network_for_host(self, host, if_class, net_list, data):
- # filter networks for this IF IP
- _nets = [n for n in net_list.keys() if if_class.ip in n]
- _masks = [n.netmask for n in _nets]
- if len(_nets) > 1:
- # There a multiple network found for this IP, Error
- self.errors.add_error(
- self.errors.NET_SUBNET_INTERSECT,
- host=host,
- networks="; ".join(_nets)
- )
- # check mask match
- if len(_nets) > 0 and if_class.netmask not in _masks:
- self.errors.add_error(
- self.errors.NET_MASK_MISMATCH,
- host=host,
- if_name=data['name'],
- if_cidr=if_class.exploded,
- if_mapped_networks=", ".join([str(_n) for _n in _nets])
- )
-
- if len(_nets) < 1:
- self._add_data(net_list, if_class.network, host, data)
- else:
- # add all data
- for net in _nets:
- self._add_data(net_list, net, host, data)
-
- return net_list
-
- def collect_network_info(self):
- """
- Collects info on the network using ifs_data.py script
-
- :return: none
- """
- logger_cli.info("# Mapping node runtime network data")
- _result = self.execute_script_on_active_nodes(
- "ifs_data.py",
- args=["json"]
- )
- self.stage = "Runtime"
- for key in self.nodes.keys():
- # check if we are to work with this node
- if not self.is_node_available(key):
- continue
- # due to much data to be passed from salt, it is happening in order
- if key in _result:
- _text = _result[key]
- _dict = json.loads(_text[_text.find('{'):])
- self.nodes[key]['routes'] = _dict.pop("routes")
- self.nodes[key]['networks'] = _dict
- else:
- self.nodes[key]['networks'] = {}
- self.nodes[key]['routes'] = {}
- logger_cli.debug("... {} has {} networks".format(
- key,
- len(self.nodes[key]['networks'].keys())
- ))
- logger_cli.info("-> done collecting networks data")
-
- # TODO: Mimic reclass structure for easy compare
- logger_cli.info("### Building network tree")
- # match interfaces by IP subnets
- _all_nets = {}
- for host, node_data in self.nodes.iteritems():
- if not self.is_node_available(host):
- continue
-
- for net_name, net_data in node_data['networks'].iteritems():
- # get ips and calculate subnets
- if net_name in ['lo']:
- # skip the localhost
- continue
- # get data and make sure that wide mask goes first
- _ip4s = sorted(
- net_data['ipv4'],
- key=lambda s: s[s.index('/'):]
- )
- for _ip_str in _ip4s:
- # create interface class
- _if = ipaddress.IPv4Interface(_ip_str)
- # check if this is a VIP
- # ...all those will have /32 mask
- net_data['vip'] = None
- if _if.network.prefixlen == 32:
- net_data['vip'] = str(_if.exploded)
- if 'name' not in net_data:
- net_data['name'] = net_name
- if 'ifs' not in net_data:
- net_data['ifs'] = [_if]
- # map it
- _all_nets = self._map_network_for_host(
- host,
- _if,
- _all_nets,
- net_data
- )
- else:
- # data is already there, just add VIP
- net_data['ifs'].append(_if)
-
- # save collected info
- self.all_nets = _all_nets
-
- def collect_reclass_networks(self):
- logger_cli.info("# Mapping reclass networks")
- self.stage = "Reclass"
- # Get networks from reclass and mark them
- _reclass_nets = {}
- # Get required pillars
- self.get_specific_pillar_for_nodes("linux:network")
- for node in self.nodes.keys():
- # check if this node
- if not self.is_node_available(node):
- continue
- # get the reclass value
- _pillar = self.nodes[node]['pillars']['linux']['network']
- # we should be ready if there is no interface in reclass for a node
- # for example on APT node
- if 'interface' in _pillar:
- _pillar = _pillar['interface']
- else:
- logger_cli.info(
- "... node '{}' skipped, no IF section in reclass".format(
- node
- )
- )
- continue
- for _if_name, _if_data in _pillar.iteritems():
- if 'address' in _if_data:
- _if = ipaddress.IPv4Interface(
- _if_data['address'] + '/' + _if_data['netmask']
- )
- _if_data['name'] = _if_name
- _if_data['ifs'] = [_if]
-
- _reclass_nets = self._map_network_for_host(
- node,
- _if,
- _reclass_nets,
- _if_data
- )
-
- self.reclass_nets = _reclass_nets
-
- def print_network_report(self):
- """
- Create text report for CLI
-
- :return: none
- """
- _all_nets = self.all_nets.keys()
- logger_cli.info("# Reclass networks")
- logger_cli.info(
- " {0:17} {1:25}: "
- "{2:19} {3:5}{4:10} {5}{6} {7} / {8} / {9}".format(
- "Hostname",
- "IF",
- "IP",
- "rtMTU",
- "rcMTU",
- "rtState",
- "rcState",
- "rtGate",
- "rtDef.Gate",
- "rcGate"
- )
- )
- # TODO: Move matching to separate function
- self.stage = "Matching"
- _reclass = [n for n in _all_nets if n in self.reclass_nets]
- for network in _reclass:
- # shortcuts
- _net = str(network)
- logger_cli.info("-> {}".format(_net))
- names = sorted(self.all_nets[network].keys())
- for hostname in names:
- if not self.is_node_available(hostname, log=False):
- logger_cli.info(
- " {0:17} {1}".format(
- hostname.split('.')[0],
- "... no data for the node"
- )
- )
- # add non-responsive node erorr
- self.errors.add_error(
- self.errors.NET_NODE_NON_RESPONSIVE,
- host=hostname
- )
-
- # print empty row
- _text = " # node non-responsive"
- logger_cli.info(
- " {0:17} {1}".format(
- hostname.split('.')[0],
- _text
- )
- )
- continue
-
- # get the gateway for current net
- _routes = self.nodes[hostname]['routes']
- _route = _routes[_net] if _net in _routes else None
- if not _route:
- _gate = "no route!"
- else:
- _gate = _route['gateway'] if _route['gateway'] else "-"
-
- # get the default gateway
- if 'default' in _routes:
- _d_gate = ipaddress.IPv4Address(
- _routes['default']['gateway']
- )
- else:
- _d_gate = None
- _d_gate_str = _d_gate if _d_gate else "No default gateway!"
-
- _a = self.all_nets[network][hostname]
- for _host in _a:
- for _if in _host['ifs']:
- # get proper reclass
- _ip_str = str(_if.exploded)
- _r = {}
- if hostname in self.reclass_nets[network]:
- for _item in self.reclass_nets[network][hostname]:
- for _item_ifs in _item['ifs']:
- if _ip_str == str(_item_ifs.exploded):
- _r = _item
- else:
- self.errors.add_error(
- self.errors.NET_NODE_UNEXPECTED_IF,
- host=hostname,
- if_name=_host['name'],
- if_ip=_ip_str
- )
-
- # check if node is UP
- if not self.is_node_available(hostname):
- _r_gate = "-"
- # get proper network from reclass
- else:
- # Lookup match for the ip
- _r_gate = "no IF in reclass!"
- # get all networks with this hostname
- _rn = self.reclass_nets
- _nets = filter(
- lambda n: hostname in _rn[n].keys(),
- self.reclass_nets
- )
- _rd = None
- for _item in _nets:
- # match ip
- _r_dat = self.reclass_nets[_item][hostname]
- for _r_ifs in _r_dat:
- for _r_if in _r_ifs['ifs']:
- if _if.ip == _r_if.ip:
- _rd = _r_ifs
- break
- if _rd:
- _gs = 'gateway'
- _e = "empty"
- _r_gate = _rd[_gs] if _gs in _rd else _e
- break
-
- # IF status in reclass
- if 'enabled' not in _r:
- _enabled = "(no record!)"
- else:
- _e = "enabled"
- _d = "disabled"
- _enabled = "("+_e+")" if _r[_e] else "("+_d+")"
-
- _name = _host['name']
- _rc_mtu = _r['mtu'] if 'mtu' in _r else None
- _rc_mtu_s = str(_rc_mtu) if _rc_mtu else '(-)'
- # check if this is a VIP address
- # no checks needed if yes.
- if _host['vip'] != _ip_str:
- if _rc_mtu:
- # if there is an MTU value, match it
- if _host['mtu'] != _rc_mtu_s:
- self.errors.add_error(
- self.errors.NET_MTU_MISMATCH,
- host=hostname,
- if_name=_name,
- if_cidr=_ip_str,
- reclass_mtu=_rc_mtu,
- runtime_mtu=_host['mtu']
- )
- elif _host['mtu'] != '1500':
- # there is no MTU value in reclass
- # and runtime value is not default
- self.errors.add_error(
- self.errors.NET_MTU_EMPTY,
- host=hostname,
- if_name=_name,
- if_cidr=_ip_str,
- if_mtu=_host['mtu']
- )
- else:
- # this is a VIP
- _name = " "*20
- _ip_str += " VIP"
- _enabled = "(-)"
- _r_gate = "-"
-
- _text = "{0:25} {1:19} {2:5}{3:10} {4:4}{5:10} " \
- "{6} / {7} / {8}".format(
- _name,
- _ip_str,
- _host['mtu'],
- _rc_mtu_s,
- _host['state'],
- _enabled,
- _gate,
- _d_gate_str,
- _r_gate
- )
- logger_cli.info(
- " {0:17} {1}".format(
- hostname.split('.')[0],
- _text
- )
- )
-
- logger_cli.info("\n# Other networks")
- _other = [n for n in _all_nets if n not in self.reclass_nets]
- for network in _other:
- logger_cli.info("-> {}".format(str(network)))
- names = sorted(self.all_nets[network].keys())
-
- for hostname in names:
- for _n in self.all_nets[network][hostname]:
- _ifs = [str(ifs.ip) for ifs in _n['ifs']]
- _text = "{0:25}: {1:19} {2:5} {3:4}".format(
- _n['name'],
- ", ".join(_ifs),
- _n['mtu'],
- _n['state']
- )
- logger_cli.info(
- " {0:17} {1}".format(hostname.split('.')[0], _text)
- )
+ self.mapper.print_map()
def print_summary(self):
logger_cli.info(self.errors.get_summary(print_zeros=False))
diff --git a/cfg_checker/modules/network/mapper.py b/cfg_checker/modules/network/mapper.py
new file mode 100644
index 0000000..c44775f
--- /dev/null
+++ b/cfg_checker/modules/network/mapper.py
@@ -0,0 +1,430 @@
+import ipaddress
+import json
+
+from cfg_checker.common import logger_cli
+from cfg_checker.common.exception import InvalidReturnException
+from cfg_checker.modules.network.network_errors import NetworkErrors
+from cfg_checker.nodes import salt_master
+
+# TODO: use templated approach
+# net interface structure should be the same
+_if_item = {
+ "name": "unnamed interface",
+ "mac": "",
+ "routes": {},
+ "ip": [],
+ "parameters": {}
+}
+
+# collection of configurations
+_network_item = {
+ "runtime": {},
+ "config": {},
+ "reclass": {}
+}
+
+
+class NetworkMapper(object):
+ RECLASS = "reclass"
+ CONFIG = "config"
+ RUNTIME = "runtime"
+
+ def __init__(self, errors_class=None):
+ logger_cli.info("# Initializing mapper")
+ self.networks = {}
+ self.nodes = salt_master.get_nodes()
+ if errors_class:
+ self.errors = errors_class
+ else:
+ logger_cli.debug("... init error logs folder")
+ self.errors = NetworkErrors()
+
+ # adding net data to tree
+ def _add_data(self, _list, _n, _h, _d):
+ if _n not in _list:
+ _list[_n] = {}
+ _list[_n][_h] = [_d]
+ elif _h not in _list[_n]:
+ # there is no such host, just create it
+ _list[_n][_h] = [_d]
+ else:
+ # there is such host... this is an error
+ self.errors.add_error(
+ self.errors.NET_DUPLICATE_IF,
+ host=_h,
+ dup_if=_d['name']
+ )
+ _list[_n][_h].append(_d)
+
+ # TODO: refactor map creation. Build one map instead of two separate
+ def _map_network_for_host(self, host, if_class, net_list, data):
+ # filter networks for this IF IP
+ _nets = [n for n in net_list.keys() if if_class.ip in n]
+ _masks = [n.netmask for n in _nets]
+ if len(_nets) > 1:
+ # There a multiple network found for this IP, Error
+ self.errors.add_error(
+ self.errors.NET_SUBNET_INTERSECT,
+ host=host,
+ ip=str(if_class.exploded),
+ networks="; ".join([str(_n) for _n in _nets])
+ )
+ # check mask match
+ if len(_nets) > 0 and if_class.netmask not in _masks:
+ self.errors.add_error(
+ self.errors.NET_MASK_MISMATCH,
+ host=host,
+ if_name=data['name'],
+ if_cidr=if_class.exploded,
+ if_mapped_networks=", ".join([str(_n) for _n in _nets])
+ )
+
+ if len(_nets) < 1:
+ self._add_data(net_list, if_class.network, host, data)
+ else:
+ # add all data
+ for net in _nets:
+ self._add_data(net_list, net, host, data)
+
+ return net_list
+
+ def _map_reclass_networks(self):
+ # class uses nodes from self.nodes dict
+ _reclass = {}
+ # Get required pillars
+ salt_master.get_specific_pillar_for_nodes("linux:network")
+ for node in salt_master.nodes.keys():
+ # check if this node
+ if not salt_master.is_node_available(node):
+ continue
+ # get the reclass value
+ _pillar = salt_master.nodes[node]['pillars']['linux']['network']
+ # we should be ready if there is no interface in reclass for a node
+ # for example on APT nohde
+ if 'interface' in _pillar:
+ _pillar = _pillar['interface']
+ else:
+ logger_cli.info(
+ "... node '{}' skipped, no IF section in reclass".format(
+ node
+ )
+ )
+ continue
+ for _if_name, _if_data in _pillar.iteritems():
+ if 'address' in _if_data:
+ _if = ipaddress.IPv4Interface(
+ _if_data['address'] + '/' + _if_data['netmask']
+ )
+ _if_data['name'] = _if_name
+ _if_data['ifs'] = [_if]
+
+ _reclass = self._map_network_for_host(
+ node,
+ _if,
+ _reclass,
+ _if_data
+ )
+
+ return _reclass
+
+ def _map_configured_networks(self):
+ # class uses nodes from self.nodes dict
+ _confs = {}
+
+ return _confs
+
+ def _map_runtime_networks(self):
+ # class uses nodes from self.nodes dict
+ _runtime = {}
+ logger_cli.info("# Mapping node runtime network data")
+ salt_master.prepare_script_on_active_nodes("ifs_data.py")
+ _result = salt_master.execute_script_on_active_nodes(
+ "ifs_data.py",
+ args=["json"]
+ )
+ for key in salt_master.nodes.keys():
+ # check if we are to work with this node
+ if not salt_master.is_node_available(key):
+ continue
+ # due to much data to be passed from salt_master,
+ # it is happening in order
+ if key in _result:
+ _text = _result[key]
+ if '{' in _text and '}' in _text:
+ _text = _text[_text.find('{'):]
+ else:
+ raise InvalidReturnException(
+ "Non-json object returned: '{}'".format(
+ _text
+ )
+ )
+ _dict = json.loads(_text[_text.find('{'):])
+ salt_master.nodes[key]['routes'] = _dict.pop("routes")
+ salt_master.nodes[key]['networks'] = _dict
+ else:
+ salt_master.nodes[key]['networks'] = {}
+ salt_master.nodes[key]['routes'] = {}
+ logger_cli.debug("... {} has {} networks".format(
+ key,
+ len(salt_master.nodes[key]['networks'].keys())
+ ))
+ logger_cli.info("-> done collecting networks data")
+
+ logger_cli.info("-> mapping IPs")
+ # match interfaces by IP subnets
+ for host, node_data in salt_master.nodes.iteritems():
+ if not salt_master.is_node_available(host):
+ continue
+
+ for net_name, net_data in node_data['networks'].iteritems():
+ # get ips and calculate subnets
+ if net_name in ['lo']:
+ # skip the localhost
+ continue
+ # get data and make sure that wide mask goes first
+ _ip4s = sorted(
+ net_data['ipv4'],
+ key=lambda s: s[s.index('/'):]
+ )
+ for _ip_str in _ip4s:
+ # create interface class
+ _if = ipaddress.IPv4Interface(_ip_str)
+ # check if this is a VIP
+ # ...all those will have /32 mask
+ net_data['vip'] = None
+ if _if.network.prefixlen == 32:
+ net_data['vip'] = str(_if.exploded)
+ if 'name' not in net_data:
+ net_data['name'] = net_name
+ if 'ifs' not in net_data:
+ net_data['ifs'] = [_if]
+ # map it
+ _runtime = self._map_network_for_host(
+ host,
+ _if,
+ _runtime,
+ net_data
+ )
+ else:
+ # data is already there, just add VIP
+ net_data['ifs'].append(_if)
+
+ return _runtime
+
+ def map_network(self, source):
+ # maps target network using given source
+ _networks = None
+
+ if source == self.RECLASS:
+ _networks = self._map_reclass_networks()
+ elif source == self.CONFIG:
+ _networks = self._map_configured_networks()
+ elif source == self.RUNTIME:
+ _networks = self._map_runtime_networks()
+
+ self.networks[source] = _networks
+ return _networks
+
+ def print_map(self):
+ """
+ Create text report for CLI
+
+ :return: none
+ """
+ _runtime = self.networks[self.RUNTIME]
+ _reclass = self.networks[self.RECLASS]
+ logger_cli.info("# Reclass networks")
+ logger_cli.info(
+ " {0:17} {1:25}: "
+ "{2:19} {3:5}{4:10} {5}{6} {7} / {8} / {9}".format(
+ "Hostname",
+ "IF",
+ "IP",
+ "rtMTU",
+ "rcMTU",
+ "rtState",
+ "rcState",
+ "rtGate",
+ "rtDef.Gate",
+ "rcGate"
+ )
+ )
+ for network in _reclass:
+ # shortcuts
+ _net = str(network)
+ logger_cli.info("-> {}".format(_net))
+ if network not in _runtime:
+ # reclass has network that not found in runtime
+ self.errors.add_error(
+ self.errors.NET_NO_RUNTIME_NETWORK,
+ reclass_net=str(network)
+ )
+ logger_cli.info(" {:-^50}".format(" No runtime network "))
+ continue
+ names = sorted(_runtime[network].keys())
+ for hostname in names:
+ if not salt_master.is_node_available(hostname, log=False):
+ logger_cli.info(
+ " {0:17} {1}".format(
+ hostname.split('.')[0],
+ "... no data for the node"
+ )
+ )
+ # add non-responsive node erorr
+ self.errors.add_error(
+ self.errors.NET_NODE_NON_RESPONSIVE,
+ host=hostname
+ )
+
+ # print empty row
+ _text = " # node non-responsive"
+ logger_cli.info(
+ " {0:17} {1}".format(
+ hostname.split('.')[0],
+ _text
+ )
+ )
+ continue
+
+ # get the gateway for current net
+ _routes = salt_master.nodes[hostname]['routes']
+ _route = _routes[_net] if _net in _routes else None
+ if not _route:
+ _gate = "no route!"
+ else:
+ _gate = _route['gateway'] if _route['gateway'] else "-"
+
+ # get the default gateway
+ if 'default' in _routes:
+ _d_gate = ipaddress.IPv4Address(
+ _routes['default']['gateway']
+ )
+ else:
+ _d_gate = None
+ _d_gate_str = _d_gate if _d_gate else "No default gateway!"
+
+ _a = _runtime[network][hostname]
+ for _host in _a:
+ for _if in _host['ifs']:
+ # get proper reclass
+ _ip_str = str(_if.exploded)
+ _r = {}
+ if hostname in _reclass[network]:
+ for _item in _reclass[network][hostname]:
+ for _item_ifs in _item['ifs']:
+ if _ip_str == str(_item_ifs.exploded):
+ _r = _item
+ else:
+ self.errors.add_error(
+ self.errors.NET_NODE_UNEXPECTED_IF,
+ host=hostname,
+ if_name=_host['name'],
+ if_ip=_ip_str
+ )
+
+ # check if node is UP
+ if not salt_master.is_node_available(hostname):
+ _r_gate = "-"
+ # get proper network from reclass
+ else:
+ # Lookup match for the ip
+ _r_gate = "no IF in reclass!"
+ # get all networks with this hostname
+ _nets = filter(
+ lambda n: hostname in _reclass[n].keys(),
+ _reclass
+ )
+ _rd = None
+ for _item in _nets:
+ # match ip
+ _r_dat = _reclass[_item][hostname]
+ for _r_ifs in _r_dat:
+ for _r_if in _r_ifs['ifs']:
+ if _if.ip == _r_if.ip:
+ _rd = _r_ifs
+ break
+ if _rd:
+ _gs = 'gateway'
+ _e = "empty"
+ _r_gate = _rd[_gs] if _gs in _rd else _e
+ break
+
+ # IF status in reclass
+ if 'enabled' not in _r:
+ _enabled = "(no record!)"
+ else:
+ _e = "enabled"
+ _d = "disabled"
+ _enabled = "("+_e+")" if _r[_e] else "("+_d+")"
+
+ _name = _host['name']
+ _rc_mtu = _r['mtu'] if 'mtu' in _r else None
+ _rc_mtu_s = str(_rc_mtu) if _rc_mtu else '(-)'
+ # check if this is a VIP address
+ # no checks needed if yes.
+ if _host['vip'] != _ip_str:
+ if _rc_mtu:
+ # if there is an MTU value, match it
+ if _host['mtu'] != _rc_mtu_s:
+ self.errors.add_error(
+ self.errors.NET_MTU_MISMATCH,
+ host=hostname,
+ if_name=_name,
+ if_cidr=_ip_str,
+ reclass_mtu=_rc_mtu,
+ runtime_mtu=_host['mtu']
+ )
+ elif _host['mtu'] != '1500':
+ # there is no MTU value in reclass
+ # and runtime value is not default
+ self.errors.add_error(
+ self.errors.NET_MTU_EMPTY,
+ host=hostname,
+ if_name=_name,
+ if_cidr=_ip_str,
+ if_mtu=_host['mtu']
+ )
+ else:
+ # this is a VIP
+ _name = " "*20
+ _ip_str += " VIP"
+ _enabled = "(-)"
+ _r_gate = "-"
+
+ _text = "{0:25} {1:19} {2:5}{3:10} {4:4}{5:10} " \
+ "{6} / {7} / {8}".format(
+ _name,
+ _ip_str,
+ _host['mtu'],
+ _rc_mtu_s,
+ _host['state'],
+ _enabled,
+ _gate,
+ _d_gate_str,
+ _r_gate
+ )
+ logger_cli.info(
+ " {0:17} {1}".format(
+ hostname.split('.')[0],
+ _text
+ )
+ )
+
+ logger_cli.info("\n# Other networks")
+ _other = [n for n in _runtime if n not in _reclass]
+ for network in _other:
+ logger_cli.info("-> {}".format(str(network)))
+ names = sorted(_runtime[network].keys())
+
+ for hostname in names:
+ for _n in _runtime[network][hostname]:
+ _ifs = [str(ifs.ip) for ifs in _n['ifs']]
+ _text = "{0:25}: {1:19} {2:5} {3:4}".format(
+ _n['name'],
+ ", ".join(_ifs),
+ _n['mtu'],
+ _n['state']
+ )
+ logger_cli.info(
+ " {0:17} {1}".format(hostname.split('.')[0], _text)
+ )
diff --git a/cfg_checker/modules/network/network_errors.py b/cfg_checker/modules/network/network_errors.py
index cb9dfef..6086be0 100644
--- a/cfg_checker/modules/network/network_errors.py
+++ b/cfg_checker/modules/network/network_errors.py
@@ -15,6 +15,7 @@
NET_MASK_MISMATCH = next(_c)
NET_NODE_NON_RESPONSIVE = next(_c)
NET_NODE_UNEXPECTED_IF = next(_c)
+ NET_NO_RUNTIME_NETWORK = next(_c)
def __init__(self):
super(NetworkErrors, self).__init__("NET")
@@ -47,6 +48,10 @@
self.NET_NODE_UNEXPECTED_IF,
"Node has unexpected IF with mapped IP"
)
+ self.add_error_type(
+ self.NET_NO_RUNTIME_NETWORK,
+ "Reclass network not found in Runtime"
+ )
del _c
diff --git a/cfg_checker/modules/network/pinger.py b/cfg_checker/modules/network/pinger.py
new file mode 100644
index 0000000..8eea315
--- /dev/null
+++ b/cfg_checker/modules/network/pinger.py
@@ -0,0 +1,180 @@
+import ipaddress
+import json
+
+from cfg_checker.common import logger, logger_cli
+from cfg_checker.helpers.console_utils import Progress
+from cfg_checker.modules.network.mapper import NetworkMapper
+from cfg_checker.nodes import salt_master
+
+
+# ping -s 9072 -M do -n -c 1 -w 1 -W 1 ctl01
+
+
+# This is independent class with a salt.nodes input
+class NetworkPinger(object):
+ def __init__(self, mtu=None, detailed=False):
+ logger_cli.info("# Initializing")
+ # all active nodes in the cloud
+ self.target_nodes = salt_master.get_nodes()
+ # default MTU value
+ self.target_mtu = mtu if mtu else 64
+ # only data
+ self.packet_size = int(self.target_mtu) - 20 - 8
+ self.detailed_summary = detailed
+
+ def _collect_node_addresses(self, target_net):
+ # use reclass model and standard methods
+ # to create list of nodes with target network
+ _mapper = NetworkMapper()
+ _reclass = _mapper.map_network(_mapper.RECLASS)
+ if target_net in _reclass:
+ return _reclass[target_net]
+ else:
+ logger_cli.info(
+ "# Target network of {} not found in reclass".format(
+ target_net.exploded
+ )
+ )
+ return None
+
+ def ping_nodes(self, network_cidr_str):
+ # Conduct actual ping using network CIDR
+ logger_cli.info("# Collecting node pairs")
+ _fake_if = ipaddress.IPv4Interface(unicode(network_cidr_str))
+ _net = _fake_if.network
+ # collect nodes and ips from reclass
+ nodes = self._collect_node_addresses(_net)
+ # build list of packets to be sent
+ # source -> target
+ _count = 0
+ _packets = {}
+ _nodes = sorted(nodes.keys())
+ _nodes_total = len(_nodes)
+ logger_cli.info("-> {} nodes found within subnet of '{}'".format(
+ _nodes_total,
+ network_cidr_str
+ ))
+ while len(_nodes) > 0:
+ src_host = _nodes.pop()
+ src_data = nodes[src_host]
+ src_if_name = src_data[0]['name']
+ src_ips = [str(_if.ip) for _if in src_data[0]['ifs']]
+ _packets[src_host] = {
+ "ip": src_ips[0],
+ "if_name": src_if_name,
+ "targets": {}
+ }
+
+ for tgt_host, tgt_data in nodes.iteritems():
+ for tgt_if in tgt_data:
+ tgt_if_name = tgt_if['name']
+ _ip_index = 0
+ for tgt_ip in tgt_if['ifs']:
+ _ip = str(tgt_ip.ip)
+ if _ip not in src_ips:
+ _packets[src_host]["targets"][tgt_host] = []
+ _tgt = {
+ "ip": _ip,
+ "tgt_host": tgt_host,
+ "ip_index": _ip_index,
+ "if_name": tgt_if_name,
+ "mtu": self.target_mtu,
+ "size": self.packet_size
+ }
+ _packets[src_host]["targets"][tgt_host].append(
+ _tgt
+ )
+ _count += 1
+ _ip_index += 1
+ else:
+ pass
+ logger_cli.info("-> {} packets to send".format(_count))
+
+ # do ping of packets
+ logger_cli.info("# Pinging nodes: MTU={}".format(self.target_mtu))
+ salt_master.prepare_script_on_active_nodes("ping.py")
+ _errors = []
+ _success = []
+ _progress = Progress(_count)
+ _progress_index = 0
+ _node_index = 0
+ for src, src_data in _packets.iteritems():
+ _targets = src_data["targets"]
+ _node_index += 1
+ # create 'targets.json' on source host
+ _path = salt_master.prepare_json_on_node(
+ src,
+ _targets,
+ "targets.json"
+ )
+ # execute ping.py
+ _results = salt_master.execute_script_on_node(
+ src,
+ "ping.py",
+ args=[_path]
+ )
+ _progress_index += len(_targets)
+ # print progress
+ _progress.write_progress(
+ _progress_index,
+ note='/ {}/{} nodes / current {}'.format(
+ _node_index,
+ _nodes_total,
+ src
+ )
+ )
+ # Parse salt output
+ _result = _results[src]
+ _result = json.loads(_result)
+ # Handle return codes
+ for tgt_node, _tgt_ips in _result.iteritems():
+ for _params in _tgt_ips:
+ _body = "{}({}) --{}--> {}({}@{})\n".format(
+ src,
+ src_data["if_name"],
+ _params["returncode"],
+ tgt_node,
+ _params["if_name"],
+ _params["ip"]
+ )
+ _stdout = ""
+ _stderr = ""
+ if len(_params["stdout"]) > 0:
+ _stdout = "stdout:\n{}\n".format(_params["stdout"])
+ if len(_params["stderr"]) > 0:
+ _stderr = "stderr:\n{}\n".format(_params["stderr"])
+
+ if _params["returncode"]:
+ _errors.append("FAIL: {}{}{}".format(
+ _body,
+ _stdout,
+ _stderr
+ ))
+ else:
+ _success.append("PASS: {}{}{}".format(
+ _body,
+ _stdout,
+ _stderr
+ ))
+
+ # Parse results back in place
+ src_data["targets"] = _result
+
+ _progress.newline()
+
+ if self.detailed_summary:
+ logger_cli.info("\n{:=^8s}".format("PASS"))
+ logger_cli.info("\n".join(_success))
+ else:
+ logger.info("\n{:=^8s}".format("PASS"))
+ logger.info("\n".join(_success))
+ if len(_errors) > 0:
+ logger_cli.info("\n{:=^8s}".format("FAIL"))
+ logger_cli.info("\n".join(_errors))
+
+ logger_cli.info(
+ "# {} failed, {} passed".format(
+ len(_errors),
+ len(_success)
+ )
+ )
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
index 3225f70..0bcb1a6 100644
--- a/cfg_checker/modules/packages/checker.py
+++ b/cfg_checker/modules/packages/checker.py
@@ -3,13 +3,13 @@
from cfg_checker.common import const, logger_cli
from cfg_checker.common.exception import ConfigException
from cfg_checker.helpers.console_utils import Progress
-from cfg_checker.nodes import SaltNodes
+from cfg_checker.nodes import salt_master
from cfg_checker.reports import reporter
from versions import DebianVersion, PkgVersions, VersionCmpResult
-class CloudPackageChecker(SaltNodes):
+class CloudPackageChecker(object):
@staticmethod
def presort_packages(all_packages, full=None):
logger_cli.info("-> Presorting packages")
@@ -52,14 +52,14 @@
# sort packages
_pn, _val = all_packages.popitem()
_c = _val['desc']['component']
- if full:
+ if not full:
# Check if this packet has errors
# if all is ok -> just skip it
_max_status = max(_val['results'].keys())
if _max_status <= const.VERSION_OK:
_max_action = max(_val['results'][_max_status].keys())
if _max_action == const.ACT_NA:
- # this package do not ha any comments
+ # this package do not has any comments
# ...just skip it from report
continue
@@ -118,9 +118,12 @@
:return: none
"""
logger_cli.info("# Collecting installed packages")
- _result = self.execute_script_on_active_nodes("pkg_versions.py")
+ if not salt_master.nodes:
+ salt_master.nodes = salt_master.get_nodes()
+ salt_master.prepare_script_on_active_nodes("pkg_versions.py")
+ _result = salt_master.execute_script_on_active_nodes("pkg_versions.py")
- for key in self.nodes.keys():
+ for key in salt_master.nodes.keys():
# due to much data to be passed from salt, it is happening in order
if key in _result:
_text = _result[key]
@@ -135,12 +138,12 @@
)
_dict = {}
- self.nodes[key]['packages'] = _dict
+ salt_master.nodes[key]['packages'] = _dict
else:
- self.nodes[key]['packages'] = {}
+ salt_master.nodes[key]['packages'] = {}
logger_cli.debug("... {} has {} packages installed".format(
key,
- len(self.nodes[key]['packages'].keys())
+ len(salt_master.nodes[key]['packages'].keys())
))
logger_cli.info("-> Done")
@@ -156,12 +159,12 @@
logger_cli.info(
"# Cross-comparing: Installed vs Candidates vs Release"
)
- _progress = Progress(len(self.nodes.keys()))
+ _progress = Progress(len(salt_master.nodes.keys()))
_progress_index = 0
_total_processed = 0
# Collect packages from all of the nodes in flat dict
_all_packages = {}
- for node_name, node_value in self.nodes.iteritems():
+ for node_name, node_value in salt_master.nodes.iteritems():
_uniq_len = len(_all_packages.keys())
_progress_index += 1
# progress will jump from node to node
@@ -182,8 +185,8 @@
# All packages list with version and node list
if _name not in _all_packages:
# shortcuts for this cloud values
- _os = self.openstack_release
- _mcp = self.mcp_release
+ _os = salt_master.openstack_release
+ _mcp = salt_master.mcp_release
_pkg_desc = {}
if _desc[_name]:
# shortcut to version library
@@ -256,9 +259,9 @@
filename
)
payload = {
- "nodes": self.nodes,
- "mcp_release": self.mcp_release,
- "openstack_release": self.openstack_release
+ "nodes": salt_master.nodes,
+ "mcp_release": salt_master.mcp_release,
+ "openstack_release": salt_master.openstack_release
}
payload.update(self.presort_packages(self._packages, full))
_report(payload)