Refactor working with Networks and Pinger class
- Mapper moved to separate module
- Other modules can use Mapper to get desired networks
- salt_master is now a separate single instance
- Updated file handling on salt
- ping.py, an scripted flexible interface to ping command
multithreaded ping execution, 15 at once
- New commands in network: 'ping' and 'list'
- New error when runtime has no network listed in reclass
Fixes:
- Master node code handling
- Unknown node codes detection
- Proper node code search and handling
- File upload procedures updated
- Packages report fix
Change-Id: I5959210aed53b20b04b05ea880218e93239bb661
Related-PROD: PROD-28199
diff --git a/cfg_checker/modules/network/mapper.py b/cfg_checker/modules/network/mapper.py
new file mode 100644
index 0000000..c44775f
--- /dev/null
+++ b/cfg_checker/modules/network/mapper.py
@@ -0,0 +1,430 @@
+import ipaddress
+import json
+
+from cfg_checker.common import logger_cli
+from cfg_checker.common.exception import InvalidReturnException
+from cfg_checker.modules.network.network_errors import NetworkErrors
+from cfg_checker.nodes import salt_master
+
+# TODO: use templated approach
+# net interface structure should be the same
+_if_item = {
+ "name": "unnamed interface",
+ "mac": "",
+ "routes": {},
+ "ip": [],
+ "parameters": {}
+}
+
+# collection of configurations
+_network_item = {
+ "runtime": {},
+ "config": {},
+ "reclass": {}
+}
+
+
+class NetworkMapper(object):
+ RECLASS = "reclass"
+ CONFIG = "config"
+ RUNTIME = "runtime"
+
+ def __init__(self, errors_class=None):
+ logger_cli.info("# Initializing mapper")
+ self.networks = {}
+ self.nodes = salt_master.get_nodes()
+ if errors_class:
+ self.errors = errors_class
+ else:
+ logger_cli.debug("... init error logs folder")
+ self.errors = NetworkErrors()
+
+ # adding net data to tree
+ def _add_data(self, _list, _n, _h, _d):
+ if _n not in _list:
+ _list[_n] = {}
+ _list[_n][_h] = [_d]
+ elif _h not in _list[_n]:
+ # there is no such host, just create it
+ _list[_n][_h] = [_d]
+ else:
+ # there is such host... this is an error
+ self.errors.add_error(
+ self.errors.NET_DUPLICATE_IF,
+ host=_h,
+ dup_if=_d['name']
+ )
+ _list[_n][_h].append(_d)
+
+ # TODO: refactor map creation. Build one map instead of two separate
+ def _map_network_for_host(self, host, if_class, net_list, data):
+ # filter networks for this IF IP
+ _nets = [n for n in net_list.keys() if if_class.ip in n]
+ _masks = [n.netmask for n in _nets]
+ if len(_nets) > 1:
+ # There a multiple network found for this IP, Error
+ self.errors.add_error(
+ self.errors.NET_SUBNET_INTERSECT,
+ host=host,
+ ip=str(if_class.exploded),
+ networks="; ".join([str(_n) for _n in _nets])
+ )
+ # check mask match
+ if len(_nets) > 0 and if_class.netmask not in _masks:
+ self.errors.add_error(
+ self.errors.NET_MASK_MISMATCH,
+ host=host,
+ if_name=data['name'],
+ if_cidr=if_class.exploded,
+ if_mapped_networks=", ".join([str(_n) for _n in _nets])
+ )
+
+ if len(_nets) < 1:
+ self._add_data(net_list, if_class.network, host, data)
+ else:
+ # add all data
+ for net in _nets:
+ self._add_data(net_list, net, host, data)
+
+ return net_list
+
+ def _map_reclass_networks(self):
+ # class uses nodes from self.nodes dict
+ _reclass = {}
+ # Get required pillars
+ salt_master.get_specific_pillar_for_nodes("linux:network")
+ for node in salt_master.nodes.keys():
+ # check if this node
+ if not salt_master.is_node_available(node):
+ continue
+ # get the reclass value
+ _pillar = salt_master.nodes[node]['pillars']['linux']['network']
+ # we should be ready if there is no interface in reclass for a node
+ # for example on APT nohde
+ if 'interface' in _pillar:
+ _pillar = _pillar['interface']
+ else:
+ logger_cli.info(
+ "... node '{}' skipped, no IF section in reclass".format(
+ node
+ )
+ )
+ continue
+ for _if_name, _if_data in _pillar.iteritems():
+ if 'address' in _if_data:
+ _if = ipaddress.IPv4Interface(
+ _if_data['address'] + '/' + _if_data['netmask']
+ )
+ _if_data['name'] = _if_name
+ _if_data['ifs'] = [_if]
+
+ _reclass = self._map_network_for_host(
+ node,
+ _if,
+ _reclass,
+ _if_data
+ )
+
+ return _reclass
+
+ def _map_configured_networks(self):
+ # class uses nodes from self.nodes dict
+ _confs = {}
+
+ return _confs
+
+ def _map_runtime_networks(self):
+ # class uses nodes from self.nodes dict
+ _runtime = {}
+ logger_cli.info("# Mapping node runtime network data")
+ salt_master.prepare_script_on_active_nodes("ifs_data.py")
+ _result = salt_master.execute_script_on_active_nodes(
+ "ifs_data.py",
+ args=["json"]
+ )
+ for key in salt_master.nodes.keys():
+ # check if we are to work with this node
+ if not salt_master.is_node_available(key):
+ continue
+ # due to much data to be passed from salt_master,
+ # it is happening in order
+ if key in _result:
+ _text = _result[key]
+ if '{' in _text and '}' in _text:
+ _text = _text[_text.find('{'):]
+ else:
+ raise InvalidReturnException(
+ "Non-json object returned: '{}'".format(
+ _text
+ )
+ )
+ _dict = json.loads(_text[_text.find('{'):])
+ salt_master.nodes[key]['routes'] = _dict.pop("routes")
+ salt_master.nodes[key]['networks'] = _dict
+ else:
+ salt_master.nodes[key]['networks'] = {}
+ salt_master.nodes[key]['routes'] = {}
+ logger_cli.debug("... {} has {} networks".format(
+ key,
+ len(salt_master.nodes[key]['networks'].keys())
+ ))
+ logger_cli.info("-> done collecting networks data")
+
+ logger_cli.info("-> mapping IPs")
+ # match interfaces by IP subnets
+ for host, node_data in salt_master.nodes.iteritems():
+ if not salt_master.is_node_available(host):
+ continue
+
+ for net_name, net_data in node_data['networks'].iteritems():
+ # get ips and calculate subnets
+ if net_name in ['lo']:
+ # skip the localhost
+ continue
+ # get data and make sure that wide mask goes first
+ _ip4s = sorted(
+ net_data['ipv4'],
+ key=lambda s: s[s.index('/'):]
+ )
+ for _ip_str in _ip4s:
+ # create interface class
+ _if = ipaddress.IPv4Interface(_ip_str)
+ # check if this is a VIP
+ # ...all those will have /32 mask
+ net_data['vip'] = None
+ if _if.network.prefixlen == 32:
+ net_data['vip'] = str(_if.exploded)
+ if 'name' not in net_data:
+ net_data['name'] = net_name
+ if 'ifs' not in net_data:
+ net_data['ifs'] = [_if]
+ # map it
+ _runtime = self._map_network_for_host(
+ host,
+ _if,
+ _runtime,
+ net_data
+ )
+ else:
+ # data is already there, just add VIP
+ net_data['ifs'].append(_if)
+
+ return _runtime
+
+ def map_network(self, source):
+ # maps target network using given source
+ _networks = None
+
+ if source == self.RECLASS:
+ _networks = self._map_reclass_networks()
+ elif source == self.CONFIG:
+ _networks = self._map_configured_networks()
+ elif source == self.RUNTIME:
+ _networks = self._map_runtime_networks()
+
+ self.networks[source] = _networks
+ return _networks
+
+ def print_map(self):
+ """
+ Create text report for CLI
+
+ :return: none
+ """
+ _runtime = self.networks[self.RUNTIME]
+ _reclass = self.networks[self.RECLASS]
+ logger_cli.info("# Reclass networks")
+ logger_cli.info(
+ " {0:17} {1:25}: "
+ "{2:19} {3:5}{4:10} {5}{6} {7} / {8} / {9}".format(
+ "Hostname",
+ "IF",
+ "IP",
+ "rtMTU",
+ "rcMTU",
+ "rtState",
+ "rcState",
+ "rtGate",
+ "rtDef.Gate",
+ "rcGate"
+ )
+ )
+ for network in _reclass:
+ # shortcuts
+ _net = str(network)
+ logger_cli.info("-> {}".format(_net))
+ if network not in _runtime:
+ # reclass has network that not found in runtime
+ self.errors.add_error(
+ self.errors.NET_NO_RUNTIME_NETWORK,
+ reclass_net=str(network)
+ )
+ logger_cli.info(" {:-^50}".format(" No runtime network "))
+ continue
+ names = sorted(_runtime[network].keys())
+ for hostname in names:
+ if not salt_master.is_node_available(hostname, log=False):
+ logger_cli.info(
+ " {0:17} {1}".format(
+ hostname.split('.')[0],
+ "... no data for the node"
+ )
+ )
+ # add non-responsive node erorr
+ self.errors.add_error(
+ self.errors.NET_NODE_NON_RESPONSIVE,
+ host=hostname
+ )
+
+ # print empty row
+ _text = " # node non-responsive"
+ logger_cli.info(
+ " {0:17} {1}".format(
+ hostname.split('.')[0],
+ _text
+ )
+ )
+ continue
+
+ # get the gateway for current net
+ _routes = salt_master.nodes[hostname]['routes']
+ _route = _routes[_net] if _net in _routes else None
+ if not _route:
+ _gate = "no route!"
+ else:
+ _gate = _route['gateway'] if _route['gateway'] else "-"
+
+ # get the default gateway
+ if 'default' in _routes:
+ _d_gate = ipaddress.IPv4Address(
+ _routes['default']['gateway']
+ )
+ else:
+ _d_gate = None
+ _d_gate_str = _d_gate if _d_gate else "No default gateway!"
+
+ _a = _runtime[network][hostname]
+ for _host in _a:
+ for _if in _host['ifs']:
+ # get proper reclass
+ _ip_str = str(_if.exploded)
+ _r = {}
+ if hostname in _reclass[network]:
+ for _item in _reclass[network][hostname]:
+ for _item_ifs in _item['ifs']:
+ if _ip_str == str(_item_ifs.exploded):
+ _r = _item
+ else:
+ self.errors.add_error(
+ self.errors.NET_NODE_UNEXPECTED_IF,
+ host=hostname,
+ if_name=_host['name'],
+ if_ip=_ip_str
+ )
+
+ # check if node is UP
+ if not salt_master.is_node_available(hostname):
+ _r_gate = "-"
+ # get proper network from reclass
+ else:
+ # Lookup match for the ip
+ _r_gate = "no IF in reclass!"
+ # get all networks with this hostname
+ _nets = filter(
+ lambda n: hostname in _reclass[n].keys(),
+ _reclass
+ )
+ _rd = None
+ for _item in _nets:
+ # match ip
+ _r_dat = _reclass[_item][hostname]
+ for _r_ifs in _r_dat:
+ for _r_if in _r_ifs['ifs']:
+ if _if.ip == _r_if.ip:
+ _rd = _r_ifs
+ break
+ if _rd:
+ _gs = 'gateway'
+ _e = "empty"
+ _r_gate = _rd[_gs] if _gs in _rd else _e
+ break
+
+ # IF status in reclass
+ if 'enabled' not in _r:
+ _enabled = "(no record!)"
+ else:
+ _e = "enabled"
+ _d = "disabled"
+ _enabled = "("+_e+")" if _r[_e] else "("+_d+")"
+
+ _name = _host['name']
+ _rc_mtu = _r['mtu'] if 'mtu' in _r else None
+ _rc_mtu_s = str(_rc_mtu) if _rc_mtu else '(-)'
+ # check if this is a VIP address
+ # no checks needed if yes.
+ if _host['vip'] != _ip_str:
+ if _rc_mtu:
+ # if there is an MTU value, match it
+ if _host['mtu'] != _rc_mtu_s:
+ self.errors.add_error(
+ self.errors.NET_MTU_MISMATCH,
+ host=hostname,
+ if_name=_name,
+ if_cidr=_ip_str,
+ reclass_mtu=_rc_mtu,
+ runtime_mtu=_host['mtu']
+ )
+ elif _host['mtu'] != '1500':
+ # there is no MTU value in reclass
+ # and runtime value is not default
+ self.errors.add_error(
+ self.errors.NET_MTU_EMPTY,
+ host=hostname,
+ if_name=_name,
+ if_cidr=_ip_str,
+ if_mtu=_host['mtu']
+ )
+ else:
+ # this is a VIP
+ _name = " "*20
+ _ip_str += " VIP"
+ _enabled = "(-)"
+ _r_gate = "-"
+
+ _text = "{0:25} {1:19} {2:5}{3:10} {4:4}{5:10} " \
+ "{6} / {7} / {8}".format(
+ _name,
+ _ip_str,
+ _host['mtu'],
+ _rc_mtu_s,
+ _host['state'],
+ _enabled,
+ _gate,
+ _d_gate_str,
+ _r_gate
+ )
+ logger_cli.info(
+ " {0:17} {1}".format(
+ hostname.split('.')[0],
+ _text
+ )
+ )
+
+ logger_cli.info("\n# Other networks")
+ _other = [n for n in _runtime if n not in _reclass]
+ for network in _other:
+ logger_cli.info("-> {}".format(str(network)))
+ names = sorted(_runtime[network].keys())
+
+ for hostname in names:
+ for _n in _runtime[network][hostname]:
+ _ifs = [str(ifs.ip) for ifs in _n['ifs']]
+ _text = "{0:25}: {1:19} {2:5} {3:4}".format(
+ _n['name'],
+ ", ".join(_ifs),
+ _n['mtu'],
+ _n['state']
+ )
+ logger_cli.info(
+ " {0:17} {1}".format(hostname.split('.')[0], _text)
+ )