Coverage for cfg_checker/modules/network/mapper.py : 7%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import ipaddress
2import json
3from copy import deepcopy
5from cfg_checker.common import logger_cli
6from cfg_checker.common.exception import InvalidReturnException
7from cfg_checker.modules.network.network_errors import NetworkErrors
8from cfg_checker.nodes import salt_master
10# TODO: use templated approach
11# net interface structure should be the same
12_if_item = {
13 "name": "unnamed interface",
14 "mac": "",
15 "routes": {},
16 "proto": "",
17 "ip": [],
18 "parameters": {}
19}
21# collection of configurations
22_network_item = {
23 "runtime": {},
24 "config": {},
25 "reclass": {}
26}
29class NetworkMapper(object):
30 RECLASS = "reclass"
31 CONFIG = "config"
32 RUNTIME = "runtime"
34 def __init__(self, errors_class=None):
35 logger_cli.info("# Initializing mapper")
36 # init networks and nodes
37 self.networks = {}
38 self.nodes = salt_master.get_nodes()
39 self.cluster = salt_master.get_info()
40 # init and pre-populate interfaces
41 self.interfaces = {k: {} for k in self.nodes}
42 # Init errors class
43 if errors_class:
44 self.errors = errors_class
45 else:
46 logger_cli.debug("... init error logs folder")
47 self.errors = NetworkErrors()
49 def prepare_all_maps(self):
50 self.map_network(self.RECLASS)
51 self.map_network(self.RUNTIME)
52 self.map_network(self.CONFIG)
54 # adding net data to tree
55 def _add_data(self, _list, _n, _h, _d):
56 if _n not in _list:
57 _list[_n] = {}
58 _list[_n][_h] = [_d]
59 elif _h not in _list[_n]:
60 # there is no such host, just create it
61 _list[_n][_h] = [_d]
62 else:
63 # there is such host... this is an error
64 self.errors.add_error(
65 self.errors.NET_DUPLICATE_IF,
66 host=_h,
67 dup_if=_d['name']
68 )
69 _list[_n][_h].append(_d)
71 # TODO: refactor map creation. Build one map instead of two separate
72 def _map_network_for_host(self, host, if_class, net_list, data):
73 # filter networks for this IF IP
74 _nets = [n for n in net_list.keys() if if_class.ip in n]
75 _masks = [n.netmask for n in _nets]
76 if len(_nets) > 1:
77 # There a multiple network found for this IP, Error
78 self.errors.add_error(
79 self.errors.NET_SUBNET_INTERSECT,
80 host=host,
81 ip=str(if_class.exploded),
82 networks="; ".join([str(_n) for _n in _nets])
83 )
84 # check mask match
85 if len(_nets) > 0 and if_class.netmask not in _masks:
86 self.errors.add_error(
87 self.errors.NET_MASK_MISMATCH,
88 host=host,
89 if_name=data['name'],
90 if_cidr=if_class.exploded,
91 if_mapped_networks=", ".join([str(_n) for _n in _nets])
92 )
94 if len(_nets) < 1:
95 self._add_data(net_list, if_class.network, host, data)
96 else:
97 # add all data
98 for net in _nets:
99 self._add_data(net_list, net, host, data)
101 return net_list
103 def _map_reclass_networks(self):
104 # class uses nodes from self.nodes dict
105 _reclass = {}
106 # Get required pillars
107 salt_master.get_specific_pillar_for_nodes("linux:network")
108 for node in salt_master.nodes.keys():
109 # check if this node
110 if not salt_master.is_node_available(node):
111 continue
112 # get the reclass value
113 _pillar = salt_master.nodes[node]['pillars']['linux']['network']
114 # we should be ready if there is no interface in reclass for a node
115 # for example on APT node
116 if 'interface' in _pillar:
117 _pillar = _pillar['interface']
118 else:
119 logger_cli.info(
120 "... node '{}' skipped, no IF section in reclass".format(
121 node
122 )
123 )
124 continue
126 # build map based on IPs and save info too
127 for if_name, _dat in _pillar.items():
128 # get proper IF name
129 _if_name = if_name if 'name' not in _dat else _dat['name']
130 # place it
131 if _if_name not in self.interfaces[node]:
132 self.interfaces[node][_if_name] = deepcopy(_network_item)
133 self.interfaces[node][_if_name]['reclass'] = deepcopy(_dat)
134 # map network if any
135 if 'address' in _dat:
136 _if = ipaddress.IPv4Interface(
137 _dat['address'] + '/' + _dat['netmask']
138 )
139 _dat['name'] = _if_name
140 _dat['ifs'] = [_if]
142 _reclass = self._map_network_for_host(
143 node,
144 _if,
145 _reclass,
146 _dat
147 )
149 return _reclass
151 def _map_configured_networks(self):
152 # class uses nodes from self.nodes dict
153 _confs = {}
155 # TODO: parse /etc/network/interfaces
157 return _confs
159 def _map_runtime_networks(self):
160 # class uses nodes from self.nodes dict
161 _runtime = {}
162 logger_cli.info("# Mapping node runtime network data")
163 salt_master.prepare_script_on_active_nodes("ifs_data.py")
164 _result = salt_master.execute_script_on_active_nodes(
165 "ifs_data.py",
166 args=["json"]
167 )
168 for key in salt_master.nodes.keys():
169 # check if we are to work with this node
170 if not salt_master.is_node_available(key):
171 continue
172 # due to much data to be passed from salt_master,
173 # it is happening in order
174 if key in _result:
175 _text = _result[key]
176 if '{' in _text and '}' in _text:
177 _text = _text[_text.find('{'):]
178 else:
179 raise InvalidReturnException(
180 "Non-json object returned: '{}'".format(
181 _text
182 )
183 )
184 _dict = json.loads(_text[_text.find('{'):])
185 salt_master.nodes[key]['routes'] = _dict.pop("routes")
186 salt_master.nodes[key]['networks'] = _dict
187 else:
188 salt_master.nodes[key]['networks'] = {}
189 salt_master.nodes[key]['routes'] = {}
190 logger_cli.debug("... {} has {} networks".format(
191 key,
192 len(salt_master.nodes[key]['networks'].keys())
193 ))
194 logger_cli.info("-> done collecting networks data")
196 logger_cli.info("-> mapping IPs")
197 # match interfaces by IP subnets
198 for host, node_data in salt_master.nodes.items():
199 if not salt_master.is_node_available(host):
200 continue
202 for net_name, net_data in node_data['networks'].items():
203 # cut net name
204 _i = net_name.find('@')
205 _name = net_name if _i < 0 else net_name[:_i]
206 # get ips and calculate subnets
207 if _name in ['lo']:
208 # skip the localhost
209 continue
210 else:
211 # add collected data to interface storage
212 if _name not in self.interfaces[host]:
213 self.interfaces[host][_name] = \
214 deepcopy(_network_item)
215 self.interfaces[host][_name]['runtime'] = \
216 deepcopy(net_data)
218 # get data and make sure that wide mask goes first
219 _ip4s = sorted(
220 net_data['ipv4'],
221 key=lambda s: s[s.index('/'):]
222 )
223 for _ip_str in _ip4s:
224 # create interface class
225 _if = ipaddress.IPv4Interface(_ip_str)
226 # check if this is a VIP
227 # ...all those will have /32 mask
228 net_data['vip'] = None
229 if _if.network.prefixlen == 32:
230 net_data['vip'] = str(_if.exploded)
231 if 'name' not in net_data:
232 net_data['name'] = _name
233 if 'ifs' not in net_data:
234 net_data['ifs'] = [_if]
235 # map it
236 _runtime = self._map_network_for_host(
237 host,
238 _if,
239 _runtime,
240 net_data
241 )
242 else:
243 # data is already there, just add VIP
244 net_data['ifs'].append(_if)
246 def process_interface(lvl, interface, tree, res):
247 # get childs for each root
248 # tree row item (<if_name>, [<parents>], [<childs>])
249 if lvl not in tree:
250 # - no level - add it
251 tree[lvl] = {}
252 # there is such interface in this level?
253 if interface not in tree[lvl]:
254 # - IF not present
255 _n = ''
256 if interface not in res:
257 _n = 'unknown IF'
258 _p = None
259 _c = None
260 else:
261 # -- get parents, add
262 _p = res[interface]['lower']
263 # -- get childs, add
264 _c = res[interface]['upper']
266 # if None, put empty list
267 _p = _p if _p else []
268 # if None, put empty list
269 _c = _c if _c else []
270 tree[lvl].update({
271 interface: {
272 "note": _n,
273 "parents": _p,
274 "children": _c,
275 "size": len(_p) if len(_p) > len(_c) else len(_c)
276 }
277 })
278 for p_if in tree[lvl][interface]["parents"]:
279 # -- cycle: execute process for next parent, lvl-1
280 process_interface(lvl-1, p_if, tree, res)
281 for c_if in tree[lvl][interface]["children"]:
282 # -- cycle: execute process for next child, lvl+1
283 process_interface(lvl+1, c_if, tree, res)
284 else:
285 # - IF present - exit (been here already)
286 return
288 def _put(cNet, cIndex, _list):
289 _added = False
290 _actual_index = -1
291 # Check list len
292 _len = len(_list)
293 if cIndex >= _len:
294 # grow list to meet index
295 _list = _list + [''] * (cIndex - _len + 1)
296 _len = len(_list)
298 for _cI in range(cIndex, _len):
299 # add child per index
300 # if space is free
301 if not _list[_cI]:
302 _list[_cI] = cNet
303 _added = True
304 _actual_index = _cI
305 break
306 if not _added:
307 # grow list by one entry
308 _list = _list + [cNet]
309 _actual_index = len(_list) - 1
310 return _actual_index, _list
312 # build network hierachy
313 nr = node_data['networks']
314 # walk interface tree
315 for _ifname in node_data['networks']:
316 _tree = {}
317 _level = 0
318 process_interface(_level, _ifname, _tree, nr)
319 # save tree for node/if
320 node_data['networks'][_ifname]['tree'] = _tree
322 # debug, print built tree
323 # logger_cli.debug("# '{}'".format(_ifname))
324 lvls = list(_tree.keys())
325 lvls.sort()
326 n = len(lvls)
327 m = max([len(_tree[k].keys()) for k in _tree.keys()])
328 matrix = [["" for i in range(m)] for j in range(n)]
329 x = 0
330 while True:
331 _lv = lvls.pop(0)
332 # get all interfaces on this level
333 nets = iter(_tree[_lv].keys())
334 while True:
335 y = 0
336 # get next interface
337 try:
338 _net = next(nets)
339 except StopIteration:
340 break
341 # all nets
342 _a = [_net]
343 # put current interface if this is only one left
344 if not _tree[_lv][_net]['children']:
345 if _net not in matrix[x]:
346 _, matrix[x] = _put(
347 _net,
348 y,
349 matrix[x]
350 )
351 y += 1
352 else:
353 # get all nets with same child
354 for _c in _tree[_lv][_net]['children']:
355 for _o_net in nets:
356 if _c in _tree[_lv][_o_net]['children']:
357 _a.append(_o_net)
358 # flush collected nets
359 for idx in range(len(_a)):
360 if _a[idx] in matrix[x]:
361 # there is such interface on this level
362 # get index
363 _nI = matrix[x].index(_a[idx])
364 _, matrix[x+1] = _put(
365 _c,
366 _nI,
367 matrix[x+1]
368 )
369 else:
370 # there is no such interface
371 # add it
372 _t, matrix[x] = _put(
373 _a[idx],
374 0,
375 matrix[x]
376 )
377 # also, put child
378 _, matrix[x+1] = _put(
379 _c,
380 _t,
381 matrix[x+1]
382 )
383 # remove collected nets from processing
384 if _a[idx] in nets:
385 nets.remove(_a[idx])
386 y += len(_a)
387 if not nets:
388 x += 1
389 break
390 if not lvls:
391 break
393 lines = []
394 _columns = [len(max([i for i in li])) for li in matrix]
395 for idx_y in range(m):
396 line = ""
397 for idx_x in range(n):
398 _fmt = "{" + ":{}".format(_columns[idx_x]) + "} "
399 line += _fmt.format(matrix[idx_x][idx_y])
400 lines.append(line)
401 node_data['networks'][_ifname]['matrix'] = matrix
402 node_data['networks'][_ifname]['lines'] = lines
403 return _runtime
405 def map_network(self, source):
406 # maps target network using given source
407 _networks = None
409 if source == self.RECLASS:
410 _networks = self._map_reclass_networks()
411 elif source == self.CONFIG:
412 _networks = self._map_configured_networks()
413 elif source == self.RUNTIME:
414 _networks = self._map_runtime_networks()
416 self.networks[source] = _networks
417 return _networks
419 def create_map(self):
420 """Create all needed elements for map output
422 :return: none
423 """
424 _runtime = self.networks[self.RUNTIME]
425 _reclass = self.networks[self.RECLASS]
427 # main networks, target vars
428 _map = {}
429 # No matter of proto, at least one IP will be present for the network
430 # we interested in, since we are to make sure that L3 level
431 # is configured according to reclass model
432 for network in _reclass:
433 # shortcuts
434 _net = str(network)
435 _map[_net] = {}
436 if network not in _runtime:
437 # reclass has network that not found in runtime
438 self.errors.add_error(
439 self.errors.NET_NO_RUNTIME_NETWORK,
440 reclass_net=str(network)
441 )
442 logger_cli.warn(
443 "WARN: {}: {}".format(
444 " No runtime network ", str(network)
445 )
446 )
447 continue
448 # hostnames
449 names = sorted(_runtime[network].keys())
450 for hostname in names:
451 _notes = []
452 node = hostname.split('.')[0]
453 if not salt_master.is_node_available(hostname, log=False):
454 logger_cli.info(
455 " {0:8} {1}".format(node, "node not available")
456 )
457 # add non-responsive node erorr
458 self.errors.add_error(
459 self.errors.NET_NODE_NON_RESPONSIVE,
460 host=hostname
461 )
462 _notes.append(
463 self.errors.get_error_type_text(
464 self.errors.NET_NODE_NON_RESPONSIVE
465 )
466 )
467 continue
468 # lookup interface name on node using network CIDR
469 _if_name = _runtime[network][hostname][0]["name"]
470 _raw = self.interfaces[hostname][_if_name]['runtime']
471 # get proper reclass
472 _r = self.interfaces[hostname][_if_name]['reclass']
473 _if_name_suffix = ""
474 # get the proto value
475 if _r:
476 _if_rc = ""
477 else:
478 self.errors.add_error(
479 self.errors.NET_NODE_UNEXPECTED_IF,
480 host=hostname,
481 if_name=_if_name
482 )
483 _notes.append(
484 self.errors.get_error_type_text(
485 self.errors.NET_NODE_UNEXPECTED_IF
486 )
487 )
488 _if_rc = "*"
490 if "proto" in _r:
491 _proto = _r['proto']
492 else:
493 _proto = "-"
495 if "type" in _r:
496 _if_name_suffix += _r["type"]
497 if "use_interfaces" in _r:
498 _if_name_suffix += "->" + ",".join(_r["use_interfaces"])
500 if _if_name_suffix:
501 _if_name_suffix = "({})".format(_if_name_suffix)
503 # get gate and routes if proto is static
504 if _proto == 'static':
505 # get the gateway for current net
506 _routes = salt_master.nodes[hostname]['routes']
507 _route = _routes[_net] if _net in _routes else None
508 # get the default gateway
509 if 'default' in _routes:
510 _d_gate = ipaddress.IPv4Address(
511 _routes['default']['gateway']
512 )
513 else:
514 _d_gate = None
515 _d_gate_str = str(_d_gate) if _d_gate else "No default!"
516 # match route with default
517 if not _route:
518 _gate = "?"
519 else:
520 _gate = _route['gateway'] if _route['gateway'] else "-"
521 else:
522 # in case of manual and dhcp, no check possible
523 _gate = "-"
524 _d_gate = "-"
525 _d_gate_str = "-"
526 # iterate through interfaces
527 _a = _runtime[network][hostname]
528 for _host in _a:
529 for _if in _host['ifs']:
530 _ip_str = str(_if.exploded)
531 _gate_error = ""
532 _up_error = ""
533 _mtu_error = ""
535 # Match gateway
536 if _proto == 'static':
537 # default reclass gate
538 _r_gate = "-"
539 if "gateway" in _r:
540 _r_gate = _r["gateway"]
542 # if values not match, put *
543 if _gate != _r_gate and _d_gate_str != _r_gate:
544 # if values not match, check if default match
545 self.errors.add_error(
546 self.errors.NET_UNEXPECTED_GATEWAY,
547 host=hostname,
548 if_name=_if_name,
549 ip=_ip_str,
550 gateway=_gate
551 )
552 _notes.append(
553 self.errors.get_error_type_text(
554 self.errors.NET_UNEXPECTED_GATEWAY
555 )
556 )
557 _gate_error = "*"
559 # IF status in reclass
560 _e = "enabled"
561 if _e not in _r:
562 self.errors.add_error(
563 self.errors.NET_NO_RC_IF_STATUS,
564 host=hostname,
565 if_name=_if_name
566 )
567 _notes.append(
568 self.errors.get_error_type_text(
569 self.errors.NET_NO_RC_IF_STATUS
570 )
571 )
572 _up_error = "*"
574 _rc_mtu = _r['mtu'] if 'mtu' in _r else None
575 _rc_mtu_s = ""
576 # check if this is a VIP address
577 # no checks needed if yes.
578 if _host['vip'] != _ip_str:
579 if _rc_mtu:
580 _rc_mtu_s = str(_rc_mtu)
581 # if there is an MTU value, match it
582 if _host['mtu'] != _rc_mtu_s:
583 self.errors.add_error(
584 self.errors.NET_MTU_MISMATCH,
585 host=hostname,
586 if_name=_if_name,
587 if_cidr=_ip_str,
588 reclass_mtu=_rc_mtu,
589 runtime_mtu=_host['mtu']
590 )
591 _notes.append(
592 self.errors.get_error_type_text(
593 self.errors.NET_MTU_MISMATCH
594 )
595 )
596 _rc_mtu_s = "/" + _rc_mtu_s
597 _mtu_error = "*"
598 else:
599 # empty the matched value
600 _rc_mtu_s = ""
601 elif _host['mtu'] != '1500' and \
602 _proto not in ["-", "dhcp"]:
603 # there is no MTU value in reclass
604 # and runtime value is not default
605 self.errors.add_error(
606 self.errors.NET_MTU_EMPTY,
607 host=hostname,
608 if_name=_if_name,
609 if_cidr=_ip_str,
610 if_mtu=_host['mtu']
611 )
612 _notes.append(
613 self.errors.get_error_type_text(
614 self.errors.NET_MTU_EMPTY
615 )
616 )
617 _mtu_error = "*"
618 else:
619 # this is a VIP
620 _if_name = " "*7
621 _if_name_suffix = ""
622 _ip_str += " VIP"
623 # Save all data
624 _values = {
625 "interface": _if_name,
626 "interface_error": _if_rc,
627 "interface_note": _if_name_suffix,
628 "interface_map": "\n".join(_host['lines']),
629 "interface_matrix": _host['matrix'],
630 "ip_address": _ip_str,
631 "address_type": _proto,
632 "rt_mtu": _host['mtu'],
633 "rc_mtu": _rc_mtu_s,
634 "mtu_error": _mtu_error,
635 "status": _host['state'],
636 "status_error": _up_error,
637 "subnet_gateway": _gate,
638 "subnet_gateway_error": _gate_error,
639 "default_gateway": _d_gate_str,
640 "raw_data": _raw,
641 "error_note": " and ".join(_notes)
642 }
643 if node in _map[_net]:
644 # add if to host
645 _map[_net][node].append(_values)
646 else:
647 _map[_net][node] = [_values]
648 _notes = []
650 # save map
651 self.map = _map
652 # other runtime networks found
653 # docker, etc
655 return
657 def print_map(self):
658 """
659 Create text report for CLI
661 :return: none
662 """
663 logger_cli.info("# Networks")
664 logger_cli.info(
665 " {0:8} {1:25} {2:25} {3:6} {4:10} {5:10} {6}/{7}".format(
666 "Host",
667 "IF",
668 "IP",
669 "Proto",
670 "MTU",
671 "State",
672 "Gate",
673 "Def.Gate"
674 )
675 )
676 for network in self.map.keys():
677 logger_cli.info("-> {}".format(network))
678 for hostname in self.map[network].keys():
679 node = hostname.split('.')[0]
680 _n = self.map[network][hostname]
681 for _i in _n:
682 # Host IF IP Proto MTU State Gate Def.Gate
683 _text = "{:7} {:17} {:25} {:6} {:10} " \
684 "{:10} {} / {}".format(
685 _i['interface'] + _i['interface_error'],
686 _i['interface_note'],
687 _i['ip_address'],
688 _i['address_type'],
689 _i['rt_mtu'] + _i['rc_mtu'] + _i['mtu_error'],
690 _i['status'] + _i['status_error'],
691 _i['subnet_gateway'] +
692 _i['subnet_gateway_error'],
693 _i['default_gateway']
694 )
695 logger_cli.info(
696 " {0:8} {1}".format(
697 node,
698 _text
699 )
700 )
702 # logger_cli.info("\n# Other networks")
703 # _other = [n for n in _runtime if n not in _reclass]
704 # for network in _other:
705 # logger_cli.info("-> {}".format(str(network)))
706 # names = sorted(_runtime[network].keys())
708 # for hostname in names:
709 # for _n in _runtime[network][hostname]:
710 # _ifs = [str(ifs.ip) for ifs in _n['ifs']]
711 # _text = "{:25} {:25} {:6} {:10} {}".format(
712 # _n['name'],
713 # ", ".join(_ifs),
714 # "-",
715 # _n['mtu'],
716 # _n['state']
717 # )
718 # logger_cli.info(
719 # " {0:8} {1}".format(hostname.split('.')[0], _text)
720 # )
721 # logger_cli.info("\n")