cfg_checker/modules/network/checker.py

import ipaddress
import json


from cfg_checker.common import logger_cli
from cfg_checker.modules.network.network_errors import NetworkErrors
from cfg_checker.nodes import SaltNodes
from cfg_checker.reports import reporter


class NetworkChecker(SaltNodes):
    def __init__(self):
        super(NetworkChecker, self).__init__()
        self.errors = NetworkErrors()

    # adding net data to tree
    def _add_data(self, _list, _n, _h, _d):
        if _n not in _list:
            _list[_n] = {}
            _list[_n][_h] = [_d]
        elif _h not in _list[_n]:
            # there is no such host, just create it
            _list[_n][_h] = [_d]
        else:
            # there is such host... this is an error
            self.errors.add_error(
                self.errors.NET_DUPLICATE_IF,
                host=_h,
                dup_if=_d['name']
            )
            _list[_n][_h].append(_d)

    # TODO: refactor map creation. Build one map instead of two separate
    def _map_network_for_host(self, host, if_class, net_list, data):
        # filter networks for this IF IP
        _nets = [n for n in net_list.keys() if if_class.ip in n]
        _masks = [n.netmask for n in _nets]
        if len(_nets) > 1:
            # There a multiple network found for this IP, Error
            self.errors.add_error(
                self.errors.NET_SUBNET_INTERSECT,
                host=host,
                networks="; ".join(_nets)
            )
        # check mask match
        if len(_nets) > 0 and if_class.netmask not in _masks:
            self.errors.add_error(
                self.errors.NET_MASK_MISMATCH,
                host=host,
                if_name=data['name'],
                if_cidr=if_class.exploded,
                if_mapped_networks=", ".join([str(_n) for _n in _nets])
            )

        if len(_nets) < 1:
            self._add_data(net_list, if_class.network, host, data)
        else:
            # add all data
            for net in _nets:
                self._add_data(net_list, net, host, data)

        return net_list

    def collect_network_info(self):
        """
        Collects info on the network using ifs_data.py script

        :return: none
        """
        logger_cli.info("# Mapping node runtime network data")
        _result = self.execute_script_on_active_nodes(
            "ifs_data.py",
            args=["json"]
        )
        self.stage = "Runtime"
        for key in self.nodes.keys():
            # check if we are to work with this node
            if not self.is_node_available(key):
                continue
            # due to much data to be passed from salt, it is happening in order
            if key in _result:
                _text = _result[key]
                _dict = json.loads(_text[_text.find('{'):])
                self.nodes[key]['routes'] = _dict.pop("routes")
                self.nodes[key]['networks'] = _dict
            else:
                self.nodes[key]['networks'] = {}
                self.nodes[key]['routes'] = {}
            logger_cli.debug("... {} has {} networks".format(
                key,
                len(self.nodes[key]['networks'].keys())
            ))
        logger_cli.info("-> done collecting networks data")

        # TODO: Mimic reclass structure for easy compare
        logger_cli.info("### Building network tree")
        # match interfaces by IP subnets
        _all_nets = {}
        for host, node_data in self.nodes.iteritems():
            if not self.is_node_available(host):
                continue

            for net_name, net_data in node_data['networks'].iteritems():
                # get ips and calculate subnets
                if net_name in ['lo']:
                    # skip the localhost
                    continue
                #  get data and make sure that wide mask goes first
                _ip4s = sorted(
                    net_data['ipv4'],
                    key=lambda s: s[s.index('/'):]
                )
                for _ip_str in _ip4s:
                    # create interface class
                    _if = ipaddress.IPv4Interface(_ip_str)
                    if 'name' not in net_data:
                        net_data['name'] = net_name
                    if 'ifs' not in net_data:
                        net_data['ifs'] = [_if]
                        # map it
                        _all_nets = self._map_network_for_host(
                            host,
                            _if,
                            _all_nets,
                            net_data
                        )
                    else:
                        # data is already there, just add VIP
                        net_data['ifs'].append(_if)

        # save collected info
        self.all_nets = _all_nets

    def collect_reclass_networks(self):
        logger_cli.info("# Mapping reclass networks")
        self.stage = "Reclass"
        # Get networks from reclass and mark them
        _reclass_nets = {}
        # Get required pillars
        self.get_specific_pillar_for_nodes("linux:network")
        for node in self.nodes.keys():
            # check if this node
            if not self.is_node_available(node):
                continue
            # get the reclass value
            _pillar = self.nodes[node]['pillars']['linux']['network']
            # we should be ready if there is no interface in reclass for a node
            # for example on APT node
            if 'interface' in _pillar:
                _pillar = _pillar['interface']
            else:
                logger_cli.info(
                    "... node '{}' skipped, no IF section in reclass".format(
                        node
                    )
                )
                continue
            for _if_name, _if_data in _pillar.iteritems():
                if 'address' in _if_data:
                    _if = ipaddress.IPv4Interface(
                        _if_data['address'] + '/' + _if_data['netmask']
                    )
                    _if_data['name'] = _if_name
                    _if_data['ifs'] = [_if]

                    _reclass_nets = self._map_network_for_host(
                        node,
                        _if,
                        _reclass_nets,
                        _if_data
                    )

        self.reclass_nets = _reclass_nets

    def print_network_report(self):
        """
        Create text report for CLI

        :return: none
        """
        _all_nets = self.all_nets.keys()
        logger_cli.info("# Reclass networks")
        logger_cli.info(
            "    {0:17} {1:25}: "
            "{2:19} {3:5}{4:10} {5}{6} {7} / {8} / {9}".format(
                "Hostname",
                "IF",
                "IP",
                "rtMTU",
                "rcMTU",
                "rtState",
                "rcState",
                "rtGate",
                "rtDef.Gate",
                "rcGate"
            )
        )
        # TODO: Move matching to separate function
        self.stage = "Matching"
        _reclass = [n for n in _all_nets if n in self.reclass_nets]
        for network in _reclass:
            # shortcuts
            _net = str(network)
            logger_cli.info("-> {}".format(_net))
            names = sorted(self.all_nets[network].keys())
            for hostname in names:
                if not self.is_node_available(hostname, log=False):
                    logger_cli.info(
                        "    {0:17} {1}".format(
                            hostname.split('.')[0],
                            "... no data for the node"
                        )
                    )
                    # add non-responsive node erorr
                    self.errors.add_error(
                        self.errors.NET_NODE_NON_RESPONSIVE,
                        host=hostname
                    )

                    # print empty row
                    _text = " # node non-responsive"
                    logger_cli.info(
                        "    {0:17} {1}".format(
                            hostname.split('.')[0],
                            _text
                        )
                    )
                    continue

                # get the gateway for current net
                _routes = self.nodes[hostname]['routes']
                _route = _routes[_net] if _net in _routes else None
                if not _route:
                    _gate = "no route!"
                else:
                    _gate = _route['gateway'] if _route['gateway'] else "-"

                # get the default gateway
                if 'default' in _routes:
                    _d_gate = ipaddress.IPv4Address(
                        _routes['default']['gateway']
                    )
                else:
                    _d_gate = None
                _d_gate_str = _d_gate if _d_gate else "No default gateway!"

                _a = self.all_nets[network][hostname]
                for _host in _a:
                    for _if in _host['ifs']:
                        # get proper reclass
                        _ip_str = str(_if.exploded)
                        _r = {}
                        if hostname in self.reclass_nets[network]:
                            for _item in self.reclass_nets[network][hostname]:
                                for _item_ifs in _item['ifs']:
                                    if _ip_str == str(_item_ifs.exploded):
                                        _r = _item
                        else:
                            self.errors.add_error(
                                self.errors.NET_NODE_UNEXPECTED_IF,
                                host=hostname,
                                if_name=_host['name'],
                                if_ip=_ip_str
                            )

                        # check if node is UP
                        if not self.is_node_available(hostname):
                            _r_gate = "-"
                        # get proper network from reclass
                        else:
                            # Lookup match for the ip
                            _r_gate = "no IF in reclass!"
                            # get all networks with this hostname
                            _rn = self.reclass_nets
                            _nets = filter(
                                lambda n: hostname in _rn[n].keys(),
                                self.reclass_nets
                            )
                            _rd = None
                            for _item in _nets:
                                # match ip
                                _r_dat = self.reclass_nets[_item][hostname]
                                for _r_ifs in _r_dat:
                                    for _r_if in _r_ifs['ifs']:
                                        if _if.ip == _r_if.ip:
                                            _rd = _r_ifs
                                            break
                                if _rd:
                                    _gs = 'gateway'
                                    _e = "empty"
                                    _r_gate = _rd[_gs] if _gs in _rd else _e
                                    break

                        # IF status in reclass
                        if 'enabled' not in _r:
                            _enabled = "(no record!)"
                        else:
                            _e = "enabled"
                            _d = "disabled"
                            _enabled = "("+_e+")" if _r[_e] else "("+_d+")"

                        _name = _host['name']
                        _rc_mtu = _r['mtu'] if 'mtu' in _r else None

                        # Check if this is a VIP
                        if _if.network.prefixlen == 32:
                            _name = " "*20
                            _ip_str += " VIP"
                            _rc_mtu = "(-)"
                            _enabled = "(-)"
                            _r_gate = "-"

                        # Check if this is a default MTU
                        elif _host['mtu'] == '1500':
                            # reclass is empty if MTU is untended to be 1500
                            _rc_mtu = "(-)"
                        elif _rc_mtu:
                            _rc_mtu_s = str(_rc_mtu)
                            # if there is an MTU value, match it
                            if _host['mtu'] != _rc_mtu_s:
                                self.errors.add_error(
                                    self.errors.NET_MTU_MISMATCH,
                                    host=hostname,
                                    if_name=_name,
                                    if_cidr=_ip_str,
                                    reclass_mtu=_rc_mtu,
                                    runtime_mtu=_host['mtu']
                                )
                        else:
                            # there is no MTU value in reclass
                            self.errors.add_error(
                                self.errors.NET_MTU_EMPTY,
                                host=hostname,
                                if_name=_name,
                                if_cidr=_ip_str,
                                if_mtu=_host['mtu']
                            )

                        _text = "{0:25} {1:19} {2:5}{3:10} {4:4}{5:10} " \
                                "{6} / {7} / {8}".format(
                                    _name,
                                    _ip_str,
                                    _host['mtu'],
                                    "("+_rc_mtu_s+")" if _rc_mtu else "(No!)",
                                    _host['state'],
                                    _enabled,
                                    _gate,
                                    _d_gate_str,
                                    _r_gate
                                )
                        logger_cli.info(
                            "    {0:17} {1}".format(
                                hostname.split('.')[0],
                                _text
                            )
                        )

        logger_cli.info("\n# Other networks")
        _other = [n for n in _all_nets if n not in self.reclass_nets]
        for network in _other:
            logger_cli.info("-> {}".format(str(network)))
            names = sorted(self.all_nets[network].keys())

            for hostname in names:
                for _n in self.all_nets[network][hostname]:
                    _ifs = [str(ifs.ip) for ifs in _n['ifs']]
                    _text = "{0:25}: {1:19} {2:5} {3:4}".format(
                        _n['name'],
                        ", ".join(_ifs),
                        _n['mtu'],
                        _n['state']
                    )
                    logger_cli.info(
                        "    {0:17} {1}".format(hostname.split('.')[0], _text)
                    )

    def print_summary(self):
        _total_errors = self.errors.get_errors_total()
        # Summary
        logger_cli.info(
            "\n{:=^8s}\n{:^8s}\n{:=^8s}".format(
                "=",
                "Totals",
                "="
            )
        )
        logger_cli.info(self.errors.get_summary(print_zeros=False))
        logger_cli.info('-'*20)
        logger_cli.info("{:5d} total errors found\n".format(_total_errors))

    def print_error_details(self):
        # Detailed errors
        if self.errors.get_errors_total() > 0:
            logger_cli.info("\n# Errors")
            for _msg in self.errors.get_errors_as_list():
                logger_cli.info("{}\n".format(_msg))
        else:
            logger_cli.info("-> No errors\n")

    def create_html_report(self, filename):
        """
        Create static html showing network schema-like report

        :return: none
        """
        logger_cli.info("### Generating report to '{}'".format(filename))
        _report = reporter.ReportToFile(
            reporter.HTMLNetworkReport(),
            filename
        )
        _report({
            "nodes": self.nodes,
            "network": {},
            "mcp_release": self.mcp_release,
            "openstack_release": self.openstack_release

        })
        logger_cli.info("-> Done")