test_set/cvp-sanity/tests/test_packet_checker.py

import pytest
import json
import utils
import logging

def is_deb_in_exception(inconsistency_rule, package_name, error_node_list):
    short_names_in_error_nodes = [n.split('.')[0] for n in error_node_list]
    for node, excluded_packages in inconsistency_rule.iteritems():
        if package_name in excluded_packages and node in short_names_in_error_nodes:
            return True
    return False

@pytest.mark.full
def test_check_package_versions(local_salt_client, nodes_in_group):
    """Validate package has same versions on all the nodes.
    Steps:
     1) Collect packages for nodes_in_group
        "salt -C '<group_of_nodes>' cmd.run 'lowpkg.list_pkgs'"
     2) Exclude nodes without packages and exceptions
     3) Go through each package and save it with version from each node to the
        list. Mark 'No version' if package is not found.
        If pachage name in the eception list or in inconsistency_rule, ignore it.
     4) Compare items in that list - they should be equal and match the amout of nodes

    """
    # defines packages specific to the concrete nodes
    inconsistency_rule = {"kvm03": ["rsync", "sysstat", "xz-utils"], "log01": ["python-elasticsearch"], "ctl01": ["python-gnocchiclient", "python-ujson"]}
    exclude_packages = utils.get_configuration().get("skipped_packages", [])
    group, nodes = nodes_in_group
    packages_versions = local_salt_client.cmd(tgt="L@"+','.join(nodes),
                                              fun='lowpkg.list_pkgs',
                                              expr_form='compound')
    # Let's exclude cid01 and dbs01 nodes from this check
    exclude_nodes = local_salt_client.test_ping(tgt="I@galera:master or I@gerrit:client",
                                                expr_form='compound').keys()
    # PROD-30833
    gtw01 = local_salt_client.pillar_get(
        param='_param:openstack_gateway_node01_hostname') or 'gtw01'
    cluster_domain = local_salt_client.pillar_get(
        param='_param:cluster_domain') or '.local'
    gtw01 += '.' + cluster_domain
    if gtw01 in nodes:
        octavia = local_salt_client.cmd(tgt="L@" + ','.join(nodes),
                                        fun='pillar.get',
                                        param='octavia:manager:enabled',
                                        expr_form='compound')
        gtws = [gtw for gtw in octavia.values() if gtw]
        if len(gtws) == 1:
            exclude_nodes.append(gtw01)
            logging.info("gtw01 node is skipped in test_check_package_versions")

    total_nodes = [i for i in nodes if i not in exclude_nodes]
    if len(total_nodes) < 2:
        pytest.skip("Nothing to compare - only 1 node")
    nodes_with_packages = []
    packages_with_different_versions = []
    packages_names = set()

    for node in total_nodes:
        if not packages_versions[node]:
            # TODO: do not skip node
            logging.warning("Node {} is skipped".format(node))
            continue
        nodes_with_packages.append(node)
        packages_names.update(packages_versions[node].keys())
    for deb in packages_names:
        if deb in exclude_packages:
            continue
        diff = []
        row = []
        for node in nodes_with_packages:
            if not packages_versions[node]:
                continue
            if deb in packages_versions[node].keys():
                diff.append(packages_versions[node][deb])
                row.append("{}: {}".format(node, packages_versions[node][deb]))
            else:
                row.append("{}: No package".format(node))

        if diff.count(diff[0]) < len(nodes_with_packages):
            if not is_deb_in_exception(inconsistency_rule, deb, row):
                row.sort()
                row.insert(0, deb)
                packages_with_different_versions.append(row)
    assert len(packages_with_different_versions) == 0, (
        "Non-uniform package versions are installed on '{}' group of nodes:\n"
        "{}".format(
            group, json.dumps(packages_with_different_versions, indent=4))
    )


@pytest.mark.full
def test_packages_are_latest(local_salt_client, nodes_in_group):
    config = utils.get_configuration()
    skip = config.get("test_packages")["skip_test"]
    if skip:
        pytest.skip("Test for the latest packages is disabled")
    skipped_pkg = config.get("test_packages")["skipped_packages"]
    group, nodes = nodes_in_group
    info_salt = local_salt_client.cmd(
        tgt='L@' + ','.join(nodes),
        param='apt list --upgradable 2>/dev/null | grep -v Listing',
        expr_form='compound')
    for node in nodes:
        result = []
        if info_salt[node]:
            upg_list = info_salt[node].split('\n')
            for i in upg_list:
                if i.split('/')[0] not in skipped_pkg:
                    result.append(i)
        assert not result, (
            "Packages are not of latest version on '{}' node:\n{}".format(
                node, "\n".join(result))
        )


@pytest.mark.full
def test_check_module_versions(local_salt_client, nodes_in_group):
    # defines modules specific to the concrete nodes
    inconsistency_rule = {"ctl01": ["gnocchiclient", "ujson"], "log01": ["elasticsearch"]}
    exclude_modules = utils.get_configuration().get("skipped_modules", [])
    group, nodes = nodes_in_group
    pre_check = local_salt_client.cmd(
        tgt="L@"+','.join(nodes),
        param='dpkg -l | grep "python-pip "',
        expr_form='compound')
    if pre_check.values().count('') > 0:
        pytest.skip("pip is not installed on one or more nodes")

    exclude_nodes = local_salt_client.test_ping(tgt="I@galera:master or I@gerrit:client",
                                                expr_form='compound').keys()

    # PROD-30833
    gtw01 = local_salt_client.pillar_get(
        param='_param:openstack_gateway_node01_hostname') or 'gtw01'
    cluster_domain = local_salt_client.pillar_get(
        param='_param:cluster_domain') or '.local'
    gtw01 += '.' + cluster_domain
    if gtw01 in nodes:
        octavia = local_salt_client.cmd(tgt="L@" + ','.join(nodes),
                                        fun='pillar.get',
                                        param='octavia:manager:enabled',
                                        expr_form='compound')
        gtws = [gtw for gtw in octavia.values() if gtw]
        if len(gtws) == 1:
            exclude_nodes.append(gtw01)
            logging.info("gtw01 node is skipped in test_check_module_versions")

    total_nodes = [i for i in pre_check.keys() if i not in exclude_nodes]

    if len(total_nodes) < 2:
        pytest.skip("Nothing to compare - only 1 node")
    list_of_pip_packages = local_salt_client.cmd(
        tgt="L@"+','.join(nodes),
        fun='pip.freeze', expr_form='compound')

    nodes_with_packages = []

    modules_with_different_versions = []
    packages_names = set()

    for node in total_nodes:
        nodes_with_packages.append(node)
        packages_names.update([x.split("=")[0] for x in list_of_pip_packages[node]])
        list_of_pip_packages[node] = dict([x.split("==") for x in list_of_pip_packages[node]])

    for deb in packages_names:
        if deb in exclude_modules:
            continue
        diff = []
        row = []
        for node in nodes_with_packages:
            if deb in list_of_pip_packages[node].keys():
                diff.append(list_of_pip_packages[node][deb])
                row.append("{}: {}".format(node, list_of_pip_packages[node][deb]))
            else:
                row.append("{}: No module".format(node))
        if diff.count(diff[0]) < len(nodes_with_packages):
            if not is_deb_in_exception(inconsistency_rule, deb, row):
                row.sort()
                row.insert(0, deb)
                modules_with_different_versions.append(row)
    assert len(modules_with_different_versions) == 0, (
        "Non-uniform pip modules are installed on '{}' group of nodes:\n"
        "{}".format(
            group, json.dumps(modules_with_different_versions, indent=4))
    )