tcp_tests/helpers/netchecker.py

#    Copyright 2017 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json
import requests

from devops.helpers import helpers
from k8sclient.client import rest

from tcp_tests import logger
from tcp_tests.helpers import utils


LOG = logger.logger


NETCHECKER_SERVICE_NAME = "netchecker-service"
NETCHECKER_CONTAINER_PORT = NETCHECKER_SERVICE_PORT = 8081
NETCHECKER_NODE_PORT = 31081
NETCHECKER_REPORT_INTERVAL = 30
NETCHECKER_SERVER_REPLICAS = 1
NETCHECKER_PROBEURLS = "http://ipinfo.io"

NETCHECKER_SVC_CFG = {
    "apiVersion": "v1",
    "kind": "Service",
    "metadata": {
        "name": NETCHECKER_SERVICE_NAME
    },
    "spec": {
        "ports": [
            {
                "nodePort": NETCHECKER_NODE_PORT,
                "port": NETCHECKER_SERVICE_PORT,
                "protocol": "TCP",
                "targetPort": NETCHECKER_CONTAINER_PORT
            }
        ],
        "selector": {
            "app": "netchecker-server"
        },
        "type": "NodePort"
    }
}

NETCHECKER_DEPLOYMENT_CFG = {
    "kind": "Deployment",
    "spec": {
        "template": {
            "spec": {
                "containers": [
                    {
                        "name": "netchecker-server",
                        "env": None,
                        "imagePullPolicy": "IfNotPresent",
                        "image": "mirantis/k8s-netchecker-server:latest",
                        "args": [
                            "-v=5",
                            "-logtostderr",
                            "-kubeproxyinit",
                            "-endpoint=0.0.0.0:{0}".format(
                                NETCHECKER_CONTAINER_PORT)
                        ],
                        "ports": [
                            {
                                "containerPort": NETCHECKER_CONTAINER_PORT,
                                "hostPort": NETCHECKER_NODE_PORT
                            }
                        ]
                    }
                ]
            },
            "metadata": {
                "labels": {
                    "app": "netchecker-server"
                },
                "name": "netchecker-server"
            }
        },
        "replicas": NETCHECKER_SERVER_REPLICAS
    },
    "apiVersion": "extensions/v1beta1",
    "metadata": {
        "name": "netchecker-server"
    }
}

NETCHECKER_DS_CFG = [
    {
        "apiVersion": "extensions/v1beta1",
        "kind": "DaemonSet",
        "metadata": {
            "labels": {
                "app": "netchecker-agent"
            },
            "name": "netchecker-agent"
        },
        "spec": {
            "template": {
                "metadata": {
                    "labels": {
                        "app": "netchecker-agent"
                    },
                    "name": "netchecker-agent"
                },
                "spec": {
                    "tolerations": [
                        {
                            "key": "node-role.kubernetes.io/master",
                            "effect": "NoSchedule"
                        }
                    ],
                    "containers": [
                        {
                            "env": [
                                {
                                    "name": "MY_POD_NAME",
                                    "valueFrom": {
                                        "fieldRef": {
                                            "fieldPath": "metadata.name"
                                        }
                                    }
                                },
                                {
                                    "name": "MY_NODE_NAME",
                                    "valueFrom": {
                                        "fieldRef": {
                                            "fieldPath": "spec.nodeName"
                                        }
                                    }
                                },
                                {
                                    "name": "REPORT_INTERVAL",
                                    "value": str(NETCHECKER_REPORT_INTERVAL)
                                },
                            ],
                            "image": "mirantis/k8s-netchecker-agent:latest",
                            "imagePullPolicy": "IfNotPresent",
                            "name": "netchecker-agent",
                            "command": ["netchecker-agent"],
                            "args": [
                                "-v=5",
                                "-logtostderr",
                                "-probeurls={0}".format(NETCHECKER_PROBEURLS)
                            ]
                        }
                    ],
                }
            },
            "updateStrategy": {
                "type": "RollingUpdate"
            }
        }
    },
    {
        "apiVersion": "extensions/v1beta1",
        "kind": "DaemonSet",
        "metadata": {
            "labels": {
                "app": "netchecker-agent-hostnet"
            },
            "name": "netchecker-agent-hostnet"
        },
        "spec": {
            "template": {
                "metadata": {
                    "labels": {
                        "app": "netchecker-agent-hostnet"
                    },
                    "name": "netchecker-agent-hostnet"
                },
                "spec": {
                    "tolerations": [
                        {
                            "key": "node-role.kubernetes.io/master",
                            "effect": "NoSchedule"
                        }
                    ],
                    "containers": [
                        {
                            "env": [
                                {
                                    "name": "MY_POD_NAME",
                                    "valueFrom": {
                                        "fieldRef": {
                                            "fieldPath": "metadata.name"
                                        }
                                    }
                                },
                                {
                                    "name": "MY_NODE_NAME",
                                    "valueFrom": {
                                        "fieldRef": {
                                            "fieldPath": "spec.nodeName"
                                        }
                                    }
                                },
                                {
                                    "name": "REPORT_INTERVAL",
                                    "value": str(NETCHECKER_REPORT_INTERVAL)
                                },
                            ],
                            "image": "mirantis/k8s-netchecker-agent:latest",
                            "imagePullPolicy": "IfNotPresent",
                            "name": "netchecker-agent",
                            "command": ["netchecker-agent"],
                            "args": [
                                "-v=5",
                                "-logtostderr",
                                "-probeurls={0}".format(NETCHECKER_PROBEURLS)
                            ]
                        }
                    ],
                    "hostNetwork": True,
                    "dnsPolicy": "ClusterFirstWithHostNet",
                    "updateStrategy": {
                        "type": "RollingUpdate"
                    }
                }
            },
            "updateStrategy": {
                "type": "RollingUpdate"
            }
        }
    }
]

NETCHECKER_BLOCK_POLICY = {
    "kind": "policy",
    "spec": {
        "ingress": [
            {
                "action": "allow"
            },
            {
                "action": "deny",
                "destination": {
                    "ports": [
                        NETCHECKER_SERVICE_PORT
                    ]
                },
                "protocol": "tcp"
            }
        ]
    },
    "apiVersion": "v1",
    "metadata": {
        "name": "deny-netchecker"
    }
}


def start_server(k8s, config, namespace=None,
                 deploy_spec=NETCHECKER_DEPLOYMENT_CFG,
                 svc_spec=NETCHECKER_SVC_CFG):
    """Start netchecker server in k8s cluster

    :param k8s: K8SManager
    :param config: fixture provides oslo.config
    :param namespace: str
    :param deploy_spec: dict
    :param svc_spec: dict
    :return: None
    """
    for container in deploy_spec['spec']['template']['spec']['containers']:
        if container['name'] == 'netchecker-server':
            container['image'] = \
                config.k8s_deploy.kubernetes_netchecker_server_image
    try:
        if k8s.api.deployments.get(name=deploy_spec['metadata']['name'],
                                   namespace=namespace):
            LOG.debug('Network checker server deployment "{}" '
                      'already exists! Skipping resource '
                      'creation'.format(deploy_spec['metadata']['name']))
    except rest.ApiException as e:
        if e.status == 404:
            n = k8s.check_deploy_create(body=deploy_spec, namespace=namespace)
            k8s.wait_deploy_ready(n.name, namespace=namespace)
        else:
            raise e
    try:
        if k8s.api.services.get(name=svc_spec['metadata']['name']):
            LOG.debug('Network checker server service {} is '
                      'already running! Skipping resource creation'
                      '.'.format(svc_spec['metadata']['name']))
    except rest.ApiException as e:
        if e.status == 404:
            k8s.check_service_create(body=svc_spec, namespace=namespace)
        else:
            raise e


def start_agent(k8s, config, namespace=None, ds_spec=NETCHECKER_DS_CFG,
                service_namespace=None):
    """Start netchecker agent in k8s cluster

    :param k8s: K8SManager
    :param config: fixture provides oslo.config
    :param namespace: str
    :param ds_spec: str
    :return: None
    """
    for ds in ds_spec:
        for container in ds['spec']['template']['spec']['containers']:
            if container['name'] == 'netchecker-agent':
                container['image'] = \
                    config.k8s_deploy.kubernetes_netchecker_agent_image
                if service_namespace is not None:
                    container['args'].append(
                        "-serverendpoint={0}.{1}.svc.cluster.local:{2}".format(
                            NETCHECKER_SERVICE_NAME,
                            service_namespace,
                            NETCHECKER_SERVICE_PORT))
        k8s.check_ds_create(body=ds, namespace=namespace)
        k8s.wait_ds_ready(dsname=ds['metadata']['name'], namespace=namespace)
    k8s.wait_pods_phase(pods=[pod for pod in k8s.api.pods.list()
                              if 'netchecker-agent' in pod.name],
                        phase='Running',
                        timeout=600)


@utils.retry(3, requests.exceptions.RequestException)
def get_connectivity_status(k8sclient,
                            netchecker_pod_port=NETCHECKER_NODE_PORT,
                            pod_name='netchecker-server', namespace='default'):

    netchecker_srv_pod_names = [pod.name for pod in
                                k8sclient.pods.list(namespace=namespace)
                                if pod_name in pod.name]

    assert len(netchecker_srv_pod_names) > 0, \
        "No netchecker-server pods found!"

    netchecker_srv_pod = k8sclient.pods.get(name=netchecker_srv_pod_names[0],
                                            namespace=namespace)
    kube_host_ip = netchecker_srv_pod.status.host_ip
    net_status_url = 'http://{0}:{1}/api/v1/connectivity_check'.format(
        kube_host_ip, netchecker_pod_port)
    response = requests.get(net_status_url, timeout=5)
    LOG.debug('Connectivity check status: [{0}] {1}'.format(
        response.status_code, response.text.strip()))
    return response


@utils.retry(3, requests.exceptions.RequestException)
def get_netchecker_pod_status(k8s,
                              pod_name='netchecker-server',
                              namespace='default'):

    k8s.wait_pods_phase(
        pods=[pod for pod in k8s.api.pods.list(namespace=namespace)
              if pod_name in pod.name], phase='Running', timeout=600)


def check_network(k8sclient, netchecker_pod_port,
                  namespace='default', works=True):
    if works:
        assert get_connectivity_status(
            k8sclient, namespace=namespace,
            netchecker_pod_port=netchecker_pod_port).status_code in (200, 204)
    else:
        assert get_connectivity_status(
            k8sclient, namespace=namespace,
            netchecker_pod_port=netchecker_pod_port).status_code == 400


def wait_check_network(k8sclient, namespace='default', works=True, timeout=300,
                       interval=10, netchecker_pod_port=NETCHECKER_NODE_PORT):
    helpers.wait_pass(
        lambda: check_network(
            k8sclient, netchecker_pod_port=netchecker_pod_port,
            namespace=namespace,
            works=works),
        timeout=timeout,
        interval=interval)


def calico_block_traffic_on_node(underlay, target_node):
    cmd = "echo '{0}' | calicoctl create -f -".format(NETCHECKER_BLOCK_POLICY)
    underlay.sudo_check_call(cmd, node_name=target_node)
    LOG.info('Blocked traffic to the network checker service from '
             'containers on node "{}".'.format(target_node))


def calico_unblock_traffic_on_node(underlay, target_node):
    cmd = "echo '{0}' | calicoctl delete -f -".format(NETCHECKER_BLOCK_POLICY)

    underlay.sudo_check_call(cmd, node_name=target_node)
    LOG.info('Unblocked traffic to the network checker service from '
             'containers on node "{}".'.format(target_node))


def calico_get_version(underlay, target_node):
    raw_version = underlay.sudo_check_call('calicoctl version',
                                           node_name=target_node)

    assert raw_version['exit_code'] == 0 and len(raw_version['stdout']) > 0, \
        "Unable to get calico version!"

    if len(raw_version['stdout']) > 1:
        ctl_version = raw_version['stdout'][0].split()[1].strip()
    else:
        ctl_version = raw_version['stdout'][0].strip()

    LOG.debug("Calico (calicoctl) version on '{0}': '{1}'".format(target_node,
                                                                  ctl_version))
    return ctl_version


def kubernetes_block_traffic_namespace(underlay, kube_host_ip, namespace):
    # TODO(apanchenko): do annotation using kubernetes API
    cmd = ('kubectl annotate ns {0} \'net.beta.kubernetes.io/'
           'network-policy={{"ingress": {{"isolation":'
           ' "DefaultDeny"}}}}\'').format(namespace)
    underlay.sudo_check_call(cmd=cmd, host=kube_host_ip)


def calico_allow_netchecker_connections(underlay, k8sclient, kube_host_ip,
                                        namespace):
    netchecker_srv_pod_names = [pod.name for pod in
                                k8sclient.pods.list(namespace=namespace)
                                if 'netchecker-server' in pod.name]

    assert len(netchecker_srv_pod_names) > 0, \
        "No netchecker-server pods found!"

    netchecker_srv_pod = k8sclient.pods.get(name=netchecker_srv_pod_names[0],
                                            namespace=namespace)
    nc_host_ip = netchecker_srv_pod.status.host_ip

    kubernetes_policy = {
        "apiVersion": "extensions/v1beta1",
        "kind": "NetworkPolicy",
        "metadata": {
            "name": "access-netchecker",
            "namespace": namespace,
        },
        "spec": {
            "ingress": [
                {
                    "from": [
                        {
                            "ipBlock": {
                                "cidr": nc_host_ip + "/24"
                            }
                        }
                    ]
                }
            ],
            "podSelector": {
                "matchLabels": {
                    "app": "netchecker-server"
                }
            }
        }
    }

    cmd_add_policy = "echo '{0}' | kubectl create -f -".format(
        json.dumps(kubernetes_policy))
    underlay.sudo_check_call(cmd=cmd_add_policy, host=kube_host_ip)


def kubernetes_allow_traffic_from_agents(underlay, kube_host_ip, namespace):
    # TODO(apanchenko): add network policies using kubernetes API
    label_namespace_cmd = "kubectl label namespace default name=default"
    underlay.sudo_check_call(cmd=label_namespace_cmd, host=kube_host_ip)
    kubernetes_policy = {
        "apiVersion": "extensions/v1beta1",
        "kind": "NetworkPolicy",
        "metadata": {
            "name": "access-netchecker-agent",
            "namespace": namespace,
        },
        "spec": {
            "ingress": [
                {
                    "from": [
                        {
                            "namespaceSelector": {
                                "matchLabels": {
                                    "name": namespace
                                }
                            }
                        },
                        {
                            "podSelector": {
                                "matchLabels": {
                                    "app": "netchecker-agent"
                                }
                            }
                        }
                    ]
                }
            ],
            "podSelector": {
                "matchLabels": {
                    "app": "netchecker-server"
                }
            }
        }
    }

    kubernetes_policy_hostnet = {
        "apiVersion": "extensions/v1beta1",
        "kind": "NetworkPolicy",
        "metadata": {
            "name": "access-netchecker-agent-hostnet",
            "namespace": namespace,
        },
        "spec": {
            "ingress": [
                {
                    "from": [
                        {
                            "namespaceSelector": {
                                "matchLabels": {
                                    "name": namespace
                                }
                            }
                        },
                        {
                            "podSelector": {
                                "matchLabels": {
                                    "app": "netchecker-agent-hostnet"
                                }
                            }
                        }
                    ]
                }
            ],
            "podSelector": {
                "matchLabels": {
                    "app": "netchecker-server"
                }
            }
        }
    }

    cmd_add_policy = "echo '{0}' | kubectl create -f -".format(
        json.dumps(kubernetes_policy))
    underlay.sudo_check_call(cmd=cmd_add_policy, host=kube_host_ip)

    cmd_add_policy_hostnet = "echo '{0}' | kubectl create -f -".format(
        json.dumps(kubernetes_policy_hostnet))
    underlay.sudo_check_call(cmd=cmd_add_policy_hostnet, host=kube_host_ip)


@utils.retry(3, requests.exceptions.RequestException)
def get_metric(k8sclient, netchecker_pod_port,
               pod_name='netchecker-server', namespace='default'):

    netchecker_srv_pod_names = [pod.name for pod in
                                k8sclient.pods.list(namespace=namespace)
                                if pod_name in pod.name]

    assert len(netchecker_srv_pod_names) > 0, \
        "No netchecker-server pods found!"
    netchecker_srv_pod = k8sclient.pods.get(name=netchecker_srv_pod_names[0],
                                            namespace=namespace)

    kube_host_ip = netchecker_srv_pod.status.host_ip
    metrics_url = 'http://{0}:{1}/metrics'.format(
        kube_host_ip, netchecker_pod_port)
    response = requests.get(metrics_url, timeout=30)
    LOG.debug('Metrics: [{0}] {1}'.format(
        response.status_code, response.text.strip()))
    return response


def get_service_port(k8sclient, service_name='netchecker',
                     namespace='netchecker'):
    full_service_name = [service.name for service
                         in k8sclient.services.list(namespace=namespace)
                         if service_name in service.name]
    assert len(full_service_name) > 0, "No netchecker service run"

    service_details = k8sclient.services.get(name=full_service_name[0],
                                             namespace=namespace)

    LOG.debug('Necthcecker service details {0}'.format(service_details))
    netchecker_port = service_details.spec.ports[0].node_port
    return netchecker_port