Merge "Add new k8s genie-cni calico+flannel test"
diff --git a/tcp_tests/managers/k8smanager.py b/tcp_tests/managers/k8smanager.py
index ff81501..9b5588d 100644
--- a/tcp_tests/managers/k8smanager.py
+++ b/tcp_tests/managers/k8smanager.py
@@ -15,7 +15,7 @@
import os
import time
from uuid import uuid4
-
+import six
import requests
import yaml
@@ -86,11 +86,13 @@
default_namespace='default')
return self._api_client
+ def ctl_hosts(self):
+ return [node for node in self.__config.underlay.ssh if
+ ext.UNDERLAY_NODE_ROLES.k8s_controller in node['roles']]
+
@property
def ctl_host(self):
- nodes = [node for node in self.__config.underlay.ssh if
- ext.UNDERLAY_NODE_ROLES.k8s_controller in node['roles']]
- return nodes[0]['node_name']
+ return self.ctl_hosts()[0]['node_name']
def get_pod_phase(self, pod_name, namespace=None):
return self.api.pods.get(
@@ -149,6 +151,8 @@
:rtype: V1Pod
"""
LOG.info("Creating pod in k8s cluster")
+ if isinstance(body, six.string_types):
+ body = yaml.load(body)
LOG.debug(
"POD spec to create:\n{}".format(
yaml.dump(body, default_flow_style=False))
@@ -156,7 +160,7 @@
LOG.debug("Timeout for creation is set to {}".format(timeout))
LOG.debug("Checking interval is set to {}".format(interval))
pod = self.api.pods.create(body=body, namespace=namespace)
- pod.wait_running(timeout=300, interval=5)
+ pod.wait_running(timeout=timeout, interval=interval)
LOG.info("Pod '{0}' is created in '{1}' namespace".format(
pod.name, pod.namespace))
return self.api.pods.get(name=pod.name, namespace=pod.namespace)
@@ -394,9 +398,8 @@
@retry(300, exception=DevopsCalledProcessError)
def nslookup(self, host, src):
- with self.__underlay.remote(
- node_name=self.ctl_host) as remote:
- remote.check_call("nslookup {0} {1}".format(host, src))
+ self.__underlay.check_call(
+ "nslookup {0} {1}".format(host, src), node_name=self.ctl_host)
@retry(300, exception=DevopsCalledProcessError)
def curl(self, url):
@@ -406,8 +409,10 @@
:param url: url to curl
:return: response string
"""
- with self.__underlay.remote(node_name=self.ctl_host) as r:
- return r.check_call("curl -s -S \"{}\"".format(url))['stdout']
+ result = self.__underlay.check_call(
+ "curl -s -S \"{}\"".format(url), node_name=self.ctl_host)
+ LOG.debug("curl \"{0}\" result: {1}".format(url, result['stdout']))
+ return result['stdout']
# ---------------------------- Virtlet methods -------------------------------
def install_jq(self):
@@ -709,11 +714,32 @@
ctl_vip_pillar = self._salt.get_pillar(
tgt="I@kubernetes:control:enabled:True",
pillar="_param:cluster_vip_address")[0]
- return [vip for minion_id, vip in ctl_vip_pillar.items()][0]
+ return ctl_vip_pillar.values()[0]
def get_sample_deployment(self, name, **kwargs):
return K8SSampleDeployment(self, name, **kwargs)
+ def is_pod_exists_with_prefix(self, prefix, namespace, phase='Running'):
+ for pod in self.api.pods.list(namespace=namespace):
+ if pod.name.startswith(prefix) and pod.phase == phase:
+ return True
+ return False
+
+ def get_pod_ips_from_container(self, pod_name, exclude_local=True):
+ """ Not all containers have 'ip' binary on-board """
+ cmd = "kubectl exec {0} ip a|grep \"inet \"|awk '{{print $2}}'".format(
+ pod_name)
+ result = self.__underlay.check_call(cmd, node_name=self.ctl_host)
+ ips = [line.strip().split('/')[0] for line in result['stdout']]
+ if exclude_local:
+ ips = [ip for ip in ips if not ip.startswith("127.")]
+ return ips
+
+ def create_pod_from_file(self, path, namespace=None):
+ with open(path) as f:
+ data = f.read()
+ return self.check_pod_create(data, namespace=namespace)
+
class K8SSampleDeployment:
def __init__(self, manager, name, replicas=2,
diff --git a/tcp_tests/tests/system/test_k8s_actions.py b/tcp_tests/tests/system/test_k8s_actions.py
index 1d28a5f..c0f1b94 100644
--- a/tcp_tests/tests/system/test_k8s_actions.py
+++ b/tcp_tests/tests/system/test_k8s_actions.py
@@ -13,6 +13,8 @@
# under the License.
import pytest
+import netaddr
+import os
from tcp_tests import logger
from tcp_tests import settings
@@ -140,16 +142,9 @@
pytest.skip("Test requires metallb addon enabled")
show_step(2)
- pods = k8s_deployed.api.pods.list(namespace="metallb-system")
-
- def is_pod_exists_with_prefix(prefix):
- for pod in pods:
- if pod.name.startswith(prefix) and pod.phase == 'Running':
- return True
- return False
-
- assert is_pod_exists_with_prefix("controller")
- assert is_pod_exists_with_prefix("speaker")
+ ns = "metallb-system"
+ assert k8s_deployed.is_pod_exists_with_prefix("controller", ns)
+ assert k8s_deployed.is_pod_exists_with_prefix("speaker", ns)
show_step(3)
samples = []
@@ -175,3 +170,111 @@
show_step(7)
for sample in samples:
assert sample.is_service_available(external=True)
+
+ @pytest.mark.grap_versions
+ @pytest.mark.fail_snapshot
+ def test_k8s_genie_flannel(self, show_step, underlay, salt_deployed,
+ k8s_deployed, k8s_copy_sample_testdata):
+ """Test genie-cni+flannel cni setup
+
+ Scenario:
+ 1. Setup Kubernetes cluster with genie cni and flannel
+ 2. Check that flannel pods created in kube-system namespace
+ 3. Create sample deployment with flannel cni annotation
+ 4. Check that the deployment have 1 ip addresses from cni provider
+ 5. Create sample deployment with calico cni annotation
+ 6. Check that the deployment have 1 ip addresses from cni provider
+ 7. Create sample deployment with multi-cni annotation
+ 8. Check that the deployment have 2 ip addresses from different
+ cni providers
+ 9. Create sample deployment without cni annotation
+ 10. Check that the deployment have 1 ip address
+ 11. Check pods availability
+ 12. Run conformance
+ 13. Check pods availability
+ """
+ show_step(1)
+
+ # Find out calico and flannel networks
+ tgt_k8s_control = "I@kubernetes:control:enabled:True"
+
+ flannel_pillar = salt_deployed.get_pillar(
+ tgt=tgt_k8s_control,
+ pillar="kubernetes:master:network:flannel:private_ip_range")[0]
+ flannel_network = netaddr.IPNetwork(flannel_pillar.values()[0])
+ LOG.info("Flannel network: {}".format(flannel_network))
+
+ calico_network_pillar = salt_deployed.get_pillar(
+ tgt=tgt_k8s_control, pillar="_param:calico_private_network")[0]
+ calico_netmask_pillar = salt_deployed.get_pillar(
+ tgt=tgt_k8s_control, pillar="_param:calico_private_netmask")[0]
+ calico_network = netaddr.IPNetwork(
+ "{0}/{1}".format(calico_network_pillar.values()[0],
+ calico_netmask_pillar.values()[0]))
+ LOG.info("Calico network: {}".format(calico_network))
+
+ show_step(2)
+ assert k8s_deployed.is_pod_exists_with_prefix("kube-flannel-",
+ "kube-system")
+
+ data_dir = os.path.join(os.path.dirname(__file__), 'testdata/k8s')
+ show_step(3)
+ flannel_pod = k8s_deployed.create_pod_from_file(
+ os.path.join(data_dir, 'pod-sample-flannel.yaml'))
+
+ show_step(4)
+ flannel_ips = k8s_deployed.get_pod_ips_from_container(flannel_pod.name)
+ assert len(flannel_ips) == 1
+ assert netaddr.IPAddress(flannel_ips[0]) in flannel_network
+
+ show_step(5)
+ calico_pod = k8s_deployed.create_pod_from_file(
+ os.path.join(data_dir, 'pod-sample-calico.yaml'))
+
+ show_step(6)
+ calico_ips = k8s_deployed.get_pod_ips_from_container(calico_pod.name)
+ assert len(calico_ips) == 1
+ assert netaddr.IPAddress(calico_ips[0]) in calico_network
+
+ show_step(7)
+ multicni_pod = k8s_deployed.create_pod_from_file(
+ os.path.join(data_dir, 'pod-sample-multicni.yaml'))
+
+ show_step(8)
+ multicni_ips = \
+ k8s_deployed.get_pod_ips_from_container(multicni_pod.name)
+ assert len(multicni_ips) == 2
+ for net in [calico_network, flannel_network]:
+ assert netaddr.IPAddress(multicni_ips[0]) in net or \
+ netaddr.IPAddress(multicni_ips[1]) in net
+
+ show_step(9)
+ nocni_pod = k8s_deployed.create_pod_from_file(
+ os.path.join(data_dir, 'pod-sample.yaml'))
+
+ show_step(10)
+ nocni_ips = k8s_deployed.get_pod_ips_from_container(nocni_pod.name)
+ assert len(nocni_ips) == 1
+ assert (netaddr.IPAddress(nocni_ips[0]) in calico_network or
+ netaddr.IPAddress(nocni_ips[0]) in flannel_network)
+
+ show_step(11)
+
+ def check_pod_availability(ip):
+ assert "Hello Kubernetes!" in k8s_deployed.curl(
+ "http://{}:8080".format(ip))
+
+ def check_pods_availability():
+ check_pod_availability(flannel_ips[0])
+ check_pod_availability(calico_ips[0])
+ check_pod_availability(multicni_ips[0])
+ check_pod_availability(multicni_ips[1])
+ check_pod_availability(nocni_ips[0])
+
+ check_pods_availability()
+
+ show_step(12)
+ k8s_deployed.run_conformance()
+
+ show_step(13)
+ check_pods_availability()
diff --git a/tcp_tests/tests/system/testdata/k8s/pod-sample-calico.yaml b/tcp_tests/tests/system/testdata/k8s/pod-sample-calico.yaml
new file mode 100644
index 0000000..2b5cbf9
--- /dev/null
+++ b/tcp_tests/tests/system/testdata/k8s/pod-sample-calico.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-sample-calico
+ annotations:
+ cni: "calico"
+spec:
+ containers:
+ - name: pod-sample-calico-container
+ image: gcr.io/google-samples/node-hello:1.0
+ ports:
+ - containerPort: 8080
diff --git a/tcp_tests/tests/system/testdata/k8s/pod-sample-flannel.yaml b/tcp_tests/tests/system/testdata/k8s/pod-sample-flannel.yaml
new file mode 100644
index 0000000..2618fc5
--- /dev/null
+++ b/tcp_tests/tests/system/testdata/k8s/pod-sample-flannel.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-sample-flannel
+ annotations:
+ cni: "flannel"
+spec:
+ containers:
+ - name: pod-sample-flannel-container
+ image: gcr.io/google-samples/node-hello:1.0
+ ports:
+ - containerPort: 8080
diff --git a/tcp_tests/tests/system/testdata/k8s/pod-sample-multicni.yaml b/tcp_tests/tests/system/testdata/k8s/pod-sample-multicni.yaml
new file mode 100644
index 0000000..4c80203
--- /dev/null
+++ b/tcp_tests/tests/system/testdata/k8s/pod-sample-multicni.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-sample-multicni
+ annotations:
+ cni: "calico,flannel"
+spec:
+ containers:
+ - name: pod-sample-multicni-container
+ image: gcr.io/google-samples/node-hello:1.0
+ ports:
+ - containerPort: 8080
diff --git a/tcp_tests/tests/system/testdata/k8s/pod-sample.yaml b/tcp_tests/tests/system/testdata/k8s/pod-sample.yaml
new file mode 100644
index 0000000..3853461
--- /dev/null
+++ b/tcp_tests/tests/system/testdata/k8s/pod-sample.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-sample
+spec:
+ containers:
+ - name: pod-sample-container
+ image: gcr.io/google-samples/node-hello:1.0
+ ports:
+ - containerPort: 8080