Implemented HW2HW network performance testing
Implemented HW2HW network performance test:
- Added K8S client manager and API methods
- Added method for collecting HW compute nodes
- Improved and extended global config
- Extended requirements.txt, pinned some modules
- Extended and improved SSH methods
Added some small improvements:
- extended .gitignore
- Added some custom exceptions instead of basic ones
- Renamed some classes
- Set iperf v2 to be default for multi-threads tests
- Updated README file
Related-PROD: PROD-36943
Change-Id: I265058967ccc01d96bf3bca532a8a0ae2a26f1f2
diff --git a/utils/__init__.py b/utils/__init__.py
index e4de308..155c03f 100644
--- a/utils/__init__.py
+++ b/utils/__init__.py
@@ -3,7 +3,11 @@
import sys
import yaml
+from kubernetes import client as kclient, config as kconfig
+
+from utils import exceptions
from utils import os_client
+from utils import k8s_client
logger = logging.getLogger(__name__)
@@ -23,8 +27,8 @@
cmp_hosts = config.get('CMP_HOSTS') or []
skipped_nodes = config.get('skipped_nodes') or []
if skipped_nodes:
- sys.stdout.write(("\nNotice: {} nodes will be skipped for vm2vm test"
- "".format(",".join(skipped_nodes))))
+ sys.stdout.write(("\nNotice: {} node(s) will be skipped for vm2vm "
+ "test.\n".format(", ".join(skipped_nodes))))
logger.info("Skipping nodes {}".format(",".join(skipped_nodes)))
if not cmp_hosts:
openstack_clients = os_client.OfficialClientManager(
@@ -38,13 +42,13 @@
os_actions = os_client.OSCliActions(openstack_clients)
nova_computes = os_actions.list_nova_computes()
if len(nova_computes) < 2:
- raise BaseException(
+ raise exceptions.NotEnoughNodes(
"At least 2 compute hosts are needed for VM2VM test, "
"now: {}.".format(len(nova_computes)))
cmp_hosts = [n.host_name for n in nova_computes
if n.host_name not in skipped_nodes]
if len(cmp_hosts) < 2:
- raise BaseException(
+ raise exceptions.NotEnoughNodes(
"At least 2 compute hosts are needed for VM2VM test. "
"Cannot create a pair from {}. Please check skip list, at "
"least 2 computes should be tested.".format(cmp_hosts))
@@ -54,6 +58,39 @@
return compile_pairs(cmp_hosts)
+def get_hw_pairs():
+ # get the K8S config, check whether the HW nodes list is set
+ config = get_configuration()
+ logger.info("Getting the K8S config path from the global_config.yaml file.")
+ k8s_config_path = config.get("mos_kubeconfig_path", "")
+ hw_nodes_list = config.get("hw_nodes_list", [])
+
+ # if the specific HW nodes list is not set in the config, get from K8S
+ hw_nodes = None
+ if not hw_nodes_list:
+ # fetch only compute nodes
+ label_selector = "openstack-compute-node=enabled," \
+ "openvswitch=enabled"
+ k8s_api = k8s_client.K8SClientManager(k8s_config_path=k8s_config_path)
+ k8s_actions = k8s_client.K8SCliActions(k8s_api.k8s_v1)
+ hw_nodes_list = k8s_actions.list_nodes_names(
+ label_selector=label_selector)
+
+ # remove some skipped nodes if any
+ skipped_nodes = config.get('skipped_nodes', [])
+ if skipped_nodes:
+ print(f"Notice: {', '.join(skipped_nodes)} node(s) will be skipped for"
+ f" hw2hw test.\n")
+ hw_nodes = [node for node in hw_nodes_list
+ if node not in skipped_nodes]
+ if len(hw_nodes) < 2:
+ raise exceptions.NotEnoughNodes(
+ f"At least 2 HW nodes are required to run hw2hw test. Cannot "
+ f"create a pair from {hw_nodes}. Check whether the cluster has at"
+ f" least 2 compute nodes, or the nodes are not in the skip list.")
+ return compile_pairs(hw_nodes)
+
+
def get_configuration():
"""function returns configuration for environment
and for test if it's specified"""
@@ -77,8 +114,9 @@
def check_iperf_utility(actual_iperf_utility):
valid_values = ["iperf", "iperf3"]
if actual_iperf_utility not in valid_values:
- raise BaseException("The iperf utility for multiple threads test case "
- "is not correct. Valid value is one of {}. Actual "
- "value is {}. Please set the correct value in "
- "global_config.yaml:multiple_threads_iperf_utility"
- "".format(valid_values, actual_iperf_utility))
+ raise exceptions.InvalidConfigException(
+ "The iperf utility for multiple threads test case is not correct. "
+ "Valid value is one of {}. Actual value is {}. Please set the "
+ "correct value in global_config.yaml:"
+ "multiple_threads_iperf_utility".format(
+ valid_values, actual_iperf_utility))
diff --git a/utils/exceptions.py b/utils/exceptions.py
new file mode 100644
index 0000000..1d014fc
--- /dev/null
+++ b/utils/exceptions.py
@@ -0,0 +1,10 @@
+class NotEnoughNodes(Exception):
+ pass
+
+
+class InvalidConfigException(Exception):
+ pass
+
+
+class NoPackageInstalled(Exception):
+ pass
diff --git a/utils/k8s_client.py b/utils/k8s_client.py
new file mode 100644
index 0000000..72c8b06
--- /dev/null
+++ b/utils/k8s_client.py
@@ -0,0 +1,57 @@
+import logging
+
+from kubernetes import client as kclient
+from kubernetes import config as kconfig
+
+from utils import exceptions
+
+logger = logging.getLogger(__name__)
+
+
+class K8SClientManager(object):
+ def __init__(self, k8s_config_path=None):
+ self.k8s_config_path = k8s_config_path
+ self._k8s_v1 = None
+
+ def get_k8s_v1_client(self):
+ if not self.k8s_config_path:
+ raise exceptions.InvalidConfigException(
+ "Please provide the Kubernetes config file path at "
+ "'mos_kubeconfig_path' config option."
+ )
+ kconfig.load_kube_config(config_file=self.k8s_config_path)
+ return kclient.CoreV1Api()
+
+ @property
+ def k8s_v1(self):
+ if self._k8s_v1 is None:
+ self._k8s_v1 = self.get_k8s_v1_client()
+ return self._k8s_v1
+
+
+class K8SCliActions(object):
+ INTERNAL_IP_TYPE = "InternalIP"
+
+ def __init__(self, k8s_v1_client):
+ self.k8s_v1_client = k8s_v1_client
+
+ def list_nodes_names(self, watch=False, label_selector=None):
+ nodes = self.k8s_v1_client.list_node(
+ watch=watch, label_selector=label_selector)
+ return [node.metadata.name for node in nodes.items]
+
+ def get_nodes_info_by_names(self, list_of_node_names):
+ # Get the nodes' info (just names and Internal IPs) for
+ # the specific list of the K8S nodes names
+ result_nodes_info = []
+ for node in list_of_node_names:
+ node_info = self.k8s_v1_client.read_node(name=node)
+ result_nodes_info.append({
+ "name": node_info.metadata.name,
+ "address": [
+ address.address
+ for address in node_info.status.addresses
+ if address.type == self.INTERNAL_IP_TYPE
+ ][0],
+ })
+ return result_nodes_info
diff --git a/utils/ssh.py b/utils/ssh.py
index e9e5f3a..caca30e 100644
--- a/utils/ssh.py
+++ b/utils/ssh.py
@@ -6,6 +6,8 @@
import time
import os
+from netaddr import IPNetwork, IPAddress
+
logger = logging.getLogger(__name__)
# Suppress paramiko logging
@@ -20,6 +22,10 @@
self.username = username
self.password = password
if private_key is not None:
+ if os.path.isfile(private_key):
+ with open(private_key, 'r') as key_file:
+ private_key_content = key_file.read()
+ private_key = private_key_content
self.private_key = paramiko.RSAKey.from_private_key(
StringIO(private_key))
else:
@@ -131,7 +137,7 @@
any(req in pack for req in required_packages) if
pack.endswith('.deb')]
if not iperf_deb_files:
- raise BaseException(
+ raise utils.exceptions.NoPackageInstalled(
"iperf3 or iperf *.deb packages are not found locally at path"
" {}. Please recheck 'iperf_deb_package_dir_path' variable in "
"global_config.yaml and check *.deb packages are manually "
@@ -178,7 +184,8 @@
"seconds.".format(floating_ip, attempts, bsleep))
time.sleep(bsleep)
- def get_mtu_from_vm(self, floating_ip, user='ubuntu', password='password',
+ @staticmethod
+ def get_mtu_from_vm(floating_ip, user='ubuntu', password='password',
private_key=None):
transport = SSHTransport(floating_ip, user, password, private_key)
iface = (transport.exec_command(
@@ -186,8 +193,29 @@
mtu = transport.exec_command('cat /sys/class/net/{}/mtu'.format(iface))
return mtu.decode("utf-8")
+ @staticmethod
+ def get_node_ip_addresses_from_cidr(ip, cidr, user='mcc-user',
+ private_key=None):
+ transport = SSHTransport(
+ ip, username=user, private_key=private_key)
+ command = "ip -4 addr show scope global"
+ output = transport.exec_command(command)
+ try:
+ all_ip_addresses = [line.split()[1] for line in
+ output.decode().splitlines()
+ if "/" in line.split()[1]]
+ for address in all_ip_addresses:
+ ip_addr = address.split('/')[0]
+ if IPAddress(ip_addr) in IPNetwork(cidr):
+ return ip_addr
+ except Exception as e:
+ raise utils.exceptions.InvalidConfigException(
+ f"Could not find the IP at the interface {cidr} at the node "
+ f"with K8S Private IP {ip}. Please check the configuration. "
+ f"\nException: {e}")
-class prepare_iperf(object):
+
+class IperfAtVM(object):
def __init__(self, fip, user='ubuntu', password='password',
private_key=None):
@@ -218,13 +246,49 @@
logger.info(check.decode('utf-8'))
if not check:
if internet_at_vms.lower() == 'true':
- info = "Please check the Internet access at VM."
+ info = "Please check the Internet access at VM"
else:
info = "Could not put offline iperf packages from {} to the " \
- "VM.".format(path_to_iperf_deb)
- raise BaseException("iperf3 is not installed at VM with FIP {}. "
- "{}.\nStdout, stderr at VM:\n{}\n{}"
- "".format(fip, info, stdout, stderr))
+ "VM".format(path_to_iperf_deb)
+ raise utils.exceptions.NoPackageInstalled(
+ "iperf3 is not installed at VM with FIP {}. {}.\nStdout, "
+ "stderr at VM:\n{}\n{}".format(fip, info, stdout, stderr))
# Staring iperf server
transport.exec_command('nohup iperf3 -s > file 2>&1 &')
transport.exec_command('nohup iperf -s > file 2>&1 &')
+
+
+class IperfAtNode(object):
+
+ def __init__(self, ip, iperf_test_ip, user='mcc-user', private_key=None):
+ transport = SSHTransport(ip, user, private_key=private_key)
+ # TODO: to avoid looping, both packages can be installed by one
+ # 'install -y iperf iperf3' command. 'which' command can also be
+ # executed for multiple packages at once.
+ packages = ["iperf", "iperf3"]
+ for p in packages:
+ check_path = transport.exec_command(f'which {p}')
+ if not check_path:
+ self.install_iperf = True
+ # Install iperf/iperf3 at MOSK nodes
+ logger.info(f"Installing {p} at MOSK nodes...")
+ _, stdout, stderr = transport.exec_sync(
+ f"sudo apt update && sudo apt install -y {p}")
+ else:
+ self.install_iperf = False
+ check_path = transport.exec_command(f'which {p}')
+ # Log whether iperf is installed
+ logger.info(f"{p} package path: {check_path.decode('utf-8')}")
+ if not check_path:
+ raise utils.exceptions.NoPackageInstalled(
+ f"{p} is not installed at the MOSK node with IP {ip}.\n"
+ f"Stdout, stderr at VM:\n{stdout}\n{stderr}")
+ # Staring iperf/iperf3 server
+ transport.exec_command(
+ f"nohup {p} -s -B {iperf_test_ip} > file 2>&1 &")
+
+ @staticmethod
+ def remove_iperf_packages(ip, user='mcc-user', private_key=None):
+ transport = SSHTransport(ip, user, private_key=private_key)
+ logger.info(f"Removing iperf,iperf3 packages from the node {ip}...")
+ transport.exec_command("sudo apt remove iperf iperf3 -y")