Implement Python helpers for creating fio resources
Related-PROD: PROD-37187
Change-Id: Iee1d85955fe10876a85da99025211946c2135178
diff --git a/fio/clouds.yaml b/fio/clouds.yaml
new file mode 100644
index 0000000..7964385
--- /dev/null
+++ b/fio/clouds.yaml
@@ -0,0 +1,45 @@
+# This is a clouds.yaml file, which can be used by OpenStack tools as a source
+# of configuration on how to connect to a cloud. If this is your only cloud,
+# just put this file in ~/.config/openstack/clouds.yaml and tools like
+# python-openstackclient will just work with no further config. (You will need
+# to add your password to the auth section)
+# If you have more than one cloud account, add the cloud entry to the clouds
+# section of your existing file and you can refer to them by name with
+# OS_CLOUD=openstack or --os-cloud=openstack
+clouds:
+ target:
+ auth:
+ auth_url: AUTH_URL
+ username: USER_NAME
+ project_name: PROJECT_NAME
+ project_domain_id: default
+ user_domain_name: "default"
+ password: PASSWORD
+ region_name: REGION_NAME
+ interface: "public"
+ insecure: true
+ identity_api_version: 3
+ custom_vars:
+ ubuntu_image_name: "Ubuntu-18.04"
+ fio_net_name: "fio-net"
+ fio_subnet_name: "fio-subnet"
+ fixed_subnet_range: "192.168.200.0/24"
+ net_ipv4: '4'
+ fio_router_name: "fio-router"
+ floating_net_name: "public"
+ sg_name: "fio-sg"
+ keypair_name: "fio-key"
+ keypair_file_location: "."
+ fio_client_name_mask: "fio-vm"
+ fio_flavor_name: "fio-flavor"
+ fio_flavor_ram: 2048
+ fio_flavor_cpus: 10
+ fio_flavor_disk: 20
+ fio_clients_count: 100
+ fio_vol_name_mask: "fio-vol"
+ fio_vol_size: 100
+ fio_vol_type: "volumes-nvme"
+ fio_vol_mountpoint: "/dev/vdc"
+ mtu_size: 8000
+ hv_suffix: "kaas-kubernetes-XXX"
+ cloud_name: "cloud-XXX"
diff --git a/fio/connection.py b/fio/connection.py
new file mode 100644
index 0000000..4007ebe
--- /dev/null
+++ b/fio/connection.py
@@ -0,0 +1,111 @@
+import os
+from typing import Any, Final, Union
+
+import openstack
+
+
+# openstack.enable_logging(True, path='openstack.log')
+
+TEST_CLOUD: Final[str] = os.getenv('OS_TEST_CLOUD', 'target')
+
+cloud = openstack.connect(cloud=TEST_CLOUD)
+config = openstack.config.loader.OpenStackConfig()
+cloud_config = config.get_one(cloud=TEST_CLOUD)
+
+
+def get_resource_value(
+ resource_key: str, default: Union[int, str]
+) -> Union[int, str]:
+ try:
+ return cloud_config.config['custom_vars'][resource_key]
+ except KeyError:
+ return default
+
+
+CLOUD_NAME: Final[Any] = get_resource_value('cloud_name', '')
+
+UBUNTU_IMAGE_NAME: Final[Any] = get_resource_value(
+ 'ubuntu_image_name', 'Ubuntu-18.04')
+FIO_SG_NAME: Final[Any] = get_resource_value('sg_name', 'fio-sg')
+FIO_KEYPAIR_NAME: Final[Union[int, str]] = "-".join(
+ [get_resource_value('keypair_name', 'fio-key'), CLOUD_NAME])
+PRIVATE_KEYPAIR_FILE: Final[str] = "{}/{}.pem".format(
+ get_resource_value('keypair_file_location', '.'),
+ FIO_KEYPAIR_NAME)
+
+FIO_NET_NAME: Final[Any] = get_resource_value('fixed_net_name', 'fio-net')
+FIO_SUBNET_NAME: Final[Any] = get_resource_value(
+ 'fixed_subnet_name', 'fio-subnet')
+FIO_SUBNET_RANGE: Final[Any] = get_resource_value(
+ 'fixed_subnet_range', '192.168.200.0/24')
+NET_IPV4: Final[Any] = get_resource_value('net_ipv4', '4')
+FIO_ROUTER_NAME: Final[Any] = get_resource_value(
+ 'fio_router_name', 'fio-router')
+FLOATING_NET_NAME: Final[Any] = get_resource_value(
+ 'floating_net_name', 'public')
+MTU_SIZE: Final[Any] = get_resource_value('mtu_size', 9000)
+
+FIO_FLAVOR_NAME: Final[Any] = get_resource_value('fio_flavor_name', 'fio')
+FIO_FLAVOR_RAM: Final[Any] = get_resource_value('fio_flavor_ram', 2048)
+FIO_FLAVOR_CPUS: Final[Any] = get_resource_value('fio_flavor_cpus', 10)
+FIO_FLAVOR_DISK: Final[Any] = get_resource_value('fio_flavor_disk', 20)
+FIO_CLIENTS_COUNT: Final[Any] = int(
+ get_resource_value('fio_clients_count', 10))
+FIO_VOL_NAME_MASK: Final[Any] = get_resource_value(
+ 'fio_vol_name_mask', 'fio-vol')
+FIO_VOL_SIZE: Final[Any] = get_resource_value('fio_vol_size', 110)
+FIO_VOL_TYPE: Final[Any] = get_resource_value(
+ 'fio_vol_type', 'volumes-nvme')
+FIO_VOL_MOUNTPOINT: Final[Any] = get_resource_value(
+ 'fio_vol_mountpoint', '/dev/vdc')
+FIO_CLIENT_NAME_MASK: Final[Any] = get_resource_value(
+ 'fio_client_name_mask', 'fio-vm')
+
+HV_SUFFIX: Final[Any] = get_resource_value('hv_suffix', '')
+
+
+def delete_server(srv: openstack.compute.v2.server.Server) -> None:
+ cloud.compute.delete_server(srv)
+ cloud.compute.wait_for_delete(srv)
+
+
+def delete_volume(vol: openstack.block_storage.v3.volume.Volume) -> None:
+ cloud.volume.delete_volume(vol)
+ cloud.volume.wait_for_delete(vol)
+
+
+def detach_volume(
+ srv: openstack.compute.v2.server.Server,
+ vol: openstack.block_storage.v3.volume.Volume
+) -> None:
+ cloud.compute.delete_volume_attachment(srv, vol)
+ cloud.volume.wait_for_status(vol, status='available')
+
+
+if __name__ == "__main__":
+ print(UBUNTU_IMAGE_NAME)
+ print(FIO_SG_NAME)
+ print(FIO_KEYPAIR_NAME)
+ print(PRIVATE_KEYPAIR_FILE)
+
+ print(FIO_NET_NAME)
+ print(FIO_SUBNET_NAME)
+ print(FIO_SUBNET_RANGE)
+ print(NET_IPV4)
+ print(FIO_ROUTER_NAME)
+ print(FLOATING_NET_NAME)
+ print(MTU_SIZE)
+
+ print(FIO_FLAVOR_NAME)
+ print(FIO_FLAVOR_RAM)
+ print(FIO_FLAVOR_CPUS)
+ print(FIO_FLAVOR_DISK)
+ print(FIO_CLIENTS_COUNT)
+ print(FIO_CLIENT_NAME_MASK)
+ print(FIO_VOL_NAME_MASK)
+ print(FIO_VOL_SIZE)
+ print(FIO_VOL_TYPE)
+ print(FIO_VOL_MOUNTPOINT)
+
+ print(HV_SUFFIX)
+ print(CLOUD_NAME)
diff --git a/fio/fio_cleanup.py b/fio/fio_cleanup.py
new file mode 100644
index 0000000..ccbe38e
--- /dev/null
+++ b/fio/fio_cleanup.py
@@ -0,0 +1,81 @@
+import connection as conn
+from openstack.exceptions import ResourceFailure
+from typing import Final
+
+
+compute = conn.cloud.compute
+network = conn.cloud.network
+volume = conn.cloud.volume
+
+CLIENT_NAME_MASK: Final[str] = conn.FIO_CLIENT_NAME_MASK
+FLAVOR_NAME: Final[str] = conn.FIO_FLAVOR_NAME
+KEYPAIR_NAME: Final[str] = conn.FIO_KEYPAIR_NAME
+SG_NAME: Final[str] = conn.FIO_SG_NAME
+
+ROUTER_NAME: Final[str] = conn.FIO_ROUTER_NAME
+NET_NAME: Final[str] = conn.FIO_NET_NAME
+
+
+if __name__ == "__main__":
+ # Find fio clients and server
+ vms = compute.servers(name=CLIENT_NAME_MASK)
+ for vm in vms:
+ attachments = compute.volume_attachments(vm)
+ # Delete fio volume attachment (and any other attachments
+ # that the VM could have)
+ # Delete the volume and the server
+ for att in attachments:
+ vol_id = att.volume_id
+ vol = volume.get_volume(vol_id)
+ try:
+ conn.detach_volume(vm, vol)
+ print(
+ f"'{vol.id}' volume has been detached from fio '{vm.name}'"
+ " server.")
+ conn.delete_volume(vol)
+ print(f"'{vol.id}' volume has been deleted.")
+ conn.delete_server(vm)
+ print(f"'{vm.name}' server has been deleted.")
+ except ResourceFailure as e:
+ print(
+ f"Cleanup of '{vm.id}' with volume '{vol.id}' attached "
+ f"failed with '{e.message}' error.")
+ conn.delete_volume(vol)
+ continue
+
+ # Remove ports from fio router (including external GW)
+ router = network.find_router(ROUTER_NAME)
+ if router:
+ network.update_router(router.id, external_gateway_info={})
+ print("Externa GW port has been deleted from fio router.")
+ router_ports = network.ports(device_id=router.id)
+ for p in router_ports:
+ network.remove_interface_from_router(router.id, port_id=p.id)
+ print(f"'{p.id}' port has been deleted from fio router.")
+
+ # Delete fio network topology
+ net = network.find_network(NET_NAME)
+ if net:
+ network.delete_network(net.id)
+ print(f"fio '{net.id}' network has been deleted.")
+ if router:
+ network.delete_router(router.id)
+ print(f"fio '{router.id}' router has been deleted.")
+
+ # Delete fio flavor
+ flavor = compute.find_flavor(FLAVOR_NAME)
+ if flavor:
+ compute.delete_flavor(flavor.id)
+ print(f"fio '{flavor.id}' flavor has been deleted.")
+
+ # # Delete fio keypair
+ kp = compute.find_keypair(KEYPAIR_NAME)
+ if kp:
+ compute.delete_keypair(kp)
+ print(f"fio '{kp.id}' keypair has been deleted.")
+
+ # Delete fio security group
+ sg = network.find_security_group(SG_NAME)
+ if sg:
+ network.delete_security_group(sg)
+ print(f"fio '{sg.id}' security group has been deleted.")
diff --git a/fio/fio_setup.py b/fio/fio_setup.py
new file mode 100644
index 0000000..d04b67b
--- /dev/null
+++ b/fio/fio_setup.py
@@ -0,0 +1,218 @@
+import os
+import sys
+from typing import Dict, Final, List
+
+import connection as conn
+import openstack
+from openstack.exceptions import ResourceFailure
+
+
+compute = conn.cloud.compute
+network = conn.cloud.network
+volume = conn.cloud.volume
+
+CLIENTS_COUNT: Final[int] = conn.FIO_CLIENTS_COUNT
+CLIENT_NAME_MASK: Final[str] = conn.FIO_CLIENT_NAME_MASK
+UBUNTU_IMAGE_NAME: Final[str] = conn.UBUNTU_IMAGE_NAME
+
+VOL_NAME_MASK: Final[str] = conn.FIO_VOL_NAME_MASK
+VOL_SIZE: Final[int] = conn.FIO_VOL_SIZE
+VOL_TYPE: Final[str] = conn.FIO_VOL_TYPE
+VOL_MOUNTPOINT: Final[str] = conn.FIO_VOL_MOUNTPOINT
+
+FLAVOR_NAME: Final[str] = conn.FIO_FLAVOR_NAME
+FLAVOR_RAM: Final[int] = conn.FIO_FLAVOR_RAM
+FLAVOR_CPUS: Final[int] = conn.FIO_FLAVOR_CPUS
+FLAVOR_DISK: Final[int] = conn.FIO_FLAVOR_DISK
+
+NET_NAME: Final[str] = conn.FIO_NET_NAME
+ROUTER_NAME: Final[str] = conn.FIO_ROUTER_NAME
+FLOATING_NET_NAME = conn.FLOATING_NET_NAME
+SUBNET_NAME = conn.FIO_SUBNET_NAME
+SUBNET_RANGE = conn.FIO_SUBNET_RANGE
+MTU_SIZE = conn.MTU_SIZE
+NET_IPV4 = conn.NET_IPV4
+
+KEYPAIR_NAME: Final[str] = conn.FIO_KEYPAIR_NAME
+PRIVATE_KEYPAIR_FILE: Final[str] = conn.PRIVATE_KEYPAIR_FILE
+
+SG_NAME: Final[str] = conn.FIO_SG_NAME
+HV_SUFFIX: Final[str] = conn.HV_SUFFIX
+CLOUD_NAME: Final[str] = conn.CLOUD_NAME
+
+NODES: Final[List[str]] = []
+SKIP_NODES: Final[List[str]] = []
+
+
+SG_ALLOW_ALL_RULES: Final[List[Dict]] = [
+ {
+ 'remote_ip_prefix': '0.0.0.0/0',
+ 'protocol': 'icmp',
+ 'port_range_max': None,
+ 'port_range_min': None,
+ 'ethertype': 'IPv4'
+ },
+ {
+ 'remote_ip_prefix': '0.0.0.0/0',
+ 'protocol': 'tcp',
+ 'port_range_max': 65535,
+ 'port_range_min': 1,
+ 'ethertype': 'IPv4'
+ },
+ {
+ 'remote_ip_prefix': '0.0.0.0/0',
+ 'protocol': 'udp',
+ 'port_range_max': 65535,
+ 'port_range_min': 1,
+ 'ethertype': 'IPv4'
+ }
+]
+
+
+def create_server(
+ name, image_id, flavor_id, networks,
+ key_name, security_groups, availability_zone
+) -> openstack.connection.Connection:
+ srv = compute.create_server(
+ name=name, image_id=image_id, flavor_id=flavor_id, networks=networks,
+ key_name=key_name, security_groups=security_groups,
+ availability_zone=availability_zone)
+ return srv
+
+
+if __name__ == "__main__":
+ # Check if any fio servers already exist on the cloud
+ servers = compute.servers(details=False, name=CLIENT_NAME_MASK)
+ srvrs = list(servers)
+ if srvrs:
+ names = [s.name for s in srvrs]
+ print("The following servers already exist in the cloud:")
+ print(*names, sep='\n')
+ sys.exit(0)
+
+ # Create fio sg if needed
+ sg = network.find_security_group(SG_NAME)
+ if not sg:
+ sg = network.create_security_group(name=SG_NAME)
+ # Add 'allow-all' kind of rules to the security group
+ pairs = [
+ (r, d) for r in SG_ALLOW_ALL_RULES for d in ('ingress', 'egress')]
+ for (rule, direction) in pairs:
+ network.create_security_group_rule(
+ security_group_id=sg.id, direction=direction, **rule)
+
+ # Create fio keypair if needed
+ kp = compute.find_keypair(KEYPAIR_NAME)
+ if not kp:
+ kp = compute.create_keypair(name=KEYPAIR_NAME)
+ with open(PRIVATE_KEYPAIR_FILE, 'w') as f:
+ f.write("{}".format(kp.private_key))
+
+ os.chmod(PRIVATE_KEYPAIR_FILE, 0o400)
+
+ # Create fio flavor if needed
+ flavor = compute.find_flavor(FLAVOR_NAME)
+ if not flavor:
+ flavor = compute.create_flavor(
+ name=FLAVOR_NAME, ram=FLAVOR_RAM,
+ vcpus=FLAVOR_CPUS, disk=FLAVOR_DISK)
+
+ # Set image property to enable virtio-net multique in created servers
+ img = compute.find_image(UBUNTU_IMAGE_NAME)
+ compute.set_image_metadata(img.id, hw_vif_multiqueue_enabled='true')
+
+ # Create fio router if needed
+ fip_net = network.find_network(FLOATING_NET_NAME)
+ router = network.find_router(ROUTER_NAME)
+ if not router:
+ router = network.create_router(
+ name=ROUTER_NAME, external_gateway_info={'network_id': fip_net.id})
+
+ # Create fio net/subnet if needed
+ fio_net = network.find_network(NET_NAME)
+ if not fio_net:
+ fio_net = network.create_network(
+ name=NET_NAME,
+ availability_zone_hints=['nova'],
+ # mtu=MTU_SIZE,
+ shared=False,
+ port_security_enabled=True)
+ fio_subnet = network.create_subnet(
+ name=SUBNET_NAME,
+ network_id=fio_net.id,
+ cidr=SUBNET_RANGE,
+ ip_version=NET_IPV4)
+ # Add fio net to fio router
+ fio_net_port = network.add_interface_to_router(
+ router.id, subnet_id=fio_subnet.id)
+
+ # Get list of running computes with enabled 'nova-compute' service
+ cmp_services = compute.services(binary='nova-compute')
+ computes = [s for s in cmp_services if
+ s.host in NODES and
+ s.host not in SKIP_NODES and
+ s.state == 'up' and s.status == 'enabled']
+
+ # Prepare list of hypervisors to be used for running fio servers
+ hypervisors = []
+ computes_num = len(computes)
+ for i in range(CLIENTS_COUNT):
+ hypervisors.append(
+ ".".join([computes[i % computes_num].host, HV_SUFFIX]))
+
+ # Create <CLIENTS_COUNT> clients, attached to fio private network
+ vms = []
+ for i in range(CLIENTS_COUNT):
+ name = f"{CLIENT_NAME_MASK}{i}"
+ az = f"::{hypervisors[i]}"
+ flavor_id = flavor.id
+ vm = create_server(
+ name=name,
+ image_id=img.id,
+ flavor_id=flavor_id,
+ networks=[{'uuid': fio_net.id}],
+ key_name=KEYPAIR_NAME,
+ security_groups=[{'name': SG_NAME}],
+ availability_zone=az)
+ try:
+ vm = compute.wait_for_server(vm, wait=180)
+ node = hypervisors[i].split('.')[0]
+ print(f"Fio client VM '{vm.name}' is created on '{node}' node")
+ # Stop and exit if any of the servers creation failed (for any reason)
+ except ResourceFailure as e:
+ print(
+ f"Fio client VM '{vm.name}' creation failed with '{e.message}'"
+ " error.")
+ conn.delete_server(vm)
+ sys.exit(0)
+ vms.append(vm)
+
+ # Create a volume of the given type
+ vol_name = f"{VOL_NAME_MASK}{i}"
+ vol = volume.create_volume(
+ name=vol_name, size=VOL_SIZE, volume_type=VOL_TYPE)
+ try:
+ vol = volume.wait_for_status(vol, status='available')
+ print(f"Volume '{vol.name}' is created")
+ # Delete a volume if its creation failed and switch to next
+ # fio client VM
+ except ResourceFailure as e:
+ print(
+ f"Volume '{vol.name}' creation failed with '{e.message}' "
+ "error.")
+ conn.delete_volume(vol)
+ continue
+
+ # Attach the volume to the fio client
+ compute.create_volume_attachment(vm, volume=vol)
+ try:
+ vol = volume.wait_for_status(vol, status='in-use')
+ print(f"Volume '{vol.name}' is attached to '{vm.name}' fio client")
+ # Delete a volume if attachment failed and switch to next
+ # fio client VM
+ except ResourceFailure as e:
+ print(
+ f"Volume '{vol.name}' attachment failed with '{e.message}' "
+ "error.")
+ conn.delete_volume(vol)
+ continue