Update fio helpers
- Create / delete fio VMs in parallel. VMs are to be created in batches, each batch
parallelism size is controlled by a new config parameter
- A server group with 'soft-anti-affinity' policy is added to fio setup.
This way VMs will be automatically scheduled on different hosts
Related-PROD: PROD-37187
Change-Id: I0a22b1a1fe279966e8370605d84783c5c49fea50
diff --git a/fio/fio_setup.py b/fio/fio_setup.py
index d04b67b..79f1d69 100644
--- a/fio/fio_setup.py
+++ b/fio/fio_setup.py
@@ -1,9 +1,10 @@
+import multiprocessing as mp
import os
+import random
import sys
from typing import Dict, Final, List
import connection as conn
-import openstack
from openstack.exceptions import ResourceFailure
@@ -13,6 +14,7 @@
CLIENTS_COUNT: Final[int] = conn.FIO_CLIENTS_COUNT
CLIENT_NAME_MASK: Final[str] = conn.FIO_CLIENT_NAME_MASK
+AA_SERVER_GROUP_NAME: Final[str] = conn.FIO_AA_SERVER_GROUP_NAME
UBUNTU_IMAGE_NAME: Final[str] = conn.UBUNTU_IMAGE_NAME
VOL_NAME_MASK: Final[str] = conn.FIO_VOL_NAME_MASK
@@ -39,6 +41,7 @@
SG_NAME: Final[str] = conn.FIO_SG_NAME
HV_SUFFIX: Final[str] = conn.HV_SUFFIX
CLOUD_NAME: Final[str] = conn.CLOUD_NAME
+CONCURRENCY: Final[int] = conn.CONCURRENCY
NODES: Final[List[str]] = []
SKIP_NODES: Final[List[str]] = []
@@ -69,15 +72,59 @@
]
-def create_server(
- name, image_id, flavor_id, networks,
- key_name, security_groups, availability_zone
-) -> openstack.connection.Connection:
- srv = compute.create_server(
- name=name, image_id=image_id, flavor_id=flavor_id, networks=networks,
- key_name=key_name, security_groups=security_groups,
- availability_zone=availability_zone)
- return srv
+def create_fio_client(
+ image_id: str, flavor_id: str, networks: List,
+ key_name: str, security_groups: List, server_group_id: str
+) -> None:
+ rand_name = str(random.randint(1, 0x7fffffff))
+ vm_name = f"{CLIENT_NAME_MASK}-{rand_name}"
+
+ vm = compute.create_server(
+ name=vm_name,
+ image_id=image_id,
+ flavor_id=flavor_id,
+ networks=networks,
+ key_name=key_name,
+ security_groups=security_groups,
+ scheduler_hints={'group': server_group_id})
+ try:
+ vm = compute.wait_for_server(vm, wait=180)
+ print(f"Fio client '{vm.name}' is created on '{vm.compute_host}' node")
+ # Stop and exit if any of the servers creation failed (for any reason)
+ except ResourceFailure as e:
+ print(
+ f"Fio client '{vm.name}' creation failed with '{e.message}'"
+ " error.")
+ conn.delete_server(vm)
+ sys.exit(0)
+
+ # Create a volume of the given type
+ vol_name = f"{VOL_NAME_MASK}-{rand_name}"
+ vol = volume.create_volume(
+ name=vol_name, size=VOL_SIZE, volume_type=VOL_TYPE)
+ try:
+ vol = volume.wait_for_status(vol, status='available')
+ print(f"Volume '{vol.name}' is created")
+ # Delete a volume if its creation failed and switch to next
+ # fio client VM
+ except ResourceFailure as e:
+ print(
+ f"Volume '{vol.name}' creation failed with '{e.message}' "
+ "error.")
+ conn.delete_volume(vol)
+
+ # Attach the volume to the fio client
+ compute.create_volume_attachment(vm, volume=vol)
+ try:
+ vol = volume.wait_for_status(vol, status='in-use')
+ print(f"Volume '{vol.name}' is attached to '{vm.name}' fio client")
+ # Delete a volume if attachment failed and switch to next
+ # fio client VM
+ except ResourceFailure as e:
+ print(
+ f"Volume '{vol.name}' attachment failed with '{e.message}' "
+ "error.")
+ conn.delete_volume(vol)
if __name__ == "__main__":
@@ -91,7 +138,8 @@
sys.exit(0)
# Create fio sg if needed
- sg = network.find_security_group(SG_NAME)
+ project_id = conn.cloud.auth['project_id']
+ sg = network.find_security_group(SG_NAME, project_id=project_id)
if not sg:
sg = network.create_security_group(name=SG_NAME)
# Add 'allow-all' kind of rules to the security group
@@ -146,73 +194,25 @@
fio_net_port = network.add_interface_to_router(
router.id, subnet_id=fio_subnet.id)
- # Get list of running computes with enabled 'nova-compute' service
- cmp_services = compute.services(binary='nova-compute')
- computes = [s for s in cmp_services if
- s.host in NODES and
- s.host not in SKIP_NODES and
- s.state == 'up' and s.status == 'enabled']
+ # Create fio server group with anti-affinity scheduling policy
+ server_group = compute.find_server_group(
+ AA_SERVER_GROUP_NAME, all_projects=True)
+ if not server_group:
+ server_group = compute.create_server_group(
+ name=AA_SERVER_GROUP_NAME, policies=['soft-anti-affinity'])
- # Prepare list of hypervisors to be used for running fio servers
- hypervisors = []
- computes_num = len(computes)
- for i in range(CLIENTS_COUNT):
- hypervisors.append(
- ".".join([computes[i % computes_num].host, HV_SUFFIX]))
+ vm_kwargs = dict(
+ image_id=img.id,
+ flavor_id=flavor.id,
+ networks=[{'uuid': fio_net.id}],
+ key_name=KEYPAIR_NAME,
+ security_groups=[{'name': SG_NAME}],
+ server_group_id=server_group.id)
- # Create <CLIENTS_COUNT> clients, attached to fio private network
- vms = []
- for i in range(CLIENTS_COUNT):
- name = f"{CLIENT_NAME_MASK}{i}"
- az = f"::{hypervisors[i]}"
- flavor_id = flavor.id
- vm = create_server(
- name=name,
- image_id=img.id,
- flavor_id=flavor_id,
- networks=[{'uuid': fio_net.id}],
- key_name=KEYPAIR_NAME,
- security_groups=[{'name': SG_NAME}],
- availability_zone=az)
- try:
- vm = compute.wait_for_server(vm, wait=180)
- node = hypervisors[i].split('.')[0]
- print(f"Fio client VM '{vm.name}' is created on '{node}' node")
- # Stop and exit if any of the servers creation failed (for any reason)
- except ResourceFailure as e:
- print(
- f"Fio client VM '{vm.name}' creation failed with '{e.message}'"
- " error.")
- conn.delete_server(vm)
- sys.exit(0)
- vms.append(vm)
-
- # Create a volume of the given type
- vol_name = f"{VOL_NAME_MASK}{i}"
- vol = volume.create_volume(
- name=vol_name, size=VOL_SIZE, volume_type=VOL_TYPE)
- try:
- vol = volume.wait_for_status(vol, status='available')
- print(f"Volume '{vol.name}' is created")
- # Delete a volume if its creation failed and switch to next
- # fio client VM
- except ResourceFailure as e:
- print(
- f"Volume '{vol.name}' creation failed with '{e.message}' "
- "error.")
- conn.delete_volume(vol)
- continue
-
- # Attach the volume to the fio client
- compute.create_volume_attachment(vm, volume=vol)
- try:
- vol = volume.wait_for_status(vol, status='in-use')
- print(f"Volume '{vol.name}' is attached to '{vm.name}' fio client")
- # Delete a volume if attachment failed and switch to next
- # fio client VM
- except ResourceFailure as e:
- print(
- f"Volume '{vol.name}' attachment failed with '{e.message}' "
- "error.")
- conn.delete_volume(vol)
- continue
+ # Create fio client VMs in parallel in batches of CONCURRENCY size
+ with mp.Pool(processes=CONCURRENCY) as pool:
+ results = [
+ pool.apply_async(create_fio_client, kwds=vm_kwargs)
+ for _ in range(CLIENTS_COUNT)]
+ # Wait for batch of fio client VMs to be created
+ _ = [r.get() for r in results]