Add an active/standby scenario test
This patch adds a tempest scenario test for the active/standby topology.
It starts an active/standby load balancer, deletes the master amphora,
and validates that the load balancer is still working.
Change-Id: Ibcd5552a67cea650edc72bfaa986357267ca2407
diff --git a/octavia_tempest_plugin/config.py b/octavia_tempest_plugin/config.py
index 7c4ed2f..63c292b 100644
--- a/octavia_tempest_plugin/config.py
+++ b/octavia_tempest_plugin/config.py
@@ -66,7 +66,7 @@
default=5,
help='Interval to check for status changes.'),
cfg.IntOpt('check_timeout',
- default=60,
+ default=120,
help='Timeout, in seconds, to wait for a status change.'),
cfg.BoolOpt('test_with_noop',
default=False,
diff --git a/octavia_tempest_plugin/tests/act_stdby_scenario/v2/test_active_standby.py b/octavia_tempest_plugin/tests/act_stdby_scenario/v2/test_active_standby.py
new file mode 100644
index 0000000..612bbe2
--- /dev/null
+++ b/octavia_tempest_plugin/tests/act_stdby_scenario/v2/test_active_standby.py
@@ -0,0 +1,287 @@
+# Copyright 2018 GoDaddy
+# Copyright 2018 Rackspace US Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import requests
+import testtools
+import time
+
+from oslo_log import log as logging
+from tempest import config
+from tempest.lib.common.utils import data_utils
+from tempest.lib import decorators
+from tempest.lib import exceptions
+
+from octavia_tempest_plugin.common import constants as const
+from octavia_tempest_plugin.tests import test_base
+from octavia_tempest_plugin.tests import waiters
+
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
+
+
+@testtools.skipUnless(
+ CONF.validation.run_validation,
+ 'Active-Standby tests will not work without run_validation enabled.')
+class ActiveStandbyScenarioTest(test_base.LoadBalancerBaseTestWithCompute):
+
+ @classmethod
+ def resource_setup(cls):
+ """Setup resources needed by the tests."""
+ super(ActiveStandbyScenarioTest, cls).resource_setup()
+
+ # We have to do this here as the api_version and clients are not
+ # setup in time to use a decorator or the skip_checks mixin
+ if not cls.lb_admin_flavor_profile_client.is_version_supported(
+ cls.api_version, '2.3'):
+ return
+
+ lb_name = data_utils.rand_name("lb_member_lb1_actstdby")
+ lb_kwargs = {const.PROVIDER: CONF.load_balancer.provider,
+ const.NAME: lb_name}
+
+ # TODO(rm_work): Make this work with ipv6 and split this test for both
+ ip_version = 4
+ cls._setup_lb_network_kwargs(lb_kwargs, ip_version)
+
+ lb = cls.mem_lb_client.create_loadbalancer(**lb_kwargs)
+ cls.lb_id = lb[const.ID]
+ cls.addClassResourceCleanup(
+ cls.mem_lb_client.cleanup_loadbalancer,
+ cls.lb_id)
+
+ if CONF.validation.connect_method == 'floating':
+ port_id = lb[const.VIP_PORT_ID]
+ result = cls.lb_mem_float_ip_client.create_floatingip(
+ floating_network_id=CONF.network.public_network_id,
+ port_id=port_id)
+ floating_ip = result['floatingip']
+ LOG.info('lb1_floating_ip: {}'.format(floating_ip))
+ cls.addClassResourceCleanup(
+ waiters.wait_for_not_found,
+ cls.lb_mem_float_ip_client.delete_floatingip,
+ cls.lb_mem_float_ip_client.show_floatingip,
+ floatingip_id=floating_ip['id'])
+ cls.lb_vip_address = floating_ip['floating_ip_address']
+ else:
+ cls.lb_vip_address = lb[const.VIP_ADDRESS]
+
+ waiters.wait_for_status(cls.mem_lb_client.show_loadbalancer,
+ cls.lb_id, const.PROVISIONING_STATUS,
+ const.ACTIVE,
+ CONF.load_balancer.lb_build_interval,
+ CONF.load_balancer.lb_build_timeout)
+
+ listener_name = data_utils.rand_name("lb_member_listener1_actstdby")
+ listener_kwargs = {
+ const.NAME: listener_name,
+ const.PROTOCOL: const.HTTP,
+ const.PROTOCOL_PORT: '80',
+ const.LOADBALANCER_ID: cls.lb_id,
+ }
+ listener = cls.mem_listener_client.create_listener(**listener_kwargs)
+ cls.listener_id = listener[const.ID]
+ cls.addClassResourceCleanup(
+ cls.mem_listener_client.cleanup_listener,
+ cls.listener_id,
+ lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
+
+ waiters.wait_for_status(cls.mem_lb_client.show_loadbalancer,
+ cls.lb_id, const.PROVISIONING_STATUS,
+ const.ACTIVE,
+ CONF.load_balancer.build_interval,
+ CONF.load_balancer.build_timeout)
+
+ pool_name = data_utils.rand_name("lb_member_pool1_actstdby")
+ pool_kwargs = {
+ const.NAME: pool_name,
+ const.PROTOCOL: const.HTTP,
+ const.LB_ALGORITHM: const.LB_ALGORITHM_ROUND_ROBIN,
+ const.LISTENER_ID: cls.listener_id,
+ }
+ pool = cls.mem_pool_client.create_pool(**pool_kwargs)
+ cls.pool_id = pool[const.ID]
+ cls.addClassResourceCleanup(
+ cls.mem_pool_client.cleanup_pool,
+ cls.pool_id,
+ lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
+
+ waiters.wait_for_status(cls.mem_lb_client.show_loadbalancer,
+ cls.lb_id, const.PROVISIONING_STATUS,
+ const.ACTIVE,
+ CONF.load_balancer.build_interval,
+ CONF.load_balancer.build_timeout)
+
+ # Set up Member 1 for Webserver 1
+ member1_name = data_utils.rand_name("lb_member_member1_actstdby")
+ member1_kwargs = {
+ const.POOL_ID: cls.pool_id,
+ const.NAME: member1_name,
+ const.ADMIN_STATE_UP: True,
+ const.ADDRESS: cls.webserver1_ip,
+ const.PROTOCOL_PORT: 80,
+ }
+ if cls.lb_member_1_subnet:
+ member1_kwargs[const.SUBNET_ID] = cls.lb_member_1_subnet[const.ID]
+
+ member1 = cls.mem_member_client.create_member(
+ **member1_kwargs)
+ cls.addClassResourceCleanup(
+ cls.mem_member_client.cleanup_member,
+ member1[const.ID], pool_id=cls.pool_id,
+ lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
+ waiters.wait_for_status(
+ cls.mem_lb_client.show_loadbalancer, cls.lb_id,
+ const.PROVISIONING_STATUS, const.ACTIVE,
+ CONF.load_balancer.check_interval,
+ CONF.load_balancer.check_timeout)
+
+ # Set up Member 2 for Webserver 2
+ member2_name = data_utils.rand_name("lb_member_member2_actstdby")
+ member2_kwargs = {
+ const.POOL_ID: cls.pool_id,
+ const.NAME: member2_name,
+ const.ADMIN_STATE_UP: True,
+ const.ADDRESS: cls.webserver2_ip,
+ const.PROTOCOL_PORT: 80,
+ }
+ if cls.lb_member_2_subnet:
+ member2_kwargs[const.SUBNET_ID] = cls.lb_member_2_subnet[const.ID]
+
+ member2 = cls.mem_member_client.create_member(
+ **member2_kwargs)
+ cls.addClassResourceCleanup(
+ cls.mem_member_client.cleanup_member,
+ member2[const.ID], pool_id=cls.pool_id,
+ lb_client=cls.mem_lb_client, lb_id=cls.lb_id)
+ waiters.wait_for_status(
+ cls.mem_lb_client.show_loadbalancer, cls.lb_id,
+ const.PROVISIONING_STATUS, const.ACTIVE,
+ CONF.load_balancer.check_interval,
+ CONF.load_balancer.check_timeout)
+
+ @testtools.skipIf(CONF.load_balancer.test_with_noop,
+ 'Active/Standby tests will not work in noop mode.')
+ @decorators.idempotent_id('e591fa7a-0eee-485a-8ca0-5cf1a556bdf0')
+ def test_active_standby_vrrp_failover(self):
+ """Tests active/standby VRRP failover
+
+ * Test the load balancer to make sure it is functioning
+ * Identifies the Master and Backup amphora
+ * Deletes the Master amphora
+ * Sends traffic through the load balancer
+ * Validates that the Backup has assumed the Master role
+ """
+ # We have to do this here as the api_version and clients are not
+ # setup in time to use a decorator or the skip_checks mixin
+ if not self.mem_listener_client.is_version_supported(
+ self.api_version, '2.4'):
+ raise self.skipException(
+ 'Active/Standby VRRP failover tests require '
+ 'Octavia API version 2.3 or newer.')
+
+ session = requests.Session()
+
+ # Send some traffic
+ self.check_members_balanced(self.lb_vip_address)
+
+ # Get the amphorae associated with this load balancer
+ amphorae = self.os_admin.amphora_client.list_amphorae(
+ query_params='{loadbalancer_id}={lb_id}'.format(
+ loadbalancer_id=const.LOADBALANCER_ID,
+ lb_id=self.lb_id))
+
+ # TODO(johnsom): Fix when LB flavors support act/stdby
+ if len(amphorae) < 2:
+ self.skipTest('Load balancer must be using active/standby '
+ 'topology for the VRRP failover test.')
+
+ # Generate traffic on the LB so we can identify the current Master
+ r = session.get('http://{0}'.format(self.lb_vip_address), timeout=2)
+
+ # Cycle through the amps to find the master
+ master_amp = None
+ backup_amp = None
+ start = int(time.time())
+ while True:
+ for amp in amphorae:
+ amphora_stats = self.os_admin.amphora_client.get_amphora_stats(
+ amp[const.ID])
+ for listener in amphora_stats:
+ if listener[const.TOTAL_CONNECTIONS] > 0:
+ master_amp = amp
+ break
+ # check if we left the listener for loop by finding the master
+ if master_amp:
+ break
+ # If we found the master and broke out of the amp for loop, break
+ # out of the while loop too.
+ if master_amp:
+ break
+ if int(time.time()) - start >= CONF.load_balancer.check_timeout:
+ message = ('Unable to find Master amphora in {timeout} '
+ 'seconds.'.format(
+ timeout=CONF.load_balancer.check_timeout))
+ raise exceptions.TimeoutException(message)
+ time.sleep(CONF.load_balancer.check_interval)
+
+ # Find the backup amphora and check it is ready for the test
+ for amp in amphorae:
+ if amp[const.ID] == master_amp[const.ID]:
+ continue
+ else:
+ backup_amp = amp
+ self.assertIsNotNone(backup_amp)
+ amphora_stats = self.os_admin.amphora_client.get_amphora_stats(
+ backup_amp[const.ID])
+ for listener in amphora_stats:
+ self.assertEqual(0, listener[const.TOTAL_CONNECTIONS])
+
+ # Delete the master amphora compute instance
+ self.os_admin_servers_client.delete_server(
+ master_amp[const.COMPUTE_ID])
+
+ # Pass some traffic through the LB
+ # Note: We want this to loop for longer than the heartbeat interval
+ # to make sure a stats update has come in to the HM
+ for x in range(0, 20):
+ try:
+ r = session.get('http://{0}'.format(self.lb_vip_address),
+ timeout=1)
+ LOG.info('Got response: %s', r.text)
+ except Exception:
+ LOG.info('Load balancer request failed. Looping')
+ time.sleep(1)
+
+ # Check that the Backup amphora is now Master
+ amphora_stats = self.os_admin.amphora_client.get_amphora_stats(
+ backup_amp[const.ID])
+ connections = 0
+ for listener in amphora_stats:
+ connections += listener[const.TOTAL_CONNECTIONS]
+ self.assertTrue(connections > 0)
+ LOG.info('Backup amphora is now Master.')
+ # Wait for the amphora failover to start
+ waiters.wait_for_status(
+ self.mem_lb_client.show_loadbalancer,
+ self.lb_id, const.PROVISIONING_STATUS,
+ const.PENDING_UPDATE, CONF.load_balancer.check_interval,
+ CONF.load_balancer.check_timeout)
+ # Wait for the load balancer to return to ACTIVE so the
+ # cleanup steps will pass
+ waiters.wait_for_status(
+ self.mem_lb_client.show_loadbalancer,
+ self.lb_id, const.PROVISIONING_STATUS,
+ const.ACTIVE, CONF.load_balancer.lb_build_interval,
+ CONF.load_balancer.lb_build_timeout)
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index d342aff..aac2460 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -556,7 +556,6 @@
loadbalancer_topology: ACTIVE_STANDBY
task_flow:
engine: parallel
- devstack_local_conf:
test-config:
"$TEMPEST_CONFIG":
load_balancer:
@@ -608,3 +607,32 @@
name: octavia-v2-act-stdby-iptables-dsvm-py2-scenario-stable-queens
parent: octavia-v2-act-stdby-iptables-dsvm-py2-scenario
override-checkout: stable/queens
+
+- job:
+ name: octavia-v2-act-stdby-dsvm-scenario
+ parent: octavia-dsvm-live-base
+ vars:
+ devstack_local_conf:
+ post-config:
+ $OCTAVIA_CONF:
+ api_settings:
+ api_v1_enabled: False
+ controller_worker:
+ loadbalancer_topology: ACTIVE_STANDBY
+ task_flow:
+ engine: parallel
+ tempest_concurrency: 2
+ tempest_test_regex: ^octavia_tempest_plugin.tests.act_stdby_scenario.v2.test_active_standby\.
+ tox_envlist: all
+
+- job:
+ name: octavia-v2-act-stdby-dsvm-py2-scenario
+ parent: octavia-v2-act-stdby-dsvm-scenario
+ vars:
+ devstack_localrc:
+ USE_PYTHON3: False
+
+- job:
+ name: octavia-v2-act-stdby-dsvm-scenario-stable-stein
+ parent: octavia-v2-act-stdby-dsvm-scenario
+ override-checkout: stable/stein
diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml
index 490d68b..4b36f01 100644
--- a/zuul.d/projects.yaml
+++ b/zuul.d/projects.yaml
@@ -38,6 +38,12 @@
voting: false
- octavia-v2-act-stdby-iptables-dsvm-py2-scenario-stable-queens:
voting: false
+ - octavia-v2-act-stdby-dsvm-scenario:
+ voting: false
+ - octavia-v2-act-stdby-dsvm-py2-scenario:
+ voting: false
+ - octavia-v2-act-stdby-dsvm-scenario-stable-stein:
+ voting: false
- octavia-v2-dsvm-tls-barbican:
voting: false
- octavia-v2-dsvm-tls-barbican-stable-stein: