Introduce PINGABLE and SSHABLE waiters and wait_until state support
This change introduces two new instance state waiters and uses them to
extend the existing tempest.common.compute.create_test_server wait_until
state support.
They are being introduced in an effort to allow the guest OS time to
start *before* we start attempting to interact with it either directly
by connecting to the instance or indirectly by hot-plugging or
hot-unplugging devices. The latter on some virt backends being an issue
if the guest OS is unable to respond to the underlying ACPI requests
sent to it.
It should be noted that these new states rely on the instance already
being ACTIVE before we begin to wait for the instance to either become
pingable or accessible over SSH. This is taken into account and will
always happen for these states even if validation isn't enabled in the
environment and thus it isn't possible to wait until the instance is
pingable or accessible over ssh.
Change-Id: Ib14fa7dc5c8093eed498049cd0a56c8ac6853660
diff --git a/tempest/common/compute.py b/tempest/common/compute.py
index d34cd6d..43e30ad 100644
--- a/tempest/common/compute.py
+++ b/tempest/common/compute.py
@@ -23,6 +23,7 @@
from oslo_log import log as logging
from oslo_utils import excutils
+from tempest.common.utils.linux import remote_client
from tempest.common import waiters
from tempest import config
from tempest import exceptions
@@ -98,7 +99,9 @@
server. Include a keypair, a security group and an IP.
:param tenant_network: Tenant network to be used for creating a server.
:param wait_until: Server status to wait for the server to reach after
- its creation.
+ its creation. Additionally PINGABLE and SSHABLE states are also
+ accepted when the server is both validatable and has the required
+ validation_resources provided.
:param volume_backed: Whether the server is volume backed or not.
If this is true, a volume will be created and create server will be
requested with 'block_device_mapping_v2' populated with below values:
@@ -125,8 +128,6 @@
:returns: a tuple
"""
- # TODO(jlanoux) add support of wait_until PINGABLE/SSHABLE
-
if name is None:
name = data_utils.rand_name(__name__ + "-instance")
if flavor is None:
@@ -259,18 +260,50 @@
server_id=servers[0]['id'])
if wait_until:
+
+ # NOTE(lyarwood): PINGABLE and SSHABLE both require the instance to
+ # go ACTIVE initially before we can setup the fip(s) etc so stash
+ # this additional wait state for later use.
+ wait_until_extra = None
+ if wait_until in ['PINGABLE', 'SSHABLE']:
+ wait_until_extra = wait_until
+ wait_until = 'ACTIVE'
+
for server in servers:
try:
waiters.wait_for_server_status(
clients.servers_client, server['id'], wait_until,
request_id=request_id)
- # Multiple validatable servers are not supported for now. Their
- # creation will fail with the condition above.
if CONF.validation.run_validation and validatable:
+
if CONF.validation.connect_method == 'floating':
_setup_validation_fip()
+ server_ip = get_server_ip(
+ server, validation_resources=validation_resources)
+
+ if wait_until_extra == 'PINGABLE':
+ waiters.wait_for_ping(
+ server_ip,
+ clients.servers_client.build_timeout,
+ clients.servers_client.build_interval
+ )
+
+ if wait_until_extra == 'SSHABLE':
+ pkey = validation_resources['keypair']['private_key']
+ ssh_client = remote_client.RemoteClient(
+ server_ip,
+ CONF.validation.image_ssh_user,
+ pkey=pkey,
+ server=server,
+ servers_client=clients.servers_client
+ )
+ waiters.wait_for_ssh(
+ ssh_client,
+ clients.servers_client.build_timeout
+ )
+
except Exception:
with excutils.save_and_reraise_exception():
for server in servers:
diff --git a/tempest/common/waiters.py b/tempest/common/waiters.py
index fbc8698..ab401fb 100644
--- a/tempest/common/waiters.py
+++ b/tempest/common/waiters.py
@@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import os
import re
import time
@@ -570,3 +571,26 @@
'in time.' % (floating_ip, server['id']))
raise lib_exc.TimeoutException(msg)
time.sleep(servers_client.build_interval)
+
+
+def wait_for_ping(server_ip, timeout=30, interval=1):
+ """Waits for an address to become pingable"""
+ start_time = int(time.time())
+ while int(time.time()) - start_time < timeout:
+ response = os.system("ping -c 1 " + server_ip)
+ if response == 0:
+ return
+ time.sleep(interval)
+ raise lib_exc.TimeoutException()
+
+
+def wait_for_ssh(ssh_client, timeout=30):
+ """Waits for SSH connection to become usable"""
+ start_time = int(time.time())
+ while int(time.time()) - start_time < timeout:
+ try:
+ ssh_client.validate_authentication()
+ return
+ except lib_exc.SSHTimeout:
+ pass
+ raise lib_exc.TimeoutException()
diff --git a/tempest/tests/common/test_waiters.py b/tempest/tests/common/test_waiters.py
index 5b0acfa..1d0ee77 100755
--- a/tempest/tests/common/test_waiters.py
+++ b/tempest/tests/common/test_waiters.py
@@ -523,6 +523,70 @@
mock_list_volume_attachments.assert_called_once_with(
mock.sentinel.server_id)
+ @mock.patch('os.system')
+ def test_wait_for_ping_host_alive(self, mock_ping):
+ mock_ping.return_value = 0
+ # Assert that nothing is raised as the host is alive
+ waiters.wait_for_ping('127.0.0.1', 10, 1)
+
+ @mock.patch('os.system')
+ def test_wait_for_ping_host_eventually_alive(self, mock_ping):
+ mock_ping.side_effect = [1, 1, 0]
+ # Assert that nothing is raised when the host is eventually alive
+ waiters.wait_for_ping('127.0.0.1', 10, 1)
+
+ @mock.patch('os.system')
+ def test_wait_for_ping_timeout(self, mock_ping):
+ mock_ping.return_value = 1
+ # Assert that TimeoutException is raised when the host is dead
+ self.assertRaises(
+ lib_exc.TimeoutException,
+ waiters.wait_for_ping,
+ '127.0.0.1',
+ .1,
+ .1
+ )
+
+ def test_wait_for_ssh(self):
+ mock_ssh_client = mock.Mock()
+ mock_ssh_client.validate_authentication.return_value = True
+ # Assert that nothing is raised when validate_authentication returns
+ waiters.wait_for_ssh(mock_ssh_client, .1)
+ mock_ssh_client.validate_authentication.assert_called_once()
+
+ def test_wait_for_ssh_eventually_up(self):
+ mock_ssh_client = mock.Mock()
+ timeout = lib_exc.SSHTimeout(
+ host='foo',
+ username='bar',
+ password='fizz'
+ )
+ mock_ssh_client.validate_authentication.side_effect = [
+ timeout,
+ timeout,
+ True
+ ]
+ # Assert that nothing is raised if validate_authentication passes
+ # before the timeout
+ waiters.wait_for_ssh(mock_ssh_client, 10)
+
+ def test_wait_for_ssh_timeout(self):
+ mock_ssh_client = mock.Mock()
+ timeout = lib_exc.SSHTimeout(
+ host='foo',
+ username='bar',
+ password='fizz'
+ )
+ mock_ssh_client.validate_authentication.side_effect = timeout
+ # Assert that TimeoutException is raised when validate_authentication
+ # doesn't pass in time.
+ self.assertRaises(
+ lib_exc.TimeoutException,
+ waiters.wait_for_ssh,
+ mock_ssh_client,
+ .1
+ )
+
class TestServerFloatingIPWaiters(base.TestCase):