diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 4b8f60b..5eac133 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -16,9 +16,11 @@
     fi
 
     go_path=$(find $DEST/tempest/.tox/tempest/ -name test_server.go)
-    bin_path=${go_path%.go}.bin
+    sudo mkdir -m755 -p /opt/octavia-tempest-plugin
+    sudo chown $STACK_USER /opt/octavia-tempest-plugin
     CGO_ENABLED=0 GOOS=linux go build \
-        -a -ldflags '-s -w -extldflags -static' -o $bin_path \
+        -a -ldflags '-s -w -extldflags -static' \
+        -o /opt/octavia-tempest-plugin/test_server.bin \
         ${DEST}/octavia-tempest-plugin/octavia_tempest_plugin/contrib/test_server/test_server.go
 }
 
@@ -26,11 +28,11 @@
     case "$2" in
         install)
             # Install dev library if
-            # - the release is more recent than stein (devstack in stein would
-            #   try to install it in a python2 env, but octavia-tempest-plugin is
-            #   now a python3-only project)
+            # - the release is more recent than train (devstack in train would
+            #   try to install it in a python2 env, but octavia-tempest-plugin
+            #   is now a python3-only project)
             # - or the user explicitly requests it (INSTALL_TEMPEST=True)
-            if [[ "$DEVSTACK_SERIES" != "stein" ]] || [[ "$(trueorfalse False INSTALL_TEMPEST)" == "True" ]]; then
+            if [[ ! "$DEVSTACK_SERIES" =~ (stein|train) ]] || [[ "$(trueorfalse False INSTALL_TEMPEST)" == "True" ]]; then
                 echo_summary "Installing octavia-tempest-plugin"
                 install_octavia_tempest_plugin
             fi
diff --git a/octavia_tempest_plugin/config.py b/octavia_tempest_plugin/config.py
index 77d2f6e..f44bf96 100644
--- a/octavia_tempest_plugin/config.py
+++ b/octavia_tempest_plugin/config.py
@@ -213,6 +213,10 @@
                default='/var/log/octavia-amphora.log',
                help='File path, on the tempest system, to the amphora admin '
                     'log file.'),
+    cfg.StrOpt('test_server_path',
+               default='/opt/octavia-tempest-plugin/test_server.bin',
+               help='Filesystem path to the test web server that will be '
+                    'installed in the web server VMs.'),
 ]
 
 lb_feature_enabled_group = cfg.OptGroup(name='loadbalancer-feature-enabled',
diff --git a/octavia_tempest_plugin/tests/test_base.py b/octavia_tempest_plugin/tests/test_base.py
index f260e88..669a33c 100644
--- a/octavia_tempest_plugin/tests/test_base.py
+++ b/octavia_tempest_plugin/tests/test_base.py
@@ -13,7 +13,6 @@
 #    under the License.
 
 import ipaddress
-import pkg_resources
 import random
 import shlex
 import string
@@ -849,8 +848,7 @@
 
     @classmethod
     def _install_start_webserver(cls, ip_address, ssh_key, start_id):
-        local_file = pkg_resources.resource_filename(
-            'octavia_tempest_plugin.contrib.test_server', 'test_server.bin')
+        local_file = CONF.load_balancer.test_server_path
         dest_file = '/dev/shm/test_server.bin'
 
         linux_client = remote_client.RemoteClient(
diff --git a/octavia_tempest_plugin/tests/waiters.py b/octavia_tempest_plugin/tests/waiters.py
index e0d9d2d..fa6c112 100644
--- a/octavia_tempest_plugin/tests/waiters.py
+++ b/octavia_tempest_plugin/tests/waiters.py
@@ -68,7 +68,7 @@
             LOG.info('{name}\'s status updated to {status}.'.format(
                 name=show_client.__name__, status=status))
             return object_details
-        elif object_details[status_key] == 'ERROR':
+        elif object_details[status_key] == 'ERROR' and not error_ok:
             message = ('{name} {field} updated to an invalid state of '
                        'ERROR'.format(name=show_client.__name__,
                                       field=status_key))
@@ -76,9 +76,9 @@
             if caller:
                 message = '({caller}) {message}'.format(caller=caller,
                                                         message=message)
-            if not error_ok:
-                raise exceptions.UnexpectedResponseCode(message)
-        elif int(time.time()) - start >= check_timeout:
+            raise exceptions.UnexpectedResponseCode(message)
+
+        if int(time.time()) - start >= check_timeout:
             message = (
                 '{name} {field} failed to update to {expected_status} within '
                 'the required time {timeout}. Current status of {name}: '
diff --git a/releasenotes/notes/test-server-path-3845f619090ba016.yaml b/releasenotes/notes/test-server-path-3845f619090ba016.yaml
new file mode 100644
index 0000000..51f79dd
--- /dev/null
+++ b/releasenotes/notes/test-server-path-3845f619090ba016.yaml
@@ -0,0 +1,7 @@
+---
+other:
+  - |
+    The Octavia tempest plugin now as a configuration setting for the path to
+    the test server. By default it will expect the test server to now be
+    located in /opt/octavia-tempest-plugin/test_server.bin. The devstack
+    plugin has been updated to place the test_server.bin in that location.
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index afffbd2..0918602 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -182,6 +182,7 @@
     required-projects:
       - openstack/diskimage-builder
     vars:
+      configure_swap_size: 8192
       devstack_localrc:
         DIB_LOCAL_ELEMENTS: openstack-ci-mirrors
         LIBVIRT_TYPE: kvm
@@ -219,6 +220,7 @@
     required-projects:
       - openstack/diskimage-builder
     vars:
+      configure_swap_size: 8192
       devstack_localrc:
         DIB_LOCAL_ELEMENTS: openstack-ci-mirrors
       devstack_local_conf:
@@ -253,6 +255,7 @@
       - openstack/diskimage-builder
     host-vars:
       controller:
+        configure_swap_size: 8192
         devstack_localrc:
           # From devstack "vars:"
           DATABASE_USER: octavia
@@ -294,6 +297,7 @@
           '/var/log/octavia-amphora.log': logs
           '/var/log/octavia-tenant-traffic.log': logs
       controller2:
+        configure_swap_size: 8192
         devstack_localrc:
           # From devstack "vars:"
           DATABASE_USER: octavia