diff --git a/devstack/README.rst b/devstack/README.rst
new file mode 100644
index 0000000..e605fcf
--- /dev/null
+++ b/devstack/README.rst
@@ -0,0 +1,21 @@
+====================
+Enabling in Devstack
+====================
+
+**WARNING**: the stack.sh script must be run in a disposable VM that is not
+being created automatically, see the README.md file in the "devstack"
+repository.  See contrib/vagrant to create a vagrant VM.
+
+1. Download DevStack::
+
+    git clone https://git.openstack.org/openstack-dev/devstack.git
+    cd devstack
+
+2. Add this repo as an external repository::
+
+     > cat local.conf
+     [[local|localrc]]
+     enable_plugin neutron-tempest-plugin https://git.openstack.org/openstack/neutron-tempest-plugin
+
+3. run ``stack.sh``
+
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
new file mode 100644
index 0000000..a2009ea
--- /dev/null
+++ b/devstack/plugin.sh
@@ -0,0 +1,13 @@
+# install_neutron_tempest_plugin
+function install_neutron_tempest_plugin {
+    setup_dev_lib "neutron-tempest-plugin"
+}
+
+if [[ "$1" == "stack" ]]; then
+    case "$2" in
+        install)
+            echo_summary "Installing neutron-tempest-plugin"
+            install_neutron_tempest_plugin
+            ;;
+    esac
+fi
diff --git a/devstack/settings b/devstack/settings
new file mode 100644
index 0000000..614376f
--- /dev/null
+++ b/devstack/settings
@@ -0,0 +1,3 @@
+GITREPO["neutron-tempest-plugin"]=${NEUTRON_TEMPEST_REPO:-${GIT_BASE}/openstack/neutron-tempest-plugin.git}
+GITDIR["neutron-tempest-plugin"]=$DEST/neutron-tempest-plugin
+GITBRANCH["neutron-tempest-plugin"]=master
diff --git a/neutron_tempest_plugin/api/admin/test_external_network_extension.py b/neutron_tempest_plugin/api/admin/test_external_network_extension.py
index b59d10e..cd4635c 100644
--- a/neutron_tempest_plugin/api/admin/test_external_network_extension.py
+++ b/neutron_tempest_plugin/api/admin/test_external_network_extension.py
@@ -110,6 +110,47 @@
         body = self.admin_client.show_network(net_id)['network']
         self.assertTrue(body['router:external'])
 
+    @decorators.idempotent_id('beddbe9d-304c-4577-8bbe-9b61d0a449b4')
+    def test_external_conversion_on_policy_delete(self):
+        net_id = self._create_network(external=False)['id']
+        policy = self.admin_client.create_rbac_policy(
+            object_type='network', object_id=net_id,
+            action='access_as_external',
+            target_tenant='*')
+        body = self.admin_client.show_network(net_id)['network']
+        self.assertTrue(body['router:external'])
+        self.admin_client.delete_rbac_policy(policy['rbac_policy']['id'])
+        body = self.admin_client.show_network(net_id)['network']
+        self.assertFalse(body['router:external'])
+
+    @decorators.idempotent_id('fb938b8e-d1d2-4bf8-823a-1b3c79c98a25')
+    def test_external_conversion_on_one_policy_delete(self):
+        net_id = self._create_network(external=False)['id']
+        self.admin_client.create_rbac_policy(
+            object_type='network', object_id=net_id,
+            action='access_as_external',
+            target_tenant=self.admin_client.tenant_id)
+        body = self.admin_client.show_network(net_id)['network']
+        self.assertTrue(body['router:external'])
+        policy2 = self.admin_client.create_rbac_policy(
+            object_type='network', object_id=net_id,
+            action='access_as_external',
+            target_tenant=self.client2.tenant_id)
+        self.admin_client.delete_rbac_policy(policy2['rbac_policy']['id'])
+        body = self.admin_client.show_network(net_id)['network']
+        self.assertTrue(body['router:external'])
+
+    @decorators.idempotent_id('046aff1a-3962-4e2b-9a3b-93a24f255fd0')
+    def test_external_network_on_shared_policy_delete(self):
+        net_id = self._create_network(external=True)['id']
+        policy = self.admin_client.create_rbac_policy(
+            object_type='network', object_id=net_id,
+            action='access_as_shared',
+            target_tenant='*')
+        self.admin_client.delete_rbac_policy(policy['rbac_policy']['id'])
+        body = self.admin_client.show_network(net_id)['network']
+        self.assertTrue(body['router:external'])
+
     @decorators.idempotent_id('01364c50-bfb6-46c4-b44c-edc4564d61cf')
     def test_policy_allows_tenant_to_allocate_floatingip(self):
         net = self._create_network(external=False)
diff --git a/neutron_tempest_plugin/api/admin/test_floating_ips_admin_actions.py b/neutron_tempest_plugin/api/admin/test_floating_ips_admin_actions.py
index 7601e7a..3607060 100644
--- a/neutron_tempest_plugin/api/admin/test_floating_ips_admin_actions.py
+++ b/neutron_tempest_plugin/api/admin/test_floating_ips_admin_actions.py
@@ -47,6 +47,7 @@
         body = self.client.create_floatingip(
             floating_network_id=self.ext_net_id)
         floating_ip = body['floatingip']
+        self.addCleanup(self.client.delete_floatingip, floating_ip['id'])
         project_id = self.create_project()['id']
 
         port = self.admin_client.create_port(network_id=self.network['id'],
diff --git a/neutron_tempest_plugin/common/utils.py b/neutron_tempest_plugin/common/utils.py
index ecccd18..d6d0aee 100644
--- a/neutron_tempest_plugin/common/utils.py
+++ b/neutron_tempest_plugin/common/utils.py
@@ -19,6 +19,7 @@
 """Utilities and helper functions."""
 
 import eventlet
+import functools
 import threading
 import time
 
@@ -70,3 +71,19 @@
             #pylint: disable=raising-bad-type
             raise exception
         raise WaitTimeout("Timed out after %d seconds" % timeout)
+
+
+# TODO(haleyb): move to neutron-lib
+# code copied from neutron repository - neutron/tests/base.py
+def unstable_test(reason):
+    def decor(f):
+        @functools.wraps(f)
+        def inner(self, *args, **kwargs):
+            try:
+                return f(self, *args, **kwargs)
+            except Exception as e:
+                msg = ("%s was marked as unstable because of %s, "
+                       "failure was: %s") % (self.id(), reason, e)
+                raise self.skipTest(msg)
+        return inner
+    return decor
diff --git a/neutron_tempest_plugin/scenario/test_floatingip.py b/neutron_tempest_plugin/scenario/test_floatingip.py
index 5fcbdc0..0febce2 100644
--- a/neutron_tempest_plugin/scenario/test_floatingip.py
+++ b/neutron_tempest_plugin/scenario/test_floatingip.py
@@ -22,6 +22,7 @@
 from testscenarios.scenarios import multiply_scenarios
 
 from neutron_tempest_plugin.common import ssh
+from neutron_tempest_plugin.common import utils as common_utils
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin.scenario import base
 from neutron_tempest_plugin.scenario import constants
@@ -134,6 +135,7 @@
 
     same_network = True
 
+    @common_utils.unstable_test("bug 1717302")
     @decorators.idempotent_id('05c4e3b3-7319-4052-90ad-e8916436c23b')
     def test_east_west(self):
         self._test_east_west()
@@ -151,6 +153,7 @@
 
     same_network = False
 
+    @common_utils.unstable_test("bug 1717302")
     @decorators.idempotent_id('f18f0090-3289-4783-b956-a0f8ac511e8b')
     def test_east_west(self):
         self._test_east_west()
diff --git a/playbooks/neutron-tempest-plugin-api/run.yaml b/playbooks/neutron-tempest-plugin-api/run.yaml
index 3470bd4..12638cc 100644
--- a/playbooks/neutron-tempest-plugin-api/run.yaml
+++ b/playbooks/neutron-tempest-plugin-api/run.yaml
@@ -27,23 +27,13 @@
         cmd: |
           set -e
           set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          TEMPEST_PLUGINS='/opt/stack/new/neutron-tempest-plugin'
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
           export PYTHONUNBUFFERED=true
           export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1
           export DEVSTACK_GATE_NEUTRON=1
           export DEVSTACK_GATE_EXERCISES=0
           export DEVSTACK_GATE_TEMPEST_REGEX="neutron_tempest_plugin.api"
+          export DEVSTACK_LOCAL_CONFIG="enable_plugin neutron-tempest-plugin git://git.openstack.org/openstack/neutron-tempest-plugin"
           export BRANCH_OVERRIDE=default
           if [ "$BRANCH_OVERRIDE" != "default" ] ; then
               export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
diff --git a/playbooks/neutron-tempest-plugin-dvr-multinode-scenario/run.yaml b/playbooks/neutron-tempest-plugin-dvr-multinode-scenario/run.yaml
index d68747c..c2d33a9 100644
--- a/playbooks/neutron-tempest-plugin-dvr-multinode-scenario/run.yaml
+++ b/playbooks/neutron-tempest-plugin-dvr-multinode-scenario/run.yaml
@@ -27,23 +27,13 @@
         cmd: |
           set -e
           set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          TEMPEST_PLUGINS='/opt/stack/new/neutron-tempest-plugin'
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
           export PYTHONUNBUFFERED=true
           export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1
           export DEVSTACK_GATE_NEUTRON=1
           export DEVSTACK_GATE_CONFIGDRIVE=0
           export DEVSTACK_GATE_TEMPEST_REGEX="(neutron_tempest_plugin.scenario)"
+          export DEVSTACK_LOCAL_CONFIG="enable_plugin neutron-tempest-plugin git://git.openstack.org/openstack/neutron-tempest-plugin"
           export TEMPEST_CONCURRENCY=2
           # Test DVR works multinode
           export DEVSTACK_GATE_NEUTRON_DVR=1
diff --git a/playbooks/neutron-tempest-plugin-scenario-linuxbridge/run.yaml b/playbooks/neutron-tempest-plugin-scenario-linuxbridge/run.yaml
index f2be130..65e8b12 100644
--- a/playbooks/neutron-tempest-plugin-scenario-linuxbridge/run.yaml
+++ b/playbooks/neutron-tempest-plugin-scenario-linuxbridge/run.yaml
@@ -31,7 +31,6 @@
           [[local|localrc]]
           Q_AGENT=linuxbridge
           PHYSICAL_NETWORK=default
-          TEMPEST_PLUGINS='/opt/stack/new/neutron-tempest-plugin'
 
           EOF
         executable: /bin/bash
@@ -44,9 +43,11 @@
           set -x
           export PYTHONUNBUFFERED=true
           export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1
           export DEVSTACK_GATE_NEUTRON=1
           export DEVSTACK_GATE_EXERCISES=0
           export DEVSTACK_GATE_TEMPEST_REGEX="(neutron_tempest_plugin.scenario)"
+          export DEVSTACK_LOCAL_CONFIG="enable_plugin neutron-tempest-plugin git://git.openstack.org/openstack/neutron-tempest-plugin"
           export BRANCH_OVERRIDE=default
           if [ "$BRANCH_OVERRIDE" != "default" ] ; then
               export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE