Merge "Make tempest-slow run on stable branch also"
diff --git a/.zuul.yaml b/.zuul.yaml
index 00dd92e..2bec8c4 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -419,10 +419,12 @@
- git.openstack.org/openstack/sahara-tests
- git.openstack.org/openstack/senlin
- git.openstack.org/openstack/senlin-tempest-plugin
+ - git.openstack.org/openstack/solum-tempest-plugin
- git.openstack.org/openstack/tap-as-a-service
- git.openstack.org/openstack/telemetry-tempest-plugin
- git.openstack.org/openstack/tempest-horizon
- git.openstack.org/openstack/tobiko
+ - git.openstack.org/openstack/trio2o
- git.openstack.org/openstack/tripleo-common-tempest-plugin
- git.openstack.org/openstack/trove-tempest-plugin
- git.openstack.org/openstack/valet
@@ -492,6 +494,7 @@
- openstack-python-jobs
- openstack-python35-jobs
- openstack-python36-jobs
+ - openstack-python37-jobs
- publish-openstack-docs-pti
- release-notes-jobs-python3
check:
diff --git a/doc/source/microversion_testing.rst b/doc/source/microversion_testing.rst
index 983fa24..4b1c145 100644
--- a/doc/source/microversion_testing.rst
+++ b/doc/source/microversion_testing.rst
@@ -338,8 +338,8 @@
.. _2.26: https://docs.openstack.org/nova/latest/reference/api-microversion-history.html#id23
-* `2.28`_
-
+ * `2.28`_
+
.. _2.28: https://docs.openstack.org/nova/latest/reference/api-microversion-history.html#id25
* `2.32`_
diff --git a/tempest/api/identity/admin/v3/test_default_project_id.py b/tempest/api/identity/admin/v3/test_default_project_id.py
index a79cbc3..73fddb7 100644
--- a/tempest/api/identity/admin/v3/test_default_project_id.py
+++ b/tempest/api/identity/admin/v3/test_default_project_id.py
@@ -21,7 +21,7 @@
CONF = config.CONF
-class TestDefaultProjectId (base.BaseIdentityV3AdminTest):
+class TestDefaultProjectId(base.BaseIdentityV3AdminTest):
@classmethod
def setup_credentials(cls):
@@ -57,9 +57,10 @@
# create a user in the domain, with the previous project as his
# default project
user_name = data_utils.rand_name('user')
+ user_pass = data_utils.rand_password()
user_body = self.users_client.create_user(
name=user_name,
- password=user_name,
+ password=user_pass,
domain_id=dom_id,
default_project_id=proj_id)['user']
user_id = user_body['id']
@@ -78,7 +79,7 @@
# create a new client with user's credentials (NOTE: unscoped token!)
creds = auth.KeystoneV3Credentials(username=user_name,
- password=user_name,
+ password=user_pass,
user_domain_name=dom_name)
auth_provider = clients.get_auth_provider(creds)
creds = auth_provider.fill_credentials()
diff --git a/tempest/api/identity/admin/v3/test_domains.py b/tempest/api/identity/admin/v3/test_domains.py
index 72b6be4..07175f4 100644
--- a/tempest/api/identity/admin/v3/test_domains.py
+++ b/tempest/api/identity/admin/v3/test_domains.py
@@ -153,18 +153,3 @@
expected_data = {'name': d_name, 'enabled': True}
self.assertEqual('', domain['description'])
self.assertDictContainsSubset(expected_data, domain)
-
-
-class DefaultDomainTestJSON(base.BaseIdentityV3AdminTest):
-
- @classmethod
- def resource_setup(cls):
- cls.domain_id = CONF.identity.default_domain_id
- super(DefaultDomainTestJSON, cls).resource_setup()
-
- @decorators.attr(type='smoke')
- @decorators.idempotent_id('17a5de24-e6a0-4e4a-a9ee-d85b6e5612b5')
- def test_default_domain_exists(self):
- domain = self.domains_client.show_domain(self.domain_id)['domain']
-
- self.assertTrue(domain['enabled'])
diff --git a/tempest/api/identity/admin/v3/test_inherits.py b/tempest/api/identity/admin/v3/test_inherits.py
index 68c0225..acc5a8c 100644
--- a/tempest/api/identity/admin/v3/test_inherits.py
+++ b/tempest/api/identity/admin/v3/test_inherits.py
@@ -30,7 +30,7 @@
u_name = data_utils.rand_name('user-')
u_desc = '%s description' % u_name
u_email = '%s@testmail.tm' % u_name
- u_password = data_utils.rand_name('pass-')
+ u_password = data_utils.rand_password()
cls.domain = cls.create_domain()
cls.project = cls.projects_client.create_project(
data_utils.rand_name('project-'),
diff --git a/tempest/api/identity/v3/test_domains.py b/tempest/api/identity/v3/test_domains.py
new file mode 100644
index 0000000..9f132dd
--- /dev/null
+++ b/tempest/api/identity/v3/test_domains.py
@@ -0,0 +1,39 @@
+# Copyright 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tempest.api.identity import base
+from tempest import config
+from tempest.lib import decorators
+
+CONF = config.CONF
+
+
+class DefaultDomainTestJSON(base.BaseIdentityV3Test):
+
+ @classmethod
+ def setup_clients(cls):
+ super(DefaultDomainTestJSON, cls).setup_clients()
+ cls.domains_client = cls.os_primary.domains_client
+
+ @classmethod
+ def resource_setup(cls):
+ super(DefaultDomainTestJSON, cls).resource_setup()
+ cls.domain_id = CONF.identity.default_domain_id
+
+ @decorators.attr(type='smoke')
+ @decorators.idempotent_id('17a5de24-e6a0-4e4a-a9ee-d85b6e5612b5')
+ def test_default_domain_exists(self):
+ domain = self.domains_client.show_domain(self.domain_id)['domain']
+ self.assertTrue(domain['enabled'])
diff --git a/tempest/api/network/admin/test_negative_quotas.py b/tempest/api/network/admin/test_negative_quotas.py
index a075b51..9d1e2a7 100644
--- a/tempest/api/network/admin/test_negative_quotas.py
+++ b/tempest/api/network/admin/test_negative_quotas.py
@@ -30,7 +30,7 @@
It is also assumed that the per-project quota extension API is configured
in /etc/neutron/neutron.conf as follows:
- quota_driver = neutron.db.quota_db.DbQuotaDriver
+ quota_driver = neutron.db.quota.driver.DbQuotaDriver
"""
@classmethod
diff --git a/tempest/api/network/admin/test_quotas.py b/tempest/api/network/admin/test_quotas.py
index b1e4a58..ef5ebb6 100644
--- a/tempest/api/network/admin/test_quotas.py
+++ b/tempest/api/network/admin/test_quotas.py
@@ -35,7 +35,7 @@
It is also assumed that the per-project quota extension API is configured
in /etc/neutron/neutron.conf as follows:
- quota_driver = neutron.db.quota_db.DbQuotaDriver
+ quota_driver = neutron.db.quota.driver.DbQuotaDriver
"""
@classmethod
diff --git a/tempest/cmd/cleanup_service.py b/tempest/cmd/cleanup_service.py
index 6b6a59b..3aed4e8 100644
--- a/tempest/cmd/cleanup_service.py
+++ b/tempest/cmd/cleanup_service.py
@@ -362,6 +362,7 @@
self.metering_label_rules_client = manager.metering_label_rules_client
self.security_groups_client = manager.security_groups_client
self.routers_client = manager.routers_client
+ self.subnetpools_client = manager.subnetpools_client
def _filter_by_conf_networks(self, item_list):
if not item_list or not all(('network_id' in i for i in item_list)):
@@ -673,6 +674,41 @@
self.data['subnets'][subnet['id']] = subnet['name']
+class NetworkSubnetPoolsService(BaseNetworkService):
+
+ def list(self):
+ client = self.subnetpools_client
+ pools = client.list_subnetpools(**self.tenant_filter)['subnetpools']
+ if not self.is_save_state:
+ # recreate list removing saved subnet pools
+ pools = [pool for pool in pools if pool['id']
+ not in self.saved_state_json['subnetpools'].keys()]
+ if self.is_preserve:
+ pools = [pool for pool in pools if pool['project_id']
+ not in CONF_PROJECTS]
+ LOG.debug("List count, %s Subnet Pools", len(pools))
+ return pools
+
+ def delete(self):
+ client = self.subnetpools_client
+ pools = self.list()
+ for pool in pools:
+ try:
+ client.delete_subnetpool(pool['id'])
+ except Exception:
+ LOG.exception("Delete Subnet Pool exception.")
+
+ def dry_run(self):
+ pools = self.list()
+ self.data['subnetpools'] = pools
+
+ def save_state(self):
+ pools = self.list()
+ self.data['subnetpools'] = {}
+ for pool in pools:
+ self.data['subnetpools'][pool['id']] = pool['name']
+
+
# begin global services
class FlavorService(BaseService):
def __init__(self, manager, **kwargs):
@@ -931,6 +967,7 @@
project_services.append(NetworkSubnetService)
project_services.append(NetworkService)
project_services.append(NetworkSecGroupService)
+ project_services.append(NetworkSubnetPoolsService)
if IS_CINDER:
project_services.append(SnapshotService)
project_services.append(VolumeService)
diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py
index 3e84b82..a8ef08c 100644
--- a/tempest/cmd/run.py
+++ b/tempest/cmd/run.py
@@ -252,6 +252,7 @@
default=False)
# execution args
parser.add_argument('--concurrency', '-w',
+ type=int,
help="The number of workers to use, defaults to "
"the number of cpus")
parallel = parser.add_mutually_exclusive_group()
diff --git a/tempest/tests/cmd/test_cleanup_services.py b/tempest/tests/cmd/test_cleanup_services.py
index 59e5636..3262b1c 100644
--- a/tempest/tests/cmd/test_cleanup_services.py
+++ b/tempest/tests/cmd/test_cleanup_services.py
@@ -104,6 +104,7 @@
cleanup_service.CONF_PROJECTS[0],
"ports": cleanup_service.CONF_PUB_NETWORK,
"routers": cleanup_service.CONF_PUB_ROUTER,
+ "subnetpools": cleanup_service.CONF_PROJECTS[0],
}
saved_state = {
@@ -139,7 +140,8 @@
"metering_labels": {u'723b346ce866-4c7q': u'saved-label'},
"ports": {u'aa74aa4v-741a': u'saved-port'},
"security_groups": {u'7q844add-3697': u'saved-sec-group'},
- "subnets": {u'55ttda4a-2584': u'saved-subnet'}
+ "subnets": {u'55ttda4a-2584': u'saved-subnet'},
+ "subnetpools": {u'8acf64c1-43fc': u'saved-subnet-pool'}
}
# Mocked methods
get_method = 'tempest.lib.common.rest_client.RestClient.get'
@@ -1097,6 +1099,74 @@
self._test_is_preserve_true([(self.get_method, self.response, 200)])
+class TestNetworkSubnetPoolsService(BaseCmdServiceTests):
+
+ service_class = 'NetworkSubnetPoolsService'
+ service_name = 'subnetpools'
+ response = {
+ "subnetpools": [
+ {
+ "min_prefixlen": "64",
+ "default_prefixlen": "64",
+ "id": "03f761e6-eee0-43fc-a921-8acf64c14988",
+ "max_prefixlen": "64",
+ "name": "my-subnet-pool-ipv6",
+ "is_default": False,
+ "project_id": "9fadcee8aa7c40cdb2114fff7d569c08",
+ "tenant_id": "9fadcee8aa7c40cdb2114fff7d569c08",
+ "prefixes": [
+ "2001:db8:0:2::/64",
+ "2001:db8::/63"
+ ],
+ "ip_version": 6,
+ "shared": False,
+ "description": "",
+ "created_at": "2016-03-08T20:19:41",
+ "updated_at": "2016-03-08T20:19:41",
+ "revision_number": 2,
+ "tags": ["tag1,tag2"]
+ },
+ {
+ "id": "8acf64c1-43fc",
+ "name": "saved-subnet-pool"
+ }
+ ]
+ }
+
+ def test_delete_fail(self):
+ delete_mock = [(self.get_method, self.response, 200),
+ (self.delete_method, 'error', None),
+ (self.log_method, 'exception', None)]
+ self._test_delete(delete_mock, fail=True)
+
+ def test_delete_pass(self):
+ delete_mock = [(self.get_method, self.response, 200),
+ (self.delete_method, None, 204),
+ (self.log_method, 'exception', None)]
+ self._test_delete(delete_mock)
+
+ def test_dry_run(self):
+ dry_mock = [(self.get_method, self.response, 200),
+ (self.delete_method, "delete", None)]
+ self._test_dry_run_true(dry_mock)
+
+ def test_save_state(self):
+ self._test_saved_state_true([(self.get_method, self.response, 200)])
+
+ def test_preserve_list(self):
+ self.response['subnetpools'].append(
+ {
+ "min_prefixlen": "64",
+ "default_prefixlen": "64",
+ "id": "9acf64c1-43fc",
+ "name": "preserve-pool",
+ "project_id": cleanup_service.CONF_PROJECTS[0],
+ "created_at": "2016-03-08T20:19:41",
+ "updated_at": "2016-03-08T20:19:41"
+ })
+ self._test_is_preserve_true([(self.get_method, self.response, 200)])
+
+
# begin global services
class TestDomainService(BaseCmdServiceTests):
diff --git a/tools/tempest-plugin-sanity.sh b/tools/tempest-plugin-sanity.sh
index 661329b..16e7b8c 100644
--- a/tools/tempest-plugin-sanity.sh
+++ b/tools/tempest-plugin-sanity.sh
@@ -18,20 +18,17 @@
# This script is intended to check the sanity of tempest plugins against
# tempest master.
# What it does:
-# * Creates the virtualenv
-# * Install tempest
# * Retrieve the project lists having tempest plugin if project name is
# given.
-# * For each project in a list, It does:
+# * For each project in a list, it does:
+# * Create virtualenv and install tempest in it
# * Clone the Project
# * Install the Project and also installs dependencies from
# test-requirements.txt.
# * Create Tempest workspace
# * List tempest plugins
# * List tempest plugins tests
-# * Uninstall the project and its dependencies
-# * Again Install tempest
-# * Again repeat the step from cloning project
+# * Delete virtualenv and project repo
#
# If one of the step fails, The script will exit with failure.
@@ -46,8 +43,39 @@
# retrieve a list of projects having tempest plugins
PROJECT_LIST="$(python tools/generate-tempest-plugins-list.py)"
-# List of projects having tempest plugin stale or unmaintained from long time
-BLACKLIST="networking-plumgrid,trio2o"
+# List of projects having tempest plugin stale or unmaintained for a long time
+# (6 months or more)
+# TODO(masayukig): Some of these can be removed from BLACKLIST in the future.
+# airship-tempest-plugin: https://review.openstack.org/#/c/634387/
+# barbican-tempest-plugin: https://review.openstack.org/#/c/634631/
+# intel-nfv-ci-tests: https://review.openstack.org/#/c/634640/
+# networking-ansible: https://review.openstack.org/#/c/634647/
+# networking-generic-switch: https://review.openstack.org/#/c/634846/
+# networking-l2gw-tempest-plugin: https://review.openstack.org/#/c/635093/
+# networking-midonet: https://review.openstack.org/#/c/635096/
+# networking-plumgrid: https://review.openstack.org/#/c/635096/
+# networking-spp: https://review.openstack.org/#/c/635098/
+# neutron-dynamic-routing: https://review.openstack.org/#/c/637718/
+# neutron-vpnaas: https://review.openstack.org/#/c/637719/
+# nova-lxd: https://review.openstack.org/#/c/638334/
+# valet: https://review.openstack.org/#/c/638339/
+# vitrage-tempest-plugin: https://review.openstack.org/#/c/639003/
+BLACKLIST="
+airship-tempest-plugin
+barbican-tempest-plugin
+intel-nfv-ci-tests
+networking-ansible
+networking-generic-switch
+networking-l2gw-tempest-plugin
+networking-midonet
+networking-plumgrid
+networking-spp
+neutron-dynamic-routing
+neutron-vpnaas
+nova-lxd
+valet
+vitrage-tempest-plugin
+"
# Function to clone project using zuul-cloner or from git
function clone_project() {
@@ -63,14 +91,16 @@
fi
}
-# Create virtualenv to perform sanity operation
-SANITY_DIR=$(pwd)
-virtualenv "$SANITY_DIR"/.venv
-export TVENV="$SANITY_DIR/tools/with_venv.sh"
-cd "$SANITY_DIR"
+# function to create virtualenv to perform sanity operation
+function prepare_workspace() {
+ SANITY_DIR=$(pwd)
+ virtualenv --clear "$SANITY_DIR"/.venv
+ export TVENV="$SANITY_DIR/tools/with_venv.sh"
+ cd "$SANITY_DIR"
-# Install tempest in a venv
-"$TVENV" pip install .
+ # Install tempest with test dependencies in a venv
+ "$TVENV" pip install -e . -r test-requirements.txt
+}
# Function to install project
function install_project() {
@@ -83,30 +113,31 @@
# Function to perform sanity checking on Tempest plugin
function tempest_sanity() {
- "$TVENV" tempest init "$SANITY_DIR"/tempest_sanity
- cd "$SANITY_DIR"/tempest_sanity
- "$TVENV" tempest list-plugins
+ "$TVENV" tempest init "$SANITY_DIR"/tempest_sanity && \
+ cd "$SANITY_DIR"/tempest_sanity && \
+ "$TVENV" tempest list-plugins && \
"$TVENV" tempest run -l
+ retval=$?
# Delete tempest workspace
+ # NOTE: Cleaning should be done even if an error occurs.
"$TVENV" tempest workspace remove --name tempest_sanity --rmdir
cd "$SANITY_DIR"
-}
-
-# Function to uninstall project
-function uninstall_project() {
- "$TVENV" pip uninstall -y "$SANITY_DIR"/openstack/"$1"
- # Check for *requirements.txt file in a project then uninstall it.
- if [ -e "$SANITY_DIR"/openstack/"$1"/*requirements.txt ]; then
- "$TVENV" pip uninstall -y -r "$SANITY_DIR"/openstack/"$1"/*requirements.txt
- fi
+ # Remove the sanity workspace in case of remaining
+ rm -fr "$SANITY_DIR"/tempest_sanity
# Remove the project directory after sanity run
rm -fr "$SANITY_DIR"/openstack/"$1"
+
+ return $retval
}
# Function to run sanity check on each project
function plugin_sanity_check() {
- clone_project "$1" && install_project "$1" && tempest_sanity "$1" \
- && uninstall_project "$1" && "$TVENV" pip install .
+ prepare_workspace && \
+ clone_project "$1" && \
+ install_project "$1" && \
+ tempest_sanity "$1"
+
+ return $?
}
# Log status
@@ -117,11 +148,12 @@
# Remove blacklisted tempest plugins
if ! [[ `echo $BLACKLIST | grep -c $project ` -gt 0 ]]; then
plugin_sanity_check $project && passed_plugin+=", $project" || \
- failed_plugin+=", $project"
+ failed_plugin+="$project, " > $SANITY_DIR/$project.txt
fi
done
# Check for failed status
if [[ -n $failed_plugin ]]; then
+ echo "Failed Plugins: $failed_plugin"
exit 1
fi