Merge "Switch testing to Victoria testing runtime"
diff --git a/.gitignore b/.gitignore
index 9767e52..8b6222e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,7 +31,7 @@
!.coveragerc
cover/
doc/source/_static/tempest.conf.sample
-doc/source/plugin-registry.rst
+doc/source/plugins/plugin-registry.rst
# Files created by releasenotes build
releasenotes/build
diff --git a/.zuul.yaml b/.zuul.yaml
index ec6b5fc..87e277c 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -154,8 +154,11 @@
Base integration test with Neutron networking.
It includes all scenarios as it was in the past.
This job runs all scenario tests in parallel!
+ timeout: 9000
vars:
tox_envlist: full-parallel
+ run_tempest_cleanup: true
+ run_tempest_dry_cleanup: true
devstack_localrc:
USE_PYTHON3: True
diff --git a/HACKING.rst b/HACKING.rst
index f8be0ad..95bcbb5 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -59,7 +59,7 @@
`relevant plugin projects`_.
.. _External Plugin Interface: https://specs.openstack.org/openstack/qa-specs/specs/tempest/implemented/tempest-external-plugin-interface.html
-.. _relevant plugin projects: https://docs.openstack.org/tempest/latest/plugin-registry.html#detected-plugins
+.. _relevant plugin projects: https://docs.openstack.org/tempest/latest/plugins/plugin-registry.html#detected-plugins
Exception Handling
------------------
diff --git a/doc/source/_extra/.htaccess b/doc/source/_extra/.htaccess
index 7745594..5422af7 100644
--- a/doc/source/_extra/.htaccess
+++ b/doc/source/_extra/.htaccess
@@ -1 +1,4 @@
redirectmatch 301 ^/developer/tempest/(.*) /tempest/latest/$1
+redirectmatch 301 ^/tempest/latest/plugin.html /tempest/latest/plugins/plugin.html
+redirectmatch 301 ^/tempest/latest/plugin-registry.html /tempest/latest/plugins/plugin-registry
+redirectmatch 301 ^/tempest/latest/#support-policy /tempest/latest/#stable-branch-support-policy
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 93fcc3a..f878888 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -87,8 +87,7 @@
.. toctree::
:maxdepth: 2
- plugin
- plugin-registry
+ plugins/index
Tempest & Plugins Compatible Version Policy
-------------------------------------------
@@ -98,6 +97,22 @@
tempest_and_plugins_compatible_version_policy
+Stable Branch Support Policy
+----------------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ stable_branch_support_policy
+
+Stable Branch Testing Policy
+----------------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ stable_branch_testing_policy
+
Library
-------
@@ -106,14 +121,6 @@
library
-Support Policy
---------------
-
-.. toctree::
- :maxdepth: 2
-
- stable_branch_support_policy
-
Search
======
diff --git a/doc/source/plugins/index.rst b/doc/source/plugins/index.rst
new file mode 100644
index 0000000..f961ac7
--- /dev/null
+++ b/doc/source/plugins/index.rst
@@ -0,0 +1,40 @@
+=====================
+Tempest Plugins Guide
+=====================
+
+.. toctree::
+ :maxdepth: 2
+
+ plugin
+
+Stable Branch Support Policy
+----------------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ ../stable_branch_support_policy
+
+Stable Branch Testing Policy
+----------------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ ../stable_branch_testing_policy
+
+Tempest & Plugins Compatible Version Policy
+-------------------------------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ ../tempest_and_plugins_compatible_version_policy
+
+Plugins Registry
+----------------
+
+.. toctree::
+ :maxdepth: 2
+
+ plugin-registry
diff --git a/doc/source/plugin.rst b/doc/source/plugins/plugin.rst
similarity index 100%
rename from doc/source/plugin.rst
rename to doc/source/plugins/plugin.rst
diff --git a/doc/source/stable_branch_testing_policy.rst b/doc/source/stable_branch_testing_policy.rst
new file mode 100644
index 0000000..02c5338
--- /dev/null
+++ b/doc/source/stable_branch_testing_policy.rst
@@ -0,0 +1,33 @@
+Stable Branch Testing Policy
+============================
+
+Tempest and its plugins need to support the stable branches
+as per :doc:`Stable Branch Support Policy </stable_branch_support_policy>`.
+
+Because of branchless model of Tempest and plugins, all the supported
+stable branches use the Tempest and plugins master version for their
+testing. That is done in devstack by using the `master branch
+<https://opendev.org/openstack/devstack/src/commit/c104afec7dd72edfd909847bee9c14eaf077a28b/stackrc#L314>`_
+for the Tempest installation. To make sure the master version of Tempest or
+plugins (for any changes or adding new tests) is compatible for all
+the supported stable branches testing, Tempest and its plugins need to
+add the stable branches job on the master gate. That way can test the stable
+branches against master code and can avoid breaking supported branches
+accidentally.
+
+Example:
+
+* `Stable jobs on Tempest master
+ <https://opendev.org/openstack/tempest/src/commit/e8f1876aa6772077f85f380677b30251c2454505/.zuul.yaml#L646-L651>`_.
+
+* `Stable job on neutron tempest plugins
+ <https://opendev.org/openstack/neutron-tempest-plugin/src/commit/4bc1b00213cf660648cad1916fe6497ac29b2e78/.zuul.yaml#L1427-L1428>`_
+
+Once any stable branch is moved to the `Extended Maintenance Phases`_
+and devstack start using the Tempest older version for that stable
+branch testing then we can remove that stable branch job from master
+gate.
+
+Example: https://review.opendev.org/#/c/722183/
+
+.. _Extended Maintenance Phases: https://docs.openstack.org/project-team-guide/stable-branches.html#extended-maintenance
diff --git a/playbooks/devstack-tempest.yaml b/playbooks/devstack-tempest.yaml
index 5f87abd..7ee7411 100644
--- a/playbooks/devstack-tempest.yaml
+++ b/playbooks/devstack-tempest.yaml
@@ -12,8 +12,41 @@
# job provided by the gabbi-tempest plugin. It can be safely ignored
# if that plugin is not being used.
GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path | default('') }}"
- roles:
- - setup-tempest-run-dir
- - setup-tempest-data-dir
- - acl-devstack-files
- - run-tempest
+ tasks:
+ - name: Setup Tempest Run Directory
+ include_role:
+ name: setup-tempest-run-dir
+
+ - name: Setup Tempest Data Directory
+ include_role:
+ name: setup-tempest-data-dir
+
+ - name: ACL devstack files
+ include_role:
+ name: acl-devstack-files
+
+ - name: Run tempest cleanup init-saved-state
+ include_role:
+ name: tempest-cleanup
+ vars:
+ init_saved_state: true
+ when:
+ - run_tempest_dry_cleanup is defined
+ - run_tempest_cleanup is defined
+
+ - name: Run Tempest
+ include_role:
+ name: run-tempest
+
+ - name: Run tempest cleanup dry-run
+ include_role:
+ name: tempest-cleanup
+ vars:
+ dry_run: true
+ when:
+ - run_tempest_dry_cleanup is defined
+
+ - name: Run tempest cleanup
+ include_role:
+ name: tempest-cleanup
+ when: run_tempest_cleanup is defined
diff --git a/roles/process-stackviz/tasks/main.yaml b/roles/process-stackviz/tasks/main.yaml
index 3724e0e..e3a0a0e 100644
--- a/roles/process-stackviz/tasks/main.yaml
+++ b/roles/process-stackviz/tasks/main.yaml
@@ -17,13 +17,18 @@
when: not subunit_input.stat.exists
- name: Install stackviz
- pip:
- name: "file://{{ stackviz_archive.stat.path }}"
- virtualenv: /tmp/stackviz
- extra_args: -U
when:
- stackviz_archive.stat.exists
- subunit_input.stat.exists
+ block:
+ - include_role:
+ name: ensure-pip
+
+ - pip:
+ name: "file://{{ stackviz_archive.stat.path }}"
+ virtualenv: /tmp/stackviz
+ virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+ extra_args: -U
- name: Deploy stackviz static html+js
command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
diff --git a/roles/tempest-cleanup/README.rst b/roles/tempest-cleanup/README.rst
new file mode 100644
index 0000000..70719ca
--- /dev/null
+++ b/roles/tempest-cleanup/README.rst
@@ -0,0 +1,33 @@
+Tempest cleanup
+===============
+
+Documentation regarding tempest cleanup can be found at the following
+link:
+https://docs.openstack.org/tempest/latest/cleanup.html
+
+When init_saved_state and dry_run variables are set to false, the role
+execution will run tempest cleanup which deletes resources not present in the
+saved_state.json file.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+ :default: /opt/stack
+
+ The devstack base directory.
+
+.. zuul:rolevar:: init_saved_state
+ :default: false
+
+ When true, tempest cleanup --init-saved-state will be executed which
+ initializes the saved state of the OpenStack deployment and will output
+ a saved_state.json file containing resources from the deployment that will
+ be preserved from the cleanup command. This should be done prior to running
+ Tempest tests.
+
+.. zuul:rolevar:: dry_run
+ :default: false
+
+ When true, tempest cleanup creates a report (./dry_run.json) of the
+ resources that would be cleaned up if the role was ran with dry_run option
+ set to false.
diff --git a/roles/tempest-cleanup/defaults/main.yaml b/roles/tempest-cleanup/defaults/main.yaml
new file mode 100644
index 0000000..fc1948a
--- /dev/null
+++ b/roles/tempest-cleanup/defaults/main.yaml
@@ -0,0 +1,3 @@
+devstack_base_dir: /opt/stack
+init_saved_state: false
+dry_run: false
diff --git a/roles/tempest-cleanup/tasks/main.yaml b/roles/tempest-cleanup/tasks/main.yaml
new file mode 100644
index 0000000..5444afc
--- /dev/null
+++ b/roles/tempest-cleanup/tasks/main.yaml
@@ -0,0 +1,31 @@
+- when: init_saved_state
+ block:
+ - name: Run tempest cleanup init-saved-state
+ become: yes
+ become_user: tempest
+ command: tox -evenv-tempest -- tempest cleanup --init-saved-state --debug
+ args:
+ chdir: "{{ devstack_base_dir }}/tempest"
+
+ - name: Cat saved_state.json
+ command: cat "{{ devstack_base_dir }}/tempest/saved_state.json"
+
+- when: dry_run
+ block:
+ - name: Run tempest cleanup dry-run
+ become: yes
+ become_user: tempest
+ command: tox -evenv-tempest -- tempest cleanup --dry-run --debug
+ args:
+ chdir: "{{ devstack_base_dir }}/tempest"
+
+ - name: Cat dry_run.json
+ command: cat "{{ devstack_base_dir }}/tempest/dry_run.json"
+
+- name: Run tempest cleanup
+ become: yes
+ become_user: tempest
+ command: tox -evenv-tempest -- tempest cleanup --debug
+ args:
+ chdir: "{{ devstack_base_dir }}/tempest"
+ when: not dry_run and not init_saved_state
diff --git a/tempest/cmd/cleanup.py b/tempest/cmd/cleanup.py
index c54b16b..0b96d9e 100644
--- a/tempest/cmd/cleanup.py
+++ b/tempest/cmd/cleanup.py
@@ -123,6 +123,16 @@
raise Exception(self.GOT_EXCEPTIONS)
def init(self, parsed_args):
+ # set new handler for logging to stdout, by default only INFO messages
+ # are logged to stdout
+ stdout_handler = logging.logging.StreamHandler()
+ # debug argument is defined in cliff already
+ if self.app_args.debug:
+ stdout_handler.level = logging.DEBUG
+ else:
+ stdout_handler.level = logging.INFO
+ LOG.handlers.append(stdout_handler)
+
cleanup_service.init_conf()
self.options = parsed_args
self.admin_mgr = clients.Manager(
@@ -149,7 +159,7 @@
self._load_json()
def _cleanup(self):
- print("Begin cleanup")
+ LOG.info("Begin cleanup")
is_dry_run = self.options.dry_run
is_preserve = not self.options.delete_tempest_conf_objects
is_save_state = False
@@ -167,7 +177,7 @@
'is_save_state': is_save_state}
project_service = cleanup_service.ProjectService(admin_mgr, **kwargs)
projects = project_service.list()
- print("Process %s projects" % len(projects))
+ LOG.info("Processing %s projects", len(projects))
# Loop through list of projects and clean them up.
for project in projects:
@@ -179,10 +189,12 @@
'is_preserve': is_preserve,
'is_save_state': is_save_state,
'got_exceptions': self.GOT_EXCEPTIONS}
+ LOG.info("Processing global services")
for service in self.global_services:
svc = service(admin_mgr, **kwargs)
svc.run()
+ LOG.info("Processing services")
for service in self.resource_cleanup_services:
svc = service(self.admin_mgr, **kwargs)
svc.run()
@@ -193,7 +205,7 @@
indent=2, separators=(',', ': ')))
def _clean_project(self, project):
- print("Cleaning project: %s " % project['name'])
+ LOG.debug("Cleaning project: %s ", project['name'])
is_dry_run = self.options.dry_run
dry_run_data = self.dry_run_data
is_preserve = not self.options.delete_tempest_conf_objects
@@ -263,7 +275,7 @@
return 'Cleanup after tempest run'
def _init_state(self):
- print("Initializing saved state.")
+ LOG.info("Initializing saved state.")
data = {}
admin_mgr = self.admin_mgr
kwargs = {'data': data,
diff --git a/tempest/cmd/cleanup_service.py b/tempest/cmd/cleanup_service.py
index 469b214..84d2492 100644
--- a/tempest/cmd/cleanup_service.py
+++ b/tempest/cmd/cleanup_service.py
@@ -23,7 +23,7 @@
from tempest import config
from tempest.lib import exceptions
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger('tempest.cmd.cleanup')
CONF = config.CONF
CONF_FLAVORS = None
@@ -167,6 +167,7 @@
client = self.client
for snap in snaps:
try:
+ LOG.debug("Deleting Snapshot with id %s", snap['id'])
client.delete_snapshot(snap['id'])
except Exception:
LOG.exception("Delete Snapshot %s exception.", snap['id'])
@@ -204,6 +205,7 @@
servers = self.list()
for server in servers:
try:
+ LOG.debug("Deleting Server with id %s", server['id'])
client.delete_server(server['id'])
except Exception:
LOG.exception("Delete Server %s exception.", server['id'])
@@ -236,6 +238,7 @@
sgs = self.list()
for sg in sgs:
try:
+ LOG.debug("Deleting Server Group with id %s", sg['id'])
client.delete_server_group(sg['id'])
except Exception:
LOG.exception("Delete Server Group %s exception.", sg['id'])
@@ -273,6 +276,7 @@
for k in keypairs:
name = k['keypair']['name']
try:
+ LOG.debug("Deleting keypair %s", name)
client.delete_keypair(name)
except Exception:
LOG.exception("Delete Keypair %s exception.", name)
@@ -309,6 +313,7 @@
vols = self.list()
for v in vols:
try:
+ LOG.debug("Deleting volume with id %s", v['id'])
client.delete_volume(v['id'])
except Exception:
LOG.exception("Delete Volume %s exception.", v['id'])
@@ -332,6 +337,8 @@
def delete(self):
client = self.client
try:
+ LOG.debug("Deleting Volume Quotas for project with id %s",
+ self.project_id)
client.delete_quota_set(self.project_id)
except Exception:
LOG.exception("Delete Volume Quotas exception for 'project %s'.",
@@ -352,9 +359,11 @@
def delete(self):
client = self.client
try:
+ LOG.debug("Deleting Nova Quotas for project with id %s",
+ self.project_id)
client.delete_quota_set(self.project_id)
except Exception:
- LOG.exception("Delete Quotas exception for 'project %s'.",
+ LOG.exception("Delete Nova Quotas exception for 'project %s'.",
self.project_id)
def dry_run(self):
@@ -371,6 +380,8 @@
def delete(self):
client = self.client
try:
+ LOG.debug("Deleting Network Quotas for project with id %s",
+ self.project_id)
client.reset_quotas(self.project_id)
except Exception:
LOG.exception("Delete Network Quotas exception for 'project %s'.",
@@ -419,7 +430,7 @@
if self.is_preserve:
networks = [network for network in networks
if network['id'] not in CONF_NETWORKS]
- LOG.debug("List count, %s Networks", networks)
+ LOG.debug("List count, %s Networks", len(networks))
return networks
def delete(self):
@@ -427,6 +438,7 @@
networks = self.list()
for n in networks:
try:
+ LOG.debug("Deleting Network with id %s", n['id'])
client.delete_network(n['id'])
except Exception:
LOG.exception("Delete Network %s exception.", n['id'])
@@ -461,6 +473,8 @@
flips = self.list()
for flip in flips:
try:
+ LOG.debug("Deleting Network Floating IP with id %s",
+ flip['id'])
client.delete_floatingip(flip['id'])
except Exception:
LOG.exception("Delete Network Floating IP %s exception.",
@@ -506,11 +520,14 @@
if net_info.is_router_interface_port(port)]
for port in ports:
try:
+ LOG.debug("Deleting port with id %s of router with id %s",
+ port['id'], rid)
client.remove_router_interface(rid, port_id=port['id'])
except Exception:
LOG.exception("Delete Router Interface exception for "
"'port %s' of 'router %s'.", port['id'], rid)
try:
+ LOG.debug("Deleting Router with id %s", rid)
client.delete_router(rid)
except Exception:
LOG.exception("Delete Router %s exception.", rid)
@@ -546,6 +563,8 @@
rules = self.list()
for rule in rules:
try:
+ LOG.debug("Deleting Metering Label Rule with id %s",
+ rule['id'])
client.delete_metering_label_rule(rule['id'])
except Exception:
LOG.exception("Delete Metering Label Rule %s exception.",
@@ -582,6 +601,7 @@
labels = self.list()
for label in labels:
try:
+ LOG.debug("Deleting Metering Label with id %s", label['id'])
client.delete_metering_label(label['id'])
except Exception:
LOG.exception("Delete Metering Label %s exception.",
@@ -622,6 +642,7 @@
ports = self.list()
for port in ports:
try:
+ LOG.debug("Deleting port with id %s", port['id'])
client.delete_port(port['id'])
except Exception:
LOG.exception("Delete Port %s exception.", port['id'])
@@ -663,6 +684,7 @@
secgroups = self.list()
for secgroup in secgroups:
try:
+ LOG.debug("Deleting security_group with id %s", secgroup['id'])
client.delete_security_group(secgroup['id'])
except Exception:
LOG.exception("Delete security_group %s exception.",
@@ -699,6 +721,7 @@
subnets = self.list()
for subnet in subnets:
try:
+ LOG.debug("Deleting subnet with id %s", subnet['id'])
client.delete_subnet(subnet['id'])
except Exception:
LOG.exception("Delete Subnet %s exception.", subnet['id'])
@@ -734,6 +757,7 @@
pools = self.list()
for pool in pools:
try:
+ LOG.debug("Deleting Subnet Pool with id %s", pool['id'])
client.delete_subnetpool(pool['id'])
except Exception:
LOG.exception("Delete Subnet Pool %s exception.", pool['id'])
@@ -762,8 +786,10 @@
if not self.is_save_state:
regions = [region for region in regions['regions'] if region['id']
not in self.saved_state_json['regions'].keys()]
+ LOG.debug("List count, %s Regions", len(regions))
return regions
else:
+ LOG.debug("List count, %s Regions", len(regions['regions']))
return regions['regions']
def delete(self):
@@ -771,6 +797,7 @@
regions = self.list()
for region in regions:
try:
+ LOG.debug("Deleting region with id %s", region['id'])
client.delete_region(region['id'])
except Exception:
LOG.exception("Delete Region %s exception.", region['id'])
@@ -812,6 +839,7 @@
flavors = self.list()
for flavor in flavors:
try:
+ LOG.debug("Deleting flavor with id %s", flavor['id'])
client.delete_flavor(flavor['id'])
except Exception:
LOG.exception("Delete Flavor %s exception.", flavor['id'])
@@ -857,6 +885,7 @@
images = self.list()
for image in images:
try:
+ LOG.debug("Deleting image with id %s", image['id'])
client.delete_image(image['id'])
except Exception:
LOG.exception("Delete Image %s exception.", image['id'])
@@ -900,6 +929,7 @@
users = self.list()
for user in users:
try:
+ LOG.debug("Deleting user with id %s", user['id'])
self.client.delete_user(user['id'])
except Exception:
LOG.exception("Delete User %s exception.", user['id'])
@@ -940,6 +970,7 @@
roles = self.list()
for role in roles:
try:
+ LOG.debug("Deleting role with id %s", role['id'])
self.client.delete_role(role['id'])
except Exception:
LOG.exception("Delete Role %s exception.", role['id'])
@@ -982,6 +1013,7 @@
projects = self.list()
for project in projects:
try:
+ LOG.debug("Deleting project with id %s", project['id'])
self.client.delete_project(project['id'])
except Exception:
LOG.exception("Delete project %s exception.", project['id'])
@@ -1018,6 +1050,7 @@
domains = self.list()
for domain in domains:
try:
+ LOG.debug("Deleting domain with id %s", domain['id'])
client.update_domain(domain['id'], enabled=False)
client.delete_domain(domain['id'])
except Exception:
diff --git a/tools/generate-tempest-plugins-list.sh b/tools/generate-tempest-plugins-list.sh
index 961cd09..33675ed 100755
--- a/tools/generate-tempest-plugins-list.sh
+++ b/tools/generate-tempest-plugins-list.sh
@@ -98,8 +98,8 @@
if [[ -r doc/source/data/tempest-plugins-registry.footer ]]; then
cat doc/source/data/tempest-plugins-registry.footer
fi
-) > doc/source/plugin-registry.rst
+) > doc/source/plugins/plugin-registry.rst
if [[ -n ${1} ]]; then
- cp doc/source/plugin-registry.rst ${1}/doc/source/plugin-registry.rst
+ cp doc/source/plugins/plugin-registry.rst ${1}/doc/source/plugins/plugin-registry.rst
fi