Merge "Improved tempest plugin sanity check script"
diff --git a/.zuul.yaml b/.zuul.yaml
index 45c957e..edaa959 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -421,10 +421,12 @@
- git.openstack.org/openstack/sahara-tests
- git.openstack.org/openstack/senlin
- git.openstack.org/openstack/senlin-tempest-plugin
+ - git.openstack.org/openstack/solum-tempest-plugin
- git.openstack.org/openstack/tap-as-a-service
- git.openstack.org/openstack/telemetry-tempest-plugin
- git.openstack.org/openstack/tempest-horizon
- git.openstack.org/openstack/tobiko
+ - git.openstack.org/openstack/trio2o
- git.openstack.org/openstack/tripleo-common-tempest-plugin
- git.openstack.org/openstack/trove-tempest-plugin
- git.openstack.org/openstack/valet
diff --git a/tools/tempest-plugin-sanity.sh b/tools/tempest-plugin-sanity.sh
index 661329b..16e7b8c 100644
--- a/tools/tempest-plugin-sanity.sh
+++ b/tools/tempest-plugin-sanity.sh
@@ -18,20 +18,17 @@
# This script is intended to check the sanity of tempest plugins against
# tempest master.
# What it does:
-# * Creates the virtualenv
-# * Install tempest
# * Retrieve the project lists having tempest plugin if project name is
# given.
-# * For each project in a list, It does:
+# * For each project in a list, it does:
+# * Create virtualenv and install tempest in it
# * Clone the Project
# * Install the Project and also installs dependencies from
# test-requirements.txt.
# * Create Tempest workspace
# * List tempest plugins
# * List tempest plugins tests
-# * Uninstall the project and its dependencies
-# * Again Install tempest
-# * Again repeat the step from cloning project
+# * Delete virtualenv and project repo
#
# If one of the step fails, The script will exit with failure.
@@ -46,8 +43,39 @@
# retrieve a list of projects having tempest plugins
PROJECT_LIST="$(python tools/generate-tempest-plugins-list.py)"
-# List of projects having tempest plugin stale or unmaintained from long time
-BLACKLIST="networking-plumgrid,trio2o"
+# List of projects having tempest plugin stale or unmaintained for a long time
+# (6 months or more)
+# TODO(masayukig): Some of these can be removed from BLACKLIST in the future.
+# airship-tempest-plugin: https://review.openstack.org/#/c/634387/
+# barbican-tempest-plugin: https://review.openstack.org/#/c/634631/
+# intel-nfv-ci-tests: https://review.openstack.org/#/c/634640/
+# networking-ansible: https://review.openstack.org/#/c/634647/
+# networking-generic-switch: https://review.openstack.org/#/c/634846/
+# networking-l2gw-tempest-plugin: https://review.openstack.org/#/c/635093/
+# networking-midonet: https://review.openstack.org/#/c/635096/
+# networking-plumgrid: https://review.openstack.org/#/c/635096/
+# networking-spp: https://review.openstack.org/#/c/635098/
+# neutron-dynamic-routing: https://review.openstack.org/#/c/637718/
+# neutron-vpnaas: https://review.openstack.org/#/c/637719/
+# nova-lxd: https://review.openstack.org/#/c/638334/
+# valet: https://review.openstack.org/#/c/638339/
+# vitrage-tempest-plugin: https://review.openstack.org/#/c/639003/
+BLACKLIST="
+airship-tempest-plugin
+barbican-tempest-plugin
+intel-nfv-ci-tests
+networking-ansible
+networking-generic-switch
+networking-l2gw-tempest-plugin
+networking-midonet
+networking-plumgrid
+networking-spp
+neutron-dynamic-routing
+neutron-vpnaas
+nova-lxd
+valet
+vitrage-tempest-plugin
+"
# Function to clone project using zuul-cloner or from git
function clone_project() {
@@ -63,14 +91,16 @@
fi
}
-# Create virtualenv to perform sanity operation
-SANITY_DIR=$(pwd)
-virtualenv "$SANITY_DIR"/.venv
-export TVENV="$SANITY_DIR/tools/with_venv.sh"
-cd "$SANITY_DIR"
+# function to create virtualenv to perform sanity operation
+function prepare_workspace() {
+ SANITY_DIR=$(pwd)
+ virtualenv --clear "$SANITY_DIR"/.venv
+ export TVENV="$SANITY_DIR/tools/with_venv.sh"
+ cd "$SANITY_DIR"
-# Install tempest in a venv
-"$TVENV" pip install .
+ # Install tempest with test dependencies in a venv
+ "$TVENV" pip install -e . -r test-requirements.txt
+}
# Function to install project
function install_project() {
@@ -83,30 +113,31 @@
# Function to perform sanity checking on Tempest plugin
function tempest_sanity() {
- "$TVENV" tempest init "$SANITY_DIR"/tempest_sanity
- cd "$SANITY_DIR"/tempest_sanity
- "$TVENV" tempest list-plugins
+ "$TVENV" tempest init "$SANITY_DIR"/tempest_sanity && \
+ cd "$SANITY_DIR"/tempest_sanity && \
+ "$TVENV" tempest list-plugins && \
"$TVENV" tempest run -l
+ retval=$?
# Delete tempest workspace
+ # NOTE: Cleaning should be done even if an error occurs.
"$TVENV" tempest workspace remove --name tempest_sanity --rmdir
cd "$SANITY_DIR"
-}
-
-# Function to uninstall project
-function uninstall_project() {
- "$TVENV" pip uninstall -y "$SANITY_DIR"/openstack/"$1"
- # Check for *requirements.txt file in a project then uninstall it.
- if [ -e "$SANITY_DIR"/openstack/"$1"/*requirements.txt ]; then
- "$TVENV" pip uninstall -y -r "$SANITY_DIR"/openstack/"$1"/*requirements.txt
- fi
+ # Remove the sanity workspace in case of remaining
+ rm -fr "$SANITY_DIR"/tempest_sanity
# Remove the project directory after sanity run
rm -fr "$SANITY_DIR"/openstack/"$1"
+
+ return $retval
}
# Function to run sanity check on each project
function plugin_sanity_check() {
- clone_project "$1" && install_project "$1" && tempest_sanity "$1" \
- && uninstall_project "$1" && "$TVENV" pip install .
+ prepare_workspace && \
+ clone_project "$1" && \
+ install_project "$1" && \
+ tempest_sanity "$1"
+
+ return $?
}
# Log status
@@ -117,11 +148,12 @@
# Remove blacklisted tempest plugins
if ! [[ `echo $BLACKLIST | grep -c $project ` -gt 0 ]]; then
plugin_sanity_check $project && passed_plugin+=", $project" || \
- failed_plugin+=", $project"
+ failed_plugin+="$project, " > $SANITY_DIR/$project.txt
fi
done
# Check for failed status
if [[ -n $failed_plugin ]]; then
+ echo "Failed Plugins: $failed_plugin"
exit 1
fi