diff --git a/tools/generate-tempest-plugins-list.py b/tools/generate-tempest-plugins-list.py
index 9ebeee2..f07410f 100644
--- a/tools/generate-tempest-plugins-list.py
+++ b/tools/generate-tempest-plugins-list.py
@@ -84,6 +84,8 @@
     except HTTPError as err:
         if err.code == 404:
             return False
+        # We should not ignore non 404 errors.
+        raise err
     p = re.compile(r'^tempest\.test_plugins', re.M)
     if p.findall(r.read().decode('utf-8')):
         return True