port servers negative tests into v3 Part2

this ports servers negative tests into v3and do some
corresponding change in the client. this also ports
shelve and create_backup tests into v3, add v3 support
in wait_for_server_status. this also removes the negative
test for personality, because it has been removed in nova
v3 api.

this also does some sync from v2 tests, applies the following
commit in v3 tests:
I4b06b148b5d9bceda0cef2d2bb5f3b72325928ac
I5b423b3424e1dfe6cf5f8fef0dc9538961c1f28d
Ibd04a1fdaab74fe59bcf16cc99d4ecf4821f4ae2

Partially implements blueprint nova-v3-api-tests

Change-Id: Ibfe22c33c26e39b86975965bbac0cc76a5f8836e
diff --git a/tempest/api/compute/v3/servers/test_server_actions.py b/tempest/api/compute/v3/servers/test_server_actions.py
index 8cd6c11..602bd5b 100644
--- a/tempest/api/compute/v3/servers/test_server_actions.py
+++ b/tempest/api/compute/v3/servers/test_server_actions.py
@@ -123,6 +123,7 @@
                                                    metadata=meta,
                                                    personality=personality,
                                                    admin_password=password)
+        self.addCleanup(self.client.rebuild, self.server_id, self.image_ref)
 
         # Verify the properties in the initial response are correct
         self.assertEqual(self.server_id, rebuilt_server['id'])
@@ -142,6 +143,35 @@
             linux_client = RemoteClient(server, self.ssh_user, password)
             linux_client.validate_authentication()
 
+    @attr(type='gate')
+    def test_rebuild_server_in_stop_state(self):
+        # The server in stop state  should be rebuilt using the provided
+        # image and remain in SHUTOFF state
+        resp, server = self.client.get_server(self.server_id)
+        old_image = server['image']['id']
+        new_image = self.image_ref_alt \
+            if old_image == self.image_ref else self.image_ref
+        resp, server = self.client.stop(self.server_id)
+        self.assertEqual(202, resp.status)
+        self.client.wait_for_server_status(self.server_id, 'SHUTOFF')
+        self.addCleanup(self.client.start, self.server_id)
+        resp, rebuilt_server = self.client.rebuild(self.server_id, new_image)
+        self.addCleanup(self.client.wait_for_server_status, self.server_id,
+                        'SHUTOFF')
+        self.addCleanup(self.client.rebuild, self.server_id, old_image)
+
+        # Verify the properties in the initial response are correct
+        self.assertEqual(self.server_id, rebuilt_server['id'])
+        rebuilt_image_id = rebuilt_server['image']['id']
+        self.assertEqual(new_image, rebuilt_image_id)
+        self.assertEqual(self.flavor_ref, rebuilt_server['flavor']['id'])
+
+        # Verify the server properties after the rebuild completes
+        self.client.wait_for_server_status(rebuilt_server['id'], 'SHUTOFF')
+        resp, server = self.client.get_server(rebuilt_server['id'])
+        rebuilt_image_id = server['image']['id']
+        self.assertEqual(new_image, rebuilt_image_id)
+
     def _detect_server_image_flavor(self, server_id):
         # Detects the current server image flavor ref.
         resp, server = self.client.get_server(self.server_id)
@@ -199,32 +229,115 @@
                 raise exceptions.TimeoutException(message)
 
     @attr(type='gate')
+    def test_create_backup(self):
+        # Positive test:create backup successfully and rotate backups correctly
+        # create the first and the second backup
+        backup1 = data_utils.rand_name('backup')
+        resp, _ = self.servers_client.create_backup(self.server_id,
+                                                    'daily',
+                                                    2,
+                                                    backup1)
+        oldest_backup_exist = True
+
+        # the oldest one should be deleted automatically in this test
+        def _clean_oldest_backup(oldest_backup):
+            if oldest_backup_exist:
+                self.images_client.delete_image(oldest_backup)
+
+        image1_id = data_utils.parse_image_id(resp['location'])
+        self.addCleanup(_clean_oldest_backup, image1_id)
+        self.assertEqual(202, resp.status)
+        self.images_client.wait_for_image_status(image1_id, 'active')
+
+        backup2 = data_utils.rand_name('backup')
+        self.servers_client.wait_for_server_status(self.server_id, 'ACTIVE')
+        resp, _ = self.servers_client.create_backup(self.server_id,
+                                                    'daily',
+                                                    2,
+                                                    backup2)
+        image2_id = data_utils.parse_image_id(resp['location'])
+        self.addCleanup(self.images_client.delete_image, image2_id)
+        self.assertEqual(202, resp.status)
+        self.images_client.wait_for_image_status(image2_id, 'active')
+
+        # verify they have been created
+        properties = {
+            'image_type': 'backup',
+            'backup_type': "daily",
+            'instance_uuid': self.server_id,
+        }
+        resp, image_list = self.images_client.image_list_detail(
+            properties,
+            sort_key='created_at',
+            sort_dir='asc')
+        self.assertEqual(200, resp.status)
+        self.assertEqual(2, len(image_list))
+        self.assertEqual((backup1, backup2),
+                         (image_list[0]['name'], image_list[1]['name']))
+
+        # create the third one, due to the rotation is 2,
+        # the first one will be deleted
+        backup3 = data_utils.rand_name('backup')
+        self.servers_client.wait_for_server_status(self.server_id, 'ACTIVE')
+        resp, _ = self.servers_client.create_backup(self.server_id,
+                                                    'daily',
+                                                    2,
+                                                    backup3)
+        image3_id = data_utils.parse_image_id(resp['location'])
+        self.addCleanup(self.images_client.delete_image, image3_id)
+        self.assertEqual(202, resp.status)
+        # the first back up should be deleted
+        self.images_client.wait_for_resource_deletion(image1_id)
+        oldest_backup_exist = False
+        resp, image_list = self.images_client.image_list_detail(
+            properties,
+            sort_key='created_at',
+            sort_dir='asc')
+        self.assertEqual(200, resp.status)
+        self.assertEqual(2, len(image_list))
+        self.assertEqual((backup2, backup3),
+                         (image_list[0]['name'], image_list[1]['name']))
+
+    def _get_output(self):
+        resp, output = self.servers_client.get_console_output(
+            self.server_id, 10)
+        self.assertEqual(200, resp.status)
+        self.assertTrue(output, "Console output was empty.")
+        lines = len(output.split('\n'))
+        self.assertEqual(lines, 10)
+
+    @attr(type='gate')
     def test_get_console_output(self):
         # Positive test:Should be able to GET the console output
         # for a given server_id and number of lines
-        def get_output():
-            resp, output = self.servers_client.get_console_output(
-                self.server_id, 10)
-            self.assertEqual(200, resp.status)
-            self.assertTrue(output, "Console output was empty.")
-            lines = len(output.split('\n'))
-            self.assertEqual(lines, 10)
-        self.wait_for(get_output)
 
-    @skip_because(bug="1014683")
+        # This reboot is necessary for outputting some console log after
+        # creating a instance backup. If a instance backup, the console
+        # log file is truncated and we cannot get any console log through
+        # "console-log" API.
+        # The detail is https://bugs.launchpad.net/nova/+bug/1251920
+        resp, body = self.servers_client.reboot(self.server_id, 'HARD')
+        self.assertEqual(202, resp.status)
+        self.servers_client.wait_for_server_status(self.server_id, 'ACTIVE')
+
+        self.wait_for(self._get_output)
+
     @attr(type='gate')
-    def test_get_console_output_server_id_in_reboot_status(self):
+    def test_get_console_output_server_id_in_shutoff_status(self):
         # Positive test:Should be able to GET the console output
-        # for a given server_id in reboot status
-        resp, output = self.servers_client.reboot(self.server_id, 'SOFT')
-        self.servers_client.wait_for_server_status(self.server_id,
-                                                   'REBOOT')
-        resp, output = self.servers_client.get_console_output(self.server_id,
-                                                              10)
-        self.assertEqual(200, resp.status)
-        self.assertIsNotNone(output)
-        lines = len(output.split('\n'))
-        self.assertEqual(lines, 10)
+        # for a given server_id in SHUTOFF status
+
+        # NOTE: SHUTOFF is irregular status. To avoid test instability,
+        #       one server is created only for this test without using
+        #       the server that was created in setupClass.
+        resp, server = self.create_test_server(wait_until='ACTIVE')
+        temp_server_id = server['id']
+
+        resp, server = self.servers_client.stop(temp_server_id)
+        self.assertEqual(202, resp.status)
+        self.servers_client.wait_for_server_status(temp_server_id, 'SHUTOFF')
+
+        self.wait_for(self._get_output)
 
     @attr(type='gate')
     def test_pause_unpause_server(self):
@@ -245,6 +358,30 @@
         self.client.wait_for_server_status(self.server_id, 'ACTIVE')
 
     @attr(type='gate')
+    def test_shelve_unshelve_server(self):
+        resp, server = self.client.shelve_server(self.server_id)
+        self.assertEqual(202, resp.status)
+
+        offload_time = self.config.compute.shelved_offload_time
+        if offload_time >= 0:
+            self.client.wait_for_server_status(self.server_id,
+                                               'SHELVED_OFFLOADED',
+                                               extra_timeout=offload_time)
+        else:
+            self.client.wait_for_server_status(self.server_id,
+                                               'SHELVED')
+
+        resp, server = self.client.get_server(self.server_id)
+        image_name = server['name'] + '-shelved'
+        resp, images = self.images_client.image_list(name=image_name)
+        self.assertEqual(1, len(images))
+        self.assertEqual(image_name, images[0]['name'])
+
+        resp, server = self.client.unshelve_server(self.server_id)
+        self.assertEqual(202, resp.status)
+        self.client.wait_for_server_status(self.server_id, 'ACTIVE')
+
+    @attr(type='gate')
     def test_stop_start_server(self):
         resp, server = self.servers_client.stop(self.server_id)
         self.assertEqual(202, resp.status)
diff --git a/tempest/api/compute/v3/servers/test_servers_negative.py b/tempest/api/compute/v3/servers/test_servers_negative.py
index 6532032..85fe47c 100644
--- a/tempest/api/compute/v3/servers/test_servers_negative.py
+++ b/tempest/api/compute/v3/servers/test_servers_negative.py
@@ -25,11 +25,11 @@
 from tempest import test
 
 
-class ServersNegativeTestJSON(base.BaseV2ComputeTest):
+class ServersNegativeV3TestJSON(base.BaseV3ComputeTest):
     _interface = 'json'
 
     def setUp(self):
-        super(ServersNegativeTestJSON, self).setUp()
+        super(ServersNegativeV3TestJSON, self).setUp()
         try:
             self.client.wait_for_server_status(self.server_id, 'ACTIVE')
         except Exception:
@@ -37,10 +37,10 @@
 
     @classmethod
     def setUpClass(cls):
-        super(ServersNegativeTestJSON, cls).setUpClass()
+        super(ServersNegativeV3TestJSON, cls).setUpClass()
         cls.client = cls.servers_client
         cls.alt_os = clients.AltManager()
-        cls.alt_client = cls.alt_os.servers_client
+        cls.alt_client = cls.alt_os.servers_v3_client
         resp, server = cls.create_test_server(wait_until='ACTIVE')
         cls.server_id = server['id']
 
@@ -53,18 +53,6 @@
                           name='')
 
     @test.attr(type=['negative', 'gate'])
-    def test_personality_file_contents_not_encoded(self):
-        # Use an unencoded file when creating a server with personality
-
-        file_contents = 'This is a test file.'
-        person = [{'path': '/etc/testfile.txt',
-                   'contents': file_contents}]
-
-        self.assertRaises(exceptions.BadRequest,
-                          self.create_test_server,
-                          personality=person)
-
-    @test.attr(type=['negative', 'gate'])
     def test_create_with_invalid_image(self):
         # Create a server with an unknown image
 
@@ -86,7 +74,7 @@
 
         IPv4 = '1.1.1.1.1.1'
         self.assertRaises(exceptions.BadRequest,
-                          self.create_test_server, accessIPv4=IPv4)
+                          self.create_test_server, access_ip_v4=IPv4)
 
     @test.attr(type=['negative', 'gate'])
     def test_invalid_ip_v6_address(self):
@@ -95,7 +83,7 @@
         IPv6 = 'notvalid'
 
         self.assertRaises(exceptions.BadRequest,
-                          self.create_test_server, accessIPv6=IPv6)
+                          self.create_test_server, access_ip_v6=IPv6)
 
     @test.attr(type=['negative', 'gate'])
     def test_resize_nonexistent_server(self):
@@ -186,12 +174,12 @@
                           self.create_test_server,
                           name=server_name)
 
+    @test.skip_because(bug="1208743")
     @test.attr(type=['negative', 'gate'])
     def test_create_with_invalid_network_uuid(self):
         # Pass invalid network uuid while creating a server
 
         networks = [{'fixed_ip': '10.0.1.1', 'uuid': 'a-b-c-d-e-f-g-h-i-j'}]
-
         self.assertRaises(exceptions.BadRequest,
                           self.create_test_server,
                           networks=networks)
@@ -421,8 +409,7 @@
 
         resp, server = self.client.get_server(self.server_id)
         image_name = server['name'] + '-shelved'
-        params = {'name': image_name}
-        resp, images = self.images_client.list_images(params)
+        resp, images = self.images_client.image_list(name=image_name)
         self.assertEqual(1, len(images))
         self.assertEqual(image_name, images[0]['name'])
 
@@ -445,5 +432,5 @@
                           self.server_id)
 
 
-class ServersNegativeTestXML(ServersNegativeTestJSON):
+class ServersNegativeV3TestXML(ServersNegativeV3TestJSON):
     _interface = 'xml'
diff --git a/tempest/common/waiters.py b/tempest/common/waiters.py
index aedba15..ce620a8 100644
--- a/tempest/common/waiters.py
+++ b/tempest/common/waiters.py
@@ -29,7 +29,10 @@
     """Waits for a server to reach a given status."""
 
     def _get_task_state(body):
-        task_state = body.get('OS-EXT-STS:task_state', None)
+        if client.service == CONF.compute.catalog_v3_type:
+            task_state = body.get("os-extended-status:task_state", None)
+        else:
+            task_state = body.get('OS-EXT-STS:task_state', None)
         return task_state
 
     # NOTE(afazekas): UNKNOWN status possible on ERROR
diff --git a/tempest/services/compute/v3/json/servers_client.py b/tempest/services/compute/v3/json/servers_client.py
index a7fcc6d..a486801 100644
--- a/tempest/services/compute/v3/json/servers_client.py
+++ b/tempest/services/compute/v3/json/servers_client.py
@@ -64,12 +64,14 @@
             'flavor_ref': flavor_ref
         }
 
-        for option in ['personality', 'admin_password', 'key_name',
-                       'security_groups', 'networks',
+        for option in ['personality', 'admin_password', 'key_name', 'networks',
+                       ('os-security-groups:security_groups',
+                        'security_groups'),
                        ('os-user-data:user_data', 'user_data'),
                        ('os-availability-zone:availability_zone',
                         'availability_zone'),
-                       'access_ip_v4', 'access_ip_v6',
+                       ('os-access-ips:access_ip_v4', 'access_ip_v4'),
+                       ('os-access-ips:access_ip_v6', 'access_ip_v6'),
                        ('os-multiple-create:min_count', 'min_count'),
                        ('os-multiple-create:max_count', 'max_count'),
                        ('metadata', 'meta'),
@@ -205,6 +207,13 @@
             body = json.loads(body)[response_key]
         return resp, body
 
+    def create_backup(self, server_id, backup_type, rotation, name):
+        """Backup a server instance."""
+        return self.action(server_id, "create_backup", None,
+                           backup_type=backup_type,
+                           rotation=rotation,
+                           name=name)
+
     def change_password(self, server_id, admin_password):
         """Changes the root password for the server."""
         return self.action(server_id, 'change_password', None,
@@ -356,6 +365,14 @@
         """Resets the state of a server to active/error."""
         return self.action(server_id, 'reset_state', None, state=state)
 
+    def shelve_server(self, server_id, **kwargs):
+        """Shelves the provided server."""
+        return self.action(server_id, 'shelve', None, **kwargs)
+
+    def unshelve_server(self, server_id, **kwargs):
+        """Un-shelves the provided server."""
+        return self.action(server_id, 'unshelve', None, **kwargs)
+
     def get_console_output(self, server_id, length):
         return self.action(server_id, 'get_console_output', 'output',
                            length=length)
@@ -388,3 +405,11 @@
                               (str(server_id), str(request_id)))
         body = json.loads(body)
         return resp, body['instance_action']
+
+    def force_delete_server(self, server_id, **kwargs):
+        """Force delete a server."""
+        return self.action(server_id, 'force_delete', None, **kwargs)
+
+    def restore_soft_deleted_server(self, server_id, **kwargs):
+        """Restore a soft-deleted server."""
+        return self.action(server_id, 'restore', None, **kwargs)
diff --git a/tempest/services/compute/v3/xml/servers_client.py b/tempest/services/compute/v3/xml/servers_client.py
index 7af4161..c3381a3 100644
--- a/tempest/services/compute/v3/xml/servers_client.py
+++ b/tempest/services/compute/v3/xml/servers_client.py
@@ -214,6 +214,14 @@
         """Resets the state of a server to active/error."""
         return self.action(server_id, 'reset_state', None, state=state)
 
+    def shelve_server(self, server_id, **kwargs):
+        """Shelves the provided server."""
+        return self.action(server_id, 'shelve', None, **kwargs)
+
+    def unshelve_server(self, server_id, **kwargs):
+        """Un-shelves the provided server."""
+        return self.action(server_id, 'unshelve', None, **kwargs)
+
     def delete_server(self, server_id):
         """Deletes the given server."""
         return self.delete("servers/%s" % str(server_id))
@@ -299,7 +307,17 @@
                          flavor_ref=flavor_ref,
                          image_ref=image_ref,
                          name=name)
-        attrs = ["admin_password", "access_ip_v4", "access_ip_v6", "key_name",
+        attrs = ["admin_password", "key_name",
+                 ('os-access-ips:access_ip_v4',
+                  'access_ip_v4',
+                  'xmlns:os-access-ips',
+                  "http://docs.openstack.org/compute/ext/"
+                  "os-access-ips/api/v3"),
+                 ('os-access-ips:access_ip_v6',
+                  'access_ip_v6',
+                  'xmlns:os-access-ips',
+                  "http://docs.openstack.org/compute/ext/"
+                  "os-access-ips/api/v3"),
                  ("os-user-data:user_data",
                   'user_data',
                   'xmlns:os-user-data',
@@ -346,7 +364,10 @@
                     server.add_attr(post_param, value)
 
         if 'security_groups' in kwargs:
-            secgroups = Element("security_groups")
+            server.add_attr("xmlns:os-security-groups",
+                            "http://docs.openstack.org/compute/ext/"
+                            "securitygroups/api/v3")
+            secgroups = Element("os-security-groups:security_groups")
             server.append(secgroups)
             for secgroup in kwargs['security_groups']:
                 s = Element("security_group", name=secgroup['name'])
@@ -441,6 +462,13 @@
             body = xml_to_json(etree.fromstring(body))
         return resp, body
 
+    def create_backup(self, server_id, backup_type, rotation, name):
+        """Backup a server instance."""
+        return self.action(server_id, "create_backup", None,
+                           backup_type=backup_type,
+                           rotation=rotation,
+                           name=name)
+
     def change_password(self, server_id, password):
         return self.action(server_id, "change_password", None,
                            admin_password=password)
@@ -621,3 +649,11 @@
                               (server_id, request_id), self.headers)
         body = xml_to_json(etree.fromstring(body))
         return resp, body
+
+    def force_delete_server(self, server_id, **kwargs):
+        """Force delete a server."""
+        return self.action(server_id, 'force_delete', None, **kwargs)
+
+    def restore_soft_deleted_server(self, server_id, **kwargs):
+        """Restore a soft-deleted server."""
+        return self.action(server_id, 'restore', None, **kwargs)