| # Copyright 2012 OpenStack Foundation |
| # Copyright 2013 IBM Corp. |
| # All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| import collections |
| import inspect |
| import json |
| from lxml import etree |
| import re |
| import time |
| |
| import jsonschema |
| |
| from tempest.common import http |
| from tempest.common import xml_utils as common |
| from tempest import config |
| from tempest import exceptions |
| from tempest.openstack.common import log as logging |
| |
| CONF = config.CONF |
| |
| # redrive rate limited calls at most twice |
| MAX_RECURSION_DEPTH = 2 |
| TOKEN_CHARS_RE = re.compile('^[-A-Za-z0-9+/=]*$') |
| |
| # All the successful HTTP status codes from RFC 2616 |
| HTTP_SUCCESS = (200, 201, 202, 203, 204, 205, 206) |
| |
| |
| class RestClient(object): |
| |
| TYPE = "json" |
| |
| # This is used by _parse_resp method |
| # Redefine it for purposes of your xml service client |
| # List should contain top-xml_tag-names of data, which is like list/array |
| # For example, in keystone it is users, roles, tenants and services |
| # All of it has children with same tag-names |
| list_tags = [] |
| |
| # This is used by _parse_resp method too |
| # Used for selection of dict-like xmls, |
| # like metadata for Vms in nova, and volumes in cinder |
| dict_tags = ["metadata", ] |
| |
| LOG = logging.getLogger(__name__) |
| |
| def __init__(self, auth_provider): |
| self.auth_provider = auth_provider |
| |
| self.endpoint_url = None |
| self.service = None |
| # The version of the API this client implements |
| self.api_version = None |
| self._skip_path = False |
| self.build_interval = CONF.compute.build_interval |
| self.build_timeout = CONF.compute.build_timeout |
| self.general_header_lc = set(('cache-control', 'connection', |
| 'date', 'pragma', 'trailer', |
| 'transfer-encoding', 'via', |
| 'warning')) |
| self.response_header_lc = set(('accept-ranges', 'age', 'etag', |
| 'location', 'proxy-authenticate', |
| 'retry-after', 'server', |
| 'vary', 'www-authenticate')) |
| dscv = CONF.identity.disable_ssl_certificate_validation |
| self.http_obj = http.ClosingHttp( |
| disable_ssl_certificate_validation=dscv) |
| |
| def _get_type(self): |
| return self.TYPE |
| |
| def get_headers(self, accept_type=None, send_type=None): |
| if accept_type is None: |
| accept_type = self._get_type() |
| if send_type is None: |
| send_type = self._get_type() |
| return {'Content-Type': 'application/%s' % send_type, |
| 'Accept': 'application/%s' % accept_type} |
| |
| def __str__(self): |
| STRING_LIMIT = 80 |
| str_format = ("config:%s, service:%s, base_url:%s, " |
| "filters: %s, build_interval:%s, build_timeout:%s" |
| "\ntoken:%s..., \nheaders:%s...") |
| return str_format % (CONF, self.service, self.base_url, |
| self.filters, self.build_interval, |
| self.build_timeout, |
| str(self.token)[0:STRING_LIMIT], |
| str(self.get_headers())[0:STRING_LIMIT]) |
| |
| def _get_region(self, service): |
| """ |
| Returns the region for a specific service |
| """ |
| service_region = None |
| for cfgname in dir(CONF._config): |
| # Find all config.FOO.catalog_type and assume FOO is a service. |
| cfg = getattr(CONF, cfgname) |
| catalog_type = getattr(cfg, 'catalog_type', None) |
| if catalog_type == service: |
| service_region = getattr(cfg, 'region', None) |
| if not service_region: |
| service_region = CONF.identity.region |
| return service_region |
| |
| def _get_endpoint_type(self, service): |
| """ |
| Returns the endpoint type for a specific service |
| """ |
| # If the client requests a specific endpoint type, then be it |
| if self.endpoint_url: |
| return self.endpoint_url |
| endpoint_type = None |
| for cfgname in dir(CONF._config): |
| # Find all config.FOO.catalog_type and assume FOO is a service. |
| cfg = getattr(CONF, cfgname) |
| catalog_type = getattr(cfg, 'catalog_type', None) |
| if catalog_type == service: |
| endpoint_type = getattr(cfg, 'endpoint_type', 'publicURL') |
| break |
| # Special case for compute v3 service which hasn't its own |
| # configuration group |
| else: |
| if service == CONF.compute.catalog_v3_type: |
| endpoint_type = CONF.compute.endpoint_type |
| return endpoint_type |
| |
| @property |
| def user(self): |
| return self.auth_provider.credentials.get('username', None) |
| |
| @property |
| def tenant_name(self): |
| return self.auth_provider.credentials.get('tenant_name', None) |
| |
| @property |
| def password(self): |
| return self.auth_provider.credentials.get('password', None) |
| |
| @property |
| def base_url(self): |
| return self.auth_provider.base_url(filters=self.filters) |
| |
| @property |
| def token(self): |
| return self.auth_provider.get_token() |
| |
| @property |
| def filters(self): |
| _filters = dict( |
| service=self.service, |
| endpoint_type=self._get_endpoint_type(self.service), |
| region=self._get_region(self.service) |
| ) |
| if self.api_version is not None: |
| _filters['api_version'] = self.api_version |
| if self._skip_path: |
| _filters['skip_path'] = self._skip_path |
| return _filters |
| |
| def skip_path(self): |
| """ |
| When set, ignore the path part of the base URL from the catalog |
| """ |
| self._skip_path = True |
| |
| def reset_path(self): |
| """ |
| When reset, use the base URL from the catalog as-is |
| """ |
| self._skip_path = False |
| |
| def expected_success(self, expected_code, read_code): |
| assert_msg = ("This function only allowed to use for HTTP status" |
| "codes which explicitly defined in the RFC 2616. {0}" |
| " is not a defined Success Code!").format(expected_code) |
| assert expected_code in HTTP_SUCCESS, assert_msg |
| |
| # NOTE(afazekas): the http status code above 400 is processed by |
| # the _error_checker method |
| if read_code < 400 and read_code != expected_code: |
| pattern = """Unexpected http success status code {0}, |
| The expected status code is {1}""" |
| details = pattern.format(read_code, expected_code) |
| raise exceptions.InvalidHttpSuccessCode(details) |
| |
| def post(self, url, body, headers=None, extra_headers=False): |
| return self.request('POST', url, extra_headers, headers, body) |
| |
| def get(self, url, headers=None, extra_headers=False): |
| return self.request('GET', url, extra_headers, headers) |
| |
| def delete(self, url, headers=None, body=None, extra_headers=False): |
| return self.request('DELETE', url, extra_headers, headers, body) |
| |
| def patch(self, url, body, headers=None, extra_headers=False): |
| return self.request('PATCH', url, extra_headers, headers, body) |
| |
| def put(self, url, body, headers=None, extra_headers=False): |
| return self.request('PUT', url, extra_headers, headers, body) |
| |
| def head(self, url, headers=None, extra_headers=False): |
| return self.request('HEAD', url, extra_headers, headers) |
| |
| def copy(self, url, headers=None, extra_headers=False): |
| return self.request('COPY', url, extra_headers, headers) |
| |
| def get_versions(self): |
| resp, body = self.get('') |
| body = self._parse_resp(body) |
| versions = map(lambda x: x['id'], body) |
| return resp, versions |
| |
| def _find_caller(self): |
| """Find the caller class and test name. |
| |
| Because we know that the interesting things that call us are |
| test_* methods, and various kinds of setUp / tearDown, we |
| can look through the call stack to find appropriate methods, |
| and the class we were in when those were called. |
| """ |
| caller_name = None |
| names = [] |
| frame = inspect.currentframe() |
| is_cleanup = False |
| # Start climbing the ladder until we hit a good method |
| while True: |
| try: |
| frame = frame.f_back |
| name = frame.f_code.co_name |
| names.append(name) |
| if re.search("^(test_|setUp|tearDown)", name): |
| cname = "" |
| if 'self' in frame.f_locals: |
| cname = frame.f_locals['self'].__class__.__name__ |
| if 'cls' in frame.f_locals: |
| cname = frame.f_locals['cls'].__name__ |
| caller_name = cname + ":" + name |
| break |
| elif re.search("^_run_cleanup", name): |
| is_cleanup = True |
| else: |
| cname = "" |
| if 'self' in frame.f_locals: |
| cname = frame.f_locals['self'].__class__.__name__ |
| if 'cls' in frame.f_locals: |
| cname = frame.f_locals['cls'].__name__ |
| |
| # the fact that we are running cleanups is indicated pretty |
| # deep in the stack, so if we see that we want to just |
| # start looking for a real class name, and declare victory |
| # once we do. |
| if is_cleanup and cname: |
| if not re.search("^RunTest", cname): |
| caller_name = cname + ":_run_cleanups" |
| break |
| except Exception: |
| break |
| # prevents frame leaks |
| del frame |
| if caller_name is None: |
| self.LOG.debug("Sane call name not found in %s" % names) |
| return caller_name |
| |
| def _get_request_id(self, resp): |
| for i in ('x-openstack-request-id', 'x-compute-request-id'): |
| if i in resp: |
| return resp[i] |
| return "" |
| |
| def _log_request(self, method, req_url, resp, |
| secs="", req_headers=None, |
| req_body=None, resp_body=None): |
| # if we have the request id, put it in the right part of the log |
| extra = dict(request_id=self._get_request_id(resp)) |
| # NOTE(sdague): while we still have 6 callers to this function |
| # we're going to just provide work around on who is actually |
| # providing timings by gracefully adding no content if they don't. |
| # Once we're down to 1 caller, clean this up. |
| caller_name = self._find_caller() |
| if secs: |
| secs = " %.3fs" % secs |
| self.LOG.info( |
| 'Request (%s): %s %s %s%s' % ( |
| caller_name, |
| resp['status'], |
| method, |
| req_url, |
| secs), |
| extra=extra) |
| |
| # We intentionally duplicate the info content because in a parallel |
| # world this is important to match |
| trace_regex = CONF.debug.trace_requests |
| if trace_regex and re.search(trace_regex, caller_name): |
| log_fmt = """Request (%s): %s %s %s%s |
| Request - Headers: %s |
| Body: %s |
| Response - Headers: %s |
| Body: %s""" |
| |
| self.LOG.debug( |
| log_fmt % ( |
| caller_name, |
| resp['status'], |
| method, |
| req_url, |
| secs, |
| str(req_headers), |
| str(req_body)[:2048], |
| str(resp), |
| str(resp_body)[:2048]), |
| extra=extra) |
| |
| def _parse_resp(self, body): |
| if self._get_type() is "json": |
| body = json.loads(body) |
| |
| # We assume, that if the first value of the deserialized body's |
| # item set is a dict or a list, that we just return the first value |
| # of deserialized body. |
| # Essentially "cutting out" the first placeholder element in a body |
| # that looks like this: |
| # |
| # { |
| # "users": [ |
| # ... |
| # ] |
| # } |
| try: |
| # Ensure there are not more than one top-level keys |
| if len(body.keys()) > 1: |
| return body |
| # Just return the "wrapped" element |
| first_key, first_item = body.items()[0] |
| if isinstance(first_item, (dict, list)): |
| return first_item |
| except (ValueError, IndexError): |
| pass |
| return body |
| elif self._get_type() is "xml": |
| element = etree.fromstring(body) |
| if any(s in element.tag for s in self.dict_tags): |
| # Parse dictionary-like xmls (metadata, etc) |
| dictionary = {} |
| for el in element.getchildren(): |
| dictionary[u"%s" % el.get("key")] = u"%s" % el.text |
| return dictionary |
| if any(s in element.tag for s in self.list_tags): |
| # Parse list-like xmls (users, roles, etc) |
| array = [] |
| for child in element.getchildren(): |
| array.append(common.xml_to_json(child)) |
| return array |
| |
| # Parse one-item-like xmls (user, role, etc) |
| return common.xml_to_json(element) |
| |
| def response_checker(self, method, resp, resp_body): |
| if (resp.status in set((204, 205, 304)) or resp.status < 200 or |
| method.upper() == 'HEAD') and resp_body: |
| raise exceptions.ResponseWithNonEmptyBody(status=resp.status) |
| # NOTE(afazekas): |
| # If the HTTP Status Code is 205 |
| # 'The response MUST NOT include an entity.' |
| # A HTTP entity has an entity-body and an 'entity-header'. |
| # In the HTTP response specification (Section 6) the 'entity-header' |
| # 'generic-header' and 'response-header' are in OR relation. |
| # All headers not in the above two group are considered as entity |
| # header in every interpretation. |
| |
| if (resp.status == 205 and |
| 0 != len(set(resp.keys()) - set(('status',)) - |
| self.response_header_lc - self.general_header_lc)): |
| raise exceptions.ResponseWithEntity() |
| # NOTE(afazekas) |
| # Now the swift sometimes (delete not empty container) |
| # returns with non json error response, we can create new rest class |
| # for swift. |
| # Usually RFC2616 says error responses SHOULD contain an explanation. |
| # The warning is normal for SHOULD/SHOULD NOT case |
| |
| # Likely it will cause an error |
| if not resp_body and resp.status >= 400: |
| self.LOG.warning("status >= 400 response with empty body") |
| |
| def _request(self, method, url, headers=None, body=None): |
| """A simple HTTP request interface.""" |
| # Authenticate the request with the auth provider |
| req_url, req_headers, req_body = self.auth_provider.auth_request( |
| method, url, headers, body, self.filters) |
| |
| # Do the actual request, and time it |
| start = time.time() |
| resp, resp_body = self.http_obj.request( |
| req_url, method, headers=req_headers, body=req_body) |
| end = time.time() |
| self._log_request(method, req_url, resp, secs=(end - start), |
| req_headers=req_headers, req_body=req_body, |
| resp_body=resp_body) |
| |
| # Verify HTTP response codes |
| self.response_checker(method, resp, resp_body) |
| |
| return resp, resp_body |
| |
| def request(self, method, url, extra_headers=False, headers=None, |
| body=None): |
| # if extra_headers is True |
| # default headers would be added to headers |
| retry = 0 |
| |
| if headers is None: |
| # NOTE(vponomaryov): if some client do not need headers, |
| # it should explicitly pass empty dict |
| headers = self.get_headers() |
| elif extra_headers: |
| try: |
| headers = headers.copy() |
| headers.update(self.get_headers()) |
| except (ValueError, TypeError): |
| headers = self.get_headers() |
| |
| resp, resp_body = self._request(method, url, |
| headers=headers, body=body) |
| |
| while (resp.status == 413 and |
| 'retry-after' in resp and |
| not self.is_absolute_limit( |
| resp, self._parse_resp(resp_body)) and |
| retry < MAX_RECURSION_DEPTH): |
| retry += 1 |
| delay = int(resp['retry-after']) |
| time.sleep(delay) |
| resp, resp_body = self._request(method, url, |
| headers=headers, body=body) |
| self._error_checker(method, url, headers, body, |
| resp, resp_body) |
| return resp, resp_body |
| |
| def _error_checker(self, method, url, |
| headers, body, resp, resp_body): |
| |
| # NOTE(mtreinish): Check for httplib response from glance_http. The |
| # object can't be used here because importing httplib breaks httplib2. |
| # If another object from a class not imported were passed here as |
| # resp this could possibly fail |
| if str(type(resp)) == "<type 'instance'>": |
| ctype = resp.getheader('content-type') |
| else: |
| try: |
| ctype = resp['content-type'] |
| # NOTE(mtreinish): Keystone delete user responses doesn't have a |
| # content-type header. (They don't have a body) So just pretend it |
| # is set. |
| except KeyError: |
| ctype = 'application/json' |
| |
| # It is not an error response |
| if resp.status < 400: |
| return |
| |
| JSON_ENC = ['application/json', 'application/json; charset=utf-8'] |
| # NOTE(mtreinish): This is for compatibility with Glance and swift |
| # APIs. These are the return content types that Glance api v1 |
| # (and occasionally swift) are using. |
| TXT_ENC = ['text/plain', 'text/html', 'text/html; charset=utf-8', |
| 'text/plain; charset=utf-8'] |
| XML_ENC = ['application/xml', 'application/xml; charset=utf-8'] |
| |
| if ctype.lower() in JSON_ENC or ctype.lower() in XML_ENC: |
| parse_resp = True |
| elif ctype.lower() in TXT_ENC: |
| parse_resp = False |
| else: |
| raise exceptions.InvalidContentType(str(resp.status)) |
| |
| if resp.status == 401 or resp.status == 403: |
| raise exceptions.Unauthorized(resp_body) |
| |
| if resp.status == 404: |
| raise exceptions.NotFound(resp_body) |
| |
| if resp.status == 400: |
| if parse_resp: |
| resp_body = self._parse_resp(resp_body) |
| raise exceptions.BadRequest(resp_body) |
| |
| if resp.status == 409: |
| if parse_resp: |
| resp_body = self._parse_resp(resp_body) |
| raise exceptions.Conflict(resp_body) |
| |
| if resp.status == 413: |
| if parse_resp: |
| resp_body = self._parse_resp(resp_body) |
| if self.is_absolute_limit(resp, resp_body): |
| raise exceptions.OverLimit(resp_body) |
| else: |
| raise exceptions.RateLimitExceeded(resp_body) |
| |
| if resp.status == 422: |
| if parse_resp: |
| resp_body = self._parse_resp(resp_body) |
| raise exceptions.UnprocessableEntity(resp_body) |
| |
| if resp.status in (500, 501): |
| message = resp_body |
| if parse_resp: |
| try: |
| resp_body = self._parse_resp(resp_body) |
| except ValueError: |
| # If response body is a non-json string message. |
| # Use resp_body as is and raise InvalidResponseBody |
| # exception. |
| raise exceptions.InvalidHTTPResponseBody(message) |
| else: |
| if isinstance(resp_body, dict): |
| # I'm seeing both computeFault |
| # and cloudServersFault come back. |
| # Will file a bug to fix, but leave as is for now. |
| if 'cloudServersFault' in resp_body: |
| message = resp_body['cloudServersFault']['message'] |
| elif 'computeFault' in resp_body: |
| message = resp_body['computeFault']['message'] |
| elif 'error' in resp_body: # Keystone errors |
| message = resp_body['error']['message'] |
| raise exceptions.IdentityError(message) |
| elif 'message' in resp_body: |
| message = resp_body['message'] |
| else: |
| message = resp_body |
| |
| raise exceptions.ServerFault(message) |
| |
| if resp.status >= 400: |
| raise exceptions.UnexpectedResponseCode(str(resp.status)) |
| |
| def is_absolute_limit(self, resp, resp_body): |
| if (not isinstance(resp_body, collections.Mapping) or |
| 'retry-after' not in resp): |
| return True |
| if self._get_type() is "json": |
| over_limit = resp_body.get('overLimit', None) |
| if not over_limit: |
| return True |
| return 'exceed' in over_limit.get('message', 'blabla') |
| elif self._get_type() is "xml": |
| return 'exceed' in resp_body.get('message', 'blabla') |
| |
| def wait_for_resource_deletion(self, id): |
| """Waits for a resource to be deleted.""" |
| start_time = int(time.time()) |
| while True: |
| if self.is_resource_deleted(id): |
| return |
| if int(time.time()) - start_time >= self.build_timeout: |
| raise exceptions.TimeoutException |
| time.sleep(self.build_interval) |
| |
| def is_resource_deleted(self, id): |
| """ |
| Subclasses override with specific deletion detection. |
| """ |
| message = ('"%s" does not implement is_resource_deleted' |
| % self.__class__.__name__) |
| raise NotImplementedError(message) |
| |
| @classmethod |
| def validate_response(cls, schema, resp, body): |
| # Only check the response if the status code is a success code |
| # TODO(cyeoh): Eventually we should be able to verify that a failure |
| # code if it exists is something that we expect. This is explicitly |
| # declared in the V3 API and so we should be able to export this in |
| # the response schema. For now we'll ignore it. |
| if resp.status in HTTP_SUCCESS: |
| response_code = schema['status_code'] |
| if resp.status not in response_code: |
| msg = ("The status code(%s) is different than the expected " |
| "one(%s)") % (resp.status, response_code) |
| raise exceptions.InvalidHttpSuccessCode(msg) |
| response_schema = schema.get('response_body') |
| if response_schema: |
| try: |
| jsonschema.validate(body, response_schema) |
| except jsonschema.ValidationError as ex: |
| msg = ("HTTP response body is invalid (%s)") % ex |
| raise exceptions.InvalidHTTPResponseBody(msg) |
| else: |
| if body: |
| msg = ("HTTP response body should not exist (%s)") % body |
| raise exceptions.InvalidHTTPResponseBody(msg) |
| |
| |
| class NegativeRestClient(RestClient): |
| """ |
| Version of RestClient that does not raise exceptions. |
| """ |
| def _error_checker(self, method, url, |
| headers, body, resp, resp_body): |
| pass |
| |
| def send_request(self, method, url_template, resources, body=None): |
| url = url_template % tuple(resources) |
| if method == "GET": |
| resp, body = self.get(url) |
| elif method == "POST": |
| resp, body = self.post(url, body) |
| elif method == "PUT": |
| resp, body = self.put(url, body) |
| elif method == "PATCH": |
| resp, body = self.patch(url, body) |
| elif method == "HEAD": |
| resp, body = self.head(url) |
| elif method == "DELETE": |
| resp, body = self.delete(url) |
| elif method == "COPY": |
| resp, body = self.copy(url) |
| else: |
| assert False |
| |
| return resp, body |