1 # Copyright 2010-2011 OpenStack Foundation
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
16 # HTTPSClientAuthConnection code comes courtesy of ActiveState website:
17 # http://code.activestate.com/recipes/
18 # 577548-https-httplib-client-connection-with-certificate-v/
29 from eventlet.green import socket
30 from eventlet.green import ssl
38 import sendfile # noqa
39 SENDFILE_SUPPORTED = True
41 SENDFILE_SUPPORTED = False
43 from oslo_log import log as logging
44 from oslo_utils import encodeutils
46 # NOTE(jokke): simplified transition to py3, behaves like py2 xrange
47 from six.moves import range
48 import six.moves.urllib.parse as urlparse
50 from escalator.common import auth
51 from escalator.common import exception
52 from escalator.common import utils
53 from escalator import i18n
55 LOG = logging.getLogger(__name__)
58 # common chunk size for get and put
61 VERSION_REGEX = re.compile(r"/?v[0-9\.]+")
64 def handle_unauthenticated(func):
66 Wrap a function to re-authenticate and retry.
68 @functools.wraps(func)
69 def wrapped(self, *args, **kwargs):
71 return func(self, *args, **kwargs)
72 except exception.NotAuthenticated:
73 self._authenticate(force_reauth=True)
74 return func(self, *args, **kwargs)
78 def handle_redirects(func):
80 Wrap the _do_request function to handle HTTP redirects.
84 @functools.wraps(func)
85 def wrapped(self, method, url, body, headers):
86 for _ in range(MAX_REDIRECTS):
88 return func(self, method, url, body, headers)
89 except exception.RedirectException as redirect:
90 if redirect.url is None:
91 raise exception.InvalidRedirect()
93 raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
97 class HTTPSClientAuthConnection(httplib.HTTPSConnection):
99 Class to make a HTTPS connection, with support for
100 full client-based SSL Authentication
102 :see http://code.activestate.com/recipes/
103 577548-https-httplib-client-connection-with-certificate-v/
106 def __init__(self, host, port, key_file, cert_file,
107 ca_file, timeout=None, insecure=False):
108 httplib.HTTPSConnection.__init__(self, host, port, key_file=key_file,
110 self.key_file = key_file
111 self.cert_file = cert_file
112 self.ca_file = ca_file
113 self.timeout = timeout
114 self.insecure = insecure
118 Connect to a host on a given (SSL) port.
119 If ca_file is pointing somewhere, use it to check Server Certificate.
121 Redefined/copied and extended from httplib.py:1105 (Python 2.6.x).
122 This is needed to pass cert_reqs=ssl.CERT_REQUIRED as parameter to
123 ssl.wrap_socket(), which forces SSL to check server certificate against
124 our client certificate.
126 sock = socket.create_connection((self.host, self.port), self.timeout)
127 if self._tunnel_host:
130 # Check CA file unless 'insecure' is specificed
131 if self.insecure is True:
132 self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
133 cert_reqs=ssl.CERT_NONE)
135 self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
136 ca_certs=self.ca_file,
137 cert_reqs=ssl.CERT_REQUIRED)
140 class BaseClient(object):
142 """A base client class"""
145 DEFAULT_DOC_ROOT = None
146 # Standard CA file locations for Debian/Ubuntu, RedHat/Fedora,
147 # Suse, FreeBSD/OpenBSD
148 DEFAULT_CA_FILE_PATH = ('/etc/ssl/certs/ca-certificates.crt:'
149 '/etc/pki/tls/certs/ca-bundle.crt:'
150 '/etc/ssl/ca-bundle.pem:'
153 OK_RESPONSE_CODES = (
160 REDIRECT_RESPONSE_CODES = (
161 httplib.MOVED_PERMANENTLY,
165 httplib.TEMPORARY_REDIRECT,
168 def __init__(self, host, port=None, timeout=None, use_ssl=False,
169 auth_token=None, creds=None, doc_root=None, key_file=None,
170 cert_file=None, ca_file=None, insecure=False,
171 configure_via_auth=True):
173 Creates a new client to some service.
175 :param host: The host where service resides
176 :param port: The port where service resides
177 :param timeout: Connection timeout.
178 :param use_ssl: Should we use HTTPS?
179 :param auth_token: The auth token to pass to the server
180 :param creds: The credentials to pass to the auth plugin
181 :param doc_root: Prefix for all URLs we request from host
182 :param key_file: Optional PEM-formatted file that contains the private
184 If use_ssl is True, and this param is None (the
185 default), then an environ variable
186 ESCALATOR_CLIENT_KEY_FILE is looked for. If no such
187 environ variable is found, ClientConnectionError
189 :param cert_file: Optional PEM-formatted certificate chain file.
190 If use_ssl is True, and this param is None (the
191 default), then an environ variable
192 ESCALATOR_CLIENT_CERT_FILE is looked for. If no such
193 environ variable is found, ClientConnectionError
195 :param ca_file: Optional CA cert file to use in SSL connections
196 If use_ssl is True, and this param is None (the
197 default), then an environ variable
198 ESCALATOR_CLIENT_CA_FILE is looked for.
199 :param insecure: Optional. If set then the server's certificate
200 will not be verified.
201 :param configure_via_auth: Optional. Defaults to True. If set, the
202 URL returned from the service catalog for the image
203 endpoint will **override** the URL supplied to in
207 self.port = port or self.DEFAULT_PORT
208 self.timeout = timeout
209 # A value of '0' implies never timeout
212 self.use_ssl = use_ssl
213 self.auth_token = auth_token
214 self.creds = creds or {}
215 self.connection = None
216 self.configure_via_auth = configure_via_auth
217 # doc_root can be a nullstring, which is valid, and why we
218 # cannot simply do doc_root or self.DEFAULT_DOC_ROOT below.
219 self.doc_root = (doc_root if doc_root is not None
220 else self.DEFAULT_DOC_ROOT)
222 self.key_file = key_file
223 self.cert_file = cert_file
224 self.ca_file = ca_file
225 self.insecure = insecure
226 self.auth_plugin = self.make_auth_plugin(self.creds, self.insecure)
227 self.connect_kwargs = self.get_connect_kwargs()
229 def get_connect_kwargs(self):
232 # Both secure and insecure connections have a timeout option
233 connect_kwargs['timeout'] = self.timeout
236 if self.key_file is None:
237 self.key_file = os.environ.get('ESCALATOR_CLIENT_KEY_FILE')
238 if self.cert_file is None:
239 self.cert_file = os.environ.get('ESCALATOR_CLIENT_CERT_FILE')
240 if self.ca_file is None:
241 self.ca_file = os.environ.get('ESCALATOR_CLIENT_CA_FILE')
243 # Check that key_file/cert_file are either both set or both unset
244 if self.cert_file is not None and self.key_file is None:
245 msg = _("You have selected to use SSL in connecting, "
246 "and you have supplied a cert, "
247 "however you have failed to supply either a "
248 "key_file parameter or set the "
249 "ESCALATOR_CLIENT_KEY_FILE environ variable")
250 raise exception.ClientConnectionError(msg)
252 if self.key_file is not None and self.cert_file is None:
253 msg = _("You have selected to use SSL in connecting, "
254 "and you have supplied a key, "
255 "however you have failed to supply either a "
256 "cert_file parameter or set the "
257 "ESCALATOR_CLIENT_CERT_FILE environ variable")
258 raise exception.ClientConnectionError(msg)
260 if (self.key_file is not None and
261 not os.path.exists(self.key_file)):
262 msg = _("The key file you specified %s does not "
263 "exist") % self.key_file
264 raise exception.ClientConnectionError(msg)
265 connect_kwargs['key_file'] = self.key_file
267 if (self.cert_file is not None and
268 not os.path.exists(self.cert_file)):
269 msg = _("The cert file you specified %s does not "
270 "exist") % self.cert_file
271 raise exception.ClientConnectionError(msg)
272 connect_kwargs['cert_file'] = self.cert_file
274 if (self.ca_file is not None and
275 not os.path.exists(self.ca_file)):
276 msg = _("The CA file you specified %s does not "
277 "exist") % self.ca_file
278 raise exception.ClientConnectionError(msg)
280 if self.ca_file is None:
281 for ca in self.DEFAULT_CA_FILE_PATH.split(":"):
282 if os.path.exists(ca):
286 connect_kwargs['ca_file'] = self.ca_file
287 connect_kwargs['insecure'] = self.insecure
289 return connect_kwargs
291 def configure_from_url(self, url):
293 Setups the connection based on the given url.
297 <http|https>://<host>:port/doc_root
299 LOG.debug("Configuring from URL: %s", url)
300 parsed = urlparse.urlparse(url)
301 self.use_ssl = parsed.scheme == 'https'
302 self.host = parsed.hostname
303 self.port = parsed.port or 80
304 self.doc_root = parsed.path.rstrip('/')
306 # We need to ensure a version identifier is appended to the doc_root
307 if not VERSION_REGEX.match(self.doc_root):
308 if self.DEFAULT_DOC_ROOT:
309 doc_root = self.DEFAULT_DOC_ROOT.lstrip('/')
310 self.doc_root += '/' + doc_root
311 msg = ("Appending doc_root %(doc_root)s to URL %(url)s" %
312 {'doc_root': doc_root, 'url': url})
315 # ensure connection kwargs are re-evaluated after the service catalog
316 # publicURL is parsed for potential SSL usage
317 self.connect_kwargs = self.get_connect_kwargs()
319 def make_auth_plugin(self, creds, insecure):
321 Returns an instantiated authentication plugin.
323 strategy = creds.get('strategy', 'noauth')
324 plugin = auth.get_plugin_from_strategy(strategy, creds, insecure,
325 self.configure_via_auth)
328 def get_connection_type(self):
330 Returns the proper connection type
333 return HTTPSClientAuthConnection
335 return httplib.HTTPConnection
337 def _authenticate(self, force_reauth=False):
339 Use the authentication plugin to authenticate and set the auth token.
341 :param force_reauth: For re-authentication to bypass cache.
343 auth_plugin = self.auth_plugin
345 if not auth_plugin.is_authenticated or force_reauth:
346 auth_plugin.authenticate()
348 self.auth_token = auth_plugin.auth_token
350 management_url = auth_plugin.management_url
351 if management_url and self.configure_via_auth:
352 self.configure_from_url(management_url)
354 @handle_unauthenticated
355 def do_request(self, method, action, body=None, headers=None,
358 Make a request, returning an HTTP response object.
360 :param method: HTTP verb (GET, POST, PUT, etc.)
361 :param action: Requested path to append to self.doc_root
362 :param body: Data to send in the body of the request
363 :param headers: Headers to send with the request
364 :param params: Key/value pairs to use in query string
365 :returns: HTTP response object
367 if not self.auth_token:
370 url = self._construct_url(action, params)
371 # NOTE(ameade): We need to copy these kwargs since they can be altered
372 # in _do_request but we need the originals if handle_unauthenticated
373 # calls this function again.
374 return self._do_request(method=method, url=url,
375 body=copy.deepcopy(body),
376 headers=copy.deepcopy(headers))
378 def _construct_url(self, action, params=None):
380 Create a URL object we can use to pass to _do_request().
382 action = urlparse.quote(action)
383 path = '/'.join([self.doc_root or '', action.lstrip('/')])
384 scheme = "https" if self.use_ssl else "http"
385 netloc = "%s:%d" % (self.host, self.port)
387 if isinstance(params, dict):
388 for (key, value) in params.items():
392 if not isinstance(value, six.string_types):
394 params[key] = encodeutils.safe_encode(value)
395 query = urlparse.urlencode(params)
399 url = urlparse.ParseResult(scheme, netloc, path, '', query, '')
400 log_msg = _("Constructed URL: %s")
401 LOG.debug(log_msg, url.geturl())
404 def _encode_headers(self, headers):
408 Note: This should be used right before
409 sending anything out.
411 :param headers: Headers to encode
412 :returns: Dictionary with encoded headers'
415 to_str = encodeutils.safe_encode
416 return dict([(to_str(h), to_str(v)) for h, v in
417 six.iteritems(headers)])
420 def _do_request(self, method, url, body, headers):
422 Connects to the server and issues a request. Handles converting
423 any returned HTTP error status codes to ESCALATOR exceptions
424 and closing the server connection. Returns the result data, or
425 raises an appropriate exception.
427 :param method: HTTP method ("GET", "POST", "PUT", etc...)
428 :param url: urlparse.ParsedResult object with URL information
429 :param body: data to send (as string, filelike or iterable),
431 :param headers: mapping of key/value pairs to add as headers
435 If the body param has a read attribute, and method is either
436 POST or PUT, this method will automatically conduct a chunked-transfer
437 encoding and use the body as a file object or iterable, transferring
438 chunks of data using the connection's send() method. This allows large
439 objects to be transferred efficiently without buffering the entire
443 path = url.path + "?" + url.query
448 connection_type = self.get_connection_type()
449 headers = self._encode_headers(headers or {})
450 headers.update(osprofiler.web.get_trace_id_headers())
452 if 'x-auth-token' not in headers and self.auth_token:
453 headers['x-auth-token'] = self.auth_token
455 c = connection_type(url.hostname, url.port, **self.connect_kwargs)
457 def _pushing(method):
458 return method.lower() in ('post', 'put')
461 return body is None or isinstance(body, six.string_types)
464 return hasattr(body, 'read')
466 def _sendbody(connection, iter):
467 connection.endheaders()
469 # iterator has done the heavy lifting
472 def _chunkbody(connection, iter):
473 connection.putheader('Transfer-Encoding', 'chunked')
474 connection.endheaders()
476 connection.send('%x\r\n%s\r\n' % (len(chunk), chunk))
477 connection.send('0\r\n\r\n')
479 # Do a simple request or a chunked request, depending
480 # on whether the body param is file-like or iterable and
481 # the method is PUT or POST
483 if not _pushing(method) or _simple(body):
485 c.request(method, path, body, headers)
486 elif _filelike(body) or self._iterable(body):
487 c.putrequest(method, path)
489 use_sendfile = self._sendable(body)
491 # According to HTTP/1.1, Content-Length and Transfer-Encoding
493 for header, value in headers.items():
494 if use_sendfile or header.lower() != 'content-length':
495 c.putheader(header, str(value))
497 iter = utils.chunkreadable(body)
500 # send actual file without copying into userspace
503 # otherwise iterate and chunk
506 raise TypeError('Unsupported image type: %s' % body.__class__)
508 res = c.getresponse()
511 return res.getheader('Retry-After')
513 status_code = self.get_status_code(res)
514 if status_code in self.OK_RESPONSE_CODES:
516 elif status_code in self.REDIRECT_RESPONSE_CODES:
517 raise exception.RedirectException(res.getheader('Location'))
518 elif status_code == httplib.UNAUTHORIZED:
519 raise exception.NotAuthenticated(res.read())
520 elif status_code == httplib.FORBIDDEN:
521 raise exception.Forbidden(res.read())
522 elif status_code == httplib.NOT_FOUND:
523 raise exception.NotFound(res.read())
524 elif status_code == httplib.CONFLICT:
525 raise exception.Duplicate(res.read())
526 elif status_code == httplib.BAD_REQUEST:
527 raise exception.Invalid(res.read())
528 elif status_code == httplib.MULTIPLE_CHOICES:
529 raise exception.MultipleChoices(body=res.read())
530 elif status_code == httplib.REQUEST_ENTITY_TOO_LARGE:
531 raise exception.LimitExceeded(retry=_retry(res),
533 elif status_code == httplib.INTERNAL_SERVER_ERROR:
534 raise exception.ServerError()
535 elif status_code == httplib.SERVICE_UNAVAILABLE:
536 raise exception.ServiceUnavailable(retry=_retry(res))
538 raise exception.UnexpectedStatus(status=status_code,
541 except (socket.error, IOError) as e:
542 raise exception.ClientConnectionError(e)
544 def _seekable(self, body):
545 # pipes are not seekable, avoids sendfile() failure on e.g.
546 # cat /path/to/image | escalator add ...
547 # or where add command is launched via popen
549 os.lseek(body.fileno(), 0, os.SEEK_CUR)
552 return (e.errno != errno.ESPIPE)
554 def _sendable(self, body):
555 return (SENDFILE_SUPPORTED and
556 hasattr(body, 'fileno') and
557 self._seekable(body) and
560 def _iterable(self, body):
561 return isinstance(body, collections.Iterable)
563 def get_status_code(self, response):
565 Returns the integer status code from the response, which
566 can be either a Webob.Response (used in testing) or httplib.Response
568 if hasattr(response, 'status_int'):
569 return response.status_int
571 return response.status
573 def _extract_params(self, actual_params, allowed_params):
575 Extract a subset of keys from a dictionary. The filters key
576 will also be extracted, and each of its values will be returned
577 as an individual param.
579 :param actual_params: dict of keys to filter
580 :param allowed_params: list of keys that 'actual_params' will be
582 :retval subset of 'params' dict
585 # expect 'filters' param to be a dict here
586 result = dict(actual_params.get('filters'))
590 for allowed_param in allowed_params:
591 if allowed_param in actual_params:
592 result[allowed_param] = actual_params[allowed_param]