|
Packit Service |
a04d08 |
# Copyright (C) 2012 Canonical Ltd.
|
|
Packit Service |
a04d08 |
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
|
|
Packit Service |
a04d08 |
# Copyright (C) 2012 Yahoo! Inc.
|
|
Packit Service |
a04d08 |
#
|
|
Packit Service |
a04d08 |
# Author: Scott Moser <scott.moser@canonical.com>
|
|
Packit Service |
a04d08 |
# Author: Juerg Haefliger <juerg.haefliger@hp.com>
|
|
Packit Service |
a04d08 |
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
|
|
Packit Service |
a04d08 |
#
|
|
Packit Service |
a04d08 |
# This file is part of cloud-init. See LICENSE file for license information.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
751c4a |
import copy
|
|
Packit Service |
a04d08 |
import json
|
|
Packit Service |
a04d08 |
import os
|
|
Packit Service |
a04d08 |
import time
|
|
Packit Service |
a04d08 |
from email.utils import parsedate
|
|
Packit Service |
a04d08 |
from errno import ENOENT
|
|
Packit Service |
a04d08 |
from functools import partial
|
|
Packit Service |
751c4a |
from http.client import NOT_FOUND
|
|
Packit Service |
a04d08 |
from itertools import count
|
|
Packit Service |
751c4a |
from urllib.parse import urlparse, urlunparse, quote
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
751c4a |
import requests
|
|
Packit Service |
751c4a |
from requests import exceptions
|
|
Packit Service |
11b429 |
|
|
Packit Service |
a04d08 |
from cloudinit import log as logging
|
|
Packit Service |
a04d08 |
from cloudinit import version
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
LOG = logging.getLogger(__name__)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
# Check if requests has ssl support (added in requests >= 0.8.8)
|
|
Packit Service |
a04d08 |
SSL_ENABLED = False
|
|
Packit Service |
a04d08 |
CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0)
|
|
Packit Service |
a04d08 |
_REQ_VER = None
|
|
Packit Service |
751c4a |
REDACTED = 'REDACTED'
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
from distutils.version import LooseVersion
|
|
Packit Service |
a04d08 |
import pkg_resources
|
|
Packit Service |
a04d08 |
_REQ = pkg_resources.get_distribution('requests')
|
|
Packit Service |
a04d08 |
_REQ_VER = LooseVersion(_REQ.version) # pylint: disable=no-member
|
|
Packit Service |
a04d08 |
if _REQ_VER >= LooseVersion('0.8.8'):
|
|
Packit Service |
a04d08 |
SSL_ENABLED = True
|
|
Packit Service |
a04d08 |
if LooseVersion('0.7.0') <= _REQ_VER < LooseVersion('1.0.0'):
|
|
Packit Service |
a04d08 |
CONFIG_ENABLED = True
|
|
Packit Service |
a04d08 |
except ImportError:
|
|
Packit Service |
a04d08 |
pass
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def _cleanurl(url):
|
|
Packit Service |
a04d08 |
parsed_url = list(urlparse(url, scheme='http'))
|
|
Packit Service |
a04d08 |
if not parsed_url[1] and parsed_url[2]:
|
|
Packit Service |
a04d08 |
# Swap these since this seems to be a common
|
|
Packit Service |
a04d08 |
# occurrence when given urls like 'www.google.com'
|
|
Packit Service |
a04d08 |
parsed_url[1] = parsed_url[2]
|
|
Packit Service |
a04d08 |
parsed_url[2] = ''
|
|
Packit Service |
a04d08 |
return urlunparse(parsed_url)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def combine_url(base, *add_ons):
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def combine_single(url, add_on):
|
|
Packit Service |
a04d08 |
url_parsed = list(urlparse(url))
|
|
Packit Service |
a04d08 |
path = url_parsed[2]
|
|
Packit Service |
a04d08 |
if path and not path.endswith("/"):
|
|
Packit Service |
a04d08 |
path += "/"
|
|
Packit Service |
751c4a |
path += quote(str(add_on), safe="/:")
|
|
Packit Service |
a04d08 |
url_parsed[2] = path
|
|
Packit Service |
a04d08 |
return urlunparse(url_parsed)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
url = base
|
|
Packit Service |
a04d08 |
for add_on in add_ons:
|
|
Packit Service |
a04d08 |
url = combine_single(url, add_on)
|
|
Packit Service |
a04d08 |
return url
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def read_file_or_url(url, **kwargs):
|
|
Packit Service |
a04d08 |
"""Wrapper function around readurl to allow passing a file path as url.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
When url is not a local file path, passthrough any kwargs to readurl.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
In the case of parameter passthrough to readurl, default values for some
|
|
Packit Service |
a04d08 |
parameters. See: call-signature of readurl in this module for param docs.
|
|
Packit Service |
a04d08 |
"""
|
|
Packit Service |
a04d08 |
url = url.lstrip()
|
|
Packit Service |
a04d08 |
if url.startswith("/"):
|
|
Packit Service |
a04d08 |
url = "file://%s" % url
|
|
Packit Service |
a04d08 |
if url.lower().startswith("file://"):
|
|
Packit Service |
a04d08 |
if kwargs.get("data"):
|
|
Packit Service |
a04d08 |
LOG.warning("Unable to post data to file resource %s", url)
|
|
Packit Service |
a04d08 |
file_path = url[len("file://"):]
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
with open(file_path, "rb") as fp:
|
|
Packit Service |
a04d08 |
contents = fp.read()
|
|
Packit Service |
a04d08 |
except IOError as e:
|
|
Packit Service |
a04d08 |
code = e.errno
|
|
Packit Service |
a04d08 |
if e.errno == ENOENT:
|
|
Packit Service |
a04d08 |
code = NOT_FOUND
|
|
Packit Service |
751c4a |
raise UrlError(cause=e, code=code, headers=None, url=url) from e
|
|
Packit Service |
a04d08 |
return FileResponse(file_path, contents=contents)
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
return readurl(url, **kwargs)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
# Made to have same accessors as UrlResponse so that the
|
|
Packit Service |
a04d08 |
# read_file_or_url can return this or that object and the
|
|
Packit Service |
a04d08 |
# 'user' of those objects will not need to know the difference.
|
|
Packit Service |
a04d08 |
class StringResponse(object):
|
|
Packit Service |
a04d08 |
def __init__(self, contents, code=200):
|
|
Packit Service |
a04d08 |
self.code = code
|
|
Packit Service |
a04d08 |
self.headers = {}
|
|
Packit Service |
a04d08 |
self.contents = contents
|
|
Packit Service |
a04d08 |
self.url = None
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def ok(self, *args, **kwargs):
|
|
Packit Service |
a04d08 |
if self.code != 200:
|
|
Packit Service |
a04d08 |
return False
|
|
Packit Service |
a04d08 |
return True
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def __str__(self):
|
|
Packit Service |
a04d08 |
return self.contents.decode('utf-8')
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
class FileResponse(StringResponse):
|
|
Packit Service |
a04d08 |
def __init__(self, path, contents, code=200):
|
|
Packit Service |
a04d08 |
StringResponse.__init__(self, contents, code=code)
|
|
Packit Service |
a04d08 |
self.url = path
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
class UrlResponse(object):
|
|
Packit Service |
a04d08 |
def __init__(self, response):
|
|
Packit Service |
a04d08 |
self._response = response
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
@property
|
|
Packit Service |
a04d08 |
def contents(self):
|
|
Packit Service |
a04d08 |
return self._response.content
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
@property
|
|
Packit Service |
a04d08 |
def url(self):
|
|
Packit Service |
a04d08 |
return self._response.url
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def ok(self, redirects_ok=False):
|
|
Packit Service |
a04d08 |
upper = 300
|
|
Packit Service |
a04d08 |
if redirects_ok:
|
|
Packit Service |
a04d08 |
upper = 400
|
|
Packit Service |
a04d08 |
if 200 <= self.code < upper:
|
|
Packit Service |
a04d08 |
return True
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
return False
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
@property
|
|
Packit Service |
a04d08 |
def headers(self):
|
|
Packit Service |
a04d08 |
return self._response.headers
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
@property
|
|
Packit Service |
a04d08 |
def code(self):
|
|
Packit Service |
a04d08 |
return self._response.status_code
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def __str__(self):
|
|
Packit Service |
a04d08 |
return self._response.text
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
class UrlError(IOError):
|
|
Packit Service |
a04d08 |
def __init__(self, cause, code=None, headers=None, url=None):
|
|
Packit Service |
a04d08 |
IOError.__init__(self, str(cause))
|
|
Packit Service |
a04d08 |
self.cause = cause
|
|
Packit Service |
a04d08 |
self.code = code
|
|
Packit Service |
a04d08 |
self.headers = headers
|
|
Packit Service |
a04d08 |
if self.headers is None:
|
|
Packit Service |
a04d08 |
self.headers = {}
|
|
Packit Service |
a04d08 |
self.url = url
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def _get_ssl_args(url, ssl_details):
|
|
Packit Service |
a04d08 |
ssl_args = {}
|
|
Packit Service |
a04d08 |
scheme = urlparse(url).scheme
|
|
Packit Service |
a04d08 |
if scheme == 'https' and ssl_details:
|
|
Packit Service |
a04d08 |
if not SSL_ENABLED:
|
|
Packit Service |
a04d08 |
LOG.warning("SSL is not supported in requests v%s, "
|
|
Packit Service |
a04d08 |
"cert. verification can not occur!", _REQ_VER)
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
if 'ca_certs' in ssl_details and ssl_details['ca_certs']:
|
|
Packit Service |
a04d08 |
ssl_args['verify'] = ssl_details['ca_certs']
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
ssl_args['verify'] = True
|
|
Packit Service |
a04d08 |
if 'cert_file' in ssl_details and 'key_file' in ssl_details:
|
|
Packit Service |
a04d08 |
ssl_args['cert'] = [ssl_details['cert_file'],
|
|
Packit Service |
a04d08 |
ssl_details['key_file']]
|
|
Packit Service |
a04d08 |
elif 'cert_file' in ssl_details:
|
|
Packit Service |
a04d08 |
ssl_args['cert'] = str(ssl_details['cert_file'])
|
|
Packit Service |
a04d08 |
return ssl_args
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
|
|
Packit Service |
751c4a |
headers=None, headers_cb=None, headers_redact=None,
|
|
Packit Service |
751c4a |
ssl_details=None, check_status=True, allow_redirects=True,
|
|
Packit Service |
751c4a |
exception_cb=None, session=None, infinite=False, log_req_resp=True,
|
|
Packit Service |
a04d08 |
request_method=None):
|
|
Packit Service |
a04d08 |
"""Wrapper around requests.Session to read the url and retry if necessary
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
:param url: Mandatory url to request.
|
|
Packit Service |
a04d08 |
:param data: Optional form data to post the URL. Will set request_method
|
|
Packit Service |
a04d08 |
to 'POST' if present.
|
|
Packit Service |
a04d08 |
:param timeout: Timeout in seconds to wait for a response
|
|
Packit Service |
a04d08 |
:param retries: Number of times to retry on exception if exception_cb is
|
|
Packit Service |
a04d08 |
None or exception_cb returns True for the exception caught. Default is
|
|
Packit Service |
a04d08 |
to fail with 0 retries on exception.
|
|
Packit Service |
a04d08 |
:param sec_between: Default 1: amount of seconds passed to time.sleep
|
|
Packit Service |
a04d08 |
between retries. None or -1 means don't sleep.
|
|
Packit Service |
a04d08 |
:param headers: Optional dict of headers to send during request
|
|
Packit Service |
a04d08 |
:param headers_cb: Optional callable returning a dict of values to send as
|
|
Packit Service |
a04d08 |
headers during request
|
|
Packit Service |
751c4a |
:param headers_redact: Optional list of header names to redact from the log
|
|
Packit Service |
a04d08 |
:param ssl_details: Optional dict providing key_file, ca_certs, and
|
|
Packit Service |
a04d08 |
cert_file keys for use on in ssl connections.
|
|
Packit Service |
a04d08 |
:param check_status: Optional boolean set True to raise when HTTPError
|
|
Packit Service |
a04d08 |
occurs. Default: True.
|
|
Packit Service |
a04d08 |
:param allow_redirects: Optional boolean passed straight to Session.request
|
|
Packit Service |
a04d08 |
as 'allow_redirects'. Default: True.
|
|
Packit Service |
a04d08 |
:param exception_cb: Optional callable which accepts the params
|
|
Packit Service |
a04d08 |
msg and exception and returns a boolean True if retries are permitted.
|
|
Packit Service |
a04d08 |
:param session: Optional exiting requests.Session instance to reuse.
|
|
Packit Service |
a04d08 |
:param infinite: Bool, set True to retry indefinitely. Default: False.
|
|
Packit Service |
a04d08 |
:param log_req_resp: Set False to turn off verbose debug messages.
|
|
Packit Service |
a04d08 |
:param request_method: String passed as 'method' to Session.request.
|
|
Packit Service |
a04d08 |
Typically GET, or POST. Default: POST if data is provided, GET
|
|
Packit Service |
a04d08 |
otherwise.
|
|
Packit Service |
a04d08 |
"""
|
|
Packit Service |
a04d08 |
url = _cleanurl(url)
|
|
Packit Service |
a04d08 |
req_args = {
|
|
Packit Service |
a04d08 |
'url': url,
|
|
Packit Service |
a04d08 |
}
|
|
Packit Service |
a04d08 |
req_args.update(_get_ssl_args(url, ssl_details))
|
|
Packit Service |
a04d08 |
req_args['allow_redirects'] = allow_redirects
|
|
Packit Service |
a04d08 |
if not request_method:
|
|
Packit Service |
a04d08 |
request_method = 'POST' if data else 'GET'
|
|
Packit Service |
a04d08 |
req_args['method'] = request_method
|
|
Packit Service |
a04d08 |
if timeout is not None:
|
|
Packit Service |
a04d08 |
req_args['timeout'] = max(float(timeout), 0)
|
|
Packit Service |
751c4a |
if headers_redact is None:
|
|
Packit Service |
751c4a |
headers_redact = []
|
|
Packit Service |
a04d08 |
# It doesn't seem like config
|
|
Packit Service |
a04d08 |
# was added in older library versions (or newer ones either), thus we
|
|
Packit Service |
a04d08 |
# need to manually do the retries if it wasn't...
|
|
Packit Service |
a04d08 |
if CONFIG_ENABLED:
|
|
Packit Service |
a04d08 |
req_config = {
|
|
Packit Service |
a04d08 |
'store_cookies': False,
|
|
Packit Service |
a04d08 |
}
|
|
Packit Service |
a04d08 |
# Don't use the retry support built-in
|
|
Packit Service |
a04d08 |
# since it doesn't allow for 'sleep_times'
|
|
Packit Service |
a04d08 |
# in between tries....
|
|
Packit Service |
a04d08 |
# if retries:
|
|
Packit Service |
a04d08 |
# req_config['max_retries'] = max(int(retries), 0)
|
|
Packit Service |
a04d08 |
req_args['config'] = req_config
|
|
Packit Service |
a04d08 |
manual_tries = 1
|
|
Packit Service |
a04d08 |
if retries:
|
|
Packit Service |
a04d08 |
manual_tries = max(int(retries) + 1, 1)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def_headers = {
|
|
Packit Service |
a04d08 |
'User-Agent': 'Cloud-Init/%s' % (version.version_string()),
|
|
Packit Service |
a04d08 |
}
|
|
Packit Service |
a04d08 |
if headers:
|
|
Packit Service |
a04d08 |
def_headers.update(headers)
|
|
Packit Service |
a04d08 |
headers = def_headers
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if not headers_cb:
|
|
Packit Service |
a04d08 |
def _cb(url):
|
|
Packit Service |
a04d08 |
return headers
|
|
Packit Service |
a04d08 |
headers_cb = _cb
|
|
Packit Service |
a04d08 |
if data:
|
|
Packit Service |
a04d08 |
req_args['data'] = data
|
|
Packit Service |
a04d08 |
if sec_between is None:
|
|
Packit Service |
a04d08 |
sec_between = -1
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
excps = []
|
|
Packit Service |
a04d08 |
# Handle retrying ourselves since the built-in support
|
|
Packit Service |
a04d08 |
# doesn't handle sleeping between tries...
|
|
Packit Service |
a04d08 |
# Infinitely retry if infinite is True
|
|
Packit Service |
a04d08 |
for i in count() if infinite else range(0, manual_tries):
|
|
Packit Service |
a04d08 |
req_args['headers'] = headers_cb(url)
|
|
Packit Service |
a04d08 |
filtered_req_args = {}
|
|
Packit Service |
a04d08 |
for (k, v) in req_args.items():
|
|
Packit Service |
a04d08 |
if k == 'data':
|
|
Packit Service |
a04d08 |
continue
|
|
Packit Service |
751c4a |
if k == 'headers' and headers_redact:
|
|
Packit Service |
751c4a |
matched_headers = [k for k in headers_redact if v.get(k)]
|
|
Packit Service |
751c4a |
if matched_headers:
|
|
Packit Service |
751c4a |
filtered_req_args[k] = copy.deepcopy(v)
|
|
Packit Service |
751c4a |
for key in matched_headers:
|
|
Packit Service |
751c4a |
filtered_req_args[k][key] = REDACTED
|
|
Packit Service |
751c4a |
else:
|
|
Packit Service |
751c4a |
filtered_req_args[k] = v
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if log_req_resp:
|
|
Packit Service |
a04d08 |
LOG.debug("[%s/%s] open '%s' with %s configuration", i,
|
|
Packit Service |
a04d08 |
"infinite" if infinite else manual_tries, url,
|
|
Packit Service |
a04d08 |
filtered_req_args)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if session is None:
|
|
Packit Service |
a04d08 |
session = requests.Session()
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
with session as sess:
|
|
Packit Service |
a04d08 |
r = sess.request(**req_args)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if check_status:
|
|
Packit Service |
a04d08 |
r.raise_for_status()
|
|
Packit Service |
a04d08 |
LOG.debug("Read from %s (%s, %sb) after %s attempts", url,
|
|
Packit Service |
a04d08 |
r.status_code, len(r.content), (i + 1))
|
|
Packit Service |
a04d08 |
# Doesn't seem like we can make it use a different
|
|
Packit Service |
a04d08 |
# subclass for responses, so add our own backward-compat
|
|
Packit Service |
a04d08 |
# attrs
|
|
Packit Service |
a04d08 |
return UrlResponse(r)
|
|
Packit Service |
a04d08 |
except exceptions.RequestException as e:
|
|
Packit Service |
a04d08 |
if (isinstance(e, (exceptions.HTTPError)) and
|
|
Packit Service |
a04d08 |
hasattr(e, 'response') and # This appeared in v 0.10.8
|
|
Packit Service |
a04d08 |
hasattr(e.response, 'status_code')):
|
|
Packit Service |
a04d08 |
excps.append(UrlError(e, code=e.response.status_code,
|
|
Packit Service |
a04d08 |
headers=e.response.headers,
|
|
Packit Service |
a04d08 |
url=url))
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
excps.append(UrlError(e, url=url))
|
|
Packit Service |
a04d08 |
if SSL_ENABLED and isinstance(e, exceptions.SSLError):
|
|
Packit Service |
a04d08 |
# ssl exceptions are not going to get fixed by waiting a
|
|
Packit Service |
a04d08 |
# few seconds
|
|
Packit Service |
a04d08 |
break
|
|
Packit Service |
a04d08 |
if exception_cb and not exception_cb(req_args.copy(), excps[-1]):
|
|
Packit Service |
a04d08 |
# if an exception callback was given, it should return True
|
|
Packit Service |
a04d08 |
# to continue retrying and False to break and re-raise the
|
|
Packit Service |
a04d08 |
# exception
|
|
Packit Service |
a04d08 |
break
|
|
Packit Service |
a04d08 |
if (infinite and sec_between > 0) or \
|
|
Packit Service |
a04d08 |
(i + 1 < manual_tries and sec_between > 0):
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if log_req_resp:
|
|
Packit Service |
a04d08 |
LOG.debug(
|
|
Packit Service |
a04d08 |
"Please wait %s seconds while we wait to try again",
|
|
Packit Service |
a04d08 |
sec_between)
|
|
Packit Service |
a04d08 |
time.sleep(sec_between)
|
|
Packit Service |
a04d08 |
if excps:
|
|
Packit Service |
a04d08 |
raise excps[-1]
|
|
Packit Service |
a04d08 |
return None # Should throw before this...
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
751c4a |
def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None,
|
|
Packit Service |
751c4a |
headers_cb=None, headers_redact=None, sleep_time=1,
|
|
Packit Service |
a04d08 |
exception_cb=None, sleep_time_cb=None, request_method=None):
|
|
Packit Service |
a04d08 |
"""
|
|
Packit Service |
a04d08 |
urls: a list of urls to try
|
|
Packit Service |
a04d08 |
max_wait: roughly the maximum time to wait before giving up
|
|
Packit Service |
a04d08 |
The max time is *actually* len(urls)*timeout as each url will
|
|
Packit Service |
a04d08 |
be tried once and given the timeout provided.
|
|
Packit Service |
a04d08 |
a number <= 0 will always result in only one try
|
|
Packit Service |
a04d08 |
timeout: the timeout provided to urlopen
|
|
Packit Service |
a04d08 |
status_cb: call method with string message when a url is not available
|
|
Packit Service |
a04d08 |
headers_cb: call method with single argument of url to get headers
|
|
Packit Service |
a04d08 |
for request.
|
|
Packit Service |
751c4a |
headers_redact: a list of header names to redact from the log
|
|
Packit Service |
a04d08 |
exception_cb: call method with 2 arguments 'msg' (per status_cb) and
|
|
Packit Service |
a04d08 |
'exception', the exception that occurred.
|
|
Packit Service |
a04d08 |
sleep_time_cb: call method with 2 arguments (response, loop_n) that
|
|
Packit Service |
a04d08 |
generates the next sleep time.
|
|
Packit Service |
a04d08 |
request_method: indicate the type of HTTP request, GET, PUT, or POST
|
|
Packit Service |
a04d08 |
returns: tuple of (url, response contents), on failure, (False, None)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
the idea of this routine is to wait for the EC2 metadata service to
|
|
Packit Service |
a04d08 |
come up. On both Eucalyptus and EC2 we have seen the case where
|
|
Packit Service |
a04d08 |
the instance hit the MD before the MD service was up. EC2 seems
|
|
Packit Service |
a04d08 |
to have permanently fixed this, though.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
In openstack, the metadata service might be painfully slow, and
|
|
Packit Service |
a04d08 |
unable to avoid hitting a timeout of even up to 10 seconds or more
|
|
Packit Service |
a04d08 |
(LP: #894279) for a simple GET.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
Offset those needs with the need to not hang forever (and block boot)
|
|
Packit Service |
a04d08 |
on a system where cloud-init is configured to look for EC2 Metadata
|
|
Packit Service |
a04d08 |
service but is not going to find one. It is possible that the instance
|
|
Packit Service |
a04d08 |
data host (169.254.169.254) may be firewalled off Entirely for a system,
|
|
Packit Service |
a04d08 |
meaning that the connection will block forever unless a timeout is set.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
A value of None for max_wait will retry indefinitely.
|
|
Packit Service |
a04d08 |
"""
|
|
Packit Service |
a04d08 |
start_time = time.time()
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def log_status_cb(msg, exc=None):
|
|
Packit Service |
a04d08 |
LOG.debug(msg)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if status_cb is None:
|
|
Packit Service |
a04d08 |
status_cb = log_status_cb
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def timeup(max_wait, start_time):
|
|
Packit Service |
a04d08 |
if (max_wait is None):
|
|
Packit Service |
a04d08 |
return False
|
|
Packit Service |
a04d08 |
return ((max_wait <= 0) or (time.time() - start_time > max_wait))
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
loop_n = 0
|
|
Packit Service |
a04d08 |
response = None
|
|
Packit Service |
a04d08 |
while True:
|
|
Packit Service |
a04d08 |
if sleep_time_cb is not None:
|
|
Packit Service |
a04d08 |
sleep_time = sleep_time_cb(response, loop_n)
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
sleep_time = int(loop_n / 5) + 1
|
|
Packit Service |
a04d08 |
for url in urls:
|
|
Packit Service |
a04d08 |
now = time.time()
|
|
Packit Service |
a04d08 |
if loop_n != 0:
|
|
Packit Service |
a04d08 |
if timeup(max_wait, start_time):
|
|
Packit Service |
a04d08 |
break
|
|
Packit Service |
a04d08 |
if (max_wait is not None and
|
|
Packit Service |
a04d08 |
timeout and (now + timeout > (start_time + max_wait))):
|
|
Packit Service |
a04d08 |
# shorten timeout to not run way over max_time
|
|
Packit Service |
a04d08 |
timeout = int((start_time + max_wait) - now)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
reason = ""
|
|
Packit Service |
a04d08 |
url_exc = None
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
if headers_cb is not None:
|
|
Packit Service |
a04d08 |
headers = headers_cb(url)
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
headers = {}
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
response = readurl(
|
|
Packit Service |
751c4a |
url, headers=headers, headers_redact=headers_redact,
|
|
Packit Service |
751c4a |
timeout=timeout, check_status=False,
|
|
Packit Service |
751c4a |
request_method=request_method)
|
|
Packit Service |
a04d08 |
if not response.contents:
|
|
Packit Service |
a04d08 |
reason = "empty response [%s]" % (response.code)
|
|
Packit Service |
a04d08 |
url_exc = UrlError(ValueError(reason), code=response.code,
|
|
Packit Service |
a04d08 |
headers=response.headers, url=url)
|
|
Packit Service |
a04d08 |
elif not response.ok():
|
|
Packit Service |
a04d08 |
reason = "bad status code [%s]" % (response.code)
|
|
Packit Service |
a04d08 |
url_exc = UrlError(ValueError(reason), code=response.code,
|
|
Packit Service |
a04d08 |
headers=response.headers, url=url)
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
return url, response.contents
|
|
Packit Service |
a04d08 |
except UrlError as e:
|
|
Packit Service |
a04d08 |
reason = "request error [%s]" % e
|
|
Packit Service |
a04d08 |
url_exc = e
|
|
Packit Service |
a04d08 |
except Exception as e:
|
|
Packit Service |
a04d08 |
reason = "unexpected error [%s]" % e
|
|
Packit Service |
a04d08 |
url_exc = e
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
time_taken = int(time.time() - start_time)
|
|
Packit Service |
a04d08 |
max_wait_str = "%ss" % max_wait if max_wait else "unlimited"
|
|
Packit Service |
a04d08 |
status_msg = "Calling '%s' failed [%s/%s]: %s" % (url,
|
|
Packit Service |
a04d08 |
time_taken,
|
|
Packit Service |
a04d08 |
max_wait_str,
|
|
Packit Service |
a04d08 |
reason)
|
|
Packit Service |
a04d08 |
status_cb(status_msg)
|
|
Packit Service |
a04d08 |
if exception_cb:
|
|
Packit Service |
a04d08 |
# This can be used to alter the headers that will be sent
|
|
Packit Service |
a04d08 |
# in the future, for example this is what the MAAS datasource
|
|
Packit Service |
a04d08 |
# does.
|
|
Packit Service |
a04d08 |
exception_cb(msg=status_msg, exception=url_exc)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if timeup(max_wait, start_time):
|
|
Packit Service |
a04d08 |
break
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
loop_n = loop_n + 1
|
|
Packit Service |
a04d08 |
LOG.debug("Please wait %s seconds while we wait to try again",
|
|
Packit Service |
a04d08 |
sleep_time)
|
|
Packit Service |
a04d08 |
time.sleep(sleep_time)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
return False, None
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
class OauthUrlHelper(object):
|
|
Packit Service |
a04d08 |
def __init__(self, consumer_key=None, token_key=None,
|
|
Packit Service |
a04d08 |
token_secret=None, consumer_secret=None,
|
|
Packit Service |
a04d08 |
skew_data_file="/run/oauth_skew.json"):
|
|
Packit Service |
a04d08 |
self.consumer_key = consumer_key
|
|
Packit Service |
a04d08 |
self.consumer_secret = consumer_secret or ""
|
|
Packit Service |
a04d08 |
self.token_key = token_key
|
|
Packit Service |
a04d08 |
self.token_secret = token_secret
|
|
Packit Service |
a04d08 |
self.skew_data_file = skew_data_file
|
|
Packit Service |
a04d08 |
self._do_oauth = True
|
|
Packit Service |
a04d08 |
self.skew_change_limit = 5
|
|
Packit Service |
a04d08 |
required = (self.token_key, self.token_secret, self.consumer_key)
|
|
Packit Service |
a04d08 |
if not any(required):
|
|
Packit Service |
a04d08 |
self._do_oauth = False
|
|
Packit Service |
a04d08 |
elif not all(required):
|
|
Packit Service |
a04d08 |
raise ValueError("all or none of token_key, token_secret, or "
|
|
Packit Service |
a04d08 |
"consumer_key can be set")
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
old = self.read_skew_file()
|
|
Packit Service |
a04d08 |
self.skew_data = old or {}
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def read_skew_file(self):
|
|
Packit Service |
a04d08 |
if self.skew_data_file and os.path.isfile(self.skew_data_file):
|
|
Packit Service |
a04d08 |
with open(self.skew_data_file, mode="r") as fp:
|
|
Packit Service |
a04d08 |
return json.load(fp)
|
|
Packit Service |
a04d08 |
return None
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def update_skew_file(self, host, value):
|
|
Packit Service |
a04d08 |
# this is not atomic
|
|
Packit Service |
a04d08 |
if not self.skew_data_file:
|
|
Packit Service |
a04d08 |
return
|
|
Packit Service |
a04d08 |
cur = self.read_skew_file()
|
|
Packit Service |
a04d08 |
if cur is None:
|
|
Packit Service |
a04d08 |
cur = {}
|
|
Packit Service |
a04d08 |
cur[host] = value
|
|
Packit Service |
a04d08 |
with open(self.skew_data_file, mode="w") as fp:
|
|
Packit Service |
a04d08 |
fp.write(json.dumps(cur))
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def exception_cb(self, msg, exception):
|
|
Packit Service |
a04d08 |
if not (isinstance(exception, UrlError) and
|
|
Packit Service |
a04d08 |
(exception.code == 403 or exception.code == 401)):
|
|
Packit Service |
a04d08 |
return
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if 'date' not in exception.headers:
|
|
Packit Service |
a04d08 |
LOG.warning("Missing header 'date' in %s response",
|
|
Packit Service |
a04d08 |
exception.code)
|
|
Packit Service |
a04d08 |
return
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
date = exception.headers['date']
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
remote_time = time.mktime(parsedate(date))
|
|
Packit Service |
a04d08 |
except Exception as e:
|
|
Packit Service |
a04d08 |
LOG.warning("Failed to convert datetime '%s': %s", date, e)
|
|
Packit Service |
a04d08 |
return
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
skew = int(remote_time - time.time())
|
|
Packit Service |
a04d08 |
host = urlparse(exception.url).netloc
|
|
Packit Service |
a04d08 |
old_skew = self.skew_data.get(host, 0)
|
|
Packit Service |
a04d08 |
if abs(old_skew - skew) > self.skew_change_limit:
|
|
Packit Service |
a04d08 |
self.update_skew_file(host, skew)
|
|
Packit Service |
a04d08 |
LOG.warning("Setting oauth clockskew for %s to %d", host, skew)
|
|
Packit Service |
a04d08 |
self.skew_data[host] = skew
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
return
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def headers_cb(self, url):
|
|
Packit Service |
a04d08 |
if not self._do_oauth:
|
|
Packit Service |
a04d08 |
return {}
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
timestamp = None
|
|
Packit Service |
a04d08 |
host = urlparse(url).netloc
|
|
Packit Service |
a04d08 |
if self.skew_data and host in self.skew_data:
|
|
Packit Service |
a04d08 |
timestamp = int(time.time()) + self.skew_data[host]
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
return oauth_headers(
|
|
Packit Service |
a04d08 |
url=url, consumer_key=self.consumer_key,
|
|
Packit Service |
a04d08 |
token_key=self.token_key, token_secret=self.token_secret,
|
|
Packit Service |
a04d08 |
consumer_secret=self.consumer_secret, timestamp=timestamp)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def _wrapped(self, wrapped_func, args, kwargs):
|
|
Packit Service |
a04d08 |
kwargs['headers_cb'] = partial(
|
|
Packit Service |
a04d08 |
self._headers_cb, kwargs.get('headers_cb'))
|
|
Packit Service |
a04d08 |
kwargs['exception_cb'] = partial(
|
|
Packit Service |
a04d08 |
self._exception_cb, kwargs.get('exception_cb'))
|
|
Packit Service |
a04d08 |
return wrapped_func(*args, **kwargs)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def wait_for_url(self, *args, **kwargs):
|
|
Packit Service |
a04d08 |
return self._wrapped(wait_for_url, args, kwargs)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def readurl(self, *args, **kwargs):
|
|
Packit Service |
a04d08 |
return self._wrapped(readurl, args, kwargs)
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def _exception_cb(self, extra_exception_cb, msg, exception):
|
|
Packit Service |
a04d08 |
ret = None
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
if extra_exception_cb:
|
|
Packit Service |
a04d08 |
ret = extra_exception_cb(msg, exception)
|
|
Packit Service |
a04d08 |
finally:
|
|
Packit Service |
a04d08 |
self.exception_cb(msg, exception)
|
|
Packit Service |
a04d08 |
return ret
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def _headers_cb(self, extra_headers_cb, url):
|
|
Packit Service |
a04d08 |
headers = {}
|
|
Packit Service |
a04d08 |
if extra_headers_cb:
|
|
Packit Service |
a04d08 |
headers = extra_headers_cb(url)
|
|
Packit Service |
a04d08 |
headers.update(self.headers_cb(url))
|
|
Packit Service |
a04d08 |
return headers
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
|
|
Packit Service |
a04d08 |
timestamp=None):
|
|
Packit Service |
a04d08 |
try:
|
|
Packit Service |
a04d08 |
import oauthlib.oauth1 as oauth1
|
|
Packit Service |
751c4a |
except ImportError as e:
|
|
Packit Service |
751c4a |
raise NotImplementedError('oauth support is not available') from e
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
if timestamp:
|
|
Packit Service |
a04d08 |
timestamp = str(timestamp)
|
|
Packit Service |
a04d08 |
else:
|
|
Packit Service |
a04d08 |
timestamp = None
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
client = oauth1.Client(
|
|
Packit Service |
a04d08 |
consumer_key,
|
|
Packit Service |
a04d08 |
client_secret=consumer_secret,
|
|
Packit Service |
a04d08 |
resource_owner_key=token_key,
|
|
Packit Service |
a04d08 |
resource_owner_secret=token_secret,
|
|
Packit Service |
a04d08 |
signature_method=oauth1.SIGNATURE_PLAINTEXT,
|
|
Packit Service |
a04d08 |
timestamp=timestamp)
|
|
Packit Service |
a04d08 |
_uri, signed_headers, _body = client.sign(url)
|
|
Packit Service |
a04d08 |
return signed_headers
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
def retry_on_url_exc(msg, exc):
|
|
Packit Service |
a04d08 |
"""readurl exception_cb that will retry on NOT_FOUND and Timeout.
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
Returns False to raise the exception from readurl, True to retry.
|
|
Packit Service |
a04d08 |
"""
|
|
Packit Service |
a04d08 |
if not isinstance(exc, UrlError):
|
|
Packit Service |
a04d08 |
return False
|
|
Packit Service |
a04d08 |
if exc.code == NOT_FOUND:
|
|
Packit Service |
a04d08 |
return True
|
|
Packit Service |
a04d08 |
if exc.cause and isinstance(exc.cause, requests.Timeout):
|
|
Packit Service |
a04d08 |
return True
|
|
Packit Service |
a04d08 |
return False
|
|
Packit Service |
a04d08 |
|
|
Packit Service |
a04d08 |
# vi: ts=4 expandtab
|