PageRenderTime 57ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 1ms

/cachecontrol/controller.py

https://gitlab.com/e0/cachecontrol
Python | 353 lines | 209 code | 56 blank | 88 comment | 56 complexity | 6e2c79bd3e42f465cd220475ef5ea946 MD5 | raw file
  1. """
  2. The httplib2 algorithms ported for use with requests.
  3. """
  4. import logging
  5. import re
  6. import calendar
  7. import time
  8. from email.utils import parsedate_tz
  9. from requests.structures import CaseInsensitiveDict
  10. from .cache import DictCache
  11. from .serialize import Serializer
  12. logger = logging.getLogger(__name__)
  13. URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
  14. def parse_uri(uri):
  15. """Parses a URI using the regex given in Appendix B of RFC 3986.
  16. (scheme, authority, path, query, fragment) = parse_uri(uri)
  17. """
  18. groups = URI.match(uri).groups()
  19. return (groups[1], groups[3], groups[4], groups[6], groups[8])
  20. class CacheController(object):
  21. """An interface to see if request should cached or not.
  22. """
  23. def __init__(self, cache=None, cache_etags=True, serializer=None):
  24. self.cache = cache or DictCache()
  25. self.cache_etags = cache_etags
  26. self.serializer = serializer or Serializer()
  27. @classmethod
  28. def _urlnorm(cls, uri):
  29. """Normalize the URL to create a safe key for the cache"""
  30. (scheme, authority, path, query, fragment) = parse_uri(uri)
  31. if not scheme or not authority:
  32. raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
  33. scheme = scheme.lower()
  34. authority = authority.lower()
  35. if not path:
  36. path = "/"
  37. # Could do syntax based normalization of the URI before
  38. # computing the digest. See Section 6.2.2 of Std 66.
  39. request_uri = query and "?".join([path, query]) or path
  40. defrag_uri = scheme + "://" + authority + request_uri
  41. return defrag_uri
  42. @classmethod
  43. def cache_url(cls, uri):
  44. return cls._urlnorm(uri)
  45. def parse_cache_control(self, headers):
  46. """
  47. Parse the cache control headers returning a dictionary with values
  48. for the different directives.
  49. """
  50. retval = {}
  51. cc_header = 'cache-control'
  52. if 'Cache-Control' in headers:
  53. cc_header = 'Cache-Control'
  54. if cc_header in headers:
  55. parts = headers[cc_header].split(',')
  56. parts_with_args = [
  57. tuple([x.strip().lower() for x in part.split("=", 1)])
  58. for part in parts if -1 != part.find("=")
  59. ]
  60. parts_wo_args = [
  61. (name.strip().lower(), 1)
  62. for name in parts if -1 == name.find("=")
  63. ]
  64. retval = dict(parts_with_args + parts_wo_args)
  65. return retval
  66. def cached_request(self, request):
  67. """
  68. Return a cached response if it exists in the cache, otherwise
  69. return False.
  70. """
  71. cache_url = self.cache_url(request.url)
  72. logger.debug('Looking up "%s" in the cache', cache_url)
  73. cc = self.parse_cache_control(request.headers)
  74. # Bail out if the request insists on fresh data
  75. if 'no-cache' in cc:
  76. logger.debug('Request header has "no-cache", cache bypassed')
  77. return False
  78. if 'max-age' in cc and cc['max-age'] == 0:
  79. logger.debug('Request header has "max_age" as 0, cache bypassed')
  80. return False
  81. # Request allows serving from the cache, let's see if we find something
  82. cache_data = self.cache.get(cache_url)
  83. if cache_data is None:
  84. logger.debug('No cache entry available')
  85. return False
  86. # Check whether it can be deserialized
  87. resp = self.serializer.loads(request, cache_data)
  88. if not resp:
  89. logger.warning('Cache entry deserialization failed, entry ignored')
  90. return False
  91. # If we have a cached 301, return it immediately. We don't
  92. # need to test our response for other headers b/c it is
  93. # intrinsically "cacheable" as it is Permanent.
  94. # See:
  95. # https://tools.ietf.org/html/rfc7231#section-6.4.2
  96. #
  97. # Client can try to refresh the value by repeating the request
  98. # with cache busting headers as usual (ie no-cache).
  99. if resp.status == 301:
  100. msg = ('Returning cached "301 Moved Permanently" response '
  101. '(ignoring date and etag information)')
  102. logger.debug(msg)
  103. return resp
  104. headers = CaseInsensitiveDict(resp.headers)
  105. if not headers or 'date' not in headers:
  106. if 'etag' not in headers:
  107. # Without date or etag, the cached response can never be used
  108. # and should be deleted.
  109. logger.debug('Purging cached response: no date or etag')
  110. self.cache.delete(cache_url)
  111. logger.debug('Ignoring cached response: no date')
  112. return False
  113. now = time.time()
  114. date = calendar.timegm(
  115. parsedate_tz(headers['date'])
  116. )
  117. current_age = max(0, now - date)
  118. logger.debug('Current age based on date: %i', current_age)
  119. # TODO: There is an assumption that the result will be a
  120. # urllib3 response object. This may not be best since we
  121. # could probably avoid instantiating or constructing the
  122. # response until we know we need it.
  123. resp_cc = self.parse_cache_control(headers)
  124. # determine freshness
  125. freshness_lifetime = 0
  126. # Check the max-age pragma in the cache control header
  127. if 'max-age' in resp_cc and resp_cc['max-age'].isdigit():
  128. freshness_lifetime = int(resp_cc['max-age'])
  129. logger.debug('Freshness lifetime from max-age: %i',
  130. freshness_lifetime)
  131. # If there isn't a max-age, check for an expires header
  132. elif 'expires' in headers:
  133. expires = parsedate_tz(headers['expires'])
  134. if expires is not None:
  135. expire_time = calendar.timegm(expires) - date
  136. freshness_lifetime = max(0, expire_time)
  137. logger.debug("Freshness lifetime from expires: %i",
  138. freshness_lifetime)
  139. # Determine if we are setting freshness limit in the
  140. # request. Note, this overrides what was in the response.
  141. if 'max-age' in cc:
  142. try:
  143. freshness_lifetime = int(cc['max-age'])
  144. logger.debug('Freshness lifetime from request max-age: %i',
  145. freshness_lifetime)
  146. except ValueError:
  147. freshness_lifetime = 0
  148. if 'min-fresh' in cc:
  149. try:
  150. min_fresh = int(cc['min-fresh'])
  151. except ValueError:
  152. min_fresh = 0
  153. # adjust our current age by our min fresh
  154. current_age += min_fresh
  155. logger.debug('Adjusted current age from min-fresh: %i',
  156. current_age)
  157. # Return entry if it is fresh enough
  158. if freshness_lifetime > current_age:
  159. logger.debug('The response is "fresh", returning cached response')
  160. logger.debug('%i > %i', freshness_lifetime, current_age)
  161. return resp
  162. # we're not fresh. If we don't have an Etag, clear it out
  163. if 'etag' not in headers:
  164. logger.debug(
  165. 'The cached response is "stale" with no etag, purging'
  166. )
  167. self.cache.delete(cache_url)
  168. # return the original handler
  169. return False
  170. def conditional_headers(self, request):
  171. cache_url = self.cache_url(request.url)
  172. resp = self.serializer.loads(request, self.cache.get(cache_url))
  173. new_headers = {}
  174. if resp:
  175. headers = CaseInsensitiveDict(resp.headers)
  176. if 'etag' in headers:
  177. new_headers['If-None-Match'] = headers['ETag']
  178. if 'last-modified' in headers:
  179. new_headers['If-Modified-Since'] = headers['Last-Modified']
  180. return new_headers
  181. def cache_response(self, request, response, body=None):
  182. """
  183. Algorithm for caching requests.
  184. This assumes a requests Response object.
  185. """
  186. # From httplib2: Don't cache 206's since we aren't going to
  187. # handle byte range requests
  188. cacheable_status_codes = [200, 203, 300, 301]
  189. if response.status not in cacheable_status_codes:
  190. logger.debug(
  191. 'Status code %s not in %s',
  192. response.status,
  193. cacheable_status_codes
  194. )
  195. return
  196. response_headers = CaseInsensitiveDict(response.headers)
  197. # If we've been given a body, our response has a Content-Length, that
  198. # Content-Length is valid then we can check to see if the body we've
  199. # been given matches the expected size, and if it doesn't we'll just
  200. # skip trying to cache it.
  201. if (body is not None and
  202. "content-length" in response_headers and
  203. response_headers["content-length"].isdigit() and
  204. int(response_headers["content-length"]) != len(body)):
  205. return
  206. cc_req = self.parse_cache_control(request.headers)
  207. cc = self.parse_cache_control(response_headers)
  208. cache_url = self.cache_url(request.url)
  209. logger.debug('Updating cache with response from "%s"', cache_url)
  210. # Delete it from the cache if we happen to have it stored there
  211. no_store = False
  212. if cc.get('no-store'):
  213. no_store = True
  214. logger.debug('Response header has "no-store"')
  215. if cc_req.get('no-store'):
  216. no_store = True
  217. logger.debug('Request header has "no-store"')
  218. if no_store and self.cache.get(cache_url):
  219. logger.debug('Purging existing cache entry to honor "no-store"')
  220. self.cache.delete(cache_url)
  221. # If we've been given an etag, then keep the response
  222. if self.cache_etags and 'etag' in response_headers:
  223. logger.debug('Caching due to etag')
  224. self.cache.set(
  225. cache_url,
  226. self.serializer.dumps(request, response, body=body),
  227. )
  228. # Add to the cache any 301s. We do this before looking that
  229. # the Date headers.
  230. elif response.status == 301:
  231. logger.debug('Caching permanant redirect')
  232. self.cache.set(
  233. cache_url,
  234. self.serializer.dumps(request, response)
  235. )
  236. # Add to the cache if the response headers demand it. If there
  237. # is no date header then we can't do anything about expiring
  238. # the cache.
  239. elif 'date' in response_headers:
  240. # cache when there is a max-age > 0
  241. if cc and cc.get('max-age'):
  242. if cc['max-age'].isdigit() and int(cc['max-age']) > 0:
  243. logger.debug('Caching b/c date exists and max-age > 0')
  244. self.cache.set(
  245. cache_url,
  246. self.serializer.dumps(request, response, body=body),
  247. )
  248. # If the request can expire, it means we should cache it
  249. # in the meantime.
  250. elif 'expires' in response_headers:
  251. if response_headers['expires']:
  252. logger.debug('Caching b/c of expires header')
  253. self.cache.set(
  254. cache_url,
  255. self.serializer.dumps(request, response, body=body),
  256. )
  257. def update_cached_response(self, request, response):
  258. """On a 304 we will get a new set of headers that we want to
  259. update our cached value with, assuming we have one.
  260. This should only ever be called when we've sent an ETag and
  261. gotten a 304 as the response.
  262. """
  263. cache_url = self.cache_url(request.url)
  264. cached_response = self.serializer.loads(
  265. request,
  266. self.cache.get(cache_url)
  267. )
  268. if not cached_response:
  269. # we didn't have a cached response
  270. return response
  271. # Lets update our headers with the headers from the new request:
  272. # http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
  273. #
  274. # The server isn't supposed to send headers that would make
  275. # the cached body invalid. But... just in case, we'll be sure
  276. # to strip out ones we know that might be problmatic due to
  277. # typical assumptions.
  278. excluded_headers = [
  279. "content-length",
  280. ]
  281. cached_response.headers.update(
  282. dict((k, v) for k, v in response.headers.items()
  283. if k.lower() not in excluded_headers)
  284. )
  285. # we want a 200 b/c we have content via the cache
  286. cached_response.status = 200
  287. # update our cache
  288. self.cache.set(
  289. cache_url,
  290. self.serializer.dumps(request, cached_response),
  291. )
  292. return cached_response