PageRenderTime 64ms CodeModel.GetById 36ms RepoModel.GetById 0ms app.codeStats 1ms

/src/optimizations/assetcache.py

https://github.com/etianen/django-optimizations
Python | 430 lines | 401 code | 1 blank | 28 comment | 3 complexity | bb767bcb88c87552a8fc65382d1e804c MD5 | raw file
  1. """
  2. An asset cache stores a copy of slow-to-obtain website assets
  3. on a fast file storage. Assets are stored based on a hash of their
  4. path and mtime, so it's safe to update the source and have the asset cache
  5. automatically cleared.
  6. A classic use of an asset cache is to copy static files from a server with
  7. a short expiry header to a server with an extremely long expiry header.
  8. """
  9. from __future__ import unicode_literals
  10. import hashlib, os.path, fnmatch, re
  11. from abc import ABCMeta, abstractmethod
  12. from contextlib import closing
  13. from django.contrib.staticfiles.finders import find as find_static_path, get_finders
  14. from django.contrib.staticfiles import storage
  15. from django.core.files.base import File, ContentFile
  16. from django.core.files.storage import default_storage
  17. from django.conf import settings
  18. from django.core.files.storage import get_storage_class
  19. from django.utils import six
  20. from django.utils.encoding import force_bytes
  21. try:
  22. staticfiles_storage = storage.staticfiles_storage
  23. except AttributeError:
  24. staticfiles_storage = get_storage_class(settings.STATICFILES_STORAGE)() # Django 1.3 compatibility.
  25. from optimizations.utils import resolve_namespaced_cache
  26. def freeze_dict(params):
  27. """Returns an invariant version of the dictionary, suitable for hashing."""
  28. return hashlib.sha1("&".join(
  29. "{key}={value}".format(
  30. key = key,
  31. value = value,
  32. )
  33. for key, value in sorted(six.iteritems(params))
  34. ).encode('utf-8')).hexdigest()
  35. class Asset(six.with_metaclass(ABCMeta)):
  36. """An asset that is available to the asset cache."""
  37. @abstractmethod
  38. def get_name(self):
  39. """
  40. Returns the name of this asset.
  41. It does not have to be globally unique.
  42. """
  43. raise NotImplementedError
  44. def get_path(self):
  45. """Returns the filesystem path of this asset."""
  46. raise NotImplementedError("This asset does not support absolute paths")
  47. def get_url(self):
  48. """Returns the frontend URL of this asset."""
  49. raise NotImplementedError("This asset does not have a URL")
  50. def get_mtime(self):
  51. """Returns the last modified time of this asset."""
  52. return os.path.getmtime(self.get_path())
  53. def get_contents_hash(self):
  54. """Returns an md5 hash of the file's contents."""
  55. md5 = hashlib.md5()
  56. with closing(self.open()) as handle:
  57. for chunk in handle.chunks():
  58. md5.update(chunk)
  59. return md5.hexdigest()
  60. def get_id_params(self):
  61. """"Returns the params which should be used to generate the id."""
  62. params = {}
  63. # Add the path.
  64. try:
  65. params["path"] = self.get_path()
  66. except NotImplementedError:
  67. pass
  68. # Add the URL.
  69. try:
  70. params["url"] = self.get_url()
  71. except NotImplementedError:
  72. pass
  73. # All done!
  74. return params
  75. def _get_and_check_id_params(self):
  76. """Retrieves the id params, and checks that some exist."""
  77. params = self.get_id_params()
  78. if not params:
  79. raise NotImplementedError("This asset does not have a path or a url.")
  80. return params
  81. def get_id(self):
  82. """Returns a globally unique id for this asset."""
  83. return freeze_dict(self._get_and_check_id_params())
  84. def get_cache_key(self):
  85. return "optimizations:assetcache:{id}".format(
  86. id = self.get_id(),
  87. )
  88. def open(self):
  89. """Returns an open File for this asset."""
  90. return File(open(self.get_path()), "rb")
  91. def get_contents(self):
  92. """Returns the contents of this asset as a string."""
  93. with closing(self.open()) as handle:
  94. return handle.read()
  95. def get_hash_params(self):
  96. """Returns the params which should be used to generate the hash."""
  97. params = self._get_and_check_id_params()
  98. try:
  99. params["mtime"] = self.get_mtime()
  100. except NotImplementedError:
  101. # Not all backends support mtime, so fall back to md5 of the contents.
  102. params["md5"] = self.get_contents_hash()
  103. return params
  104. def get_hash(self):
  105. """Returns the sha1 hash of this asset's contents."""
  106. return freeze_dict(self.get_hash_params())
  107. def get_save_meta(self):
  108. """Returns the meta parameters to associate with the asset in the asset cache."""
  109. return {}
  110. def get_save_extension(self):
  111. """Returns the file extension to use when saving the asset."""
  112. _, asset_ext = os.path.splitext(self.get_name())
  113. return asset_ext.lower()
  114. def save(self, storage, name, meta):
  115. """Saves this asset to the given storage."""
  116. with closing(self.open()) as handle:
  117. storage.save(name, handle)
  118. class StaticAsset(Asset):
  119. """An asset that wraps a Django static file."""
  120. @staticmethod
  121. def get_static_path(name):
  122. """Returns the full static path of the given name."""
  123. path = find_static_path(name)
  124. if path is None:
  125. path = staticfiles_storage.path(name)
  126. return os.path.abspath(path)
  127. @staticmethod
  128. def load(type, assets="default"):
  129. """Resolves the given asset name into a list of static assets."""
  130. namespaces = StaticAsset._load_namespaces()
  131. # Adapt a single asset to a list.
  132. if isinstance(assets, (six.string_types, Asset)):
  133. assets = [assets]
  134. # Adapt asset names to assets.
  135. asset_objs = []
  136. for asset in assets:
  137. # Leave actual assets as they are.
  138. if isinstance(asset, Asset):
  139. asset_objs.append(asset)
  140. else:
  141. # Convert asset group ids into assets.
  142. asset_namespace = namespaces.get(asset)
  143. if asset_namespace is not None:
  144. asset_group = asset_namespace.get(type)
  145. if asset_group is not None:
  146. asset_objs.extend(asset_group)
  147. else:
  148. asset_objs.append(StaticAsset(asset))
  149. return asset_objs
  150. @staticmethod
  151. def get_namespaces():
  152. """Returns a list of all namespaces in the static asset loader."""
  153. return list(StaticAsset._load_namespaces().keys())
  154. @staticmethod
  155. def get_urls(type, assets="default"):
  156. """Returns a list of cached urls for the given static assets."""
  157. return [
  158. default_asset_cache.get_url(asset)
  159. for asset in StaticAsset.load(type, assets)
  160. ]
  161. @staticmethod
  162. def _load_namespaces():
  163. namespaces = getattr(StaticAsset, "_namespace_cache", None)
  164. if namespaces is None:
  165. namespaces = {}
  166. # Find all the assets.
  167. all_asset_names = []
  168. for finder in get_finders():
  169. for path, storage in finder.list(()):
  170. if getattr(storage, "prefix", None):
  171. path = os.path.join(storage.prefix, path)
  172. all_asset_names.append(path)
  173. all_asset_names.sort()
  174. # Loads the assets.
  175. def do_load(type, include=(), exclude=()):
  176. include = [re.compile(fnmatch.translate(pattern)) for pattern in include]
  177. exclude = [re.compile(fnmatch.translate(pattern)) for pattern in exclude]
  178. # Create the loaded list of assets.
  179. asset_names = []
  180. seen_asset_names = set()
  181. for pattern in include:
  182. new_asset_names = [a for a in all_asset_names if pattern.match(a) and not a in seen_asset_names]
  183. asset_names.extend(new_asset_names)
  184. seen_asset_names.update(new_asset_names)
  185. for pattern in exclude:
  186. asset_names = [a for a in asset_names if not pattern.match(a)]
  187. # Create the assets.
  188. return [StaticAsset(asset_name) for asset_name in asset_names]
  189. # Load in all namespaces.
  190. for namespace, types in six.iteritems(getattr(settings, "STATIC_ASSETS", {})):
  191. type_cache = namespaces[namespace] = {}
  192. for type, config in six.iteritems(types):
  193. type_cache[type] = do_load(type, **config)
  194. # Save in the cache.
  195. StaticAsset._namespace_cache = namespaces
  196. return namespaces
  197. def __init__(self, name):
  198. """Initializes the static asset."""
  199. self._name = name
  200. def open(self):
  201. return staticfiles_storage.open(self._name)
  202. def get_name(self):
  203. """Returns the name of this static asset."""
  204. return self._name
  205. def get_path(self):
  206. """Returns the path of this static asset."""
  207. return StaticAsset.get_static_path(self._name)
  208. def get_url(self):
  209. """Returns the URL of this static asset."""
  210. return staticfiles_storage.url(self._name)
  211. def get_mtime(self):
  212. """Returns the last modified time of this asset."""
  213. if settings.DEBUG:
  214. return os.path.getmtime(self.get_path())
  215. return staticfiles_storage.modified_time(self.get_name())
  216. class FileAsset(Asset):
  217. """An asset that wraps a file."""
  218. def __init__(self, file):
  219. """Initializes the file asset."""
  220. self._file = file
  221. def get_name(self):
  222. """Returns the name of this asset."""
  223. return self._file.name
  224. def get_path(self):
  225. """Returns the path of this asset."""
  226. try:
  227. return self._file.path
  228. except AttributeError:
  229. return os.path.abspath(self._file.name)
  230. def get_url(self):
  231. """Returns the URL of this asset."""
  232. try:
  233. return self._file.url
  234. except AttributeError:
  235. raise NotImplementedError("Underlying file does not have a URL.")
  236. def get_mtime(self):
  237. """Returns the mtime of this asset."""
  238. storage = getattr(self._file, "storage", None)
  239. if storage:
  240. return storage.modified_time(self._file.name)
  241. return super(FileAsset, self).get_mtime()
  242. def open(self):
  243. """Opens this asset."""
  244. self._file.open("rb")
  245. return self._file
  246. class GroupedAsset(Asset):
  247. """An asset composed of multiple sub-assets."""
  248. join_str = ""
  249. def __init__(self, assets):
  250. """Initializes the grouped asset."""
  251. self._assets = assets
  252. def get_name(self):
  253. """Returns the name of this asset."""
  254. return self._assets[0].get_name()
  255. def get_id_params(self):
  256. """"Returns the params which should be used to generate the id."""
  257. params = {}
  258. # Add in the assets.
  259. for n, asset in enumerate(self._assets):
  260. params.update(
  261. ("{n}_{key}".format(
  262. n = n,
  263. key = key,
  264. ), value)
  265. for key, value
  266. in six.iteritems(asset._get_and_check_id_params())
  267. )
  268. # All done.
  269. return params
  270. def get_mtime(self):
  271. """Returns the modified time for this asset."""
  272. return max(asset.get_mtime() for asset in self._assets)
  273. def get_contents(self):
  274. """Loads all the js code."""
  275. return force_bytes(self.join_str).join(asset.get_contents() for asset in self._assets)
  276. def get_hash(self):
  277. """Returns the sha1 hash of this asset's contents."""
  278. return hashlib.sha1("".join(asset.get_hash() for asset in self._assets).encode("utf-8")).hexdigest()
  279. def open(self):
  280. """Returns an open file pointer."""
  281. return ContentFile(self.get_contents())
  282. class AdaptiveAsset(Asset):
  283. """An asset that adapts to wrap as many types as possible."""
  284. def __new__(cls, asset):
  285. """Creates the new asset."""
  286. if isinstance(asset, Asset):
  287. return asset
  288. if isinstance(asset, File):
  289. return FileAsset(asset)
  290. if isinstance(asset, six.string_types):
  291. return StaticAsset(asset)
  292. raise TypeError("{!r} is not a valid asset".format(asset))
  293. class AssetCache(object):
  294. """A cache of assets."""
  295. def __init__(self, storage=default_storage, prefix="assets", cache_name="optimizations.assetcache"):
  296. """Initializes the asset cache."""
  297. self._storage = storage
  298. self._prefix = prefix
  299. self._cache = resolve_namespaced_cache(cache_name)
  300. def get_name_and_meta(self, asset):
  301. """Returns the name and associated parameters of an asset."""
  302. # Get the asset ID.
  303. asset_cache_key = asset.get_cache_key()
  304. name_and_meta = self._cache.get(asset_cache_key)
  305. if name_and_meta is None:
  306. # Generate the name.
  307. asset_hash = asset.get_hash()
  308. asset_ext = asset.get_save_extension()
  309. name = "{prefix}/{folder}/{hash}{ext}".format(
  310. prefix = self._prefix,
  311. folder = asset_hash[:2],
  312. hash = asset_hash[2:],
  313. ext = asset_ext,
  314. )
  315. # Save the asset's params.
  316. meta = asset.get_save_meta()
  317. # Save the file to the asset cache.
  318. if not self._storage.exists(name):
  319. asset.save(self._storage, name, meta)
  320. # Cache the name.
  321. name_and_meta = (name, meta)
  322. self._cache.set(asset_cache_key, name_and_meta)
  323. return name_and_meta
  324. def get_name(self, asset):
  325. """Returns the cached name of the given asset."""
  326. return self.get_name_and_meta(asset)[0]
  327. def get_meta(self, asset):
  328. """Returns the cached meta of the given asset."""
  329. return self.get_name_and_meta(asset)[1]
  330. def get_path(self, asset, force_save=None):
  331. """Returns the cached path of the given asset."""
  332. if force_save is None:
  333. force_save = not settings.DEBUG
  334. asset = AdaptiveAsset(asset)
  335. if not force_save:
  336. try:
  337. return asset.get_path()
  338. except NotImplementedError:
  339. pass
  340. return self._storage.path(self.get_name(asset))
  341. def get_url(self, asset, force_save=None):
  342. """Returns the cached url of the given asset."""
  343. if force_save is None:
  344. force_save = not settings.DEBUG
  345. asset = AdaptiveAsset(asset)
  346. if not force_save:
  347. try:
  348. return asset.get_url()
  349. except NotImplementedError:
  350. pass
  351. return self._storage.url(self.get_name(asset))
  352. # The default asset cache.
  353. default_asset_cache = AssetCache()