PageRenderTime 24ms CodeModel.GetById 29ms RepoModel.GetById 1ms app.codeStats 0ms

/django/contrib/staticfiles/storage.py

https://github.com/gregmuellegger/django
Python | 290 lines | 199 code | 31 blank | 60 comment | 34 complexity | 150665df5701ec3dd3a202cb9b6704b1 MD5 | raw file
  1. from __future__ import unicode_literals
  2. import hashlib
  3. import os
  4. import posixpath
  5. import re
  6. from urllib import unquote
  7. from urlparse import urlsplit, urlunsplit, urldefrag
  8. from django.conf import settings
  9. from django.core.cache import (get_cache, InvalidCacheBackendError,
  10. cache as default_cache)
  11. from django.core.exceptions import ImproperlyConfigured
  12. from django.core.files.base import ContentFile
  13. from django.core.files.storage import FileSystemStorage, get_storage_class
  14. from django.utils.datastructures import SortedDict
  15. from django.utils.encoding import force_unicode, smart_str
  16. from django.utils.functional import LazyObject
  17. from django.utils.importlib import import_module
  18. from django.contrib.staticfiles.utils import check_settings, matches_patterns
  19. class StaticFilesStorage(FileSystemStorage):
  20. """
  21. Standard file system storage for static files.
  22. The defaults for ``location`` and ``base_url`` are
  23. ``STATIC_ROOT`` and ``STATIC_URL``.
  24. """
  25. def __init__(self, location=None, base_url=None, *args, **kwargs):
  26. if location is None:
  27. location = settings.STATIC_ROOT
  28. if base_url is None:
  29. base_url = settings.STATIC_URL
  30. check_settings(base_url)
  31. super(StaticFilesStorage, self).__init__(location, base_url,
  32. *args, **kwargs)
  33. def path(self, name):
  34. if not self.location:
  35. raise ImproperlyConfigured("You're using the staticfiles app "
  36. "without having set the STATIC_ROOT "
  37. "setting to a filesystem path.")
  38. return super(StaticFilesStorage, self).path(name)
  39. class CachedFilesMixin(object):
  40. patterns = (
  41. ("*.css", (
  42. br"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""",
  43. br"""(@import\s*["']\s*(.*?)["'])""",
  44. )),
  45. )
  46. def __init__(self, *args, **kwargs):
  47. super(CachedFilesMixin, self).__init__(*args, **kwargs)
  48. try:
  49. self.cache = get_cache('staticfiles')
  50. except InvalidCacheBackendError:
  51. # Use the default backend
  52. self.cache = default_cache
  53. self._patterns = SortedDict()
  54. for extension, patterns in self.patterns:
  55. for pattern in patterns:
  56. compiled = re.compile(pattern)
  57. self._patterns.setdefault(extension, []).append(compiled)
  58. def file_hash(self, name, content=None):
  59. """
  60. Retuns a hash of the file with the given name and optional content.
  61. """
  62. if content is None:
  63. return None
  64. md5 = hashlib.md5()
  65. for chunk in content.chunks():
  66. md5.update(chunk)
  67. return md5.hexdigest()[:12]
  68. def hashed_name(self, name, content=None):
  69. parsed_name = urlsplit(unquote(name))
  70. clean_name = parsed_name.path.strip()
  71. if content is None:
  72. if not self.exists(clean_name):
  73. raise ValueError("The file '%s' could not be found with %r." %
  74. (clean_name, self))
  75. try:
  76. content = self.open(clean_name)
  77. except IOError:
  78. # Handle directory paths and fragments
  79. return name
  80. path, filename = os.path.split(clean_name)
  81. root, ext = os.path.splitext(filename)
  82. file_hash = self.file_hash(clean_name, content)
  83. if file_hash is not None:
  84. file_hash = ".%s" % file_hash
  85. hashed_name = os.path.join(path, "%s%s%s" %
  86. (root, file_hash, ext))
  87. unparsed_name = list(parsed_name)
  88. unparsed_name[2] = hashed_name
  89. # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
  90. # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
  91. if '?#' in name and not unparsed_name[3]:
  92. unparsed_name[2] += '?'
  93. return urlunsplit(unparsed_name)
  94. def cache_key(self, name):
  95. return 'staticfiles:%s' % hashlib.md5(smart_str(name)).hexdigest()
  96. def url(self, name, force=False):
  97. """
  98. Returns the real URL in DEBUG mode.
  99. """
  100. if settings.DEBUG and not force:
  101. hashed_name, fragment = name, ''
  102. else:
  103. clean_name, fragment = urldefrag(name)
  104. if urlsplit(clean_name).path.endswith('/'): # don't hash paths
  105. hashed_name = name
  106. else:
  107. cache_key = self.cache_key(name)
  108. hashed_name = self.cache.get(cache_key)
  109. if hashed_name is None:
  110. hashed_name = self.hashed_name(clean_name).replace('\\', '/')
  111. # set the cache if there was a miss
  112. # (e.g. if cache server goes down)
  113. self.cache.set(cache_key, hashed_name)
  114. final_url = super(CachedFilesMixin, self).url(hashed_name)
  115. # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
  116. # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
  117. query_fragment = '?#' in name # [sic!]
  118. if fragment or query_fragment:
  119. urlparts = list(urlsplit(final_url))
  120. if fragment and not urlparts[4]:
  121. urlparts[4] = fragment
  122. if query_fragment and not urlparts[3]:
  123. urlparts[2] += '?'
  124. final_url = urlunsplit(urlparts)
  125. return unquote(final_url)
  126. def url_converter(self, name):
  127. """
  128. Returns the custom URL converter for the given file name.
  129. """
  130. def converter(matchobj):
  131. """
  132. Converts the matched URL depending on the parent level (`..`)
  133. and returns the normalized and hashed URL using the url method
  134. of the storage.
  135. """
  136. matched, url = matchobj.groups()
  137. # Completely ignore http(s) prefixed URLs,
  138. # fragments and data-uri URLs
  139. if url.startswith(('#', 'http:', 'https:', 'data:')):
  140. return matched
  141. name_parts = name.split(os.sep)
  142. # Using posix normpath here to remove duplicates
  143. url = posixpath.normpath(url)
  144. url_parts = url.split('/')
  145. parent_level, sub_level = url.count('..'), url.count('/')
  146. if url.startswith('/'):
  147. sub_level -= 1
  148. url_parts = url_parts[1:]
  149. if parent_level or not url.startswith('/'):
  150. start, end = parent_level + 1, parent_level
  151. else:
  152. if sub_level:
  153. if sub_level == 1:
  154. parent_level -= 1
  155. start, end = parent_level, 1
  156. else:
  157. start, end = 1, sub_level - 1
  158. joined_result = '/'.join(name_parts[:-start] + url_parts[end:])
  159. hashed_url = self.url(unquote(joined_result), force=True)
  160. file_name = hashed_url.split('/')[-1:]
  161. relative_url = '/'.join(url.split('/')[:-1] + file_name)
  162. # Return the hashed version to the file
  163. return 'url("%s")' % unquote(relative_url)
  164. return converter
  165. def post_process(self, paths, dry_run=False, **options):
  166. """
  167. Post process the given list of files (called from collectstatic).
  168. Processing is actually two separate operations:
  169. 1. renaming files to include a hash of their content for cache-busting,
  170. and copying those files to the target storage.
  171. 2. adjusting files which contain references to other files so they
  172. refer to the cache-busting filenames.
  173. If either of these are performed on a file, then that file is considered
  174. post-processed.
  175. """
  176. # don't even dare to process the files if we're in dry run mode
  177. if dry_run:
  178. return
  179. # where to store the new paths
  180. hashed_paths = {}
  181. # build a list of adjustable files
  182. matches = lambda path: matches_patterns(path, self._patterns.keys())
  183. adjustable_paths = [path for path in paths if matches(path)]
  184. # then sort the files by the directory level
  185. path_level = lambda name: len(name.split(os.sep))
  186. for name in sorted(paths.keys(), key=path_level, reverse=True):
  187. # use the original, local file, not the copied-but-unprocessed
  188. # file, which might be somewhere far away, like S3
  189. storage, path = paths[name]
  190. with storage.open(path) as original_file:
  191. # generate the hash with the original content, even for
  192. # adjustable files.
  193. hashed_name = self.hashed_name(name, original_file)
  194. # then get the original's file content..
  195. if hasattr(original_file, 'seek'):
  196. original_file.seek(0)
  197. hashed_file_exists = self.exists(hashed_name)
  198. processed = False
  199. # ..to apply each replacement pattern to the content
  200. if name in adjustable_paths:
  201. content = original_file.read()
  202. converter = self.url_converter(name)
  203. for patterns in self._patterns.values():
  204. for pattern in patterns:
  205. content = pattern.sub(converter, content)
  206. if hashed_file_exists:
  207. self.delete(hashed_name)
  208. # then save the processed result
  209. content_file = ContentFile(smart_str(content))
  210. saved_name = self._save(hashed_name, content_file)
  211. hashed_name = force_unicode(saved_name.replace('\\', '/'))
  212. processed = True
  213. else:
  214. # or handle the case in which neither processing nor
  215. # a change to the original file happened
  216. if not hashed_file_exists:
  217. processed = True
  218. saved_name = self._save(hashed_name, original_file)
  219. hashed_name = force_unicode(saved_name.replace('\\', '/'))
  220. # and then set the cache accordingly
  221. hashed_paths[self.cache_key(name)] = hashed_name
  222. yield name, hashed_name, processed
  223. # Finally set the cache
  224. self.cache.set_many(hashed_paths)
  225. class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
  226. """
  227. A static file system storage backend which also saves
  228. hashed copies of the files it saves.
  229. """
  230. pass
  231. class AppStaticStorage(FileSystemStorage):
  232. """
  233. A file system storage backend that takes an app module and works
  234. for the ``static`` directory of it.
  235. """
  236. prefix = None
  237. source_dir = 'static'
  238. def __init__(self, app, *args, **kwargs):
  239. """
  240. Returns a static file storage if available in the given app.
  241. """
  242. # app is the actual app module
  243. mod = import_module(app)
  244. mod_path = os.path.dirname(mod.__file__)
  245. location = os.path.join(mod_path, self.source_dir)
  246. super(AppStaticStorage, self).__init__(location, *args, **kwargs)
  247. class ConfiguredStorage(LazyObject):
  248. def _setup(self):
  249. self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()
  250. staticfiles_storage = ConfiguredStorage()