PageRenderTime 58ms CodeModel.GetById 24ms RepoModel.GetById 0ms app.codeStats 0ms

/source/ftrack_api/cache.py

https://gitlab.com/themill/ftrack-python-api
Python | 579 lines | 361 code | 63 blank | 155 comment | 33 complexity | 186f614c65c9dcd888883a8585566931 MD5 | raw file
  1. # :coding: utf-8
  2. # :copyright: Copyright (c) 2014 ftrack
  3. '''Caching framework.
  4. Defines a standardised :class:`Cache` interface for storing data against
  5. specific keys. Key generation is also standardised using a :class:`KeyMaker`
  6. interface.
  7. Combining a Cache and KeyMaker allows for memoisation of function calls with
  8. respect to the arguments used by using a :class:`Memoiser`.
  9. As a convenience a simple :func:`memoise` decorator is included for quick
  10. memoisation of function using a global cache and standard key maker.
  11. '''
  12. import collections
  13. import functools
  14. import abc
  15. import copy
  16. import inspect
  17. import re
  18. import anydbm
  19. import contextlib
  20. try:
  21. import cPickle as pickle
  22. except ImportError: # pragma: no cover
  23. import pickle
  24. import ftrack_api.inspection
  25. import ftrack_api.symbol
  26. class Cache(object):
  27. '''Cache interface.
  28. Derive from this to define concrete cache implementations. A cache is
  29. centered around the concept of key:value pairings where the key is unique
  30. across the cache.
  31. '''
  32. __metaclass__ = abc.ABCMeta
  33. @abc.abstractmethod
  34. def get(self, key):
  35. '''Return value for *key*.
  36. Raise :exc:`KeyError` if *key* not found.
  37. '''
  38. @abc.abstractmethod
  39. def set(self, key, value):
  40. '''Set *value* for *key*.'''
  41. @abc.abstractmethod
  42. def remove(self, key):
  43. '''Remove *key* and return stored value.
  44. Raise :exc:`KeyError` if *key* not found.
  45. '''
  46. def keys(self):
  47. '''Return list of keys at this current time.
  48. .. warning::
  49. Actual keys may differ from those returned due to timing of access.
  50. '''
  51. raise NotImplementedError() # pragma: no cover
  52. def values(self):
  53. '''Return values for current keys.'''
  54. values = []
  55. for key in self.keys():
  56. try:
  57. value = self.get(key)
  58. except KeyError:
  59. continue
  60. else:
  61. values.append(value)
  62. return values
  63. def clear(self, pattern=None):
  64. '''Remove all keys matching *pattern*.
  65. *pattern* should be a regular expression string.
  66. If *pattern* is None then all keys will be removed.
  67. '''
  68. if pattern is not None:
  69. pattern = re.compile(pattern)
  70. for key in self.keys():
  71. if pattern is not None:
  72. if not pattern.search(key):
  73. continue
  74. try:
  75. self.remove(key)
  76. except KeyError:
  77. pass
  78. class ProxyCache(Cache):
  79. '''Proxy another cache.'''
  80. def __init__(self, proxied):
  81. '''Initialise cache with *proxied* cache instance.'''
  82. self.proxied = proxied
  83. super(ProxyCache, self).__init__()
  84. def get(self, key):
  85. '''Return value for *key*.
  86. Raise :exc:`KeyError` if *key* not found.
  87. '''
  88. return self.proxied.get(key)
  89. def set(self, key, value):
  90. '''Set *value* for *key*.'''
  91. return self.proxied.set(key, value)
  92. def remove(self, key):
  93. '''Remove *key* and return stored value.
  94. Raise :exc:`KeyError` if *key* not found.
  95. '''
  96. return self.proxied.remove(key)
  97. def keys(self):
  98. '''Return list of keys at this current time.
  99. .. warning::
  100. Actual keys may differ from those returned due to timing of access.
  101. '''
  102. return self.proxied.keys()
  103. class LayeredCache(Cache):
  104. '''Layered cache.'''
  105. def __init__(self, caches):
  106. '''Initialise cache with *caches*.'''
  107. super(LayeredCache, self).__init__()
  108. self.caches = caches
  109. def get(self, key):
  110. '''Return value for *key*.
  111. Raise :exc:`KeyError` if *key* not found.
  112. Attempt to retrieve from cache layers in turn, starting with shallowest.
  113. If value retrieved, then also set the value in each higher level cache
  114. up from where retrieved.
  115. '''
  116. target_caches = []
  117. value = ftrack_api.symbol.NOT_SET
  118. for cache in self.caches:
  119. try:
  120. value = cache.get(key)
  121. except KeyError:
  122. target_caches.append(cache)
  123. continue
  124. else:
  125. break
  126. if value is ftrack_api.symbol.NOT_SET:
  127. raise KeyError(key)
  128. # Set value on all higher level caches.
  129. for cache in target_caches:
  130. cache.set(key, value)
  131. return value
  132. def set(self, key, value):
  133. '''Set *value* for *key*.'''
  134. for cache in self.caches:
  135. cache.set(key, value)
  136. def remove(self, key):
  137. '''Remove *key*.
  138. Raise :exc:`KeyError` if *key* not found in any layer.
  139. '''
  140. removed = False
  141. for cache in self.caches:
  142. try:
  143. cache.remove(key)
  144. except KeyError:
  145. pass
  146. else:
  147. removed = True
  148. if not removed:
  149. raise KeyError(key)
  150. def keys(self):
  151. '''Return list of keys at this current time.
  152. .. warning::
  153. Actual keys may differ from those returned due to timing of access.
  154. '''
  155. keys = []
  156. for cache in self.caches:
  157. keys.extend(cache.keys())
  158. return list(set(keys))
  159. class MemoryCache(Cache):
  160. '''Memory based cache.'''
  161. def __init__(self):
  162. '''Initialise cache.'''
  163. self._cache = {}
  164. super(MemoryCache, self).__init__()
  165. def get(self, key):
  166. '''Return value for *key*.
  167. Raise :exc:`KeyError` if *key* not found.
  168. '''
  169. return self._cache[key]
  170. def set(self, key, value):
  171. '''Set *value* for *key*.'''
  172. self._cache[key] = value
  173. def remove(self, key):
  174. '''Remove *key*.
  175. Raise :exc:`KeyError` if *key* not found.
  176. '''
  177. del self._cache[key]
  178. def keys(self):
  179. '''Return list of keys at this current time.
  180. .. warning::
  181. Actual keys may differ from those returned due to timing of access.
  182. '''
  183. return self._cache.keys()
  184. class FileCache(Cache):
  185. '''File based cache that uses :mod:`anydbm` module.
  186. .. note::
  187. No locking of the underlying file is performed.
  188. '''
  189. def __init__(self, path):
  190. '''Initialise cache at *path*.'''
  191. self.path = path
  192. # Initialise cache.
  193. cache = anydbm.open(self.path, 'c')
  194. cache.close()
  195. super(FileCache, self).__init__()
  196. @contextlib.contextmanager
  197. def _database(self):
  198. '''Yield opened database file.'''
  199. cache = anydbm.open(self.path, 'w')
  200. try:
  201. yield cache
  202. finally:
  203. cache.close()
  204. def get(self, key):
  205. '''Return value for *key*.
  206. Raise :exc:`KeyError` if *key* not found.
  207. '''
  208. with self._database() as cache:
  209. return cache[key]
  210. def set(self, key, value):
  211. '''Set *value* for *key*.'''
  212. with self._database() as cache:
  213. cache[key] = value
  214. def remove(self, key):
  215. '''Remove *key*.
  216. Raise :exc:`KeyError` if *key* not found.
  217. '''
  218. with self._database() as cache:
  219. del cache[key]
  220. def keys(self):
  221. '''Return list of keys at this current time.
  222. .. warning::
  223. Actual keys may differ from those returned due to timing of access.
  224. '''
  225. with self._database() as cache:
  226. return cache.keys()
  227. class SerialisedCache(ProxyCache):
  228. '''Proxied cache that stores values as serialised data.'''
  229. def __init__(self, proxied, encode=None, decode=None):
  230. '''Initialise cache with *encode* and *decode* callables.
  231. *proxied* is the underlying cache to use for storage.
  232. '''
  233. self.encode = encode
  234. self.decode = decode
  235. super(SerialisedCache, self).__init__(proxied)
  236. def get(self, key):
  237. '''Return value for *key*.
  238. Raise :exc:`KeyError` if *key* not found.
  239. '''
  240. value = super(SerialisedCache, self).get(key)
  241. if self.decode:
  242. value = self.decode(value)
  243. return value
  244. def set(self, key, value):
  245. '''Set *value* for *key*.'''
  246. if self.encode:
  247. value = self.encode(value)
  248. super(SerialisedCache, self).set(key, value)
  249. class KeyMaker(object):
  250. '''Generate unique keys.'''
  251. __metaclass__ = abc.ABCMeta
  252. def __init__(self):
  253. '''Initialise key maker.'''
  254. super(KeyMaker, self).__init__()
  255. self.item_separator = ''
  256. def key(self, *items):
  257. '''Return key for *items*.'''
  258. keys = []
  259. for item in items:
  260. keys.append(self._key(item))
  261. return self.item_separator.join(keys)
  262. @abc.abstractmethod
  263. def _key(self, obj):
  264. '''Return key for *obj*.'''
  265. class StringKeyMaker(KeyMaker):
  266. '''Generate string key.'''
  267. def _key(self, obj):
  268. '''Return key for *obj*.'''
  269. return str(obj)
  270. class ObjectKeyMaker(KeyMaker):
  271. '''Generate unique keys for objects.'''
  272. def __init__(self):
  273. '''Initialise key maker.'''
  274. super(ObjectKeyMaker, self).__init__()
  275. self.item_separator = '\0'
  276. self.mapping_identifier = '\1'
  277. self.mapping_pair_separator = '\2'
  278. self.iterable_identifier = '\3'
  279. self.name_identifier = '\4'
  280. def _key(self, item):
  281. '''Return key for *item*.
  282. Returned key will be a pickle like string representing the *item*. This
  283. allows for typically non-hashable objects to be used in key generation
  284. (such as dictionaries).
  285. If *item* is iterable then each item in it shall also be passed to this
  286. method to ensure correct key generation.
  287. Special markers are used to distinguish handling of specific cases in
  288. order to ensure uniqueness of key corresponds directly to *item*.
  289. Example::
  290. >>> key_maker = ObjectKeyMaker()
  291. >>> def add(x, y):
  292. ... "Return sum of *x* and *y*."
  293. ... return x + y
  294. ...
  295. >>> key_maker.key(add, (1, 2))
  296. '\x04add\x00__main__\x00\x03\x80\x02K\x01.\x00\x80\x02K\x02.\x03'
  297. >>> key_maker.key(add, (1, 3))
  298. '\x04add\x00__main__\x00\x03\x80\x02K\x01.\x00\x80\x02K\x03.\x03'
  299. '''
  300. # TODO: Consider using a more robust and comprehensive solution such as
  301. # dill (https://github.com/uqfoundation/dill).
  302. if isinstance(item, collections.Iterable):
  303. if isinstance(item, basestring):
  304. return pickle.dumps(item, pickle.HIGHEST_PROTOCOL)
  305. if isinstance(item, collections.Mapping):
  306. contents = self.item_separator.join([
  307. (
  308. self._key(key) +
  309. self.mapping_pair_separator +
  310. self._key(value)
  311. )
  312. for key, value in sorted(item.items())
  313. ])
  314. return (
  315. self.mapping_identifier +
  316. contents +
  317. self.mapping_identifier
  318. )
  319. else:
  320. contents = self.item_separator.join([
  321. self._key(item) for item in item
  322. ])
  323. return (
  324. self.iterable_identifier +
  325. contents +
  326. self.iterable_identifier
  327. )
  328. elif inspect.ismethod(item):
  329. return ''.join((
  330. self.name_identifier,
  331. item.__name__,
  332. self.item_separator,
  333. item.im_class.__name__,
  334. self.item_separator,
  335. item.__module__
  336. ))
  337. elif inspect.isfunction(item) or inspect.isclass(item):
  338. return ''.join((
  339. self.name_identifier,
  340. item.__name__,
  341. self.item_separator,
  342. item.__module__
  343. ))
  344. elif inspect.isbuiltin(item):
  345. return self.name_identifier + item.__name__
  346. else:
  347. return pickle.dumps(item, pickle.HIGHEST_PROTOCOL)
  348. class Memoiser(object):
  349. '''Memoise function calls using a :class:`KeyMaker` and :class:`Cache`.
  350. Example::
  351. >>> memoiser = Memoiser(MemoryCache(), ObjectKeyMaker())
  352. >>> def add(x, y):
  353. ... "Return sum of *x* and *y*."
  354. ... print 'Called'
  355. ... return x + y
  356. ...
  357. >>> memoiser.call(add, (1, 2), {})
  358. Called
  359. >>> memoiser.call(add, (1, 2), {})
  360. >>> memoiser.call(add, (1, 3), {})
  361. Called
  362. '''
  363. def __init__(self, cache=None, key_maker=None, return_copies=True):
  364. '''Initialise with *cache* and *key_maker* to use.
  365. If *cache* is not specified a default :class:`MemoryCache` will be
  366. used. Similarly, if *key_maker* is not specified a default
  367. :class:`ObjectKeyMaker` will be used.
  368. If *return_copies* is True then all results returned from the cache will
  369. be deep copies to avoid indirect mutation of cached values.
  370. '''
  371. self.cache = cache
  372. if self.cache is None:
  373. self.cache = MemoryCache()
  374. self.key_maker = key_maker
  375. if self.key_maker is None:
  376. self.key_maker = ObjectKeyMaker()
  377. self.return_copies = return_copies
  378. super(Memoiser, self).__init__()
  379. def call(self, function, args=None, kw=None):
  380. '''Call *function* with *args* and *kw* and return result.
  381. If *function* was previously called with exactly the same arguments
  382. then return cached result if available.
  383. Store result for call in cache.
  384. '''
  385. if args is None:
  386. args = ()
  387. if kw is None:
  388. kw = {}
  389. # Support arguments being passed as positionals or keywords.
  390. arguments = inspect.getcallargs(function, *args, **kw)
  391. key = self.key_maker.key(function, arguments)
  392. try:
  393. value = self.cache.get(key)
  394. except KeyError:
  395. value = function(*args, **kw)
  396. self.cache.set(key, value)
  397. # If requested, deep copy value to return in order to avoid cached value
  398. # being inadvertently altered by the caller.
  399. if self.return_copies:
  400. value = copy.deepcopy(value)
  401. return value
  402. def memoise_decorator(memoiser):
  403. '''Decorator to memoise function calls using *memoiser*.'''
  404. def outer(function):
  405. @functools.wraps(function)
  406. def inner(*args, **kw):
  407. return memoiser.call(function, args, kw)
  408. return inner
  409. return outer
  410. #: Default memoiser.
  411. memoiser = Memoiser()
  412. #: Default memoise decorator using standard cache and key maker.
  413. memoise = memoise_decorator(memoiser)