/hyde/fs.py

http://github.com/hyde/hyde · Python · 618 lines · 500 code · 22 blank · 96 comment · 15 complexity · 087b5e421038e3a96a1a80ec14f146e9 MD5 · raw file

  1. # -*- coding: utf-8 -*-
  2. """
  3. Unified object oriented interface for interacting with file system objects.
  4. File system operations in python are distributed across modules: os, os.path,
  5. fnamtch, shutil and distutils. This module attempts to make the right choices
  6. for common operations to provide a single interface.
  7. """
  8. import codecs
  9. from datetime import datetime
  10. import mimetypes
  11. import os
  12. import shutil
  13. from distutils import dir_util
  14. import functools
  15. import fnmatch
  16. from hyde.util import getLoggerWithNullHandler
  17. logger = getLoggerWithNullHandler('fs')
  18. # pylint: disable-msg=E0611
  19. __all__ = ['File', 'Folder']
  20. class FS(object):
  21. """
  22. The base file system object
  23. """
  24. def __init__(self, path):
  25. super(FS, self).__init__()
  26. if path == os.sep:
  27. self.path = path
  28. else:
  29. self.path = os.path.expandvars(os.path.expanduser(
  30. unicode(path).strip().rstrip(os.sep)))
  31. def __str__(self):
  32. return self.path
  33. def __repr__(self):
  34. return self.path
  35. def __eq__(self, other):
  36. return unicode(self) == unicode(other)
  37. def __ne__(self, other):
  38. return unicode(self) != unicode(other)
  39. @property
  40. def fully_expanded_path(self):
  41. """
  42. Returns the absolutely absolute path. Calls os.(
  43. normpath, normcase, expandvars and expanduser).
  44. """
  45. return os.path.abspath(
  46. os.path.normpath(
  47. os.path.normcase(
  48. os.path.expandvars(
  49. os.path.expanduser(self.path)))))
  50. @property
  51. def exists(self):
  52. """
  53. Does the file system object exist?
  54. """
  55. return os.path.exists(self.path)
  56. @property
  57. def name(self):
  58. """
  59. Returns the name of the FS object with its extension
  60. """
  61. return os.path.basename(self.path)
  62. @property
  63. def parent(self):
  64. """
  65. The parent folder. Returns a `Folder` object.
  66. """
  67. return Folder(os.path.dirname(self.path))
  68. @property
  69. def depth(self):
  70. """
  71. Returns the number of ancestors of this directory.
  72. """
  73. return len(self.path.rstrip(os.sep).split(os.sep))
  74. def ancestors(self, stop=None):
  75. """
  76. Generates the parents until stop or the absolute
  77. root directory is reached.
  78. """
  79. folder = self
  80. while folder.parent != stop:
  81. if folder.parent == folder:
  82. return
  83. yield folder.parent
  84. folder = folder.parent
  85. def is_descendant_of(self, ancestor):
  86. """
  87. Checks if this folder is inside the given ancestor.
  88. """
  89. stop = Folder(ancestor)
  90. for folder in self.ancestors():
  91. if folder == stop:
  92. return True
  93. if stop.depth > folder.depth:
  94. return False
  95. return False
  96. def get_relative_path(self, root):
  97. """
  98. Gets the fragment of the current path starting at root.
  99. """
  100. if self.path == root:
  101. return ''
  102. ancestors = self.ancestors(stop=root)
  103. return functools.reduce(lambda f, p: Folder(p.name).child(f),
  104. ancestors,
  105. self.name)
  106. def get_mirror(self, target_root, source_root=None):
  107. """
  108. Returns a File or Folder object that reperesents if the entire
  109. fragment of this directory starting with `source_root` were copied
  110. to `target_root`.
  111. >>> Folder('/usr/local/hyde/stuff').get_mirror('/usr/tmp',
  112. source_root='/usr/local/hyde')
  113. Folder('/usr/tmp/stuff')
  114. """
  115. fragment = self.get_relative_path(
  116. source_root if source_root else self.parent)
  117. return Folder(target_root).child(fragment)
  118. @staticmethod
  119. def file_or_folder(path):
  120. """
  121. Returns a File or Folder object that would represent the given path.
  122. """
  123. target = unicode(path)
  124. return Folder(target) if os.path.isdir(target) else File(target)
  125. def __get_destination__(self, destination):
  126. """
  127. Returns a File or Folder object that would represent this entity
  128. if it were copied or moved to `destination`.
  129. """
  130. if isinstance(destination, File) or os.path.isfile(unicode(destination)):
  131. return destination
  132. else:
  133. return FS.file_or_folder(Folder(destination).child(self.name))
  134. class File(FS):
  135. """
  136. The File object.
  137. """
  138. def __init__(self, path):
  139. super(File, self).__init__(path)
  140. @property
  141. def name_without_extension(self):
  142. """
  143. Returns the name of the FS object without its extension
  144. """
  145. return os.path.splitext(self.name)[0]
  146. @property
  147. def extension(self):
  148. """
  149. File extension prefixed with a dot.
  150. """
  151. return os.path.splitext(self.path)[1]
  152. @property
  153. def kind(self):
  154. """
  155. File extension without dot prefix.
  156. """
  157. return self.extension.lstrip(".")
  158. @property
  159. def size(self):
  160. """
  161. Size of this file.
  162. """
  163. if not self.exists:
  164. return -1
  165. return os.path.getsize(self.path)
  166. @property
  167. def mimetype(self):
  168. """
  169. Gets the mimetype of this file.
  170. """
  171. (mime, _) = mimetypes.guess_type(self.path)
  172. return mime
  173. @property
  174. def is_binary(self):
  175. """Return true if this is a binary file."""
  176. with open(self.path, 'rb') as fin:
  177. CHUNKSIZE = 1024
  178. while 1:
  179. chunk = fin.read(CHUNKSIZE)
  180. if '\0' in chunk:
  181. return True
  182. if len(chunk) < CHUNKSIZE:
  183. break
  184. return False
  185. @property
  186. def is_text(self):
  187. """Return true if this is a text file."""
  188. return (not self.is_binary)
  189. @property
  190. def is_image(self):
  191. """Return true if this is an image file."""
  192. return self.mimetype.split("/")[0] == "image"
  193. @property
  194. def last_modified(self):
  195. """
  196. Returns a datetime object representing the last modified time.
  197. Calls os.path.getmtime.
  198. """
  199. return datetime.fromtimestamp(os.path.getmtime(self.path))
  200. def has_changed_since(self, basetime):
  201. """
  202. Returns True if the file has been changed since the given time.
  203. """
  204. return self.last_modified > basetime
  205. def older_than(self, another_file):
  206. """
  207. Checks if this file is older than the given file. Uses last_modified to
  208. determine age.
  209. """
  210. return self.last_modified < File(unicode(another_file)).last_modified
  211. @staticmethod
  212. def make_temp(text):
  213. """
  214. Creates a temprorary file and writes the `text` into it
  215. """
  216. import tempfile
  217. (handle, path) = tempfile.mkstemp(text=True)
  218. os.close(handle)
  219. afile = File(path)
  220. afile.write(text)
  221. return afile
  222. def read_all(self, encoding='utf-8'):
  223. """
  224. Reads from the file and returns the content as a string.
  225. """
  226. logger.info("Reading everything from %s" % self)
  227. with codecs.open(self.path, 'r', encoding) as fin:
  228. read_text = fin.read()
  229. return read_text
  230. def write(self, text, encoding="utf-8"):
  231. """
  232. Writes the given text to the file using the given encoding.
  233. """
  234. logger.info("Writing to %s" % self)
  235. with codecs.open(self.path, 'w', encoding) as fout:
  236. fout.write(text)
  237. def copy_to(self, destination):
  238. """
  239. Copies the file to the given destination. Returns a File
  240. object that represents the target file. `destination` must
  241. be a File or Folder object.
  242. """
  243. target = self.__get_destination__(destination)
  244. logger.info("Copying %s to %s" % (self, target))
  245. shutil.copy(self.path, unicode(destination))
  246. return target
  247. def delete(self):
  248. """
  249. Delete the file if it exists.
  250. """
  251. if self.exists:
  252. os.remove(self.path)
  253. class FSVisitor(object):
  254. """
  255. Implements syntactic sugar for walking and listing folders
  256. """
  257. def __init__(self, folder, pattern=None):
  258. super(FSVisitor, self).__init__()
  259. self.folder = folder
  260. self.pattern = pattern
  261. def folder_visitor(self, function):
  262. """
  263. Decorator for `visit_folder` protocol
  264. """
  265. self.visit_folder = function
  266. return function
  267. def file_visitor(self, function):
  268. """
  269. Decorator for `visit_file` protocol
  270. """
  271. self.visit_file = function
  272. return function
  273. def finalizer(self, function):
  274. """
  275. Decorator for `visit_complete` protocol
  276. """
  277. self.visit_complete = function
  278. return function
  279. def __enter__(self):
  280. return self
  281. def __exit__(self, exc_type, exc_val, exc_tb):
  282. pass
  283. class FolderWalker(FSVisitor):
  284. """
  285. Walks the entire hirearchy of this directory starting with itself.
  286. If a pattern is provided, only the files that match the pattern are
  287. processed.
  288. """
  289. def walk(self, walk_folders=False, walk_files=False):
  290. """
  291. A simple generator that yields a File or Folder object based on
  292. the arguments.
  293. """
  294. if not walk_files and not walk_folders:
  295. return
  296. for root, _, a_files in os.walk(self.folder.path, followlinks=True):
  297. folder = Folder(root)
  298. if walk_folders:
  299. yield folder
  300. if walk_files:
  301. for a_file in a_files:
  302. if (not self.pattern or
  303. fnmatch.fnmatch(a_file, self.pattern)):
  304. yield File(folder.child(a_file))
  305. def walk_all(self):
  306. """
  307. Yield both Files and Folders as the tree is walked.
  308. """
  309. return self.walk(walk_folders=True, walk_files=True)
  310. def walk_files(self):
  311. """
  312. Yield only Files.
  313. """
  314. return self.walk(walk_folders=False, walk_files=True)
  315. def walk_folders(self):
  316. """
  317. Yield only Folders.
  318. """
  319. return self.walk(walk_folders=True, walk_files=False)
  320. def __exit__(self, exc_type, exc_val, exc_tb):
  321. """
  322. Automatically walk the folder when the context manager is exited.
  323. Calls self.visit_folder first and then calls self.visit_file for
  324. any files found. After all files and folders have been exhausted
  325. self.visit_complete is called.
  326. If visitor.visit_folder returns False, the files in the folder are not
  327. processed.
  328. """
  329. def __visit_folder__(folder):
  330. process_folder = True
  331. if hasattr(self, 'visit_folder'):
  332. process_folder = self.visit_folder(folder)
  333. # If there is no return value assume true
  334. #
  335. if process_folder is None:
  336. process_folder = True
  337. return process_folder
  338. def __visit_file__(a_file):
  339. if hasattr(self, 'visit_file'):
  340. self.visit_file(a_file)
  341. def __visit_complete__():
  342. if hasattr(self, 'visit_complete'):
  343. self.visit_complete()
  344. for root, dirs, a_files in os.walk(self.folder.path, followlinks=True):
  345. folder = Folder(root)
  346. if not __visit_folder__(folder):
  347. dirs[:] = []
  348. continue
  349. for a_file in a_files:
  350. if not self.pattern or fnmatch.fnmatch(a_file, self.pattern):
  351. __visit_file__(File(folder.child(a_file)))
  352. __visit_complete__()
  353. class FolderLister(FSVisitor):
  354. """
  355. Lists the contents of this directory.
  356. If a pattern is provided, only the files that match the pattern are
  357. processed.
  358. """
  359. def list(self, list_folders=False, list_files=False):
  360. """
  361. A simple generator that yields a File or Folder object based on
  362. the arguments.
  363. """
  364. a_files = os.listdir(self.folder.path)
  365. for a_file in a_files:
  366. path = self.folder.child(a_file)
  367. if os.path.isdir(path):
  368. if list_folders:
  369. yield Folder(path)
  370. elif list_files:
  371. if not self.pattern or fnmatch.fnmatch(a_file, self.pattern):
  372. yield File(path)
  373. def list_all(self):
  374. """
  375. Yield both Files and Folders as the folder is listed.
  376. """
  377. return self.list(list_folders=True, list_files=True)
  378. def list_files(self):
  379. """
  380. Yield only Files.
  381. """
  382. return self.list(list_folders=False, list_files=True)
  383. def list_folders(self):
  384. """
  385. Yield only Folders.
  386. """
  387. return self.list(list_folders=True, list_files=False)
  388. def __exit__(self, exc_type, exc_val, exc_tb):
  389. """
  390. Automatically list the folder contents when the context manager
  391. is exited.
  392. Calls self.visit_folder first and then calls self.visit_file for
  393. any files found. After all files and folders have been exhausted
  394. self.visit_complete is called.
  395. """
  396. a_files = os.listdir(self.folder.path)
  397. for a_file in a_files:
  398. path = self.folder.child(a_file)
  399. if os.path.isdir(path) and hasattr(self, 'visit_folder'):
  400. self.visit_folder(Folder(path))
  401. elif hasattr(self, 'visit_file'):
  402. if not self.pattern or fnmatch.fnmatch(a_file, self.pattern):
  403. self.visit_file(File(path))
  404. if hasattr(self, 'visit_complete'):
  405. self.visit_complete()
  406. class Folder(FS):
  407. """
  408. Represents a directory.
  409. """
  410. def __init__(self, path):
  411. super(Folder, self).__init__(path)
  412. def child_folder(self, fragment):
  413. """
  414. Returns a folder object by combining the fragment to this folder's path
  415. """
  416. return Folder(os.path.join(self.path, Folder(fragment).path))
  417. def child(self, fragment):
  418. """
  419. Returns a path of a child item represented by `fragment`.
  420. """
  421. return os.path.join(self.path, FS(fragment).path)
  422. def make(self):
  423. """
  424. Creates this directory and any of the missing directories in the path.
  425. Any errors that may occur are eaten.
  426. """
  427. try:
  428. if not self.exists:
  429. logger.info("Creating %s" % self.path)
  430. os.makedirs(self.path)
  431. except os.error:
  432. pass
  433. return self
  434. def delete(self):
  435. """
  436. Deletes the directory if it exists.
  437. """
  438. if self.exists:
  439. logger.info("Deleting %s" % self.path)
  440. shutil.rmtree(self.path)
  441. def copy_to(self, destination):
  442. """
  443. Copies this directory to the given destination. Returns a Folder object
  444. that represents the moved directory.
  445. """
  446. target = self.__get_destination__(destination)
  447. logger.info("Copying %s to %s" % (self, target))
  448. shutil.copytree(self.path, unicode(target))
  449. return target
  450. def move_to(self, destination):
  451. """
  452. Moves this directory to the given destination. Returns a Folder object
  453. that represents the moved directory.
  454. """
  455. target = self.__get_destination__(destination)
  456. logger.info("Move %s to %s" % (self, target))
  457. shutil.move(self.path, unicode(target))
  458. return target
  459. def rename_to(self, destination_name):
  460. """
  461. Moves this directory to the given destination. Returns a Folder object
  462. that represents the moved directory.
  463. """
  464. target = self.parent.child_folder(destination_name)
  465. logger.info("Rename %s to %s" % (self, target))
  466. shutil.move(self.path, unicode(target))
  467. return target
  468. def _create_target_tree(self, target):
  469. """
  470. There is a bug in dir_util that makes `copy_tree` crash if a folder in
  471. the tree has been deleted before and readded now. To workaround the
  472. bug, we first walk the tree and create directories that are needed.
  473. """
  474. source = self
  475. with source.walker as walker:
  476. @walker.folder_visitor
  477. def visit_folder(folder):
  478. """
  479. Create the mirror directory
  480. """
  481. if folder != source:
  482. Folder(folder.get_mirror(target, source)).make()
  483. def copy_contents_to(self, destination):
  484. """
  485. Copies the contents of this directory to the given destination.
  486. Returns a Folder object that represents the moved directory.
  487. """
  488. logger.info("Copying contents of %s to %s" % (self, destination))
  489. target = Folder(destination)
  490. target.make()
  491. self._create_target_tree(target)
  492. dir_util.copy_tree(self.path, unicode(target))
  493. return target
  494. def get_walker(self, pattern=None):
  495. """
  496. Return a `FolderWalker` object with a set pattern.
  497. """
  498. return FolderWalker(self, pattern)
  499. @property
  500. def walker(self):
  501. """
  502. Return a `FolderWalker` object
  503. """
  504. return FolderWalker(self)
  505. def get_lister(self, pattern=None):
  506. """
  507. Return a `FolderLister` object with a set pattern.
  508. """
  509. return FolderLister(self, pattern)
  510. @property
  511. def lister(self):
  512. """
  513. Return a `FolderLister` object
  514. """
  515. return FolderLister(self)