/hyde/fs.py

  1# -*- coding: utf-8 -*-
  2"""
  3Unified object oriented interface for interacting with file system objects.
  4File system operations in python are distributed across modules: os, os.path,
  5fnamtch, shutil and distutils. This module attempts to make the right choices
  6for common operations to provide a single interface.
  7"""
  8
  9import codecs
 10from datetime import datetime
 11import mimetypes
 12import os
 13import shutil
 14from distutils import dir_util
 15import functools
 16import fnmatch
 17
 18from hyde.util import getLoggerWithNullHandler
 19
 20logger = getLoggerWithNullHandler('fs')
 21
 22# pylint: disable-msg=E0611
 23
 24
 25__all__ = ['File', 'Folder']
 26
 27
 28class FS(object):
 29    """
 30    The base file system object
 31    """
 32
 33    def __init__(self, path):
 34        super(FS, self).__init__()
 35        if path == os.sep:
 36            self.path = path
 37        else:
 38            self.path = os.path.expandvars(os.path.expanduser(
 39                        unicode(path).strip().rstrip(os.sep)))
 40
 41    def __str__(self):
 42        return self.path
 43
 44    def __repr__(self):
 45        return self.path
 46
 47    def __eq__(self, other):
 48        return unicode(self) == unicode(other)
 49
 50    def __ne__(self, other):
 51        return unicode(self) != unicode(other)
 52
 53    @property
 54    def fully_expanded_path(self):
 55        """
 56        Returns the absolutely absolute path. Calls os.(
 57        normpath, normcase, expandvars and expanduser).
 58        """
 59        return os.path.abspath(
 60        os.path.normpath(
 61        os.path.normcase(
 62        os.path.expandvars(
 63        os.path.expanduser(self.path)))))
 64
 65    @property
 66    def exists(self):
 67        """
 68        Does the file system object exist?
 69        """
 70        return os.path.exists(self.path)
 71
 72    @property
 73    def name(self):
 74        """
 75        Returns the name of the FS object with its extension
 76        """
 77        return os.path.basename(self.path)
 78
 79    @property
 80    def parent(self):
 81        """
 82        The parent folder. Returns a `Folder` object.
 83        """
 84        return Folder(os.path.dirname(self.path))
 85
 86    @property
 87    def depth(self):
 88        """
 89        Returns the number of ancestors of this directory.
 90        """
 91        return len(self.path.rstrip(os.sep).split(os.sep))
 92
 93    def ancestors(self, stop=None):
 94        """
 95        Generates the parents until stop or the absolute
 96        root directory is reached.
 97        """
 98        folder = self
 99        while folder.parent != stop:
100            if folder.parent == folder:
101                return
102            yield folder.parent
103            folder = folder.parent
104
105    def is_descendant_of(self, ancestor):
106        """
107        Checks if this folder is inside the given ancestor.
108        """
109        stop = Folder(ancestor)
110        for folder in self.ancestors():
111            if folder == stop:
112                return True
113            if stop.depth > folder.depth:
114                return False
115        return False
116
117    def get_relative_path(self, root):
118        """
119        Gets the fragment of the current path starting at root.
120        """
121        if self.path == root:
122            return ''
123        ancestors = self.ancestors(stop=root)
124        return functools.reduce(lambda f, p: Folder(p.name).child(f),
125                                            ancestors,
126                                            self.name)
127
128    def get_mirror(self, target_root, source_root=None):
129        """
130        Returns a File or Folder object that reperesents if the entire
131        fragment of this directory starting with `source_root` were copied
132        to `target_root`.
133
134        >>> Folder('/usr/local/hyde/stuff').get_mirror('/usr/tmp',
135                                                source_root='/usr/local/hyde')
136        Folder('/usr/tmp/stuff')
137        """
138        fragment = self.get_relative_path(
139                        source_root if source_root else self.parent)
140        return Folder(target_root).child(fragment)
141
142    @staticmethod
143    def file_or_folder(path):
144        """
145        Returns a File or Folder object that would represent the given path.
146        """
147        target = unicode(path)
148        return Folder(target) if os.path.isdir(target) else File(target)
149
150    def __get_destination__(self, destination):
151        """
152        Returns a File or Folder object that would represent this entity
153        if it were copied or moved to `destination`.
154        """
155        if isinstance(destination, File) or os.path.isfile(unicode(destination)):
156            return destination
157        else:
158            return FS.file_or_folder(Folder(destination).child(self.name))
159
160
161class File(FS):
162    """
163    The File object.
164    """
165
166    def __init__(self, path):
167        super(File, self).__init__(path)
168
169    @property
170    def name_without_extension(self):
171        """
172        Returns the name of the FS object without its extension
173        """
174        return os.path.splitext(self.name)[0]
175
176    @property
177    def extension(self):
178        """
179        File extension prefixed with a dot.
180        """
181        return os.path.splitext(self.path)[1]
182
183    @property
184    def kind(self):
185        """
186        File extension without dot prefix.
187        """
188        return self.extension.lstrip(".")
189
190    @property
191    def size(self):
192        """
193        Size of this file.
194        """
195
196        if not self.exists:
197            return -1
198        return os.path.getsize(self.path)
199
200    @property
201    def mimetype(self):
202        """
203        Gets the mimetype of this file.
204        """
205        (mime, _) = mimetypes.guess_type(self.path)
206        return mime
207
208    @property
209    def is_binary(self):
210        """Return true if this is a binary file."""
211        with open(self.path, 'rb') as fin:
212            CHUNKSIZE = 1024
213            while 1:
214                chunk = fin.read(CHUNKSIZE)
215                if '\0' in chunk:
216                    return True
217                if len(chunk) < CHUNKSIZE:
218                    break
219        return False
220
221    @property
222    def is_text(self):
223        """Return true if this is a text file."""
224        return (not self.is_binary)
225
226    @property
227    def is_image(self):
228        """Return true if this is an image file."""
229        return self.mimetype.split("/")[0] == "image"
230
231
232    @property
233    def last_modified(self):
234        """
235        Returns a datetime object representing the last modified time.
236        Calls os.path.getmtime.
237
238        """
239        return datetime.fromtimestamp(os.path.getmtime(self.path))
240
241    def has_changed_since(self, basetime):
242        """
243        Returns True if the file has been changed since the given time.
244
245        """
246        return self.last_modified > basetime
247
248    def older_than(self, another_file):
249        """
250        Checks if this file is older than the given file. Uses last_modified to
251        determine age.
252
253        """
254        return self.last_modified < File(unicode(another_file)).last_modified
255
256    @staticmethod
257    def make_temp(text):
258        """
259        Creates a temprorary file and writes the `text` into it
260        """
261        import tempfile
262        (handle, path) = tempfile.mkstemp(text=True)
263        os.close(handle)
264        afile = File(path)
265        afile.write(text)
266        return afile
267
268    def read_all(self, encoding='utf-8'):
269        """
270        Reads from the file and returns the content as a string.
271        """
272        logger.info("Reading everything from %s" % self)
273        with codecs.open(self.path, 'r', encoding) as fin:
274            read_text = fin.read()
275        return read_text
276
277    def write(self, text, encoding="utf-8"):
278        """
279        Writes the given text to the file using the given encoding.
280        """
281        logger.info("Writing to %s" % self)
282        with codecs.open(self.path, 'w', encoding) as fout:
283            fout.write(text)
284
285    def copy_to(self, destination):
286        """
287        Copies the file to the given destination. Returns a File
288        object that represents the target file. `destination` must
289        be a File or Folder object.
290        """
291        target = self.__get_destination__(destination)
292        logger.info("Copying %s to %s" % (self, target))
293        shutil.copy(self.path, unicode(destination))
294        return target
295
296    def delete(self):
297        """
298        Delete the file if it exists.
299        """
300        if self.exists:
301            os.remove(self.path)
302
303
304class FSVisitor(object):
305    """
306    Implements syntactic sugar for walking and listing folders
307    """
308
309    def __init__(self, folder, pattern=None):
310        super(FSVisitor, self).__init__()
311        self.folder = folder
312        self.pattern = pattern
313
314    def folder_visitor(self, function):
315        """
316        Decorator for `visit_folder` protocol
317        """
318        self.visit_folder = function
319        return function
320
321    def file_visitor(self, function):
322        """
323        Decorator for `visit_file` protocol
324        """
325        self.visit_file = function
326        return function
327
328    def finalizer(self, function):
329        """
330        Decorator for `visit_complete` protocol
331        """
332        self.visit_complete = function
333        return function
334
335    def __enter__(self):
336        return self
337
338    def __exit__(self, exc_type, exc_val, exc_tb):
339        pass
340
341
342class FolderWalker(FSVisitor):
343    """
344    Walks the entire hirearchy of this directory starting with itself.
345
346    If a pattern is provided, only the files that match the pattern are
347    processed.
348    """
349
350    def walk(self, walk_folders=False, walk_files=False):
351        """
352        A simple generator that yields a File or Folder object based on
353        the arguments.
354        """
355
356        if not walk_files and not walk_folders:
357            return
358
359        for root, _, a_files in os.walk(self.folder.path, followlinks=True):
360            folder = Folder(root)
361            if walk_folders:
362                yield folder
363            if walk_files:
364                for a_file in a_files:
365                    if (not self.pattern or
366                        fnmatch.fnmatch(a_file, self.pattern)):
367                        yield File(folder.child(a_file))
368
369    def walk_all(self):
370        """
371        Yield both Files and Folders as the tree is walked.
372        """
373
374        return self.walk(walk_folders=True, walk_files=True)
375
376    def walk_files(self):
377        """
378        Yield only Files.
379        """
380        return self.walk(walk_folders=False, walk_files=True)
381
382    def walk_folders(self):
383        """
384        Yield only Folders.
385        """
386        return self.walk(walk_folders=True, walk_files=False)
387
388    def __exit__(self, exc_type, exc_val, exc_tb):
389        """
390        Automatically walk the folder when the context manager is exited.
391
392        Calls self.visit_folder first and then calls self.visit_file for
393        any files found. After all files and folders have been exhausted
394        self.visit_complete is called.
395
396        If visitor.visit_folder returns False, the files in the folder are not
397        processed.
398        """
399
400        def __visit_folder__(folder):
401            process_folder = True
402            if hasattr(self, 'visit_folder'):
403                process_folder = self.visit_folder(folder)
404                # If there is no return value assume true
405                #
406                if process_folder is None:
407                    process_folder = True
408            return process_folder
409
410        def __visit_file__(a_file):
411            if hasattr(self, 'visit_file'):
412                self.visit_file(a_file)
413
414        def __visit_complete__():
415            if hasattr(self, 'visit_complete'):
416                self.visit_complete()
417
418        for root, dirs, a_files in os.walk(self.folder.path, followlinks=True):
419            folder = Folder(root)
420            if not __visit_folder__(folder):
421                dirs[:] = []
422                continue
423            for a_file in a_files:
424                if not self.pattern or fnmatch.fnmatch(a_file, self.pattern):
425                    __visit_file__(File(folder.child(a_file)))
426        __visit_complete__()
427
428
429class FolderLister(FSVisitor):
430    """
431    Lists the contents of this directory.
432
433    If a pattern is provided, only the files that match the pattern are
434    processed.
435    """
436
437    def list(self, list_folders=False, list_files=False):
438        """
439        A simple generator that yields a File or Folder object based on
440        the arguments.
441        """
442
443        a_files = os.listdir(self.folder.path)
444        for a_file in a_files:
445            path = self.folder.child(a_file)
446            if os.path.isdir(path):
447                if list_folders:
448                    yield Folder(path)
449            elif list_files:
450                if not self.pattern or fnmatch.fnmatch(a_file, self.pattern):
451                    yield File(path)
452
453    def list_all(self):
454        """
455        Yield both Files and Folders as the folder is listed.
456        """
457
458        return self.list(list_folders=True, list_files=True)
459
460    def list_files(self):
461        """
462        Yield only Files.
463        """
464        return self.list(list_folders=False, list_files=True)
465
466    def list_folders(self):
467        """
468        Yield only Folders.
469        """
470        return self.list(list_folders=True, list_files=False)
471
472    def __exit__(self, exc_type, exc_val, exc_tb):
473        """
474        Automatically list the folder contents when the context manager
475        is exited.
476
477        Calls self.visit_folder first and then calls self.visit_file for
478        any files found. After all files and folders have been exhausted
479        self.visit_complete is called.
480        """
481
482        a_files = os.listdir(self.folder.path)
483        for a_file in a_files:
484            path = self.folder.child(a_file)
485            if os.path.isdir(path) and hasattr(self, 'visit_folder'):
486                self.visit_folder(Folder(path))
487            elif hasattr(self, 'visit_file'):
488                if not self.pattern or fnmatch.fnmatch(a_file, self.pattern):
489                    self.visit_file(File(path))
490        if hasattr(self, 'visit_complete'):
491            self.visit_complete()
492
493
494class Folder(FS):
495    """
496    Represents a directory.
497    """
498
499    def __init__(self, path):
500        super(Folder, self).__init__(path)
501
502    def child_folder(self, fragment):
503        """
504        Returns a folder object by combining the fragment to this folder's path
505        """
506        return Folder(os.path.join(self.path, Folder(fragment).path))
507
508    def child(self, fragment):
509        """
510        Returns a path of a child item represented by `fragment`.
511        """
512        return os.path.join(self.path, FS(fragment).path)
513
514    def make(self):
515        """
516        Creates this directory and any of the missing directories in the path.
517        Any errors that may occur are eaten.
518        """
519        try:
520            if not self.exists:
521                logger.info("Creating %s" % self.path)
522                os.makedirs(self.path)
523        except os.error:
524            pass
525        return self
526
527    def delete(self):
528        """
529        Deletes the directory if it exists.
530        """
531        if self.exists:
532            logger.info("Deleting %s" % self.path)
533            shutil.rmtree(self.path)
534
535    def copy_to(self, destination):
536        """
537        Copies this directory to the given destination. Returns a Folder object
538        that represents the moved directory.
539        """
540        target = self.__get_destination__(destination)
541        logger.info("Copying %s to %s" % (self, target))
542        shutil.copytree(self.path, unicode(target))
543        return target
544
545    def move_to(self, destination):
546        """
547        Moves this directory to the given destination. Returns a Folder object
548        that represents the moved directory.
549        """
550        target = self.__get_destination__(destination)
551        logger.info("Move %s to %s" % (self, target))
552        shutil.move(self.path, unicode(target))
553        return target
554
555    def rename_to(self, destination_name):
556        """
557        Moves this directory to the given destination. Returns a Folder object
558        that represents the moved directory.
559        """
560        target = self.parent.child_folder(destination_name)
561        logger.info("Rename %s to %s" % (self, target))
562        shutil.move(self.path, unicode(target))
563        return target
564
565    def _create_target_tree(self, target):
566        """
567        There is a bug in dir_util that makes `copy_tree` crash if a folder in
568        the tree has been deleted before and readded now. To workaround the
569        bug, we first walk the tree and create directories that are needed.
570        """
571        source = self
572        with source.walker as walker:
573
574            @walker.folder_visitor
575            def visit_folder(folder):
576                """
577                Create the mirror directory
578                """
579                if folder != source:
580                    Folder(folder.get_mirror(target, source)).make()
581
582    def copy_contents_to(self, destination):
583        """
584        Copies the contents of this directory to the given destination.
585        Returns a Folder object that represents the moved directory.
586        """
587        logger.info("Copying contents of %s to %s" % (self, destination))
588        target = Folder(destination)
589        target.make()
590        self._create_target_tree(target)
591        dir_util.copy_tree(self.path, unicode(target))
592        return target
593
594    def get_walker(self, pattern=None):
595        """
596        Return a `FolderWalker` object with a set pattern.
597        """
598        return FolderWalker(self, pattern)
599
600    @property
601    def walker(self):
602        """
603        Return a `FolderWalker` object
604        """
605        return FolderWalker(self)
606
607    def get_lister(self, pattern=None):
608        """
609        Return a `FolderLister` object with a set pattern.
610        """
611        return FolderLister(self, pattern)
612
613    @property
614    def lister(self):
615        """
616        Return a `FolderLister` object
617        """
618        return FolderLister(self)