/Lib/pkgutil.py

  1"""Utilities to support packages."""
  2
  3# NOTE: This module must remain compatible with Python 2.3, as it is shared
  4# by setuptools for distribution with Python 2.3 and up.
  5
  6import os
  7import sys
  8import imp
  9import os.path
 10from types import ModuleType
 11
 12__all__ = [
 13    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
 14    'walk_packages', 'iter_modules',
 15    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
 16]
 17
 18def read_code(stream):
 19    # This helper is needed in order for the PEP 302 emulation to
 20    # correctly handle compiled files
 21    import marshal
 22
 23    magic = stream.read(4)
 24    if magic != imp.get_magic():
 25        return None
 26
 27    stream.read(4) # Skip timestamp
 28    return marshal.load(stream)
 29
 30
 31def simplegeneric(func):
 32    """Make a trivial single-dispatch generic function"""
 33    registry = {}
 34    def wrapper(*args, **kw):
 35        ob = args[0]
 36        try:
 37            cls = ob.__class__
 38        except AttributeError:
 39            cls = type(ob)
 40        try:
 41            mro = cls.__mro__
 42        except AttributeError:
 43            try:
 44                class cls(cls, object):
 45                    pass
 46                mro = cls.__mro__[1:]
 47            except TypeError:
 48                mro = object,   # must be an ExtensionClass or some such  :(
 49        for t in mro:
 50            if t in registry:
 51                return registry[t](*args, **kw)
 52        else:
 53            return func(*args, **kw)
 54    try:
 55        wrapper.__name__ = func.__name__
 56    except (TypeError, AttributeError):
 57        pass    # Python 2.3 doesn't allow functions to be renamed
 58
 59    def register(typ, func=None):
 60        if func is None:
 61            return lambda f: register(typ, f)
 62        registry[typ] = func
 63        return func
 64
 65    wrapper.__dict__ = func.__dict__
 66    wrapper.__doc__ = func.__doc__
 67    wrapper.register = register
 68    return wrapper
 69
 70
 71def walk_packages(path=None, prefix='', onerror=None):
 72    """Yields (module_loader, name, ispkg) for all modules recursively
 73    on path, or, if path is None, all accessible modules.
 74
 75    'path' should be either None or a list of paths to look for
 76    modules in.
 77
 78    'prefix' is a string to output on the front of every module name
 79    on output.
 80
 81    Note that this function must import all *packages* (NOT all
 82    modules!) on the given path, in order to access the __path__
 83    attribute to find submodules.
 84
 85    'onerror' is a function which gets called with one argument (the
 86    name of the package which was being imported) if any exception
 87    occurs while trying to import a package.  If no onerror function is
 88    supplied, ImportErrors are caught and ignored, while all other
 89    exceptions are propagated, terminating the search.
 90
 91    Examples:
 92
 93    # list all modules python can access
 94    walk_packages()
 95
 96    # list all submodules of ctypes
 97    walk_packages(ctypes.__path__, ctypes.__name__+'.')
 98    """
 99
100    def seen(p, m={}):
101        if p in m:
102            return True
103        m[p] = True
104
105    for importer, name, ispkg in iter_modules(path, prefix):
106        yield importer, name, ispkg
107
108        if ispkg:
109            try:
110                __import__(name)
111            except ImportError:
112                if onerror is not None:
113                    onerror(name)
114            except Exception:
115                if onerror is not None:
116                    onerror(name)
117                else:
118                    raise
119            else:
120                path = getattr(sys.modules[name], '__path__', None) or []
121
122                # don't traverse path items we've seen before
123                path = [p for p in path if not seen(p)]
124
125                for item in walk_packages(path, name+'.', onerror):
126                    yield item
127
128
129def iter_modules(path=None, prefix=''):
130    """Yields (module_loader, name, ispkg) for all submodules on path,
131    or, if path is None, all top-level modules on sys.path.
132
133    'path' should be either None or a list of paths to look for
134    modules in.
135
136    'prefix' is a string to output on the front of every module name
137    on output.
138    """
139
140    if path is None:
141        importers = iter_importers()
142    else:
143        importers = map(get_importer, path)
144
145    yielded = {}
146    for i in importers:
147        for name, ispkg in iter_importer_modules(i, prefix):
148            if name not in yielded:
149                yielded[name] = 1
150                yield i, name, ispkg
151
152
153#@simplegeneric
154def iter_importer_modules(importer, prefix=''):
155    if not hasattr(importer, 'iter_modules'):
156        return []
157    return importer.iter_modules(prefix)
158
159iter_importer_modules = simplegeneric(iter_importer_modules)
160
161
162class ImpImporter:
163    """PEP 302 Importer that wraps Python's "classic" import algorithm
164
165    ImpImporter(dirname) produces a PEP 302 importer that searches that
166    directory.  ImpImporter(None) produces a PEP 302 importer that searches
167    the current sys.path, plus any modules that are frozen or built-in.
168
169    Note that ImpImporter does not currently support being used by placement
170    on sys.meta_path.
171    """
172
173    def __init__(self, path=None):
174        self.path = path
175
176    def find_module(self, fullname, path=None):
177        # Note: we ignore 'path' argument since it is only used via meta_path
178        subname = fullname.split(".")[-1]
179        if subname != fullname and self.path is None:
180            return None
181        if self.path is None:
182            path = None
183        else:
184            path = [os.path.realpath(self.path)]
185        try:
186            file, filename, etc = imp.find_module(subname, path)
187        except ImportError:
188            return None
189        return ImpLoader(fullname, file, filename, etc)
190
191    def iter_modules(self, prefix=''):
192        if self.path is None or not os.path.isdir(self.path):
193            return
194
195        yielded = {}
196        import inspect
197
198        filenames = os.listdir(self.path)
199        filenames.sort()  # handle packages before same-named modules
200
201        for fn in filenames:
202            modname = inspect.getmodulename(fn)
203            if modname=='__init__' or modname in yielded:
204                continue
205
206            path = os.path.join(self.path, fn)
207            ispkg = False
208
209            if not modname and os.path.isdir(path) and '.' not in fn:
210                modname = fn
211                for fn in os.listdir(path):
212                    subname = inspect.getmodulename(fn)
213                    if subname=='__init__':
214                        ispkg = True
215                        break
216                else:
217                    continue    # not a package
218
219            if modname and '.' not in modname:
220                yielded[modname] = 1
221                yield prefix + modname, ispkg
222
223
224class ImpLoader:
225    """PEP 302 Loader that wraps Python's "classic" import algorithm
226    """
227    code = source = None
228
229    def __init__(self, fullname, file, filename, etc):
230        self.file = file
231        self.filename = filename
232        self.fullname = fullname
233        self.etc = etc
234
235    def load_module(self, fullname):
236        self._reopen()
237        try:
238            mod = imp.load_module(fullname, self.file, self.filename, self.etc)
239        finally:
240            if self.file:
241                self.file.close()
242        # Note: we don't set __loader__ because we want the module to look
243        # normal; i.e. this is just a wrapper for standard import machinery
244        return mod
245
246    def get_data(self, pathname):
247        return open(pathname, "rb").read()
248
249    def _reopen(self):
250        if self.file and self.file.closed:
251            mod_type = self.etc[2]
252            if mod_type==imp.PY_SOURCE:
253                self.file = open(self.filename, 'rU')
254            elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
255                self.file = open(self.filename, 'rb')
256
257    def _fix_name(self, fullname):
258        if fullname is None:
259            fullname = self.fullname
260        elif fullname != self.fullname:
261            raise ImportError("Loader for module %s cannot handle "
262                              "module %s" % (self.fullname, fullname))
263        return fullname
264
265    def is_package(self, fullname):
266        fullname = self._fix_name(fullname)
267        return self.etc[2]==imp.PKG_DIRECTORY
268
269    def get_code(self, fullname=None):
270        fullname = self._fix_name(fullname)
271        if self.code is None:
272            mod_type = self.etc[2]
273            if mod_type==imp.PY_SOURCE:
274                source = self.get_source(fullname)
275                self.code = compile(source, self.filename, 'exec')
276            elif mod_type==imp.PY_COMPILED:
277                self._reopen()
278                try:
279                    self.code = read_code(self.file)
280                finally:
281                    self.file.close()
282            elif mod_type==imp.PKG_DIRECTORY:
283                self.code = self._get_delegate().get_code()
284        return self.code
285
286    def get_source(self, fullname=None):
287        fullname = self._fix_name(fullname)
288        if self.source is None:
289            mod_type = self.etc[2]
290            if mod_type==imp.PY_SOURCE:
291                self._reopen()
292                try:
293                    self.source = self.file.read()
294                finally:
295                    self.file.close()
296            elif mod_type==imp.PY_COMPILED:
297                if os.path.exists(self.filename[:-1]):
298                    f = open(self.filename[:-1], 'rU')
299                    self.source = f.read()
300                    f.close()
301            elif mod_type==imp.PKG_DIRECTORY:
302                self.source = self._get_delegate().get_source()
303        return self.source
304
305
306    def _get_delegate(self):
307        return ImpImporter(self.filename).find_module('__init__')
308
309    def get_filename(self, fullname=None):
310        fullname = self._fix_name(fullname)
311        mod_type = self.etc[2]
312        if self.etc[2]==imp.PKG_DIRECTORY:
313            return self._get_delegate().get_filename()
314        elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
315            return self.filename
316        return None
317
318
319try:
320    import zipimport
321    from zipimport import zipimporter
322
323    def iter_zipimport_modules(importer, prefix=''):
324        dirlist = zipimport._zip_directory_cache[importer.archive].keys()
325        dirlist.sort()
326        _prefix = importer.prefix
327        plen = len(_prefix)
328        yielded = {}
329        import inspect
330        for fn in dirlist:
331            if not fn.startswith(_prefix):
332                continue
333
334            fn = fn[plen:].split(os.sep)
335
336            if len(fn)==2 and fn[1].startswith('__init__.py'):
337                if fn[0] not in yielded:
338                    yielded[fn[0]] = 1
339                    yield fn[0], True
340
341            if len(fn)!=1:
342                continue
343
344            modname = inspect.getmodulename(fn[0])
345            if modname=='__init__':
346                continue
347
348            if modname and '.' not in modname and modname not in yielded:
349                yielded[modname] = 1
350                yield prefix + modname, False
351
352    iter_importer_modules.register(zipimporter, iter_zipimport_modules)
353
354except ImportError:
355    pass
356
357
358def get_importer(path_item):
359    """Retrieve a PEP 302 importer for the given path item
360
361    The returned importer is cached in sys.path_importer_cache
362    if it was newly created by a path hook.
363
364    If there is no importer, a wrapper around the basic import
365    machinery is returned. This wrapper is never inserted into
366    the importer cache (None is inserted instead).
367
368    The cache (or part of it) can be cleared manually if a
369    rescan of sys.path_hooks is necessary.
370    """
371    try:
372        importer = sys.path_importer_cache[path_item]
373    except KeyError:
374        for path_hook in sys.path_hooks:
375            try:
376                importer = path_hook(path_item)
377                break
378            except ImportError:
379                pass
380        else:
381            importer = None
382        sys.path_importer_cache.setdefault(path_item, importer)
383
384    if importer is None:
385        try:
386            importer = ImpImporter(path_item)
387        except ImportError:
388            importer = None
389    return importer
390
391
392def iter_importers(fullname=""):
393    """Yield PEP 302 importers for the given module name
394
395    If fullname contains a '.', the importers will be for the package
396    containing fullname, otherwise they will be importers for sys.meta_path,
397    sys.path, and Python's "classic" import machinery, in that order.  If
398    the named module is in a package, that package is imported as a side
399    effect of invoking this function.
400
401    Non PEP 302 mechanisms (e.g. the Windows registry) used by the
402    standard import machinery to find files in alternative locations
403    are partially supported, but are searched AFTER sys.path. Normally,
404    these locations are searched BEFORE sys.path, preventing sys.path
405    entries from shadowing them.
406
407    For this to cause a visible difference in behaviour, there must
408    be a module or package name that is accessible via both sys.path
409    and one of the non PEP 302 file system mechanisms. In this case,
410    the emulation will find the former version, while the builtin
411    import mechanism will find the latter.
412
413    Items of the following types can be affected by this discrepancy:
414        imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
415    """
416    if fullname.startswith('.'):
417        raise ImportError("Relative module names not supported")
418    if '.' in fullname:
419        # Get the containing package's __path__
420        pkg = '.'.join(fullname.split('.')[:-1])
421        if pkg not in sys.modules:
422            __import__(pkg)
423        path = getattr(sys.modules[pkg], '__path__', None) or []
424    else:
425        for importer in sys.meta_path:
426            yield importer
427        path = sys.path
428    for item in path:
429        yield get_importer(item)
430    if '.' not in fullname:
431        yield ImpImporter()
432
433def get_loader(module_or_name):
434    """Get a PEP 302 "loader" object for module_or_name
435
436    If the module or package is accessible via the normal import
437    mechanism, a wrapper around the relevant part of that machinery
438    is returned.  Returns None if the module cannot be found or imported.
439    If the named module is not already imported, its containing package
440    (if any) is imported, in order to establish the package __path__.
441
442    This function uses iter_importers(), and is thus subject to the same
443    limitations regarding platform-specific special import locations such
444    as the Windows registry.
445    """
446    if module_or_name in sys.modules:
447        module_or_name = sys.modules[module_or_name]
448    if isinstance(module_or_name, ModuleType):
449        module = module_or_name
450        loader = getattr(module, '__loader__', None)
451        if loader is not None:
452            return loader
453        fullname = module.__name__
454    else:
455        fullname = module_or_name
456    return find_loader(fullname)
457
458def find_loader(fullname):
459    """Find a PEP 302 "loader" object for fullname
460
461    If fullname contains dots, path must be the containing package's __path__.
462    Returns None if the module cannot be found or imported. This function uses
463    iter_importers(), and is thus subject to the same limitations regarding
464    platform-specific special import locations such as the Windows registry.
465    """
466    for importer in iter_importers(fullname):
467        loader = importer.find_module(fullname)
468        if loader is not None:
469            return loader
470
471    return None
472
473
474def extend_path(path, name):
475    """Extend a package's path.
476
477    Intended use is to place the following code in a package's __init__.py:
478
479        from pkgutil import extend_path
480        __path__ = extend_path(__path__, __name__)
481
482    This will add to the package's __path__ all subdirectories of
483    directories on sys.path named after the package.  This is useful
484    if one wants to distribute different parts of a single logical
485    package as multiple directories.
486
487    It also looks for *.pkg files beginning where * matches the name
488    argument.  This feature is similar to *.pth files (see site.py),
489    except that it doesn't special-case lines starting with 'import'.
490    A *.pkg file is trusted at face value: apart from checking for
491    duplicates, all entries found in a *.pkg file are added to the
492    path, regardless of whether they are exist the filesystem.  (This
493    is a feature.)
494
495    If the input path is not a list (as is the case for frozen
496    packages) it is returned unchanged.  The input path is not
497    modified; an extended copy is returned.  Items are only appended
498    to the copy at the end.
499
500    It is assumed that sys.path is a sequence.  Items of sys.path that
501    are not (unicode or 8-bit) strings referring to existing
502    directories are ignored.  Unicode items of sys.path that cause
503    errors when used as filenames may cause this function to raise an
504    exception (in line with os.path.isdir() behavior).
505    """
506
507    if not isinstance(path, list):
508        # This could happen e.g. when this is called from inside a
509        # frozen package.  Return the path unchanged in that case.
510        return path
511
512    pname = os.path.join(*name.split('.')) # Reconstitute as relative path
513    # Just in case os.extsep != '.'
514    sname = os.extsep.join(name.split('.'))
515    sname_pkg = sname + os.extsep + "pkg"
516    init_py = "__init__" + os.extsep + "py"
517
518    path = path[:] # Start with a copy of the existing path
519
520    for dir in sys.path:
521        if not isinstance(dir, basestring) or not os.path.isdir(dir):
522            continue
523        subdir = os.path.join(dir, pname)
524        # XXX This may still add duplicate entries to path on
525        # case-insensitive filesystems
526        initfile = os.path.join(subdir, init_py)
527        if subdir not in path and os.path.isfile(initfile):
528            path.append(subdir)
529        # XXX Is this the right thing for subpackages like zope.app?
530        # It looks for a file named "zope.app.pkg"
531        pkgfile = os.path.join(dir, sname_pkg)
532        if os.path.isfile(pkgfile):
533            try:
534                f = open(pkgfile)
535            except IOError, msg:
536                sys.stderr.write("Can't open %s: %s\n" %
537                                 (pkgfile, msg))
538            else:
539                for line in f:
540                    line = line.rstrip('\n')
541                    if not line or line.startswith('#'):
542                        continue
543                    path.append(line) # Don't check for existence!
544                f.close()
545
546    return path
547
548def get_data(package, resource):
549    """Get a resource from a package.
550
551    This is a wrapper round the PEP 302 loader get_data API. The package
552    argument should be the name of a package, in standard module format
553    (foo.bar). The resource argument should be in the form of a relative
554    filename, using '/' as the path separator. The parent directory name '..'
555    is not allowed, and nor is a rooted name (starting with a '/').
556
557    The function returns a binary string, which is the contents of the
558    specified resource.
559
560    For packages located in the filesystem, which have already been imported,
561    this is the rough equivalent of
562
563        d = os.path.dirname(sys.modules[package].__file__)
564        data = open(os.path.join(d, resource), 'rb').read()
565
566    If the package cannot be located or loaded, or it uses a PEP 302 loader
567    which does not support get_data(), then None is returned.
568    """
569
570    loader = get_loader(package)
571    if loader is None or not hasattr(loader, 'get_data'):
572        return None
573    mod = sys.modules.get(package) or loader.load_module(package)
574    if mod is None or not hasattr(mod, '__file__'):
575        return None
576
577    # Modify the resource name to be compatible with the loader.get_data
578    # signature - an os.path format "filename" starting with the dirname of
579    # the package's __file__
580    parts = resource.split('/')
581    parts.insert(0, os.path.dirname(mod.__file__))
582    resource_name = os.path.join(*parts)
583    return loader.get_data(resource_name)