/Lib/modulefinder.py

  1"""Find modules used by a script, using introspection."""
  2
  3from __future__ import generators
  4import dis
  5import imp
  6import marshal
  7import os
  8import sys
  9import types
 10import struct
 11
 12if hasattr(sys.__stdout__, "newlines"):
 13    READ_MODE = "U"  # universal line endings
 14else:
 15    # remain compatible with Python  < 2.3
 16    READ_MODE = "r"
 17
 18IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME'))
 19LOAD_GLOBAL = chr(dis.opname.index('LOAD_GLOBAL'))
 20LOAD_CONST = chr(dis.opname.index('LOAD_CONST'))
 21STORE_NAME = chr(dis.opname.index('STORE_NAME'))
 22STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL'))
 23STORE_OPS = [STORE_NAME, STORE_GLOBAL]
 24HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT)
 25
 26# Modulefinder does a good job at simulating Python's, but it can not
 27# handle __path__ modifications packages make at runtime.  Therefore there
 28# is a mechanism whereby you can register extra paths in this map for a
 29# package, and it will be honored.
 30
 31# Note this is a mapping is lists of paths.
 32packagePathMap = {}
 33
 34# A Public interface
 35def AddPackagePath(packagename, path):
 36    paths = packagePathMap.get(packagename, [])
 37    paths.append(path)
 38    packagePathMap[packagename] = paths
 39
 40replacePackageMap = {}
 41
 42# This ReplacePackage mechanism allows modulefinder to work around the
 43# way the _xmlplus package injects itself under the name "xml" into
 44# sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml")
 45# before running ModuleFinder.
 46
 47def ReplacePackage(oldname, newname):
 48    replacePackageMap[oldname] = newname
 49
 50
 51class Module:
 52
 53    def __init__(self, name, file=None, path=None):
 54        self.__name__ = name
 55        self.__file__ = file
 56        self.__path__ = path
 57        self.__code__ = None
 58        # The set of global names that are assigned to in the module.
 59        # This includes those names imported through starimports of
 60        # Python modules.
 61        self.globalnames = {}
 62        # The set of starimports this module did that could not be
 63        # resolved, ie. a starimport from a non-Python module.
 64        self.starimports = {}
 65
 66    def __repr__(self):
 67        s = "Module(%r" % (self.__name__,)
 68        if self.__file__ is not None:
 69            s = s + ", %r" % (self.__file__,)
 70        if self.__path__ is not None:
 71            s = s + ", %r" % (self.__path__,)
 72        s = s + ")"
 73        return s
 74
 75class ModuleFinder:
 76
 77    def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]):
 78        if path is None:
 79            path = sys.path
 80        self.path = path
 81        self.modules = {}
 82        self.badmodules = {}
 83        self.debug = debug
 84        self.indent = 0
 85        self.excludes = excludes
 86        self.replace_paths = replace_paths
 87        self.processed_paths = []   # Used in debugging only
 88
 89    def msg(self, level, str, *args):
 90        if level <= self.debug:
 91            for i in range(self.indent):
 92                print "   ",
 93            print str,
 94            for arg in args:
 95                print repr(arg),
 96            print
 97
 98    def msgin(self, *args):
 99        level = args[0]
100        if level <= self.debug:
101            self.indent = self.indent + 1
102            self.msg(*args)
103
104    def msgout(self, *args):
105        level = args[0]
106        if level <= self.debug:
107            self.indent = self.indent - 1
108            self.msg(*args)
109
110    def run_script(self, pathname):
111        self.msg(2, "run_script", pathname)
112        fp = open(pathname, READ_MODE)
113        stuff = ("", "r", imp.PY_SOURCE)
114        self.load_module('__main__', fp, pathname, stuff)
115
116    def load_file(self, pathname):
117        dir, name = os.path.split(pathname)
118        name, ext = os.path.splitext(name)
119        fp = open(pathname, READ_MODE)
120        stuff = (ext, "r", imp.PY_SOURCE)
121        self.load_module(name, fp, pathname, stuff)
122
123    def import_hook(self, name, caller=None, fromlist=None, level=-1):
124        self.msg(3, "import_hook", name, caller, fromlist, level)
125        parent = self.determine_parent(caller, level=level)
126        q, tail = self.find_head_package(parent, name)
127        m = self.load_tail(q, tail)
128        if not fromlist:
129            return q
130        if m.__path__:
131            self.ensure_fromlist(m, fromlist)
132        return None
133
134    def determine_parent(self, caller, level=-1):
135        self.msgin(4, "determine_parent", caller, level)
136        if not caller or level == 0:
137            self.msgout(4, "determine_parent -> None")
138            return None
139        pname = caller.__name__
140        if level >= 1: # relative import
141            if caller.__path__:
142                level -= 1
143            if level == 0:
144                parent = self.modules[pname]
145                assert parent is caller
146                self.msgout(4, "determine_parent ->", parent)
147                return parent
148            if pname.count(".") < level:
149                raise ImportError, "relative importpath too deep"
150            pname = ".".join(pname.split(".")[:-level])
151            parent = self.modules[pname]
152            self.msgout(4, "determine_parent ->", parent)
153            return parent
154        if caller.__path__:
155            parent = self.modules[pname]
156            assert caller is parent
157            self.msgout(4, "determine_parent ->", parent)
158            return parent
159        if '.' in pname:
160            i = pname.rfind('.')
161            pname = pname[:i]
162            parent = self.modules[pname]
163            assert parent.__name__ == pname
164            self.msgout(4, "determine_parent ->", parent)
165            return parent
166        self.msgout(4, "determine_parent -> None")
167        return None
168
169    def find_head_package(self, parent, name):
170        self.msgin(4, "find_head_package", parent, name)
171        if '.' in name:
172            i = name.find('.')
173            head = name[:i]
174            tail = name[i+1:]
175        else:
176            head = name
177            tail = ""
178        if parent:
179            qname = "%s.%s" % (parent.__name__, head)
180        else:
181            qname = head
182        q = self.import_module(head, qname, parent)
183        if q:
184            self.msgout(4, "find_head_package ->", (q, tail))
185            return q, tail
186        if parent:
187            qname = head
188            parent = None
189            q = self.import_module(head, qname, parent)
190            if q:
191                self.msgout(4, "find_head_package ->", (q, tail))
192                return q, tail
193        self.msgout(4, "raise ImportError: No module named", qname)
194        raise ImportError, "No module named " + qname
195
196    def load_tail(self, q, tail):
197        self.msgin(4, "load_tail", q, tail)
198        m = q
199        while tail:
200            i = tail.find('.')
201            if i < 0: i = len(tail)
202            head, tail = tail[:i], tail[i+1:]
203            mname = "%s.%s" % (m.__name__, head)
204            m = self.import_module(head, mname, m)
205            if not m:
206                self.msgout(4, "raise ImportError: No module named", mname)
207                raise ImportError, "No module named " + mname
208        self.msgout(4, "load_tail ->", m)
209        return m
210
211    def ensure_fromlist(self, m, fromlist, recursive=0):
212        self.msg(4, "ensure_fromlist", m, fromlist, recursive)
213        for sub in fromlist:
214            if sub == "*":
215                if not recursive:
216                    all = self.find_all_submodules(m)
217                    if all:
218                        self.ensure_fromlist(m, all, 1)
219            elif not hasattr(m, sub):
220                subname = "%s.%s" % (m.__name__, sub)
221                submod = self.import_module(sub, subname, m)
222                if not submod:
223                    raise ImportError, "No module named " + subname
224
225    def find_all_submodules(self, m):
226        if not m.__path__:
227            return
228        modules = {}
229        # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"].
230        # But we must also collect Python extension modules - although
231        # we cannot separate normal dlls from Python extensions.
232        suffixes = []
233        for triple in imp.get_suffixes():
234            suffixes.append(triple[0])
235        for dir in m.__path__:
236            try:
237                names = os.listdir(dir)
238            except os.error:
239                self.msg(2, "can't list directory", dir)
240                continue
241            for name in names:
242                mod = None
243                for suff in suffixes:
244                    n = len(suff)
245                    if name[-n:] == suff:
246                        mod = name[:-n]
247                        break
248                if mod and mod != "__init__":
249                    modules[mod] = mod
250        return modules.keys()
251
252    def import_module(self, partname, fqname, parent):
253        self.msgin(3, "import_module", partname, fqname, parent)
254        try:
255            m = self.modules[fqname]
256        except KeyError:
257            pass
258        else:
259            self.msgout(3, "import_module ->", m)
260            return m
261        if fqname in self.badmodules:
262            self.msgout(3, "import_module -> None")
263            return None
264        if parent and parent.__path__ is None:
265            self.msgout(3, "import_module -> None")
266            return None
267        try:
268            fp, pathname, stuff = self.find_module(partname,
269                                                   parent and parent.__path__, parent)
270        except ImportError:
271            self.msgout(3, "import_module ->", None)
272            return None
273        try:
274            m = self.load_module(fqname, fp, pathname, stuff)
275        finally:
276            if fp: fp.close()
277        if parent:
278            setattr(parent, partname, m)
279        self.msgout(3, "import_module ->", m)
280        return m
281
282    def load_module(self, fqname, fp, pathname, file_info):
283        suffix, mode, type = file_info
284        self.msgin(2, "load_module", fqname, fp and "fp", pathname)
285        if type == imp.PKG_DIRECTORY:
286            m = self.load_package(fqname, pathname)
287            self.msgout(2, "load_module ->", m)
288            return m
289        if type == imp.PY_SOURCE:
290            co = compile(fp.read()+'\n', pathname, 'exec')
291        elif type == imp.PY_COMPILED:
292            if fp.read(4) != imp.get_magic():
293                self.msgout(2, "raise ImportError: Bad magic number", pathname)
294                raise ImportError, "Bad magic number in %s" % pathname
295            fp.read(4)
296            co = marshal.load(fp)
297        else:
298            co = None
299        m = self.add_module(fqname)
300        m.__file__ = pathname
301        if co:
302            if self.replace_paths:
303                co = self.replace_paths_in_code(co)
304            m.__code__ = co
305            self.scan_code(co, m)
306        self.msgout(2, "load_module ->", m)
307        return m
308
309    def _add_badmodule(self, name, caller):
310        if name not in self.badmodules:
311            self.badmodules[name] = {}
312        if caller:
313            self.badmodules[name][caller.__name__] = 1
314        else:
315            self.badmodules[name]["-"] = 1
316
317    def _safe_import_hook(self, name, caller, fromlist, level=-1):
318        # wrapper for self.import_hook() that won't raise ImportError
319        if name in self.badmodules:
320            self._add_badmodule(name, caller)
321            return
322        try:
323            self.import_hook(name, caller, level=level)
324        except ImportError, msg:
325            self.msg(2, "ImportError:", str(msg))
326            self._add_badmodule(name, caller)
327        else:
328            if fromlist:
329                for sub in fromlist:
330                    if sub in self.badmodules:
331                        self._add_badmodule(sub, caller)
332                        continue
333                    try:
334                        self.import_hook(name, caller, [sub], level=level)
335                    except ImportError, msg:
336                        self.msg(2, "ImportError:", str(msg))
337                        fullname = name + "." + sub
338                        self._add_badmodule(fullname, caller)
339
340    def scan_opcodes(self, co, unpack=struct.unpack):
341        # Scan the code, and yield 'interesting' opcode combinations.
342        # This supports the absolute and relative imports introduced in
343        # Python 2.5.
344        code = co.co_code
345        names = co.co_names
346        consts = co.co_consts
347        OPCODE_SIG = LOAD_CONST + LOAD_CONST + LOAD_CONST + IMPORT_NAME
348        while code:
349            c = code[0]
350            if c in STORE_OPS:
351                oparg, = unpack('<H', code[1:3])
352                yield "store", (names[oparg],)
353                code = code[3:]
354                continue
355            if code[:12:3] == OPCODE_SIG:
356                oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9])
357                level = consts[oparg_1]
358                if level == -1: # normal import
359                    yield "import", (consts[oparg_2], consts[oparg_3])
360                elif level == 0: # absolute import
361                    yield "absolute_import", (consts[oparg_2],
362                                              consts[oparg_3])
363                else: # relative import
364                    yield "relative_import", (level,
365                                              consts[oparg_2],
366                                              consts[oparg_3])
367                code = code[9:]
368                continue
369            if c >= HAVE_ARGUMENT:
370                code = code[3:]
371            else:
372                code = code[1:]
373
374    def scan_code(self, co, m):
375        code = co.co_code
376        for what, args in self.scan_opcodes(co):
377            if what == "store":
378                name, = args
379                m.globalnames[name] = 1
380            elif what in ("import", "absolute_import"):
381                fromlist, name = args
382                have_star = 0
383                if fromlist is not None:
384                    if "*" in fromlist:
385                        have_star = 1
386                    fromlist = [f for f in fromlist if f != "*"]
387                if what == "absolute_import": level = 0
388                else: level = -1
389                self._safe_import_hook(name, m, fromlist, level=level)
390                if have_star:
391                    # We've encountered an "import *". If it is a Python
392                    # module, the code has already been parsed and we can suck
393                    # out the global names.
394                    mm = None
395                    if m.__path__:
396                        # At this point we don't know whether 'name' is a
397                        # submodule of 'm' or a global module. Let's just try
398                        # the full name first.
399                        mm = self.modules.get(m.__name__ + "." + name)
400                    if mm is None:
401                        mm = self.modules.get(name)
402                    if mm is not None:
403                        m.globalnames.update(mm.globalnames)
404                        m.starimports.update(mm.starimports)
405                        if mm.__code__ is None:
406                            m.starimports[name] = 1
407                    else:
408                        m.starimports[name] = 1
409            elif what == "relative_import":
410                level, fromlist, name = args
411                if name:
412                    self._safe_import_hook(name, m, fromlist, level=level)
413                else:
414                    parent = self.determine_parent(m, level=level)
415                    self._safe_import_hook(parent.__name__, None, fromlist, level=0)
416            else:
417                # We don't expect anything else from the generator.
418                raise RuntimeError(what)
419
420        for c in co.co_consts:
421            if isinstance(c, type(co)):
422                self.scan_code(c, m)
423
424    def load_package(self, fqname, pathname):
425        self.msgin(2, "load_package", fqname, pathname)
426        newname = replacePackageMap.get(fqname)
427        if newname:
428            fqname = newname
429        m = self.add_module(fqname)
430        m.__file__ = pathname
431        m.__path__ = [pathname]
432
433        # As per comment at top of file, simulate runtime __path__ additions.
434        m.__path__ = m.__path__ + packagePathMap.get(fqname, [])
435
436        fp, buf, stuff = self.find_module("__init__", m.__path__)
437        self.load_module(fqname, fp, buf, stuff)
438        self.msgout(2, "load_package ->", m)
439        return m
440
441    def add_module(self, fqname):
442        if fqname in self.modules:
443            return self.modules[fqname]
444        self.modules[fqname] = m = Module(fqname)
445        return m
446
447    def find_module(self, name, path, parent=None):
448        if parent is not None:
449            # assert path is not None
450            fullname = parent.__name__+'.'+name
451        else:
452            fullname = name
453        if fullname in self.excludes:
454            self.msgout(3, "find_module -> Excluded", fullname)
455            raise ImportError, name
456
457        if path is None:
458            if name in sys.builtin_module_names:
459                return (None, None, ("", "", imp.C_BUILTIN))
460
461            path = self.path
462        return imp.find_module(name, path)
463
464    def report(self):
465        """Print a report to stdout, listing the found modules with their
466        paths, as well as modules that are missing, or seem to be missing.
467        """
468        print
469        print "  %-25s %s" % ("Name", "File")
470        print "  %-25s %s" % ("----", "----")
471        # Print modules found
472        keys = self.modules.keys()
473        keys.sort()
474        for key in keys:
475            m = self.modules[key]
476            if m.__path__:
477                print "P",
478            else:
479                print "m",
480            print "%-25s" % key, m.__file__ or ""
481
482        # Print missing modules
483        missing, maybe = self.any_missing_maybe()
484        if missing:
485            print
486            print "Missing modules:"
487            for name in missing:
488                mods = self.badmodules[name].keys()
489                mods.sort()
490                print "?", name, "imported from", ', '.join(mods)
491        # Print modules that may be missing, but then again, maybe not...
492        if maybe:
493            print
494            print "Submodules thay appear to be missing, but could also be",
495            print "global names in the parent package:"
496            for name in maybe:
497                mods = self.badmodules[name].keys()
498                mods.sort()
499                print "?", name, "imported from", ', '.join(mods)
500
501    def any_missing(self):
502        """Return a list of modules that appear to be missing. Use
503        any_missing_maybe() if you want to know which modules are
504        certain to be missing, and which *may* be missing.
505        """
506        missing, maybe = self.any_missing_maybe()
507        return missing + maybe
508
509    def any_missing_maybe(self):
510        """Return two lists, one with modules that are certainly missing
511        and one with modules that *may* be missing. The latter names could
512        either be submodules *or* just global names in the package.
513
514        The reason it can't always be determined is that it's impossible to
515        tell which names are imported when "from module import *" is done
516        with an extension module, short of actually importing it.
517        """
518        missing = []
519        maybe = []
520        for name in self.badmodules:
521            if name in self.excludes:
522                continue
523            i = name.rfind(".")
524            if i < 0:
525                missing.append(name)
526                continue
527            subname = name[i+1:]
528            pkgname = name[:i]
529            pkg = self.modules.get(pkgname)
530            if pkg is not None:
531                if pkgname in self.badmodules[name]:
532                    # The package tried to import this module itself and
533                    # failed. It's definitely missing.
534                    missing.append(name)
535                elif subname in pkg.globalnames:
536                    # It's a global in the package: definitely not missing.
537                    pass
538                elif pkg.starimports:
539                    # It could be missing, but the package did an "import *"
540                    # from a non-Python module, so we simply can't be sure.
541                    maybe.append(name)
542                else:
543                    # It's not a global in the package, the package didn't
544                    # do funny star imports, it's very likely to be missing.
545                    # The symbol could be inserted into the package from the
546                    # outside, but since that's not good style we simply list
547                    # it missing.
548                    missing.append(name)
549            else:
550                missing.append(name)
551        missing.sort()
552        maybe.sort()
553        return missing, maybe
554
555    def replace_paths_in_code(self, co):
556        new_filename = original_filename = os.path.normpath(co.co_filename)
557        for f, r in self.replace_paths:
558            if original_filename.startswith(f):
559                new_filename = r + original_filename[len(f):]
560                break
561
562        if self.debug and original_filename not in self.processed_paths:
563            if new_filename != original_filename:
564                self.msgout(2, "co_filename %r changed to %r" \
565                                    % (original_filename,new_filename,))
566            else:
567                self.msgout(2, "co_filename %r remains unchanged" \
568                                    % (original_filename,))
569            self.processed_paths.append(original_filename)
570
571        consts = list(co.co_consts)
572        for i in range(len(consts)):
573            if isinstance(consts[i], type(co)):
574                consts[i] = self.replace_paths_in_code(consts[i])
575
576        return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize,
577                         co.co_flags, co.co_code, tuple(consts), co.co_names,
578                         co.co_varnames, new_filename, co.co_name,
579                         co.co_firstlineno, co.co_lnotab,
580                         co.co_freevars, co.co_cellvars)
581
582
583def test():
584    # Parse command line
585    import getopt
586    try:
587        opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:")
588    except getopt.error, msg:
589        print msg
590        return
591
592    # Process options
593    debug = 1
594    domods = 0
595    addpath = []
596    exclude = []
597    for o, a in opts:
598        if o == '-d':
599            debug = debug + 1
600        if o == '-m':
601            domods = 1
602        if o == '-p':
603            addpath = addpath + a.split(os.pathsep)
604        if o == '-q':
605            debug = 0
606        if o == '-x':
607            exclude.append(a)
608
609    # Provide default arguments
610    if not args:
611        script = "hello.py"
612    else:
613        script = args[0]
614
615    # Set the path based on sys.path and the script directory
616    path = sys.path[:]
617    path[0] = os.path.dirname(script)
618    path = addpath + path
619    if debug > 1:
620        print "path:"
621        for item in path:
622            print "   ", repr(item)
623
624    # Create the module finder and turn its crank
625    mf = ModuleFinder(path, debug, exclude)
626    for arg in args[1:]:
627        if arg == '-m':
628            domods = 1
629            continue
630        if domods:
631            if arg[-2:] == '.*':
632                mf.import_hook(arg[:-2], None, ["*"])
633            else:
634                mf.import_hook(arg)
635        else:
636            mf.load_file(arg)
637    mf.run_script(script)
638    mf.report()
639    return mf  # for -i debugging
640
641
642if __name__ == '__main__':
643    try:
644        mf = test()
645    except KeyboardInterrupt:
646        print "\n[interrupt]"