/pypy/interpreter/baseobjspace.py

Large files files are truncated, but you can click here to view the full file

import sys

from rpython.rlib.cache import Cache
from rpython.tool.uid import HUGEVAL_BYTES
from rpython.rlib import jit, types
from rpython.rlib.debug import make_sure_not_resized
from rpython.rlib.objectmodel import (we_are_translated, newlist_hint,
     compute_unique_id, specialize)
from rpython.rlib.signature import signature
from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
    INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX

from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
    UserDelAction)
from pypy.interpreter.error import OperationError, new_exception_class, oefmt
from pypy.interpreter.argument import Arguments
from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary


__all__ = ['ObjSpace', 'OperationError', 'W_Root']

unpackiterable_driver = jit.JitDriver(name='unpackiterable',
                                      greens=['tp'],
                                      reds=['items', 'w_iterator'])


class W_Root(object):
    """This is the abstract root class of all wrapped objects that live
    in a 'normal' object space like StdObjSpace."""
    __slots__ = ()
    _settled_ = True
    user_overridden_class = False

    def getdict(self, space):
        return None

    def getdictvalue(self, space, attr):
        w_dict = self.getdict(space)
        if w_dict is not None:
            return space.finditem_str(w_dict, attr)
        return None

    def setdictvalue(self, space, attr, w_value):
        w_dict = self.getdict(space)
        if w_dict is not None:
            space.setitem_str(w_dict, attr, w_value)
            return True
        return False

    def deldictvalue(self, space, attr):
        w_dict = self.getdict(space)
        if w_dict is not None:
            try:
                space.delitem(w_dict, space.wrap(attr))
                return True
            except OperationError, ex:
                if not ex.match(space, space.w_KeyError):
                    raise
        return False

    def setdict(self, space, w_dict):
        raise oefmt(space.w_TypeError,
                     "attribute '__dict__' of %T objects is not writable",
                     self)

    # to be used directly only by space.type implementations
    def getclass(self, space):
        return space.gettypeobject(self.typedef)

    def setclass(self, space, w_subtype):
        raise OperationError(space.w_TypeError,
                             space.wrap("__class__ assignment: only for heap types"))

    def user_setup(self, space, w_subtype):
        raise NotImplementedError("only for interp-level user subclasses "
                                  "from typedef.py")

    def getname(self, space):
        try:
            return space.str_w(space.getattr(self, space.wrap('__name__')))
        except OperationError, e:
            if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                return '?'
            raise

    def getaddrstring(self, space):
        # slowish
        w_id = space.id(self)
        w_4 = space.wrap(4)
        w_0x0F = space.wrap(0x0F)
        i = 2 * HUGEVAL_BYTES
        addrstring = [' '] * i
        while True:
            n = space.int_w(space.and_(w_id, w_0x0F), allow_conversion=False)
            n += ord('0')
            if n > ord('9'):
                n += (ord('a') - ord('9') - 1)
            i -= 1
            addrstring[i] = chr(n)
            if i == 0:
                break
            w_id = space.rshift(w_id, w_4)
        return ''.join(addrstring)

    def getrepr(self, space, info, moreinfo=''):
        addrstring = self.getaddrstring(space)
        return space.wrap("<%s at 0x%s%s>" % (info, addrstring,
                                              moreinfo))

    def getslotvalue(self, index):
        raise NotImplementedError

    def setslotvalue(self, index, w_val):
        raise NotImplementedError

    def delslotvalue(self, index):
        raise NotImplementedError

    def descr_call_mismatch(self, space, opname, RequiredClass, args):
        if RequiredClass is None:
            classname = '?'
        else:
            classname = wrappable_class_name(RequiredClass)
        raise oefmt(space.w_TypeError,
                    "'%s' object expected, got '%T' instead", classname, self)

    # used by _weakref implemenation

    def getweakref(self):
        return None

    def setweakref(self, space, weakreflifeline):
        raise oefmt(space.w_TypeError,
                    "cannot create weak reference to '%T' object", self)

    def delweakref(self):
        pass

    def clear_all_weakrefs(self):
        """Call this at the beginning of interp-level __del__() methods
        in subclasses.  It ensures that weakrefs (if any) are cleared
        before the object is further destroyed.
        """
        lifeline = self.getweakref()
        if lifeline is not None:
            # Clear all weakrefs to this object before we proceed with
            # the destruction of the object.  We detach the lifeline
            # first: if the code following before_del() calls the
            # app-level, e.g. a user-defined __del__(), and this code
            # tries to use weakrefs again, it won't reuse the broken
            # (already-cleared) weakrefs from this lifeline.
            self.delweakref()
            lifeline.clear_all_weakrefs()

    __already_enqueued_for_destruction = ()

    def enqueue_for_destruction(self, space, callback, descrname):
        """Put the object in the destructor queue of the space.
        At a later, safe point in time, UserDelAction will call
        callback(self).  If that raises OperationError, prints it
        to stderr with the descrname string.

        Note that 'callback' will usually need to start with:
            assert isinstance(self, W_SpecificClass)
        """
        # this function always resurect the object, so when
        # running on top of CPython we must manually ensure that
        # we enqueue it only once
        if not we_are_translated():
            if callback in self.__already_enqueued_for_destruction:
                return
            self.__already_enqueued_for_destruction += (callback,)
        space.user_del_action.register_callback(self, callback, descrname)

    # hooks that the mapdict implementations needs:
    def _get_mapdict_map(self):
        return None
    def _set_mapdict_map(self, map):
        raise NotImplementedError
    def _mapdict_read_storage(self, index):
        raise NotImplementedError
    def _mapdict_write_storage(self, index, value):
        raise NotImplementedError
    def _mapdict_storage_length(self):
        raise NotImplementedError
    def _set_mapdict_storage_and_map(self, storage, map):
        raise NotImplementedError

    # -------------------------------------------------------------------

    def is_w(self, space, w_other):
        return self is w_other

    def immutable_unique_id(self, space):
        return None

    def buffer_w(self, space, flags):
        w_impl = space.lookup(self, '__buffer__')
        if w_impl is not None:
            w_result = space.get_and_call_function(w_impl, self)
            if space.isinstance_w(w_result, space.w_buffer):
                return w_result.buffer_w(space, flags)
        raise BufferInterfaceNotFound

    def readbuf_w(self, space):
        w_impl = space.lookup(self, '__buffer__')
        if w_impl is not None:
            w_result = space.get_and_call_function(w_impl, self)
            if space.isinstance_w(w_result, space.w_buffer):
                return w_result.readbuf_w(space)
        raise BufferInterfaceNotFound

    def writebuf_w(self, space):
        w_impl = space.lookup(self, '__buffer__')
        if w_impl is not None:
            w_result = space.get_and_call_function(w_impl, self)
            if space.isinstance_w(w_result, space.w_buffer):
                return w_result.writebuf_w(space)
        raise BufferInterfaceNotFound

    def charbuf_w(self, space):
        w_impl = space.lookup(self, '__buffer__')
        if w_impl is not None:
            w_result = space.get_and_call_function(w_impl, self)
            if space.isinstance_w(w_result, space.w_buffer):
                return w_result.charbuf_w(space)
        raise BufferInterfaceNotFound

    def str_w(self, space):
        self._typed_unwrap_error(space, "string")

    def unicode_w(self, space):
        self._typed_unwrap_error(space, "unicode")

    def int_w(self, space, allow_conversion=True):
        # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w
        # raises w_TypeError
        w_obj = self
        if allow_conversion:
            w_obj = space.int(self)
        return w_obj._int_w(space)

    def _int_w(self, space):
        self._typed_unwrap_error(space, "integer")

    def float_w(self, space, allow_conversion=True):
        w_obj = self
        if allow_conversion:
            w_obj = space.float(self)
        return w_obj._float_w(space)

    def _float_w(self, space):
        self._typed_unwrap_error(space, "float")

    def uint_w(self, space):
        self._typed_unwrap_error(space, "integer")

    def bigint_w(self, space, allow_conversion=True):
        # note that W_IntObject and W_LongObject have fast paths,
        # W_FloatObject.rbigint_w raises w_TypeError raises
        w_obj = self
        if allow_conversion:
            w_obj = space.long(self)
        return w_obj._bigint_w(space)

    def _bigint_w(self, space):
        self._typed_unwrap_error(space, "integer")

    def _typed_unwrap_error(self, space, expected):
        raise oefmt(space.w_TypeError,
                    "expected %s, got %T object", expected, self)

    def int(self, space):
        w_impl = space.lookup(self, '__int__')
        if w_impl is None:
            self._typed_unwrap_error(space, "integer")
        w_result = space.get_and_call_function(w_impl, self)

        if (space.isinstance_w(w_result, space.w_int) or
            space.isinstance_w(w_result, space.w_long)):
            return w_result
        raise oefmt(space.w_TypeError,
                    "__int__ returned non-int (type '%T')", w_result)

    def ord(self, space):
        raise oefmt(space.w_TypeError,
                    "ord() expected string of length 1, but %T found", self)

    def __spacebind__(self, space):
        return self

    def unwrap(self, space):
        """NOT_RPYTHON"""
        # _____ this code is here to support testing only _____
        return self

    def unpackiterable_int(self, space):
        lst = space.listview_int(self)
        if lst:
            return lst[:]
        return None

    def unpackiterable_float(self, space):
        lst = space.listview_float(self)
        if lst:
            return lst[:]
        return None


class W_InterpIterable(W_Root):
    def __init__(self, space, w_iterable):
        self.w_iter = space.iter(w_iterable)
        self.space = space

    def __iter__(self):
        return self

    def next(self):
        space = self.space
        try:
            return space.next(self.w_iter)
        except OperationError, e:
            if not e.match(space, space.w_StopIteration):
                raise
            raise StopIteration

class InternalSpaceCache(Cache):
    """A generic cache for an object space.  Arbitrary information can
    be attached to the space by defining a function or class 'f' which
    can be called as 'f(space)'.  Its result is stored in this
    ObjSpaceCache.
    """
    def __init__(self, space):
        Cache.__init__(self)
        self.space = space
    def _build(self, callable):
        return callable(self.space)

class SpaceCache(Cache):
    """A base class for all our concrete caches."""
    def __init__(self, space):
        Cache.__init__(self)
        self.space = space

    def _build(self, key):
        return self.build(key)

    def _ready(self, result):
        return self.ready(result)

    def ready(self, result):
        pass

class DescrMismatch(Exception):
    pass

class BufferInterfaceNotFound(Exception):
    pass

def wrappable_class_name(Class):
    try:
        return Class.typedef.name
    except AttributeError:
        return 'internal subclass of %s' % (Class.__name__,)
wrappable_class_name._annspecialcase_ = 'specialize:memo'

class CannotHaveLock(Exception):
    """Raised by space.allocate_lock() if we're translating."""

# ____________________________________________________________

class ObjSpace(object):
    """Base class for the interpreter-level implementations of object spaces.
    http://pypy.readthedocs.org/en/latest/objspace.html"""

    def __init__(self, config=None):
        "NOT_RPYTHON: Basic initialization of objects."
        self.fromcache = InternalSpaceCache(self).getorbuild
        self.threadlocals = ThreadLocals()
        # set recursion limit
        # sets all the internal descriptors
        if config is None:
            from pypy.config.pypyoption import get_pypy_config
            config = get_pypy_config(translating=False)
        self.config = config

        self.builtin_modules = {}
        self.reloading_modules = {}

        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
        self.actionflag = ActionFlag()    # changed by the signal module
        self.check_signal_action = None   # changed by the signal module
        self.user_del_action = UserDelAction(self)
        self._code_of_sys_exc_info = None

        # can be overridden to a subclass
        self.initialize()

    def startup(self):
        # To be called before using the space
        self.threadlocals.enter_thread(self)

        # Initialize already imported builtin modules
        from pypy.interpreter.module import Module
        w_modules = self.sys.get('modules')
        for w_modname in self.unpackiterable(
                                self.sys.get('builtin_module_names')):
            try:
                w_mod = self.getitem(w_modules, w_modname)
            except OperationError, e:
                if e.match(self, self.w_KeyError):
                    continue
                raise
            if isinstance(w_mod, Module) and not w_mod.startup_called:
                w_mod.init(self)

    def finish(self):
        self.wait_for_thread_shutdown()
        w_exitfunc = self.sys.getdictvalue(self, 'exitfunc')
        if w_exitfunc is not None:
            self.call_function(w_exitfunc)
        from pypy.interpreter.module import Module
        for w_mod in self.builtin_modules.values():
            if isinstance(w_mod, Module) and w_mod.startup_called:
                w_mod.shutdown(self)

    def wait_for_thread_shutdown(self):
        """Wait until threading._shutdown() completes, provided the threading
        module was imported in the first place.  The shutdown routine will
        wait until all non-daemon 'threading' threads have completed."""
        if not self.config.translation.thread:
            return

        w_modules = self.sys.get('modules')
        w_mod = self.finditem_str(w_modules, 'threading')
        if w_mod is None:
            return

        try:
            self.call_method(w_mod, "_shutdown")
        except OperationError, e:
            e.write_unraisable(self, "threading._shutdown()")

    def __repr__(self):
        try:
            return self._this_space_repr_
        except AttributeError:
            return self.__class__.__name__

    def setbuiltinmodule(self, importname):
        """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules"""
        if '.' in importname:
            fullname = importname
            importname = fullname.rsplit('.', 1)[1]
        else:
            fullname = "pypy.module.%s" % importname

        Module = __import__(fullname,
                            None, None, ["Module"]).Module
        if Module.applevel_name is not None:
            name = Module.applevel_name
        else:
            name = importname

        mod = Module(self, self.wrap(name))
        mod.install()

        return name

    def getbuiltinmodule(self, name, force_init=False, reuse=True):
        w_name = self.wrap(name)
        w_modules = self.sys.get('modules')
        if not force_init:
            assert reuse
            try:
                return self.getitem(w_modules, w_name)
            except OperationError, e:
                if not e.match(self, self.w_KeyError):
                    raise

        # If the module is a builtin but not yet imported,
        # retrieve it and initialize it
        try:
            w_mod = self.builtin_modules[name]
        except KeyError:
            raise oefmt(self.w_SystemError,
                        "getbuiltinmodule() called with non-builtin module %s",
                        name)

        # Add the module to sys.modules and initialize the module. The
        # order is important to avoid recursions.
        from pypy.interpreter.module import Module
        if isinstance(w_mod, Module):
            if not reuse and w_mod.startup_called:
                # create a copy of the module.  (see issue1514) eventlet
                # patcher relies on this behaviour.
                w_mod2 = self.wrap(Module(self, w_name))
                self.setitem(w_modules, w_name, w_mod2)
                w_mod.getdict(self)  # unlazy w_initialdict
                self.call_method(w_mod2.getdict(self), 'update',
                                 w_mod.w_initialdict)
                return w_mod2
            self.setitem(w_modules, w_name, w_mod)
            w_mod.init(self)
        else:
            self.setitem(w_modules, w_name, w_mod)
        return w_mod

    def get_builtinmodule_to_install(self):
        """NOT_RPYTHON"""
        try:
            return self._builtinmodule_list
        except AttributeError:
            pass

        modules = []

        # You can enable more modules by specifying --usemodules=xxx,yyy
        for name, value in self.config.objspace.usemodules:
            if value and name not in modules:
                modules.append(name)

        if self.config.objspace.extmodules:
            for name in self.config.objspace.extmodules.split(','):
                if name not in modules:
                    modules.append(name)

        self._builtinmodule_list = modules
        return self._builtinmodule_list

    ALL_BUILTIN_MODULES = [
        'posix', 'nt', 'os2', 'mac', 'ce', 'riscos',
        'math', 'array', 'select',
        '_random', '_sre', 'time', '_socket', 'errno',
        'unicodedata',
        'parser', 'fcntl', '_codecs', 'binascii'
    ]

    # These modules are treated like CPython treats built-in modules,
    # i.e. they always shadow any xx.py.  The other modules are treated
    # like CPython treats extension modules, and are loaded in sys.path
    # order by the fake entry '.../lib_pypy/__extensions__'.
    MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([
        '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings',
        '_weakref', 'errno', 'exceptions', 'gc', 'imp', 'marshal',
        'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport',
    ], None)

    def make_builtins(self):
        "NOT_RPYTHON: only for initializing the space."

        from pypy.module.exceptions import Module
        w_name = self.wrap('exceptions')
        self.exceptions_module = Module(self, w_name)
        self.exceptions_module.install()

        from pypy.module.sys import Module
        w_name = self.wrap('sys')
        self.sys = Module(self, w_name)
        self.sys.install()

        from pypy.module.imp import Module
        w_name = self.wrap('imp')
        mod = Module(self, w_name)
        mod.install()

        from pypy.module.__builtin__ import Module
        w_name = self.wrap('__builtin__')
        self.builtin = Module(self, w_name)
        w_builtin = self.wrap(self.builtin)
        w_builtin.install()
        self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)

        bootstrap_modules = set(('sys', 'imp', '__builtin__', 'exceptions'))
        installed_builtin_modules = list(bootstrap_modules)

        exception_types_w = self.export_builtin_exceptions()

        # initialize with "bootstrap types" from objspace  (e.g. w_None)
        types_w = (self.get_builtin_types().items() +
                   exception_types_w.items())
        for name, w_type in types_w:
            self.setitem(self.builtin.w_dict, self.wrap(name), w_type)

        # install mixed modules
        for mixedname in self.get_builtinmodule_to_install():
            if mixedname not in bootstrap_modules:
                self.install_mixedmodule(mixedname, installed_builtin_modules)

        installed_builtin_modules.sort()
        w_builtin_module_names = self.newtuple(
            [self.wrap(fn) for fn in installed_builtin_modules])

        # force this value into the dict without unlazyfying everything
        self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'),
                     w_builtin_module_names)

    def get_builtin_types(self):
        """Get a dictionary mapping the names of builtin types to the type
        objects."""
        raise NotImplementedError

    def export_builtin_exceptions(self):
        """NOT_RPYTHON"""
        w_dic = self.exceptions_module.getdict(self)
        w_keys = self.call_method(w_dic, "keys")
        exc_types_w = {}
        for w_name in self.unpackiterable(w_keys):
            name = self.str_w(w_name)
            if not name.startswith('__'):
                excname = name
                w_exc = self.getitem(w_dic, w_name)
                exc_types_w[name] = w_exc
                setattr(self, "w_" + excname, w_exc)
        return exc_types_w

    def install_mixedmodule(self, mixedname, installed_builtin_modules):
        """NOT_RPYTHON"""
        modname = self.setbuiltinmodule(mixedname)
        if modname:
            assert modname not in installed_builtin_modules, (
                "duplicate interp-level module enabled for the "
                "app-level module %r" % (modname,))
            installed_builtin_modules.append(modname)

    def setup_builtin_modules(self):
        "NOT_RPYTHON: only for initializing the space."
        if self.config.objspace.usemodules.cpyext:
            from pypy.module.cpyext.state import State
            self.fromcache(State).build_api(self)
        self.getbuiltinmodule('sys')
        self.getbuiltinmodule('imp')
        self.getbuiltinmodule('__builtin__')
        for mod in self.builtin_modules.values():
            mod.setup_after_space_initialization()

    def initialize(self):
        """NOT_RPYTHON: Abstract method that should put some minimal
        content into the w_builtins."""

    def getexecutioncontext(self):
        "Return what we consider to be the active execution context."
        # Important: the annotator must not see a prebuilt ExecutionContext:
        # you should not see frames while you translate
        # so we make sure that the threadlocals never *have* an
        # ExecutionContext during translation.
        if not we_are_translated():
            if self.config.translating:
                assert self.threadlocals.get_ec() is None, (
                    "threadlocals got an ExecutionContext during translation!")
                try:
                    return self._ec_during_translation
                except AttributeError:
                    ec = self.createexecutioncontext()
                    self._ec_during_translation = ec
                    return ec
            else:
                ec = self.threadlocals.get_ec()
                if ec is None:
                    self.threadlocals.enter_thread(self)
                    ec = self.threadlocals.get_ec()
                return ec
        else:
            # translated case follows.  self.threadlocals is either from
            # 'pypy.interpreter.miscutils' or 'pypy.module.thread.threadlocals'.
            # the result is assumed to be non-null: enter_thread() was called
            # by space.startup().
            ec = self.threadlocals.get_ec()
            assert ec is not None
            return ec

    def _freeze_(self):
        return True

    def createexecutioncontext(self):
        "Factory function for execution contexts."
        return ExecutionContext(self)

    def createcompiler(self):
        "Factory function creating a compiler object."
        try:
            return self.default_compiler
        except AttributeError:
            from pypy.interpreter.pycompiler import PythonAstCompiler
            compiler = PythonAstCompiler(self)
            self.default_compiler = compiler
            return compiler

    def createframe(self, code, w_globals, outer_func=None):
        "Create an empty PyFrame suitable for this code object."
        return self.FrameClass(self, code, w_globals, outer_func)

    def allocate_lock(self):
        """Return an interp-level Lock object if threads are enabled,
        and a dummy object if they are not."""
        from rpython.rlib import rthread
        if not self.config.objspace.usemodules.thread:
            return rthread.dummy_lock
        # hack: we can't have prebuilt locks if we're translating.
        # In this special situation we should just not lock at all
        # (translation is not multithreaded anyway).
        if not we_are_translated() and self.config.translating:
            raise CannotHaveLock()
        try:
            return rthread.allocate_lock()
        except rthread.error:
            raise OperationError(self.w_RuntimeError,
                                 self.wrap("out of resources"))

    # Following is a friendly interface to common object space operations
    # that can be defined in term of more primitive ones.  Subclasses
    # may also override specific functions for performance.

    def not_(self, w_obj):
        return self.wrap(not self.is_true(w_obj))

    def eq_w(self, w_obj1, w_obj2):
        """Implements equality with the double check 'x is y or x == y'."""
        return self.is_w(w_obj1, w_obj2) or self.is_true(self.eq(w_obj1, w_obj2))

    def is_(self, w_one, w_two):
        return self.newbool(self.is_w(w_one, w_two))

    def is_w(self, w_one, w_two):
        # done by a method call on w_two (and not on w_one, because of the
        # expected programming style where we say "if x is None" or
        # "if x is object").
        assert w_two is not None
        return w_two.is_w(self, w_one)

    def is_none(self, w_obj):
        """ mostly for checking inputargs that have unwrap_spec and
        can accept both w_None and None
        """
        return w_obj is None or self.is_w(w_obj, self.w_None)

    def id(self, w_obj):
        w_result = w_obj.immutable_unique_id(self)
        if w_result is None:
            # in the common case, returns an unsigned value
            w_result = self.wrap(r_uint(compute_unique_id(w_obj)))
        return w_result

    def hash_w(self, w_obj):
        """shortcut for space.int_w(space.hash(w_obj))"""
        return self.int_w(self.hash(w_obj))

    def len_w(self, w_obj):
        """shotcut for space.int_w(space.len(w_obj))"""
        return self.int_w(self.len(w_obj))

    def setitem_str(self, w_obj, key, w_value):
        return self.setitem(w_obj, self.wrap(key), w_value)

    def finditem_str(self, w_obj, key):
        return self.finditem(w_obj, self.wrap(key))

    def finditem(self, w_obj, w_key):
        try:
            return self.getitem(w_obj, w_key)
        except OperationError, e:
            if e.match(self, self.w_KeyError):
                return None
            raise

    def findattr(self, w_object, w_name):
        try:
            return self.getattr(w_object, w_name)
        except OperationError, e:
            # a PyPy extension: let SystemExit and KeyboardInterrupt go through
            if e.async(self):
                raise
            return None

    @signature(types.any(), types.bool(), returns=types.instance(W_Root))
    def newbool(self, b):
        if b:
            return self.w_True
        else:
            return self.w_False

    def new_interned_w_str(self, w_s):
        assert isinstance(w_s, W_Root)   # and is not None
        s = self.str_w(w_s)
        if not we_are_translated():
            assert type(s) is str
        w_s1 = self.interned_strings.get(s)
        if w_s1 is None:
            w_s1 = w_s
            self.interned_strings.set(s, w_s1)
        return w_s1

    def new_interned_str(self, s):
        if not we_are_translated():
            assert type(s) is str
        w_s1 = self.interned_strings.get(s)
        if w_s1 is None:
            w_s1 = self.wrap(s)
            self.interned_strings.set(s, w_s1)
        return w_s1

    def is_interned_str(self, s):
        # interface for marshal_impl
        if not we_are_translated():
            assert type(s) is str
        return self.interned_strings.get(s) is not None

    def descr_self_interp_w(self, RequiredClass, w_obj):
        if not isinstance(w_obj, RequiredClass):
            raise DescrMismatch()
        return w_obj
    descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)'

    def interp_w(self, RequiredClass, w_obj, can_be_None=False):
        """
        Unwrap w_obj, checking that it is an instance of the required internal
        interpreter class.
        """
        assert RequiredClass is not None
        if can_be_None and self.is_none(w_obj):
            return None
        if not isinstance(w_obj, RequiredClass):   # or obj is None
            raise oefmt(self.w_TypeError,
                        "'%s' object expected, got '%N' instead",
                        wrappable_class_name(RequiredClass),
                        w_obj.getclass(self))
        return w_obj
    interp_w._annspecialcase_ = 'specialize:arg(1)'

    def unpackiterable(self, w_iterable, expected_length=-1):
        """Unpack an iterable into a real (interpreter-level) list.

        Raise an OperationError(w_ValueError) if the length is wrong."""
        w_iterator = self.iter(w_iterable)
        if expected_length == -1:
            # xxx special hack for speed
            from pypy.interpreter.generator import GeneratorIterator
            if isinstance(w_iterator, GeneratorIterator):
                lst_w = []
                w_iterator.unpack_into(lst_w)
                return lst_w
            # /xxx
            return self._unpackiterable_unknown_length(w_iterator, w_iterable)
        else:
            lst_w = self._unpackiterable_known_length(w_iterator,
                                                      expected_length)
            return lst_w[:]     # make the resulting list resizable

    def iteriterable(self, w_iterable):
        return W_InterpIterable(self, w_iterable)

    def _unpackiterable_unknown_length(self, w_iterator, w_iterable):
        """Unpack an iterable of unknown length into an interp-level
        list.
        """
        # If we can guess the expected length we can preallocate.
        try:
            items = newlist_hint(self.length_hint(w_iterable, 0))
        except MemoryError:
            items = [] # it might have lied

        tp = self.type(w_iterator)
        while True:
            unpackiterable_driver.jit_merge_point(tp=tp,
                                                  w_iterator=w_iterator,
                                                  items=items)
            try:
                w_item = self.next(w_iterator)
            except OperationError, e:
                if not e.match(self, self.w_StopIteration):
                    raise
                break  # done
            items.append(w_item)
        #
        return items

    @jit.dont_look_inside
    def _unpackiterable_known_length(self, w_iterator, expected_length):
        # Unpack a known length list, without letting the JIT look inside.
        # Implemented by just calling the @jit.unroll_safe version, but
        # the JIT stopped looking inside already.
        return self._unpackiterable_known_length_jitlook(w_iterator,
                                                         expected_length)

    @jit.unroll_safe
    def _unpackiterable_known_length_jitlook(self, w_iterator,
                                             expected_length):
        items = [None] * expected_length
        idx = 0
        while True:
            try:
                w_item = self.next(w_iterator)
            except OperationError, e:
                if not e.match(self, self.w_StopIteration):
                    raise
                break  # done
            if idx == expected_length:
                raise OperationError(self.w_ValueError,
                                    self.wrap("too many values to unpack"))
            items[idx] = w_item
            idx += 1
        if idx < expected_length:
            raise oefmt(self.w_ValueError,
                        "need more than %d value%s to unpack",
                        idx, "" if idx == 1 else "s")
        return items

    def unpackiterable_unroll(self, w_iterable, expected_length):
        # Like unpackiterable(), but for the cases where we have
        # an expected_length and want to unroll when JITted.
        # Returns a fixed-size list.
        w_iterator = self.iter(w_iterable)
        assert expected_length != -1
        return self._unpackiterable_known_length_jitlook(w_iterator,
                                                         expected_length)


    def unpackiterable_int(self, w_obj):
        """
        Return a RPython list of unwrapped ints out of w_obj. The list is
        guaranteed to be acopy of the actual data contained in w_obj, so you
        can freely modify it. It might return None if not supported.
        """
        return w_obj.unpackiterable_int(self)

    def unpackiterable_float(self, w_obj):
        """
        Same as unpackiterable_int, but for floats.
        """
        return w_obj.unpackiterable_float(self)


    def length_hint(self, w_obj, default):
        """Return the length of an object, consulting its __length_hint__
        method if necessary.
        """
        try:
            return self.len_w(w_obj)
        except OperationError, e:
            if not (e.match(self, self.w_TypeError) or
                    e.match(self, self.w_AttributeError)):
                raise

        w_descr = self.lookup(w_obj, '__length_hint__')
        if w_descr is None:
            return default
        try:
            w_hint = self.get_and_call_function(w_descr, w_obj)
        except OperationError, e:
            if not (e.match(self, self.w_TypeError) or
                    e.match(self, self.w_AttributeError)):
                raise
            return default
        if self.is_w(w_hint, self.w_NotImplemented):
            return default

        hint = self.int_w(w_hint)
        if hint < 0:
            raise OperationError(self.w_ValueError, self.wrap(
                    "__length_hint__() should return >= 0"))
        return hint

    def fixedview(self, w_iterable, expected_length=-1):
        """ A fixed list view of w_iterable. Don't modify the result
        """
        return make_sure_not_resized(self.unpackiterable(w_iterable,
                                                         expected_length)[:])

    fixedview_unroll = fixedview

    def listview(self, w_iterable, expected_length=-1):
        """ A non-fixed view of w_iterable. Don't modify the result
        """
        return self.unpackiterable(w_iterable, expected_length)

    def listview_no_unpack(self, w_iterable):
        """ Same as listview() if cheap.  If 'w_iterable' is something like
        a generator, for example, then return None instead.
        May return None anyway.
        """
        return None

    def listview_bytes(self, w_list):
        """ Return a list of unwrapped strings out of a list of strings. If the
        argument is not a list or does not contain only strings, return None.
        May return None anyway.
        """
        return None

    def listview_unicode(self, w_list):
        """ Return a list of unwrapped unicode out of a list of unicode. If the
        argument is not a list or does not contain only unicode, return None.
        May return None anyway.
        """
        return None

    def listview_int(self, w_list):
        """ Return a list of unwrapped int out of a list of int. If the
        argument is not a list or does not contain only int, return None.
        May return None anyway.
        """
        return None

    def listview_float(self, w_list):
        """ Return a list of unwrapped float out of a list of float. If the
        argument is not a list or does not contain only float, return None.
        May return None anyway.
        """
        return None

    def view_as_kwargs(self, w_dict):
        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
        strings and one of wrapped values. otherwise return (None, None)
        """
        return (None, None)

    def newlist_bytes(self, list_s):
        return self.newlist([self.wrap(s) for s in list_s])

    def newlist_unicode(self, list_u):
        return self.newlist([self.wrap(u) for u in list_u])

    def newlist_int(self, list_i):
        return self.newlist([self.wrap(i) for i in list_i])

    def newlist_hint(self, sizehint):
        from pypy.objspace.std.listobject import make_empty_list_with_size
        return make_empty_list_with_size(self, sizehint)

    @jit.unroll_safe
    def exception_match(self, w_exc_type, w_check_class):
        """Checks if the given exception type matches 'w_check_class'."""
        if self.is_w(w_exc_type, w_check_class):
            return True   # fast path (also here to handle string exceptions)
        try:
            if self.isinstance_w(w_check_class, self.w_tuple):
                for w_t in self.fixedview(w_check_class):
                    if self.exception_match(w_exc_type, w_t):
                        return True
                else:
                    return False
            return self.exception_issubclass_w(w_exc_type, w_check_class)
        except OperationError, e:
            if e.match(self, self.w_TypeError):   # string exceptions maybe
                return False
            raise

    def call_obj_args(self, w_callable, w_obj, args):
        if not self.config.objspace.disable_call_speedhacks:
            # start of hack for performance
            from pypy.interpreter.function import Function
            if isinstance(w_callable, Function):
                return w_callable.call_obj_args(w_obj, args)
            # end of hack for performance
        return self.call_args(w_callable, args.prepend(w_obj))

    def call(self, w_callable, w_args, w_kwds=None):
        args = Arguments.frompacked(self, w_args, w_kwds)
        return self.call_args(w_callable, args)

    def call_function(self, w_func, *args_w):
        nargs = len(args_w) # used for pruning funccall versions
        if not self.config.objspace.disable_call_speedhacks and nargs < 5:
            # start of hack for performance
            from pypy.interpreter.function import Function, Method
            if isinstance(w_func, Method):
                w_inst = w_func.w_instance
                if w_inst is not None:
                    if nargs < 4:
                        func = w_func.w_function
                        if isinstance(func, Function):
                            return func.funccall(w_inst, *args_w)
                elif args_w and (
                        self.abstract_isinstance_w(args_w[0], w_func.w_class)):
                    w_func = w_func.w_function

            if isinstance(w_func, Function):
                return w_func.funccall(*args_w)
            # end of hack for performance

        args = Arguments(self, list(args_w))
        return self.call_args(w_func, args)

    def call_valuestack(self, w_func, nargs, frame):
        from pypy.interpreter.function import Function, Method, is_builtin_code
        if frame.get_is_being_profiled() and is_builtin_code(w_func):
            # XXX: this code is copied&pasted :-( from the slow path below
            # call_valuestack().
            args = frame.make_arguments(nargs)
            return self.call_args_and_c_profile(frame, w_func, args)

        if not self.config.objspace.disable_call_speedhacks:
            # start of hack for performance
            if isinstance(w_func, Method):
                w_inst = w_func.w_instance
                if w_inst is not None:
                    w_func = w_func.w_function
                    # reuse callable stack place for w_inst
                    frame.settopvalue(w_inst, nargs)
                    nargs += 1
                elif nargs > 0 and (
                    self.abstract_isinstance_w(frame.peekvalue(nargs-1),   #    :-(
                                               w_func.w_class)):
                    w_func = w_func.w_function

            if isinstance(w_func, Function):
                return w_func.funccall_valuestack(nargs, frame)
            # end of hack for performance

        args = frame.make_arguments(nargs)
        return self.call_args(w_func, args)

    def call_args_and_c_profile(self, frame, w_func, args):
        ec = self.getexecutioncontext()
        ec.c_call_trace(frame, w_func, args)
        try:
            w_res = self.call_args(w_func, args)
        except OperationError:
            ec.c_exception_trace(frame, w_func)
            raise
        ec.c_return_trace(frame, w_func, args)
        return w_res

    def call_method(self, w_obj, methname, *arg_w):
        w_meth = self.getattr(w_obj, self.wrap(methname))
        return self.call_function(w_meth, *arg_w)

    def raise_key_error(self, w_key):
        e = self.call_function(self.w_KeyError, w_key)
        raise OperationError(self.w_KeyError, e)

    def lookup(self, w_obj, name):
        w_type = self.type(w_obj)
        w_mro = self.getattr(w_type, self.wrap("__mro__"))
        for w_supertype in self.fixedview(w_mro):
            w_value = w_supertype.getdictvalue(self, name)
            if w_value is not None:
                return w_value
        return None

    def is_oldstyle_instance(self, w_obj):
        # xxx hack hack hack
        from pypy.module.__builtin__.interp_classobj import W_InstanceObject
        return isinstance(w_obj, W_InstanceObject)

    def callable(self, w_obj):
        if self.lookup(w_obj, "__call__") is not None:
            if self.is_oldstyle_instance(w_obj):
                # ugly old style class special treatment, but well ...
                try:
                    self.getattr(w_obj, self.wrap("__call__"))
                    return self.w_True
                except OperationError, e:
                    if not e.match(self, self.w_AttributeError):
                        raise
                    return self.w_False
            else:
                return self.w_True
        return self.w_False

    def issequence_w(self, w_obj):
        return (self.findattr(w_obj, self.wrap("__getitem__")) is not None)

    # The code below only works
    # for the simple case (new-style instance).
    # These methods are patched with the full logic by the __builtin__
    # module when it is loaded

    def abstract_issubclass_w(self, w_cls1, w_cls2):
        # Equivalent to 'issubclass(cls1, cls2)'.
        return self.is_true(self.issubtype(w_cls1, w_cls2))

    def abstract_isinstance_w(self, w_obj, w_cls):
        # Equivalent to 'isinstance(obj, cls)'.
        return self.isinstance_w(w_obj, w_cls)

    def abstract_isclass_w(self, w_obj):
        # Equivalent to 'isinstance(obj, type)'.
        return self.isinstance_w(w_obj, self.w_type)

    def abstract_getclass(self, w_obj):
        # Equivalent to 'obj.__class__'.
        return self.type(w_obj)

    # CPython rules allows old style classes or subclasses
    # of BaseExceptions to be exceptions.
    # This is slightly less general than the case above, so we prefix
    # it with exception_

    def exception_is_valid_obj_as_class_w(self, w_obj):
        if not self.isinstance_w(w_obj, self.w_type):
            return False
        return self.is_true(self.issubtype(w_obj, self.w_BaseException))

    def exception_is_valid_class_w(self, w_cls):
        return self.is_true(self.issubtype(w_cls, self.w_BaseException))

    def exception_getclass(self, w_obj):
        return self.type(w_obj)

    def exception_issubclass_w(self, w_cls1, w_cls2):
        return self.is_true(self.issubtype(w_cls1, w_cls2))

    def new_exception_class(self, *args, **kwargs):
        "NOT_RPYTHON; convenience method to create excceptions in modules"
        return new_exception_class(self, *args, **kwargs)

    # end of special support code

    def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
        "NOT_RPYTHON: For internal debugging."
        if isinstance(expression, str):
            compiler = self.createcompiler()
            expression = compiler.compile(expression, '?', 'eval', 0,
                                         hidden_applevel=hidden_applevel)
        else:
            raise TypeError('space.eval(): expected a string, code or PyCode object')
        return expression.exec_code(self, w_globals, w_locals)

    def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
              filename=None):
        "NOT_RPYTHON: For internal debugging."
        if filename is None:
            filename = '?'
        from pypy.interpreter.pycode import PyCode
        if isinstance(statement, str):
            compiler = self.createcompiler()
            statement = compiler.compile(statement, filename, 'exec', 0,
                                         hidden_applevel=hidden_applevel)
        if not isinstance(statement, PyCode):
            raise TypeError('space.exec_(): expected a string, code or PyCode object')
        w_key = self.wrap('__builtins__')
        if not self.is_true(self.contains(w_globals, w_key)):
            self.setitem(w_globals, w_key, self.wrap(self.builtin))
        return statement.exec_code(self, w_globals, w_locals)

    def appexec(self, posargs_w, source):
        """ return value from executing given source at applevel.
            EXPERIMENTAL. The source must look like
               '''(x, y):
                       do_stuff...
                       return result
               '''
        """
        w_func = self.fromcache(AppExecCache).getorbuild(source)
        args = Arguments(self, list(posargs_w))
        return self.call_args(w_func, args)
    appexec._annspecialcase_ = 'specialize:arg(2)'

    def _next_or_none(self, w_it):
        try:
            return self.next(w_it)
        except OperationError, e:
            if not e.match(self, self.w_StopIteration):
                raise
            return None

    def compare_by_iteration(self, w_iterable1, w_iterable2, op):
        w_it1 = self.iter(w_iterable1)
        w_it2 = self.iter(w_iterable2)
        while True:
            w_x1 = self._next_or_none(w_it1)
            w_x2 = self._next_or_none(w_it2)
            if w_x1 is None or w_x2 is None:
                if op == 'eq': return self.newbool(w_x1 is w_x2)  # both None
                if op == 'ne': return self.newbool(w_x1 is not w_x2)
                if op == 'lt': return self.newbool(w_x2 is not None)
                if op == 'le': return self.newbool(w_x1 is None)
                if op == 'gt': return self.newbool(w_x1 is not None)
                if op == 'ge': return self.newbool(w_x2 is None)
                assert False, "bad value for op"
            if not self.eq_w(w_x1, w_x2):
                if op == 'eq': return self.w_False
                if op == 'ne': return self.w_True
                if op == 'lt': return self.lt(w_x1, w_x2)
                if op == 'le': return self.le(w_x1, w_x2)
                if op == 'gt': return self.gt(w_x1, w_x2)
                if op == 'ge': return self.ge(w_x1, w_x2)
                assert False, "bad value for op"
    compare_by_iteration._annspecialcase_ = 'specialize:arg(3)'

    def decode_index(self, w_index_or_slice, seqlength):
        """Helper for custom sequence implementations
             -> (index, 0, 0) or
                (start, stop, step)
        """
        if self.isinstance_w(w_index_or_slice, self.w_slice):
            from pypy.objspace.std.sliceobject import W_SliceObject
            assert isinstance(w_index_or_slice, W_SliceObject)
            start, stop, step = w_index_or_slice.indices3(self, seqlength)
        else:
            start = self.int_w(w_index_or_slice, allow_conversion=False)
            if start < 0:
                start += seqlength
            if not (0 <= start < seqlength):
                raise OperationError(self.w_IndexError,
                                     self.wrap("index out of range"))
            stop = 0
            step = 0
        return start, stop, step

    def decode_index4(self, w_index_or_slice, seqlength):
        """Helper for custom sequence implementations
             -> (index, 0, 0, 1) or
                (start, stop, step, slice_length)
        """
        if self.isinstance_w(w_index_or_slice, self.w_slice):
            from pypy.objspace.std.sliceobject import W_SliceObject
            assert isinstance(w_index_or_slice, W_SliceObject)
            start, stop, step, length = w_index_or_slice.indices4(self,
                                                                  seqlength)
        else:
            start = self.int_w(w_index_or_slice, allow_conversion=False)
            if start < 0:
                start += seqlength
            if not (0 <= start < seqlength):
                raise OperationError(self.w_IndexError,
                                     self.wrap…