/rpython/memory/gctransform/asmgcroot.py

from rpython.flowspace.model import (Constant, Variable, Block, Link,
     copygraph, SpaceOperation, checkgraph)
from rpython.rlib.debug import ll_assert
from rpython.rlib.nonconst import NonConstant
from rpython.rlib import rgil
from rpython.rtyper.annlowlevel import llhelper
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.memory.gctransform.framework import (
     BaseFrameworkGCTransformer, BaseRootWalker)
from rpython.rtyper.llannotation import SomeAddress
from rpython.rtyper.rbuiltin import gen_cast
from rpython.translator.unsimplify import varoftype
from rpython.translator.tool.cbuild import ExternalCompilationInfo
import sys


#
#  This transformer avoids the use of a shadow stack in a completely
#  platform-specific way, by directing genc to insert asm() special
#  instructions in the C source, which are recognized by GCC.
#  The .s file produced by GCC is then parsed by trackgcroot.py.
#

IS_64_BITS = sys.maxint > 2147483647

class AsmGcRootFrameworkGCTransformer(BaseFrameworkGCTransformer):
    _asmgcc_save_restore_arguments = None
    _seen_gctransformer_hint_close_stack = None

    def push_roots(self, hop, keep_current_args=False):
        livevars = self.get_livevars_for_roots(hop, keep_current_args)
        self.num_pushs += len(livevars)
        return livevars

    def pop_roots(self, hop, livevars):
        if not livevars:
            return
        # mark the values as gc roots
        for var in livevars:
            v_adr = gen_cast(hop.llops, llmemory.Address, var)
            v_newaddr = hop.genop("direct_call", [c_asm_gcroot, v_adr],
                                  resulttype=llmemory.Address)
            hop.genop("gc_reload_possibly_moved", [v_newaddr, var])

    def build_root_walker(self):
        return AsmStackRootWalker(self)

    def mark_call_cannotcollect(self, hop, name):
        hop.genop("direct_call", [c_asm_nocollect, name])

    def gct_direct_call(self, hop):
        fnptr = hop.spaceop.args[0].value
        try:
            close_stack = fnptr._obj._callable._gctransformer_hint_close_stack_
        except AttributeError:
            close_stack = False
        if close_stack:
            self.handle_call_with_close_stack(hop)
        else:
            BaseFrameworkGCTransformer.gct_direct_call(self, hop)

    def handle_call_with_close_stack(self, hop):
        fnptr = hop.spaceop.args[0].value
        if self._seen_gctransformer_hint_close_stack is None:
            self._seen_gctransformer_hint_close_stack = {}
        if fnptr._obj.graph not in self._seen_gctransformer_hint_close_stack:
            self._transform_hint_close_stack(fnptr)
            self._seen_gctransformer_hint_close_stack[fnptr._obj.graph] = True
        #
        livevars = self.push_roots(hop)
        self.default(hop)
        self.pop_roots(hop, livevars)

    def _transform_hint_close_stack(self, fnptr):
        # We cannot easily pass variable amount of arguments of the call
        # across the call to the pypy_asm_stackwalk helper.  So we store
        # them away and restore them.  More precisely, we need to
        # replace 'graph' with code that saves the arguments, and make
        # a new graph that starts with restoring the arguments.
        if self._asmgcc_save_restore_arguments is None:
            self._asmgcc_save_restore_arguments = {}
        sradict = self._asmgcc_save_restore_arguments
        sra = []     # list of pointers to raw-malloced containers for args
        seen = {}
        FUNC1 = lltype.typeOf(fnptr).TO
        for TYPE in FUNC1.ARGS:
            if isinstance(TYPE, lltype.Ptr):
                TYPE = llmemory.Address
            num = seen.get(TYPE, 0)
            seen[TYPE] = num + 1
            key = (TYPE, num)
            if key not in sradict:
                CONTAINER = lltype.FixedSizeArray(TYPE, 1)
                p = lltype.malloc(CONTAINER, flavor='raw', zero=True,
                                  immortal=True)
                sradict[key] = Constant(p, lltype.Ptr(CONTAINER))
            sra.append(sradict[key])
        #
        # make a copy of the graph that will reload the values
        graph = fnptr._obj.graph
        graph2 = copygraph(graph)
        #
        # edit the original graph to only store the value of the arguments
        block = Block(graph.startblock.inputargs)
        c_item0 = Constant('item0', lltype.Void)
        assert len(block.inputargs) == len(sra)
        for v_arg, c_p in zip(block.inputargs, sra):
            if isinstance(v_arg.concretetype, lltype.Ptr):
                v_adr = varoftype(llmemory.Address)
                block.operations.append(
                    SpaceOperation("cast_ptr_to_adr", [v_arg], v_adr))
                v_arg = v_adr
            v_void = varoftype(lltype.Void)
            block.operations.append(
                SpaceOperation("bare_setfield", [c_p, c_item0, v_arg], v_void))
        #
        # call asm_stackwalk(graph2)
        FUNC2 = lltype.FuncType([], FUNC1.RESULT)
        fnptr2 = lltype.functionptr(FUNC2,
                                    fnptr._obj._name + '_reload',
                                    graph=graph2)
        c_fnptr2 = Constant(fnptr2, lltype.Ptr(FUNC2))
        HELPERFUNC = lltype.FuncType([lltype.Ptr(FUNC2),
                                      ASM_FRAMEDATA_HEAD_PTR], FUNC1.RESULT)
        v_asm_stackwalk = varoftype(lltype.Ptr(HELPERFUNC), "asm_stackwalk")
        block.operations.append(
            SpaceOperation("cast_pointer", [c_asm_stackwalk], v_asm_stackwalk))
        v_result = varoftype(FUNC1.RESULT)
        block.operations.append(
            SpaceOperation("indirect_call", [v_asm_stackwalk, c_fnptr2,
                                             c_gcrootanchor,
                                             Constant(None, lltype.Void)],
                           v_result))
        block.closeblock(Link([v_result], graph.returnblock))
        graph.startblock = block
        #
        # edit the copy of the graph to reload the values
        block2 = graph2.startblock
        block1 = Block([])
        reloadedvars = []
        for v, c_p in zip(block2.inputargs, sra):
            v = v.copy()
            if isinstance(v.concretetype, lltype.Ptr):
                w = varoftype(llmemory.Address)
            else:
                w = v
            block1.operations.append(SpaceOperation('getfield',
                                                    [c_p, c_item0], w))
            if w is not v:
                block1.operations.append(SpaceOperation('cast_adr_to_ptr',
                                                        [w], v))
            reloadedvars.append(v)
        block1.closeblock(Link(reloadedvars, block2))
        graph2.startblock = block1
        #
        checkgraph(graph)
        checkgraph(graph2)


class AsmStackRootWalker(BaseRootWalker):

    def __init__(self, gctransformer):
        BaseRootWalker.__init__(self, gctransformer)

        def _asm_callback():
            self.walk_stack_from()
        self._asm_callback = _asm_callback
        self._shape_decompressor = ShapeDecompressor()
        self._with_jit = hasattr(gctransformer.translator, '_jit2gc')
        if self._with_jit:
            jit2gc = gctransformer.translator._jit2gc
            self.frame_tid = jit2gc['frame_tid']
        self.gctransformer = gctransformer
        #
        # unless overridden in need_thread_support():
        self.belongs_to_current_thread = lambda framedata: True

    def need_stacklet_support(self, gctransformer, getfn):
        from rpython.annotator import model as annmodel
        from rpython.rlib import _stacklet_asmgcc
        # stacklet support: BIG HACK for rlib.rstacklet
        _stacklet_asmgcc._asmstackrootwalker = self     # as a global! argh
        _stacklet_asmgcc.complete_destrptr(gctransformer)
        #
        def gc_detach_callback_pieces():
            anchor = llmemory.cast_ptr_to_adr(gcrootanchor)
            result = llmemory.NULL
            framedata = anchor.address[1]
            while framedata != anchor:
                next = framedata.address[1]
                if self.belongs_to_current_thread(framedata):
                    # detach it
                    prev = framedata.address[0]
                    prev.address[1] = next
                    next.address[0] = prev
                    # update the global stack counter
                    rffi.stackcounter.stacks_counter -= 1
                    # reattach framedata into the singly-linked list 'result'
                    framedata.address[0] = rffi.cast(llmemory.Address, -1)
                    framedata.address[1] = result
                    result = framedata
                framedata = next
            return result
        #
        def gc_reattach_callback_pieces(pieces):
            anchor = llmemory.cast_ptr_to_adr(gcrootanchor)
            while pieces != llmemory.NULL:
                framedata = pieces
                pieces = pieces.address[1]
                # attach 'framedata' into the normal doubly-linked list
                following = anchor.address[1]
                following.address[0] = framedata
                framedata.address[1] = following
                anchor.address[1] = framedata
                framedata.address[0] = anchor
                # update the global stack counter
                rffi.stackcounter.stacks_counter += 1
        #
        s_addr = SomeAddress()
        s_None = annmodel.s_None
        self.gc_detach_callback_pieces_ptr = getfn(gc_detach_callback_pieces,
                                                   [], s_addr)
        self.gc_reattach_callback_pieces_ptr=getfn(gc_reattach_callback_pieces,
                                                   [s_addr], s_None)

    def need_thread_support(self, gctransformer, getfn):
        # Threads supported "out of the box" by the rest of the code.
        # The whole code in this function is only there to support
        # fork()ing in a multithreaded process :-(
        # For this, we need to handle gc_thread_start and gc_thread_die
        # to record the mapping {thread_id: stack_start}, and
        # gc_thread_before_fork and gc_thread_after_fork to get rid of
        # all ASM_FRAMEDATA structures that do no belong to the current
        # thread after a fork().
        from rpython.rlib import rthread
        from rpython.memory.support import AddressDict
        from rpython.memory.support import copy_without_null_values
        from rpython.annotator import model as annmodel
        gcdata = self.gcdata

        def get_aid():
            """Return the thread identifier, cast to an (opaque) address."""
            return llmemory.cast_int_to_adr(rthread.get_ident())

        def thread_start():
            value = llmemory.cast_int_to_adr(llop.stack_current(lltype.Signed))
            gcdata.aid2stack.setitem(get_aid(), value)
        thread_start._always_inline_ = True

        def thread_setup():
            gcdata.aid2stack = AddressDict()
            gcdata.dead_threads_count = 0
            # to also register the main thread's stack
            thread_start()
        thread_setup._always_inline_ = True

        def thread_die():
            gcdata.aid2stack.setitem(get_aid(), llmemory.NULL)
            # from time to time, rehash the dictionary to remove
            # old NULL entries
            gcdata.dead_threads_count += 1
            if (gcdata.dead_threads_count & 511) == 0:
                copy = copy_without_null_values(gcdata.aid2stack)
                gcdata.aid2stack.delete()
                gcdata.aid2stack = copy

        def belongs_to_current_thread(framedata):
            # xxx obscure: the answer is Yes if, as a pointer, framedata
            # lies between the start of the current stack and the top of it.
            stack_start = gcdata.aid2stack.get(get_aid(), llmemory.NULL)
            ll_assert(stack_start != llmemory.NULL,
                      "current thread not found in gcdata.aid2stack!")
            stack_stop = llmemory.cast_int_to_adr(
                             llop.stack_current(lltype.Signed))
            return (stack_start <= framedata <= stack_stop or
                    stack_start >= framedata >= stack_stop)
        self.belongs_to_current_thread = belongs_to_current_thread

        def thread_before_fork():
            # before fork(): collect all ASM_FRAMEDATA structures that do
            # not belong to the current thread, and move them out of the
            # way, i.e. out of the main circular doubly linked list.
            detached_pieces = llmemory.NULL
            anchor = llmemory.cast_ptr_to_adr(gcrootanchor)
            initialframedata = anchor.address[1]
            while initialframedata != anchor:   # while we have not looped back
                if not belongs_to_current_thread(initialframedata):
                    # Unlink it
                    prev = initialframedata.address[0]
                    next = initialframedata.address[1]
                    prev.address[1] = next
                    next.address[0] = prev
                    # Link it to the singly linked list 'detached_pieces'
                    initialframedata.address[0] = detached_pieces
                    detached_pieces = initialframedata
                    rffi.stackcounter.stacks_counter -= 1
                # Then proceed to the next piece of stack
                initialframedata = initialframedata.address[1]
            return detached_pieces

        def thread_after_fork(result_of_fork, detached_pieces):
            if result_of_fork == 0:
                # We are in the child process.  Assumes that only the
                # current thread survived.  All the detached_pieces
                # are pointers in other stacks, so have likely been
                # freed already by the multithreaded library.
                # Nothing more for us to do.
                pass
            else:
                # We are still in the parent process.  The fork() may
                # have succeeded or not, but that's irrelevant here.
                # We need to reattach the detached_pieces now, to the
                # circular doubly linked list at 'gcrootanchor'.  The
                # order is not important.
                anchor = llmemory.cast_ptr_to_adr(gcrootanchor)
                while detached_pieces != llmemory.NULL:
                    reattach = detached_pieces
                    detached_pieces = detached_pieces.address[0]
                    a_next = anchor.address[1]
                    reattach.address[0] = anchor
                    reattach.address[1] = a_next
                    anchor.address[1] = reattach
                    a_next.address[0] = reattach
                    rffi.stackcounter.stacks_counter += 1

        self.thread_setup = thread_setup
        self.thread_start_ptr = getfn(thread_start, [], annmodel.s_None,
                                      inline=True)
        self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None)
        self.thread_before_fork_ptr = getfn(thread_before_fork, [],
                                            SomeAddress())
        self.thread_after_fork_ptr = getfn(thread_after_fork,
                                           [annmodel.SomeInteger(),
                                            SomeAddress()],
                                           annmodel.s_None)
        #
        # check that the order of the need_*() is correct for us: if we
        # need both threads and stacklets, need_thread_support() must be
        # called first, to initialize self.belongs_to_current_thread.
        assert not hasattr(self, 'gc_detach_callback_pieces_ptr')

    def walk_stack_roots(self, collect_stack_root, is_minor=False):
        gcdata = self.gcdata
        gcdata._gc_collect_stack_root = collect_stack_root
        gcdata._gc_collect_is_minor = is_minor
        pypy_asm_stackwalk(llhelper(ASM_CALLBACK_PTR, self._asm_callback),
                           gcrootanchor)

    def walk_stack_from(self):
        curframe = lltype.malloc(WALKFRAME, flavor='raw')
        otherframe = lltype.malloc(WALKFRAME, flavor='raw')

        # Walk over all the pieces of stack.  They are in a circular linked
        # list of structures of 7 words, the 2 first words being prev/next.
        # The anchor of this linked list is:
        anchor = llmemory.cast_ptr_to_adr(gcrootanchor)
        initialframedata = anchor.address[1]
        stackscount = 0
        while initialframedata != anchor:     # while we have not looped back
            self.walk_frames(curframe, otherframe, initialframedata)
            # Then proceed to the next piece of stack
            initialframedata = initialframedata.address[1]
            stackscount += 1
        #
        # for the JIT: rpy_fastgil may contain an extra framedata
        rpy_fastgil = rgil.gil_fetch_fastgil().signed[0]
        if rpy_fastgil != 1:
            ll_assert(rpy_fastgil != 0, "walk_stack_from doesn't have the GIL")
            initialframedata = rffi.cast(llmemory.Address, rpy_fastgil)
            #
            # very rare issue: initialframedata.address[0] is uninitialized
            # in this case, but "retaddr = callee.frame_address.address[0]"
            # reads it.  If it happens to be exactly a valid return address
            # inside the C code, then bad things occur.
            initialframedata.address[0] = llmemory.NULL
            #
            self.walk_frames(curframe, otherframe, initialframedata)
            stackscount += 1
        #
        expected = rffi.stackcounter.stacks_counter
        if NonConstant(0):
            rffi.stackcounter.stacks_counter += 42    # hack to force it
        ll_assert(not (stackscount < expected), "non-closed stacks around")
        ll_assert(not (stackscount > expected), "stacks counter corruption?")
        lltype.free(otherframe, flavor='raw')
        lltype.free(curframe, flavor='raw')

    def walk_frames(self, curframe, otherframe, initialframedata):
        self.fill_initial_frame(curframe, initialframedata)
        # Loop over all the frames in the stack
        while self.walk_to_parent_frame(curframe, otherframe):
            swap = curframe
            curframe = otherframe    # caller becomes callee
            otherframe = swap

    def fill_initial_frame(self, curframe, initialframedata):
        # Read the information provided by initialframedata
        initialframedata += 2*sizeofaddr #skip the prev/next words at the start
        reg = 0
        while reg < CALLEE_SAVED_REGS:
            # NB. 'initialframedata' stores the actual values of the
            # registers %ebx etc., and if these values are modified
            # they are reloaded by pypy_asm_stackwalk().  By contrast,
            # 'regs_stored_at' merely points to the actual values
            # from the 'initialframedata'.
            curframe.regs_stored_at[reg] = initialframedata + reg*sizeofaddr
            reg += 1
        curframe.frame_address = initialframedata.address[CALLEE_SAVED_REGS]

    def walk_to_parent_frame(self, callee, caller):
        """Starting from 'callee', walk the next older frame on the stack
        and fill 'caller' accordingly.  Also invokes the collect_stack_root()
        callback from the GC code for each GC root found in 'caller'.
        """
        #
        # The gcmap table is a list of entries, two machine words each:
        #     void *SafePointAddress;
        #     int Shape;
        #
        # A "safe point" is the return address of a call.
        # The "shape" of a safe point is a list of integers
        # that represent "locations".  A "location" can be
        # either in the stack or in a register.  See
        # getlocation() for the decoding of this integer.
        # The locations stored in a "shape" are as follows:
        #
        #   * The "location" of the return address.  This is just
        #     after the end of the frame of 'callee'; it is the
        #     first word of the frame of 'caller' (see picture
        #     below).
        #
        #   * Four "locations" that specify where the function saves
        #     each of the four callee-saved registers (%ebx, %esi,
        #     %edi, %ebp).
        #
        #   * The number of live GC roots around the call.
        #
        #   * For each GC root, an integer that specify where the
        #     GC pointer is stored.  This is a "location" too.
        #
        # XXX the details are completely specific to X86!!!
        # a picture of the stack may help:
        #                                           ^ ^ ^
        #     |     ...      |                 to older frames
        #     +--------------+
        #     |   ret addr   |  <------ caller_frame (addr of retaddr)
        #     |     ...      |
        #     | caller frame |
        #     |     ...      |
        #     +--------------+
        #     |   ret addr   |  <------ callee_frame (addr of retaddr)
        #     |     ...      |
        #     | callee frame |
        #     |     ...      |                 lower addresses
        #     +--------------+                      v v v
        #

        retaddr = callee.frame_address.address[0]
        #
        # try to locate the caller function based on retaddr.
        # set up self._shape_decompressor.
        #
        ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP].address[0]
        self.locate_caller_based_on_retaddr(retaddr, ebp_in_caller)
        #
        # found!  Enumerate the GC roots in the caller frame
        #
        collect_stack_root = self.gcdata._gc_collect_stack_root
        gc = self.gc
        while True:
            location = self._shape_decompressor.next()
            if location == 0:
                break
            addr = self.getlocation(callee, ebp_in_caller, location)
            if gc.points_to_valid_gc_object(addr):
                collect_stack_root(gc, addr)
        #
        # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
        # us.  We use it to store an "already traced past this frame"
        # flag.
        if self._with_jit and self.gcdata._gc_collect_is_minor:
            if self.mark_jit_frame_can_stop(callee):
                return False
        #
        # track where the caller_frame saved the registers from its own
        # caller
        #
        reg = CALLEE_SAVED_REGS - 1
        while reg >= 0:
            location = self._shape_decompressor.next()
            addr = self.getlocation(callee, ebp_in_caller, location)
            caller.regs_stored_at[reg] = addr
            reg -= 1

        location = self._shape_decompressor.next()
        caller.frame_address = self.getlocation(callee, ebp_in_caller,
                                                location)
        # we get a NULL marker to mean "I'm the frame
        # of the entry point, stop walking"
        return caller.frame_address != llmemory.NULL

    def locate_caller_based_on_retaddr(self, retaddr, ebp_in_caller):
        gcmapstart = llop.gc_asmgcroot_static(llmemory.Address, 0)
        gcmapend   = llop.gc_asmgcroot_static(llmemory.Address, 1)
        item = search_in_gcmap(gcmapstart, gcmapend, retaddr)
        if item:
            self._shape_decompressor.setpos(item.signed[1])
            return

        if not self._shape_decompressor.sorted:
            # the item may have been not found because the main array was
            # not sorted.  Sort it and try again.
            win32_follow_gcmap_jmp(gcmapstart, gcmapend)
            sort_gcmap(gcmapstart, gcmapend)
            self._shape_decompressor.sorted = True
            item = search_in_gcmap(gcmapstart, gcmapend, retaddr)
            if item:
                self._shape_decompressor.setpos(item.signed[1])
                return

        if self._with_jit:
            # item not found.  We assume that it's a JIT-generated
            # location -- but we check for consistency that ebp points
            # to a JITFRAME object.
            from rpython.jit.backend.llsupport.jitframe import STACK_DEPTH_OFS

            tid = self.gc.get_possibly_forwarded_type_id(ebp_in_caller)
            if (rffi.cast(lltype.Signed, tid) ==
                    rffi.cast(lltype.Signed, self.frame_tid)):
                # fish the depth
                extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0]
                ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1))
                           == 0, "asmgcc: misaligned extra_stack_depth")
                extra_stack_depth //= rffi.sizeof(lltype.Signed)
                self._shape_decompressor.setjitframe(extra_stack_depth)
                return
        llop.debug_fatalerror(lltype.Void, "cannot find gc roots!")

    def getlocation(self, callee, ebp_in_caller, location):
        """Get the location in the 'caller' frame of a variable, based
        on the integer 'location' that describes it.  All locations are
        computed based on information saved by the 'callee'.
        """
        ll_assert(location >= 0, "negative location")
        kind = location & LOC_MASK
        offset = location & ~ LOC_MASK
        if IS_64_BITS:
            offset <<= 1
        if kind == LOC_REG:   # register
            if location == LOC_NOWHERE:
                return llmemory.NULL
            reg = (location >> 2) - 1
            ll_assert(reg < CALLEE_SAVED_REGS, "bad register location")
            return callee.regs_stored_at[reg]
        elif kind == LOC_ESP_PLUS:    # in the caller stack frame at N(%esp)
            esp_in_caller = callee.frame_address + sizeofaddr
            return esp_in_caller + offset
        elif kind == LOC_EBP_PLUS:    # in the caller stack frame at N(%ebp)
            return ebp_in_caller + offset
        else:  # kind == LOC_EBP_MINUS:   at -N(%ebp)
            return ebp_in_caller - offset

    def mark_jit_frame_can_stop(self, callee):
        location = self._shape_decompressor.get_threadlocal_loc()
        if location == LOC_NOWHERE:
            return False
        addr = self.getlocation(callee, llmemory.NULL, location)
        #
        x = addr.signed[0]
        if x & 1:
            return True            # this JIT stack frame is already marked!
        else:
            addr.signed[0] = x | 1    # otherwise, mark it but don't stop
            return False


LOC_REG       = 0
LOC_ESP_PLUS  = 1
LOC_EBP_PLUS  = 2
LOC_EBP_MINUS = 3
LOC_MASK      = 0x03
LOC_NOWHERE   = LOC_REG | 0

# ____________________________________________________________

sizeofaddr = llmemory.sizeof(llmemory.Address)
arrayitemsize = 2 * sizeofaddr


def binary_search(start, end, addr1):
    """Search for an element in a sorted array.

    The interval from the start address (included) to the end address
    (excluded) is assumed to be a sorted arrays of pairs (addr1, addr2).
    This searches for the item with a given addr1 and returns its
    address.  If not found exactly, it tries to return the address
    of the item left of addr1 (i.e. such that result.address[0] < addr1).
    """
    count = (end - start) // arrayitemsize
    while count > 1:
        middleindex = count // 2
        middle = start + middleindex * arrayitemsize
        if addr1 < middle.address[0]:
            count = middleindex
        else:
            start = middle
            count -= middleindex
    return start

def search_in_gcmap(gcmapstart, gcmapend, retaddr):
    item = binary_search(gcmapstart, gcmapend, retaddr)
    if item.address[0] == retaddr:
        return item     # found
    # 'retaddr' not exactly found.  Check that 'item' is the start of a
    # compressed range that includes 'retaddr'.
    if retaddr > item.address[0] and item.signed[1] < 0:
        return item     # ok
    else:
        return llmemory.NULL    # failed

def search_in_gcmap2(gcmapstart, gcmapend, retaddr):
    # same as 'search_in_gcmap', but without range checking support
    # (item.signed[1] is an address in this case, not a signed at all!)
    item = binary_search(gcmapstart, gcmapend, retaddr)
    if item.address[0] == retaddr:
        return item.address[1]     # found
    else:
        return llmemory.NULL    # failed

def sort_gcmap(gcmapstart, gcmapend):
    count = (gcmapend - gcmapstart) // arrayitemsize
    qsort(gcmapstart,
          rffi.cast(rffi.SIZE_T, count),
          rffi.cast(rffi.SIZE_T, arrayitemsize),
          c_compare_gcmap_entries)

def replace_dead_entries_with_nulls(start, end):
    # replace the dead entries (null value) with a null key.
    count = (end - start) // arrayitemsize - 1
    while count >= 0:
        item = start + count * arrayitemsize
        if item.address[1] == llmemory.NULL:
            item.address[0] = llmemory.NULL
        count -= 1

if sys.platform == 'win32':
    def win32_follow_gcmap_jmp(start, end):
        # The initial gcmap table contains addresses to a JMP
        # instruction that jumps indirectly to the real code.
        # Replace them with the target addresses.
        assert rffi.SIGNEDP is rffi.LONGP, "win64 support missing"
        while start < end:
            code = rffi.cast(rffi.CCHARP, start.address[0])[0]
            if code == '\xe9': # jmp
                rel32 = rffi.cast(rffi.SIGNEDP, start.address[0]+1)[0]
                target = start.address[0] + (rel32 + 5)
                start.address[0] = target
            start += arrayitemsize
else:
    def win32_follow_gcmap_jmp(start, end):
        pass

# ____________________________________________________________

class ShapeDecompressor:
    _alloc_flavor_ = "raw"

    sorted = False

    def setpos(self, pos):
        if pos < 0:
            pos = ~ pos     # can ignore this "range" marker here
        gccallshapes = llop.gc_asmgcroot_static(llmemory.Address, 2)
        self.addr = gccallshapes + pos
        self.jit_index = -1

    def setjitframe(self, extra_stack_depth):
        self.jit_index = 0
        self.extra_stack_depth = extra_stack_depth

    def next(self):
        index = self.jit_index
        if index < 0:
            # case "outside the jit"
            addr = self.addr
            value = 0
            while True:
                b = ord(addr.char[0])
                addr += 1
                value += b
                if b < 0x80:
                    break
                value = (value - 0x80) << 7
            self.addr = addr
            return value
        else:
            # case "in the jit"
            from rpython.jit.backend.x86.arch import FRAME_FIXED_SIZE
            from rpython.jit.backend.x86.arch import PASS_ON_MY_FRAME
            self.jit_index = index + 1
            if index == 0:
                # the jitframe is an object in EBP
                return LOC_REG | ((INDEX_OF_EBP + 1) << 2)
            if index == 1:
                return 0
            # the remaining returned values should be:
            #      saved %rbp
            #      saved %r15           or on 32bit:
            #      saved %r14             saved %ebp
            #      saved %r13             saved %edi
            #      saved %r12             saved %esi
            #      saved %rbx             saved %ebx
            #      return addr            return addr
            stack_depth = PASS_ON_MY_FRAME + self.extra_stack_depth
            if IS_64_BITS:
                if index == 2:   # rbp
                    return LOC_ESP_PLUS | (stack_depth << 2)
                if index == 3:   # r15
                    return LOC_ESP_PLUS | ((stack_depth + 5) << 2)
                if index == 4:   # r14
                    return LOC_ESP_PLUS | ((stack_depth + 4) << 2)
                if index == 5:   # r13
                    return LOC_ESP_PLUS | ((stack_depth + 3) << 2)
                if index == 6:   # r12
                    return LOC_ESP_PLUS | ((stack_depth + 2) << 2)
                if index == 7:   # rbx
                    return LOC_ESP_PLUS | ((stack_depth + 1) << 2)
                if index == 8:   # return addr
                    return (LOC_ESP_PLUS |
                        ((FRAME_FIXED_SIZE + self.extra_stack_depth) << 2))
            else:
                if index == 2:   # ebp
                    return LOC_ESP_PLUS | (stack_depth << 2)
                if index == 3:   # edi
                    return LOC_ESP_PLUS | ((stack_depth + 3) << 2)
                if index == 4:   # esi
                    return LOC_ESP_PLUS | ((stack_depth + 2) << 2)
                if index == 5:   # ebx
                    return LOC_ESP_PLUS | ((stack_depth + 1) << 2)
                if index == 6:   # return addr
                    return (LOC_ESP_PLUS |
                        ((FRAME_FIXED_SIZE + self.extra_stack_depth) << 2))
            llop.debug_fatalerror(lltype.Void, "asmgcroot: invalid index")
            return 0   # annotator fix

    def get_threadlocal_loc(self):
        index = self.jit_index
        if index < 0:
            return LOC_NOWHERE     # case "outside the jit"
        else:
            # case "in the jit"
            from rpython.jit.backend.x86.arch import THREADLOCAL_OFS, WORD
            return (LOC_ESP_PLUS |
                    ((THREADLOCAL_OFS // WORD + self.extra_stack_depth) << 2))


# ____________________________________________________________

#
# The special pypy_asm_stackwalk(), implemented directly in
# assembler, fills information about the current stack top in an
# ASM_FRAMEDATA array and invokes an RPython callback with it.
# An ASM_FRAMEDATA is an array of 5 values that describe everything
# we need to know about a stack frame:
#
#   - the value that %ebx had when the current function started
#   - the value that %esi had when the current function started
#   - the value that %edi had when the current function started
#   - the value that %ebp had when the current function started
#   - frame address (actually the addr of the retaddr of the current function;
#                    that's the last word of the frame in memory)
#
# On 64 bits, it is an array of 7 values instead of 5:
#
#   - %rbx, %r12, %r13, %r14, %r15, %rbp; and the frame address
#

if IS_64_BITS:
    CALLEE_SAVED_REGS = 6
    INDEX_OF_EBP      = 5
    FRAME_PTR         = CALLEE_SAVED_REGS
else:
    CALLEE_SAVED_REGS = 4       # there are 4 callee-saved registers
    INDEX_OF_EBP      = 3
    FRAME_PTR         = CALLEE_SAVED_REGS    # the frame is at index 4 in the array

JIT_USE_WORDS = 2 + FRAME_PTR + 1

ASM_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([], lltype.Void))

# used internally by walk_stack_from()
WALKFRAME = lltype.Struct('WALKFRAME',
        ('regs_stored_at',    # address of where the registers have been saved
             lltype.FixedSizeArray(llmemory.Address, CALLEE_SAVED_REGS)),
        ('frame_address',
             llmemory.Address),
    )

# We have a circular doubly-linked list of all the ASM_FRAMEDATAs currently
# alive.  The list's starting point is given by 'gcrootanchor', which is not
# a full ASM_FRAMEDATA but only contains the prev/next pointers:
ASM_FRAMEDATA_HEAD_PTR = lltype.Ptr(lltype.ForwardReference())
ASM_FRAMEDATA_HEAD_PTR.TO.become(lltype.Struct('ASM_FRAMEDATA_HEAD',
        ('prev', ASM_FRAMEDATA_HEAD_PTR),
        ('next', ASM_FRAMEDATA_HEAD_PTR)
    ))
gcrootanchor = lltype.malloc(ASM_FRAMEDATA_HEAD_PTR.TO, immortal=True)
gcrootanchor.prev = gcrootanchor
gcrootanchor.next = gcrootanchor
c_gcrootanchor = Constant(gcrootanchor, ASM_FRAMEDATA_HEAD_PTR)

eci = ExternalCompilationInfo(compile_extra=['-DPYPY_USE_ASMGCC'],
                              post_include_bits=["""
static int pypy_compare_gcmap_entries(const void *addr1, const void *addr2)
{
    char *key1 = * (char * const *) addr1;
    char *key2 = * (char * const *) addr2;
    if (key1 < key2)
        return -1;
    else if (key1 == key2)
        return 0;
    else
        return 1;
}
"""])

pypy_asm_stackwalk = rffi.llexternal('pypy_asm_stackwalk',
                                     [ASM_CALLBACK_PTR,
                                      ASM_FRAMEDATA_HEAD_PTR],
                                     lltype.Signed,
                                     sandboxsafe=True,
                                     _nowrapper=True,
                                     random_effects_on_gcobjs=True,
                                     compilation_info=eci)
c_asm_stackwalk = Constant(pypy_asm_stackwalk,
                           lltype.typeOf(pypy_asm_stackwalk))

pypy_asm_gcroot = rffi.llexternal('pypy_asm_gcroot',
                                  [llmemory.Address],
                                  llmemory.Address,
                                  sandboxsafe=True,
                                  _nowrapper=True)
c_asm_gcroot = Constant(pypy_asm_gcroot, lltype.typeOf(pypy_asm_gcroot))

pypy_asm_nocollect = rffi.llexternal('pypy_asm_gc_nocollect',
                                     [rffi.CCHARP], lltype.Void,
                                     sandboxsafe=True,
                                     _nowrapper=True)
c_asm_nocollect = Constant(pypy_asm_nocollect, lltype.typeOf(pypy_asm_nocollect))

QSORT_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address,
                                                 llmemory.Address], rffi.INT))
c_compare_gcmap_entries = rffi.llexternal('pypy_compare_gcmap_entries',
                                          [llmemory.Address, llmemory.Address],
                                          rffi.INT, compilation_info=eci,
                                          _nowrapper=True, sandboxsafe=True)
qsort = rffi.llexternal('qsort',
                        [llmemory.Address,
                         rffi.SIZE_T,
                         rffi.SIZE_T,
                         QSORT_CALLBACK_PTR],
                        lltype.Void,
                        sandboxsafe=True,
                        random_effects_on_gcobjs=False,  # but has a callback
                        _nowrapper=True)