pypy /rpython/rlib/rthread.py

Language Python Lines 447
MD5 Hash d57f9c03382b9b0af0981d58298ffdec Estimated Cost $7,872 (why?)
Repository https://bitbucket.org/pypy/pypy/ View Raw File View Project SPDX
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.translator import cdir
import py, sys
from rpython.rlib import jit, rgc
from rpython.rlib.debug import ll_assert
from rpython.rlib.objectmodel import we_are_translated, specialize
from rpython.rlib.objectmodel import CDefinedIntSymbolic
from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rtyper.tool import rffi_platform
from rpython.rtyper.extregistry import ExtRegistryEntry

class RThreadError(Exception):
    pass
error = RThreadError

translator_c_dir = py.path.local(cdir)

eci = ExternalCompilationInfo(
    includes = ['src/thread.h'],
    separate_module_files = [translator_c_dir / 'src' / 'thread.c'],
    include_dirs = [translator_c_dir],
)

def llexternal(name, args, result, **kwds):
    kwds.setdefault('sandboxsafe', True)
    return rffi.llexternal(name, args, result, compilation_info=eci,
                           **kwds)

def _emulated_start_new_thread(func):
    "NOT_RPYTHON"
    import thread
    try:
        ident = thread.start_new_thread(func, ())
    except thread.error:
        ident = -1
    return rffi.cast(rffi.LONG, ident)

CALLBACK = lltype.Ptr(lltype.FuncType([], lltype.Void))
c_thread_start = llexternal('RPyThreadStart', [CALLBACK], rffi.LONG,
                            _callable=_emulated_start_new_thread,
                            releasegil=True)  # release the GIL, but most
                                              # importantly, reacquire it
                                              # around the callback

TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock',
                          compilation_info=eci)
TLOCKP_SIZE = rffi_platform.sizeof('struct RPyOpaque_ThreadLock', eci)
c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], rffi.INT,
                                releasegil=False)   # may add in a global list
c_thread_lock_dealloc_NOAUTO = llexternal('RPyOpaqueDealloc_ThreadLock',
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT],
                                  rffi.INT,
                                  releasegil=True)    # release the GIL
c_thread_acquirelock_timed = llexternal('RPyThreadAcquireLockTimed',
                                        [TLOCKP, rffi.LONGLONG, rffi.INT],
                                        rffi.INT,
                                        releasegil=True)    # release the GIL
c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP],
                                  lltype.Signed,
                                  _nowrapper=True)   # *don't* release the GIL

# another set of functions, this time in versions that don't cause the
# GIL to be released.  Used to be there to handle the GIL lock itself,
# but that was changed (see rgil.py).  Now here for performance only.
c_thread_acquirelock_NOAUTO = llexternal('RPyThreadAcquireLock',
                                         [TLOCKP, rffi.INT], rffi.INT,
                                         _nowrapper=True)
c_thread_acquirelock_timed_NOAUTO = llexternal('RPyThreadAcquireLockTimed',
                                         [TLOCKP, rffi.LONGLONG, rffi.INT],
                                         rffi.INT, _nowrapper=True)
c_thread_releaselock_NOAUTO = c_thread_releaselock


def allocate_lock():
    return Lock(allocate_ll_lock())

@specialize.arg(0)
def ll_start_new_thread(func):
    from rpython.rlib import rgil
    _check_thread_enabled()
    rgil.allocate()
    # ^^^ convenience: any RPython program which uses explicitly
    # rthread.start_new_thread() will initialize the GIL at that
    # point.
    ident = c_thread_start(func)
    if ident == -1:
        raise error("can't start new thread")
    return ident

# wrappers...

def get_ident():
    if we_are_translated():
        return tlfield_thread_ident.getraw()
    else:
        import thread
        return thread.get_ident()

def get_or_make_ident():
    if we_are_translated():
        return tlfield_thread_ident.get_or_make_raw()
    else:
        import thread
        return thread.get_ident()

@specialize.arg(0)
def start_new_thread(x, y):
    """In RPython, no argument can be passed.  You have to use global
    variables to pass information to the new thread.  That's not very
    nice, but at least it avoids some levels of GC issues.
    """
    assert len(y) == 0
    return rffi.cast(lltype.Signed, ll_start_new_thread(x))

class DummyLock(object):
    def acquire(self, flag):
        return True

    def release(self):
        pass

    def _freeze_(self):
        return True

    def __enter__(self):
        pass

    def __exit__(self, *args):
        pass

dummy_lock = DummyLock()

class Lock(object):
    """ Container for low-level implementation
    of a lock object
    """
    _immutable_fields_ = ["_lock"]

    def __init__(self, ll_lock):
        self._lock = ll_lock

    def acquire(self, flag):
        if flag:
            c_thread_acquirelock(self._lock, 1)
            return True
        else:
            res = c_thread_acquirelock_timed_NOAUTO(
                self._lock,
                rffi.cast(rffi.LONGLONG, 0),
                rffi.cast(rffi.INT, 0))
            res = rffi.cast(lltype.Signed, res)
            return bool(res)

    def acquire_timed(self, timeout):
        """Timeout is in microseconds.  Returns 0 in case of failure,
        1 in case it works, 2 if interrupted by a signal."""
        res = c_thread_acquirelock_timed(self._lock, timeout, 1)
        res = rffi.cast(lltype.Signed, res)
        return res

    def release(self):
        if c_thread_releaselock(self._lock) != 0:
            raise error("the lock was not previously acquired")

    def __del__(self):
        if free_ll_lock is None:  # happens when tests are shutting down
            return
        free_ll_lock(self._lock)

    def __enter__(self):
        self.acquire(True)

    def __exit__(self, *args):
        self.release()

    def _cleanup_(self):
        raise Exception("seeing a prebuilt rpython.rlib.rthread.Lock instance")

def _check_thread_enabled():
    pass
class Entry(ExtRegistryEntry):
    _about_ = _check_thread_enabled
    def compute_result_annotation(self):
        translator = self.bookkeeper.annotator.translator
        if not translator.config.translation.thread:
            raise Exception(
                "this RPython program uses threads: translate with '--thread'")
    def specialize_call(self, hop):
        hop.exception_cannot_occur()

# ____________________________________________________________
#
# Stack size

get_stacksize = llexternal('RPyThreadGetStackSize', [], lltype.Signed)
set_stacksize = llexternal('RPyThreadSetStackSize', [lltype.Signed],
                           lltype.Signed)

# ____________________________________________________________
#
# Hack

thread_after_fork = llexternal('RPyThreadAfterFork', [], lltype.Void)

# ____________________________________________________________
#
# GIL support wrappers

null_ll_lock = lltype.nullptr(TLOCKP.TO)

def allocate_ll_lock():
    # track_allocation=False here; be careful to lltype.free() it.  The
    # reason it is set to False is that we get it from all app-level
    # lock objects, as well as from the GIL, which exists at shutdown.
    ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw', track_allocation=False)
    res = c_thread_lock_init(ll_lock)
    if rffi.cast(lltype.Signed, res) <= 0:
        lltype.free(ll_lock, flavor='raw', track_allocation=False)
        raise error("out of resources")
    # Add some memory pressure for the size of the lock because it is an
    # Opaque object
    rgc.add_memory_pressure(TLOCKP_SIZE)
    return ll_lock

def free_ll_lock(ll_lock):
    acquire_NOAUTO(ll_lock, False)
    release_NOAUTO(ll_lock)
    c_thread_lock_dealloc_NOAUTO(ll_lock)
    lltype.free(ll_lock, flavor='raw', track_allocation=False)

def acquire_NOAUTO(ll_lock, flag):
    flag = rffi.cast(rffi.INT, int(flag))
    res = c_thread_acquirelock_NOAUTO(ll_lock, flag)
    res = rffi.cast(lltype.Signed, res)
    return bool(res)

def release_NOAUTO(ll_lock):
    if not we_are_translated():
        ll_assert(not acquire_NOAUTO(ll_lock, False), "NOAUTO lock not held!")
    c_thread_releaselock_NOAUTO(ll_lock)

# ____________________________________________________________
#
# Thread integration.
# These are five completely ad-hoc operations at the moment.

@jit.dont_look_inside
def gc_thread_run():
    """To call whenever the current thread (re-)acquired the GIL.
    """
    if we_are_translated():
        llop.gc_thread_run(lltype.Void)
gc_thread_run._always_inline_ = True

@jit.dont_look_inside
def gc_thread_start():
    """To call at the beginning of a new thread.
    """
    if we_are_translated():
        llop.gc_thread_start(lltype.Void)

@jit.dont_look_inside
def gc_thread_die():
    """To call just before the final GIL release done by a dying
    thread.  After a thread_die(), no more gc operation should
    occur in this thread.
    """
    if we_are_translated():
        llop.gc_thread_die(lltype.Void)
gc_thread_die._always_inline_ = True

@jit.dont_look_inside
def gc_thread_before_fork():
    """To call just before fork().  Prepares for forking, after
    which only the current thread will be alive.
    """
    if we_are_translated():
        return llop.gc_thread_before_fork(llmemory.Address)
    else:
        return llmemory.NULL

@jit.dont_look_inside
def gc_thread_after_fork(result_of_fork, opaqueaddr):
    """To call just after fork().
    """
    if we_are_translated():
        llop.gc_thread_after_fork(lltype.Void, result_of_fork, opaqueaddr)
    else:
        assert opaqueaddr == llmemory.NULL

# ____________________________________________________________
#
# Thread-locals.


class ThreadLocalField(object):
    def __init__(self, FIELDTYPE, fieldname, loop_invariant=False):
        "NOT_RPYTHON: must be prebuilt"
        try:
            from thread import _local
        except ImportError:
            class _local(object):
                pass
        self.FIELDTYPE = FIELDTYPE
        self.fieldname = fieldname
        self.local = _local()      # <- NOT_RPYTHON
        zero = rffi.cast(FIELDTYPE, 0)
        offset = CDefinedIntSymbolic('RPY_TLOFS_%s' % self.fieldname,
                                     default='?')
        offset.loop_invariant = loop_invariant
        self._offset = offset

        def getraw():
            if we_are_translated():
                _threadlocalref_seeme(self)
                return llop.threadlocalref_get(FIELDTYPE, offset)
            else:
                return getattr(self.local, 'rawvalue', zero)

        @jit.dont_look_inside
        def get_or_make_raw():
            if we_are_translated():
                _threadlocalref_seeme(self)
                addr = llop.threadlocalref_addr(llmemory.Address)
                return llop.raw_load(FIELDTYPE, addr, offset)
            else:
                return getattr(self.local, 'rawvalue', zero)

        @jit.dont_look_inside
        def setraw(value):
            if we_are_translated():
                _threadlocalref_seeme(self)
                addr = llop.threadlocalref_addr(llmemory.Address)
                llop.raw_store(lltype.Void, addr, offset, value)
            else:
                self.local.rawvalue = value

        def getoffset():
            _threadlocalref_seeme(self)
            return offset

        self.getraw = getraw
        self.get_or_make_raw = get_or_make_raw
        self.setraw = setraw
        self.getoffset = getoffset

    def _freeze_(self):
        return True


class ThreadLocalReference(ThreadLocalField):
    # A thread-local that points to an object.  The object stored in such
    # a thread-local is kept alive as long as the thread is not finished
    # (but only with our own GCs!  it seems not to work with Boehm...)
    # (also, on Windows, if you're not making a DLL but an EXE, it will
    # leak the objects when a thread finishes; see threadlocal.c.)
    _COUNT = 1

    def __init__(self, Cls, loop_invariant=False):
        "NOT_RPYTHON: must be prebuilt"
        self.Cls = Cls
        unique_id = ThreadLocalReference._COUNT
        ThreadLocalReference._COUNT += 1
        ThreadLocalField.__init__(self, lltype.Signed, 'tlref%d' % unique_id,
                                  loop_invariant=loop_invariant)
        setraw = self.setraw
        offset = self._offset

        def get():
            if we_are_translated():
                from rpython.rtyper import rclass
                from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
                _threadlocalref_seeme(self)
                ptr = llop.threadlocalref_get(rclass.OBJECTPTR, offset)
                return cast_base_ptr_to_instance(Cls, ptr)
            else:
                return getattr(self.local, 'value', None)

        @jit.dont_look_inside
        def set(value):
            assert isinstance(value, Cls) or value is None
            if we_are_translated():
                from rpython.rtyper.annlowlevel import cast_instance_to_gcref
                gcref = cast_instance_to_gcref(value)
                value = lltype.cast_ptr_to_int(gcref)
                setraw(value)
                rgc.register_custom_trace_hook(TRACETLREF, _lambda_trace_tlref)
                rgc.ll_writebarrier(_tracetlref_obj)
            else:
                self.local.value = value

        self.get = get
        self.set = set

        def _trace_tlref(gc, obj, callback, arg):
            p = llmemory.NULL
            llop.threadlocalref_acquire(lltype.Void)
            while True:
                p = llop.threadlocalref_enum(llmemory.Address, p)
                if not p:
                    break
                gc._trace_callback(callback, arg, p + offset)
            llop.threadlocalref_release(lltype.Void)
        _lambda_trace_tlref = lambda: _trace_tlref
        TRACETLREF = lltype.GcStruct('TRACETLREF')
        _tracetlref_obj = lltype.malloc(TRACETLREF, immortal=True)

    @staticmethod
    def automatic_keepalive(config):
        """Returns True if translated with a GC that keeps alive
        the set() value until the end of the thread.  Returns False
        if you need to keep it alive yourself (but in that case, you
        should also reset it to None before the thread finishes).
        """
        return (config.translation.gctransformer == "framework" and
                # see translator/c/src/threadlocal.c for the following line
                (not _win32 or config.translation.shared))


tlfield_thread_ident = ThreadLocalField(lltype.Signed, "thread_ident",
                                        loop_invariant=True)
tlfield_p_errno = ThreadLocalField(rffi.CArrayPtr(rffi.INT), "p_errno",
                                   loop_invariant=True)
tlfield_rpy_errno = ThreadLocalField(rffi.INT, "rpy_errno")
tlfield_alt_errno = ThreadLocalField(rffi.INT, "alt_errno")
_win32 = (sys.platform == "win32")
if _win32:
    from rpython.rlib import rwin32
    tlfield_rpy_lasterror = ThreadLocalField(rwin32.DWORD, "rpy_lasterror")
    tlfield_alt_lasterror = ThreadLocalField(rwin32.DWORD, "alt_lasterror")

def _threadlocalref_seeme(field):
    "NOT_RPYTHON"

class _Entry(ExtRegistryEntry):
    _about_ = _threadlocalref_seeme

    def compute_result_annotation(self, s_field):
        field = s_field.const
        self.bookkeeper.thread_local_fields.add(field)

    def specialize_call(self, hop):
        hop.exception_cannot_occur()
Back to Top