PageRenderTime 80ms CodeModel.GetById 13ms app.highlight 63ms RepoModel.GetById 1ms app.codeStats 0ms

/src/rt/rust_upcall.cpp

http://github.com/jruderman/rust
C++ | 516 lines | 360 code | 78 blank | 78 comment | 6 complexity | c7f97becc0c9b998b1cb2f3b2cc52858 MD5 | raw file
  1/*
  2  Upcalls
  3
  4  These are runtime functions that the compiler knows about and generates
  5  calls to. They are called on the Rust stack and, in most cases, immediately
  6  switch to the C stack.
  7 */
  8
  9#include "rust_globals.h"
 10#include "rust_task.h"
 11#include "rust_cc.h"
 12#include "rust_sched_loop.h"
 13#include "rust_unwind.h"
 14#include "rust_upcall.h"
 15#include "rust_util.h"
 16
 17#ifdef __GNUC__
 18#define LOG_UPCALL_ENTRY(task)                            \
 19    LOG(task, upcall,                                     \
 20        "> UPCALL %s - task: %s 0x%" PRIxPTR              \
 21        " retpc: x%" PRIxPTR,                             \
 22        __FUNCTION__,                                     \
 23        (task)->name, (task),                             \
 24        __builtin_return_address(0));
 25#else
 26#define LOG_UPCALL_ENTRY(task)                            \
 27    LOG(task, upcall, "> UPCALL task: %s @x%" PRIxPTR,    \
 28        (task)->name, (task));
 29#endif
 30
 31#define UPCALL_SWITCH_STACK(T, A, F) \
 32    call_upcall_on_c_stack(T, (void*)A, (void*)F)
 33
 34inline void
 35call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) {
 36    task->call_on_c_stack(args, fn_ptr);
 37}
 38
 39/**********************************************************************
 40 * Switches to the C-stack and invokes |fn_ptr|, passing |args| as argument.
 41 * This is used by the C compiler to call foreign functions and by other
 42 * upcalls to switch to the C stack.  The return value is passed through a
 43 * field in the args parameter. This upcall is specifically for switching
 44 * to the shim functions generated by rustc.
 45 */
 46extern "C" CDECL void
 47upcall_call_shim_on_c_stack(void *args, void *fn_ptr) {
 48    rust_task *task = rust_get_current_task();
 49
 50    try {
 51        task->call_on_c_stack(args, fn_ptr);
 52    } catch (...) {
 53        // Logging here is not reliable
 54        assert(false && "Foreign code threw an exception");
 55    }
 56}
 57
 58/*
 59 * The opposite of above. Starts on a C stack and switches to the Rust
 60 * stack. This is the only upcall that runs from the C stack.
 61 */
 62extern "C" CDECL void
 63upcall_call_shim_on_rust_stack(void *args, void *fn_ptr) {
 64    rust_task *task = rust_get_current_task();
 65
 66    try {
 67        task->call_on_rust_stack(args, fn_ptr);
 68    } catch (...) {
 69        // We can't count on being able to unwind through arbitrary
 70        // code. Our best option is to just fail hard.
 71        // Logging here is not reliable
 72        assert(false && "Rust task failed after reentering the Rust stack");
 73    }
 74}
 75
 76/**********************************************************************/
 77
 78struct s_fail_args {
 79    rust_task *task;
 80    char const *expr;
 81    char const *file;
 82    size_t line;
 83};
 84
 85extern "C" CDECL void
 86upcall_s_fail(s_fail_args *args) {
 87    rust_task *task = args->task;
 88    LOG_UPCALL_ENTRY(task);
 89    task->fail(args->expr, args->file, args->line);
 90}
 91
 92extern "C" CDECL void
 93upcall_fail(char const *expr,
 94            char const *file,
 95            size_t line) {
 96    rust_task *task = rust_get_current_task();
 97    s_fail_args args = {task,expr,file,line};
 98    UPCALL_SWITCH_STACK(task, &args, upcall_s_fail);
 99}
100
101// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
102// autogenerated wrappers for upcall_fail. Remove this when we fully move away
103// away from the C upcall path.
104extern "C" CDECL void
105rust_upcall_fail(char const *expr,
106                 char const *file,
107                 size_t line) {
108    upcall_fail(expr, file, line);
109}
110
111struct s_trace_args {
112    rust_task *task;
113    char const *msg;
114    char const *file;
115    size_t line;
116};
117
118extern "C" CDECL void
119upcall_s_trace(s_trace_args *args) {
120    rust_task *task = args->task;
121    LOG_UPCALL_ENTRY(task);
122    LOG(task, trace, "Trace %s:%d: %s",
123        args->file, args->line, args->msg);
124}
125
126extern "C" CDECL void
127upcall_trace(char const *msg,
128             char const *file,
129             size_t line) {
130    rust_task *task = rust_get_current_task();
131    s_trace_args args = {task,msg,file,line};
132    UPCALL_SWITCH_STACK(task, &args, upcall_s_trace);
133}
134
135/**********************************************************************
136 * Allocate an object in the exchange heap
137 */
138
139struct s_exchange_malloc_args {
140    rust_task *task;
141    uintptr_t retval;
142    type_desc *td;
143    uintptr_t size;
144};
145
146extern "C" CDECL void
147upcall_s_exchange_malloc(s_exchange_malloc_args *args) {
148    rust_task *task = args->task;
149    LOG_UPCALL_ENTRY(task);
150    LOG(task, mem, "upcall exchange malloc(0x%" PRIxPTR ")", args->td);
151
152    size_t total_size = get_box_size(args->size, args->td->align);
153    // FIXME--does this have to be calloc? (Issue #2682)
154    void *p = task->kernel->calloc(total_size, "exchange malloc");
155
156    rust_opaque_box *header = static_cast<rust_opaque_box*>(p);
157    header->ref_count = -1; // This is not ref counted
158    header->td = args->td;
159    header->prev = 0;
160    header->next = 0;
161
162    args->retval = (uintptr_t)header;
163}
164
165extern "C" CDECL uintptr_t
166upcall_exchange_malloc(type_desc *td, uintptr_t size) {
167    rust_task *task = rust_get_current_task();
168    s_exchange_malloc_args args = {task, 0, td, size};
169    UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_malloc);
170    return args.retval;
171}
172
173// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
174// autogenerated wrappers for upcall_exchange_malloc. Remove this when we
175// fully move away away from the C upcall path.
176extern "C" CDECL uintptr_t
177rust_upcall_exchange_malloc(type_desc *td, uintptr_t size) {
178    return upcall_exchange_malloc(td, size);
179}
180
181struct s_exchange_free_args {
182    rust_task *task;
183    void *ptr;
184};
185
186extern "C" CDECL void
187upcall_s_exchange_free(s_exchange_free_args *args) {
188    rust_task *task = args->task;
189    LOG_UPCALL_ENTRY(task);
190    task->kernel->free(args->ptr);
191}
192
193extern "C" CDECL void
194upcall_exchange_free(void *ptr) {
195    rust_task *task = rust_get_current_task();
196    s_exchange_free_args args = {task,ptr};
197    UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_free);
198}
199
200// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
201// autogenerated wrappers for upcall_exchange_free. Remove this when we fully
202// move away away from the C upcall path.
203extern "C" CDECL void
204rust_upcall_exchange_free(void *ptr) {
205    return upcall_exchange_free(ptr);
206}
207
208/**********************************************************************
209 * Allocate an object in the task-local heap.
210 */
211
212struct s_malloc_args {
213    rust_task *task;
214    uintptr_t retval;
215    type_desc *td;
216    uintptr_t size;
217};
218
219extern "C" CDECL void
220upcall_s_malloc(s_malloc_args *args) {
221    rust_task *task = args->task;
222    LOG_UPCALL_ENTRY(task);
223    LOG(task, mem, "upcall malloc(0x%" PRIxPTR ")", args->td);
224
225    cc::maybe_cc(task);
226
227    // FIXME--does this have to be calloc? (Issue #2682)
228    rust_opaque_box *box = task->boxed.calloc(args->td, args->size);
229    void *body = box_body(box);
230
231    debug::maybe_track_origin(task, box);
232
233    LOG(task, mem,
234        "upcall malloc(0x%" PRIxPTR ") = box 0x%" PRIxPTR
235        " with body 0x%" PRIxPTR,
236        args->td, (uintptr_t)box, (uintptr_t)body);
237
238    args->retval = (uintptr_t)box;
239}
240
241extern "C" CDECL uintptr_t
242upcall_malloc(type_desc *td, uintptr_t size) {
243    rust_task *task = rust_get_current_task();
244    s_malloc_args args = {task, 0, td, size};
245    UPCALL_SWITCH_STACK(task, &args, upcall_s_malloc);
246    return args.retval;
247}
248
249// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
250// autogenerated wrappers for upcall_malloc. Remove this when we fully move
251// away away from the C upcall path.
252extern "C" CDECL uintptr_t
253rust_upcall_malloc(type_desc *td, uintptr_t size) {
254    return upcall_malloc(td, size);
255}
256
257/**********************************************************************
258 * Called whenever an object in the task-local heap is freed.
259 */
260
261struct s_free_args {
262    rust_task *task;
263    void *ptr;
264};
265
266extern "C" CDECL void
267upcall_s_free(s_free_args *args) {
268    rust_task *task = args->task;
269    LOG_UPCALL_ENTRY(task);
270
271    rust_sched_loop *sched_loop = task->sched_loop;
272    DLOG(sched_loop, mem,
273             "upcall free(0x%" PRIxPTR ", is_gc=%" PRIdPTR ")",
274             (uintptr_t)args->ptr);
275
276    debug::maybe_untrack_origin(task, args->ptr);
277
278    rust_opaque_box *box = (rust_opaque_box*) args->ptr;
279    task->boxed.free(box);
280}
281
282extern "C" CDECL void
283upcall_free(void* ptr) {
284    rust_task *task = rust_get_current_task();
285    s_free_args args = {task,ptr};
286    UPCALL_SWITCH_STACK(task, &args, upcall_s_free);
287}
288
289// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
290// autogenerated wrappers for upcall_free. Remove this when we fully move away
291// away from the C upcall path.
292extern "C" CDECL void
293rust_upcall_free(void* ptr) {
294    upcall_free(ptr);
295}
296
297/**********************************************************************
298 * Sanity checks on boxes, insert when debugging possible
299 * use-after-free bugs.  See maybe_validate_box() in trans.rs.
300 */
301
302extern "C" CDECL void
303upcall_validate_box(rust_opaque_box* ptr) {
304    if (ptr) {
305        assert(ptr->ref_count > 0);
306        assert(ptr->td != NULL);
307        assert(ptr->td->align <= 8);
308        assert(ptr->td->size <= 4096); // might not really be true...
309    }
310}
311
312/**********************************************************************/
313
314struct s_str_new_uniq_args {
315    rust_task *task;
316    const char *cstr;
317    size_t len;
318    rust_str *retval;
319};
320
321extern "C" CDECL void
322upcall_s_str_new_uniq(s_str_new_uniq_args *args) {
323    rust_task *task = args->task;
324    LOG_UPCALL_ENTRY(task);
325    args->retval = make_str(task->kernel, args->cstr, args->len,
326                            "str_new_uniq");
327}
328
329extern "C" CDECL rust_str*
330upcall_str_new_uniq(const char *cstr, size_t len) {
331    rust_task *task = rust_get_current_task();
332    s_str_new_uniq_args args = { task, cstr, len, 0 };
333    UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_uniq);
334    return args.retval;
335}
336
337extern "C" CDECL rust_str*
338upcall_str_new(const char *cstr, size_t len) {
339    rust_task *task = rust_get_current_task();
340    s_str_new_uniq_args args = { task, cstr, len, 0 };
341    UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_uniq);
342    return args.retval;
343}
344
345
346
347struct s_str_new_shared_args {
348    rust_task *task;
349    const char *cstr;
350    size_t len;
351    rust_opaque_box *retval;
352};
353
354extern "C" CDECL void
355upcall_s_str_new_shared(s_str_new_shared_args *args) {
356    rust_task *task = args->task;
357    LOG_UPCALL_ENTRY(task);
358
359    size_t str_fill = args->len + 1;
360    size_t str_alloc = str_fill;
361    args->retval = (rust_opaque_box *)
362        task->boxed.malloc(&str_body_tydesc,
363                           str_fill + sizeof(rust_vec));
364    rust_str *str = (rust_str *)args->retval;
365    str->body.fill = str_fill;
366    str->body.alloc = str_alloc;
367    memcpy(&str->body.data, args->cstr, args->len);
368    str->body.data[args->len] = '\0';
369}
370
371extern "C" CDECL rust_opaque_box*
372upcall_str_new_shared(const char *cstr, size_t len) {
373    rust_task *task = rust_get_current_task();
374    s_str_new_shared_args args = { task, cstr, len, 0 };
375    UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_shared);
376    return args.retval;
377}
378
379
380extern "C" _Unwind_Reason_Code
381__gxx_personality_v0(int version,
382                     _Unwind_Action actions,
383                     uint64_t exception_class,
384                     _Unwind_Exception *ue_header,
385                     _Unwind_Context *context);
386
387struct s_rust_personality_args {
388    _Unwind_Reason_Code retval;
389    int version;
390    _Unwind_Action actions;
391    uint64_t exception_class;
392    _Unwind_Exception *ue_header;
393    _Unwind_Context *context;
394};
395
396extern "C" void
397upcall_s_rust_personality(s_rust_personality_args *args) {
398    args->retval = __gxx_personality_v0(args->version,
399                                        args->actions,
400                                        args->exception_class,
401                                        args->ue_header,
402                                        args->context);
403}
404
405/**
406   The exception handling personality function. It figures
407   out what to do with each landing pad. Just a stack-switching
408   wrapper around the C++ personality function.
409*/
410extern "C" _Unwind_Reason_Code
411upcall_rust_personality(int version,
412                        _Unwind_Action actions,
413                        uint64_t exception_class,
414                        _Unwind_Exception *ue_header,
415                        _Unwind_Context *context) {
416    s_rust_personality_args args = {(_Unwind_Reason_Code)0,
417                                    version, actions, exception_class,
418                                    ue_header, context};
419    rust_task *task = rust_get_current_task();
420
421    // The personality function is run on the stack of the
422    // last function that threw or landed, which is going
423    // to sometimes be the C stack. If we're on the Rust stack
424    // then switch to the C stack.
425
426    if (task->on_rust_stack()) {
427        UPCALL_SWITCH_STACK(task, &args, upcall_s_rust_personality);
428    } else {
429        upcall_s_rust_personality(&args);
430    }
431    return args.retval;
432}
433
434extern "C" void
435shape_cmp_type(int8_t *result, const type_desc *tydesc,
436               uint8_t *data_0, uint8_t *data_1, uint8_t cmp_type);
437
438struct s_cmp_type_args {
439    int8_t *result;
440    const type_desc *tydesc;
441    uint8_t *data_0;
442    uint8_t *data_1;
443    uint8_t cmp_type;
444};
445
446extern "C" void
447upcall_s_cmp_type(s_cmp_type_args *args) {
448    shape_cmp_type(args->result, args->tydesc,
449                   args->data_0, args->data_1, args->cmp_type);
450}
451
452extern "C" void
453upcall_cmp_type(int8_t *result, const type_desc *tydesc,
454                uint8_t *data_0, uint8_t *data_1, uint8_t cmp_type) {
455    rust_task *task = rust_get_current_task();
456    s_cmp_type_args args = {result, tydesc,
457                            data_0, data_1, cmp_type};
458    UPCALL_SWITCH_STACK(task, &args, upcall_s_cmp_type);
459}
460
461extern "C" void
462shape_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level);
463
464struct s_log_type_args {
465    const type_desc *tydesc;
466    uint8_t *data;
467    uint32_t level;
468};
469
470extern "C" void
471upcall_s_log_type(s_log_type_args *args) {
472    shape_log_type(args->tydesc, args->data, args->level);
473}
474
475extern "C" void
476upcall_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level) {
477    rust_task *task = rust_get_current_task();
478    s_log_type_args args = {tydesc, data, level};
479    UPCALL_SWITCH_STACK(task, &args, upcall_s_log_type);
480}
481
482// NB: This needs to be blazing fast. Don't switch stacks
483extern "C" CDECL void *
484upcall_new_stack(size_t stk_sz, void *args_addr, size_t args_sz) {
485    rust_task *task = rust_get_current_task();
486    return task->next_stack(stk_sz,
487                            args_addr,
488                            args_sz);
489}
490
491// NB: This needs to be blazing fast. Don't switch stacks
492extern "C" CDECL void
493upcall_del_stack() {
494    rust_task *task = rust_get_current_task();
495    task->prev_stack();
496}
497
498// Landing pads need to call this to insert the
499// correct limit into TLS.
500// NB: This must run on the Rust stack because it
501// needs to acquire the value of the stack pointer
502extern "C" CDECL void
503upcall_reset_stack_limit() {
504    rust_task *task = rust_get_current_task();
505    task->reset_stack_limit();
506}
507
508//
509// Local Variables:
510// mode: C++
511// fill-column: 78;
512// indent-tabs-mode: nil
513// c-basic-offset: 4
514// buffer-file-coding-system: utf-8-unix
515// End:
516//