/src/rt/rust_upcall.cpp
C++ | 516 lines | 360 code | 78 blank | 78 comment | 6 complexity | c7f97becc0c9b998b1cb2f3b2cc52858 MD5 | raw file
1/* 2 Upcalls 3 4 These are runtime functions that the compiler knows about and generates 5 calls to. They are called on the Rust stack and, in most cases, immediately 6 switch to the C stack. 7 */ 8 9#include "rust_globals.h" 10#include "rust_task.h" 11#include "rust_cc.h" 12#include "rust_sched_loop.h" 13#include "rust_unwind.h" 14#include "rust_upcall.h" 15#include "rust_util.h" 16 17#ifdef __GNUC__ 18#define LOG_UPCALL_ENTRY(task) \ 19 LOG(task, upcall, \ 20 "> UPCALL %s - task: %s 0x%" PRIxPTR \ 21 " retpc: x%" PRIxPTR, \ 22 __FUNCTION__, \ 23 (task)->name, (task), \ 24 __builtin_return_address(0)); 25#else 26#define LOG_UPCALL_ENTRY(task) \ 27 LOG(task, upcall, "> UPCALL task: %s @x%" PRIxPTR, \ 28 (task)->name, (task)); 29#endif 30 31#define UPCALL_SWITCH_STACK(T, A, F) \ 32 call_upcall_on_c_stack(T, (void*)A, (void*)F) 33 34inline void 35call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) { 36 task->call_on_c_stack(args, fn_ptr); 37} 38 39/********************************************************************** 40 * Switches to the C-stack and invokes |fn_ptr|, passing |args| as argument. 41 * This is used by the C compiler to call foreign functions and by other 42 * upcalls to switch to the C stack. The return value is passed through a 43 * field in the args parameter. This upcall is specifically for switching 44 * to the shim functions generated by rustc. 45 */ 46extern "C" CDECL void 47upcall_call_shim_on_c_stack(void *args, void *fn_ptr) { 48 rust_task *task = rust_get_current_task(); 49 50 try { 51 task->call_on_c_stack(args, fn_ptr); 52 } catch (...) { 53 // Logging here is not reliable 54 assert(false && "Foreign code threw an exception"); 55 } 56} 57 58/* 59 * The opposite of above. Starts on a C stack and switches to the Rust 60 * stack. This is the only upcall that runs from the C stack. 61 */ 62extern "C" CDECL void 63upcall_call_shim_on_rust_stack(void *args, void *fn_ptr) { 64 rust_task *task = rust_get_current_task(); 65 66 try { 67 task->call_on_rust_stack(args, fn_ptr); 68 } catch (...) { 69 // We can't count on being able to unwind through arbitrary 70 // code. Our best option is to just fail hard. 71 // Logging here is not reliable 72 assert(false && "Rust task failed after reentering the Rust stack"); 73 } 74} 75 76/**********************************************************************/ 77 78struct s_fail_args { 79 rust_task *task; 80 char const *expr; 81 char const *file; 82 size_t line; 83}; 84 85extern "C" CDECL void 86upcall_s_fail(s_fail_args *args) { 87 rust_task *task = args->task; 88 LOG_UPCALL_ENTRY(task); 89 task->fail(args->expr, args->file, args->line); 90} 91 92extern "C" CDECL void 93upcall_fail(char const *expr, 94 char const *file, 95 size_t line) { 96 rust_task *task = rust_get_current_task(); 97 s_fail_args args = {task,expr,file,line}; 98 UPCALL_SWITCH_STACK(task, &args, upcall_s_fail); 99} 100 101// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with 102// autogenerated wrappers for upcall_fail. Remove this when we fully move away 103// away from the C upcall path. 104extern "C" CDECL void 105rust_upcall_fail(char const *expr, 106 char const *file, 107 size_t line) { 108 upcall_fail(expr, file, line); 109} 110 111struct s_trace_args { 112 rust_task *task; 113 char const *msg; 114 char const *file; 115 size_t line; 116}; 117 118extern "C" CDECL void 119upcall_s_trace(s_trace_args *args) { 120 rust_task *task = args->task; 121 LOG_UPCALL_ENTRY(task); 122 LOG(task, trace, "Trace %s:%d: %s", 123 args->file, args->line, args->msg); 124} 125 126extern "C" CDECL void 127upcall_trace(char const *msg, 128 char const *file, 129 size_t line) { 130 rust_task *task = rust_get_current_task(); 131 s_trace_args args = {task,msg,file,line}; 132 UPCALL_SWITCH_STACK(task, &args, upcall_s_trace); 133} 134 135/********************************************************************** 136 * Allocate an object in the exchange heap 137 */ 138 139struct s_exchange_malloc_args { 140 rust_task *task; 141 uintptr_t retval; 142 type_desc *td; 143 uintptr_t size; 144}; 145 146extern "C" CDECL void 147upcall_s_exchange_malloc(s_exchange_malloc_args *args) { 148 rust_task *task = args->task; 149 LOG_UPCALL_ENTRY(task); 150 LOG(task, mem, "upcall exchange malloc(0x%" PRIxPTR ")", args->td); 151 152 size_t total_size = get_box_size(args->size, args->td->align); 153 // FIXME--does this have to be calloc? (Issue #2682) 154 void *p = task->kernel->calloc(total_size, "exchange malloc"); 155 156 rust_opaque_box *header = static_cast<rust_opaque_box*>(p); 157 header->ref_count = -1; // This is not ref counted 158 header->td = args->td; 159 header->prev = 0; 160 header->next = 0; 161 162 args->retval = (uintptr_t)header; 163} 164 165extern "C" CDECL uintptr_t 166upcall_exchange_malloc(type_desc *td, uintptr_t size) { 167 rust_task *task = rust_get_current_task(); 168 s_exchange_malloc_args args = {task, 0, td, size}; 169 UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_malloc); 170 return args.retval; 171} 172 173// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with 174// autogenerated wrappers for upcall_exchange_malloc. Remove this when we 175// fully move away away from the C upcall path. 176extern "C" CDECL uintptr_t 177rust_upcall_exchange_malloc(type_desc *td, uintptr_t size) { 178 return upcall_exchange_malloc(td, size); 179} 180 181struct s_exchange_free_args { 182 rust_task *task; 183 void *ptr; 184}; 185 186extern "C" CDECL void 187upcall_s_exchange_free(s_exchange_free_args *args) { 188 rust_task *task = args->task; 189 LOG_UPCALL_ENTRY(task); 190 task->kernel->free(args->ptr); 191} 192 193extern "C" CDECL void 194upcall_exchange_free(void *ptr) { 195 rust_task *task = rust_get_current_task(); 196 s_exchange_free_args args = {task,ptr}; 197 UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_free); 198} 199 200// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with 201// autogenerated wrappers for upcall_exchange_free. Remove this when we fully 202// move away away from the C upcall path. 203extern "C" CDECL void 204rust_upcall_exchange_free(void *ptr) { 205 return upcall_exchange_free(ptr); 206} 207 208/********************************************************************** 209 * Allocate an object in the task-local heap. 210 */ 211 212struct s_malloc_args { 213 rust_task *task; 214 uintptr_t retval; 215 type_desc *td; 216 uintptr_t size; 217}; 218 219extern "C" CDECL void 220upcall_s_malloc(s_malloc_args *args) { 221 rust_task *task = args->task; 222 LOG_UPCALL_ENTRY(task); 223 LOG(task, mem, "upcall malloc(0x%" PRIxPTR ")", args->td); 224 225 cc::maybe_cc(task); 226 227 // FIXME--does this have to be calloc? (Issue #2682) 228 rust_opaque_box *box = task->boxed.calloc(args->td, args->size); 229 void *body = box_body(box); 230 231 debug::maybe_track_origin(task, box); 232 233 LOG(task, mem, 234 "upcall malloc(0x%" PRIxPTR ") = box 0x%" PRIxPTR 235 " with body 0x%" PRIxPTR, 236 args->td, (uintptr_t)box, (uintptr_t)body); 237 238 args->retval = (uintptr_t)box; 239} 240 241extern "C" CDECL uintptr_t 242upcall_malloc(type_desc *td, uintptr_t size) { 243 rust_task *task = rust_get_current_task(); 244 s_malloc_args args = {task, 0, td, size}; 245 UPCALL_SWITCH_STACK(task, &args, upcall_s_malloc); 246 return args.retval; 247} 248 249// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with 250// autogenerated wrappers for upcall_malloc. Remove this when we fully move 251// away away from the C upcall path. 252extern "C" CDECL uintptr_t 253rust_upcall_malloc(type_desc *td, uintptr_t size) { 254 return upcall_malloc(td, size); 255} 256 257/********************************************************************** 258 * Called whenever an object in the task-local heap is freed. 259 */ 260 261struct s_free_args { 262 rust_task *task; 263 void *ptr; 264}; 265 266extern "C" CDECL void 267upcall_s_free(s_free_args *args) { 268 rust_task *task = args->task; 269 LOG_UPCALL_ENTRY(task); 270 271 rust_sched_loop *sched_loop = task->sched_loop; 272 DLOG(sched_loop, mem, 273 "upcall free(0x%" PRIxPTR ", is_gc=%" PRIdPTR ")", 274 (uintptr_t)args->ptr); 275 276 debug::maybe_untrack_origin(task, args->ptr); 277 278 rust_opaque_box *box = (rust_opaque_box*) args->ptr; 279 task->boxed.free(box); 280} 281 282extern "C" CDECL void 283upcall_free(void* ptr) { 284 rust_task *task = rust_get_current_task(); 285 s_free_args args = {task,ptr}; 286 UPCALL_SWITCH_STACK(task, &args, upcall_s_free); 287} 288 289// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with 290// autogenerated wrappers for upcall_free. Remove this when we fully move away 291// away from the C upcall path. 292extern "C" CDECL void 293rust_upcall_free(void* ptr) { 294 upcall_free(ptr); 295} 296 297/********************************************************************** 298 * Sanity checks on boxes, insert when debugging possible 299 * use-after-free bugs. See maybe_validate_box() in trans.rs. 300 */ 301 302extern "C" CDECL void 303upcall_validate_box(rust_opaque_box* ptr) { 304 if (ptr) { 305 assert(ptr->ref_count > 0); 306 assert(ptr->td != NULL); 307 assert(ptr->td->align <= 8); 308 assert(ptr->td->size <= 4096); // might not really be true... 309 } 310} 311 312/**********************************************************************/ 313 314struct s_str_new_uniq_args { 315 rust_task *task; 316 const char *cstr; 317 size_t len; 318 rust_str *retval; 319}; 320 321extern "C" CDECL void 322upcall_s_str_new_uniq(s_str_new_uniq_args *args) { 323 rust_task *task = args->task; 324 LOG_UPCALL_ENTRY(task); 325 args->retval = make_str(task->kernel, args->cstr, args->len, 326 "str_new_uniq"); 327} 328 329extern "C" CDECL rust_str* 330upcall_str_new_uniq(const char *cstr, size_t len) { 331 rust_task *task = rust_get_current_task(); 332 s_str_new_uniq_args args = { task, cstr, len, 0 }; 333 UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_uniq); 334 return args.retval; 335} 336 337extern "C" CDECL rust_str* 338upcall_str_new(const char *cstr, size_t len) { 339 rust_task *task = rust_get_current_task(); 340 s_str_new_uniq_args args = { task, cstr, len, 0 }; 341 UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_uniq); 342 return args.retval; 343} 344 345 346 347struct s_str_new_shared_args { 348 rust_task *task; 349 const char *cstr; 350 size_t len; 351 rust_opaque_box *retval; 352}; 353 354extern "C" CDECL void 355upcall_s_str_new_shared(s_str_new_shared_args *args) { 356 rust_task *task = args->task; 357 LOG_UPCALL_ENTRY(task); 358 359 size_t str_fill = args->len + 1; 360 size_t str_alloc = str_fill; 361 args->retval = (rust_opaque_box *) 362 task->boxed.malloc(&str_body_tydesc, 363 str_fill + sizeof(rust_vec)); 364 rust_str *str = (rust_str *)args->retval; 365 str->body.fill = str_fill; 366 str->body.alloc = str_alloc; 367 memcpy(&str->body.data, args->cstr, args->len); 368 str->body.data[args->len] = '\0'; 369} 370 371extern "C" CDECL rust_opaque_box* 372upcall_str_new_shared(const char *cstr, size_t len) { 373 rust_task *task = rust_get_current_task(); 374 s_str_new_shared_args args = { task, cstr, len, 0 }; 375 UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_shared); 376 return args.retval; 377} 378 379 380extern "C" _Unwind_Reason_Code 381__gxx_personality_v0(int version, 382 _Unwind_Action actions, 383 uint64_t exception_class, 384 _Unwind_Exception *ue_header, 385 _Unwind_Context *context); 386 387struct s_rust_personality_args { 388 _Unwind_Reason_Code retval; 389 int version; 390 _Unwind_Action actions; 391 uint64_t exception_class; 392 _Unwind_Exception *ue_header; 393 _Unwind_Context *context; 394}; 395 396extern "C" void 397upcall_s_rust_personality(s_rust_personality_args *args) { 398 args->retval = __gxx_personality_v0(args->version, 399 args->actions, 400 args->exception_class, 401 args->ue_header, 402 args->context); 403} 404 405/** 406 The exception handling personality function. It figures 407 out what to do with each landing pad. Just a stack-switching 408 wrapper around the C++ personality function. 409*/ 410extern "C" _Unwind_Reason_Code 411upcall_rust_personality(int version, 412 _Unwind_Action actions, 413 uint64_t exception_class, 414 _Unwind_Exception *ue_header, 415 _Unwind_Context *context) { 416 s_rust_personality_args args = {(_Unwind_Reason_Code)0, 417 version, actions, exception_class, 418 ue_header, context}; 419 rust_task *task = rust_get_current_task(); 420 421 // The personality function is run on the stack of the 422 // last function that threw or landed, which is going 423 // to sometimes be the C stack. If we're on the Rust stack 424 // then switch to the C stack. 425 426 if (task->on_rust_stack()) { 427 UPCALL_SWITCH_STACK(task, &args, upcall_s_rust_personality); 428 } else { 429 upcall_s_rust_personality(&args); 430 } 431 return args.retval; 432} 433 434extern "C" void 435shape_cmp_type(int8_t *result, const type_desc *tydesc, 436 uint8_t *data_0, uint8_t *data_1, uint8_t cmp_type); 437 438struct s_cmp_type_args { 439 int8_t *result; 440 const type_desc *tydesc; 441 uint8_t *data_0; 442 uint8_t *data_1; 443 uint8_t cmp_type; 444}; 445 446extern "C" void 447upcall_s_cmp_type(s_cmp_type_args *args) { 448 shape_cmp_type(args->result, args->tydesc, 449 args->data_0, args->data_1, args->cmp_type); 450} 451 452extern "C" void 453upcall_cmp_type(int8_t *result, const type_desc *tydesc, 454 uint8_t *data_0, uint8_t *data_1, uint8_t cmp_type) { 455 rust_task *task = rust_get_current_task(); 456 s_cmp_type_args args = {result, tydesc, 457 data_0, data_1, cmp_type}; 458 UPCALL_SWITCH_STACK(task, &args, upcall_s_cmp_type); 459} 460 461extern "C" void 462shape_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level); 463 464struct s_log_type_args { 465 const type_desc *tydesc; 466 uint8_t *data; 467 uint32_t level; 468}; 469 470extern "C" void 471upcall_s_log_type(s_log_type_args *args) { 472 shape_log_type(args->tydesc, args->data, args->level); 473} 474 475extern "C" void 476upcall_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level) { 477 rust_task *task = rust_get_current_task(); 478 s_log_type_args args = {tydesc, data, level}; 479 UPCALL_SWITCH_STACK(task, &args, upcall_s_log_type); 480} 481 482// NB: This needs to be blazing fast. Don't switch stacks 483extern "C" CDECL void * 484upcall_new_stack(size_t stk_sz, void *args_addr, size_t args_sz) { 485 rust_task *task = rust_get_current_task(); 486 return task->next_stack(stk_sz, 487 args_addr, 488 args_sz); 489} 490 491// NB: This needs to be blazing fast. Don't switch stacks 492extern "C" CDECL void 493upcall_del_stack() { 494 rust_task *task = rust_get_current_task(); 495 task->prev_stack(); 496} 497 498// Landing pads need to call this to insert the 499// correct limit into TLS. 500// NB: This must run on the Rust stack because it 501// needs to acquire the value of the stack pointer 502extern "C" CDECL void 503upcall_reset_stack_limit() { 504 rust_task *task = rust_get_current_task(); 505 task->reset_stack_limit(); 506} 507 508// 509// Local Variables: 510// mode: C++ 511// fill-column: 78; 512// indent-tabs-mode: nil 513// c-basic-offset: 4 514// buffer-file-coding-system: utf-8-unix 515// End: 516//