/src/rt/rust_upcall.cpp

http://github.com/jruderman/rust · C++ · 516 lines · 360 code · 78 blank · 78 comment · 6 complexity · c7f97becc0c9b998b1cb2f3b2cc52858 MD5 · raw file

  1. /*
  2. Upcalls
  3. These are runtime functions that the compiler knows about and generates
  4. calls to. They are called on the Rust stack and, in most cases, immediately
  5. switch to the C stack.
  6. */
  7. #include "rust_globals.h"
  8. #include "rust_task.h"
  9. #include "rust_cc.h"
  10. #include "rust_sched_loop.h"
  11. #include "rust_unwind.h"
  12. #include "rust_upcall.h"
  13. #include "rust_util.h"
  14. #ifdef __GNUC__
  15. #define LOG_UPCALL_ENTRY(task) \
  16. LOG(task, upcall, \
  17. "> UPCALL %s - task: %s 0x%" PRIxPTR \
  18. " retpc: x%" PRIxPTR, \
  19. __FUNCTION__, \
  20. (task)->name, (task), \
  21. __builtin_return_address(0));
  22. #else
  23. #define LOG_UPCALL_ENTRY(task) \
  24. LOG(task, upcall, "> UPCALL task: %s @x%" PRIxPTR, \
  25. (task)->name, (task));
  26. #endif
  27. #define UPCALL_SWITCH_STACK(T, A, F) \
  28. call_upcall_on_c_stack(T, (void*)A, (void*)F)
  29. inline void
  30. call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) {
  31. task->call_on_c_stack(args, fn_ptr);
  32. }
  33. /**********************************************************************
  34. * Switches to the C-stack and invokes |fn_ptr|, passing |args| as argument.
  35. * This is used by the C compiler to call foreign functions and by other
  36. * upcalls to switch to the C stack. The return value is passed through a
  37. * field in the args parameter. This upcall is specifically for switching
  38. * to the shim functions generated by rustc.
  39. */
  40. extern "C" CDECL void
  41. upcall_call_shim_on_c_stack(void *args, void *fn_ptr) {
  42. rust_task *task = rust_get_current_task();
  43. try {
  44. task->call_on_c_stack(args, fn_ptr);
  45. } catch (...) {
  46. // Logging here is not reliable
  47. assert(false && "Foreign code threw an exception");
  48. }
  49. }
  50. /*
  51. * The opposite of above. Starts on a C stack and switches to the Rust
  52. * stack. This is the only upcall that runs from the C stack.
  53. */
  54. extern "C" CDECL void
  55. upcall_call_shim_on_rust_stack(void *args, void *fn_ptr) {
  56. rust_task *task = rust_get_current_task();
  57. try {
  58. task->call_on_rust_stack(args, fn_ptr);
  59. } catch (...) {
  60. // We can't count on being able to unwind through arbitrary
  61. // code. Our best option is to just fail hard.
  62. // Logging here is not reliable
  63. assert(false && "Rust task failed after reentering the Rust stack");
  64. }
  65. }
  66. /**********************************************************************/
  67. struct s_fail_args {
  68. rust_task *task;
  69. char const *expr;
  70. char const *file;
  71. size_t line;
  72. };
  73. extern "C" CDECL void
  74. upcall_s_fail(s_fail_args *args) {
  75. rust_task *task = args->task;
  76. LOG_UPCALL_ENTRY(task);
  77. task->fail(args->expr, args->file, args->line);
  78. }
  79. extern "C" CDECL void
  80. upcall_fail(char const *expr,
  81. char const *file,
  82. size_t line) {
  83. rust_task *task = rust_get_current_task();
  84. s_fail_args args = {task,expr,file,line};
  85. UPCALL_SWITCH_STACK(task, &args, upcall_s_fail);
  86. }
  87. // FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
  88. // autogenerated wrappers for upcall_fail. Remove this when we fully move away
  89. // away from the C upcall path.
  90. extern "C" CDECL void
  91. rust_upcall_fail(char const *expr,
  92. char const *file,
  93. size_t line) {
  94. upcall_fail(expr, file, line);
  95. }
  96. struct s_trace_args {
  97. rust_task *task;
  98. char const *msg;
  99. char const *file;
  100. size_t line;
  101. };
  102. extern "C" CDECL void
  103. upcall_s_trace(s_trace_args *args) {
  104. rust_task *task = args->task;
  105. LOG_UPCALL_ENTRY(task);
  106. LOG(task, trace, "Trace %s:%d: %s",
  107. args->file, args->line, args->msg);
  108. }
  109. extern "C" CDECL void
  110. upcall_trace(char const *msg,
  111. char const *file,
  112. size_t line) {
  113. rust_task *task = rust_get_current_task();
  114. s_trace_args args = {task,msg,file,line};
  115. UPCALL_SWITCH_STACK(task, &args, upcall_s_trace);
  116. }
  117. /**********************************************************************
  118. * Allocate an object in the exchange heap
  119. */
  120. struct s_exchange_malloc_args {
  121. rust_task *task;
  122. uintptr_t retval;
  123. type_desc *td;
  124. uintptr_t size;
  125. };
  126. extern "C" CDECL void
  127. upcall_s_exchange_malloc(s_exchange_malloc_args *args) {
  128. rust_task *task = args->task;
  129. LOG_UPCALL_ENTRY(task);
  130. LOG(task, mem, "upcall exchange malloc(0x%" PRIxPTR ")", args->td);
  131. size_t total_size = get_box_size(args->size, args->td->align);
  132. // FIXME--does this have to be calloc? (Issue #2682)
  133. void *p = task->kernel->calloc(total_size, "exchange malloc");
  134. rust_opaque_box *header = static_cast<rust_opaque_box*>(p);
  135. header->ref_count = -1; // This is not ref counted
  136. header->td = args->td;
  137. header->prev = 0;
  138. header->next = 0;
  139. args->retval = (uintptr_t)header;
  140. }
  141. extern "C" CDECL uintptr_t
  142. upcall_exchange_malloc(type_desc *td, uintptr_t size) {
  143. rust_task *task = rust_get_current_task();
  144. s_exchange_malloc_args args = {task, 0, td, size};
  145. UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_malloc);
  146. return args.retval;
  147. }
  148. // FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
  149. // autogenerated wrappers for upcall_exchange_malloc. Remove this when we
  150. // fully move away away from the C upcall path.
  151. extern "C" CDECL uintptr_t
  152. rust_upcall_exchange_malloc(type_desc *td, uintptr_t size) {
  153. return upcall_exchange_malloc(td, size);
  154. }
  155. struct s_exchange_free_args {
  156. rust_task *task;
  157. void *ptr;
  158. };
  159. extern "C" CDECL void
  160. upcall_s_exchange_free(s_exchange_free_args *args) {
  161. rust_task *task = args->task;
  162. LOG_UPCALL_ENTRY(task);
  163. task->kernel->free(args->ptr);
  164. }
  165. extern "C" CDECL void
  166. upcall_exchange_free(void *ptr) {
  167. rust_task *task = rust_get_current_task();
  168. s_exchange_free_args args = {task,ptr};
  169. UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_free);
  170. }
  171. // FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
  172. // autogenerated wrappers for upcall_exchange_free. Remove this when we fully
  173. // move away away from the C upcall path.
  174. extern "C" CDECL void
  175. rust_upcall_exchange_free(void *ptr) {
  176. return upcall_exchange_free(ptr);
  177. }
  178. /**********************************************************************
  179. * Allocate an object in the task-local heap.
  180. */
  181. struct s_malloc_args {
  182. rust_task *task;
  183. uintptr_t retval;
  184. type_desc *td;
  185. uintptr_t size;
  186. };
  187. extern "C" CDECL void
  188. upcall_s_malloc(s_malloc_args *args) {
  189. rust_task *task = args->task;
  190. LOG_UPCALL_ENTRY(task);
  191. LOG(task, mem, "upcall malloc(0x%" PRIxPTR ")", args->td);
  192. cc::maybe_cc(task);
  193. // FIXME--does this have to be calloc? (Issue #2682)
  194. rust_opaque_box *box = task->boxed.calloc(args->td, args->size);
  195. void *body = box_body(box);
  196. debug::maybe_track_origin(task, box);
  197. LOG(task, mem,
  198. "upcall malloc(0x%" PRIxPTR ") = box 0x%" PRIxPTR
  199. " with body 0x%" PRIxPTR,
  200. args->td, (uintptr_t)box, (uintptr_t)body);
  201. args->retval = (uintptr_t)box;
  202. }
  203. extern "C" CDECL uintptr_t
  204. upcall_malloc(type_desc *td, uintptr_t size) {
  205. rust_task *task = rust_get_current_task();
  206. s_malloc_args args = {task, 0, td, size};
  207. UPCALL_SWITCH_STACK(task, &args, upcall_s_malloc);
  208. return args.retval;
  209. }
  210. // FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
  211. // autogenerated wrappers for upcall_malloc. Remove this when we fully move
  212. // away away from the C upcall path.
  213. extern "C" CDECL uintptr_t
  214. rust_upcall_malloc(type_desc *td, uintptr_t size) {
  215. return upcall_malloc(td, size);
  216. }
  217. /**********************************************************************
  218. * Called whenever an object in the task-local heap is freed.
  219. */
  220. struct s_free_args {
  221. rust_task *task;
  222. void *ptr;
  223. };
  224. extern "C" CDECL void
  225. upcall_s_free(s_free_args *args) {
  226. rust_task *task = args->task;
  227. LOG_UPCALL_ENTRY(task);
  228. rust_sched_loop *sched_loop = task->sched_loop;
  229. DLOG(sched_loop, mem,
  230. "upcall free(0x%" PRIxPTR ", is_gc=%" PRIdPTR ")",
  231. (uintptr_t)args->ptr);
  232. debug::maybe_untrack_origin(task, args->ptr);
  233. rust_opaque_box *box = (rust_opaque_box*) args->ptr;
  234. task->boxed.free(box);
  235. }
  236. extern "C" CDECL void
  237. upcall_free(void* ptr) {
  238. rust_task *task = rust_get_current_task();
  239. s_free_args args = {task,ptr};
  240. UPCALL_SWITCH_STACK(task, &args, upcall_s_free);
  241. }
  242. // FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
  243. // autogenerated wrappers for upcall_free. Remove this when we fully move away
  244. // away from the C upcall path.
  245. extern "C" CDECL void
  246. rust_upcall_free(void* ptr) {
  247. upcall_free(ptr);
  248. }
  249. /**********************************************************************
  250. * Sanity checks on boxes, insert when debugging possible
  251. * use-after-free bugs. See maybe_validate_box() in trans.rs.
  252. */
  253. extern "C" CDECL void
  254. upcall_validate_box(rust_opaque_box* ptr) {
  255. if (ptr) {
  256. assert(ptr->ref_count > 0);
  257. assert(ptr->td != NULL);
  258. assert(ptr->td->align <= 8);
  259. assert(ptr->td->size <= 4096); // might not really be true...
  260. }
  261. }
  262. /**********************************************************************/
  263. struct s_str_new_uniq_args {
  264. rust_task *task;
  265. const char *cstr;
  266. size_t len;
  267. rust_str *retval;
  268. };
  269. extern "C" CDECL void
  270. upcall_s_str_new_uniq(s_str_new_uniq_args *args) {
  271. rust_task *task = args->task;
  272. LOG_UPCALL_ENTRY(task);
  273. args->retval = make_str(task->kernel, args->cstr, args->len,
  274. "str_new_uniq");
  275. }
  276. extern "C" CDECL rust_str*
  277. upcall_str_new_uniq(const char *cstr, size_t len) {
  278. rust_task *task = rust_get_current_task();
  279. s_str_new_uniq_args args = { task, cstr, len, 0 };
  280. UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_uniq);
  281. return args.retval;
  282. }
  283. extern "C" CDECL rust_str*
  284. upcall_str_new(const char *cstr, size_t len) {
  285. rust_task *task = rust_get_current_task();
  286. s_str_new_uniq_args args = { task, cstr, len, 0 };
  287. UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_uniq);
  288. return args.retval;
  289. }
  290. struct s_str_new_shared_args {
  291. rust_task *task;
  292. const char *cstr;
  293. size_t len;
  294. rust_opaque_box *retval;
  295. };
  296. extern "C" CDECL void
  297. upcall_s_str_new_shared(s_str_new_shared_args *args) {
  298. rust_task *task = args->task;
  299. LOG_UPCALL_ENTRY(task);
  300. size_t str_fill = args->len + 1;
  301. size_t str_alloc = str_fill;
  302. args->retval = (rust_opaque_box *)
  303. task->boxed.malloc(&str_body_tydesc,
  304. str_fill + sizeof(rust_vec));
  305. rust_str *str = (rust_str *)args->retval;
  306. str->body.fill = str_fill;
  307. str->body.alloc = str_alloc;
  308. memcpy(&str->body.data, args->cstr, args->len);
  309. str->body.data[args->len] = '\0';
  310. }
  311. extern "C" CDECL rust_opaque_box*
  312. upcall_str_new_shared(const char *cstr, size_t len) {
  313. rust_task *task = rust_get_current_task();
  314. s_str_new_shared_args args = { task, cstr, len, 0 };
  315. UPCALL_SWITCH_STACK(task, &args, upcall_s_str_new_shared);
  316. return args.retval;
  317. }
  318. extern "C" _Unwind_Reason_Code
  319. __gxx_personality_v0(int version,
  320. _Unwind_Action actions,
  321. uint64_t exception_class,
  322. _Unwind_Exception *ue_header,
  323. _Unwind_Context *context);
  324. struct s_rust_personality_args {
  325. _Unwind_Reason_Code retval;
  326. int version;
  327. _Unwind_Action actions;
  328. uint64_t exception_class;
  329. _Unwind_Exception *ue_header;
  330. _Unwind_Context *context;
  331. };
  332. extern "C" void
  333. upcall_s_rust_personality(s_rust_personality_args *args) {
  334. args->retval = __gxx_personality_v0(args->version,
  335. args->actions,
  336. args->exception_class,
  337. args->ue_header,
  338. args->context);
  339. }
  340. /**
  341. The exception handling personality function. It figures
  342. out what to do with each landing pad. Just a stack-switching
  343. wrapper around the C++ personality function.
  344. */
  345. extern "C" _Unwind_Reason_Code
  346. upcall_rust_personality(int version,
  347. _Unwind_Action actions,
  348. uint64_t exception_class,
  349. _Unwind_Exception *ue_header,
  350. _Unwind_Context *context) {
  351. s_rust_personality_args args = {(_Unwind_Reason_Code)0,
  352. version, actions, exception_class,
  353. ue_header, context};
  354. rust_task *task = rust_get_current_task();
  355. // The personality function is run on the stack of the
  356. // last function that threw or landed, which is going
  357. // to sometimes be the C stack. If we're on the Rust stack
  358. // then switch to the C stack.
  359. if (task->on_rust_stack()) {
  360. UPCALL_SWITCH_STACK(task, &args, upcall_s_rust_personality);
  361. } else {
  362. upcall_s_rust_personality(&args);
  363. }
  364. return args.retval;
  365. }
  366. extern "C" void
  367. shape_cmp_type(int8_t *result, const type_desc *tydesc,
  368. uint8_t *data_0, uint8_t *data_1, uint8_t cmp_type);
  369. struct s_cmp_type_args {
  370. int8_t *result;
  371. const type_desc *tydesc;
  372. uint8_t *data_0;
  373. uint8_t *data_1;
  374. uint8_t cmp_type;
  375. };
  376. extern "C" void
  377. upcall_s_cmp_type(s_cmp_type_args *args) {
  378. shape_cmp_type(args->result, args->tydesc,
  379. args->data_0, args->data_1, args->cmp_type);
  380. }
  381. extern "C" void
  382. upcall_cmp_type(int8_t *result, const type_desc *tydesc,
  383. uint8_t *data_0, uint8_t *data_1, uint8_t cmp_type) {
  384. rust_task *task = rust_get_current_task();
  385. s_cmp_type_args args = {result, tydesc,
  386. data_0, data_1, cmp_type};
  387. UPCALL_SWITCH_STACK(task, &args, upcall_s_cmp_type);
  388. }
  389. extern "C" void
  390. shape_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level);
  391. struct s_log_type_args {
  392. const type_desc *tydesc;
  393. uint8_t *data;
  394. uint32_t level;
  395. };
  396. extern "C" void
  397. upcall_s_log_type(s_log_type_args *args) {
  398. shape_log_type(args->tydesc, args->data, args->level);
  399. }
  400. extern "C" void
  401. upcall_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level) {
  402. rust_task *task = rust_get_current_task();
  403. s_log_type_args args = {tydesc, data, level};
  404. UPCALL_SWITCH_STACK(task, &args, upcall_s_log_type);
  405. }
  406. // NB: This needs to be blazing fast. Don't switch stacks
  407. extern "C" CDECL void *
  408. upcall_new_stack(size_t stk_sz, void *args_addr, size_t args_sz) {
  409. rust_task *task = rust_get_current_task();
  410. return task->next_stack(stk_sz,
  411. args_addr,
  412. args_sz);
  413. }
  414. // NB: This needs to be blazing fast. Don't switch stacks
  415. extern "C" CDECL void
  416. upcall_del_stack() {
  417. rust_task *task = rust_get_current_task();
  418. task->prev_stack();
  419. }
  420. // Landing pads need to call this to insert the
  421. // correct limit into TLS.
  422. // NB: This must run on the Rust stack because it
  423. // needs to acquire the value of the stack pointer
  424. extern "C" CDECL void
  425. upcall_reset_stack_limit() {
  426. rust_task *task = rust_get_current_task();
  427. task->reset_stack_limit();
  428. }
  429. //
  430. // Local Variables:
  431. // mode: C++
  432. // fill-column: 78;
  433. // indent-tabs-mode: nil
  434. // c-basic-offset: 4
  435. // buffer-file-coding-system: utf-8-unix
  436. // End:
  437. //