/Modules/_ctypes/libffi/src/x86/ffi64.c

http://unladen-swallow.googlecode.com/ · C · 572 lines · 394 code · 77 blank · 101 comment · 140 complexity · 9aef4b263be7d350d72ca2733e318ca9 MD5 · raw file

  1. /* -----------------------------------------------------------------------
  2. ffi.c - Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
  3. Copyright (c) 2008 Red Hat, Inc.
  4. x86-64 Foreign Function Interface
  5. Permission is hereby granted, free of charge, to any person obtaining
  6. a copy of this software and associated documentation files (the
  7. ``Software''), to deal in the Software without restriction, including
  8. without limitation the rights to use, copy, modify, merge, publish,
  9. distribute, sublicense, and/or sell copies of the Software, and to
  10. permit persons to whom the Software is furnished to do so, subject to
  11. the following conditions:
  12. The above copyright notice and this permission notice shall be included
  13. in all copies or substantial portions of the Software.
  14. THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  15. EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16. MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  17. NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  18. HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  19. WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21. DEALINGS IN THE SOFTWARE.
  22. ----------------------------------------------------------------------- */
  23. #include <ffi.h>
  24. #include <ffi_common.h>
  25. #include <stdlib.h>
  26. #include <stdarg.h>
  27. #ifdef __x86_64__
  28. #define MAX_GPR_REGS 6
  29. #define MAX_SSE_REGS 8
  30. struct register_args
  31. {
  32. /* Registers for argument passing. */
  33. UINT64 gpr[MAX_GPR_REGS];
  34. __int128_t sse[MAX_SSE_REGS];
  35. };
  36. extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
  37. void *raddr, void (*fnaddr)(void), unsigned ssecount);
  38. /* All reference to register classes here is identical to the code in
  39. gcc/config/i386/i386.c. Do *not* change one without the other. */
  40. /* Register class used for passing given 64bit part of the argument.
  41. These represent classes as documented by the PS ABI, with the exception
  42. of SSESF, SSEDF classes, that are basically SSE class, just gcc will
  43. use SF or DFmode move instead of DImode to avoid reformatting penalties.
  44. Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
  45. whenever possible (upper half does contain padding). */
  46. enum x86_64_reg_class
  47. {
  48. X86_64_NO_CLASS,
  49. X86_64_INTEGER_CLASS,
  50. X86_64_INTEGERSI_CLASS,
  51. X86_64_SSE_CLASS,
  52. X86_64_SSESF_CLASS,
  53. X86_64_SSEDF_CLASS,
  54. X86_64_SSEUP_CLASS,
  55. X86_64_X87_CLASS,
  56. X86_64_X87UP_CLASS,
  57. X86_64_COMPLEX_X87_CLASS,
  58. X86_64_MEMORY_CLASS
  59. };
  60. #define MAX_CLASSES 4
  61. #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
  62. /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
  63. of this code is to classify each 8bytes of incoming argument by the register
  64. class and assign registers accordingly. */
  65. /* Return the union class of CLASS1 and CLASS2.
  66. See the x86-64 PS ABI for details. */
  67. static enum x86_64_reg_class
  68. merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
  69. {
  70. /* Rule #1: If both classes are equal, this is the resulting class. */
  71. if (class1 == class2)
  72. return class1;
  73. /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
  74. the other class. */
  75. if (class1 == X86_64_NO_CLASS)
  76. return class2;
  77. if (class2 == X86_64_NO_CLASS)
  78. return class1;
  79. /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
  80. if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
  81. return X86_64_MEMORY_CLASS;
  82. /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
  83. if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
  84. || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
  85. return X86_64_INTEGERSI_CLASS;
  86. if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
  87. || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
  88. return X86_64_INTEGER_CLASS;
  89. /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
  90. MEMORY is used. */
  91. if (class1 == X86_64_X87_CLASS
  92. || class1 == X86_64_X87UP_CLASS
  93. || class1 == X86_64_COMPLEX_X87_CLASS
  94. || class2 == X86_64_X87_CLASS
  95. || class2 == X86_64_X87UP_CLASS
  96. || class2 == X86_64_COMPLEX_X87_CLASS)
  97. return X86_64_MEMORY_CLASS;
  98. /* Rule #6: Otherwise class SSE is used. */
  99. return X86_64_SSE_CLASS;
  100. }
  101. /* Classify the argument of type TYPE and mode MODE.
  102. CLASSES will be filled by the register class used to pass each word
  103. of the operand. The number of words is returned. In case the parameter
  104. should be passed in memory, 0 is returned. As a special case for zero
  105. sized containers, classes[0] will be NO_CLASS and 1 is returned.
  106. See the x86-64 PS ABI for details.
  107. */
  108. static int
  109. classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
  110. size_t byte_offset)
  111. {
  112. switch (type->type)
  113. {
  114. case FFI_TYPE_UINT8:
  115. case FFI_TYPE_SINT8:
  116. case FFI_TYPE_UINT16:
  117. case FFI_TYPE_SINT16:
  118. case FFI_TYPE_UINT32:
  119. case FFI_TYPE_SINT32:
  120. case FFI_TYPE_UINT64:
  121. case FFI_TYPE_SINT64:
  122. case FFI_TYPE_POINTER:
  123. if (byte_offset + type->size <= 4)
  124. classes[0] = X86_64_INTEGERSI_CLASS;
  125. else
  126. classes[0] = X86_64_INTEGER_CLASS;
  127. return 1;
  128. case FFI_TYPE_FLOAT:
  129. if (byte_offset == 0)
  130. classes[0] = X86_64_SSESF_CLASS;
  131. else
  132. classes[0] = X86_64_SSE_CLASS;
  133. return 1;
  134. case FFI_TYPE_DOUBLE:
  135. classes[0] = X86_64_SSEDF_CLASS;
  136. return 1;
  137. case FFI_TYPE_LONGDOUBLE:
  138. classes[0] = X86_64_X87_CLASS;
  139. classes[1] = X86_64_X87UP_CLASS;
  140. return 2;
  141. case FFI_TYPE_STRUCT:
  142. {
  143. const int UNITS_PER_WORD = 8;
  144. int words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
  145. ffi_type **ptr;
  146. int i;
  147. enum x86_64_reg_class subclasses[MAX_CLASSES];
  148. /* If the struct is larger than 16 bytes, pass it on the stack. */
  149. if (type->size > 16)
  150. return 0;
  151. for (i = 0; i < words; i++)
  152. classes[i] = X86_64_NO_CLASS;
  153. /* Merge the fields of structure. */
  154. for (ptr = type->elements; *ptr != NULL; ptr++)
  155. {
  156. int num;
  157. byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
  158. num = classify_argument (*ptr, subclasses, byte_offset % 8);
  159. if (num == 0)
  160. return 0;
  161. for (i = 0; i < num; i++)
  162. {
  163. int pos = byte_offset / 8;
  164. classes[i + pos] =
  165. merge_classes (subclasses[i], classes[i + pos]);
  166. }
  167. byte_offset += (*ptr)->size;
  168. }
  169. /* Final merger cleanup. */
  170. for (i = 0; i < words; i++)
  171. {
  172. /* If one class is MEMORY, everything should be passed in
  173. memory. */
  174. if (classes[i] == X86_64_MEMORY_CLASS)
  175. return 0;
  176. /* The X86_64_SSEUP_CLASS should be always preceded by
  177. X86_64_SSE_CLASS. */
  178. if (classes[i] == X86_64_SSEUP_CLASS
  179. && (i == 0 || classes[i - 1] != X86_64_SSE_CLASS))
  180. classes[i] = X86_64_SSE_CLASS;
  181. /* X86_64_X87UP_CLASS should be preceded by X86_64_X87_CLASS. */
  182. if (classes[i] == X86_64_X87UP_CLASS
  183. && (i == 0 || classes[i - 1] != X86_64_X87_CLASS))
  184. classes[i] = X86_64_SSE_CLASS;
  185. }
  186. return words;
  187. }
  188. default:
  189. FFI_ASSERT(0);
  190. }
  191. return 0; /* Never reached. */
  192. }
  193. /* Examine the argument and return set number of register required in each
  194. class. Return zero iff parameter should be passed in memory, otherwise
  195. the number of registers. */
  196. static int
  197. examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
  198. _Bool in_return, int *pngpr, int *pnsse)
  199. {
  200. int i, n, ngpr, nsse;
  201. n = classify_argument (type, classes, 0);
  202. if (n == 0)
  203. return 0;
  204. ngpr = nsse = 0;
  205. for (i = 0; i < n; ++i)
  206. switch (classes[i])
  207. {
  208. case X86_64_INTEGER_CLASS:
  209. case X86_64_INTEGERSI_CLASS:
  210. ngpr++;
  211. break;
  212. case X86_64_SSE_CLASS:
  213. case X86_64_SSESF_CLASS:
  214. case X86_64_SSEDF_CLASS:
  215. nsse++;
  216. break;
  217. case X86_64_NO_CLASS:
  218. case X86_64_SSEUP_CLASS:
  219. break;
  220. case X86_64_X87_CLASS:
  221. case X86_64_X87UP_CLASS:
  222. case X86_64_COMPLEX_X87_CLASS:
  223. return in_return != 0;
  224. default:
  225. abort ();
  226. }
  227. *pngpr = ngpr;
  228. *pnsse = nsse;
  229. return n;
  230. }
  231. /* Perform machine dependent cif processing. */
  232. ffi_status
  233. ffi_prep_cif_machdep (ffi_cif *cif)
  234. {
  235. int gprcount, ssecount, i, avn, n, ngpr, nsse, flags;
  236. enum x86_64_reg_class classes[MAX_CLASSES];
  237. size_t bytes;
  238. gprcount = ssecount = 0;
  239. flags = cif->rtype->type;
  240. if (flags != FFI_TYPE_VOID)
  241. {
  242. n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
  243. if (n == 0)
  244. {
  245. /* The return value is passed in memory. A pointer to that
  246. memory is the first argument. Allocate a register for it. */
  247. gprcount++;
  248. /* We don't have to do anything in asm for the return. */
  249. flags = FFI_TYPE_VOID;
  250. }
  251. else if (flags == FFI_TYPE_STRUCT)
  252. {
  253. /* Mark which registers the result appears in. */
  254. _Bool sse0 = SSE_CLASS_P (classes[0]);
  255. _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
  256. if (sse0 && !sse1)
  257. flags |= 1 << 8;
  258. else if (!sse0 && sse1)
  259. flags |= 1 << 9;
  260. else if (sse0 && sse1)
  261. flags |= 1 << 10;
  262. /* Mark the true size of the structure. */
  263. flags |= cif->rtype->size << 12;
  264. }
  265. }
  266. /* Go over all arguments and determine the way they should be passed.
  267. If it's in a register and there is space for it, let that be so. If
  268. not, add it's size to the stack byte count. */
  269. for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
  270. {
  271. if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
  272. || gprcount + ngpr > MAX_GPR_REGS
  273. || ssecount + nsse > MAX_SSE_REGS)
  274. {
  275. long align = cif->arg_types[i]->alignment;
  276. if (align < 8)
  277. align = 8;
  278. bytes = ALIGN(bytes, align);
  279. bytes += cif->arg_types[i]->size;
  280. }
  281. else
  282. {
  283. gprcount += ngpr;
  284. ssecount += nsse;
  285. }
  286. }
  287. if (ssecount)
  288. flags |= 1 << 11;
  289. cif->flags = flags;
  290. cif->bytes = bytes;
  291. return FFI_OK;
  292. }
  293. void
  294. ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
  295. {
  296. enum x86_64_reg_class classes[MAX_CLASSES];
  297. char *stack, *argp;
  298. ffi_type **arg_types;
  299. int gprcount, ssecount, ngpr, nsse, i, avn;
  300. _Bool ret_in_memory;
  301. struct register_args *reg_args;
  302. /* Can't call 32-bit mode from 64-bit mode. */
  303. FFI_ASSERT (cif->abi == FFI_UNIX64);
  304. /* If the return value is a struct and we don't have a return value
  305. address then we need to make one. Note the setting of flags to
  306. VOID above in ffi_prep_cif_machdep. */
  307. ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
  308. && (cif->flags & 0xff) == FFI_TYPE_VOID);
  309. if (rvalue == NULL && ret_in_memory)
  310. rvalue = alloca (cif->rtype->size);
  311. /* Allocate the space for the arguments, plus 4 words of temp space. */
  312. stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
  313. reg_args = (struct register_args *) stack;
  314. argp = stack + sizeof (struct register_args);
  315. gprcount = ssecount = 0;
  316. /* If the return value is passed in memory, add the pointer as the
  317. first integer argument. */
  318. if (ret_in_memory)
  319. reg_args->gpr[gprcount++] = (long) rvalue;
  320. avn = cif->nargs;
  321. arg_types = cif->arg_types;
  322. for (i = 0; i < avn; ++i)
  323. {
  324. size_t size = arg_types[i]->size;
  325. int n;
  326. n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
  327. if (n == 0
  328. || gprcount + ngpr > MAX_GPR_REGS
  329. || ssecount + nsse > MAX_SSE_REGS)
  330. {
  331. long align = arg_types[i]->alignment;
  332. /* Stack arguments are *always* at least 8 byte aligned. */
  333. if (align < 8)
  334. align = 8;
  335. /* Pass this argument in memory. */
  336. argp = (void *) ALIGN (argp, align);
  337. memcpy (argp, avalue[i], size);
  338. argp += size;
  339. }
  340. else
  341. {
  342. /* The argument is passed entirely in registers. */
  343. char *a = (char *) avalue[i];
  344. int j;
  345. for (j = 0; j < n; j++, a += 8, size -= 8)
  346. {
  347. switch (classes[j])
  348. {
  349. case X86_64_INTEGER_CLASS:
  350. case X86_64_INTEGERSI_CLASS:
  351. reg_args->gpr[gprcount] = 0;
  352. memcpy (&reg_args->gpr[gprcount], a, size < 8 ? size : 8);
  353. gprcount++;
  354. break;
  355. case X86_64_SSE_CLASS:
  356. case X86_64_SSEDF_CLASS:
  357. reg_args->sse[ssecount++] = *(UINT64 *) a;
  358. break;
  359. case X86_64_SSESF_CLASS:
  360. reg_args->sse[ssecount++] = *(UINT32 *) a;
  361. break;
  362. default:
  363. abort();
  364. }
  365. }
  366. }
  367. }
  368. ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
  369. cif->flags, rvalue, fn, ssecount);
  370. }
  371. extern void ffi_closure_unix64(void);
  372. ffi_status
  373. ffi_prep_closure_loc (ffi_closure* closure,
  374. ffi_cif* cif,
  375. void (*fun)(ffi_cif*, void*, void**, void*),
  376. void *user_data,
  377. void *codeloc)
  378. {
  379. volatile unsigned short *tramp;
  380. tramp = (volatile unsigned short *) &closure->tramp[0];
  381. tramp[0] = 0xbb49; /* mov <code>, %r11 */
  382. *(void * volatile *) &tramp[1] = ffi_closure_unix64;
  383. tramp[5] = 0xba49; /* mov <data>, %r10 */
  384. *(void * volatile *) &tramp[6] = codeloc;
  385. /* Set the carry bit iff the function uses any sse registers.
  386. This is clc or stc, together with the first byte of the jmp. */
  387. tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
  388. tramp[11] = 0xe3ff; /* jmp *%r11 */
  389. closure->cif = cif;
  390. closure->fun = fun;
  391. closure->user_data = user_data;
  392. return FFI_OK;
  393. }
  394. int
  395. ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
  396. struct register_args *reg_args, char *argp)
  397. {
  398. ffi_cif *cif;
  399. void **avalue;
  400. ffi_type **arg_types;
  401. long i, avn;
  402. int gprcount, ssecount, ngpr, nsse;
  403. int ret;
  404. cif = closure->cif;
  405. avalue = alloca(cif->nargs * sizeof(void *));
  406. gprcount = ssecount = 0;
  407. ret = cif->rtype->type;
  408. if (ret != FFI_TYPE_VOID)
  409. {
  410. enum x86_64_reg_class classes[MAX_CLASSES];
  411. int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
  412. if (n == 0)
  413. {
  414. /* The return value goes in memory. Arrange for the closure
  415. return value to go directly back to the original caller. */
  416. rvalue = (void *) reg_args->gpr[gprcount++];
  417. /* We don't have to do anything in asm for the return. */
  418. ret = FFI_TYPE_VOID;
  419. }
  420. else if (ret == FFI_TYPE_STRUCT && n == 2)
  421. {
  422. /* Mark which register the second word of the structure goes in. */
  423. _Bool sse0 = SSE_CLASS_P (classes[0]);
  424. _Bool sse1 = SSE_CLASS_P (classes[1]);
  425. if (!sse0 && sse1)
  426. ret |= 1 << 8;
  427. else if (sse0 && !sse1)
  428. ret |= 1 << 9;
  429. }
  430. }
  431. avn = cif->nargs;
  432. arg_types = cif->arg_types;
  433. for (i = 0; i < avn; ++i)
  434. {
  435. enum x86_64_reg_class classes[MAX_CLASSES];
  436. int n;
  437. n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
  438. if (n == 0
  439. || gprcount + ngpr > MAX_GPR_REGS
  440. || ssecount + nsse > MAX_SSE_REGS)
  441. {
  442. long align = arg_types[i]->alignment;
  443. /* Stack arguments are *always* at least 8 byte aligned. */
  444. if (align < 8)
  445. align = 8;
  446. /* Pass this argument in memory. */
  447. argp = (void *) ALIGN (argp, align);
  448. avalue[i] = argp;
  449. argp += arg_types[i]->size;
  450. }
  451. /* If the argument is in a single register, or two consecutive
  452. registers, then we can use that address directly. */
  453. else if (n == 1
  454. || (n == 2
  455. && SSE_CLASS_P (classes[0]) == SSE_CLASS_P (classes[1])))
  456. {
  457. /* The argument is in a single register. */
  458. if (SSE_CLASS_P (classes[0]))
  459. {
  460. avalue[i] = &reg_args->sse[ssecount];
  461. ssecount += n;
  462. }
  463. else
  464. {
  465. avalue[i] = &reg_args->gpr[gprcount];
  466. gprcount += n;
  467. }
  468. }
  469. /* Otherwise, allocate space to make them consecutive. */
  470. else
  471. {
  472. char *a = alloca (16);
  473. int j;
  474. avalue[i] = a;
  475. for (j = 0; j < n; j++, a += 8)
  476. {
  477. if (SSE_CLASS_P (classes[j]))
  478. memcpy (a, &reg_args->sse[ssecount++], 8);
  479. else
  480. memcpy (a, &reg_args->gpr[gprcount++], 8);
  481. }
  482. }
  483. }
  484. /* Invoke the closure. */
  485. closure->fun (cif, rvalue, avalue, closure->user_data);
  486. /* Tell assembly how to perform return type promotions. */
  487. return ret;
  488. }
  489. #endif /* __x86_64__ */