PageRenderTime 87ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 1ms

/erts/emulator/beam/beam_emu.c

https://github.com/system/erlang-otp
C | 5921 lines | 4453 code | 636 blank | 832 comment | 852 complexity | a1d0c5b6e5b28b84d7bfa68f8386ca83 MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, LGPL-2.1 
    

  1. /* ``The contents of this file are subject to the Erlang Public License,
    2. 

  2.  * Version 1.1, (the "License"); you may not use this file except in
    3. 

  3.  * compliance with the License. You should have received a copy of the
    4. 

  4.  * Erlang Public License along with this software. If not, it can be
    5. 

  5.  * retrieved via the world wide web at http://www.erlang.org/.
    6. 

  6.  * 
    7. 

  7.  * Software distributed under the License is distributed on an "AS IS"
    8. 

  8.  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
    9. 

  9.  * the License for the specific language governing rights and limitations
    10. 

  10.  * under the License.
    11. 

  11.  * 
    12. 

  12.  * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
    13. 

  13.  * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
    14. 

  14.  * AB. All Rights Reserved.''
    15. 

  15.  * 
    16. 

  16.  *     $Id$
    17. 

  17.  */
    18. 

  18. 

  19. #ifdef HAVE_CONFIG_H
    20. 

  20. #  include "config.h"
    21. 

  21. #endif
    22. 

  22. 

  23. #include <stddef.h> /* offsetof() */
    24. 

  24. #include "sys.h"
    25. 

  25. #include "erl_vm.h"
    26. 

  26. #include "global.h"
    27. 

  27. #include "erl_process.h"
    28. 

  28. #include "erl_nmgc.h"
    29. 

  29. #include "error.h"
    30. 

  30. #include "bif.h"
    31. 

  31. #include "big.h"
    32. 

  32. #include "beam_load.h"
    33. 

  33. #include "erl_binary.h"
    34. 

  34. #include "erl_bits.h"
    35. 

  35. #include "dist.h"
    36. 

  36. #include "beam_bp.h"
    37. 

  37. #include "beam_catches.h"
    38. 

  38. #ifdef HIPE
    39. 

  39. #include "hipe_mode_switch.h"
    40. 

  40. #include "hipe_bif1.h"
    41. 

  41. #endif
    42. 

  42. 

  43. /* #define HARDDEBUG 1 */
    44. 

  44. 

  45. #if defined(NO_JUMP_TABLE)
    46. 

  46. #  define OpCase(OpCode)    case op_##OpCode: lb_##OpCode
    47. 

  47. #  define CountCase(OpCode) case op_count_##OpCode
    48. 

  48. #  define OpCode(OpCode)    ((Uint*)op_##OpCode)
    49. 

  49. #  define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;}
    50. 

  50. #  define LabelAddr(Addr) &&##Addr
    51. 

  51. #else
    52. 

  52. #  define OpCase(OpCode)    lb_##OpCode
    53. 

  53. #  define CountCase(OpCode) lb_count_##OpCode
    54. 

  54. #  define Goto(Rel) goto *(Rel)
    55. 

  55. #  define LabelAddr(Label) &&Label
    56. 

  56. #  define OpCode(OpCode)  (&&lb_##OpCode)
    57. 

  57. #endif
    58. 

  58. 

  59. #ifdef ERTS_ENABLE_LOCK_CHECK
    60. 

  60. #  ifdef ERTS_SMP
    61. 

  61. #    define PROCESS_MAIN_CHK_LOCKS(P)					\
    62. 

  62. do {									\
    63. 

  63.     if ((P))								\
    64. 

  64. 	erts_proc_lc_chk_only_proc_main((P));				\
    65. 

  65.     else								\
    66. 

  66. 	erts_lc_check_exact(NULL, 0);					\
    67. 

  67.     ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);				\
    68. 

  68. } while (0)
    69. 

  69. #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
    70. 

  70.         if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
    71. 

  71. #    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
    72. 

  72.         if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
    73. 

  73. #  else
    74. 

  74. #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
    75. 

  75. #    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
    76. 

  76. #    define PROCESS_MAIN_CHK_LOCKS(P) erts_lc_check_exact(NULL, 0)
    77. 

  77. #  endif
    78. 

  78. #else
    79. 

  79. #  define PROCESS_MAIN_CHK_LOCKS(P)
    80. 

  80. #  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
    81. 

  81. #  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
    82. 

  82. #endif
    83. 

  83. 

  84. /*
    85. 

  85.  * Shallow copy to heap if possible; otherwise,
    86. 

  86.  * move to heap via garbage collection.
    87. 

  87.  */
    88. 

  88. 

  89. #define MV_MSG_MBUF_INTO_PROC(M)					\
    90. 

  90. do {									\
    91. 

  91.     if ((M)->bp) {							\
    92. 

  92. 	Uint need = (M)->bp->size;					\
    93. 

  93.  	if (E - HTOP >= need) {						\
    94. 

  94. 	    Uint *htop = HTOP;						\
    95. 

  95. 	    erts_move_msg_mbuf_to_heap(&htop, &MSO(c_p), (M));		\
    96. 

  96. 	    ASSERT(htop - HTOP == need);				\
    97. 

  97. 	    HTOP = htop;						\
    98. 

  98. 	}								\
    99. 

  99. 	else {								\
    100. 

  100. 	    SWAPOUT;							\
    101. 

  101. 	    reg[0] = r(0);						\
    102. 

  102. 	    PROCESS_MAIN_CHK_LOCKS(c_p);				\
    103. 

  103. 	    FCALLS -= erts_garbage_collect(c_p, 0, NULL, 0);		\
    104. 

  104. 	    PROCESS_MAIN_CHK_LOCKS(c_p);				\
    105. 

  105. 	    r(0) = reg[0];						\
    106. 

  106. 	    SWAPIN;							\
    107. 

  107. 	    ASSERT(!(M)->bp);						\
    108. 

  108. 	}								\
    109. 

  109.     }									\
    110. 

  110.     ASSERT(!(M)->bp);							\
    111. 

  111. } while (0)
    112. 

  112. 

  113. /*
    114. 

  114.  * Define macros for deep checking of terms.
    115. 

  115.  */
    116. 

  116. 

  117. #if defined(HARDDEBUG)
    118. 

  118. 

  119. #  define CHECK_TERM(T) size_object(T)
    120. 

  120. 

  121. #  define CHECK_ARGS(PC)                 \
    122. 

  122. do {                                     \
    123. 

  123.   int i_;                                \
    124. 

  124.   int Arity_ = PC[-1];                   \
    125. 

  125.   if (Arity_ > 0) {                      \
    126. 

  126. 	CHECK_TERM(r(0));                \
    127. 

  127.   }                                      \
    128. 

  128.   for (i_ = 1; i_ < Arity_; i_++) {      \
    129. 

  129. 	CHECK_TERM(x(i_));               \
    130. 

  130.   }                                      \
    131. 

  131. } while (0)
    132. 

  132.     
    133. 

  133. #else
    134. 

  134. #  define CHECK_TERM(T) ASSERT(!is_CP(T))
    135. 

  135. #  define CHECK_ARGS(T)
    136. 

  136. #endif
    137. 

  137. 

  138. #ifndef MAX
    139. 

  139. #define MAX(x, y) (((x) > (y)) ? (x) : (y))
    140. 

  140. #endif
    141. 

  141. 

  142. #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
    143. 

  143. 

  144. 

  145. /*
    146. 

  146.  * We reuse some of fields in the save area in the process structure.
    147. 

  147.  * This is safe to do, since this space is only activly used when
    148. 

  148.  * the process is switched out.
    149. 

  149.  */
    150. 

  150. #define REDS_IN(p)  ((p)->def_arg_reg[5])
    151. 

  151. 

  152. /*
    153. 

  153.  * Add a byte offset to a pointer to Eterm.  This is useful when the
    154. 

  154.  * the loader has precalculated a byte offset.
    155. 

  155.  */
    156. 

  156. #define ADD_BYTE_OFFSET(ptr, offset) \
    157. 

  157.    ((Eterm *) (((unsigned char *)ptr) + (offset)))
    158. 

  158. 

  159. /* We don't check the range if an ordinary switch is used */
    160. 

  160. #ifdef NO_JUMP_TABLE
    161. 

  161. #define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10)))
    162. 

  162. #else
    163. 

  163. #define VALID_INSTR(IP) \
    164. 

  164.    ((Sint)LabelAddr(emulator_loop) <= (Sint)(IP) && \
    165. 

  165.     (Sint)(IP) < (Sint)LabelAddr(end_emulator_loop))
    166. 

  166. #endif /* NO_JUMP_TABLE */
    167. 

  167. 

  168. #define SET_CP(p, ip)           \
    169. 

  169.    ASSERT(VALID_INSTR(*(ip)));  \
    170. 

  170.    (p)->cp = (ip)
    171. 

  171. 

  172. #define SET_I(ip) \
    173. 

  173.    ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
    174. 

  174.    I = (ip)
    175. 

  175. 

  176. #define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2)
    177. 

  177. 

  178. /*
    179. 

  179.  * Store a result into a register given a destination descriptor.
    180. 

  180.  */
    181. 

  181. 

  182. #define StoreResult(Result, DestDesc)               \
    183. 

  183.   do {                                              \
    184. 

  184.     Eterm stb_reg;                                  \
    185. 

  185.     stb_reg = (DestDesc);                           \
    186. 

  186.     CHECK_TERM(Result);                             \
    187. 

  187.     switch (beam_reg_tag(stb_reg)) {                \
    188. 

  188.     case R_REG_DEF:                                 \
    189. 

  189.       r(0) = (Result); break;                       \
    190. 

  190.     case X_REG_DEF:                                 \
    191. 

  191.       xb(x_reg_offset(stb_reg)) = (Result); break;  \
    192. 

  192.     default:                                        \
    193. 

  193.       yb(y_reg_offset(stb_reg)) = (Result); break;  \
    194. 

  194.     }                                               \
    195. 

  195.   } while (0)
    196. 

  196. 

  197. #define StoreSimpleDest(Src, Dest) Dest = (Src)
    198. 

  198. 

  199. /*
    200. 

  200.  * Store a result into a register and execute the next instruction.
    201. 

  201.  * Dst points to the word with a destination descriptor, which MUST
    202. 

  202.  * be just before the next instruction.
    203. 

  203.  */
    204. 

  204.  
    205. 

  205. #define StoreBifResult(Dst, Result)                          \
    206. 

  206.   do {                                                       \
    207. 

  207.     Eterm* stb_next;                                         \
    208. 

  208.     Eterm stb_reg;                                           \
    209. 

  209.     stb_reg = Arg(Dst);                                      \
    210. 

  210.     I += (Dst) + 2;                                          \
    211. 

  211.     stb_next = (Eterm *) *I;                                 \
    212. 

  212.     CHECK_TERM(Result);                                      \
    213. 

  213.     switch (beam_reg_tag(stb_reg)) {                         \
    214. 

  214.     case R_REG_DEF:                                          \
    215. 

  215.       r(0) = (Result); Goto(stb_next);                       \
    216. 

  216.     case X_REG_DEF:                                          \
    217. 

  217.       xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
    218. 

  218.     default:                                                 \
    219. 

  219.       yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
    220. 

  220.     }                                                        \
    221. 

  221.   } while (0)
    222. 

  222. 

  223. #define ClauseFail() goto lb_jump_f
    224. 

  224. 

  225. #define SAVE_CP(X)		*(X) = make_cp(c_p->cp)
    226. 

  226. #define RESTORE_CP(X)		SET_CP(c_p, cp_val(*(X)))
    227. 

  227. 

  228. #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
    229. 

  229. 

  230. /*
    231. 

  231.  * Special Beam instructions.
    232. 

  232.  */
    233. 

  233. 

  234. Eterm beam_apply[2];
    235. 

  235. Eterm beam_exit[1];
    236. 

  236. 

  237. Eterm* em_call_error_handler;
    238. 

  238. Eterm* em_apply_bif;
    239. 

  239. Eterm* em_call_traced_function;
    240. 

  240. 

  241. 

  242. /* NOTE These should be the only variables containing trace instructions.
    243. 

  243. **      Sometimes tests are form the instruction value, and sometimes
    244. 

  244. **      for the refering variable (one of these), and rouge references
    245. 

  245. **      will most likely cause chaos.
    246. 

  246. */
    247. 

  247. Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */
    248. 

  248. Eterm beam_return_trace[1];    /* OpCode(i_return_trace) */
    249. 

  249. Eterm beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */
    250. 

  250. 

  251. /*
    252. 

  252.  * All Beam instructions in numerical order.
    253. 

  253.  */
    254. 

  254. 

  255. #ifndef NO_JUMP_TABLE
    256. 

  256. void** beam_ops;
    257. 

  257. #endif
    258. 

  258. 

  259. #ifndef ERTS_SMP /* Not supported with smp emulator */
    260. 

  260. extern int count_instructions;
    261. 

  261. #endif
    262. 

  262. 

  263. #if defined(HYBRID)
    264. 

  264. #define SWAPIN             \
    265. 

  265.     g_htop = global_htop;  \
    266. 

  266.     g_hend = global_hend;  \
    267. 

  267.     HTOP = HEAP_TOP(c_p);  \
    268. 

  268.     E = c_p->stop
    269. 

  269. 

  270. #define SWAPOUT            \
    271. 

  271.     global_htop = g_htop;  \
    272. 

  272.     global_hend = g_hend;  \
    273. 

  273.     HEAP_TOP(c_p) = HTOP;  \
    274. 

  274.     c_p->stop = E
    275. 

  275. 

  276. #else
    277. 

  277. #define SWAPIN             \
    278. 

  278.     HTOP = HEAP_TOP(c_p);  \
    279. 

  279.     E = c_p->stop
    280. 

  280. 

  281. #define SWAPOUT            \
    282. 

  282.     HEAP_TOP(c_p) = HTOP;  \
    283. 

  283.     c_p->stop = E
    284. 

  284. #endif
    285. 

  285. 

  286. #define PRE_BIF_SWAPOUT(P)						\
    287. 

  287.      HEAP_TOP((P)) = HTOP;  						\
    288. 

  288.      (P)->stop = E;  							\
    289. 

  289.      PROCESS_MAIN_CHK_LOCKS((P));					\
    290. 

  290.      ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P))
    291. 

  291. 

  292. #if defined(HYBRID)
    293. 

  293. #  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
    294. 

  294.      if ((_p)->mbuf) {							\
    295. 

  295.        _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
    296. 

  296.      }									\
    297. 

  297.      SWAPIN
    298. 

  298. 

  299. #  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
    300. 

  300.      if ((_p)->mbuf) {							\
    301. 

  301.        _regs[0] = r(0);							\
    302. 

  302.        _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
    303. 

  303.        r(0) = _regs[0];							\
    304. 

  304.      }									\
    305. 

  305.      SWAPIN
    306. 

  306. #else
    307. 

  307. #  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
    308. 

  308.      ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
    309. 

  309.      PROCESS_MAIN_CHK_LOCKS((_p));					\
    310. 

  310.      if ((_p)->mbuf) {							\
    311. 

  311.        _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
    312. 

  312.        E = (_p)->stop;							\
    313. 

  313.      }									\
    314. 

  314.      HTOP = HEAP_TOP((_p))
    315. 

  315. 

  316. #  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
    317. 

  317.      ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
    318. 

  318.      PROCESS_MAIN_CHK_LOCKS((_p));					\
    319. 

  319.      if ((_p)->mbuf) {							\
    320. 

  320.        _regs[0] = r(0);							\
    321. 

  321.        _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
    322. 

  322.        r(0) = _regs[0];							\
    323. 

  323.        E = (_p)->stop;							\
    324. 

  324.      }									\
    325. 

  325.      HTOP = HEAP_TOP((_p))
    326. 

  326. #endif
    327. 

  327. 

  328. #define SAVE_HTOP HEAP_TOP(c_p) = HTOP
    329. 

  329. 

  330. #define db(N) (N)
    331. 

  331. #define tb(N) (N)
    332. 

  332. #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
    333. 

  333. #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
    334. 

  334. #define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
    335. 

  335. #define x(N) reg[N]
    336. 

  336. #define y(N) E[N]
    337. 

  337. #define r(N) x##N
    338. 

  338. 

  339. /*
    340. 

  340.  * Makes sure that there are StackNeed + HeapNeed + 1 words available
    341. 

  341.  * on the combined heap/stack segment, then allocates StackNeed + 1
    342. 

  342.  * words on the stack and saves CP.
    343. 

  343.  *
    344. 

  344.  * M is number of live registers to preserve during garbage collection
    345. 

  345.  */
    346. 

  346. 

  347. #define AH(StackNeed, HeapNeed, M) \
    348. 

  348.   do { \
    349. 

  349.      int needed; \
    350. 

  350.      needed = (StackNeed) + 1; \
    351. 

  351.      if (E - HTOP < (needed + (HeapNeed))) { \
    352. 

  352.            SWAPOUT; \
    353. 

  353.            reg[0] = r(0); \
    354. 

  354.            PROCESS_MAIN_CHK_LOCKS(c_p); \
    355. 

  355.            FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
    356. 

  356.            PROCESS_MAIN_CHK_LOCKS(c_p); \
    357. 

  357.            r(0) = reg[0]; \
    358. 

  358.            SWAPIN; \
    359. 

  359.      } \
    360. 

  360.      E -= needed; \
    361. 

  361.      SAVE_CP(E); \
    362. 

  362.   } while (0)
    363. 

  363. 

  364. #define Allocate(Ns, Live) AH(Ns, 0, Live)
    365. 

  365. 

  366. #define AllocateZero(Ns, Live)             \
    367. 

  367.  do { Eterm* ptr;                          \
    368. 

  368.       int i = (Ns);                        \
    369. 

  369.       AH(i, 0, Live);                      \
    370. 

  370.       for (ptr = E + i; ptr > E; ptr--) {  \
    371. 

  371. 	 make_blank(*ptr);                 \
    372. 

  372.      }                                     \
    373. 

  373.   } while (0)
    374. 

  374. 

  375. #define AllocateHeap(Ns, Nh, Live) AH(Ns, Nh, Live)
    376. 

  376. 

  377. #define AllocateHeapZero(Ns, Nh, Live)     \
    378. 

  378.  do { Eterm* ptr;                          \
    379. 

  379.       int i = (Ns);                        \
    380. 

  380.       AH(i, Nh, Live);                     \
    381. 

  381.       for (ptr = E + i; ptr > E; ptr--) {  \
    382. 

  382. 	 make_blank(*ptr);                 \
    383. 

  383.      }                                     \
    384. 

  384.   } while (0)
    385. 

  385. 

  386. #define AllocateInit(Ns, Live, Y) \
    387. 

  387.   do { AH(Ns, 0, Live); make_blank(Y); } while (0)
    388. 

  388. 

  389. /*
    390. 

  390.  * Like the AH macro, but allocates no additional heap space.
    391. 

  391.  */
    392. 

  392. 

  393. #define A(StackNeed, M) AH(StackNeed, 0, M)
    394. 

  394. 

  395. #define D(N)             \
    396. 

  396.      RESTORE_CP(E);      \
    397. 

  397.      E += (N) + 1;
    398. 

  398. 

  399. 

  400. /*
    401. 

  401.  * Check if Nh words of heap are available; if not, do a garbage collection.
    402. 

  402.  * Live is number of active argument registers to be preserved.
    403. 

  403.  */
    404. 

  404. 

  405. #define TestHeap(Nh, Live)                                      \
    406. 

  406.   do {                                                          \
    407. 

  407.     unsigned need = (Nh);                                       \
    408. 

  408.     if (E - HTOP < need) {                                      \
    409. 

  409.        SWAPOUT;                                                 \
    410. 

  410.        reg[0] = r(0);                                           \
    411. 

  411.        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
    412. 

  412.        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
    413. 

  413.        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
    414. 

  414.        r(0) = reg[0];                                           \
    415. 

  415.        SWAPIN;                                                  \
    416. 

  416.     }                                                           \
    417. 

  417.   } while (0)
    418. 

  418. 

  419. /*
    420. 

  420.  * Check if Nh words of heap are available; if not, do a garbage collection.
    421. 

  421.  * Live is number of active argument registers to be preserved.
    422. 

  422.  * Takes special care to preserve Extra if a garbage collection occurs.
    423. 

  423.  */
    424. 

  424. 

  425. #define TestHeapPreserve(Nh, Live, Extra)				\
    426. 

  426.   do {									\
    427. 

  427.     unsigned need = (Nh);						\
    428. 

  428.     if (E - HTOP < need) {						\
    429. 

  429.        SWAPOUT;								\
    430. 

  430.        reg[0] = r(0);							\
    431. 

  431.        reg[Live] = Extra;						\
    432. 

  432.        PROCESS_MAIN_CHK_LOCKS(c_p);					\
    433. 

  433.        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
    434. 

  434.        PROCESS_MAIN_CHK_LOCKS(c_p);					\
    435. 

  435.        if (Live > 0) {							\
    436. 

  436. 	   r(0) = reg[0];						\
    437. 

  437.        }								\
    438. 

  438.        Extra = reg[Live];						\
    439. 

  439.        SWAPIN;								\
    440. 

  440.     }									\
    441. 

  441.   } while (0)
    442. 

  442. 

  443. #ifdef HYBRID
    444. 

  444. #ifdef INCREMENTAL
    445. 

  445. #define TestGlobalHeap(Nh, Live, hp)                                    \
    446. 

  446.   do {                                                                  \
    447. 

  447.     unsigned need = (Nh);                                               \
    448. 

  448.     ASSERT(global_heap <= g_htop && g_htop <= global_hend);             \
    449. 

  449.     SWAPOUT;                                                            \
    450. 

  450.     reg[0] = r(0);                                                      \
    451. 

  451.     FCALLS -= need;                                                     \
    452. 

  452.     (hp) = IncAlloc(c_p,need,reg,(Live));                               \
    453. 

  453.     r(0) = reg[0];                                                      \
    454. 

  454.     SWAPIN;                                                             \
    455. 

  455.   } while (0)
    456. 

  456. #else
    457. 

  457. #define TestGlobalHeap(Nh, Live, hp)                                    \
    458. 

  458.   do {                                                                  \
    459. 

  459.     unsigned need = (Nh);                                               \
    460. 

  460.     ASSERT(global_heap <= g_htop && g_htop <= global_hend);             \
    461. 

  461.     if (g_hend - g_htop < need) {                                       \
    462. 

  462.        SWAPOUT;                                                         \
    463. 

  463.        reg[0] = r(0);                                                   \
    464. 

  464.        FCALLS -= erts_global_garbage_collect(c_p, need, reg, (Live));   \
    465. 

  465.        r(0) = reg[0];                                                   \
    466. 

  466.        SWAPIN;                                                          \
    467. 

  467.     }                                                                   \
    468. 

  468.     (hp) = global_htop;                                                 \
    469. 

  469.   } while (0)
    470. 

  470. #endif
    471. 

  471. #endif /* HYBRID */
    472. 

  472. 

  473. #define Init(N) make_blank(yb(N))
    474. 

  474. 

  475. #define Init2(Y1, Y2) do { make_blank(Y1); make_blank(Y2); } while (0)
    476. 

  476. #define Init3(Y1, Y2, Y3) \
    477. 

  477.    do { make_blank(Y1); make_blank(Y2); make_blank(Y3); } while (0)
    478. 

  478. 

  479. #define MakeFun(FunP, NumFree)					\
    480. 

  480.   do {								\
    481. 

  481.      SWAPOUT;							\
    482. 

  482.      reg[0] = r(0);						\
    483. 

  483.      r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
    484. 

  484.      SWAPIN;							\
    485. 

  485.   } while (0)
    486. 

  486. 

  487. 

  488. /*
    489. 

  489.  * Check that we haven't used the reductions and jump to function pointed to by
    490. 

  490.  * the I register.  If we are out of reductions, do a context switch.
    491. 

  491.  */
    492. 

  492. 

  493. #define DispatchMacro()				\
    494. 

  494.   do {						\
    495. 

  495.      Eterm* dis_next;				\
    496. 

  496.      dis_next = (Eterm *) *I;			\
    497. 

  497.      CHECK_ARGS(I);				\
    498. 

  498.      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
    499. 

  499.         FCALLS--;				\
    500. 

  500.         Goto(dis_next);				\
    501. 

  501.      } else {					\
    502. 

  502. 	goto context_switch;			\
    503. 

  503.      }						\
    504. 

  504.  } while (0)
    505. 

  505. 

  506. #define DispatchMacroFun()			\
    507. 

  507.   do {						\
    508. 

  508.      Eterm* dis_next;				\
    509. 

  509.      dis_next = (Eterm *) *I;			\
    510. 

  510.      CHECK_ARGS(I);				\
    511. 

  511.      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
    512. 

  512.         FCALLS--;				\
    513. 

  513.         Goto(dis_next);				\
    514. 

  514.      } else {					\
    515. 

  515. 	goto context_switch_fun;		\
    516. 

  516.      }						\
    517. 

  517.  } while (0)
    518. 

  518. 

  519. #define DispatchMacrox()					\
    520. 

  520.   do {								\
    521. 

  521.      if (FCALLS > 0) {						\
    522. 

  522.         Eterm* dis_next;					\
    523. 

  523.         SET_I(((Export *) Arg(0))->address);			\
    524. 

  524.         dis_next = (Eterm *) *I;				\
    525. 

  525.         FCALLS--;						\
    526. 

  526.         CHECK_ARGS(I);						\
    527. 

  527.         Goto(dis_next);						\
    528. 

  528.      } else if (c_p->ct != NULL && FCALLS > neg_o_reds) {	\
    529. 

  529.         goto save_calls1;					\
    530. 

  530.      } else {							\
    531. 

  531.         SET_I(((Export *) Arg(0))->address);			\
    532. 

  532.         CHECK_ARGS(I);						\
    533. 

  533. 	goto context_switch;					\
    534. 

  534.      }								\
    535. 

  535.  } while (0)
    536. 

  536. 

  537. #ifdef DEBUG
    538. 

  538. /*
    539. 

  539.  * To simplify breakpoint setting, put the code in one place only and jump to it.
    540. 

  540.  */
    541. 

  541. #  define Dispatch() goto do_dispatch
    542. 

  542. #  define Dispatchx() goto do_dispatchx
    543. 

  543. #  define Dispatchfun() goto do_dispatchfun
    544. 

  544. #else
    545. 

  545. /*
    546. 

  546.  * Inline for speed.
    547. 

  547.  */
    548. 

  548. #  define Dispatch() DispatchMacro()
    549. 

  549. #  define Dispatchx() DispatchMacrox()
    550. 

  550. #  define Dispatchfun() DispatchMacroFun()
    551. 

  551. #endif
    552. 

  552. 

  553. #define Self(R) R = c_p->id
    554. 

  554. #define Node(R) R = erts_this_node->sysname
    555. 

  555. 

  556. #define Arg(N)       I[(N)+1]
    557. 

  557. #define Next(N)                \
    558. 

  558.     I += (N) + 1;              \
    559. 

  559.     ASSERT(VALID_INSTR(*I));   \
    560. 

  560.     Goto(*I)
    561. 

  561. 

  562. #define PreFetch(N, Dst) do { Dst = (Eterm *) *(I + N + 1); } while (0)
    563. 

  563. #define NextPF(N, Dst)         \
    564. 

  564.     I += N + 1;                \
    565. 

  565.     ASSERT(VALID_INSTR(Dst));  \
    566. 

  566.     Goto(Dst)
    567. 

  567. 

  568. #define GetR(pos, tr) \
    569. 

  569.    do { \
    570. 

  570.      tr = Arg(pos); \
    571. 

  571.      switch (beam_reg_tag(tr)) { \
    572. 

  572.      case R_REG_DEF: tr = r(0); break; \
    573. 

  573.      case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \
    574. 

  574.      case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \
    575. 

  575.      } \
    576. 

  576.      CHECK_TERM(tr); \
    577. 

  577.    } while (0)
    578. 

  578. 

  579. #define GetArg1(N, Dst) GetR((N), Dst)
    580. 

  580. 

  581. #define GetArg2(N, Dst1, Dst2)     \
    582. 

  582.    do {                            \
    583. 

  583.      GetR(N, Dst1);                \
    584. 

  584.      GetR((N)+1, Dst2);            \
    585. 

  585.    } while (0)
    586. 

  586. 

  587. #define PutList(H, T, Dst, Store)  \
    588. 

  588.   do {                             \
    589. 

  589.    HTOP[0] = (H); HTOP[1] = (T);   \
    590. 

  590.    Store(make_list(HTOP), Dst);    \
    591. 

  591.    HTOP += 2;                      \
    592. 

  592.   } while (0)
    593. 

  593. 

  594. #define Move(Src, Dst, Store)      \
    595. 

  595.    do {                            \
    596. 

  596.        Eterm term = (Src);         \
    597. 

  597.        Store(term, Dst);           \
    598. 

  598.    } while (0)
    599. 

  599. 

  600. #define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2)
    601. 

  601. 

  602. #define MoveGenDest(src, dstp) \
    603. 

  603.    if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; }
    604. 

  604. 

  605. #define MoveReturn(Src, Dest)       \
    606. 

  606.     (Dest) = (Src);                 \
    607. 

  607.     I = c_p->cp;                    \
    608. 

  608.     ASSERT(VALID_INSTR(*c_p->cp));  \
    609. 

  609.     CHECK_TERM(r(0));               \
    610. 

  610.     Goto(*I)
    611. 

  611. 

  612. #define DeallocateReturn(Deallocate)       \
    613. 

  613.   do {                                     \
    614. 

  614.     int words_to_pop = (Deallocate);       \
    615. 

  615.     SET_I(cp_val(*E));                     \
    616. 

  616.     E = ADD_BYTE_OFFSET(E, words_to_pop);  \
    617. 

  617.     CHECK_TERM(r(0));                      \
    618. 

  618.     Goto(*I);                              \
    619. 

  619.   } while (0)
    620. 

  620. 

  621. #define MoveDeallocateReturn(Src, Dest, Deallocate)  \
    622. 

  622.     (Dest) = (Src);                                  \
    623. 

  623.     DeallocateReturn(Deallocate)
    624. 

  624. 

  625. #define MoveCall(Src, Dest, CallDest, Size)	\
    626. 

  626.     (Dest) = (Src);				\
    627. 

  627.     SET_CP(c_p, I+Size+1);			\
    628. 

  628.     SET_I((Eterm *) CallDest);			\
    629. 

  629.     Dispatch();
    630. 

  630. 

  631. #define MoveCallLast(Src, Dest, CallDest, Deallocate)	\
    632. 

  632.     (Dest) = (Src);					\
    633. 

  633.     RESTORE_CP(E);					\
    634. 

  634.     E = ADD_BYTE_OFFSET(E, (Deallocate));		\
    635. 

  635.     SET_I((Eterm *) CallDest);				\
    636. 

  636.     Dispatch();
    637. 

  637. 

  638. #define MoveCallOnly(Src, Dest, CallDest)	\
    639. 

  639.     (Dest) = (Src);				\
    640. 

  640.     SET_I((Eterm *) CallDest);			\
    641. 

  641.     Dispatch();
    642. 

  642. 

  643. #define GetList(Src, H, T) do {			\
    644. 

  644.    Eterm* tmp_ptr = list_val(Src);		\
    645. 

  645.    H = CAR(tmp_ptr);				\
    646. 

  646.    T = CDR(tmp_ptr); } while (0)
    647. 

  647. 

  648. #define GetTupleElement(Src, Element, Dest)				  \
    649. 

  649.   do {									  \
    650. 

  650.     tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element));  \
    651. 

  651.     (Dest) = (*(Eterm *)tmp_arg1);					  \
    652. 

  652.   } while (0)
    653. 

  653. 

  654. #define ExtractNextElement(Dest)				\
    655. 

  655.     tmp_arg1 += sizeof(Eterm);					\
    656. 

  656.     (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1)))
    657. 

  657. 

  658. #define ExtractNextElement2(Dest)		\
    659. 

  659.   do {						\
    660. 

  660.     Eterm* ene_dstp = &(Dest);			\
    661. 

  661.     ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
    662. 

  662.     ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
    663. 

  663.     tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);	\
    664. 

  664.   } while (0)
    665. 

  665. 

  666. #define ExtractNextElement3(Dest)		\
    667. 

  667.   do {						\
    668. 

  668.     Eterm* ene_dstp = &(Dest);			\
    669. 

  669.     ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
    670. 

  670.     ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
    671. 

  671.     ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
    672. 

  672.     tmp_arg1 += 3*sizeof(Eterm);		\
    673. 

  673.   } while (0)
    674. 

  674. 

  675. #define ExtractNextElement4(Dest)		\
    676. 

  676.   do {						\
    677. 

  677.     Eterm* ene_dstp = &(Dest);			\
    678. 

  678.     ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
    679. 

  679.     ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
    680. 

  680.     ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
    681. 

  681.     ene_dstp[3] = ((Eterm *) tmp_arg1)[4];	\
    682. 

  682.     tmp_arg1 += 4*sizeof(Eterm);		\
    683. 

  683.   } while (0)
    684. 

  684. 

  685. #define ExtractElement(Element, Dest)		\
    686. 

  686.   do {						\
    687. 

  687.      tmp_arg1 += (Element);			\
    688. 

  688.      (Dest) = (* (Eterm *) tmp_arg1);		\
    689. 

  689.   } while (0)
    690. 

  690. 

  691. #define PutTuple(Arity, Src, Dest)		\
    692. 

  692.      ASSERT(is_arity_value(Arity));		\
    693. 

  693.      Dest = make_tuple(HTOP);			\
    694. 

  694.      HTOP[0] = (Arity);				\
    695. 

  695.      HTOP[1] = (Src);				\
    696. 

  696.      HTOP += 2
    697. 

  697. 

  698. #define Put(Word) *HTOP++ = (Word)
    699. 

  699. 

  700. #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
    701. 

  701. 

  702. #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
    703. 

  703. 

  704. #define IsInteger(Src, Fail) if (is_not_integer(Src)) { Fail; }
    705. 

  705. 

  706. #define IsNumber(X, Fail) if (is_not_integer(X) && is_not_float(X)) { Fail; }
    707. 

  707. 

  708. #define IsConstant(X, Fail) if (is_list(X) || is_nil(X) || is_tuple(X)) { Fail; }
    709. 

  709. 

  710. #define IsAtom(Src, Fail) if (is_not_atom(Src)) { Fail; }
    711. 

  711. 

  712. #define IsIntegerAllocate(Src, Need, Alive, Fail)  \
    713. 

  713.     if (is_not_integer(Src)) { Fail; }             \
    714. 

  714.     A(Need, Alive)
    715. 

  715. 

  716. #define IsNil(Src, Fail) if (is_not_nil(Src)) { Fail; }
    717. 

  717. 

  718. #define IsList(Src, Fail) if (is_not_list(Src) && is_not_nil(Src)) { Fail; }
    719. 

  719. 

  720. #define IsNonemptyList(Src, Fail) if (is_not_list(Src)) { Fail; }
    721. 

  721. 

  722. #define IsNonemptyListAllocate(Src, Need, Alive, Fail)  \
    723. 

  723.     if (is_not_list(Src)) { Fail; }                     \
    724. 

  724.     A(Need, Alive)
    725. 

  725. 

  726. #define IsNonemptyListTestHeap(Src, Need, Alive, Fail)  \
    727. 

  727.     if (is_not_list(Src)) { Fail; }                     \
    728. 

  728.     TestHeap(Need, Alive)
    729. 

  729. 

  730. #define IsTuple(X, Action) if (is_not_tuple(X)) Action
    731. 

  731. 

  732. #define IsArity(Pointer, Arity, Fail) \
    733. 

  733.     if (*(Eterm *)(tmp_arg1 = (Eterm)tuple_val(Pointer)) != (Arity)) { Fail; }
    734. 

  734. 

  735. #define IsFunction(X, Action)			\
    736. 

  736.   do {						\
    737. 

  737.      if ( !(is_any_fun(X)) ) {			\
    738. 

  738.           Action;				\
    739. 

  739.      }						\
    740. 

  740.   } while (0)
    741. 

  741. 

  742. #define IsFunction2(F, A, Action)		\
    743. 

  743.   do {						\
    744. 

  744.      if (is_function_2(c_p, F, A) != am_true ) {\
    745. 

  745.           Action;				\
    746. 

  746.      }						\
    747. 

  747.   } while (0)
    748. 

  748. 

  749. #define IsTupleOfArity(Src, Arity, Fail) \
    750. 

  750.   do { \
    751. 

  751.     if (is_not_tuple(Src) || *(Eterm *)(tmp_arg1 = (Eterm) tuple_val(Src)) != Arity) { \
    752. 

  752.         Fail; \
    753. 

  753.     } \
    754. 

  754.   } while (0)
    755. 

  755. 

  756. #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
    757. 

  757. 

  758. #define IsBinary(Src, Fail) \
    759. 

  759.  if (is_not_binary(Src) || binary_bitsize(Src) != 0) { Fail; }
    760. 

  760. 

  761. #define IsBitstring(Src, Fail) \
    762. 

  762.   if (is_not_binary(Src)) { Fail; }
    763. 

  763. 

  764. #ifdef ARCH_64
    765. 

  765. #define BsSafeMul(A, B, Fail, Target)		\
    766. 

  766.    do { Uint64 _res = (A) * (B);		\
    767. 

  767.       if (_res / B != A) { Fail; }		\
    768. 

  768.       Target = _res;				\
    769. 

  769.    } while (0)
    770. 

  770. #else
    771. 

  771. #define BsSafeMul(A, B, Fail, Target)			\
    772. 

  772.    do { Uint64 _res = (Uint64)(A) * (Uint64)(B);	\
    773. 

  773.       if ((_res >> (8*sizeof(Uint))) != 0) { Fail; }	\
    774. 

  774.       Target = _res;					\
    775. 

  775.    } while (0)
    776. 

  776. #endif
    777. 

  777. 

  778. #define BsGetFieldSize(Bits, Unit, Fail, Target)	\
    779. 

  779.    do {							\
    780. 

  780.       Sint _signed_size; Uint _uint_size;		\
    781. 

  781.       if (is_small(Bits)) {				\
    782. 

  782.         _signed_size = signed_val(Bits);		\
    783. 

  783.          if (_signed_size < 0) { Fail; }		\
    784. 

  784.          _uint_size = (Uint) _signed_size;		\
    785. 

  785.       } else {						\
    786. 

  786.          if (!term_to_Uint(Bits, &temp_bits)) { Fail; }	\
    787. 

  787.          _uint_size = temp_bits;			\
    788. 

  788.       }							\
    789. 

  789.       BsSafeMul(_uint_size, Unit, Fail, Target);	\
    790. 

  790.    } while (0)
    791. 

  791. 

  792. #define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target)	\
    793. 

  793.    do {								\
    794. 

  794.       Sint _signed_size; Uint _uint_size;			\
    795. 

  795.       if (is_small(Bits)) {					\
    796. 

  796.         _signed_size = signed_val(Bits);			\
    797. 

  797.          if (_signed_size < 0) { Fail; }			\
    798. 

  798.          _uint_size = (Uint) _signed_size;			\
    799. 

  799.       } else {							\
    800. 

  800.          if (!term_to_Uint(Bits, &temp_bits)) { Fail; }		\
    801. 

  801.          _uint_size = (Uint) temp_bits;				\
    802. 

  802.       }								\
    803. 

  803.       Target = _uint_size * Unit;				\
    804. 

  804.    } while (0)
    805. 

  805. 

  806. #define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Store, Fail)		\
    807. 

  807.  do {									\
    808. 

  808.    ErlBinMatchBuffer *_mb;						\
    809. 

  809.    Eterm _result; Sint _size;						\
    810. 

  810.    if (!is_small(Sz) || (_size = unsigned_val(Sz)) > 64) { Fail; }	\
    811. 

  811.    _size *= ((Flags) >> 3);						\
    812. 

  812.    TestHeap(FLOAT_SIZE_OBJECT, Live);					\
    813. 

  813.    _mb = ms_matchbuffer(Ms);						\
    814. 

  814.    SWAPOUT;								\
    815. 

  815.    _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb);		\
    816. 

  816.    HTOP = HEAP_TOP(c_p);						\
    817. 

  817.    if (is_non_value(_result)) { Fail; }					\
    818. 

  818.    else { Store(_result, Dst); }					\
    819. 

  819.  } while (0)
    820. 

  820. 

  821. #define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
    822. 

  822.   do {								\
    823. 

  823.     ErlBinMatchBuffer *_mb;					\
    824. 

  824.     Eterm _result;						\
    825. 

  825.     TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), Live);	\
    826. 

  826.     _mb = ms_matchbuffer(Ms);					\
    827. 

  827.     SWAPOUT;							\
    828. 

  828.     _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb);	\
    829. 

  829.     HTOP = HEAP_TOP(c_p);					\
    830. 

  830.     if (is_non_value(_result)) { Fail; }			\
    831. 

  831.     else { Store(_result, Dst); }				\
    832. 

  832.   } while (0)
    833. 

  833. 

  834. #define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
    835. 

  835.   do {								\
    836. 

  836.     ErlBinMatchBuffer *_mb;					\
    837. 

  837.     Eterm _result; Uint _size;					\
    838. 

  838.     BsGetFieldSize(Sz, ((Flags) >> 3), Fail, _size);		\
    839. 

  839.     TestHeap(ERL_SUB_BIN_SIZE, Live);				\
    840. 

  840.     _mb = ms_matchbuffer(Ms);					\
    841. 

  841.     SWAPOUT;							\
    842. 

  842.     _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb);	\
    843. 

  843.     HTOP = HEAP_TOP(c_p);					\
    844. 

  844.     if (is_non_value(_result)) { Fail; }			\
    845. 

  845.     else { Store(_result, Dst); }				\
    846. 

  846.   } while (0)
    847. 

  847. 

  848. #define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Store, Fail)	\
    849. 

  849.   do {								\
    850. 

  850.     ErlBinMatchBuffer *_mb;					\
    851. 

  851.     Eterm _result;						\
    852. 

  852.     TestHeap(ERL_SUB_BIN_SIZE, Live);				\
    853. 

  853.     _mb = ms_matchbuffer(Ms);					\
    854. 

  854.     if (((_mb->size - _mb->offset) % Unit) == 0) {		\
    855. 

  855.       SWAPOUT;							\
    856. 

  856.       _result = erts_bs_get_binary_all_2(c_p, _mb);		\
    857. 

  857.       HTOP = HEAP_TOP(c_p);					\
    858. 

  858.       ASSERT(is_value(_result));				\
    859. 

  859.       Store(_result, Dst);					\
    860. 

  860.     } else { Fail; }						\
    861. 

  861.  } while (0)
    862. 

  862. 

  863. #define BsSkipBits2(Ms, Bits, Unit, Fail)			\
    864. 

  864.  do {								\
    865. 

  865.    ErlBinMatchBuffer *_mb;					\
    866. 

  866.    size_t new_offset;						\
    867. 

  867.    Uint _size;							\
    868. 

  868.     _mb = ms_matchbuffer(Ms);					\
    869. 

  869.    BsGetFieldSize(Bits, Unit, Fail, _size);			\
    870. 

  870.    new_offset = _mb->offset + _size;				\
    871. 

  871.    if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
    872. 

  872.    else { Fail; }						\
    873. 

  873.  } while (0)
    874. 

  874. 

  875. #define BsSkipBitsAll2(Ms, Unit, Fail)		\
    876. 

  876.  do {						\
    877. 

  877.     ErlBinMatchBuffer *_mb;			\
    878. 

  878.    _mb = ms_matchbuffer(Ms);			\
    879. 

  879.    if (((_mb->size - _mb->offset) % Unit) == 0) {_mb->offset = _mb->size; } \
    880. 

  880.    else { Fail; }					\
    881. 

  881.  } while (0)
    882. 

  882. 

  883. #define BsSkipBitsImm2(Ms, Bits, Fail)				\
    884. 

  884.  do {								\
    885. 

  885.    ErlBinMatchBuffer *_mb;					\
    886. 

  886.    size_t new_offset;						\
    887. 

  887.    _mb = ms_matchbuffer(Ms);					\
    888. 

  888.    new_offset = _mb->offset + (Bits);				\
    889. 

  889.    if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
    890. 

  890.    else { Fail; }						\
    891. 

  891.  } while (0)
    892. 

  892. 

  893. #define NewBsPutIntegerImm(Sz, Flags, Src)					\
    894. 

  894.  do {									\
    895. 

  895.     if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), (Sz), (Flags)))) { goto badarg; }	\
    896. 

  896.  } while (0)
    897. 

  897. 

  898. #define NewBsPutInteger(Sz, Flags, Src)						\
    899. 

  899.  do {										\
    900. 

  900.     Sint _size;									\
    901. 

  901.     BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
    902. 

  902.     if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), _size, (Flags))))	\
    903. 

  903.        { goto badarg; }								\
    904. 

  904.  } while (0)
    905. 

  905. 

  906. #define NewBsPutFloatImm(Sz, Flags, Src)					\
    907. 

  907.  do {									\
    908. 

  908.     if (!erts_new_bs_put_float(c_p, (Src), (Sz), (Flags))) { goto badarg; }	\
    909. 

  909.  } while (0)
    910. 

  910. 

  911. #define NewBsPutFloat(Sz, Flags, Src)						\
    912. 

  912.  do {										\
    913. 

  913.     Sint _size;									\
    914. 

  914.     BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
    915. 

  915.     if (!erts_new_bs_put_float(c_p, (Src), _size, (Flags))) { goto badarg; }	\
    916. 

  916.  } while (0)
    917. 

  917. 

  918. #define NewBsPutBinary(Sz, Flags, Src)							\
    919. 

  919.  do {											\
    920. 

  920.     Sint _size;										\
    921. 

  921.     BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);			\
    922. 

  922.     if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), _size))) { goto badarg; }	\
    923. 

  923.  } while (0)
    924. 

  924. 

  925. #define NewBsPutBinaryImm(Sz, Src)				        \
    926. 

  926.  do {							        \
    927. 

  927.     if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), (Sz)))) { goto badarg; }	\
    928. 

  928.  } while (0)
    929. 

  929. 

  930. #define NewBsPutBinaryAll(Src, Unit)							\
    931. 

  931.  do {											\
    932. 

  932.     if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2((Src), (Unit)))) { goto badarg; }	\
    933. 

  933.  } while (0)
    934. 

  934. 

  935. 

  936. #define IsPort(Src, Fail) if (is_not_port(Src)) { Fail; }
    937. 

  937. #define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
    938. 

  938. #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }
    939. 

  939. 

  940. static BifFunction translate_gc_bif(void* gcf);
    941. 

  941. static Eterm* handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf);
    942. 

  942. static Eterm* next_catch(Process* c_p, Eterm *reg);
    943. 

  943. static void terminate_proc(Process* c_p, Eterm Value);
    944. 

  944. static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
    945. 

  945. static void save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg,
    946. 

  946. 			     BifFunction bf, Eterm args);
    947. 

  947. static struct StackTrace * get_trace_from_exc(Eterm exc);
    948. 

  948. static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
    949. 

  949. static Eterm call_error_handler(Process* p, Eterm* ip, Eterm* reg);
    950. 

  950. static Eterm call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg);
    951. 

  951. static Uint* fixed_apply(Process* p, Eterm* reg, Uint arity);
    952. 

  952. static Eterm* apply(Process* p, Eterm module, Eterm function,
    953. 

  953. 		     Eterm args, Eterm* reg);
    954. 

  954. static int hibernate(Process* c_p, Eterm module, Eterm function,
    955. 

  955. 		     Eterm args, Eterm* reg);
    956. 

  956. static Eterm* call_fun(Process* p, int arity, Eterm* reg, Eterm args);
    957. 

  957. static Eterm* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg);
    958. 

  958. static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free);
    959. 

  959. 

  960. #if defined(_OSE_) || defined(VXWORKS)
    961. 

  961. static int init_done;
    962. 

  962. #endif
    963. 

  963. 

  964. void
    965. 

  965. init_emulator(void)
    966. 

  966. {
    967. 

  967. #if defined(_OSE_) || defined(VXWORKS)
    968. 

  968.     init_done = 0;
    969. 

  969. #endif
    970. 

  970.     process_main();
    971. 

  971. }
    972. 

  972. 

  973. /*
    974. 

  974.  * On certain platforms, make sure that the main variables really are placed
    975. 

  975.  * in registers.
    976. 

  976.  */
    977. 

  977. 

  978. #if defined(__GNUC__) && defined(sparc) && !defined(DEBUG)
    979. 

  979. #  define REG_x0 asm("%l0")
    980. 

  980. #  define REG_xregs asm("%l1")
    981. 

  981. #  define REG_htop asm("%l2")
    982. 

  982. #  define REG_stop asm("%l3")
    983. 

  983. #  define REG_I asm("%l4")
    984. 

  984. #  define REG_fcalls asm("%l5")
    985. 

  985. #  define REG_tmp_arg1 asm("%l6")
    986. 

  986. #  define REG_tmp_arg2 asm("%l7")
    987. 

  987. #else
    988. 

  988. #  define REG_x0
    989. 

  989. #  define REG_xregs
    990. 

  990. #  define REG_htop
    991. 

  991. #  define REG_stop
    992. 

  992. #  define REG_I
    993. 

  993. #  define REG_fcalls
    994. 

  994. #  define REG_tmp_arg1
    995. 

  995. #  define REG_tmp_arg2
    996. 

  996. #endif
    997. 

  997. 

  998. /*
    999. 

  999.  * process_main() is called twice:
    1000. 

  1000.  * The first call performs some initialisation, including exporting
    1001. 

  1001.  * the instructions' C labels to the loader.
    1002. 

  1002.  * The second call starts execution of BEAM code. This call never returns.
    1003. 

  1003.  */
    1004. 

  1004. void process_main(void)
    1005. 

  1005. {
    1006. 

  1006. #if !defined(_OSE_) && !defined(VXWORKS)
    1007. 

  1007.     static int init_done = 0;
    1008. 

  1008. #endif
    1009. 

  1009.     Process* c_p = NULL;
    1010. 

  1010.     int reds_used;
    1011. 

  1011. 

  1012.     /*
    1013. 

  1013.      * X register zero; also called r(0)
    1014. 

  1014.      */
    1015. 

  1015.     register Eterm x0 REG_x0 = NIL;
    1016. 

  1016. 

  1017.     /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
    1018. 

  1018.      * in all other cases x0 is used.
    1019. 

  1019.      */
    1020. 

  1020.     register Eterm* reg REG_xregs = NULL;
    1021. 

  1021. 

  1022.     /*
    1023. 

  1023.      * Top of heap (next free location); grows upwards.
    1024. 

  1024.      */
    1025. 

  1025.     register Eterm* HTOP REG_htop = NULL;
    1026. 

  1026. 

  1027. 

  1028. #ifdef HYBRID
    1029. 

  1029.      Eterm *g_htop;
    1030. 

  1030.      Eterm *g_hend;
    1031. 

  1031. #endif
    1032. 

  1032. 

  1033.     /* Stack pointer.  Grows downwards; points
    1034. 

  1034.      * to last item pushed (normally a saved
    1035. 

  1035.      * continuation pointer).
    1036. 

  1036.      */
    1037. 

  1037.     register Eterm* E REG_stop = NULL;
    1038. 

  1038. 

  1039.     /*
    1040. 

  1040.      * Pointer to next threaded instruction.
    1041. 

  1041.      */
    1042. 

  1042.     register Eterm *I REG_I = NULL;
    1043. 

  1043. 

  1044.     /* Number of reductions left.  This function
    1045. 

  1045.      * returns to the scheduler when FCALLS reaches zero.
    1046. 

  1046.      */
    1047. 

  1047.     register Sint FCALLS REG_fcalls = 0;
    1048. 

  1048. 

  1049.     /*
    1050. 

  1050.      * Temporaries used for picking up arguments for instructions.
    1051. 

  1051.      */
    1052. 

  1052.     register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
    1053. 

  1053.     register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
    1054. 

  1054.     Eterm tmp_big[2];		/* Temporary buffer for small bignums. */
    1055. 

  1055. 

  1056. #ifndef ERTS_SMP
    1057. 

  1057.     static Eterm save_reg[ERTS_X_REGS_ALLOCATED];
    1058. 

  1058.     /* X registers -- not used directly, but
    1059. 

  1059.      * through 'reg', because using it directly
    1060. 

  1060.      * needs two instructions on a SPARC,
    1061. 

  1061.      * while using it through reg needs only
    1062. 

  1062.      * one.
    1063. 

  1063.      */
    1064. 

  1064. 

  1065.     /*
    1066. 

  1066.      * Floating point registers.
    1067. 

  1067.      */
    1068. 

  1068.     static FloatDef freg[MAX_REG];
    1069. 

  1069. #else
    1070. 

  1070.     /* X regisers and floating point registers are located in
    1071. 

  1071.      * scheduler specific data.
    1072. 

  1072.      */
    1073. 

  1073.     register FloatDef *freg;
    1074. 

  1074. #endif
    1075. 

  1075. 

  1076.     /*
    1077. 

  1077.      * For keeping the negative old value of 'reds' when call saving is active.
    1078. 

  1078.      */
    1079. 

  1079.     int neg_o_reds = 0;
    1080. 

  1080. 

  1081.     Eterm (*arith_func)(Process* p, Eterm* reg, Uint live);
    1082. 

  1082. 

  1083. #ifndef NO_JUMP_TABLE
    1084. 

  1084.     static void* opcodes[] = { DEFINE_OPCODES };
    1085. 

  1085. #ifdef ERTS_OPCODE_COUNTER_SUPPORT
    1086. 

  1086.     static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
    1087. 

  1087. #endif
    1088. 

  1088. #else
    1089. 

  1089.     int Go;
    1090. 

  1090. #endif
    1091. 

  1091. 

  1092.     Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
    1093. 

  1093. 

  1094.     ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
    1095. 

  1095. 

  1096. 

  1097.     /*
    1098. 

  1098.      * Note: In this function, we attempt to place rarely executed code towards
    1099. 

  1099.      * the end of the function, in the hope that the cache hit rate will be better.
    1100. 

  1100.      * The initialization code is only run once, so it is at the very end.
    1101. 

  1101.      *
    1102. 

  1102.      * Note: c_p->arity must be set to reflect the number of useful terms in
    1103. 

  1103.      * c_p->arg_reg before calling the scheduler.
    1104. 

  1104.      */
    1105. 

  1105. 

  1106.     if (!init_done) {
    1107. 

  1107. 	init_done = 1;
    1108. 

  1108. 	goto init_emulator;
    1109. 

  1109.     }
    1110. 

  1110. #ifndef ERTS_SMP
    1111. 

  1111.     reg = save_reg;	/* XXX: probably wastes a register on x86 */
    1112. 

  1112. #endif
    1113. 

  1113.     c_p = NULL;
    1114. 

  1114.     reds_used = 0;
    1115. 

  1115.     goto do_schedule1;
    1116. 

  1116. 

  1117.  do_schedule:
    1118. 

  1118.     reds_used = REDS_IN(c_p) - FCALLS;
    1119. 

  1119.  do_schedule1:
    1120. 

  1120.     PROCESS_MAIN_CHK_LOCKS(c_p);
    1121. 

  1121.     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    1122. 

  1122.     c_p = schedule(c_p, reds_used);
    1123. 

  1123.     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    1124. 

  1124.     PROCESS_MAIN_CHK_LOCKS(c_p);
    1125. 

  1125. #ifdef ERTS_SMP
    1126. 

  1126.     reg = c_p->scheduler_data->save_reg;
    1127. 

  1127.     freg = c_p->scheduler_data->freg;
    1128. 

  1128. #endif
    1129. 

  1129.     ERL_BITS_RELOAD_STATEP(c_p);
    1130. 

  1130.     {
    1131. 

  1131. 	int reds;
    1132. 

  1132. 	Eterm* argp;
    1133. 

  1133. 	Eterm* next;
    1134. 

  1134. 	int i;
    1135. 

  1135. 

  1136. 	argp = c_p->arg_reg;
    1137. 

  1137. 	for (i = c_p->arity - 1; i > 0; i--) {
    1138. 

  1138. 	    reg[i] = argp[i];
    1139. 

  1139. 	    CHECK_TERM(reg[i]);
    1140. 

  1140. 	}
    1141. 

  1141. 

  1142. 	/*
    1143. 

  1143. 	 * We put the original reduction count in the process structure, to reduce
    1144. 

  1144. 	 * the code size (referencing a field in a struct through a pointer stored
    1145. 

  1145. 	 * in a register gives smaller code than referencing a global variable).
    1146. 

  1146. 	 */
    1147. 

  1147. 

  1148. 	SET_I(c_p->i);
    1149. 

  1149. 

  1150. 	reds = c_p->fcalls;
    1151. 

  1151. 	if (c_p->ct != NULL && (c_p->trace_flags & F_SENSITIVE) == 0) {
    1152. 

  1152. 	    neg_o_reds = -reds;
    1153. 

  1153. 	    FCALLS = REDS_IN(c_p) = 0;
    1154. 

  1154. 	} else {
    1155. 

  1155. 	    neg_o_reds = 0;
    1156. 

  1156. 	    FCALLS = REDS_IN(c_p) = reds;
    1157. 

  1157. 	}
    1158. 

  1158. 

  1159. 	next = (Eterm *) *I;
    1160. 

  1160. 	r(0) = c_p->arg_reg[0];
    1161. 

  1161. #ifdef HARDDEBUG
    1162. 

  1162. 	if (c_p->arity > 0) {
    1163. 

  1163. 	    CHECK_TERM(r(0));
    1164. 

  1164. 	}
    1165. 

  1165. #endif
    1166. 

  1166. 	SWAPIN;
    1167. 

  1167. 	ASSERT(VALID_INSTR(next));
    1168. 

  1168. 	Goto(next);
    1169. 

  1169.     }
    1170. 

  1170. 

  1171. #if defined(DEBUG) || defined(NO_JUMP_TABLE)
    1172. 

  1172.  emulator_loop:
    1173. 

  1173. #endif
    1174. 

  1174. 

  1175. #ifdef NO_JUMP_TABLE
    1176. 

  1176.     switch (Go) {
    1177. 

  1177. #endif
    1178. 

  1178. #include "beam_hot.h"
    1179. 

  1179. 

  1180. #define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
    1181. 

  1181. #define ARITH_FUNC(name) erts_gc_##name
    1182. 

  1182. 

  1183.  OpCase(i_plus_jId):
    1184. 

  1184.  {
    1185. 

  1185.      Eterm result;
    1186. 

  1186. 

  1187.      if (is_both_small(tmp_arg1, tmp_arg2)) {
    1188. 

  1188. 	 Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2);
    1189. 

  1189. 	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
    1190. 

  1190. 	 if (MY_IS_SSMALL(i)) {
    1191. 

  1191. 	     result = make_small(i);
    1192. 

  1192. 	     STORE_ARITH_RESULT(result);
    1193. 

  1193. 	 }
    1194. 

  1194.      
    1195. 

  1195.      }
    1196. 

  1196.      arith_func = ARITH_FUNC(mixed_plus);
    1197. 

  1197.      goto do_big_arith2;
    1198. 

  1198.  }
    1199. 

  1199. 

  1200.  OpCase(i_minus_jId):
    1201. 

  1201.  {
    1202. 

  1202.      Eterm result;
    1203. 

  1203. 

  1204.      if (is_both_small(tmp_arg1, tmp_arg2)) {
    1205. 

  1205. 	 Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2);
    1206. 

  1206. 	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
    1207. 

  1207. 	 if (MY_IS_SSMALL(i)) {
    1208. 

  1208. 	     result = make_small(i);
    1209. 

  1209. 	     STORE_ARITH_RESULT(result);
    1210. 

  1210. 	 }
    1211. 

  1211.      }
    1212. 

  1212.      arith_func = ARITH_FUNC(mixed_minus);
    1213. 

  1213.      goto do_big_arith2;
    1214. 

  1214.  }
    1215. 

  1215. 

  1216.  OpCase(i_is_lt_f):
    1217. 

  1217.     if (CMP_GE(tmp_arg1, tmp_arg2)) {
    1218. 

  1218. 	ClauseFail();
    1219. 

  1219.     }
    1220. 

  1220.     Next(1);
    1221. 

  1221. 

  1222.  OpCase(i_is_ge_f):
    1223. 

  1223.     if (CMP_LT(tmp_arg1, tmp_arg2)) {
    1224. 

  1224. 	ClauseFail();
    1225. 

  1225.     }
    1226. 

  1226.     Next(1);
    1227. 

  1227. 

  1228.  OpCase(i_is_eq_f):
    1229. 

  1229.     if (CMP_NE(tmp_arg1, tmp_arg2)) {
    1230. 

  1230. 	ClauseFail();
    1231. 

  1231.     }
    1232. 

  1232.     Next(1);
    1233. 

  1233. 

  1234.  OpCase(i_is_ne_f):
    1235. 

  1235.     if (CMP_EQ(tmp_arg1, tmp_arg2)) {
    1236. 

  1236. 	ClauseFail();
    1237. 

  1237.     }
    1238. 

  1238.     Next(1);
    1239. 

  1239. 

  1240.  OpCase(i_is_eq_exact_f):
    1241. 

  1241.     if (!EQ(tmp_arg1, tmp_arg2)) {
    1242. 

  1242. 	ClauseFail();
    1243. 

  1243.     }
    1244. 

  1244.     Next(1);
    1245. 

  1245. 

  1246.  OpCase(i_move_call_only_fcr): {
    1247. 

  1247.      r(0) = Arg(1);
    1248. 

  1248.  }
    1249. 

  1249.  /* FALL THROUGH */
    1250. 

  1250.  OpCase(i_call_only_f): {
    1251. 

  1251.      SET_I((Eterm *) Arg(0));
    1252. 

  1252.      Dispatch();
    1253. 

  1253.  }
    1254. 

  1254. 

  1255.  OpCase(i_move_call_last_fPcr): {
    1256. 

  1256.      r(0) = Arg(2);
    1257. 

  1257.  }
    1258. 

  1258.  /* FALL THROUGH */
    1259. 

  1259.  OpCase(i_call_last_fP): {
    1260. 

  1260.      RESTORE_CP(E);
    1261. 

  1261.      E = ADD_BYTE_OFFSET(E, Arg(1));
    1262. 

  1262.      SET_I((Eterm *) Arg(0));
    1263. 

  1263.      Dispatch();
    1264. 

  1264.  }
    1265. 

  1265. 

  1266.  OpCase(i_move_call_crf): {
    1267. 

  1267.      r(0) = Arg(0);
    1268. 

  1268.      I++;
    1269. 

  1269.  }
    1270. 

  1270.  /* FALL THROUGH */
    1271. 

  1271.  OpCase(i_call_f): {
    1272. 

  1272.      SET_CP(c_p, I+2);
    1273. 

  1273.      SET_I((Eterm *) Arg(0));
    1274. 

  1274.      Dispatch();
    1275. 

  1275.  }
    1276. 

  1276. 

  1277.  OpCase(i_move_call_ext_last_ePcr): {
    1278. 

  1278.      r(0) = Arg(2);
    1279. 

  1279.  }
    1280. 

  1280.  /* FALL THROUGH */
    1281. 

  1281.  OpCase(i_call_ext_last_eP):
    1282. 

  1282.     RESTORE_CP(E);
    1283. 

  1283.     E = ADD_BYTE_OFFSET(E, Arg(1));
    1284. 

  1284. 

  1285.     /*
    1286. 

  1286.      * Note: The pointer to the export entry is never NULL; if the module
    1287. 

  1287.      * is not loaded, it points to code which will invoke the error handler
    1288. 

  1288.      * (see lb_call_error_handler below).
    1289. 

  1289.      */
    1290. 

  1290.     Dispatchx();
    1291. 

  1291. 

  1292.  OpCase(i_move_call_ext_cre): {
    1293. 

  1293.      r(0) = Arg(0);
    1294. 

  1294.      I++;
    1295. 

  1295.  }
    1296. 

  1296.  /* FALL THROUGH */
    1297. 

  1297.  OpCase(i_call_ext_e):
    1298. 

  1298.     SET_CP(c_p, I+2);
    1299. 

  1299.     Dispatchx();
    1300. 

  1300. 

  1301.  OpCase(i_move_call_ext_only_ecr): {
    1302. 

  1302.      r(0) = Arg(1);
    1303. 

  1303.  }
    1304. 

  1304.  /* FALL THROUGH */
    1305. 

  1305.  OpCase(i_call_ext_only_e):
    1306. 

  1306.     Dispatchx();
    1307. 

  1307. 

  1308.  OpCase(init_y): {
    1309. 

  1309.      Eterm* next;
    1310. 

  1310. 

  1311.      PreFetch(1, next);
    1312. 

  1312.      make_blank(yb(Arg(0)));
    1313. 

  1313.      NextPF(1, next);
    1314. 

  1314.  }
    1315. 

  1315. 

  1316.  OpCase(i_trim_I): {
    1317. 

  1317.      Eterm* next;
    1318. 

  1318.      Uint words;
    1319. 

  1319.      Uint cp;
    1320. 

  1320. 

  1321.      words = Arg(0);
    1322. 

  1322.      cp = E[0];
    1323. 

  1323.      PreFetch(1, next);
    1324. 

  1324.      E += words;
    1325. 

  1325.      E[0] = cp;
    1326. 

  1326.      NextPF(1, next);
    1327. 

  1327.  }
    1328. 

  1328. 

  1329.  OpCase(return):
    1330. 

  1330.     SET_I(c_p->cp);
    1331. 

  1331.     CHECK_TERM(r(0));
    1332. 

  1332.     Goto(*I);
    1333. 

  1333. 

  1334.  OpCase(test_heap_1_put_list_Iy): {
    1335. 

  1335.      Eterm* next;
    1336. 

  1336. 

  1337.      PreFetch(2, next);
    1338. 

  1338.      TestHeap(Arg(0), 1);
    1339. 

  1339.      PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest);
    1340. 

  1340.      CHECK_TERM(r(0));
    1341. 

  1341.      NextPF(2, next);
    1342. 

  1342.  }
    1343. 

  1343. 

  1344.  OpCase(put_string_IId):
    1345. 

  1345.     {
    1346. 

  1346.       unsigned char* s;
    1347. 

  1347.       int len;
    1348. 

  1348.       Eterm result;
    1349. 

  1349. 

  1350.       len = Arg(0);		/* Length. */
    1351. 

  1351.       result = NIL;
    1352. 

  1352.       for (s = (unsigned char *) Arg(1); len > 0; s--, len--) {
    1353. 

  1353. 	  PutList(make_small(*s), result, result, StoreSimpleDest);
    1354. 

  1354.       }
    1355. 

  1355.       StoreBifResult(2, result);
    1356. 

  1356.     }
    1357. 

  1357. 

  1358.     /*
    1359. 

  1359.      * Send is almost a standard call-BIF with two arguments, except for:
    1360. 

  1360.      *    1) It cannot be traced.
    1361. 

  1361.      *	  2) There is no pointer to the send_2 function stored in
    1362. 

  1362.      *       the instruction.
    1363. 

  1363.      */
    1364. 

  1364. 

  1365.  OpCase(send): {
    1366. 

  1366.      Eterm* next;
    1367. 

  1367.      Eterm result;
    1368. 

  1368. 

  1369.      PRE_BIF_SWAPOUT(c_p);
    1370. 

  1370.      c_p->fcalls = FCALLS - 1;
    1371. 

  1371.      result = send_2(c_p, r(0), x(1));
    1372. 

  1372.      PreFetch(0, next);
    1373. 

  1373.      POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
    1374. 

  1374.      FCALLS = c_p->fcalls;
    1375. 

  1375.      if (is_value(result)) {
    1376. 

  1376. 	 r(0) = result;
    1377. 

  1377. 	 CHECK_TERM(r(0));
    1378. 

  1378. 	 NextPF(0, next);
    1379. 

  1379.      } else if (c_p->freason == RESCHEDULE) {
    1380. 

  1380. 	 Eterm* argp;
    1381. 

  1381. 

  1382. 	 c_p->arity = 2;
    1383. 

  1383. 

  1384. 	 /*
    1385. 

  1385. 	  * Moving c_p->arg to a register is shorter than using c_p->arg_reg
    1386. 

  1386. 	  * directly, since c_p->arg_reg is a pointer (not an array)
    1387. 

  1387. 	  * and the compiler generates code to fetch the pointer every time.
    1388. 

  1388. 	  */
    1389. 

  1389. 	 argp = c_p->arg_reg;
    1390. 

  1390. 	 argp[0] = r(0);
    1391. 

  1391. 	 argp[1] = x(1);
    1392. 

  1392. 	 SWAPOUT;
    1393. 

  1393. 	 c_p->i = I;
    1394. 

  1394. 	 c_p->current = NULL;
    1395. 

  1395. 	 goto do_schedule;
    1396. 

  1396.      } else if (c_p->freason == TRAP) {
    1397. 

  1397. 	 SET_CP(c_p, I+1);
    1398. 

  1398. 	 SET_I(((Export *)(c_p->def_arg_reg[3]))->address);
    1399. 

  1399. 	 SWAPIN;
    1400. 

  1400. 	 r(0) = c_p->def_arg_reg[0];
    1401. 

  1401. 	 x(1) = c_p->def_arg_reg[1];
    1402. 

  1402. 	 Dispatch();
    1403. 

  1403.      }
    1404. 

  1404.      goto find_func_info;
    1405. 

  1405.  }
    1406. 

  1406. 

  1407.  OpCase(i_element_jssd): {
    1408. 

  1408.      Eterm index;
    1409. 

  1409.      Eterm tuple;
    1410. 

  1410. 

  1411.      /*
    1412. 

  1412.       * Inlined version of element/2 for speed.
    1413. 

  1413.       */
    1414. 

  1414.      GetArg2(1, index, tuple);
    1415. 

  1415.      if (is_small(index) && is_tuple(tuple)) {
    1416. 

  1416. 	 Eterm* tp = tuple_val(tuple);
    1417. 

  1417. 

  1418. 	 if ((signed_val(index) >= 1) &&
    1419. 

  1419. 	     (signed_val(index) <= arityval(*tp))) {
    1420. 

  1420. 	     Eterm result = tp[signed_val(index)];
    1421. 

  1421. 	     StoreBifResult(3, result);
    1422. 

  1422. 	 }
    1423. 

  1423.      }
    1424. 

  1424.  }
    1425. 

  1425.  /* Fall through */
    1426. 

  1426. 

  1427.  OpCase(badarg_j):
    1428. 

  1428.  badarg:
    1429. 

  1429.     c_p->freason = BADARG;
    1430. 

  1430.     goto lb_Cl_error;
    1431. 

  1431. 

  1432.  OpCase(i_fast_element_jIsd): {
    1433. 

  1433.      Eterm tuple;
    1434. 

  1434. 

  1435.      /*
    1436. 

  1436.       * Inlined version of element/2 for even more speed.
    1437. 

  1437.       * The first argument is an untagged integer >= 1.
    1438. 

  1438.       * The second argument is guaranteed to be a register operand.
    1439. 

  1439.       */
    1440. 

  1440.      GetArg1(2, tuple);
    1441. 

  1441.      if (is_tuple(tuple)) {
    1442. 

  1442. 	 Eterm* tp = tuple_val(tuple);
    1443. 

  1443. 	 tmp_arg2 = Arg(1);
    1444. 

  1444. 	 if (tmp_arg2 <= arityval(*tp)) {
    1445. 

  1445. 	     Eterm result = tp[tmp_arg2];
    1446. 

  1446. 	     StoreBifResult(3, result);
    1447. 

  1447. 	 }
    1448. 

  1448.      }
    1449. 

  1449.      goto badarg;
    1450. 

  1450.  }
    1451. 

  1451. 

  1452.  OpCase(catch_yf):
    1453. 

  1453.      c_p->catches++;
    1454. 

  1454.      yb(Arg(0)) = Arg(1);
    1455. 

  1455.      Next(2);
    1456. 

  1456. 

  1457.  OpCase(catch_end_y): {
    1458. 

  1458.      c_p->catches--;
    1459. 

  1459.      make_blank(yb(Arg(0)));
    1460. 

  1460.      if (is_non_value(r(0))) {
    1461. 

  1461. 	 if (x(1) == am_throw) {
    1462. 

  1462. 	     r(0) = x(2);
    1463. 

  1463. 	 } else {
    1464. 

  1464. 	     if (x(1) == am_error) {
    1465. 

  1465. 	         SWAPOUT;
    1466. 

  1466. 		 x(2) = add_stacktrace(c_p, x(2), x(3));
    1467. 

  1467. 		 SWAPIN;
    1468. 

  1468. 	     }
    1469. 

  1469. 	     /* only x(2) is included in the rootset here */
    1470. 

  1470. 	     if (E - HTOP < 3 || c_p->mbuf) {	/* Force GC in case add_stacktrace()
    1471. 

  1471. 						 * created heap fragments */
    1472. 

  1472. 		 SWAPOUT;
    1473. 

  1473. 		 PROCESS_MAIN_CHK_LOCKS(c_p);
    1474. 

  1474. 		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
    1475. 

  1475. 		 PROCESS_MAIN_CHK_LOCKS(c_p);
    1476. 

  1476. 		 SWAPIN;
    1477. 

  1477. 	     }
    1478. 

  1478. 	     r(0) = TUPLE2(HTOP, am_EXIT, x(2));
    1479. 

  1479. 	     HTOP += 3;
    1480. 

  1480. 	 }
    1481. 

  1481.      }
    1482. 

  1482.      CHECK_TERM(r(0));
    1483. 

  1483.      Next(1);
    1484. 

  1484.  }
    1485. 

  1485. 

  1486.  OpCase(try_end_y): {
    1487. 

  1487.      c_p->catches--;
    1488. 

  1488.      make_blank(yb(Arg(0)));
    1489. 

  1489.      if (is_non_value(r(0))) {
    1490. 

  1490. 	 r(0) = x(1);
    1491. 

  1491. 	 x(1) = x(2);
    1492. 

  1492. 	 x(2) = x(3);
    1493. 

  1493.      }
    1494. 

  1494.      Next(1);
    1495. 

  1495.  }
    1496. 

  1496. 

  1497.  /*
    1498. 

  1498.   * Skeleton for receive statement:
    1499. 

  1499.   *
    1500. 

  1500.   *      L1:          <-------------------+
    1501. 

  1501.   *                   <-----------+       |
    1502. 

  1502.   *     	     	       	  |   	  |
    1503. 

  1503.   *             loop_rec L2 ------+---+   |
    1504. 

  1504.   *             ...               |   |   |
    1505. 

  1505.   *             remove_message 	  |   |	  |
    1506. 

  1506.   *             jump L3           |   |   |
    1507. 

  1507.   *		...	          |   |   |
    1508. 

  1508.   *		loop_rec_end L1 --+   |   |
    1509. 

  1509.   *      L2:          <---------------+   |
    1510. 

  1510.   *	   	wait L1  -----------------+      or wait_timeout
    1511. 

  1511.   *		timeout
    1512. 

  1512.   *
    1513. 

  1513.   *	 L3:    Code after receive...
    1514. 

  1514.   *
    1515. 

  1515.   *
    1516. 

  1516.   */
    1517. 

  1517. 

  1518.     /*
    1519. 

  1519.      * Pick up the next message and place it in x(0).
    1520. 

  1520.      * If no message, jump to a wait or wait_timeout instruction.
    1521. 

  1521.      */
    1522. 

  1522.  OpCase(i_loop_rec_fr):
    1523. 

  1523.  {
    1524. 

  1524.      Eterm* next;
    1525. 

  1525.      ErlMessage* msgp;
    1526. 

  1526. 

  1527.  loop_rec__:
    1528. 

  1528. 

  1529.      PROCESS_MAIN_CHK_LOCKS(c_p);
    1530. 

  1530. 

  1531.      msgp = PEEK_MESSAGE(c_p);
    1532. 

  1532. 

  1533.      if (!msgp) {
    1534. 

  1534. #ifdef ERTS_SMP
    1535. 

  1535. 	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1536. 

  1536. 	 /* Make sure messages wont pass exit signals... */
    1537. 

  1537. 	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
    1538. 

  1538. 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1539. 

  1539. 	     SWAPOUT;
    1540. 

  1540. 	     goto do_schedule; /* Will be rescheduled for exit */
    1541. 

  1541. 	 }
    1542. 

  1542. 	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
    1543. 

  1543. 	 msgp = PEEK_MESSAGE(c_p);
    1544. 

  1544. 	 if (msgp)
    1545. 

  1545. 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1546. 

  1546. 	 else {
    1547. 

  1547. #endif
    1548. 

  1548. 	     SET_I((Eterm *) Arg(0));
    1549. 

  1549. 	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
    1550. 

  1550. #ifdef ERTS_SMP
    1551. 

  1551. 	 }
    1552. 

  1552. #endif
    1553. 

  1553.      }
    1554. 

  1554.      MV_MSG_MBUF_INTO_PROC(msgp);
    1555. 

  1555.      PreFetch(1, next);
    1556. 

  1556.      r(0) = ERL_MESSAGE_TERM(msgp);
    1557. 

  1557.      NextPF(1, next);
    1558. 

  1558.  }
    1559. 

  1559. 

  1560.  /*
    1561. 

  1561.   * Remove a (matched) message from the message queue.
    1562. 

  1562.   */
    1563. 

  1563.  OpCase(remove_message): {
    1564. 

  1564.      Eterm* next;
    1565. 

  1565.      ErlMessage* msgp;
    1566. 

  1566. 

  1567.      PROCESS_MAIN_CHK_LOCKS(c_p);
    1568. 

  1568. 

  1569.      PreFetch(0, next);
    1570. 

  1570.      msgp = PEEK_MESSAGE(c_p);
    1571. 

  1571. 

  1572.      if (c_p->ct != NULL) {
    1573. 

  1573. 	 save_calls(c_p, &exp_receive);
    1574. 

  1574.      }
    1575. 

  1575.      if (ERL_MESSAGE_TOKEN(msgp) == NIL) {
    1576. 

  1576. 	 SEQ_TRACE_TOKEN(c_p) = NIL;
    1577. 

  1577.      } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) {
    1578. 

  1578. 	 Eterm msg;
    1579. 

  1579. 	 SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp);
    1580. 

  1580. 	 ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
    1581. 

  1581. 	 ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
    1582. 

  1582. 	 ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p)));
    1583. 

  1583. 	 ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
    1584. 

  1584. 	 ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p)));
    1585. 

  1585. 	 ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)));
    1586. 

  1586. 	 c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
    1587. 

  1587. 	 if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) {
    1588. 

  1588. 	     c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
    1589. 

  1589. 	 }
    1590. 

  1590. 	 msg = ERL_MESSAGE_TERM(msgp);
    1591. 

  1591. 	 seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, 
    1592. 

  1592. 			  c_p->id, c_p);
    1593. 

  1593.      }
    1594. 

  1594.      UNLINK_MESSAGE(c_p, msgp);
    1595. 

  1595.      JOIN_MESSAGE(c_p);
    1596. 

  1596.      CANCEL_TIMER(c_p);
    1597. 

  1597.      free_message(msgp);
    1598. 

  1598. 

  1599.      PROCESS_MAIN_CHK_LOCKS(c_p);
    1600. 

  1600. 

  1601.      NextPF(0, next);
    1602. 

  1602.  }
    1603. 

  1603. 

  1604.     /*
    1605. 

  1605.      * Advance the save pointer to the next message (the current
    1606. 

  1606.      * message didn't match), then jump to the loop_rec instruction.
    1607. 

  1607.      */
    1608. 

  1608.  OpCase(loop_rec_end_f): {
    1609. 

  1609.      SET_I((Eterm *) Arg(0));
    1610. 

  1610.      SAVE_MESSAGE(c_p);
    1611. 

  1611.      goto loop_rec__;
    1612. 

  1612.  }
    1613. 

  1613.     /*
    1614. 

  1614.      * Prepare to wait for a message or a timeout, whichever occurs first.
    1615. 

  1615.      *
    1616. 

  1616.      * Note: In order to keep the compatibility between 32 and 64 bits
    1617. 

  1617.      * emulators, only timeout values that can be represented in 32 bits
    1618. 

  1618.      * (unsigned) or less are allowed.
    1619. 

  1619.      */
    1620. 

  1620. 

  1621. 

  1622.  OpCase(i_wait_timeout_fs): {
    1623. 

  1623.      erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1624. 

  1624. 

  1625.      /* Fall through */
    1626. 

  1626.  }
    1627. 

  1627.  OpCase(i_wait_timeout_locked_fs): {
    1628. 

  1628.      Eterm timeout_value;
    1629. 

  1629. 

  1630.      /*
    1631. 

  1631.       * If we have already set the timer, we must NOT set it again.  Therefore,
    1632. 

  1632.       * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
    1633. 

  1633.       */
    1634. 

  1634.      if (c_p->flags & (F_INSLPQUEUE | F_TIMO)) {
    1635. 

  1635. 	 goto wait2;
    1636. 

  1636.      }
    1637. 

  1637.      GetArg1(1, timeout_value);
    1638. 

  1638.      if (timeout_value != make_small(0)) {
    1639. 

  1639. #if !defined(ARCH_64)
    1640. 

  1640. 	 Uint time_val;
    1641. 

  1641. #endif
    1642. 

  1642. 

  1643. 	 if (is_small(timeout_value) && signed_val(timeout_value) > 0 &&
    1644. 

  1644. #if defined(ARCH_64)
    1645. 

  1645. 	     ((unsigned_val(timeout_value) >> 32) == 0)
    1646. 

  1646. #else
    1647. 

  1647. 	     1
    1648. 

  1648. #endif
    1649. 

  1649. 	     ) {
    1650. 

  1650. 	     /*
    1651. 

  1651. 	      * The timer routiner will set c_p->i to the value in
    1652. 

  1652. 	      * c_p->def_arg_reg[0].  Note that it is safe to use this
    1653. 

  1653. 	      * location because there are no living x registers in
    1654. 

  1654. 	      * a receive statement.
    1655. 

  1655. 	      */
    1656. 

  1656. 	     c_p->def_arg_reg[0] = (Eterm) (I+3);
    1657. 

  1657. 	     set_timer(c_p, unsigned_val(timeout_value));
    1658. 

  1658. 	 } else if (timeout_value == am_infinity) {
    1659. 

  1659. 	     c_p->flags |= F_TIMO;
    1660. 

  1660. #if !defined(ARCH_64)
    1661. 

  1661. 	 } else if (term_to_Uint(timeout_value, &time_val)) {
    1662. 

  1662. 	     c_p->def_arg_reg[0] = (Eterm) (I+3);
    1663. 

  1663. 	     set_timer(c_p, time_val);
    1664. 

  1664. #endif
    1665. 

  1665. 	 } else {		/* Wrong time */
    1666. 

  1666. 	     OpCase(i_wait_error_locked): {
    1667. 

  1667. 		 erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1668. 

  1668. 		 /* Fall through */
    1669. 

  1669. 	     }
    1670. 

  1670. 	     OpCase(i_wait_error): {
    1671. 

  1671. 		 c_p->freason = EXC_TIMEOUT_VALUE;
    1672. 

  1672. 		 goto find_func_info;
    1673. 

  1673. 	     }
    1674. 

  1674. 	 }
    1675. 

  1675. 

  1676. 	 /*
    1677. 

  1677. 	  * Prepare to wait indefinitely for a new message to arrive
    1678. 

  1678. 	  * (or the time set above if falling through from above).
    1679. 

  1679. 	  *
    1680. 

  1680. 	  * When a new message arrives, control will be transferred
    1681. 

  1681. 	  * the loop_rec instruction (at label L1).  In case of
    1682. 

  1682. 	  * of timeout, control will be transferred to the timeout
    1683. 

  1683. 	  * instruction following the wait_timeout instruction.
    1684. 

  1684. 	  */
    1685. 

  1685. 

  1686. 	 OpCase(wait_locked_f):
    1687. 

  1687. 	 OpCase(wait_f):
    1688. 

  1688. 

  1689. 	 wait2: {
    1690. 

  1690. 	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    1691. 

  1691. 	     c_p->i = (Eterm *) Arg(0); /* L1 */
    1692. 

  1692. 	     SWAPOUT;
    1693. 

  1693. 	     c_p->arity = 0;
    1694. 

  1694. 	     c_p->status = P_WAITING;
    1695. 

  1695. 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1696. 

  1696. 	     c_p->current = NULL;
    1697. 

  1697. 	     goto do_schedule;
    1698. 

  1698. 	 }
    1699. 

  1699. 	 OpCase(wait_unlocked_f): {
    1700. 

  1700. 	     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1701. 

  1701. 	     goto wait2;
    1702. 

  1702. 	 }
    1703. 

  1703.      }
    1704. 

  1704.      erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1705. 

  1705.      Next(2);
    1706. 

  1706.  }
    1707. 

  1707. 

  1708.  OpCase(i_wait_timeout_fI): {
    1709. 

  1709.      erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1710. 

  1710.  }
    1711. 

  1711. 

  1712.  OpCase(i_wait_timeout_locked_fI):
    1713. 

  1713.  {
    1714. 

  1714.      /*
    1715. 

  1715.       * If we have already set the timer, we must NOT set it again.  Therefore,
    1716. 

  1716.       * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
    1717. 

  1717.       */
    1718. 

  1718.      if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
    1719. 

  1719. 	 c_p->def_arg_reg[0] = (Eterm) (I+3);
    1720. 

  1720. 	 set_timer(c_p, Arg(1));
    1721. 

  1721.      }
    1722. 

  1722.      goto wait2;
    1723. 

  1723.  }
    1724. 

  1724. 

  1725.     /*
    1726. 

  1726.      * A timeout has occurred.  Reset the save pointer so that the next
    1727. 

  1727.      * receive statement will examine the first message first.
    1728. 

  1728.      */
    1729. 

  1729.  OpCase(timeout_locked): {
    1730. 

  1730.      erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
    1731. 

  1731.  }
    1732. 

  1732. 

  1733.  OpCase(timeout): {
    1734. 

  1734.      Eterm* next;
    1735. 

  1735. 

  1736.      PreFetch(0, next);
    1737. 

  1737.      if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
    1738. 

  1738. 	 trace_receive(c_p, am_timeout);
    1739. 

  1739.      }
    1740. 

  1740.      if (c_p->ct != NULL) {
    1741. 

  1741. 	 save_calls(c_p, &exp_timeout);
    1742. 

  1742.      }
    1743. 

  1743.      c_p->flags &= ~F_TIMO;
    1744. 

  1744.      JOIN_MESSAGE(c_p);
    1745. 

  1745.      NextPF(0, next);
    1746. 

  1746.  }
    1747. 

  1747. 

  1748.  OpCase(i_select_val_sfI):
    1749. 

  1749.      GetArg1(0, tmp_arg1);
    1750. 

  1750. 

  1751.  do_binary_search:
    1752. 

  1752.  {
    1753. 

  1753.      struct Pairs {
    1754. 

  1754. 	 Eterm val;
    1755. 

  1755. 	 Eterm* addr;
    1756. 

  1756.      };
    1757. 

  1757.      struct Pairs* low;
    1758. 

  1758.      struct Pairs* high;
    1759. 

  1759.      struct Pairs* mid;
    1760. 

  1760.      int bdiff; /* int not long because the arrays aren't that large */
    1761. 

  1761. 

  1762.      low = (struct Pairs *) &Arg(3);
    1763. 

  1763.      high = low + Arg(2);
    1764. 

  1764. 

  1765.      /* The pointer subtraction (high-low) below must produce
    1766. 

  1766.       * a signed result, because high could be < low. That
    1767. 

  1767.       * requires the compiler to insert quite a bit of code.
    1768. 

  1768.       *
    1769. 

  1769.       * However, high will be > low so the result will be
    1770. 

  1770.       * positive. We can use that knowledge to optimise the
    1771. 

  1771.       * entire sequence, from the initial comparison to the
    1772. 

  1772.       * computation of mid.
    1773. 

  1773.       *
    1774. 

  1774.       * -- Mikael Pettersson, Acumem AB
    1775. 

  1775.       *
    1776. 

  1776.       * Original loop control code:
    1777. 

  1777.       *
    1778. 

  1778.       * while (low < high) {
    1779. 

  1779.       *    mid = low + (high-low) / 2;
    1780. 

  1780.       *
    1781. 

  1781.       */
    1782. 

  1782.      while ((bdiff = (int)((char*)high - (char*)low)) > 0) {
    1783. 

  1783. 	 unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
    1784. 

  1784. 

  1785. 	 mid = (struct Pairs*)((char*)low + boffset);
    1786. 

  1786. 	 if (tmp_arg1 < mid->val) {
    1787. 

  1787. 	     high = mid;
    1788. 

  1788. 	 } else if (tmp_arg1 > mid->val) {
    1789. 

  1789. 	     low = mid + 1;
    1790. 

  1790. 	 } else {
    1791. 

  1791. 	     SET_I(mid->addr);
    1792. 

  1792. 	     Goto(*I);
    1793. 

  1793. 	 }
    1794. 

  1794.      }
    1795. 

  1795.      SET_I((Eterm *) Arg(1));
    1796. 

  1796.      Goto(*I);
    1797. 

  1797.  }
    1798. 

  1798. 

  1799.  OpCase(i_jump_on_val_zero_sfI):
    1800. 

  1800.  {
    1801. 

  1801.      Eterm index;
    1802. 

  1802. 

  1803.      GetArg1(0, index);
    1804. 

  1804.      if (is_small(index)) {
    1805. 

  1805. 	 index = signed_val(index);
    1806. 

  1806. 	 if (index < Arg(2)) {
    1807. 

  1807. 	     SET_I((Eterm *) (&Arg(3))[index]);
    1808. 

  1808. 	     Goto(*I);
    1809. 

  1809. 	 }
    1810. 

  1810.      }
    1811. 

  1811.      SET_I((Eterm *) Arg(1));
    1812. 

  1812.      Goto(*I);
    1813. 

  1813.  }
    1814. 

  1814. 

  1815.  OpCase(i_jump_on_val_sfII):
    1816. 

  1816.  {
    1817. 

  1817.      Eterm index;
    1818. 

  1818. 

  1819.      GetArg1(0, index);
    1820. 

  1820.      if (is_small(index)) {
    1821. 

  1821. 	 index = (Uint) (signed_val(index) - Arg(3));
    1822. 

  1822. 	 if (index < Arg(2)) {
    1823. 

  1823. 	     SET_I((Eterm *) (&Arg(4))[index]);
    1824. 

  1824. 	     Goto(*I);
    1825. 

  1825. 	 }
    1826. 

  1826.      }
    1827. 

  1827.      SET_I((Eterm *) Arg(1));
    1828. 

  1828.      Goto(*I);
    1829. 

  1829.  }
    1830. 

  1830. 

  1831.     /*
    1832. 

  1832.      * All guards with zero arguments have special instructions:
    1833. 

  1833.      * 	self/0
    1834. 

  1834.      * 	node/0
    1835. 

  1835.      *
    1836. 

  1836.      * All other guard BIFs take one or two arguments.
    1837. 

  1837.      */
    1838. 

  1838. 

  1839.     /*
    1840. 

  1840.      * Guard BIF in head.  On failure, ignore the error and jump
    1841. 

  1841.      * to the code for the next clause.  We don't support tracing
    1842. 

  1842.      * of guard BIFs.
    1843. 

  1843.      */
    1844. 

  1844. 

  1845.  OpCase(bif1_fbsd):
    1846. 

  1846.     {
    1847. 

  1847. 	Eterm (*bf)(Process*, Eterm);
    1848. 

  1848. 	Eterm arg;
    1849. 

  1849. 	Eterm result;
    1850. 

  1850. 

  1851. 	GetArg1(2, arg);
    1852. 

  1852. 	bf = (BifFunction) Arg(1);
    1853. 

  1853. 	SAVE_HTOP;
    1854. 

  1854. 	c_p->fcalls = FCALLS;
    1855. 

  1855. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1856. 

  1856. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    1857. 

  1857. 	result = (*bf)(c_p, arg);
    1858. 

  1858. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    1859. 

  1859. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1860. 

  1860. 	ERTS_HOLE_CHECK(c_p);
    1861. 

  1861. 	FCALLS = c_p->fcalls;
    1862. 

  1862. 	if (is_value(result)) {
    1863. 

  1863. 	    StoreBifResult(3, result);
    1864. 

  1864. 	}
    1865. 

  1865. 	SET_I((Eterm *) Arg(0));
    1866. 

  1866. 	Goto(*I);
    1867. 

  1867.     }
    1868. 

  1868. 

  1869.     /*
    1870. 

  1870.      * Guard BIF in body.  It can fail like any BIF.  No trace support.
    1871. 

  1871.      */
    1872. 

  1872. 

  1873.  OpCase(bif1_body_bsd):
    1874. 

  1874.     {
    1875. 

  1875. 	Eterm (*bf)(Process*, Eterm);
    1876. 

  1876. 

  1877. 	Eterm arg;
    1878. 

  1878. 	Eterm result;
    1879. 

  1879. 

  1880. 	GetArg1(1, arg);
    1881. 

  1881. 	bf = (BifFunction) Arg(0);
    1882. 

  1882. 	SAVE_HTOP;
    1883. 

  1883. 	c_p->fcalls = FCALLS;
    1884. 

  1884. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1885. 

  1885. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    1886. 

  1886. 	result = (*bf)(c_p, arg);
    1887. 

  1887. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    1888. 

  1888. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1889. 

  1889. 	ERTS_HOLE_CHECK(c_p);
    1890. 

  1890. 	FCALLS = c_p->fcalls;
    1891. 

  1891. 	if (is_value(result)) {
    1892. 

  1892. 	    StoreBifResult(2, result);
    1893. 

  1893. 	}
    1894. 

  1894. 	reg[0] = arg;
    1895. 

  1895. 	SWAPOUT;
    1896. 

  1896. 	I = handle_error(c_p, I, reg, bf);
    1897. 

  1897. 	goto post_error_handling;
    1898. 

  1898.     }
    1899. 

  1899. 

  1900.  OpCase(i_gc_bif1_jIsId):
    1901. 

  1901.     {
    1902. 

  1902. 	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
    1903. 

  1903. 	GcBifFunction bf;
    1904. 

  1904. 	Eterm arg;
    1905. 

  1905. 	Eterm result;
    1906. 

  1906. 	Uint live = Arg(3);
    1907. 

  1907. 

  1908. 	GetArg1(2, arg);
    1909. 

  1909. 	reg[0] = r(0);
    1910. 

  1910. 	reg[live] = arg;
    1911. 

  1911. 	bf = (GcBifFunction) Arg(1);
    1912. 

  1912. 	c_p->fcalls = FCALLS;
    1913. 

  1913. 	SWAPOUT;
    1914. 

  1914. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1915. 

  1915. 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    1916. 

  1916. 	result = (*bf)(c_p, reg, live);
    1917. 

  1917. 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    1918. 

  1918. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1919. 

  1919. 	SWAPIN;
    1920. 

  1920. 	r(0) = reg[0];
    1921. 

  1921. 	ERTS_HOLE_CHECK(c_p);
    1922. 

  1922. 	FCALLS = c_p->fcalls;
    1923. 

  1923. 	if (is_value(result)) {
    1924. 

  1924. 	    StoreBifResult(4, result);
    1925. 

  1925. 	}
    1926. 

  1926. 	if (Arg(0) != 0) {
    1927. 

  1927. 	    SET_I((Eterm *) Arg(0));
    1928. 

  1928. 	    Goto(*I);
    1929. 

  1929. 	}
    1930. 

  1930. 	reg[0] = arg;
    1931. 

  1931. 	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
    1932. 

  1932. 	goto post_error_handling;
    1933. 

  1933.     }
    1934. 

  1934. 

  1935.  /*
    1936. 

  1936.   * Guards bifs and, or, xor in guards.
    1937. 

  1937.   */
    1938. 

  1938.  OpCase(i_bif2_fbd):
    1939. 

  1939.     {
    1940. 

  1940. 	Eterm (*bf)(Process*, Eterm, Eterm);
    1941. 

  1941. 	Eterm result;
    1942. 

  1942. 

  1943. 	bf = (BifFunction) Arg(1);
    1944. 

  1944. 	SAVE_HTOP;
    1945. 

  1945. 	c_p->fcalls = FCALLS;
    1946. 

  1946. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1947. 

  1947. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    1948. 

  1948. 	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
    1949. 

  1949. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    1950. 

  1950. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1951. 

  1951. 	ERTS_HOLE_CHECK(c_p);
    1952. 

  1952. 	FCALLS = c_p->fcalls;
    1953. 

  1953. 	if (is_value(result)) {
    1954. 

  1954. 	    StoreBifResult(2, result);
    1955. 

  1955. 	}
    1956. 

  1956. 	SET_I((Eterm *) Arg(0));
    1957. 

  1957. 	Goto(*I);
    1958. 

  1958.     }
    1959. 

  1959. 

  1960.  /*
    1961. 

  1961.   * Guards bifs and, or, xor, relational operators in body.
    1962. 

  1962.   */
    1963. 

  1963.  OpCase(i_bif2_body_bd):
    1964. 

  1964.     {
    1965. 

  1965. 	Eterm (*bf)(Process*, Eterm, Eterm);
    1966. 

  1966. 	Eterm result;
    1967. 

  1967. 

  1968. 	bf = (BifFunction) Arg(0);
    1969. 

  1969. 	SAVE_HTOP;
    1970. 

  1970. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1971. 

  1971. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    1972. 

  1972. 	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
    1973. 

  1973. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    1974. 

  1974. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    1975. 

  1975. 	ERTS_HOLE_CHECK(c_p);
    1976. 

  1976. 	if (is_value(result)) {
    1977. 

  1977. 	    ASSERT(!is_CP(result));
    1978. 

  1978. 	    StoreBifResult(1, result);
    1979. 

  1979. 	}
    1980. 

  1980. 	reg[0] = tmp_arg1;
    1981. 

  1981. 	reg[1] = tmp_arg2;
    1982. 

  1982. 	SWAPOUT;
    1983. 

  1983. 	I = handle_error(c_p, I, reg, bf);
    1984. 

  1984. 	goto post_error_handling;
    1985. 

  1985.     }
    1986. 

  1986. 

  1987.     /*
    1988. 

  1988.      * The most general BIF call.  The BIF may build any amount of data
    1989. 

  1989.      * on the heap.  The result is always returned in r(0).
    1990. 

  1990.      */
    1991. 

  1991.  OpCase(call_bif0_e):
    1992. 

  1992.     {
    1993. 

  1993. 	Eterm (*bf)(Process*, Uint*) = GET_BIF_ADDRESS(Arg(0));
    1994. 

  1994. 

  1995. 	PRE_BIF_SWAPOUT(c_p);
    1996. 

  1996. 	c_p->fcalls = FCALLS - 1;
    1997. 

  1997. 	if (FCALLS <= 0) {
    1998. 

  1998. 	    save_calls(c_p, (Export *) Arg(0));
    1999. 

  1999. 	}
    2000. 

  2000. 

  2001. 	/*
    2002. 

  2002. 	 * A BIF with no arguments cannot fail (especially not with badarg).
    2003. 

  2003. 	 */
    2004. 

  2004. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2005. 

  2005. 	r(0) = (*bf)(c_p, I);
    2006. 

  2006. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(r(0)));
    2007. 

  2007. 	ERTS_HOLE_CHECK(c_p);
    2008. 

  2008. 	POST_BIF_GC_SWAPIN_0(c_p, r(0));
    2009. 

  2009. 	FCALLS = c_p->fcalls;
    2010. 

  2010. 	if (is_value(r(0))) {
    2011. 

  2011. 	    CHECK_TERM(r(0));
    2012. 

  2012. 	    Next(1);
    2013. 

  2013. 	}
    2014. 

  2014. 	else {
    2015. 

  2015. 	    ASSERT(c_p->freason == TRAP);
    2016. 

  2016. 	    goto call_bif_trap3;
    2017. 

  2017. 	}
    2018. 

  2018.     }
    2019. 

  2019. 

  2020.  OpCase(call_bif1_e):
    2021. 

  2021.     {
    2022. 

  2022. 	Eterm (*bf)(Process*, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
    2023. 

  2023. 	Eterm result;
    2024. 

  2024. 	Eterm* next;
    2025. 

  2025. 

  2026. 	c_p->fcalls = FCALLS - 1;
    2027. 

  2027. 	if (FCALLS <= 0) {
    2028. 

  2028. 	    save_calls(c_p, (Export *) Arg(0));
    2029. 

  2029. 	}
    2030. 

  2030. 	PreFetch(1, next);
    2031. 

  2031. 	PRE_BIF_SWAPOUT(c_p);
    2032. 

  2032. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2033. 

  2033. 	result = (*bf)(c_p, r(0), I);
    2034. 

  2034. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    2035. 

  2035. 	ERTS_HOLE_CHECK(c_p);
    2036. 

  2036. 	POST_BIF_GC_SWAPIN(c_p, result, reg, 1);
    2037. 

  2037. 	FCALLS = c_p->fcalls;
    2038. 

  2038.         if (is_value(result)) {
    2039. 

  2039. 	    r(0) = result;
    2040. 

  2040. 	    CHECK_TERM(r(0));
    2041. 

  2041. 	    NextPF(1, next);
    2042. 

  2042. 	} else if (c_p->freason == RESCHEDULE) {
    2043. 

  2043. 	    c_p->arity = 1;
    2044. 

  2044. 	    goto suspend_bif;
    2045. 

  2045. 	} else if (c_p->freason == TRAP) {
    2046. 

  2046. 	    goto call_bif_trap3;
    2047. 

  2047. 	}
    2048. 

  2048. 

  2049. 	/*
    2050. 

  2050. 	 * Error handling.  SWAPOUT is not needed because it was done above.
    2051. 

  2051. 	 */
    2052. 

  2052. 	ASSERT(c_p->stop == E);
    2053. 

  2053. 	reg[0] = r(0);
    2054. 

  2054. 	I = handle_error(c_p, I, reg, bf);
    2055. 

  2055. 	goto post_error_handling;
    2056. 

  2056.     }
    2057. 

  2057. 

  2058.  OpCase(call_bif2_e):
    2059. 

  2059.     {
    2060. 

  2060. 	Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
    2061. 

  2061. 	Eterm result;
    2062. 

  2062. 	Eterm* next;
    2063. 

  2063. 

  2064. 	PRE_BIF_SWAPOUT(c_p);
    2065. 

  2065. 	c_p->fcalls = FCALLS - 1;
    2066. 

  2066. 	if (FCALLS <= 0) {
    2067. 

  2067. 	   save_calls(c_p, (Export *) Arg(0));
    2068. 

  2068. 	}
    2069. 

  2069. 	PreFetch(1, next);
    2070. 

  2070. 	CHECK_TERM(r(0));
    2071. 

  2071. 	CHECK_TERM(x(1));
    2072. 

  2072. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2073. 

  2073. 	result = (*bf)(c_p, r(0), x(1), I);
    2074. 

  2074. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    2075. 

  2075. 	ERTS_HOLE_CHECK(c_p);
    2076. 

  2076. 	POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
    2077. 

  2077. 	FCALLS = c_p->fcalls;
    2078. 

  2078. 	if (is_value(result)) {
    2079. 

  2079. 	    r(0) = result;
    2080. 

  2080. 	    CHECK_TERM(r(0));
    2081. 

  2081. 	    NextPF(1, next);
    2082. 

  2082. 	} else if (c_p->freason == RESCHEDULE) {
    2083. 

  2083. 	    c_p->arity = 2;
    2084. 

  2084. 	    goto suspend_bif;
    2085. 

  2085. 	} else if (c_p->freason == TRAP) {
    2086. 

  2086. 	    goto call_bif_trap3;
    2087. 

  2087. 	}
    2088. 

  2088. 

  2089. 	/*
    2090. 

  2090. 	 * Error handling.  SWAPOUT is not needed because it was done above.
    2091. 

  2091. 	 */
    2092. 

  2092. 	ASSERT(c_p->stop == E);
    2093. 

  2093. 	reg[0] = r(0);
    2094. 

  2094. 	I = handle_error(c_p, I, reg, bf);
    2095. 

  2095. 	goto post_error_handling;
    2096. 

  2096.     }
    2097. 

  2097. 

  2098.  OpCase(call_bif3_e):
    2099. 

  2099.     {
    2100. 

  2100. 	Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
    2101. 

  2101. 	Eterm result;
    2102. 

  2102. 	Eterm* next;
    2103. 

  2103. 

  2104. 	PRE_BIF_SWAPOUT(c_p);
    2105. 

  2105. 	c_p->fcalls = FCALLS - 1;
    2106. 

  2106. 	if (FCALLS <= 0) {
    2107. 

  2107. 	   save_calls(c_p, (Export *) Arg(0));
    2108. 

  2108. 	}
    2109. 

  2109. 	PreFetch(1, next);
    2110. 

  2110. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2111. 

  2111. 	result = (*bf)(c_p, r(0), x(1), x(2), I);
    2112. 

  2112. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
    2113. 

  2113. 	ERTS_HOLE_CHECK(c_p);
    2114. 

  2114. 	POST_BIF_GC_SWAPIN(c_p, result, reg, 3);
    2115. 

  2115. 	FCALLS = c_p->fcalls;
    2116. 

  2116. 	if (is_value(result)) {
    2117. 

  2117. 	    r(0) = result;
    2118. 

  2118. 	    CHECK_TERM(r(0));
    2119. 

  2119. 	    NextPF(1, next);
    2120. 

  2120. 	} else if (c_p->freason == RESCHEDULE) {
    2121. 

  2121. 	    c_p->arity = 3;
    2122. 

  2122. 	    goto suspend_bif;
    2123. 

  2123. 	} else if (c_p->freason == TRAP) {
    2124. 

  2124. 	call_bif_trap3:
    2125. 

  2125. 	    SET_CP(c_p, I+2);
    2126. 

  2126. 	    SET_I(((Export *)(c_p->def_arg_reg[3]))->address);
    2127. 

  2127. 	    SWAPIN;
    2128. 

  2128. 	    r(0) = c_p->def_arg_reg[0];
    2129. 

  2129. 	    x(1) = c_p->def_arg_reg[1];
    2130. 

  2130. 	    x(2) = c_p->def_arg_reg[2];
    2131. 

  2131. 	    Dispatch();
    2132. 

  2132. 	}
    2133. 

  2133. 

  2134. 	/*
    2135. 

  2135. 	 * Error handling.  SWAPOUT is not needed because it was done above.
    2136. 

  2136. 	 */
    2137. 

  2137. 	ASSERT(c_p->stop == E);
    2138. 

  2138. 	reg[0] = r(0);
    2139. 

  2139. 	I = handle_error(c_p, I, reg, bf);
    2140. 

  2140. 	goto post_error_handling;
    2141. 

  2141.     }
    2142. 

  2142. 

  2143.  /*
    2144. 

  2144.   * Arithmetic operations.
    2145. 

  2145.   */
    2146. 

  2146. 

  2147.  OpCase(i_times_jId):
    2148. 

  2148.  {
    2149. 

  2149.      arith_func = ARITH_FUNC(mixed_times);
    2150. 

  2150.      goto do_big_arith2;
    2151. 

  2151.  }
    2152. 

  2152. 

  2153.  OpCase(i_m_div_jId):
    2154. 

  2154.  {
    2155. 

  2155.      arith_func = ARITH_FUNC(mixed_div);
    2156. 

  2156.      goto do_big_arith2;
    2157. 

  2157.  }
    2158. 

  2158. 

  2159.  OpCase(i_int_div_jId):
    2160. 

  2160.  {
    2161. 

  2161.      Eterm result;
    2162. 

  2162. 

  2163.      if (tmp_arg2 == SMALL_ZERO) {
    2164. 

  2164. 	 goto badarith;
    2165. 

  2165.      } else if (is_both_small(tmp_arg1, tmp_arg2)) {
    2166. 

  2166. 	 Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2);
    2167. 

  2167. 	 if (MY_IS_SSMALL(ires)) {
    2168. 

  2168. 	     result = make_small(ires);
    2169. 

  2169. 	     STORE_ARITH_RESULT(result);
    2170. 

  2170. 	 }
    2171. 

  2171.      }
    2172. 

  2172.      arith_func = ARITH_FUNC(int_div);
    2173. 

  2173.      goto do_big_arith2;
    2174. 

  2174.  }
    2175. 

  2175. 

  2176.  OpCase(i_rem_jId):
    2177. 

  2177.  {
    2178. 

  2178.      Eterm result;
    2179. 

  2179. 

  2180.      if (tmp_arg2 == SMALL_ZERO) {
    2181. 

  2181. 	 goto badarith;
    2182. 

  2182.      } else if (is_both_small(tmp_arg1, tmp_arg2)) {
    2183. 

  2183. 	 result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2));
    2184. 

  2184. 	 STORE_ARITH_RESULT(result);
    2185. 

  2185.      } else {
    2186. 

  2186. 	 arith_func = ARITH_FUNC(int_rem);
    2187. 

  2187. 	 goto do_big_arith2;
    2188. 

  2188.      }
    2189. 

  2189.  }
    2190. 

  2190. 

  2191.  OpCase(i_band_jId):
    2192. 

  2192.  {
    2193. 

  2193.      Eterm result;
    2194. 

  2194. 

  2195.      if (is_both_small(tmp_arg1, tmp_arg2)) {
    2196. 

  2196. 	 /*
    2197. 

  2197. 	  * No need to untag -- TAG & TAG == TAG.
    2198. 

  2198. 	  */
    2199. 

  2199. 	 result = tmp_arg1 & tmp_arg2;
    2200. 

  2200. 	 STORE_ARITH_RESULT(result);
    2201. 

  2201.      }
    2202. 

  2202.      arith_func = ARITH_FUNC(band);
    2203. 

  2203.      goto do_big_arith2;
    2204. 

  2204.  }
    2205. 

  2205. 

  2206.  do_big_arith2:
    2207. 

  2207.  {
    2208. 

  2208.      Eterm result;
    2209. 

  2209.      Uint live = Arg(1);
    2210. 

  2210. 

  2211.      SWAPOUT;
    2212. 

  2212.      reg[0] = r(0);
    2213. 

  2213.      reg[live] = tmp_arg1;
    2214. 

  2214.      reg[live+1] = tmp_arg2;
    2215. 

  2215.      result = arith_func(c_p, reg, live);
    2216. 

  2216.      r(0) = reg[0];
    2217. 

  2217.      SWAPIN;
    2218. 

  2218.      ERTS_HOLE_CHECK(c_p);
    2219. 

  2219.      if (is_value(result)) {
    2220. 

  2220. 	 STORE_ARITH_RESULT(result);
    2221. 

  2221.      }
    2222. 

  2222.      goto lb_Cl_error;
    2223. 

  2223.  }
    2224. 

  2224. 

  2225.  /*
    2226. 

  2226.   * An error occured in an arithmetic operation or test that could
    2227. 

  2227.   * appear either in a head or in a body.
    2228. 

  2228.   * In a head, execution should continue at failure address in Arg(0).
    2229. 

  2229.   * In a body, Arg(0) == 0 and an exception should be raised.
    2230. 

  2230.   */
    2231. 

  2231.  lb_Cl_error: {
    2232. 

  2232.      if (Arg(0) != 0) {
    2233. 

  2233. 	 OpCase(jump_f): {
    2234. 

  2234. 	     SET_I((Eterm *) Arg(0));
    2235. 

  2235. 	     Goto(*I);
    2236. 

  2236. 	 }
    2237. 

  2237.      }
    2238. 

  2238.      ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
    2239. 

  2239.      goto find_func_info;
    2240. 

  2240.  }
    2241. 

  2241. 

  2242.  OpCase(i_bor_jId):
    2243. 

  2243.  {
    2244. 

  2244.      Eterm result;
    2245. 

  2245. 

  2246.      if (is_both_small(tmp_arg1, tmp_arg2)) {
    2247. 

  2247. 	 /*
    2248. 

  2248. 	  * No need to untag -- TAG | TAG == TAG.
    2249. 

  2249. 	  */
    2250. 

  2250. 	 result = tmp_arg1 | tmp_arg2;
    2251. 

  2251. 	 STORE_ARITH_RESULT(result);
    2252. 

  2252.      }
    2253. 

  2253.      arith_func = ARITH_FUNC(bor);
    2254. 

  2254.      goto do_big_arith2;
    2255. 

  2255.  }
    2256. 

  2256. 

  2257.  OpCase(i_bxor_jId):
    2258. 

  2258.  {
    2259. 

  2259.      Eterm result;
    2260. 

  2260. 

  2261.      if (is_both_small(tmp_arg1, tmp_arg2)) {
    2262. 

  2262. 	 /*
    2263. 

  2263. 	  * We could extract the tag from one argument, but a tag extraction
    2264. 

  2264. 	  * could mean a shift.  Therefore, play it safe here.
    2265. 

  2265. 	  */
    2266. 

  2266. 	 result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2));
    2267. 

  2267. 	 STORE_ARITH_RESULT(result);
    2268. 

  2268.      }
    2269. 

  2269.      arith_func = ARITH_FUNC(bxor);
    2270. 

  2270.      goto do_big_arith2;
    2271. 

  2271.  }
    2272. 

  2272. 

  2273.  {
    2274. 

  2274.      Sint i;
    2275. 

  2275.      Sint ires;
    2276. 

  2276.      Eterm* bigp;
    2277. 

  2277. 

  2278.      OpCase(i_bsr_jId):
    2279. 

  2279. 	 if (is_small(tmp_arg2)) {
    2280. 

  2280. 	     i = -signed_val(tmp_arg2);
    2281. 

  2281. 	     if (is_small(tmp_arg1)) {
    2282. 

  2282. 		 goto small_shift;
    2283. 

  2283. 	     } else if (is_big(tmp_arg1)) {
    2284. 

  2284. 		 if (i == 0) {
    2285. 

  2285. 		     StoreBifResult(2, tmp_arg1);
    2286. 

  2286. 		 }
    2287. 

  2287. 		 goto big_shift;
    2288. 

  2288. 	     }
    2289. 

  2289. 	 }
    2290. 

  2290.      goto badarith;
    2291. 

  2291.      
    2292. 

  2292.      OpCase(i_bsl_jId):
    2293. 

  2293. 	 if (is_small(tmp_arg2)) {
    2294. 

  2294. 	     i = signed_val(tmp_arg2);
    2295. 

  2295. 

  2296. 	     if (is_small(tmp_arg1)) {
    2297. 

  2297. 	     small_shift:
    2298. 

  2298. 		 ires = signed_val(tmp_arg1);
    2299. 

  2299. 	     
    2300. 

  2300. 		 if (i == 0 || ires == 0) {
    2301. 

  2301. 		     StoreBifResult(2, tmp_arg1);
    2302. 

  2302. 		 } else if (i < 0)  { /* Right shift */
    2303. 

  2303. 		     i = -i;
    2304. 

  2304. 		     if (i >= SMALL_BITS-1) {
    2305. 

  2305. 			 tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
    2306. 

  2306. 		     } else {
    2307. 

  2307. 			 tmp_arg1 = make_small(ires >> i);
    2308. 

  2308. 		     }
    2309. 

  2309. 		     StoreBifResult(2, tmp_arg1);
    2310. 

  2310. 		 } else if (i < SMALL_BITS-1) { /* Left shift */
    2311. 

  2311. 		     if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) ||
    2312. 

  2312. 			 ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) {
    2313. 

  2313. 			 tmp_arg1 = make_small(ires << i);
    2314. 

  2314. 			 StoreBifResult(2, tmp_arg1);
    2315. 

  2315. 		     }
    2316. 

  2316. 		 }
    2317. 

  2317. 		 tmp_arg1 = small_to_big(ires, tmp_big);
    2318. 

  2318. 

  2319. 	     big_shift:
    2320. 

  2320. 		 if (i > 0) {	/* Left shift. */
    2321. 

  2321. 		     ires = big_size(tmp_arg1) + (i / D_EXP);
    2322. 

  2322. 		 } else {	/* Right shift. */
    2323. 

  2323. 		     ires = big_size(tmp_arg1);
    2324. 

  2324. 		     if (ires <= (-i / D_EXP))
    2325. 

  2325. 			 ires = 3; /* ??? */
    2326. 

  2326. 		     else
    2327. 

  2327. 			 ires -= (-i / D_EXP);
    2328. 

  2328. 		 }
    2329. 

  2329. 		 {
    2330. 

  2330. 		     Uint live = Arg(1);
    2331. 

  2331. 		     ires = BIG_NEED_SIZE(ires+1);
    2332. 

  2332. 		     SWAPOUT;
    2333. 

  2333. 		     reg[0] = r(0);
    2334. 

  2334. 		     reg[live] = tmp_arg1;
    2335. 

  2335. 		     
    2336. 

  2336. 		     PROCESS_MAIN_CHK_LOCKS(c_p);
    2337. 

  2337. 		     FCALLS -= erts_garbage_collect(c_p, ires, reg, live+1);
    2338. 

  2338. 		     PROCESS_MAIN_CHK_LOCKS(c_p);
    2339. 

  2339. 		     r(0) = reg[0];
    2340. 

  2340. 		     SWAPIN;
    2341. 

  2341. 		     bigp = HTOP;
    2342. 

  2342. 		     tmp_arg1 = big_lshift(reg[live], i, bigp);
    2343. 

  2343. 		     if (is_big(tmp_arg1)) {
    2344. 

  2344. 			 HTOP += bignum_header_arity(*HTOP) + 1;
    2345. 

  2345. 		     }
    2346. 

  2346. 		     if (is_nil(tmp_arg1)) {
    2347. 

  2347. 			 c_p->freason = SYSTEM_LIMIT;
    2348. 

  2348. 			 goto lb_Cl_error;
    2349. 

  2349. 		     }
    2350. 

  2350. 		     ERTS_HOLE_CHECK(c_p);
    2351. 

  2351. 		     StoreBifResult(2, tmp_arg1);
    2352. 

  2352. 		 }
    2353. 

  2353. 	     } else if (is_big(tmp_arg1)) {
    2354. 

  2354. 		 if (i == 0) {
    2355. 

  2355. 		     StoreBifResult(2, tmp_arg1);
    2356. 

  2356. 		 }
    2357. 

  2357. 		 goto big_shift;
    2358. 

  2358. 	     }
    2359. 

  2359. 	 }
    2360. 

  2360.      goto badarith;
    2361. 

  2361.  }
    2362. 

  2362. 

  2363.  OpCase(i_int_bnot_jsId):
    2364. 

  2364.  {
    2365. 

  2365.      GetArg1(1, tmp_arg1);
    2366. 

  2366.      if (is_small(tmp_arg1)) {
    2367. 

  2367. 	 tmp_arg1 = make_small(~signed_val(tmp_arg1));
    2368. 

  2368.      } else {
    2369. 

  2369. 	 Uint live = Arg(2);
    2370. 

  2370. 	 SWAPOUT;
    2371. 

  2371. 	 reg[0] = r(0);
    2372. 

  2372. 	 reg[live] = tmp_arg1;
    2373. 

  2373. 	 tmp_arg1 = erts_gc_bnot(c_p, reg, live);
    2374. 

  2374. 	 r(0) = reg[0];
    2375. 

  2375. 	 SWAPIN;
    2376. 

  2376. 	 ERTS_HOLE_CHECK(c_p);
    2377. 

  2377. 	 if (is_nil(tmp_arg1)) {
    2378. 

  2378. 	     goto lb_Cl_error;
    2379. 

  2379. 	 }
    2380. 

  2380.      }
    2381. 

  2381.      StoreBifResult(3, tmp_arg1);
    2382. 

  2382.  }
    2383. 

  2383. 

  2384.  badarith:
    2385. 

  2385.     c_p->freason = BADARITH;
    2386. 

  2386.     goto lb_Cl_error;
    2387. 

  2387. 

  2388.  OpCase(i_apply): {
    2389. 

  2389.      Eterm* next;
    2390. 

  2390.      SWAPOUT;
    2391. 

  2391.      next = apply(c_p, r(0), x(1), x(2), reg);
    2392. 

  2392.      SWAPIN;
    2393. 

  2393.      if (next != NULL) {
    2394. 

  2394. 	 r(0) = reg[0];
    2395. 

  2395. 	 SET_CP(c_p, I+1);
    2396. 

  2396. 	 SET_I(next);
    2397. 

  2397. 	 Dispatch();
    2398. 

  2398.      }
    2399. 

  2399.      I = handle_error(c_p, I, reg, apply_3);
    2400. 

  2400.      goto post_error_handling;
    2401. 

  2401.  }
    2402. 

  2402. 

  2403.  OpCase(i_apply_last_P): {
    2404. 

  2404.      Eterm* next;
    2405. 

  2405.      SWAPOUT;
    2406. 

  2406.      next = apply(c_p, r(0), x(1), x(2), reg);
    2407. 

  2407.      SWAPIN;
    2408. 

  2408.      if (next != NULL) {
    2409. 

  2409. 	 r(0) = reg[0];
    2410. 

  2410. 	 SET_CP(c_p, (Eterm *) E[0]);
    2411. 

  2411. 	 E = ADD_BYTE_OFFSET(E, Arg(0));
    2412. 

  2412. 	 SET_I(next);
    2413. 

  2413. 	 Dispatch();
    2414. 

  2414.      }
    2415. 

  2415.      I = handle_error(c_p, I, reg, apply_3);
    2416. 

  2416.      goto post_error_handling;
    2417. 

  2417.  }
    2418. 

  2418. 

  2419.  OpCase(i_apply_only): {
    2420. 

  2420.      Eterm* next;
    2421. 

  2421.      SWAPOUT;
    2422. 

  2422.      next = apply(c_p, r(0), x(1), x(2), reg);
    2423. 

  2423.      SWAPIN;
    2424. 

  2424.      if (next != NULL) {
    2425. 

  2425. 	 r(0) = reg[0];
    2426. 

  2426. 	 SET_I(next);
    2427. 

  2427. 	 Dispatch();
    2428. 

  2428.      }
    2429. 

  2429.      I = handle_error(c_p, I, reg, apply_3);
    2430. 

  2430.      goto post_error_handling;
    2431. 

  2431.  }
    2432. 

  2432. 

  2433.  OpCase(apply_I): {
    2434. 

  2434.      Eterm* next;
    2435. 

  2435. 

  2436.      reg[0] = r(0);
    2437. 

  2437.      SWAPOUT;
    2438. 

  2438.      next = fixed_apply(c_p, reg, Arg(0));
    2439. 

  2439.      SWAPIN;
    2440. 

  2440.      if (next != NULL) {
    2441. 

  2441. 	 r(0) = reg[0];
    2442. 

  2442. 	 SET_CP(c_p, I+2);
    2443. 

  2443. 	 SET_I(next);
    2444. 

  2444. 	 Dispatch();
    2445. 

  2445.      }
    2446. 

  2446.      I = handle_error(c_p, I, reg, apply_3);
    2447. 

  2447.      goto post_error_handling;
    2448. 

  2448.  }
    2449. 

  2449. 

  2450.  OpCase(apply_last_IP): {
    2451. 

  2451.      Eterm* next;
    2452. 

  2452. 

  2453.      reg[0] = r(0);
    2454. 

  2454.      SWAPOUT;
    2455. 

  2455.      next = fixed_apply(c_p, reg, Arg(0));
    2456. 

  2456.      SWAPIN;
    2457. 

  2457.      if (next != NULL) {
    2458. 

  2458. 	 r(0) = reg[0];
    2459. 

  2459. 	 SET_CP(c_p, (Eterm *) E[0]);
    2460. 

  2460. 	 E = ADD_BYTE_OFFSET(E, Arg(1));
    2461. 

  2461. 	 SET_I(next);
    2462. 

  2462. 	 Dispatch();
    2463. 

  2463.      }
    2464. 

  2464.      I = handle_error(c_p, I, reg, apply_3);
    2465. 

  2465.      goto post_error_handling;
    2466. 

  2466.  }
    2467. 

  2467. 

  2468.  OpCase(i_apply_fun): {
    2469. 

  2469.      Eterm* next;
    2470. 

  2470. 

  2471.      SWAPOUT;
    2472. 

  2472.      next = apply_fun(c_p, r(0), x(1), reg);
    2473. 

  2473.      SWAPIN;
    2474. 

  2474.      if (next != NULL) {
    2475. 

  2475. 	 r(0) = reg[0];
    2476. 

  2476. 	 SET_CP(c_p, I+1);
    2477. 

  2477. 	 SET_I(next);
    2478. 

  2478. 	 Dispatchfun();
    2479. 

  2479.      }
    2480. 

  2480.      goto find_func_info;
    2481. 

  2481.  }
    2482. 

  2482. 

  2483.  OpCase(i_apply_fun_last_P): {
    2484. 

  2484.      Eterm* next;
    2485. 

  2485. 

  2486.      SWAPOUT;
    2487. 

  2487.      next = apply_fun(c_p, r(0), x(1), reg);
    2488. 

  2488.      SWAPIN;
    2489. 

  2489.      if (next != NULL) {
    2490. 

  2490. 	 r(0) = reg[0];
    2491. 

  2491. 	 SET_CP(c_p, (Eterm *) E[0]);
    2492. 

  2492. 	 E = ADD_BYTE_OFFSET(E, Arg(0));
    2493. 

  2493. 	 SET_I(next);
    2494. 

  2494. 	 Dispatchfun();
    2495. 

  2495.      }
    2496. 

  2496.      goto find_func_info;
    2497. 

  2497.  }
    2498. 

  2498. 

  2499.  OpCase(i_apply_fun_only): {
    2500. 

  2500.      Eterm* next;
    2501. 

  2501. 

  2502.      SWAPOUT;
    2503. 

  2503.      next = apply_fun(c_p, r(0), x(1), reg);
    2504. 

  2504.      SWAPIN;
    2505. 

  2505.      if (next != NULL) {
    2506. 

  2506. 	 r(0) = reg[0];
    2507. 

  2507. 	 SET_I(next);
    2508. 

  2508. 	 Dispatchfun();
    2509. 

  2509.      }
    2510. 

  2510.      goto find_func_info;
    2511. 

  2511.  }
    2512. 

  2512. 

  2513.  OpCase(i_call_fun_I): {
    2514. 

  2514.      Eterm* next;
    2515. 

  2515. 

  2516.      SWAPOUT;
    2517. 

  2517.      reg[0] = r(0);
    2518. 

  2518.      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
    2519. 

  2519.      SWAPIN;
    2520. 

  2520.      if (next != NULL) {
    2521. 

  2521. 	 r(0) = reg[0];
    2522. 

  2522. 	 SET_CP(c_p, I+2);
    2523. 

  2523. 	 SET_I(next);
    2524. 

  2524. 	 Dispatchfun();
    2525. 

  2525.      }
    2526. 

  2526.      goto find_func_info;
    2527. 

  2527.  }
    2528. 

  2528. 

  2529.  OpCase(i_call_fun_last_IP): {
    2530. 

  2530.      Eterm* next;
    2531. 

  2531. 

  2532.      SWAPOUT;
    2533. 

  2533.      reg[0] = r(0);
    2534. 

  2534.      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
    2535. 

  2535.      SWAPIN;
    2536. 

  2536.      if (next != NULL) {
    2537. 

  2537. 	r(0) = reg[0];
    2538. 

  2538. 	SET_CP(c_p, (Eterm *) E[0]);
    2539. 

  2539. 	E = ADD_BYTE_OFFSET(E, Arg(1));
    2540. 

  2540. 	SET_I(next);
    2541. 

  2541. 	Dispatchfun();
    2542. 

  2542.      }
    2543. 

  2543.      goto find_func_info;
    2544. 

  2544.  }
    2545. 

  2545. 

  2546. #ifdef DEBUG
    2547. 

  2547.     /*
    2548. 

  2548.      * Set a breakpoint here to get control just after a call instruction.
    2549. 

  2549.      * I points to the first instruction in the called function.
    2550. 

  2550.      *
    2551. 

  2551.      * In gdb, use 'call dis(I-5, 1)' to show the name of the function.
    2552. 

  2552.      */
    2553. 

  2553.  do_dispatch:
    2554. 

  2554.      DispatchMacro();
    2555. 

  2555. 

  2556.  do_dispatchx:
    2557. 

  2557.      DispatchMacrox();
    2558. 

  2558. 

  2559.  do_dispatchfun:
    2560. 

  2560.      DispatchMacroFun();
    2561. 

  2561. 

  2562. #endif
    2563. 

  2563. 

  2564.     /*
    2565. 

  2565.      * Jumped to from the Dispatch() macro when the reductions are used up.
    2566. 

  2566.      *
    2567. 

  2567.      * Since the I register points just beyond the FuncBegin instruction, we
    2568. 

  2568.      * can get the module, function, and arity for the function being
    2569. 

  2569.      * called from I[-3], I[-2], and I[-1] respectively.
    2570. 

  2570.      */
    2571. 

  2571.  context_switch_fun:
    2572. 

  2572.     c_p->arity = I[-1] + 1;
    2573. 

  2573.     goto context_switch2;
    2574. 

  2574. 

  2575.  context_switch:
    2576. 

  2576.     c_p->arity = I[-1];
    2577. 

  2577. 

  2578.  context_switch2:		/* Entry for fun calls. */
    2579. 

  2579.     c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
    2580. 

  2580. 

  2581.  {
    2582. 

  2582.      Eterm* argp;
    2583. 

  2583.      int i;
    2584. 

  2584. 

  2585.      /*
    2586. 

  2586.       * Make sure that there is enough room for the argument registers to be saved.
    2587. 

  2587.       */
    2588. 

  2588.      if (c_p->arity > c_p->max_arg_reg) {
    2589. 

  2589. 	 /*
    2590. 

  2590. 	  * Yes, this is an expensive operation, but you only pay it the first
    2591. 

  2591. 	  * time you call a function with more than 6 arguments which is
    2592. 

  2592. 	  * scheduled out.  This is better than paying for 26 words of wasted
    2593. 

  2593. 	  * space for most processes which never call functions with more than
    2594. 

  2594. 	  * 6 arguments.
    2595. 

  2595. 	  */
    2596. 

  2596. 	 Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
    2597. 

  2597. 	 if (c_p->arg_reg != c_p->def_arg_reg) {
    2598. 

  2598. 	     ERTS_PROC_LESS_MEM(c_p->max_arg_reg * sizeof(c_p->arg_reg[0]));
    2599. 

  2599. 	     ERTS_PROC_MORE_MEM(size);
    2600. 

  2600. 	     c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
    2601. 

  2601. 						   (void *) c_p->arg_reg,
    2602. 

  2602. 						   size);
    2603. 

  2603. 	 } else {
    2604. 

  2604. 	     ERTS_PROC_MORE_MEM(size);
    2605. 

  2605. 	     c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
    2606. 

  2606. 	 }
    2607. 

  2607. 	 c_p->max_arg_reg = c_p->arity;
    2608. 

  2608.      }
    2609. 

  2609. 

  2610.      /*
    2611. 

  2611.       * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
    2612. 

  2612.       * now before saving registers.
    2613. 

  2613.       *
    2614. 

  2614.       * The '+ 1' compensates for the last increment which was not done
    2615. 

  2615.       * (beacuse the code for the Dispatch() macro becomes shorter that way).
    2616. 

  2616.       */
    2617. 

  2617. 

  2618.      reds_used = REDS_IN(c_p) - FCALLS + 1;
    2619. 

  2619.      
    2620. 

  2620.      /*
    2621. 

  2621.       * Save the argument registers and everything else.
    2622. 

  2622.       */
    2623. 

  2623. 

  2624.      argp = c_p->arg_reg;
    2625. 

  2625.      for (i = c_p->arity - 1; i > 0; i--) {
    2626. 

  2626. 	 argp[i] = reg[i];
    2627. 

  2627.      }
    2628. 

  2628.      c_p->arg_reg[0] = r(0);
    2629. 

  2629.      SWAPOUT;
    2630. 

  2630.      c_p->i = I;
    2631. 

  2631.      erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
    2632. 

  2632.      add_to_schedule_q(c_p);
    2633. 

  2633.      erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
    2634. 

  2634.      goto do_schedule1;
    2635. 

  2635.  }
    2636. 

  2636. 

  2637.  OpCase(i_select_tuple_arity_sfI):
    2638. 

  2638.  {
    2639. 

  2639.      GetArg1(0, tmp_arg1);
    2640. 

  2640. 

  2641.      if (is_tuple(tmp_arg1)) {
    2642. 

  2642. 	 tmp_arg1 = *tuple_val(tmp_arg1);
    2643. 

  2643. 	 goto do_binary_search;
    2644. 

  2644.      }
    2645. 

  2645.      SET_I((Eterm *) Arg(1));
    2646. 

  2646.      Goto(*I);
    2647. 

  2647.  }     
    2648. 

  2648. 

  2649.  OpCase(i_select_big_sf):
    2650. 

  2650.     {
    2651. 

  2651. 	Eterm* bigp;
    2652. 

  2652. 	Uint arity;
    2653. 

  2653. 	Eterm* given;
    2654. 

  2654. 	Uint given_arity;
    2655. 

  2655. 	Uint given_size;
    2656. 

  2656. 

  2657. 	GetArg1(0, tmp_arg1);
    2658. 

  2658. 	if (is_big(tmp_arg1)) {
    2659. 

  2659. 

  2660. 	    /*
    2661. 

  2661. 	     * The loader has sorted the bignumbers in descending order
    2662. 

  2662. 	     * on the arity word.  Therefore, we know that the search
    2663. 

  2663. 	     * has failed as soon as we encounter an arity word less than
    2664. 

  2664. 	     * the arity word of the given number.  There is a zero word
    2665. 

  2665. 	     * (less than any valid arity word) stored after the last bignumber.
    2666. 

  2666. 	     */
    2667. 

  2667. 

  2668.  	    given = big_val(tmp_arg1);
    2669. 

  2669. 	    given_arity = given[0];
    2670. 

  2670. 	    given_size = thing_arityval(given_arity);
    2671. 

  2671. 	    bigp = &Arg(2);
    2672. 

  2672. 	    while ((arity = bigp[0]) > given_arity) {
    2673. 

  2673. 		bigp += thing_arityval(arity) + 2;
    2674. 

  2674. 	    }
    2675. 

  2675. 	    while (bigp[0] == given_arity) {
    2676. 

  2676. 		if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) {
    2677. 

  2677. 		    SET_I((Eterm *) bigp[given_size+1]);
    2678. 

  2678. 		    Goto(*I);
    2679. 

  2679. 		}
    2680. 

  2680. 		bigp += thing_arityval(arity) + 2;
    2681. 

  2681. 	    }
    2682. 

  2682. 	}
    2683. 

  2683. 

  2684. 	/*
    2685. 

  2685. 	 * Failed.
    2686. 

  2686. 	 */
    2687. 

  2687. 

  2688. 	SET_I((Eterm *) Arg(1));
    2689. 

  2689. 	Goto(*I);
    2690. 

  2690.     }
    2691. 

  2691. 

  2692. #ifdef ARCH_64
    2693. 

  2693.  OpCase(i_select_float_sfI):
    2694. 

  2694.  {
    2695. 

  2695.      Uint f;
    2696. 

  2696.      int n;
    2697. 

  2697.      struct ValLabel {
    2698. 

  2698. 	 Uint f;
    2699. 

  2699. 	 Eterm* addr;
    2700. 

  2700.      };
    2701. 

  2701.      struct ValLabel* ptr;
    2702. 

  2702. 

  2703.      GetArg1(0, tmp_arg1);
    2704. 

  2704.      ASSERT(is_float(tmp_arg1));
    2705. 

  2705.      f = float_val(tmp_arg1)[1];
    2706. 

  2706.      n = Arg(2);
    2707. 

  2707.      ptr = (struct ValLabel *) &Arg(3);
    2708. 

  2708.      while (n-- > 0) {
    2709. 

  2709. 	 if (ptr->f == f) {
    2710. 

  2710. 	     SET_I(ptr->addr);
    2711. 

  2711. 	     Goto(*I);
    2712. 

  2712. 	 }
    2713. 

  2713. 	 ptr++;
    2714. 

  2714.      }
    2715. 

  2715.      SET_I((Eterm *) Arg(1));
    2716. 

  2716.      Goto(*I);
    2717. 

  2717.  }
    2718. 

  2718. #else
    2719. 

  2719.  OpCase(i_select_float_sfI):
    2720. 

  2720.  {
    2721. 

  2721.      Uint fpart1;
    2722. 

  2722.      Uint fpart2;
    2723. 

  2723.      int n;
    2724. 

  2724.      struct ValLabel {
    2725. 

  2725. 	 Uint fpart1;
    2726. 

  2726. 	 Uint fpart2;
    2727. 

  2727. 	 Eterm* addr;
    2728. 

  2728.      };
    2729. 

  2729.      struct ValLabel* ptr;
    2730. 

  2730. 

  2731.      GetArg1(0, tmp_arg1);
    2732. 

  2732.      ASSERT(is_float(tmp_arg1));
    2733. 

  2733.      fpart1 = float_val(tmp_arg1)[1];
    2734. 

  2734.      fpart2 = float_val(tmp_arg1)[2];
    2735. 

  2735. 

  2736.      n = Arg(2);
    2737. 

  2737.      ptr = (struct ValLabel *) &Arg(3);
    2738. 

  2738.      while (n-- > 0) {
    2739. 

  2739. 	 if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) {
    2740. 

  2740. 	     SET_I(ptr->addr);
    2741. 

  2741. 	     Goto(*I);
    2742. 

  2742. 	 }
    2743. 

  2743. 	 ptr++;
    2744. 

  2744.      }
    2745. 

  2745.      SET_I((Eterm *) Arg(1));
    2746. 

  2746.      Goto(*I);
    2747. 

  2747.  }
    2748. 

  2748. #endif
    2749. 

  2749. 

  2750.  OpCase(set_tuple_element_sdP): {
    2751. 

  2751.      Eterm element;
    2752. 

  2752.      Eterm tuple;
    2753. 

  2753.      Eterm* next;
    2754. 

  2754.      Eterm* p;
    2755. 

  2755.      
    2756. 

  2756.      PreFetch(3, next);
    2757. 

  2757.      GetArg2(0, element, tuple);
    2758. 

  2758.      ASSERT(is_tuple(tuple));
    2759. 

  2759.      p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2));
    2760. 

  2760.      *p = element;
    2761. 

  2761.      NextPF(3, next);
    2762. 

  2762.  }
    2763. 

  2763. 

  2764.  OpCase(i_is_ne_exact_f):
    2765. 

  2765.     if (EQ(tmp_arg1, tmp_arg2)) {
    2766. 

  2766. 	ClauseFail();
    2767. 

  2767.     }
    2768. 

  2768.     Next(1);
    2769. 

  2769. 

  2770.  OpCase(normal_exit): {
    2771. 

  2771.      SWAPOUT;
    2772. 

  2772.      c_p->freason = EXC_NORMAL;
    2773. 

  2773.      c_p->arity = 0;		/* In case this process will ever be garbed again. */
    2774. 

  2774.      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    2775. 

  2775.      erts_do_exit_process(c_p, am_normal);
    2776. 

  2776.      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    2777. 

  2777.      goto do_schedule;
    2778. 

  2778.  }
    2779. 

  2779. 

  2780.     /*
    2781. 

  2781.      * Suspend BIF and prepare BIF to be rescheduled.
    2782. 

  2782.      */
    2783. 

  2783.  suspend_bif: {
    2784. 

  2784.      Eterm* argp = c_p->arg_reg;
    2785. 

  2785.      argp[0] = r(0);
    2786. 

  2786.      argp[1] = x(1);
    2787. 

  2787.      argp[2] = x(2);
    2788. 

  2788.      SWAPOUT;
    2789. 

  2789.      c_p->i = I;
    2790. 

  2790.      c_p->current = NULL;
    2791. 

  2791.      goto do_schedule;
    2792. 

  2792.  }
    2793. 

  2793. 

  2794.  OpCase(raise_ss): {
    2795. 

  2795.      /* This was not done very well in R10-0; then, we passed the tag in
    2796. 

  2796. 	the first argument and hoped that the existing c_p->ftrace was
    2797. 

  2797. 	still correct. But the ftrace-object already includes the tag
    2798. 

  2798. 	(or rather, the freason). Now, we pass the original ftrace in
    2799. 

  2799. 	the first argument. We also handle atom tags in the first
    2800. 

  2800. 	argument for backwards compatibility.
    2801. 

  2801.      */
    2802. 

  2802.      GetArg2(0, tmp_arg1, tmp_arg2);
    2803. 

  2803.      c_p->fvalue = tmp_arg2;
    2804. 

  2804.      if (c_p->freason == EXC_NULL) {
    2805. 

  2805.        /* a safety check for the R10-0 case; should not happen */
    2806. 

  2806.        c_p->ftrace = NIL;
    2807. 

  2807.        c_p->freason = EXC_ERROR;
    2808. 

  2808.      }
    2809. 

  2809.      /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
    2810. 

  2810.      switch (tmp_arg1) {
    2811. 

  2811.      case am_throw:
    2812. 

  2812.        c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
    2813. 

  2813.        break;
    2814. 

  2814.      case am_error:
    2815. 

  2815.        c_p->freason = EXC_ERROR & ~EXF_SAVETRACE;
    2816. 

  2816.        break;
    2817. 

  2817.      case am_exit:
    2818. 

  2818.        c_p->freason = EXC_EXIT & ~EXF_SAVETRACE;
    2819. 

  2819.        break;
    2820. 

  2820.      default:
    2821. 

  2821.        {/* R10-1 and later
    2822. 

  2822. 	   XXX note: should do sanity check on given trace if it can be
    2823. 

  2823. 	   passed from a user! Currently only expecting generated calls.
    2824. 

  2824. 	*/
    2825. 

  2825. 	 struct StackTrace *s;
    2826. 

  2826. 	 c_p->ftrace = tmp_arg1;
    2827. 

  2827. 	 s = get_trace_from_exc(tmp_arg1);
    2828. 

  2828. 	 if (s == NULL) {
    2829. 

  2829. 	   c_p->freason = EXC_ERROR;
    2830. 

  2830. 	 } else {
    2831. 

  2831. 	   c_p->freason = PRIMARY_EXCEPTION(s->freason);
    2832. 

  2832. 	 }
    2833. 

  2833.        }
    2834. 

  2834.      }
    2835. 

  2835.      goto find_func_info;
    2836. 

  2836.  }
    2837. 

  2837. 

  2838.  OpCase(badmatch_s): {
    2839. 

  2839.      GetArg1(0, tmp_arg1);
    2840. 

  2840.      c_p->fvalue = tmp_arg1;
    2841. 

  2841.      c_p->freason = BADMATCH;
    2842. 

  2842.  }
    2843. 

  2843.  /* Fall through here */
    2844. 

  2844. 

  2845.  find_func_info: {
    2846. 

  2846.      reg[0] = r(0);
    2847. 

  2847.      SWAPOUT;
    2848. 

  2848.      I = handle_error(c_p, I, reg, NULL);
    2849. 

  2849.      goto post_error_handling;
    2850. 

  2850.  }
    2851. 

  2851. 

  2852.  OpCase(call_error_handler):
    2853. 

  2853.     /*
    2854. 

  2854.      * At this point, I points to the code[3] in the export entry for
    2855. 

  2855.      * a function which is not loaded.
    2856. 

  2856.      *
    2857. 

  2857.      * code[0]: Module
    2858. 

  2858.      * code[1]: Function
    2859. 

  2859.      * code[2]: Arity
    2860. 

  2860.      * code[3]: &&call_error_handler
    2861. 

  2861.      * code[4]: Not used
    2862. 

  2862.      */
    2863. 

  2863.     SWAPOUT;
    2864. 

  2864.     reg[0] = r(0);
    2865. 

  2865.     tmp_arg1 = call_error_handler(c_p, I-3, reg);
    2866. 

  2866.     r(0) = reg[0];
    2867. 

  2867.     SWAPIN;
    2868. 

  2868.     if (tmp_arg1) {
    2869. 

  2869. 	SET_I(c_p->i);
    2870. 

  2870. 	Dispatch();
    2871. 

  2871.     }
    2872. 

  2872. 

  2873.  /* Fall through */
    2874. 

  2874.  OpCase(error_action_code): {
    2875. 

  2875.  no_error_handler:
    2876. 

  2876.      reg[0] = r(0);
    2877. 

  2877.      SWAPOUT;
    2878. 

  2878.      I = handle_error(c_p, NULL, reg, NULL);
    2879. 

  2879.  post_error_handling:
    2880. 

  2880.      if (I == 0) {
    2881. 

  2881. 	 goto do_schedule;
    2882. 

  2882.      } else {
    2883. 

  2883. 	 r(0) = reg[0];
    2884. 

  2884. 	 ASSERT(!is_value(r(0)));
    2885. 

  2885. 	 if (c_p->mbuf) {
    2886. 

  2886. 	     erts_garbage_collect(c_p, 0, reg+1, 3);
    2887. 

  2887. 	 }
    2888. 

  2888. 	 SWAPIN;
    2889. 

  2889. 	 Goto(*I);
    2890. 

  2890.      }
    2891. 

  2891.  }
    2892. 

  2892. 

  2893.  OpCase(apply_bif):
    2894. 

  2894.     /*
    2895. 

  2895.      * At this point, I points to the code[3] in the export entry for
    2896. 

  2896.      * the BIF:
    2897. 

  2897.      *
    2898. 

  2898.      * code[0]: Module
    2899. 

  2899.      * code[1]: Function
    2900. 

  2900.      * code[2]: Arity
    2901. 

  2901.      * code[3]: &&apply_bif
    2902. 

  2902.      * code[4]: Function pointer to BIF function
    2903. 

  2903.      */
    2904. 

  2904.      {
    2905. 

  2905. 	BifFunction vbf;
    2906. 

  2906. 

  2907. 	c_p->current = I-3;	/* In case we apply process_info/1,2. */
    2908. 

  2908. 	c_p->i = I;		/* In case we apply check_process_code/2. */
    2909. 

  2909. 	c_p->arity = 0;		/* To allow garbage collection on ourselves
    2910. 

  2910. 				 * (check_process_code/2).
    2911. 

  2911. 				 */
    2912. 

  2912. 				   
    2913. 

  2913. 	SWAPOUT;
    2914. 

  2914. 	c_p->fcalls = FCALLS - 1;
    2915. 

  2915. 	vbf = (BifFunction) Arg(0);
    2916. 

  2916. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    2917. 

  2917. 	tmp_arg2 = I[-1];
    2918. 

  2918. 	ASSERT(tmp_arg2 <= 3);
    2919. 

  2919. 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    2920. 

  2920. 	switch (tmp_arg2) {
    2921. 

  2921. 	case 3:
    2922. 

  2922. 	    {
    2923. 

  2923. 		Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = vbf;
    2924. 

  2924. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2925. 

  2925. 		tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I);
    2926. 

  2926. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
    2927. 

  2927. 		PROCESS_MAIN_CHK_LOCKS(c_p);
    2928. 

  2928. 	    }
    2929. 

  2929. 	    break;
    2930. 

  2930. 	case 2:
    2931. 

  2931. 	    {
    2932. 

  2932. 		Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = vbf;
    2933. 

  2933. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2934. 

  2934. 		tmp_arg1 = (*bf)(c_p, r(0), x(1), I);
    2935. 

  2935. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
    2936. 

  2936. 		PROCESS_MAIN_CHK_LOCKS(c_p);
    2937. 

  2937. 	    }
    2938. 

  2938. 	    break;
    2939. 

  2939. 	case 1:
    2940. 

  2940. 	    {
    2941. 

  2941. 		Eterm (*bf)(Process*, Eterm, Uint*) = vbf;
    2942. 

  2942. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2943. 

  2943. 		tmp_arg1 = (*bf)(c_p, r(0), I);
    2944. 

  2944. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
    2945. 

  2945. 		PROCESS_MAIN_CHK_LOCKS(c_p);
    2946. 

  2946. 	    }
    2947. 

  2947. 	    break;
    2948. 

  2948. 	case 0:
    2949. 

  2949. 	    {
    2950. 

  2950. 		Eterm (*bf)(Process*, Uint*) = vbf;
    2951. 

  2951. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    2952. 

  2952. 		tmp_arg1 = (*bf)(c_p, I);
    2953. 

  2953. 		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
    2954. 

  2954. 		PROCESS_MAIN_CHK_LOCKS(c_p);
    2955. 

  2955. 		break;
    2956. 

  2956. 	    }
    2957. 

  2957. 	}
    2958. 

  2958. 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    2959. 

  2959. 	ERTS_HOLE_CHECK(c_p);
    2960. 

  2960. 	if (c_p->mbuf) {
    2961. 

  2961. 	    reg[0] = r(0);
    2962. 

  2962. 	    tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2);
    2963. 

  2963. 	    r(0) = reg[0];
    2964. 

  2964. 	}
    2965. 

  2965. 	SWAPIN;			/* There might have been a garbage collection. */
    2966. 

  2966. 	FCALLS = c_p->fcalls;
    2967. 

  2967. 	if (is_value(tmp_arg1)) {
    2968. 

  2968. 	    r(0) = tmp_arg1;
    2969. 

  2969. 	    CHECK_TERM(r(0));
    2970. 

  2970. 	    SET_I(c_p->cp);
    2971. 

  2971. 	    Goto(*I);
    2972. 

  2972. 	} else if (c_p->freason == RESCHEDULE) {
    2973. 

  2973. 	    Eterm* argp = c_p->arg_reg;
    2974. 

  2974. 

  2975. 	    c_p->arity = I[-1];
    2976. 

  2976. 	    argp[0] = r(0);
    2977. 

  2977. 	    argp[1] = x(1);
    2978. 

  2978. 	    argp[2] = x(2);
    2979. 

  2979. 	    SWAPOUT;
    2980. 

  2980. 	    c_p->i = I;
    2981. 

  2981. 	    c_p->current = NULL;
    2982. 

  2982. 	    goto do_schedule;
    2983. 

  2983. 	} else if (c_p->freason == TRAP) {
    2984. 

  2984. 	    SET_I(((Export *)(c_p->def_arg_reg[3]))->address);
    2985. 

  2985. 	    r(0) = c_p->def_arg_reg[0];
    2986. 

  2986. 	    x(1) = c_p->def_arg_reg[1];
    2987. 

  2987. 	    x(2) = c_p->def_arg_reg[2];
    2988. 

  2988. 	    Dispatch();
    2989. 

  2989. 	}
    2990. 

  2990. 	reg[0] = r(0);
    2991. 

  2991. 	I = handle_error(c_p, c_p->cp, reg, vbf);
    2992. 

  2992. 	goto post_error_handling;
    2993. 

  2993.     }
    2994. 

  2994. 

  2995.  OpCase(i_get_sd):
    2996. 

  2996.     {
    2997. 

  2997. 	Eterm arg;
    2998. 

  2998. 	Eterm result;
    2999. 

  2999. 

  3000. 	GetArg1(0, arg);
    3001. 

  3001. 	result = erts_pd_hash_get(c_p, arg);
    3002. 

  3002. 	StoreBifResult(1, result);
    3003. 

  3003.     }
    3004. 

  3004. 

  3005.  OpCase(i_put_tuple_only_Ad): {
    3006. 

  3006.      tmp_arg1 = make_tuple(HTOP);
    3007. 

  3007.      *HTOP++ = Arg(0);
    3008. 

  3008.      StoreBifResult(1, tmp_arg1);
    3009. 

  3009.  }
    3010. 

  3010. 

  3011.  OpCase(case_end_s):
    3012. 

  3012.     GetArg1(0, tmp_arg1);
    3013. 

  3013.     c_p->fvalue = tmp_arg1;
    3014. 

  3014.     c_p->freason = EXC_CASE_CLAUSE;
    3015. 

  3015.     goto find_func_info;
    3016. 

  3016. 

  3017.  OpCase(if_end):
    3018. 

  3018.     c_p->freason = EXC_IF_CLAUSE;
    3019. 

  3019.     goto find_func_info;
    3020. 

  3020. 

  3021.  OpCase(i_func_info_IaaI): {
    3022. 

  3022.      c_p->freason = EXC_FUNCTION_CLAUSE;
    3023. 

  3023.      c_p->current = I + 2;
    3024. 

  3024.      goto lb_error_action_code;
    3025. 

  3025.  }
    3026. 

  3026. 

  3027.  OpCase(try_case_end_s):
    3028. 

  3028.     GetArg1(0, tmp_arg1);
    3029. 

  3029.     c_p->fvalue = tmp_arg1;
    3030. 

  3030.     c_p->freason = EXC_TRY_CLAUSE;
    3031. 

  3031.     goto find_func_info;
    3032. 

  3032. 

  3033.  /*
    3034. 

  3034.   * Construction of binaries using new instructions.
    3035. 

  3035.   */
    3036. 

  3036.  {
    3037. 

  3037.      Eterm new_binary;
    3038. 

  3038.      Eterm num_bits_term;
    3039. 

  3039.      Uint num_bits;
    3040. 

  3040.      Uint alloc;
    3041. 

  3041.      Uint num_bytes;
    3042. 

  3042. 

  3043.      OpCase(i_bs_init_bits_heap_IIId): {
    3044. 

  3044. 	 num_bits = Arg(0);
    3045. 

  3045. 	 alloc = Arg(1);
    3046. 

  3046. 	 I++;
    3047. 

  3047. 	 goto do_bs_init_bits_known;
    3048. 

  3048.      }
    3049. 

  3049.      
    3050. 

  3050.      OpCase(i_bs_init_bits_IId): {
    3051. 

  3051. 	 num_bits = Arg(0);
    3052. 

  3052. 	 alloc = 0;
    3053. 

  3053. 	 goto do_bs_init_bits_known;
    3054. 

  3054.      }
    3055. 

  3055. 

  3056.      OpCase(i_bs_init_bits_fail_heap_IjId): {
    3057. 

  3057. 	 /* tmp_arg1 was fetched by an i_fetch instruction */
    3058. 

  3058. 	 num_bits_term = tmp_arg1;
    3059. 

  3059. 	 alloc = Arg(0);
    3060. 

  3060. 	 I++;
    3061. 

  3061. 	 goto do_bs_init_bits;
    3062. 

  3062.      }
    3063. 

  3063. 

  3064.      OpCase(i_bs_init_bits_fail_rjId): {
    3065. 

  3065. 	 num_bits_term = r(0);
    3066. 

  3066. 	 alloc = 0;
    3067. 

  3067. 	 goto do_bs_init_bits;
    3068. 

  3068.      }
    3069. 

  3069.      OpCase(i_bs_init_bits_fail_yjId): {
    3070. 

  3070. 	 num_bits_term = yb(Arg(0));
    3071. 

  3071. 	 I++;
    3072. 

  3072. 	 alloc = 0;
    3073. 

  3073. 	 goto do_bs_init_bits;
    3074. 

  3074.      }
    3075. 

  3075.      OpCase(i_bs_init_bits_fail_xjId): {
    3076. 

  3076. 	 num_bits_term = xb(Arg(0));
    3077. 

  3077. 	 I++;
    3078. 

  3078. 	 alloc = 0;
    3079. 

  3079.      /* FALL THROUGH */
    3080. 

  3080.      }
    3081. 

  3081. 

  3082.      /* num_bits_term = Term for number of bits to build (small/big)
    3083. 

  3083.       * alloc = Number of words to allocate on heap
    3084. 

  3084.       * Operands: Fail Live Dst
    3085. 

  3085.       */
    3086. 

  3086. 

  3087.  do_bs_init_bits:
    3088. 

  3088.      if (is_small(num_bits_term)) {
    3089. 

  3089. 	 Sint size = signed_val(num_bits_term);
    3090. 

  3090. 	 if (size < 0) {
    3091. 

  3091. 	     goto badarg;
    3092. 

  3092. 	 }
    3093. 

  3093. 	 num_bits = (Uint) size;
    3094. 

  3094.      } else if (is_big(num_bits_term) &&
    3095. 

  3095. 		!bignum_header_is_neg(*big_val(num_bits_term))) {
    3096. 

  3096. 	 Uint bits;
    3097. 

  3097. 

  3098. 	 if (!term_to_Uint(num_bits_term, &bits)) {
    3099. 

  3099. 	     goto system_limit;
    3100. 

  3100. 	 }
    3101. 

  3101. 	 num_bits = (Eterm) bits;
    3102. 

  3102.      } else {
    3103. 

  3103. 	 goto badarg;
    3104. 

  3104.      }
    3105. 

  3105. 

  3106.      /* num_bits = Number of bits to build
    3107. 

  3107.       * alloc = Number of extra words to allocate on heap
    3108. 

  3108.       * Operands: NotUsed Live Dst
    3109. 

  3109.       */
    3110. 

  3110.  do_bs_init_bits_known:
    3111. 

  3111.      num_bytes = (num_bits+7) >> 3;
    3112. 

  3112.      if (num_bits & 7) {
    3113. 

  3113. 	 alloc += ERL_SUB_BIN_SIZE;
    3114. 

  3114.      }
    3115. 

  3115.      if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
    3116. 

  3116. 	 alloc += heap_bin_size(num_bytes);
    3117. 

  3117.      } else {
    3118. 

  3118. 	 alloc += PROC_BIN_SIZE;
    3119. 

  3119.      }
    3120. 

  3120.      TestHeap(alloc, Arg(1));
    3121. 

  3121. 

  3122.      /* num_bits = Number of bits to build
    3123. 

  3123.       * num_bytes = Number of bytes to allocate in the binary
    3124. 

  3124.       * alloc = Total number of words to allocate on heap
    3125. 

  3125.       * Operands: NotUsed NotUsed Dst
    3126. 

  3126.       */
    3127. 

  3127.      if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
    3128. 

  3128. 	 ErlHeapBin* hb;
    3129. 

  3129. 

  3130. 	 erts_bin_offset = 0;
    3131. 

  3131. 	 erts_writable_bin = 0;
    3132. 

  3132. 	 hb = (ErlHeapBin *) HTOP;
    3133. 

  3133. 	 HTOP += heap_bin_size(num_bytes);
    3134. 

  3134. 	 hb->thing_word = header_heap_bin(num_bytes);
    3135. 

  3135. 	 hb->size = num_bytes;
    3136. 

  3136. 	 erts_current_bin = (byte *) hb->data;
    3137. 

  3137. 	 new_binary = make_binary(hb);
    3138. 

  3138. 

  3139.      do_bits_sub_bin:
    3140. 

  3140. 	 if (num_bits & 7) {
    3141. 

  3141. 	     ErlSubBin* sb;
    3142. 

  3142. 

  3143. 	     sb = (ErlSubBin *) HTOP;
    3144. 

  3144. 	     HTOP += ERL_SUB_BIN_SIZE;
    3145. 

  3145. 	     sb->thing_word = HEADER_SUB_BIN;
    3146. 

  3146. 	     sb->size = num_bytes - 1;
    3147. 

  3147. 	     sb->bitsize = num_bits & 7;
    3148. 

  3148. 	     sb->offs = 0;
    3149. 

  3149. 	     sb->bitoffs = 0;
    3150. 

  3150. 	     sb->is_writable = 0;
    3151. 

  3151. 	     sb->orig = new_binary;
    3152. 

  3152. 	     new_binary = make_binary(sb);
    3153. 

  3153. 	 }
    3154. 

  3154. 	 StoreBifResult(2, new_binary);
    3155. 

  3155.      } else {
    3156. 

  3156. 	 Binary* bptr;
    3157. 

  3157. 	 ProcBin* pb;
    3158. 

  3158. 

  3159. 	 erts_bin_offset = 0;
    3160. 

  3160. 	 erts_writable_bin = 0;
    3161. 

  3161. 

  3162. 	 /*
    3163. 

  3163. 	  * Allocate the binary struct itself.
    3164. 

  3164. 	  */
    3165. 

  3165. 	 bptr = erts_bin_nrml_alloc(num_bytes);
    3166. 

  3166. 	 bptr->flags = 0;
    3167. 

  3167. 	 bptr->orig_size = num_bytes;
    3168. 

  3168. 	 erts_refc_init(&bptr->refc, 1);
    3169. 

  3169. 	 erts_current_bin = (byte *) bptr->orig_bytes;
    3170. 

  3170. 

  3171. 	 /*
    3172. 

  3172. 	  * Now allocate the ProcBin on the heap.
    3173. 

  3173. 	  */
    3174. 

  3174. 	 pb = (ProcBin *) HTOP;
    3175. 

  3175. 	 HTOP += PROC_BIN_SIZE;
    3176. 

  3176. 	 pb->thing_word = HEADER_PROC_BIN;
    3177. 

  3177. 	 pb->size = num_bytes;
    3178. 

  3178. 	 pb->next = MSO(c_p).mso;
    3179. 

  3179. 	 MSO(c_p).mso = pb;
    3180. 

  3180. 	 pb->val = bptr;
    3181. 

  3181. 	 pb->bytes = (byte*) bptr->orig_bytes;
    3182. 

  3182. 	 pb->flags = 0;
    3183. 

  3183. 	 MSO(c_p).overhead += pb->size / BINARY_OVERHEAD_FACTOR / sizeof(Eterm);
    3184. 

  3184. 	 new_binary = make_binary(pb);
    3185. 

  3185. 	 goto do_bits_sub_bin;
    3186. 

  3186.      }
    3187. 

  3187.  }
    3188. 

  3188. 

  3189.  {
    3190. 

  3190.      OpCase(i_bs_init_fail_heap_IjId): {
    3191. 

  3191. 	 /* tmp_arg1 was fetched by an i_fetch instruction */
    3192. 

  3192. 	 tmp_arg2 = Arg(0);
    3193. 

  3193. 	 I++;
    3194. 

  3194. 	 goto do_bs_init;
    3195. 

  3195.      }
    3196. 

  3196. 

  3197.      OpCase(i_bs_init_fail_rjId): {
    3198. 

  3198. 	 tmp_arg1 = r(0);
    3199. 

  3199. 	 tmp_arg2 = 0;
    3200. 

  3200. 	 goto do_bs_init;
    3201. 

  3201.      }
    3202. 

  3202. 

  3203.      OpCase(i_bs_init_fail_yjId): {
    3204. 

  3204. 	 tmp_arg1 = yb(Arg(0));
    3205. 

  3205. 	 tmp_arg2 = 0;
    3206. 

  3206. 	 I++;
    3207. 

  3207. 	 goto do_bs_init;
    3208. 

  3208.      }
    3209. 

  3209. 

  3210.      OpCase(i_bs_init_fail_xjId): {
    3211. 

  3211. 	 tmp_arg1 = xb(Arg(0));
    3212. 

  3212. 	 tmp_arg2 = 0;
    3213. 

  3213. 	 I++;
    3214. 

  3214.      }
    3215. 

  3215. 	 /* FALL THROUGH */
    3216. 

  3216.      do_bs_init:
    3217. 

  3217.          if (is_small(tmp_arg1)) {
    3218. 

  3218. 	     Sint size = signed_val(tmp_arg1);
    3219. 

  3219. 	     if (size < 0) {
    3220. 

  3220. 		 goto badarg;
    3221. 

  3221. 	     }
    3222. 

  3222. 	     tmp_arg1 = (Eterm) size;
    3223. 

  3223. 	 } else if (is_big(tmp_arg1) && !bignum_header_is_neg(*big_val(tmp_arg1))) {
    3224. 

  3224. 	     Sint bytes;
    3225. 

  3225. 	     if (!term_to_Sint(tmp_arg1, &bytes)) {
    3226. 

  3226. 		 goto system_limit;
    3227. 

  3227. 	     }
    3228. 

  3228. 	     if (bytes < 0) {
    3229. 

  3229. 		 goto badarg;
    3230. 

  3230. 	     }
    3231. 

  3231. 	     if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
    3232. 

  3232. 		 goto system_limit;
    3233. 

  3233. 	     }
    3234. 

  3234. 	     tmp_arg1 = (Eterm) bytes;
    3235. 

  3235. 	 } else {
    3236. 

  3236. 	     goto badarg;
    3237. 

  3237. 	 }
    3238. 

  3238. 

  3239. 	 if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) {
    3240. 

  3240. 	     goto do_heap_bin_alloc;
    3241. 

  3241. 	 } else {
    3242. 

  3242. 	     goto do_proc_bin_alloc;
    3243. 

  3243. 	 }
    3244. 

  3244. 

  3245. 

  3246.      OpCase(i_bs_init_heap_IIId): {
    3247. 

  3247. 	 tmp_arg1 = Arg(0);
    3248. 

  3248. 	 tmp_arg2 = Arg(1);
    3249. 

  3249. 	 I++;
    3250. 

  3250. 	 goto do_proc_bin_alloc;
    3251. 

  3251.      }
    3252. 

  3252. 

  3253.      OpCase(i_bs_init_IId): {
    3254. 

  3254. 	 tmp_arg1 = Arg(0);
    3255. 

  3255. 	 tmp_arg2 = 0;
    3256. 

  3256.      }
    3257. 

  3257.      /* FALL THROUGH */
    3258. 

  3258.      do_proc_bin_alloc: {
    3259. 

  3259. 	 Binary* bptr;
    3260. 

  3260. 	 ProcBin* pb;
    3261. 

  3261. 

  3262. 	 erts_bin_offset = 0;
    3263. 

  3263. 	 erts_writable_bin = 0;
    3264. 

  3264. 	 TestHeap(tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));
    3265. 

  3265. 

  3266. 	 /*
    3267. 

  3267. 	  * Allocate the binary struct itself.
    3268. 

  3268. 	  */
    3269. 

  3269. 	 bptr = erts_bin_nrml_alloc(tmp_arg1);
    3270. 

  3270. 	 bptr->flags = 0;
    3271. 

  3271. 	 bptr->orig_size = tmp_arg1;
    3272. 

  3272. 	 erts_refc_init(&bptr->refc, 1);
    3273. 

  3273. 	 erts_current_bin = (byte *) bptr->orig_bytes;
    3274. 

  3274. 

  3275. 	 /*
    3276. 

  3276. 	  * Now allocate the ProcBin on the heap.
    3277. 

  3277. 	  */
    3278. 

  3278. 	 pb = (ProcBin *) HTOP;
    3279. 

  3279. 	 HTOP += PROC_BIN_SIZE;
    3280. 

  3280. 	 pb->thing_word = HEADER_PROC_BIN;
    3281. 

  3281. 	 pb->size = tmp_arg1;
    3282. 

  3282. 	 pb->next = MSO(c_p).mso;
    3283. 

  3283. 	 MSO(c_p).mso = pb;
    3284. 

  3284. 	 pb->val = bptr;
    3285. 

  3285. 	 pb->bytes = (byte*) bptr->orig_bytes;
    3286. 

  3286. 	 pb->flags = 0;
    3287. 

  3287. 

  3288. 	 MSO(c_p).overhead += pb->size / BINARY_OVERHEAD_FACTOR / sizeof(Eterm);
    3289. 

  3289. 	 StoreBifResult(2, make_binary(pb));
    3290. 

  3290.      }
    3291. 

  3291. 

  3292.      OpCase(i_bs_init_heap_bin_heap_IIId): {
    3293. 

  3293. 	 tmp_arg1 = Arg(0);
    3294. 

  3294. 	 tmp_arg2 = Arg(1);
    3295. 

  3295. 	 I++;
    3296. 

  3296. 	 goto do_heap_bin_alloc;
    3297. 

  3297.      }
    3298. 

  3298. 

  3299.      OpCase(i_bs_init_heap_bin_IId): {
    3300. 

  3300. 	 tmp_arg1 = Arg(0);
    3301. 

  3301. 	 tmp_arg2 = 0;
    3302. 

  3302.      }
    3303. 

  3303.      /* Fall through */
    3304. 

  3304.      do_heap_bin_alloc:
    3305. 

  3305. 	 {
    3306. 

  3306. 	     ErlHeapBin* hb;
    3307. 

  3307. 	     Uint bin_need;
    3308. 

  3308. 

  3309. 	     bin_need = heap_bin_size(tmp_arg1);
    3310. 

  3310. 	     erts_bin_offset = 0;
    3311. 

  3311. 	     erts_writable_bin = 0;
    3312. 

  3312. 	     TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1));
    3313. 

  3313. 	     hb = (ErlHeapBin *) HTOP;
    3314. 

  3314. 	     HTOP += bin_need;
    3315. 

  3315. 	     hb->thing_word = header_heap_bin(tmp_arg1);
    3316. 

  3316. 	     hb->size = tmp_arg1;
    3317. 

  3317. 	     erts_current_bin = (byte *) hb->data;
    3318. 

  3318. 	     tmp_arg1 = make_binary(hb);
    3319. 

  3319. 	     StoreBifResult(2, tmp_arg1);
    3320. 

  3320. 	 }
    3321. 

  3321.  }
    3322. 

  3322. 

  3323.  OpCase(i_bs_bits_to_bytes_rjd): {
    3324. 

  3324.      tmp_arg1 = r(0);
    3325. 

  3325.      goto do_bits_to_bytes;
    3326. 

  3326.  }
    3327. 

  3327. 

  3328.  OpCase(i_bs_bits_to_bytes_yjd): {
    3329. 

  3329.      tmp_arg1 = yb(Arg(0));
    3330. 

  3330.      I++;
    3331. 

  3331.      goto do_bits_to_bytes;
    3332. 

  3332. 

  3333.  OpCase(i_bs_bits_to_bytes_xjd): {
    3334. 

  3334.      tmp_arg1 = xb(Arg(0));
    3335. 

  3335.      I++;
    3336. 

  3336.  }
    3337. 

  3337. 

  3338.  do_bits_to_bytes:
    3339. 

  3339.      {
    3340. 

  3340. 	 if (is_valid_bit_size(tmp_arg1)) {
    3341. 

  3341. 	     tmp_arg1 = make_small(unsigned_val(tmp_arg1) >> 3);
    3342. 

  3342. 	 } else {
    3343. 

  3343. 	     Uint bytes;
    3344. 

  3344. 	     if (!term_to_Uint(tmp_arg1, &bytes)) {
    3345. 

  3345. 		 goto badarg;
    3346. 

  3346. 	     }
    3347. 

  3347. 	     tmp_arg1 = bytes;
    3348. 

  3348. 	     if ((tmp_arg1 & 0x07) != 0) {
    3349. 

  3349. 		 goto badarg;
    3350. 

  3350. 	     }
    3351. 

  3351. 	     SWAPOUT;
    3352. 

  3352. 	     tmp_arg1 = erts_make_integer(tmp_arg1 >> 3, c_p);
    3353. 

  3353. 	     HTOP = HEAP_TOP(c_p);
    3354. 

  3354. 	 }
    3355. 

  3355. 	 StoreBifResult(1, tmp_arg1);
    3356. 

  3356.      }
    3357. 

  3357.  }
    3358. 

  3358. 

  3359.  OpCase(i_bs_add_jId): {
    3360. 

  3360.      Uint Unit = Arg(1);
    3361. 

  3361.      if (is_both_small(tmp_arg1, tmp_arg2)) {
    3362. 

  3362. 	 Sint Arg1 = signed_val(tmp_arg1);
    3363. 

  3363. 	 Sint Arg2 = signed_val(tmp_arg2);
    3364. 

  3364. 

  3365. 	 if (Arg1 >= 0 && Arg2 >= 0) {
    3366. 

  3366. 	     BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1);
    3367. 

  3367. 	     if ((Sint) tmp_arg1 < 0) {
    3368. 

  3368. 		 goto system_limit;
    3369. 

  3369. 	     }
    3370. 

  3370. 	     tmp_arg1 += Arg1;
    3371. 

  3371. 

  3372. 	 store_bs_add_result:
    3373. 

  3373. 	     if (MY_IS_SSMALL((Sint) tmp_arg1)) {
    3374. 

  3374. 		 tmp_arg1 = make_small(tmp_arg1);
    3375. 

  3375. 	     } else {
    3376. 

  3376. 		 /*
    3377. 

  3377. 		  * May generate a heap fragment, but in this particular case it is OK,
    3378. 

  3378. 		  * since the value will be stored into an x register (the GC will
    3379. 

  3379. 		  * scan x registers for references to heap fragments) and there is no
    3380. 

  3380. 		  * risk that value can be stored into a location that is not scanned
    3381. 

  3381. 		  * for heap-fragment references (such as the heap).
    3382. 

  3382. 		  */
    3383. 

  3383. 		 SWAPOUT;
    3384. 

  3384. 		 tmp_arg1 = erts_make_integer(tmp_arg1, c_p);
    3385. 

  3385. 		 HTOP = HEAP_TOP(c_p);
    3386. 

  3386. 	     }
    3387. 

  3387. 	     StoreBifResult(2, tmp_arg1);
    3388. 

  3388. 	 }
    3389. 

  3389.      } else {
    3390. 

  3390. 	 Uint a;
    3391. 

  3391. 	 Uint b;
    3392. 

  3392. 	 if (term_to_Uint(tmp_arg1, &a) && term_to_Uint(tmp_arg2, &b)) {
    3393. 

  3393. 	     BsSafeMul(b, Unit, goto system_limit, tmp_arg1);
    3394. 

  3394. 	     if ((Sint) tmp_arg1 < 0) {
    3395. 

  3395. 		 goto system_limit;
    3396. 

  3396. 	     }
    3397. 

  3397. 	     tmp_arg1 += a;
    3398. 

  3398. 	     goto store_bs_add_result;
    3399. 

  3399. 	 }
    3400. 

  3400.      }
    3401. 

  3401. 

  3402.      /*
    3403. 

  3403.       * Fall through to here if there were any errors in the arguments.
    3404. 

  3404.       */
    3405. 

  3405.      goto badarg;
    3406. 

  3406.  }
    3407. 

  3407. 

  3408.  OpCase(bs_put_string_II):
    3409. 

  3409.     {
    3410. 

  3410. 	Eterm* next;
    3411. 

  3411. 	PreFetch(2, next);
    3412. 

  3412. 	erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0)));
    3413. 

  3413. 	NextPF(2, next);
    3414. 

  3414.     }
    3415. 

  3415. 

  3416.  /*
    3417. 

  3417.   * tmp_arg1 = Number of bytes to build
    3418. 

  3418.   * tmp_arg2 = Source binary
    3419. 

  3419.   * Operands: Fail ExtraHeap Live Unit Dst
    3420. 

  3420.   */
    3421. 

  3421. 

  3422.  OpCase(i_bs_append_jIIId): {
    3423. 

  3423.      Uint live = Arg(2);
    3424. 

  3424.      Uint res;
    3425. 

  3425. 

  3426.      SWAPOUT;
    3427. 

  3427.      reg[0] = r(0);
    3428. 

  3428.      reg[live] = tmp_arg2;
    3429. 

  3429.      res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3));
    3430. 

  3430.      r(0) = reg[0];
    3431. 

  3431.      SWAPIN;
    3432. 

  3432.      if (is_non_value(res)) {
    3433. 

  3433. 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
    3434. 

  3434. 	 goto lb_Cl_error;
    3435. 

  3435.      }
    3436. 

  3436.      StoreBifResult(4, res);
    3437. 

  3437.  }
    3438. 

  3438. 

  3439.  /*
    3440. 

  3440.   * tmp_arg1 = Number of bytes to build
    3441. 

  3441.   * tmp_arg2 = Source binary
    3442. 

  3442.   * Operands: Fail Unit Dst
    3443. 

  3443.   */
    3444. 

  3444.  OpCase(i_bs_private_append_jId): {
    3445. 

  3445.      Eterm res;
    3446. 

  3446. 

  3447.      res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1));
    3448. 

  3448.      if (is_non_value(res)) {
    3449. 

  3449. 	 goto system_limit;
    3450. 

  3450.      }
    3451. 

  3451.      StoreBifResult(2, res);
    3452. 

  3452.  }
    3453. 

  3453. 

  3454.  /*
    3455. 

  3455.   * tmp_arg1 = Initial size of writable binary
    3456. 

  3456.   * Operands: Live Dst
    3457. 

  3457.   */
    3458. 

  3458.  OpCase(bs_init_writable): {
    3459. 

  3459.      SWAPOUT;
    3460. 

  3460.      r(0) = erts_bs_init_writable(c_p, r(0));
    3461. 

  3461.      SWAPIN;
    3462. 

  3462.      Next(0);
    3463. 

  3463.  }
    3464. 

  3464. 

  3465.  /*
    3466. 

  3466.   * Matching of binaries.
    3467. 

  3467.   */
    3468. 

  3468. 

  3469.  {
    3470. 

  3470.      Eterm header;
    3471. 

  3471.      Eterm* next;
    3472. 

  3472.      Uint slots;
    3473. 

  3473. 

  3474.      OpCase(i_bs_start_match2_rfIId): {
    3475. 

  3475. 	 tmp_arg1 = r(0);
    3476. 

  3476. 

  3477.      do_start_match:
    3478. 

  3478. 	 slots = Arg(2);
    3479. 

  3479. 	 if (!is_boxed(tmp_arg1)) {
    3480. 

  3480. 	     ClauseFail();
    3481. 

  3481. 	 }
    3482. 

  3482. 	 PreFetch(4, next);
    3483. 

  3483. 	 header = *boxed_val(tmp_arg1);
    3484. 

  3484. 	 if (header_is_bin_matchstate(header)) {
    3485. 

  3485. 	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
    3486. 

  3486. 	     Uint actual_slots = HEADER_NUM_SLOTS(header);
    3487. 

  3487. 	     ms->save_offset[0] = ms->mb.offset;
    3488. 

  3488. 	     if (actual_slots < slots) {
    3489. 

  3489. 		 ErlBinMatchState* dst;
    3490. 

  3490. 		 Uint live = Arg(1);
    3491. 

  3491. 		 Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
    3492. 

  3492. 

  3493. 		 TestHeapPreserve(wordsneeded, live, tmp_arg1);
    3494. 

  3494. 		 ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
    3495. 

  3495. 		 dst = (ErlBinMatchState *) HTOP;
    3496. 

  3496. 		 *dst = *ms;
    3497. 

  3497. 		 *HTOP = HEADER_BIN_MATCHSTATE(slots);
    3498. 

  3498. 		 HTOP += wordsneeded;
    3499. 

  3499. 		 StoreResult(make_matchstate(dst), Arg(3));
    3500. 

  3500. 	     }
    3501. 

  3501. 	 } else if (is_binary_header(header)) {
    3502. 

  3502. 	     Eterm result;
    3503. 

  3503. 	     Uint live = Arg(1);
    3504. 

  3504. 	     Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
    3505. 

  3505. 	     TestHeapPreserve(wordsneeded, live, tmp_arg1);
    3506. 

  3506. 	     SWAPOUT;
    3507. 

  3507. 	     result = erts_bs_start_match_2(c_p, tmp_arg1, slots);
    3508. 

  3508. 	     HTOP = HEAP_TOP(c_p);
    3509. 

  3509. 	     if (is_non_value(result)) {
    3510. 

  3510. 		 ClauseFail();
    3511. 

  3511. 	     } else {
    3512. 

  3512. 		 StoreResult(result, Arg(3));
    3513. 

  3513. 	     }
    3514. 

  3514. 	 } else {
    3515. 

  3515. 	     ClauseFail();
    3516. 

  3516. 	 }
    3517. 

  3517. 	 NextPF(4, next);
    3518. 

  3518.      }
    3519. 

  3519.      OpCase(i_bs_start_match2_xfIId): {
    3520. 

  3520. 	 tmp_arg1 = xb(Arg(0));
    3521. 

  3521. 	 I++;
    3522. 

  3522. 	 goto do_start_match;
    3523. 

  3523.      }
    3524. 

  3524.      OpCase(i_bs_start_match2_yfIId): {
    3525. 

  3525. 	 tmp_arg1 = yb(Arg(0));
    3526. 

  3526. 	 I++;
    3527. 

  3527. 	 goto do_start_match;
    3528. 

  3528.      }
    3529. 

  3529.  }
    3530. 

  3530. 

  3531.  OpCase(bs_test_zero_tail2_fr): {
    3532. 

  3532.      Eterm* next;
    3533. 

  3533.      ErlBinMatchBuffer *_mb;
    3534. 

  3534.      
    3535. 

  3535.      PreFetch(1, next);
    3536. 

  3536.      _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0));
    3537. 

  3537.      if (_mb->size != _mb->offset) {
    3538. 

  3538. 	 ClauseFail();
    3539. 

  3539.      }
    3540. 

  3540.      NextPF(1, next);
    3541. 

  3541.  }
    3542. 

  3542. 

  3543.  OpCase(bs_test_zero_tail2_fx): {
    3544. 

  3544.      Eterm* next;
    3545. 

  3545.      ErlBinMatchBuffer *_mb;
    3546. 

  3546.      
    3547. 

  3547.      PreFetch(2, next);
    3548. 

  3548.      _mb = (ErlBinMatchBuffer*) ms_matchbuffer(xb(Arg(1)));
    3549. 

  3549.      if (_mb->size != _mb->offset) {
    3550. 

  3550. 	 ClauseFail();
    3551. 

  3551.      }
    3552. 

  3552.      NextPF(2, next);
    3553. 

  3553.  }
    3554. 

  3554. 

  3555.  OpCase(bs_test_tail_imm2_frI): {
    3556. 

  3556.      Eterm* next;
    3557. 

  3557.      ErlBinMatchBuffer *_mb;
    3558. 

  3558.      PreFetch(2, next);
    3559. 

  3559.      _mb = ms_matchbuffer(r(0));
    3560. 

  3560.      if (_mb->size - _mb->offset != Arg(1)) {
    3561. 

  3561. 	 ClauseFail();
    3562. 

  3562.      }
    3563. 

  3563.      NextPF(2, next);
    3564. 

  3564.  }
    3565. 

  3565.  OpCase(bs_test_tail_imm2_fxI): {
    3566. 

  3566.      Eterm* next;
    3567. 

  3567.      ErlBinMatchBuffer *_mb;
    3568. 

  3568.      PreFetch(3, next);
    3569. 

  3569.      _mb = ms_matchbuffer(xb(Arg(1)));
    3570. 

  3570.      if (_mb->size - _mb->offset != Arg(2)) {
    3571. 

  3571. 	 ClauseFail();
    3572. 

  3572.      }
    3573. 

  3573.      NextPF(3, next);
    3574. 

  3574.  }
    3575. 

  3575. 

  3576.  OpCase(bs_test_unit_frI): {
    3577. 

  3577.      Eterm* next;
    3578. 

  3578.      ErlBinMatchBuffer *_mb;
    3579. 

  3579.      PreFetch(2, next);
    3580. 

  3580.      _mb = ms_matchbuffer(r(0));
    3581. 

  3581.      if ((_mb->size - _mb->offset) % Arg(1)) {
    3582. 

  3582. 	 ClauseFail();
    3583. 

  3583.      }
    3584. 

  3584.      NextPF(2, next);
    3585. 

  3585.  }
    3586. 

  3586.  OpCase(bs_test_unit_fxI): {
    3587. 

  3587.      Eterm* next;
    3588. 

  3588.      ErlBinMatchBuffer *_mb;
    3589. 

  3589.      PreFetch(3, next);
    3590. 

  3590.      _mb = ms_matchbuffer(xb(Arg(1)));
    3591. 

  3591.      if ((_mb->size - _mb->offset) % Arg(2)) {
    3592. 

  3592. 	 ClauseFail();
    3593. 

  3593.      }
    3594. 

  3594.      NextPF(3, next);
    3595. 

  3595.  }
    3596. 

  3596. 

  3597.  OpCase(bs_test_unit8_fr): {
    3598. 

  3598.      Eterm* next;
    3599. 

  3599.      ErlBinMatchBuffer *_mb;
    3600. 

  3600.      PreFetch(1, next);
    3601. 

  3601.      _mb = ms_matchbuffer(r(0));
    3602. 

  3602.      if ((_mb->size - _mb->offset) & 7) {
    3603. 

  3603. 	 ClauseFail();
    3604. 

  3604.      }
    3605. 

  3605.      NextPF(1, next);
    3606. 

  3606.  }
    3607. 

  3607.  OpCase(bs_test_unit8_fx): {
    3608. 

  3608.      Eterm* next;
    3609. 

  3609.      ErlBinMatchBuffer *_mb;
    3610. 

  3610.      PreFetch(2, next);
    3611. 

  3611.      _mb = ms_matchbuffer(xb(Arg(1)));
    3612. 

  3612.      if ((_mb->size - _mb->offset) & 7) {
    3613. 

  3613. 	 ClauseFail();
    3614. 

  3614.      }
    3615. 

  3615.      NextPF(2, next);
    3616. 

  3616.  }
    3617. 

  3617. 

  3618.  OpCase(i_bs_get_integer_8_rfd): {
    3619. 

  3619.      tmp_arg1 = r(0);
    3620. 

  3620.      goto do_bs_get_integer_8;
    3621. 

  3621.  }
    3622. 

  3622. 

  3623.  OpCase(i_bs_get_integer_8_xfd): {
    3624. 

  3624.      tmp_arg1 = xb(Arg(0));
    3625. 

  3625.      I++;
    3626. 

  3626.  }
    3627. 

  3627. 

  3628.  do_bs_get_integer_8: {
    3629. 

  3629.      ErlBinMatchBuffer *_mb;
    3630. 

  3630.      Eterm _result;
    3631. 

  3631.      _mb = ms_matchbuffer(tmp_arg1);
    3632. 

  3632.      if (_mb->size - _mb->offset < 8) {
    3633. 

  3633. 	 ClauseFail();
    3634. 

  3634.      }
    3635. 

  3635.      if (BIT_OFFSET(_mb->offset) != 0) {
    3636. 

  3636. 	 _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
    3637. 

  3637.      } else {
    3638. 

  3638. 	 _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
    3639. 

  3639. 	 _mb->offset += 8;
    3640. 

  3640.      }
    3641. 

  3641.      StoreBifResult(1, _result);
    3642. 

  3642.  }
    3643. 

  3643. 

  3644.  OpCase(i_bs_get_integer_16_rfd): {
    3645. 

  3645.      tmp_arg1 = r(0);
    3646. 

  3646.      goto do_bs_get_integer_16;
    3647. 

  3647.  }
    3648. 

  3648. 

  3649.  OpCase(i_bs_get_integer_16_xfd): {
    3650. 

  3650.      tmp_arg1 = xb(Arg(0));
    3651. 

  3651.      I++;
    3652. 

  3652.  }
    3653. 

  3653. 

  3654.  do_bs_get_integer_16: {
    3655. 

  3655.      ErlBinMatchBuffer *_mb;
    3656. 

  3656.      Eterm _result;
    3657. 

  3657.      _mb = ms_matchbuffer(tmp_arg1);
    3658. 

  3658.      if (_mb->size - _mb->offset < 16) {
    3659. 

  3659. 	 ClauseFail();
    3660. 

  3660.      }
    3661. 

  3661.      if (BIT_OFFSET(_mb->offset) != 0) {
    3662. 

  3662. 	 _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
    3663. 

  3663.      } else {
    3664. 

  3664. 	 _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
    3665. 

  3665. 	 _mb->offset += 16;
    3666. 

  3666.      }
    3667. 

  3667.      StoreBifResult(1, _result);
    3668. 

  3668.  }
    3669. 

  3669. 

  3670.  OpCase(i_bs_get_integer_32_rfId): {
    3671. 

  3671.      tmp_arg1 = r(0);
    3672. 

  3672.      goto do_bs_get_integer_32;
    3673. 

  3673.  }
    3674. 

  3674.      
    3675. 

  3675.  OpCase(i_bs_get_integer_32_xfId): {
    3676. 

  3676.      tmp_arg1 = xb(Arg(0));
    3677. 

  3677.      I++;
    3678. 

  3678.  }
    3679. 

  3679. 

  3680.  do_bs_get_integer_32: {
    3681. 

  3681.      ErlBinMatchBuffer *_mb;
    3682. 

  3682.      Uint32 _integer;
    3683. 

  3683.      Eterm _result;
    3684. 

  3684.      _mb = ms_matchbuffer(tmp_arg1);
    3685. 

  3685.      if (_mb->size - _mb->offset < 32) { ClauseFail(); }
    3686. 

  3686.      if (BIT_OFFSET(_mb->offset) != 0) {
    3687. 

  3687. 	 _integer = erts_bs_get_unaligned_uint32(c_p, _mb);
    3688. 

  3688.      } else {
    3689. 

  3689. 	 _integer = get_int32(_mb->base + _mb->offset/8);
    3690. 

  3690.      }
    3691. 

  3691.      _mb->offset += 32;
    3692. 

  3692. #ifndef ARCH_64
    3693. 

  3693.      if (IS_USMALL(0, _integer)) {
    3694. 

  3694. #endif
    3695. 

  3695. 	 _result = make_small(_integer);
    3696. 

  3696. #ifndef ARCH_64
    3697. 

  3697.      } else {
    3698. 

  3698. 	 TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
    3699. 

  3699. 	 _result = uint_to_big((Uint) _integer, HTOP);
    3700. 

  3700. 	 HTOP += BIG_UINT_HEAP_SIZE;
    3701. 

  3701.      }
    3702. 

  3702. #endif
    3703. 

  3703.      StoreBifResult(2, _result);
    3704. 

  3704.  }
    3705. 

  3705. 

  3706.  /* Operands: Size Live Fail Flags Dst */
    3707. 

  3707.  OpCase(i_bs_get_integer_imm_rIIfId): {
    3708. 

  3708.      tmp_arg1 = r(0);
    3709. 

  3709.      /* Operands: Size Live Fail Flags Dst */
    3710. 

  3710.      goto do_bs_get_integer_imm_test_heap;
    3711. 

  3711.  }
    3712. 

  3712. 

  3713.  /* Operands: x(Reg) Size Live Fail Flags Dst */
    3714. 

  3714.  OpCase(i_bs_get_integer_imm_xIIfId): {
    3715. 

  3715.      tmp_arg1 = xb(Arg(0));
    3716. 

  3716.      I++;
    3717. 

  3717.      /* Operands: Size Live Fail Flags Dst */
    3718. 

  3718.      goto do_bs_get_integer_imm_test_heap;
    3719. 

  3719.  }
    3720. 

  3720. 

  3721.  /*
    3722. 

  3722.   * tmp_arg1 = match context
    3723. 

  3723.   * Operands: Size Live Fail Flags Dst
    3724. 

  3724.   */
    3725. 

  3725.  do_bs_get_integer_imm_test_heap: {
    3726. 

  3726.      Uint wordsneeded;
    3727. 

  3727.      tmp_arg2 = Arg(0);
    3728. 

  3728.      wordsneeded = 1+WSIZE(NBYTES(tmp_arg2));
    3729. 

  3729.      TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
    3730. 

  3730.      I += 2;
    3731. 

  3731.      /* Operands: Fail Flags Dst */
    3732. 

  3732.      goto do_bs_get_integer_imm;
    3733. 

  3733.  }
    3734. 

  3734. 

  3735.  /* Operands: Size Fail Flags Dst */
    3736. 

  3736.  OpCase(i_bs_get_integer_small_imm_rIfId): {
    3737. 

  3737.      tmp_arg1 = r(0);
    3738. 

  3738.      tmp_arg2 = Arg(0);
    3739. 

  3739.      I++;
    3740. 

  3740.      /* Operands: Fail Flags Dst */
    3741. 

  3741.      goto do_bs_get_integer_imm;
    3742. 

  3742.  }
    3743. 

  3743. 

  3744.  /* Operands: x(Reg) Size Fail Flags Dst */
    3745. 

  3745.  OpCase(i_bs_get_integer_small_imm_xIfId): {
    3746. 

  3746.      tmp_arg1 = xb(Arg(0));
    3747. 

  3747.      tmp_arg2 = Arg(1);
    3748. 

  3748.      I += 2;
    3749. 

  3749.      /* Operands: Fail Flags Dst */
    3750. 

  3750.      goto do_bs_get_integer_imm;
    3751. 

  3751.  }
    3752. 

  3752.  
    3753. 

  3753.  /*
    3754. 

  3754.   * tmp_arg1 = match context
    3755. 

  3755.   * tmp_arg2 = size of field
    3756. 

  3756.   * Operands: Fail Flags Dst
    3757. 

  3757.   */
    3758. 

  3758.  do_bs_get_integer_imm: {
    3759. 

  3759.      ErlBinMatchBuffer* mb;
    3760. 

  3760.      Eterm result;
    3761. 

  3761. 

  3762.      mb = ms_matchbuffer(tmp_arg1);
    3763. 

  3763.      SWAPOUT;
    3764. 

  3764.      result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb);
    3765. 

  3765.      HTOP = HEAP_TOP(c_p);
    3766. 

  3766.      if (is_non_value(result)) {
    3767. 

  3767. 	 ClauseFail();
    3768. 

  3768.      }
    3769. 

  3769.      StoreBifResult(2, result);
    3770. 

  3770.  }
    3771. 

  3771. 

  3772.  /*
    3773. 

  3773.   * tmp_arg1 = Match context
    3774. 

  3774.   * tmp_arg2 = Size field
    3775. 

  3775.   * Operands: Fail Live FlagsAndUnit Dst
    3776. 

  3776.   */
    3777. 

  3777.  OpCase(i_bs_get_integer_fIId): {
    3778. 

  3778.      Uint flags;
    3779. 

  3779.      Uint size;
    3780. 

  3780.      ErlBinMatchBuffer* mb;
    3781. 

  3781.      Eterm result;
    3782. 

  3782. 

  3783.      flags = Arg(2);
    3784. 

  3784.      BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size);
    3785. 

  3785.      if (size >= SMALL_BITS) {
    3786. 

  3786. 	 Uint wordsneeded = 1+WSIZE(NBYTES((Uint) size));
    3787. 

  3787. 	 TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
    3788. 

  3788.      }
    3789. 

  3789.      mb = ms_matchbuffer(tmp_arg1);
    3790. 

  3790.      SWAPOUT;
    3791. 

  3791.      result = erts_bs_get_integer_2(c_p, size, flags, mb);
    3792. 

  3792.      HTOP = HEAP_TOP(c_p);
    3793. 

  3793.      if (is_non_value(result)) {
    3794. 

  3794. 	 ClauseFail();
    3795. 

  3795.      }
    3796. 

  3796.      StoreBifResult(3, result);
    3797. 

  3797.  }
    3798. 

  3798. 

  3799.  {
    3800. 

  3800.      ErlBinMatchBuffer* mb;
    3801. 

  3801.      ErlSubBin* sb;
    3802. 

  3802.      Uint size;
    3803. 

  3803.      Uint offs;
    3804. 

  3804.      Uint orig;
    3805. 

  3805.      Uint hole_size;
    3806. 

  3806. 

  3807.      OpCase(bs_context_to_binary_r): {
    3808. 

  3808. 	 tmp_arg1 = x0;
    3809. 

  3809. 	 I -= 2;
    3810. 

  3810. 	 goto do_context_to_binary;
    3811. 

  3811.      }
    3812. 

  3812. 

  3813.      /* Unfortunately, inlining can generate this instruction. */
    3814. 

  3814.      OpCase(bs_context_to_binary_y): {
    3815. 

  3815. 	 tmp_arg1 = yb(Arg(0));
    3816. 

  3816. 	 goto do_context_to_binary0;
    3817. 

  3817.      }
    3818. 

  3818. 

  3819.      OpCase(bs_context_to_binary_x): {
    3820. 

  3820. 	 tmp_arg1 = xb(Arg(0));
    3821. 

  3821.      
    3822. 

  3822.      do_context_to_binary0:
    3823. 

  3823. 	 I--;
    3824. 

  3824.      }
    3825. 

  3825. 

  3826.  do_context_to_binary:
    3827. 

  3827.      if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) {
    3828. 

  3828. 	 ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
    3829. 

  3829. 	 mb = &ms->mb;
    3830. 

  3830. 	 offs = ms->save_offset[0];
    3831. 

  3831. 	 size = mb->size - offs;
    3832. 

  3832. 	 goto do_bs_get_binary_all_reuse_common;
    3833. 

  3833.      }
    3834. 

  3834.      Next(2);
    3835. 

  3835. 

  3836.      OpCase(i_bs_get_binary_all_reuse_rfI): {
    3837. 

  3837. 	 tmp_arg1 = x0;
    3838. 

  3838. 	 goto do_bs_get_binary_all_reuse;
    3839. 

  3839.      }
    3840. 

  3840. 

  3841.      OpCase(i_bs_get_binary_all_reuse_xfI): {
    3842. 

  3842. 	 tmp_arg1 = xb(Arg(0));
    3843. 

  3843. 	 I++;
    3844. 

  3844.      }
    3845. 

  3845. 

  3846.  do_bs_get_binary_all_reuse:
    3847. 

  3847.      mb = ms_matchbuffer(tmp_arg1);
    3848. 

  3848.      size = mb->size - mb->offset;
    3849. 

  3849.      if (size % Arg(1) != 0) {
    3850. 

  3850. 	 ClauseFail();
    3851. 

  3851.      }
    3852. 

  3852.      offs = mb->offset;
    3853. 

  3853. 

  3854.  do_bs_get_binary_all_reuse_common:
    3855. 

  3855.      orig = mb->orig;
    3856. 

  3856.      sb = (ErlSubBin *) boxed_val(tmp_arg1);
    3857. 

  3857.      hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
    3858. 

  3858.      sb->thing_word = HEADER_SUB_BIN;
    3859. 

  3859.      sb->size = BYTE_OFFSET(size);
    3860. 

  3860.      sb->bitsize = BIT_OFFSET(size);
    3861. 

  3861.      sb->offs = BYTE_OFFSET(offs);
    3862. 

  3862.      sb->bitoffs = BIT_OFFSET(offs);
    3863. 

  3863.      sb->is_writable = 0;
    3864. 

  3864.      sb->orig = orig;
    3865. 

  3865.      if (hole_size) {
    3866. 

  3866. 	 sb[1].thing_word = make_pos_bignum_header(hole_size-1);
    3867. 

  3867.      }
    3868. 

  3868.      Next(2);
    3869. 

  3869.  }
    3870. 

  3870. 

  3871.  {
    3872. 

  3872.      OpCase(i_bs_match_string_rfII): {
    3873. 

  3873. 	 tmp_arg1 = r(0);
    3874. 

  3874. 	 goto do_bs_match_string;
    3875. 

  3875.      }
    3876. 

  3876.      OpCase(i_bs_match_string_xfII): {
    3877. 

  3877. 	 tmp_arg1 = xb(Arg(0));
    3878. 

  3878. 	 I++;
    3879. 

  3879.      }
    3880. 

  3880. 

  3881.  do_bs_match_string:
    3882. 

  3882.      {
    3883. 

  3883. 	 Eterm* next;
    3884. 

  3884. 	 byte* bytes;
    3885. 

  3885. 	 Uint bits;
    3886. 

  3886. 	 ErlBinMatchBuffer* mb;
    3887. 

  3887. 	 Uint offs;
    3888. 

  3888. 

  3889. 	 PreFetch(3, next);
    3890. 

  3890. 	 bits = Arg(1);
    3891. 

  3891. 	 bytes = (byte *) Arg(2);
    3892. 

  3892. 	 mb = ms_matchbuffer(tmp_arg1);
    3893. 

  3893. 	 if (mb->size - mb->offset < bits) {
    3894. 

  3894. 	     ClauseFail();
    3895. 

  3895. 	 }
    3896. 

  3896. 	 offs = mb->offset & 7;
    3897. 

  3897. 	 if (offs == 0 && (bits & 7) == 0) {
    3898. 

  3898. 	     if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) {
    3899. 

  3899. 		 ClauseFail();
    3900. 

  3900. 	     }
    3901. 

  3901. 	 } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) {
    3902. 

  3902. 	     ClauseFail();
    3903. 

  3903. 	 }
    3904. 

  3904. 	 mb->offset += bits;
    3905. 

  3905. 	 NextPF(3, next);
    3906. 

  3906.      }
    3907. 

  3907.  }
    3908. 

  3908. 

  3909.  OpCase(i_bs_save2_rI): {
    3910. 

  3910.      Eterm* next;
    3911. 

  3911.      ErlBinMatchState *_ms;
    3912. 

  3912.      PreFetch(1, next);
    3913. 

  3913.      _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
    3914. 

  3914.      _ms->save_offset[Arg(0)] = _ms->mb.offset;
    3915. 

  3915.      NextPF(1, next);
    3916. 

  3916.  }
    3917. 

  3917.  OpCase(i_bs_save2_xI): {
    3918. 

  3918.      Eterm* next;
    3919. 

  3919.      ErlBinMatchState *_ms;
    3920. 

  3920.      PreFetch(2, next);
    3921. 

  3921.      _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
    3922. 

  3922.      _ms->save_offset[Arg(1)] = _ms->mb.offset;
    3923. 

  3923.      NextPF(2, next);
    3924. 

  3924.  }
    3925. 

  3925. 

  3926.  OpCase(i_bs_restore2_rI): {
    3927. 

  3927.      Eterm* next;
    3928. 

  3928.      ErlBinMatchState *_ms;
    3929. 

  3929.      PreFetch(1, next);
    3930. 

  3930.      _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
    3931. 

  3931.      _ms->mb.offset = _ms->save_offset[Arg(0)];
    3932. 

  3932.      NextPF(1, next);
    3933. 

  3933.  }
    3934. 

  3934.  OpCase(i_bs_restore2_xI): {
    3935. 

  3935.      Eterm* next;
    3936. 

  3936.      ErlBinMatchState *_ms;
    3937. 

  3937.      PreFetch(2, next);
    3938. 

  3938.      _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
    3939. 

  3939.      _ms->mb.offset = _ms->save_offset[Arg(1)];
    3940. 

  3940.      NextPF(2, next);
    3941. 

  3941.  }
    3942. 

  3942. 

  3943. #include "beam_cold.h"
    3944. 

  3944. 

  3945. 

  3946.  /*
    3947. 

  3947.   * This instruction is probably never used (because it is combined with a
    3948. 

  3948.   * a return). However, a future compiler might for some reason emit a
    3949. 

  3949.   * deallocate not followed by a return, and that should work.
    3950. 

  3950.   */
    3951. 

  3951.  OpCase(deallocate_I): {
    3952. 

  3952.      Eterm* next;
    3953. 

  3953. 

  3954.      PreFetch(1, next);
    3955. 

  3955.      D(Arg(0));
    3956. 

  3956.      NextPF(1, next);
    3957. 

  3957.  }
    3958. 

  3958. 

  3959.     /*
    3960. 

  3960.      * Trace and debugging support.
    3961. 

  3961.      */
    3962. 

  3962. 

  3963.     /*
    3964. 

  3964.      * At this point, I points to the code[3] in the export entry for
    3965. 

  3965.      * a trace-enabled function.
    3966. 

  3966.      *
    3967. 

  3967.      * code[0]: Module
    3968. 

  3968.      * code[1]: Function
    3969. 

  3969.      * code[2]: Arity
    3970. 

  3970.      * code[3]: &&call_traced_function
    3971. 

  3971.      * code[4]: Address of function.
    3972. 

  3972.      */
    3973. 

  3973.  OpCase(call_traced_function): {
    3974. 

  3974.      if (IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
    3975. 

  3975. 	 unsigned offset = offsetof(Export, code) + 3*sizeof(Eterm);
    3976. 

  3976. 	 Export* ep = (Export *) (((char *)I)-offset);
    3977. 

  3977. 	 Uint32 flags;
    3978. 

  3978. 

  3979. 	 SWAPOUT;
    3980. 

  3980. 	 reg[0] = r(0);
    3981. 

  3981. 	 PROCESS_MAIN_CHK_LOCKS(c_p);
    3982. 

  3982. 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    3983. 

  3983. 	 flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg,
    3984. 

  3984. 				 0, &c_p->tracer_proc);
    3985. 

  3985. 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    3986. 

  3986. 	 PROCESS_MAIN_CHK_LOCKS(c_p);
    3987. 

  3987. 	 ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    3988. 

  3988. 	 SWAPIN;
    3989. 

  3989. 	 
    3990. 

  3990. 	 if (flags & MATCH_SET_RX_TRACE) {
    3991. 

  3991. 	     ASSERT(c_p->htop <= E && E <= c_p->hend);
    3992. 

  3992. 	     if (E - 3 < HTOP) {
    3993. 

  3993. 		 /* SWAPOUT, SWAPIN was done and r(0) was saved above */
    3994. 

  3994. 		 PROCESS_MAIN_CHK_LOCKS(c_p);
    3995. 

  3995. 		 FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]);
    3996. 

  3996. 		 PROCESS_MAIN_CHK_LOCKS(c_p);
    3997. 

  3997. 		 r(0) = reg[0];
    3998. 

  3998. 		 SWAPIN;
    3999. 

  3999. 	     }
    4000. 

  4000. 	     E -= 3;
    4001. 

  4001. 	     ASSERT(c_p->htop <= E && E <= c_p->hend);
    4002. 

  4002. 	     ASSERT(is_CP((Eterm)(ep->code)));
    4003. 

  4003. 	     ASSERT(is_internal_pid(c_p->tracer_proc) || 
    4004. 

  4004. 		    is_internal_port(c_p->tracer_proc));
    4005. 

  4005. 	     E[2] = make_cp(c_p->cp);
    4006. 

  4006. 	     E[1] = am_true; /* Process tracer */
    4007. 

  4007. 	     E[0] = make_cp(ep->code);
    4008. 

  4008. 	     c_p->cp = (Eterm*)
    4009. 

  4009. 		 make_cp(flags & MATCH_SET_EXCEPTION_TRACE
    4010. 

  4010. 			 ? beam_exception_trace : beam_return_trace);
    4011. 

  4011. 	     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
    4012. 

  4012. 	     c_p->trace_flags |= F_EXCEPTION_TRACE;
    4013. 

  4013. 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
    4014. 

  4014. 	 }
    4015. 

  4015.      }
    4016. 

  4016.      SET_I((Uint *) Arg(0));
    4017. 

  4017.      Dispatch();
    4018. 

  4018.  }
    4019. 

  4019. 

  4020.  OpCase(return_trace): {
    4021. 

  4021.      Uint* code = (Uint *) E[0];
    4022. 

  4022.      
    4023. 

  4023.      SWAPOUT;		/* Needed for shared heap */
    4024. 

  4024.      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    4025. 

  4025.      erts_trace_return(c_p, code, r(0), E+1/*Process tracer*/);
    4026. 

  4026.      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    4027. 

  4027.      SWAPIN;
    4028. 

  4028.      c_p->cp = NULL;
    4029. 

  4029.      SET_I((Eterm *) E[2]);
    4030. 

  4030.      E += 3;
    4031. 

  4031.      Goto(*I);
    4032. 

  4032.  }
    4033. 

  4033. 

  4034.  OpCase(i_count_breakpoint): {
    4035. 

  4035.      Uint real_I;
    4036. 

  4036.      
    4037. 

  4037.      ErtsCountBreak((Uint *) I, &real_I);
    4038. 

  4038.      ASSERT(VALID_INSTR(real_I));
    4039. 

  4039.      Goto(real_I);
    4040. 

  4040.  }
    4041. 

  4041. 

  4042.  OpCase(i_trace_breakpoint):
    4043. 

  4043.      if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
    4044. 

  4044. 	 Uint real_I;
    4045. 

  4045. 	 
    4046. 

  4046. 	 ErtsBreakSkip((Uint *) I, &real_I);
    4047. 

  4047. 	 Goto(real_I);
    4048. 

  4048.      }
    4049. 

  4049.  /* Fall through to next case */
    4050. 

  4050.  OpCase(i_mtrace_breakpoint): {
    4051. 

  4051.      Uint real_I;
    4052. 

  4052.      Uint32 flags;
    4053. 

  4053.      Eterm tracer_pid;
    4054. 

  4054.      Uint *cpp;
    4055. 

  4055.      int return_to_trace = 0, need = 0;
    4056. 

  4056.      flags = 0;
    4057. 

  4057.      SWAPOUT;
    4058. 

  4058.      reg[0] = r(0);
    4059. 

  4059. 

  4060.      if (*cp_val((Eterm)c_p->cp) 
    4061. 

  4061. 	 == (Uint) OpCode(return_trace)) {
    4062. 

  4062. 	 cpp = (Uint*)&E[2];
    4063. 

  4063.      } else if (*cp_val((Eterm)c_p->cp) 
    4064. 

  4064. 		== (Uint) OpCode(i_return_to_trace)) {
    4065. 

  4065. 	 return_to_trace = !0;
    4066. 

  4066. 	 cpp = (Uint*)&E[0];
    4067. 

  4067.      } else {
    4068. 

  4068. 	 cpp = NULL;
    4069. 

  4069.      }
    4070. 

  4070.      if (cpp) {
    4071. 

  4071. 	 /* This _IS_ a tail recursive call, if there are
    4072. 

  4072. 	  * return_trace and/or i_return_to_trace stackframes
    4073. 

  4073. 	  * on the stack, they are not intermixed with y registers
    4074. 

  4074. 	  */
    4075. 

  4075. 	 Eterm *cp_save = c_p->cp;
    4076. 

  4076. 	 for (;;) {
    4077. 

  4077. 	     ASSERT(is_CP(*cpp));
    4078. 

  4078. 	     if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) {
    4079. 

  4079. 		 cpp += 3;
    4080. 

  4080. 	     } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) {
    4081. 

  4081. 		 return_to_trace = !0;
    4082. 

  4082. 		 cpp += 1;
    4083. 

  4083. 	     } else
    4084. 

  4084. 		 break;
    4085. 

  4085. 	 }
    4086. 

  4086. 	 c_p->cp = (Eterm *) *cpp;
    4087. 

  4087. 	 ASSERT(is_CP((Eterm)c_p->cp));
    4088. 

  4088. 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    4089. 

  4089. 	 real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid);
    4090. 

  4090. 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    4091. 

  4091. 	 SWAPIN;		/* Needed by shared heap. */
    4092. 

  4092. 	 c_p->cp = cp_save;
    4093. 

  4093.      } else {
    4094. 

  4094. 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    4095. 

  4095. 	 real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid);
    4096. 

  4096. 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    4097. 

  4097. 	 SWAPIN;		/* Needed by shared heap. */
    4098. 

  4098.      }
    4099. 

  4099. 

  4100.      ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    4101. 

  4101. 

  4102.      if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) {
    4103. 

  4103. 	 need += 1;
    4104. 

  4104.      }
    4105. 

  4105.      if (flags & MATCH_SET_RX_TRACE) {
    4106. 

  4106. 	 need += 3;
    4107. 

  4107.      }
    4108. 

  4108.      if (need) {
    4109. 

  4109. 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
    4110. 

  4110. 	 if (E - need < HTOP) {
    4111. 

  4111. 	     /* SWAPOUT was done and r(0) was saved above */
    4112. 

  4112. 	     PROCESS_MAIN_CHK_LOCKS(c_p);
    4113. 

  4113. 	     FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]);
    4114. 

  4114. 	     PROCESS_MAIN_CHK_LOCKS(c_p);
    4115. 

  4115. 	     r(0) = reg[0];
    4116. 

  4116. 	     SWAPIN;
    4117. 

  4117. 	 }
    4118. 

  4118.      }
    4119. 

  4119.      if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) {
    4120. 

  4120. 	 E -= 1;
    4121. 

  4121. 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
    4122. 

  4122. 	 E[0] = make_cp(c_p->cp);
    4123. 

  4123. 	 c_p->cp = (Eterm *) make_cp(beam_return_to_trace);
    4124. 

  4124.      }
    4125. 

  4125.      if (flags & MATCH_SET_RX_TRACE) {
    4126. 

  4126. 	 E -= 3;
    4127. 

  4127. 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
    4128. 

  4128. 	 ASSERT(is_CP((Eterm) (I - 3)));
    4129. 

  4129. 	 ASSERT(am_true == tracer_pid || 
    4130. 

  4130. 		is_internal_pid(tracer_pid) || is_internal_port(tracer_pid));
    4131. 

  4131. 	 E[2] = make_cp(c_p->cp);
    4132. 

  4132. 	 E[1] = tracer_pid;
    4133. 

  4133. 	 E[0] = make_cp(I - 3); /* We ARE at the beginning of an 
    4134. 

  4134. 				   instruction,
    4135. 

  4135. 				   the funcinfo is above i. */
    4136. 

  4136. 	 c_p->cp = (Eterm*)
    4137. 

  4137. 	     make_cp(flags & MATCH_SET_EXCEPTION_TRACE
    4138. 

  4138. 		     ? beam_exception_trace : beam_return_trace);
    4139. 

  4139. 	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
    4140. 

  4140. 	 c_p->trace_flags |= F_EXCEPTION_TRACE;
    4141. 

  4141. 	 erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
    4142. 

  4142.      }
    4143. 

  4143.      Goto(real_I);
    4144. 

  4144.  }
    4145. 

  4145.  
    4146. 

  4146.  OpCase(i_return_to_trace): {
    4147. 

  4147.      if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) {
    4148. 

  4148. 	 Uint *cpp = (Uint*) E;
    4149. 

  4149. 	 for(;;) {
    4150. 

  4150. 	     ASSERT(is_CP(*cpp));
    4151. 

  4151. 	     if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) {
    4152. 

  4152. 		 do ++cpp; while(is_not_CP(*cpp));
    4153. 

  4153. 		 cpp += 2;
    4154. 

  4154. 	     } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) {
    4155. 

  4155. 		 do ++cpp; while(is_not_CP(*cpp));
    4156. 

  4156. 	     } else break;
    4157. 

  4157. 	 }
    4158. 

  4158. 	 SWAPOUT;		/* Needed for shared heap */
    4159. 

  4159. 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    4160. 

  4160. 	 erts_trace_return_to(c_p, cp_val(*cpp));
    4161. 

  4161. 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    4162. 

  4162. 	 SWAPIN;
    4163. 

  4163.      }
    4164. 

  4164.      c_p->cp = NULL;
    4165. 

  4165.      SET_I((Eterm *) E[0]);
    4166. 

  4166.      E += 1;
    4167. 

  4167.      Goto(*I);
    4168. 

  4168.  }
    4169. 

  4169. 

  4170.  /*
    4171. 

  4171.   * Instructions for module_info/0,1.
    4172. 

  4172.   */
    4173. 

  4173. 

  4174.  OpCase(i_module_info_0): {
    4175. 

  4175.      SWAPOUT;
    4176. 

  4176.      r(0) = erts_module_info_0(c_p, I[-3]);
    4177. 

  4177.      HTOP = HEAP_TOP(c_p);
    4178. 

  4178.      SET_I(c_p->cp);
    4179. 

  4179.      Goto(*I);
    4180. 

  4180.  }
    4181. 

  4181. 

  4182.  OpCase(i_module_info_1): {
    4183. 

  4183.      Eterm res;
    4184. 

  4184. 

  4185.      SWAPOUT;
    4186. 

  4186.      res = erts_module_info_1(c_p, I[-3], r(0));
    4187. 

  4187.      HTOP = HEAP_TOP(c_p);
    4188. 

  4188.      if (is_value(res)) {
    4189. 

  4189. 	 r(0) = res;
    4190. 

  4190. 	 SET_I(c_p->cp);
    4191. 

  4191. 	 Goto(*I);
    4192. 

  4192.      }
    4193. 

  4193.      c_p->freason = EXC_FUNCTION_CLAUSE;
    4194. 

  4194.      c_p->current = I-3;
    4195. 

  4195.      goto lb_error_action_code;
    4196. 

  4196.  }
    4197. 

  4197. 

  4198.  /*
    4199. 

  4199.   * Instructions for allocating on the message area.
    4200. 

  4200.   */
    4201. 

  4201. 

  4202.  OpCase(i_global_cons):
    4203. 

  4203.  {
    4204. 

  4204.      Eterm *next;
    4205. 

  4205. #ifdef HYBRID
    4206. 

  4206.      Eterm *hp;
    4207. 

  4207. 

  4208.      PreFetch(0,next);
    4209. 

  4209.      TestGlobalHeap(2,2,hp);
    4210. 

  4210.      hp[0] = r(0);
    4211. 

  4211.      hp[1] = x(1);
    4212. 

  4212.      r(0) = make_list(hp);
    4213. 

  4213. #ifndef INCREMENTAL
    4214. 

  4214.      global_htop += 2;
    4215. 

  4215. #endif
    4216. 

  4216.      NextPF(0,next);
    4217. 

  4217. #else
    4218. 

  4218.      PreFetch(0,next);
    4219. 

  4219.      c_p->freason = EXC_INTERNAL_ERROR;
    4220. 

  4220.      goto find_func_info;
    4221. 

  4221. #endif
    4222. 

  4222.  }
    4223. 

  4223. 

  4224.  OpCase(i_global_tuple):
    4225. 

  4225.  {
    4226. 

  4226.      Eterm *next;
    4227. 

  4227.      int len;
    4228. 

  4228. #ifdef HYBRID
    4229. 

  4229.      Eterm list;
    4230. 

  4230.      Eterm *hp;
    4231. 

  4231. #endif
    4232. 

  4232. 

  4233.      if ((len = list_length(r(0))) < 0) {
    4234. 

  4234.          goto badarg;
    4235. 

  4235.      }
    4236. 

  4236. 

  4237.      PreFetch(0,next);
    4238. 

  4238. #ifdef HYBRID
    4239. 

  4239.      TestGlobalHeap(len + 1,1,hp);
    4240. 

  4240.      list = r(0);
    4241. 

  4241.      r(0) = make_tuple(hp);
    4242. 

  4242.      *hp++ = make_arityval(len);
    4243. 

  4243.      while(is_list(list))
    4244. 

  4244.      {
    4245. 

  4245.          Eterm* cons = list_val(list);
    4246. 

  4246.          *hp++ = CAR(cons);
    4247. 

  4247.          list = CDR(cons);
    4248. 

  4248.      }
    4249. 

  4249. #ifndef INCREMENTAL
    4250. 

  4250.      global_htop += len + 1;
    4251. 

  4251. #endif
    4252. 

  4252.      NextPF(0,next);
    4253. 

  4253. #else
    4254. 

  4254.      c_p->freason = EXC_INTERNAL_ERROR;
    4255. 

  4255.      goto find_func_info;
    4256. 

  4256. #endif
    4257. 

  4257.  }
    4258. 

  4258. 

  4259.  OpCase(i_global_copy):
    4260. 

  4260.  {
    4261. 

  4261.      Eterm *next;
    4262. 

  4262.      PreFetch(0,next);
    4263. 

  4263. #ifdef HYBRID
    4264. 

  4264.      if (!IS_CONST(r(0)))
    4265. 

  4265.      {
    4266. 

  4266.          BM_SWAP_TIMER(system,copy);
    4267. 

  4267.          SWAPOUT;
    4268. 

  4268.          reg[0] = r(0);
    4269. 

  4269.          reg[1] = NIL;
    4270. 

  4270.          r(0) = copy_struct_lazy(c_p,r(0),0);
    4271. 

  4271.          ASSERT(ma_src_top == 0);
    4272. 

  4272.          ASSERT(ma_dst_top == 0);
    4273. 

  4273.          ASSERT(ma_offset_top == 0);
    4274. 

  4274.          SWAPIN;
    4275. 

  4275.          BM_SWAP_TIMER(copy,system);
    4276. 

  4276.      }
    4277. 

  4277.      NextPF(0,next);
    4278. 

  4278. #else
    4279. 

  4279.      c_p->freason = EXC_INTERNAL_ERROR;
    4280. 

  4280.      goto find_func_info;
    4281. 

  4281. #endif
    4282. 

  4282.  }
    4283. 

  4283. 

  4284.  /*
    4285. 

  4285.   * New floating point instructions.
    4286. 

  4286.   */
    4287. 

  4287. 

  4288.  OpCase(fmove_ql): {
    4289. 

  4289.      Eterm fr = Arg(1);
    4290. 

  4290.      Eterm* next;
    4291. 

  4291. 

  4292.      PreFetch(2, next);
    4293. 

  4293.      GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
    4294. 

  4294.      NextPF(2, next);
    4295. 

  4295.  }
    4296. 

  4296. 

  4297.  OpCase(fmove_dl): {
    4298. 

  4298.      Eterm targ1;
    4299. 

  4299.      Eterm fr = Arg(1);
    4300. 

  4300.      Eterm* next;
    4301. 

  4301. 

  4302.      PreFetch(2, next);
    4303. 

  4303.      GetR(0, targ1);
    4304. 

  4304.      /* Arg(0) == HEADER_FLONUM */
    4305. 

  4305.      GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
    4306. 

  4306.      NextPF(2, next);
    4307. 

  4307.  }
    4308. 

  4308. 

  4309.  OpCase(fmove_new_ld): {
    4310. 

  4310.      Eterm fr = Arg(0);
    4311. 

  4311.      Eterm dest = make_float(HTOP);
    4312. 

  4312. 

  4313.      PUT_DOUBLE(*(FloatDef*)ADD_BYTE_OFFSET(freg, fr), HTOP);
    4314. 

  4314.      HTOP += FLOAT_SIZE_OBJECT;
    4315. 

  4315.      StoreBifResult(1, dest);
    4316. 

  4316.  }
    4317. 

  4317. 

  4318.  OpCase(fconv_dl): {
    4319. 

  4319.      Eterm targ1;
    4320. 

  4320.      Eterm fr = Arg(1);
    4321. 

  4321.      Eterm* next;
    4322. 

  4322. 

  4323.      GetR(0, targ1);
    4324. 

  4324.      PreFetch(2, next);
    4325. 

  4325.      if (is_small(targ1)) {
    4326. 

  4326. 	 fb(fr) = (double) signed_val(targ1);
    4327. 

  4327.      } else if (is_big(targ1)) {
    4328. 

  4328. 	 if (big_to_double(targ1, &fb(fr)) < 0) {
    4329. 

  4329. 	     goto fbadarith;
    4330. 

  4330. 	 }
    4331. 

  4331.      } else if (is_float(targ1)) {
    4332. 

  4332. 	 GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
    4333. 

  4333.      } else {
    4334. 

  4334. 	 goto fbadarith;
    4335. 

  4335.      }
    4336. 

  4336.      NextPF(2, next);
    4337. 

  4337.  }
    4338. 

  4338. 

  4339.  /*
    4340. 

  4340.   * Old allocating fmove.
    4341. 

  4341.   */
    4342. 

  4342. 

  4343. 

  4344. #ifdef NO_FPE_SIGNALS
    4345. 

  4345.      OpCase(fclearerror):
    4346. 

  4346.      OpCase(i_fcheckerror):
    4347. 

  4347. 	 erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
    4348. 

  4348. #else
    4349. 

  4349.      OpCase(fclearerror): {
    4350. 

  4350. 	 Eterm* next;
    4351. 

  4351. 

  4352. 	 PreFetch(0, next);
    4353. 

  4353. 	 ERTS_FP_CHECK_INIT(c_p);
    4354. 

  4354. 	 NextPF(0, next);
    4355. 

  4355.      }
    4356. 

  4356. 

  4357.      OpCase(i_fcheckerror): {
    4358. 

  4358. 	 Eterm* next;
    4359. 

  4359. 

  4360. 	 PreFetch(0, next);
    4361. 

  4361. 	 ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith);
    4362. 

  4362. 	 NextPF(0, next);
    4363. 

  4363.      }
    4364. 

  4364. #  undef ERTS_FP_CHECK_INIT
    4365. 

  4365. #  undef ERTS_FP_ERROR
    4366. 

  4366. #  define ERTS_FP_CHECK_INIT(p)
    4367. 

  4367. #  define ERTS_FP_ERROR(p, a, b)
    4368. 

  4368. #endif
    4369. 

  4369. 

  4370. 

  4371.  OpCase(i_fadd_lll): {
    4372. 

  4372.      Eterm* next;
    4373. 

  4373. 

  4374.      PreFetch(3, next);
    4375. 

  4375.      ERTS_FP_CHECK_INIT(c_p);
    4376. 

  4376.      fb(Arg(2)) = fb(Arg(0)) + fb(Arg(1));
    4377. 

  4377.      ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
    4378. 

  4378.      NextPF(3, next);
    4379. 

  4379.  }
    4380. 

  4380.  OpCase(i_fsub_lll): {
    4381. 

  4381.      Eterm* next;
    4382. 

  4382. 

  4383.      PreFetch(3, next);
    4384. 

  4384.      ERTS_FP_CHECK_INIT(c_p);
    4385. 

  4385.      fb(Arg(2)) = fb(Arg(0)) - fb(Arg(1));
    4386. 

  4386.      ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
    4387. 

  4387.      NextPF(3, next);
    4388. 

  4388.  }
    4389. 

  4389.  OpCase(i_fmul_lll): {
    4390. 

  4390.      Eterm* next;
    4391. 

  4391. 

  4392.      PreFetch(3, next);
    4393. 

  4393.      ERTS_FP_CHECK_INIT(c_p);
    4394. 

  4394.      fb(Arg(2)) = fb(Arg(0)) * fb(Arg(1));
    4395. 

  4395.      ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
    4396. 

  4396.      NextPF(3, next);
    4397. 

  4397.  }
    4398. 

  4398.  OpCase(i_fdiv_lll): {
    4399. 

  4399.      Eterm* next;
    4400. 

  4400. 

  4401.      PreFetch(3, next);
    4402. 

  4402.      ERTS_FP_CHECK_INIT(c_p);
    4403. 

  4403.      fb(Arg(2)) = fb(Arg(0)) / fb(Arg(1));
    4404. 

  4404.      ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
    4405. 

  4405.      NextPF(3, next);
    4406. 

  4406.  }
    4407. 

  4407.  OpCase(i_fnegate_ll): {
    4408. 

  4408.      Eterm* next;
    4409. 

  4409. 

  4410.      PreFetch(2, next);
    4411. 

  4411.      ERTS_FP_CHECK_INIT(c_p);
    4412. 

  4412.      fb(Arg(1)) = -fb(Arg(0));
    4413. 

  4413.      ERTS_FP_ERROR(c_p, fb(Arg(1)), goto fbadarith);
    4414. 

  4414.      NextPF(2, next);
    4415. 

  4415. 

  4416.  fbadarith:
    4417. 

  4417.      c_p->freason = BADARITH;
    4418. 

  4418.      goto find_func_info;
    4419. 

  4419.  }
    4420. 

  4420. 

  4421. #ifdef HIPE
    4422. 

  4422.  {
    4423. 

  4423.      unsigned cmd;
    4424. 

  4424. 

  4425.      OpCase(hipe_trap_call): {
    4426. 

  4426. 	 /*
    4427. 

  4427. 	  * I[-5]: &&lb_i_func_info_IaaI
    4428. 

  4428. 	  * I[-4]: Native code callee (inserted by HiPE)
    4429. 

  4429. 	  * I[-3]: Module (tagged atom)
    4430. 

  4430. 	  * I[-2]: Function (tagged atom)
    4431. 

  4431. 	  * I[-1]: Arity (untagged integer)
    4432. 

  4432. 	  * I[ 0]: &&lb_hipe_trap_call
    4433. 

  4433. 	  * ... remainder of original BEAM code
    4434. 

  4434. 	  */
    4435. 

  4435. 	 ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
    4436. 

  4436. 	 c_p->hipe.ncallee = (void(*)(void)) I[-4];
    4437. 

  4437. 	 cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8);
    4438. 

  4438. 	 ++hipe_trap_count;
    4439. 

  4439. 	 goto L_hipe_mode_switch;
    4440. 

  4440.      }
    4441. 

  4441.      OpCase(hipe_trap_call_closure): {
    4442. 

  4442.        ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
    4443. 

  4443.        c_p->hipe.ncallee = (void(*)(void)) I[-4];
    4444. 

  4444.        cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8);
    4445. 

  4445.        ++hipe_trap_count;
    4446. 

  4446.        goto L_hipe_mode_switch;
    4447. 

  4447.      }
    4448. 

  4448.      OpCase(hipe_trap_return): {
    4449. 

  4449. 	 cmd = HIPE_MODE_SWITCH_CMD_RETURN;
    4450. 

  4450. 	 goto L_hipe_mode_switch;
    4451. 

  4451.      }
    4452. 

  4452.      OpCase(hipe_trap_throw): {
    4453. 

  4453. 	 cmd = HIPE_MODE_SWITCH_CMD_THROW;
    4454. 

  4454. 	 goto L_hipe_mode_switch;
    4455. 

  4455.      }
    4456. 

  4456.      OpCase(hipe_trap_resume): {
    4457. 

  4457. 	 cmd = HIPE_MODE_SWITCH_CMD_RESUME;
    4458. 

  4458. 	 goto L_hipe_mode_switch;
    4459. 

  4459.      }
    4460. 

  4460.      OpCase(hipe_trap_reschedule): {
    4461. 

  4461. 	 cmd = HIPE_MODE_SWITCH_CMD_RESCHEDULE;
    4462. 

  4462. 	 goto L_hipe_mode_switch;
    4463. 

  4463.      }
    4464. 

  4464.  L_hipe_mode_switch:
    4465. 

  4465.      /* XXX: this abuse of def_arg_reg[] is horrid! */
    4466. 

  4466.      SWAPOUT;
    4467. 

  4467.      c_p->fcalls = FCALLS;
    4468. 

  4468.      c_p->def_arg_reg[4] = -neg_o_reds;
    4469. 

  4469.      reg[0] = r(0);
    4470. 

  4470.      c_p = hipe_mode_switch(c_p, cmd, reg);
    4471. 

  4471. #ifdef ERTS_SMP
    4472. 

  4472.      reg = c_p->scheduler_data->save_reg;
    4473. 

  4473.      freg = c_p->scheduler_data->freg;
    4474. 

  4474. #endif
    4475. 

  4475.      ERL_BITS_RELOAD_STATEP(c_p);
    4476. 

  4476.      neg_o_reds = -c_p->def_arg_reg[4];
    4477. 

  4477.      FCALLS = c_p->fcalls;
    4478. 

  4478.      SWAPIN;
    4479. 

  4479.      switch( c_p->def_arg_reg[3] ) {
    4480. 

  4480.        case HIPE_MODE_SWITCH_RES_RETURN:
    4481. 

  4481. 	 ASSERT(is_value(reg[0]));
    4482. 

  4482. 	 MoveReturn(reg[0], r(0));
    4483. 

  4483.        case HIPE_MODE_SWITCH_RES_CALL:
    4484. 

  4484. 	 SET_I(c_p->i);
    4485. 

  4485. 	 r(0) = reg[0];
    4486. 

  4486. 	 Dispatch();
    4487. 

  4487.        case HIPE_MODE_SWITCH_RES_CALL_CLOSURE:
    4488. 

  4488. 	 /* This can be used to call any function value, but currently it's
    4489. 

  4489. 	    only used to call closures referring to unloaded modules. */
    4490. 

  4490. 	 {
    4491. 

  4491. 	     Eterm *next;
    4492. 

  4492. 

  4493. 	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
    4494. 

  4494. 	     SWAPIN;
    4495. 

  4495. 	     if (next != NULL) {
    4496. 

  4496. 		 r(0) = reg[0];
    4497. 

  4497. 		 SET_I(next);
    4498. 

  4498. 		 Dispatchfun();
    4499. 

  4499. 	     }
    4500. 

  4500. 	     goto find_func_info;
    4501. 

  4501. 	 }
    4502. 

  4502.        case HIPE_MODE_SWITCH_RES_THROW:
    4503. 

  4503. 	 c_p->cp = NULL;
    4504. 

  4504. 	 I = handle_error(c_p, I, reg, NULL);
    4505. 

  4505. 	 goto post_error_handling;
    4506. 

  4506.        default:
    4507. 

  4507. 	 erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
    4508. 

  4508.      }
    4509. 

  4509.  }
    4510. 

  4510.  OpCase(hipe_call_count): {
    4511. 

  4511.      /*
    4512. 

  4512.       * I[-5]: &&lb_i_func_info_IaaI
    4513. 

  4513.       * I[-4]: pointer to struct hipe_call_count (inserted by HiPE)
    4514. 

  4514.       * I[-3]: Module (tagged atom)
    4515. 

  4515.       * I[-2]: Function (tagged atom)
    4516. 

  4516.       * I[-1]: Arity (untagged integer)
    4517. 

  4517.       * I[ 0]: &&lb_hipe_call_count
    4518. 

  4518.       * ... remainder of original BEAM code
    4519. 

  4519.       */
    4520. 

  4520.      struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4];
    4521. 

  4521.      ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
    4522. 

  4522.      ASSERT(hcc != NULL);
    4523. 

  4523.      ASSERT(VALID_INSTR(hcc->opcode));
    4524. 

  4524.      ++(hcc->count);
    4525. 

  4525.      Goto(hcc->opcode);
    4526. 

  4526.  }
    4527. 

  4527. #endif /* HIPE */
    4528. 

  4528. 

  4529.  OpCase(i_yield):
    4530. 

  4530.  {
    4531. 

  4531.      /* This is safe as long as REDS_IN(c_p) is never stored 
    4532. 

  4532.       * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5],
    4533. 

  4533.       * which may be c_p->arg_reg[5], which is close, but no banana.
    4534. 

  4534.       */
    4535. 

  4535.      c_p->arg_reg[0] = am_true;
    4536. 

  4536.      c_p->arity = 1; /* One living register (the 'true' return value) */
    4537. 

  4537.      SWAPOUT;
    4538. 

  4538.      c_p->i = I + 1; /* Next instruction */
    4539. 

  4539.      erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
    4540. 

  4540.      add_to_schedule_q(c_p);
    4541. 

  4541.      erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
    4542. 

  4542.      c_p->current = NULL;
    4543. 

  4543.      goto do_schedule;
    4544. 

  4544.  }
    4545. 

  4545. 

  4546.  OpCase(i_hibernate): {
    4547. 

  4547.      SWAPOUT;
    4548. 

  4548.      if (hibernate(c_p, r(0), x(1), x(2), reg)) {
    4549. 

  4549. 	 goto do_schedule;
    4550. 

  4550.      } else {
    4551. 

  4551. 	 I = handle_error(c_p, I, reg, hibernate_3);
    4552. 

  4552. 	 goto post_error_handling;
    4553. 

  4553.      }
    4554. 

  4554.  }
    4555. 

  4555. 

  4556.  OpCase(i_debug_breakpoint): {
    4557. 

  4557.      SWAPOUT;
    4558. 

  4558.      reg[0] = r(0);
    4559. 

  4559.      tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg);
    4560. 

  4560.      r(0) = reg[0];
    4561. 

  4561.      SWAPIN;
    4562. 

  4562.      if (tmp_arg1) {
    4563. 

  4563. 	 SET_I(c_p->i);
    4564. 

  4564. 	 Dispatch();
    4565. 

  4565.      }
    4566. 

  4566.      goto no_error_handler;
    4567. 

  4567.  }
    4568. 

  4568. 

  4569. 

  4570.  OpCase(system_limit_j):
    4571. 

  4571.  system_limit:
    4572. 

  4572.     c_p->freason = SYSTEM_LIMIT;
    4573. 

  4573.     goto lb_Cl_error;
    4574. 

  4574. 

  4575. 

  4576. #ifdef ERTS_OPCODE_COUNTER_SUPPORT
    4577. 

  4577.     DEFINE_COUNTING_LABELS;
    4578. 

  4578. #endif
    4579. 

  4579. 

  4580. #ifndef NO_JUMP_TABLE
    4581. 

  4581. #ifdef DEBUG
    4582. 

  4582.  end_emulator_loop:
    4583. 

  4583. #endif
    4584. 

  4584. #endif
    4585. 

  4585. 

  4586.  OpCase(int_code_end):
    4587. 

  4587.  OpCase(label_L):
    4588. 

  4588.  OpCase(too_old_compiler):
    4589. 

  4589.     erl_exit(1, "meta op\n");
    4590. 

  4590. 

  4591.     /*
    4592. 

  4592.      * One-time initialization of Beam emulator.
    4593. 

  4593.      */
    4594. 

  4594. 

  4595.  init_emulator:
    4596. 

  4596.  {
    4597. 

  4597.      int i;
    4598. 

  4598.      Export* ep;
    4599. 

  4599. 

  4600. #ifndef NO_JUMP_TABLE
    4601. 

  4601. #ifdef ERTS_OPCODE_COUNTER_SUPPORT
    4602. 

  4602. 

  4603.      /* Are tables correctly generated by beam_makeops? */
    4604. 

  4604.      ASSERT(sizeof(counting_opcodes) == sizeof(opcodes));
    4605. 

  4605. 

  4606.      if (count_instructions) {
    4607. 

  4607. #ifdef DEBUG
    4608. 

  4608. 	 counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
    4609. 

  4609. #endif
    4610. 

  4610. 	 counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
    4611. 

  4611. 	 beam_ops = counting_opcodes;
    4612. 

  4612.      }
    4613. 

  4613.      else
    4614. 

  4614. #endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
    4615. 

  4615.      {
    4616. 

  4616. 	 beam_ops = opcodes;
    4617. 

  4617.      }
    4618. 

  4618. #endif /* NO_JUMP_TABLE */
    4619. 

  4619.      
    4620. 

  4620.      em_call_error_handler = OpCode(call_error_handler);
    4621. 

  4621.      em_call_traced_function = OpCode(call_traced_function);
    4622. 

  4622.      em_apply_bif = OpCode(apply_bif);
    4623. 

  4623.      beam_apply[0] = (Eterm) OpCode(i_apply);
    4624. 

  4624.      beam_apply[1] = (Eterm) OpCode(normal_exit);
    4625. 

  4625.      beam_exit[0] = (Eterm) OpCode(error_action_code);
    4626. 

  4626.      beam_return_to_trace[0] = (Eterm) OpCode(i_return_to_trace);
    4627. 

  4627.      beam_return_trace[0] = (Eterm) OpCode(return_trace);
    4628. 

  4628.      beam_exception_trace[0] = (Eterm) OpCode(return_trace); /* UGLY */
    4629. 

  4629. 

  4630.      /*
    4631. 

  4631.       * Enter all BIFs into the export table.
    4632. 

  4632.       */
    4633. 

  4633.      for (i = 0; i < BIF_SIZE; i++) {
    4634. 

  4634. 	 ep = erts_export_put(bif_table[i].module,
    4635. 

  4635. 			      bif_table[i].name,
    4636. 

  4636. 			      bif_table[i].arity);
    4637. 

  4637. 	 bif_export[i] = ep;
    4638. 

  4638. 	 ep->code[3] = (Eterm) OpCode(apply_bif);
    4639. 

  4639. 	 ep->code[4] = (Eterm) bif_table[i].f;
    4640. 

  4640.      }
    4641. 

  4641. 

  4642.      return;
    4643. 

  4643.  }
    4644. 

  4644. #ifdef NO_JUMP_TABLE
    4645. 

  4645.  default:
    4646. 

  4646.     erl_exit(1, "unexpected op code %d\n",Go);
    4647. 

  4647.   }
    4648. 

  4648. #endif
    4649. 

  4649.     return;			/* Never executed */
    4650. 

  4650. 

  4651.   save_calls1:
    4652. 

  4652.     {
    4653. 

  4653. 	Eterm* dis_next;
    4654. 

  4654. 

  4655. 	save_calls(c_p, (Export *) Arg(0));
    4656. 

  4656. 

  4657. 	SET_I(((Export *) Arg(0))->address);
    4658. 

  4658. 

  4659. 	dis_next = (Eterm *) *I;
    4660. 

  4660. 	FCALLS--;
    4661. 

  4661. 	Goto(dis_next);
    4662. 

  4662.     }
    4663. 

  4663. }
    4664. 

  4664. 

  4665. static BifFunction
    4666. 

  4666. translate_gc_bif(void* gcf)
    4667. 

  4667. {
    4668. 

  4668.     if (gcf == erts_gc_length_1) {
    4669. 

  4669. 	return length_1;
    4670. 

  4670.     } else if (gcf == erts_gc_size_1) {
    4671. 

  4671. 	return size_1;
    4672. 

  4672.     } else if (gcf == erts_gc_bit_size_1) {
    4673. 

  4673. 	return bit_size_1;
    4674. 

  4674.     } else if (gcf == erts_gc_byte_size_1) {
    4675. 

  4675. 	return byte_size_1;
    4676. 

  4676.     } else if (gcf == erts_gc_abs_1) {
    4677. 

  4677. 	return abs_1;
    4678. 

  4678.     } else if (gcf == erts_gc_float_1) {
    4679. 

  4679. 	return float_1;
    4680. 

  4680.     } else if (gcf == erts_gc_round_1) {
    4681. 

  4681. 	return round_1;
    4682. 

  4682.     } else if (gcf == erts_gc_trunc_1) {
    4683. 

  4683. 	return round_1;
    4684. 

  4684.     } else {
    4685. 

  4685. 	erl_exit(1, "bad gc bif");
    4686. 

  4686.     }
    4687. 

  4687. }
    4688. 

  4688. 

  4689. /*
    4690. 

  4690.  * Mapping from the error code 'class tag' to atoms.
    4691. 

  4691.  */
    4692. 

  4692. Eterm exception_tag[NUMBER_EXC_TAGS] = {
    4693. 

  4693.   am_error,	/* 0 */
    4694. 

  4694.   am_exit,	/* 1 */
    4695. 

  4695.   am_throw,	/* 2 */
    4696. 

  4696. };
    4697. 

  4697. 

  4698. /*
    4699. 

  4699.  * Mapping from error code 'index' to atoms.
    4700. 

  4700.  */
    4701. 

  4701. Eterm error_atom[NUMBER_EXIT_CODES] = {
    4702. 

  4702.   am_internal_error,	/* 0 */
    4703. 

  4703.   am_normal,		/* 1 */
    4704. 

  4704.   am_internal_error,	/* 2 */
    4705. 

  4705.   am_badarg,		/* 3 */
    4706. 

  4706.   am_badarith,		/* 4 */
    4707. 

  4707.   am_badmatch,		/* 5 */
    4708. 

  4708.   am_function_clause,	/* 6 */
    4709. 

  4709.   am_case_clause,	/* 7 */
    4710. 

  4710.   am_if_clause,		/* 8 */
    4711. 

  4711.   am_undef,		/* 9 */
    4712. 

  4712.   am_badfun,		/* 10 */
    4713. 

  4713.   am_badarity,		/* 11 */
    4714. 

  4714.   am_timeout_value,	/* 12 */
    4715. 

  4715.   am_noproc,		/* 13 */
    4716. 

  4716.   am_notalive,		/* 14 */
    4717. 

  4717.   am_system_limit,	/* 15 */
    4718. 

  4718.   am_try_clause,	/* 16 */
    4719. 

  4719. };
    4720. 

  4720. 

  4721. /*
    4722. 

  4722.  * To fully understand the error handling, one must keep in mind that
    4723. 

  4723.  * when an exception is thrown, the search for a handler can jump back
    4724. 

  4724.  * and forth between Beam and native code. Upon each mode switch, a
    4725. 

  4725.  * dummy handler is inserted so that if an exception reaches that point,
    4726. 

  4726.  * the handler is invoked (like any handler) and transfers control so
    4727. 

  4727.  * that the search for a real handler is continued in the other mode.
    4728. 

  4728.  * Therefore, c_p->freason and c_p->fvalue must still hold the exception
    4729. 

  4729.  * info when the handler is executed, but normalized so that creation of
    4730. 

  4730.  * error terms and saving of the stack trace is only done once, even if
    4731. 

  4731.  * we pass through the error handling code several times.
    4732. 

  4732.  *
    4733. 

  4733.  * When a new exception is raised, the current stack trace information
    4734. 

  4734.  * is quick-saved in a small structure allocated on the heap. Depending
    4735. 

  4735.  * on how the exception is eventually caught (perhaps by causing the
    4736. 

  4736.  * current process to terminate), the saved information may be used to
    4737. 

  4737.  * create a symbolic (human-readable) representation of the stack trace
    4738. 

  4738.  * at the point of the original exception.
    4739. 

  4739.  */
    4740. 

  4740. 

  4741. static Eterm*
    4742. 

  4742. handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
    4743. 

  4743. {
    4744. 

  4744.     Eterm* hp;
    4745. 

  4745.     Eterm Value = c_p->fvalue;
    4746. 

  4746.     Eterm Args = am_true;
    4747. 

  4747.     c_p->i = pc;    /* In case we call erl_exit(). */
    4748. 

  4748. 

  4749.     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
    4750. 

  4750.     ASSERT(c_p->freason != RESCHEDULE); /* Should have been handled earlier. */
    4751. 

  4751. 

  4752.     /*
    4753. 

  4753.      * Check if we have an arglist for the top level call. If so, this
    4754. 

  4754.      * is encoded in Value, so we have to dig out the real Value as well
    4755. 

  4755.      * as the Arglist.
    4756. 

  4756.      */
    4757. 

  4757.     if (c_p->freason & EXF_ARGLIST) {
    4758. 

  4758. 	  Eterm* tp;
    4759. 

  4759. 	  ASSERT(is_tuple(Value));
    4760. 

  4760. 	  tp = tuple_val(Value);
    4761. 

  4761. 	  Value = tp[1];
    4762. 

  4762. 	  Args = tp[2];
    4763. 

  4763.     }
    4764. 

  4764. 

  4765.     /*
    4766. 

  4766.      * Save the stack trace info if the EXF_SAVETRACE flag is set. The
    4767. 

  4767.      * main reason for doing this separately is to allow throws to later
    4768. 

  4768.      * become promoted to errors without losing the original stack
    4769. 

  4769.      * trace, even if they have passed through one or more catch and
    4770. 

  4770.      * rethrow. It also makes the creation of symbolic stack traces much
    4771. 

  4771.      * more modular.
    4772. 

  4772.      */
    4773. 

  4773.     if (c_p->freason & EXF_SAVETRACE) {
    4774. 

  4774.         save_stacktrace(c_p, pc, reg, bf, Args);
    4775. 

  4775.     }
    4776. 

  4776. 

  4777.     /*
    4778. 

  4778.      * Throws that are not caught are turned into 'nocatch' errors
    4779. 

  4779.      */
    4780. 

  4780.     if ((c_p->freason & EXF_THROWN) && (c_p->catches <= 0) ) {
    4781. 

  4781. 	hp = HAlloc(c_p, 3);
    4782. 

  4782.         Value = TUPLE2(hp, am_nocatch, Value);
    4783. 

  4783.         c_p->freason = EXC_ERROR;
    4784. 

  4784.     }
    4785. 

  4785. 

  4786.     /* Get the fully expanded error term */
    4787. 

  4787.     Value = expand_error_value(c_p, c_p->freason, Value);
    4788. 

  4788. 

  4789.     /* Save final error term and stabilize the exception flags so no
    4790. 

  4790.        further expansion is done. */
    4791. 

  4791.     c_p->fvalue = Value;
    4792. 

  4792.     c_p->freason = PRIMARY_EXCEPTION(c_p->freason);
    4793. 

  4793. 

  4794.     /* Find a handler or die */
    4795. 

  4795.     if ((c_p->catches > 0 || IS_TRACED_FL(c_p, F_EXCEPTION_TRACE))
    4796. 

  4796. 	&& !(c_p->freason & EXF_PANIC)) {
    4797. 

  4797. 	Eterm *new_pc;
    4798. 

  4798.         /* The Beam handler code (catch_end or try_end) checks reg[0]
    4799. 

  4799. 	   for THE_NON_VALUE to see if the previous code finished
    4800. 

  4800. 	   abnormally. If so, reg[1], reg[2] and reg[3] should hold the
    4801. 

  4801. 	   exception class, term and trace, respectively. (If the
    4802. 

  4802. 	   handler is just a trap to native code, these registers will
    4803. 

  4803. 	   be ignored.) */
    4804. 

  4804. 	reg[0] = THE_NON_VALUE;
    4805. 

  4805. 	reg[1] = exception_tag[GET_EXC_CLASS(c_p->freason)];
    4806. 

  4806. 	reg[2] = Value;
    4807. 

  4807. 	reg[3] = c_p->ftrace;
    4808. 

  4808.         if ( (new_pc = next_catch(c_p, reg))) return new_pc;
    4809. 

  4809. 	if (c_p->catches > 0) erl_exit(1, "Catch not found");
    4810. 

  4810.     }
    4811. 

  4811.     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    4812. 

  4812.     terminate_proc(c_p, Value);
    4813. 

  4813.     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    4814. 

  4814.     return NULL;
    4815. 

  4815. }
    4816. 

  4816. 

  4817. /*
    4818. 

  4818.  * Find the nearest catch handler
    4819. 

  4819.  */
    4820. 

  4820. static Eterm*
    4821. 

  4821. next_catch(Process* c_p, Eterm *reg) {
    4822. 

  4822.     int active_catches = c_p->catches > 0;
    4823. 

  4823.     int have_return_to_trace = 0;
    4824. 

  4824.     Eterm *ptr, *prev, *return_to_trace_ptr = NULL;
    4825. 

  4825.     Uint i_return_trace = beam_return_trace[0];
    4826. 

  4826.     Uint i_return_to_trace = beam_return_to_trace[0];
    4827. 

  4827.     ptr = prev = c_p->stop;
    4828. 

  4828.     ASSERT(is_CP(*ptr));
    4829. 

  4829.     ASSERT(ptr <= STACK_START(c_p));
    4830. 

  4830.     if (ptr == STACK_START(c_p)) return NULL;
    4831. 

  4831.     if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace &&
    4832. 

  4832. 			     *cp_val(*ptr) != i_return_to_trace))
    4833. 

  4833. 	&& c_p->cp) {
    4834. 

  4834. 	/* Can not follow cp here - code may be unloaded */
    4835. 

  4835. 	Uint *cpp = cp_val((Eterm) c_p->cp);
    4836. 

  4836. 	if (cpp == beam_exception_trace) {
    4837. 

  4837. 	    erts_trace_exception(c_p, (Eterm*) ptr[0],
    4838. 

  4838. 				 reg[1], reg[2], ptr+1);
    4839. 

  4839. 	    /* Skip return_trace parameters */
    4840. 

  4840. 	    ptr += 2;
    4841. 

  4841. 	} else if (cpp == beam_return_trace) {
    4842. 

  4842. 	    /* Skip return_trace parameters */
    4843. 

  4843. 	    ptr += 2;
    4844. 

  4844. 	} else if (cpp == beam_return_to_trace) {
    4845. 

  4845. 	    have_return_to_trace = !0; /* Record next cp */
    4846. 

  4846. 	}
    4847. 

  4847.     }
    4848. 

  4848.     while (ptr < STACK_START(c_p)) {
    4849. 

  4849. 	if (is_catch(*ptr)) {
    4850. 

  4850. 	    if (active_catches) goto found_catch;
    4851. 

  4851. 	    ptr++;
    4852. 

  4852. 	}
    4853. 

  4853. 	else if (is_CP(*ptr)) {
    4854. 

  4854. 	    prev = ptr;
    4855. 

  4855. 	    if (*cp_val(*prev) == i_return_trace) {
    4856. 

  4856. 		/* Skip stack frame variables */
    4857. 

  4857. 		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
    4858. 

  4858. 		    if (is_catch(*ptr) && active_catches) goto found_catch;
    4859. 

  4859. 		}
    4860. 

  4860. 		if (cp_val(*prev) == beam_exception_trace) {
    4861. 

  4861. 		    erts_trace_exception(c_p, (Eterm*) ptr[0],
    4862. 

  4862. 					 reg[1], reg[2], ptr+1);
    4863. 

  4863. 		}
    4864. 

  4864. 		/* Skip return_trace parameters */
    4865. 

  4865. 		ptr += 2;
    4866. 

  4866. 	    } else if (*cp_val(*prev) == i_return_to_trace) {
    4867. 

  4867. 		/* Skip stack frame variables */
    4868. 

  4868. 		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
    4869. 

  4869. 		    if (is_catch(*ptr) && active_catches) goto found_catch;
    4870. 

  4870. 		}
    4871. 

  4871. 		have_return_to_trace = !0; /* Record next cp */
    4872. 

  4872. 		return_to_trace_ptr = NULL;
    4873. 

  4873. 	    } else {
    4874. 

  4874. 		if (have_return_to_trace) {
    4875. 

  4875. 		    /* Record this cp as possible return_to trace cp */
    4876. 

  4876. 		    have_return_to_trace = 0;
    4877. 

  4877. 		    return_to_trace_ptr = ptr;
    4878. 

  4878. 		} else return_to_trace_ptr = NULL;
    4879. 

  4879. 		ptr++;
    4880. 

  4880. 	    }
    4881. 

  4881. 	} else ptr++;
    4882. 

  4882.     }
    4883. 

  4883.     return NULL;
    4884. 

  4884.     
    4885. 

  4885.  found_catch:
    4886. 

  4886.     ASSERT(ptr < STACK_START(c_p));
    4887. 

  4887.     c_p->stop = prev;
    4888. 

  4888.     if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO) && return_to_trace_ptr) {
    4889. 

  4889. 	/* The stackframe closest to the catch contained an
    4890. 

  4890. 	 * return_to_trace entry, so since the execution now
    4891. 

  4891. 	 * continues after the catch, a return_to trace message 
    4892. 

  4892. 	 * would be appropriate.
    4893. 

  4893. 	 */
    4894. 

  4894. 	erts_trace_return_to(c_p, cp_val(*return_to_trace_ptr));
    4895. 

  4895.     }
    4896. 

  4896.     return catch_pc(*ptr);
    4897. 

  4897. }
    4898. 

  4898. 

  4899. /*
    4900. 

  4900.  * Terminating the process when an exception is not caught
    4901. 

  4901.  */
    4902. 

  4902. static void
    4903. 

  4903. terminate_proc(Process* c_p, Eterm Value)
    4904. 

  4904. {
    4905. 

  4905.     /* Add a stacktrace if this is an error. */
    4906. 

  4906.     if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) {
    4907. 

  4907.         Value = add_stacktrace(c_p, Value, c_p->ftrace);
    4908. 

  4908.     }
    4909. 

  4909.     /* EXF_LOG is a primary exception flag */
    4910. 

  4910.     if (c_p->freason & EXF_LOG) {
    4911. 

  4911. 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
    4912. 

  4912. 	erts_dsprintf(dsbufp, "Error in process %T ", c_p->id);
    4913. 

  4913. 	if (erts_is_alive())
    4914. 

  4914. 	    erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname);
    4915. 

  4915. 	erts_dsprintf(dsbufp,"with exit value: %0.*T\n", display_items, Value);
    4916. 

  4916. 	erts_send_error_to_logger(c_p->group_leader, dsbufp);
    4917. 

  4917.     }
    4918. 

  4918.     /*
    4919. 

  4919.      * If we use a shared heap, the process will be garbage-collected.
    4920. 

  4920.      * Must zero c_p->arity to indicate that there are no live registers.
    4921. 

  4921.      */
    4922. 

  4922.     c_p->arity = 0;
    4923. 

  4923.     erts_do_exit_process(c_p, Value);
    4924. 

  4924. }
    4925. 

  4925. 

  4926. /*
    4927. 

  4927.  * Build and add a symbolic stack trace to the error value.
    4928. 

  4928.  */
    4929. 

  4929. static Eterm
    4930. 

  4930. add_stacktrace(Process* c_p, Eterm Value, Eterm exc) {
    4931. 

  4931.     Eterm Where = build_stacktrace(c_p, exc);
    4932. 

  4932.     Eterm* hp = HAlloc(c_p, 3);
    4933. 

  4933.     return TUPLE2(hp, Value, Where);
    4934. 

  4934. }
    4935. 

  4935. 

  4936. /*
    4937. 

  4937.  * Forming the correct error value from the internal error code.
    4938. 

  4938.  * This does not update c_p->fvalue or c_p->freason.
    4939. 

  4939.  */
    4940. 

  4940. Eterm
    4941. 

  4941. expand_error_value(Process* c_p, Uint freason, Eterm Value) {
    4942. 

  4942.     Eterm* hp;
    4943. 

  4943.     Uint r;
    4944. 

  4944. 

  4945.     r = GET_EXC_INDEX(freason);
    4946. 

  4946.     ASSERT(r < NUMBER_EXIT_CODES); /* range check */
    4947. 

  4947.     ASSERT(is_value(Value));
    4948. 

  4948. 

  4949.     switch (r) {
    4950. 

  4950.     case (GET_EXC_INDEX(EXC_PRIMARY)):
    4951. 

  4951.         /* Primary exceptions use fvalue as it is */
    4952. 

  4952. 	break;
    4953. 

  4953.     case (GET_EXC_INDEX(EXC_BADMATCH)):
    4954. 

  4954.     case (GET_EXC_INDEX(EXC_CASE_CLAUSE)):
    4955. 

  4955.     case (GET_EXC_INDEX(EXC_TRY_CLAUSE)):
    4956. 

  4956.     case (GET_EXC_INDEX(EXC_BADFUN)):
    4957. 

  4957.     case (GET_EXC_INDEX(EXC_BADARITY)):
    4958. 

  4958.         /* Some common exceptions: value -> {atom, value} */
    4959. 

  4959.         ASSERT(is_value(Value));
    4960. 

  4960. 	hp = HAlloc(c_p, 3);
    4961. 

  4961. 	Value = TUPLE2(hp, error_atom[r], Value);
    4962. 

  4962. 	break;
    4963. 

  4963.     default:
    4964. 

  4964.         /* Other exceptions just use an atom as descriptor */
    4965. 

  4965.         Value = error_atom[r];
    4966. 

  4966. 	break;
    4967. 

  4967.     }
    4968. 

  4968. #ifdef DEBUG
    4969. 

  4969.     ASSERT(Value != am_internal_error);
    4970. 

  4970. #endif
    4971. 

  4971.     return Value;
    4972. 

  4972. }
    4973. 

  4973. 

  4974. /*
    4975. 

  4975.  * Quick-saving the stack trace in an internal form on the heap. Note
    4976. 

  4976.  * that c_p->ftrace will point to a cons cell which holds the given args
    4977. 

  4977.  * and the saved data (encoded as a bignum).
    4978. 

  4978.  *
    4979. 

  4979.  * (It would be much better to put the arglist - when it exists - in the
    4980. 

  4980.  * error value instead of in the actual trace; e.g. '{badarg, Args}'
    4981. 

  4981.  * instead of using 'badarg' with Args in the trace. The arglist may
    4982. 

  4982.  * contain very large values, and right now they will be kept alive as
    4983. 

  4983.  * long as the stack trace is live. Preferably, the stack trace should
    4984. 

  4984.  * always be small, so that it does not matter if it is long-lived.
    4985. 

  4985.  * However, it is probably not possible to ever change the format of
    4986. 

  4986.  * error terms.)
    4987. 

  4987.  */
    4988. 

  4988. 

  4989. static void
    4990. 

  4990. save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
    4991. 

  4991. 		Eterm args) {
    4992. 

  4992.     struct StackTrace* s;
    4993. 

  4993.     int sz;
    4994. 

  4994.     int depth = erts_backtrace_depth;    /* max depth (never negative) */
    4995. 

  4995.     if (depth > 0) {
    4996. 

  4996. 	/* There will always be a current function */
    4997. 

  4997. 	depth --;
    4998. 

  4998.     }
    4999. 

  4999. 

  5000.     /* Create a container for the exception data */
    5001. 

  5001.     sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm)*depth
    5002. 

  5002.           + sizeof(Eterm) - 1) / sizeof(Eterm);
    5003. 

  5003.     s = (struct StackTrace *) HAlloc(c_p, 1 + sz);
    5004. 

  5004.     /* The following fields are inside the bignum */
    5005. 

  5005.     s->header = make_pos_bignum_header(sz);
    5006. 

  5006.     s->freason = c_p->freason;
    5007. 

  5007.     s->depth = 0;
    5008. 

  5008. 

  5009.     /*
    5010. 

  5010.      * If the failure was in a BIF other than 'error', 'exit' or
    5011. 

  5011.      * 'throw', find the bif-table index and save the argument
    5012. 

  5012.      * registers by consing up an arglist.
    5013. 

  5013.      */
    5014. 

  5014.     if (bf != NULL && bf != error_1 && bf != error_2
    5015. 

  5015. 	&& bf != fault_1 && bf != fault_2
    5016. 

  5016. 	&& bf != exit_1 && bf != throw_1) {
    5017. 

  5017.         int i;
    5018. 

  5018. 	int a = 0;
    5019. 

  5019. 	for (i = 0; i < BIF_SIZE; i++) {
    5020. 

  5020. 	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
    5021. 

  5021. 		Export *ep = bif_export[i];
    5022. 

  5022. 		s->current = ep->code;
    5023. 

  5023. 	        a = bif_table[i].arity;
    5024. 

  5024. 		break;
    5025. 

  5025. 	    }
    5026. 

  5026. 	}
    5027. 

  5027. 	if (i >= BIF_SIZE) {
    5028. 

  5028. 	    /* 
    5029. 

  5029. 	     * The Bif does not really exist (no BIF entry).  It is a
    5030. 

  5030. 	     * TRAP and traps are called through apply_bif, which also
    5031. 

  5031. 	     * sets c_p->current (luckily).
    5032. 

  5032. 	     */
    5033. 

  5033. 	    ASSERT(c_p->current);
    5034. 

  5034. 	    s->current = c_p->current;
    5035. 

  5035. 	    a = s->current[2];
    5036. 

  5036. 	    ASSERT(s->current[2] <= 3);
    5037. 

  5037. 	}
    5038. 

  5038. 	/* Save first stack entry */
    5039. 

  5039. 	ASSERT(pc);
    5040. 

  5040. 	if (depth > 0) {
    5041. 

  5041. 	    s->trace[s->depth++] = pc;
    5042. 

  5042. 	    depth--;
    5043. 

  5043. 	}
    5044. 

  5044. 	s->pc = NULL;
    5045. 

  5045. 	args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
    5046. 

  5046.     } else {
    5047. 

  5047. 	s->current = c_p->current;
    5048. 

  5048.         /* 
    5049. 

  5049. 	 * For a function_clause error, the arguments are in the beam
    5050. 

  5050. 	 * registers, c_p->cp is valid, and c_p->current is set.
    5051. 

  5051. 	 */
    5052. 

  5052. 	if ( (GET_EXC_INDEX(s->freason)) ==
    5053. 

  5053. 	     (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) {
    5054. 

  5054. 	    int a;
    5055. 

  5055. 	    ASSERT(s->current);
    5056. 

  5056. 	    a = s->current[2];
    5057. 

  5057. 	    args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
    5058. 

  5058. 	    /* Save first stack entry */
    5059. 

  5059. 	    ASSERT(c_p->cp);
    5060. 

  5060. 	    if (depth > 0) {
    5061. 

  5061. 		s->trace[s->depth++] = c_p->cp;
    5062. 

  5062. 		depth--;
    5063. 

  5063. 	    }
    5064. 

  5064. 	    s->pc = NULL; /* Ignore pc */
    5065. 

  5065. 	} else {
    5066. 

  5066. 	    s->pc = pc;
    5067. 

  5067. 	}
    5068. 

  5068.     }
    5069. 

  5069. 

  5070.     /* Package args and stack trace */
    5071. 

  5071.     {
    5072. 

  5072. 	Eterm *hp;
    5073. 

  5073. 	hp = HAlloc(c_p, 2);
    5074. 

  5074. 	c_p->ftrace = CONS(hp, args, make_big((Eterm *) s));
    5075. 

  5075.     }
    5076. 

  5076. 

  5077.     /* Save the actual stack trace */
    5078. 

  5078.     if (depth > 0) {
    5079. 

  5079. 	Eterm *ptr, *prev = s->depth ? s->trace[s->depth-1] : NULL;
    5080. 

  5080. 	Uint i_return_trace = beam_return_trace[0];
    5081. 

  5081. 	Uint i_return_to_trace = beam_return_to_trace[0];
    5082. 

  5082. 	/*
    5083. 

  5083. 	 * Traverse the stack backwards and add all unique continuation
    5084. 

  5084. 	 * pointers to the buffer, up to the maximum stack trace size.
    5085. 

  5085. 	 * 
    5086. 

  5086. 	 * Skip trace stack frames.
    5087. 

  5087. 	 */
    5088. 

  5088. 	ptr = c_p->stop;
    5089. 

  5089. 	if (ptr < STACK_START(c_p) 
    5090. 

  5090. 	    && (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace &&
    5091. 

  5091. 				   *cp_val(*ptr) != i_return_to_trace))
    5092. 

  5092. 	    && c_p->cp) {
    5093. 

  5093. 	    /* Can not follow cp here - code may be unloaded */
    5094. 

  5094. 	    Uint *cpp = cp_val((Eterm) c_p->cp);
    5095. 

  5095. 	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
    5096. 

  5096. 		/* Skip return_trace parameters */
    5097. 

  5097. 		ptr += 2;
    5098. 

  5098. 	    } else if (cpp == beam_return_to_trace) {
    5099. 

  5099. 		/* Skip return_to_trace parameters */
    5100. 

  5100. 		ptr += 1;
    5101. 

  5101. 	    }
    5102. 

  5102. 	}
    5103. 

  5103. 	while (ptr < STACK_START(c_p) && depth > 0) {
    5104. 

  5104. 	    if (is_CP(*ptr)) {
    5105. 

  5105. 		if (*cp_val(*ptr) == i_return_trace) {
    5106. 

  5106. 		    /* Skip stack frame variables */
    5107. 

  5107. 		    do ++ptr; while (is_not_CP(*ptr));
    5108. 

  5108. 		    /* Skip return_trace parameters */
    5109. 

  5109. 		    ptr += 2;
    5110. 

  5110. 		} else if (*cp_val(*ptr) == i_return_to_trace) {
    5111. 

  5111. 		    /* Skip stack frame variables */
    5112. 

  5112. 		    do ++ptr; while (is_not_CP(*ptr));
    5113. 

  5113. 		} else {
    5114. 

  5114. 		    Eterm *cp = (Eterm *)(*ptr);
    5115. 

  5115. 		    if (cp != prev) {
    5116. 

  5116. 			/* Record non-duplicates only */
    5117. 

  5117. 			prev = cp;
    5118. 

  5118. 			s->trace[s->depth++] = cp;
    5119. 

  5119. 			depth--;
    5120. 

  5120. 		    }
    5121. 

  5121. 		    ptr++;
    5122. 

  5122. 		}
    5123. 

  5123. 	    } else ptr++;
    5124. 

  5124. 	}
    5125. 

  5125.     }
    5126. 

  5126. }
    5127. 

  5127. 

  5128. /*
    5129. 

  5129.  * Getting the relevant fields from the term pointed to by ftrace
    5130. 

  5130.  */
    5131. 

  5131. 

  5132. static struct StackTrace *get_trace_from_exc(Eterm exc) {
    5133. 

  5133.     if (exc == NIL) {
    5134. 

  5134. 	return NULL;
    5135. 

  5135.     } else {
    5136. 

  5136. 	ASSERT(is_list(exc));
    5137. 

  5137. 	return (struct StackTrace *) big_val(CDR(list_val(exc)));
    5138. 

  5138.     }
    5139. 

  5139. }
    5140. 

  5140. 

  5141. static Eterm get_args_from_exc(Eterm exc) {
    5142. 

  5142.     if (exc == NIL) {
    5143. 

  5143. 	return NIL;
    5144. 

  5144.     } else {
    5145. 

  5145. 	ASSERT(is_list(exc));
    5146. 

  5146. 	return CAR(list_val(exc));
    5147. 

  5147.     }
    5148. 

  5148. }
    5149. 

  5149. 

  5150. static int is_raised_exc(Eterm exc) {
    5151. 

  5151.     if (exc == NIL) {
    5152. 

  5152.         return 0;
    5153. 

  5153.     } else {
    5154. 

  5154.         ASSERT(is_list(exc));
    5155. 

  5155.         return bignum_header_is_neg(*big_val(CDR(list_val(exc))));
    5156. 

  5156.     }
    5157. 

  5157. }
    5158. 

  5158. 

  5159. /*
    5160. 

  5160.  * Creating a list with the argument registers
    5161. 

  5161.  */
    5162. 

  5162. static Eterm
    5163. 

  5163. make_arglist(Process* c_p, Eterm* reg, int a) {
    5164. 

  5164.     Eterm args = NIL;
    5165. 

  5165.     Eterm* hp = HAlloc(c_p, 2*a);
    5166. 

  5166.     while (a > 0) {
    5167. 

  5167.         args = CONS(hp, reg[a-1], args);
    5168. 

  5168. 	hp += 2;
    5169. 

  5169. 	a--;
    5170. 

  5170.     }
    5171. 

  5171.     return args;
    5172. 

  5172. }
    5173. 

  5173. 

  5174. /*
    5175. 

  5175.  * Building a symbolic representation of a saved stack trace. Note that
    5176. 

  5176.  * the exception object 'exc', unless NIL, points to a cons cell which
    5177. 

  5177.  * holds the given args and the quick-saved data (encoded as a bignum).
    5178. 

  5178.  *
    5179. 

  5179.  * If the bignum is negative, the given args is a complete stacktrace.
    5180. 

  5180.  */
    5181. 

  5181. Eterm
    5182. 

  5182. build_stacktrace(Process* c_p, Eterm exc) {
    5183. 

  5183.     struct StackTrace* s;
    5184. 

  5184.     Eterm  args;
    5185. 

  5185.     int    depth;
    5186. 

  5186.     Eterm* current;
    5187. 

  5187.     Eterm  Where = NIL;
    5188. 

  5188.     Eterm* next_p = &Where;
    5189. 

  5189. 

  5190.     if (! (s = get_trace_from_exc(exc))) {
    5191. 

  5191.         return NIL;
    5192. 

  5192.     }
    5193. 

  5193. #ifdef HIPE
    5194. 

  5194.     if (s->freason & EXF_NATIVE) {
    5195. 

  5195. 	return hipe_build_stacktrace(c_p, s);
    5196. 

  5196.     }
    5197. 

  5197. #endif
    5198. 

  5198.     if (is_raised_exc(exc)) {
    5199. 

  5199. 	return get_args_from_exc(exc);
    5200. 

  5200.     }
    5201. 

  5201. 

  5202.     /*
    5203. 

  5203.      * Find the current function. If the saved s->pc is null, then the
    5204. 

  5204.      * saved s->current should already contain the proper value.
    5205. 

  5205.      */
    5206. 

  5206.     if (s->pc != NULL) {
    5207. 

  5207. 	current = find_function_from_pc(s->pc);
    5208. 

  5208.     } else {
    5209. 

  5209. 	current = s->current;
    5210. 

  5210.     }
    5211. 

  5211.     /*
    5212. 

  5212.      * If current is still NULL, default to the initial function
    5213. 

  5213.      * (e.g. spawn_link(erlang, abs, [1])).
    5214. 

  5214.      */
    5215. 

  5215.     if (current == NULL) {
    5216. 

  5216. 	current = c_p->initial;
    5217. 

  5217. 	args = am_true; /* Just in case */
    5218. 

  5218.     } else {
    5219. 

  5219. 	args = get_args_from_exc(exc);
    5220. 

  5220.     }
    5221. 

  5221. 

  5222.     depth = s->depth;
    5223. 

  5223.     
    5224. 

  5224.     /*
    5225. 

  5225.      * Add the {M,F,A} for the current function 
    5226. 

  5226.      * (where A is arity or [Argument]).
    5227. 

  5227.      */
    5228. 

  5228.     {
    5229. 

  5229. 	int i;
    5230. 

  5230. 	Eterm mfa;
    5231. 

  5231. 	Uint heap_size = 6*(depth+1);
    5232. 

  5232. 	Eterm* hp = HAlloc(c_p, heap_size);
    5233. 

  5233. 	Eterm* hp_end = hp + heap_size;
    5234. 

  5234. 

  5235. 	if (args != am_true) {
    5236. 

  5236. 	    /* We have an arglist - use it */
    5237. 

  5237. 	    mfa = TUPLE3(hp, current[0], current[1], args);
    5238. 

  5238. 	} else {
    5239. 

  5239. 	    Eterm arity = make_small(current[2]);
    5240. 

  5240. 	    mfa = TUPLE3(hp, current[0], current[1], arity);
    5241. 

  5241. 	}
    5242. 

  5242. 	hp += 4;
    5243. 

  5243. 	ASSERT(*next_p == NIL);
    5244. 

  5244. 	*next_p = CONS(hp, mfa, NIL);
    5245. 

  5245. 	next_p = &CDR(list_val(*next_p));
    5246. 

  5246. 	hp += 2;
    5247. 

  5247. 

  5248. 	/* 
    5249. 

  5249. 	 * Finally, we go through the saved continuation pointers.
    5250. 

  5250. 	 */
    5251. 

  5251. 	for (i = 0; i < depth; i++) {
    5252. 

  5252. 	    Eterm *fi = find_function_from_pc((Eterm *) s->trace[i]);
    5253. 

  5253. 	    if (fi == NULL) continue;
    5254. 

  5254. 	    mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2]));
    5255. 

  5255. 	    hp += 4;
    5256. 

  5256. 	    ASSERT(*next_p == NIL);
    5257. 

  5257. 	    *next_p = CONS(hp, mfa, NIL);
    5258. 

  5258. 	    next_p = &CDR(list_val(*next_p));
    5259. 

  5259. 	    hp += 2;
    5260. 

  5260. 	}
    5261. 

  5261. 	ASSERT(hp <= hp_end);
    5262. 

  5262. 	HRelease(c_p, hp_end, hp);
    5263. 

  5263.     }
    5264. 

  5264.     return Where;
    5265. 

  5265. }
    5266. 

  5266. 

  5267. 
    5268. 

  5268. static Eterm
    5269. 

  5269. call_error_handler(Process* p, Eterm* fi, Eterm* reg)
    5270. 

  5270. {
    5271. 

  5271.     Eterm* hp;
    5272. 

  5272.     Export* ep;
    5273. 

  5273.     int arity;
    5274. 

  5274.     Eterm args;
    5275. 

  5275.     Uint sz;
    5276. 

  5276.     int i;
    5277. 

  5277. 

  5278.     /*
    5279. 

  5279.      * Search for the error_handler module.
    5280. 

  5280.      */
    5281. 

  5281.     ep = erts_find_function(p->error_handler, am_undefined_function, 3);
    5282. 

  5282.     if (ep == NULL) {		/* No error handler */
    5283. 

  5283. 	p->current = fi;
    5284. 

  5284. 	p->freason = EXC_UNDEF;
    5285. 

  5285. 	return 0;
    5286. 

  5286.     }
    5287. 

  5287.     p->i = ep->address;
    5288. 

  5288. 

  5289.     /*
    5290. 

  5290.      * Create a list with all arguments in the x registers.
    5291. 

  5291.      */
    5292. 

  5292. 

  5293.     arity = fi[2];
    5294. 

  5294.     sz = 2 * arity;
    5295. 

  5295.     if (HeapWordsLeft(p) < sz) {
    5296. 

  5296. 	erts_garbage_collect(p, sz, reg, arity);
    5297. 

  5297.     }
    5298. 

  5298.     hp = HEAP_TOP(p);
    5299. 

  5299.     HEAP_TOP(p) += sz;
    5300. 

  5300.     args = NIL;
    5301. 

  5301.     for (i = arity-1; i >= 0; i--) {
    5302. 

  5302. 	args = CONS(hp, reg[i], args);
    5303. 

  5303. 	hp += 2;
    5304. 

  5304.     }
    5305. 

  5305. 

  5306.     /*
    5307. 

  5307.      * Set up registers for call to error_handler:undefined_function/3.
    5308. 

  5308.      */
    5309. 

  5309.     reg[0] = fi[0];
    5310. 

  5310.     reg[1] = fi[1];
    5311. 

  5311.     reg[2] = args;
    5312. 

  5312.     return 1;
    5313. 

  5313. }
    5314. 

  5314. 

  5315. static Eterm
    5316. 

  5316. call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg)
    5317. 

  5317. {
    5318. 

  5318.     Eterm* hp;
    5319. 

  5319.     Export* ep;
    5320. 

  5320.     int arity;
    5321. 

  5321.     Eterm args;
    5322. 

  5322.     Uint sz;
    5323. 

  5323.     int i;
    5324. 

  5324. 

  5325.     /*
    5326. 

  5326.      * Search for error handler module.
    5327. 

  5327.      */
    5328. 

  5328.     ep = erts_find_function(p->error_handler, am_breakpoint, 3);
    5329. 

  5329.     if (ep == NULL) {		/* No error handler */
    5330. 

  5330. 	p->current = fi;
    5331. 

  5331. 	p->freason = EXC_UNDEF;
    5332. 

  5332. 	return 0;
    5333. 

  5333.     }
    5334. 

  5334.     p->i = ep->address;
    5335. 

  5335. 

  5336.     /*
    5337. 

  5337.      * Create a list with all arguments in the x registers.
    5338. 

  5338.      */
    5339. 

  5339. 

  5340.     arity = fi[2];
    5341. 

  5341.     sz = 2 * arity;
    5342. 

  5342.     if (HeapWordsLeft(p) < sz) {
    5343. 

  5343. 	erts_garbage_collect(p, sz, reg, arity);
    5344. 

  5344.     }
    5345. 

  5345.     hp = HEAP_TOP(p);
    5346. 

  5346.     HEAP_TOP(p) += sz;
    5347. 

  5347.     args = NIL;
    5348. 

  5348.     for (i = arity-1; i >= 0; i--) {
    5349. 

  5349. 	args = CONS(hp, reg[i], args);
    5350. 

  5350. 	hp += 2;
    5351. 

  5351.     }
    5352. 

  5352. 

  5353.     /*
    5354. 

  5354.      * Set up registers for call to error_handler:breakpoint/3.
    5355. 

  5355.      */
    5356. 

  5356.     reg[0] = fi[0];
    5357. 

  5357.     reg[1] = fi[1];
    5358. 

  5358.     reg[2] = args;
    5359. 

  5359.     return 1;
    5360. 

  5360. }
    5361. 

  5361. 

  5362. 

  5363. 
    5364. 

  5364. static Export*
    5365. 

  5365. apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
    5366. 

  5366. {
    5367. 

  5367.     Export* ep;
    5368. 

  5368. 

  5369.     /*
    5370. 

  5370.      * Find the export table index for the error handler. Return NULL if
    5371. 

  5371.      * there is no error handler module.
    5372. 

  5372.      */
    5373. 

  5373. 

  5374.     if ((ep = erts_find_export_entry(p->error_handler,
    5375. 

  5375. 				     am_undefined_function, 3)) == NULL) {
    5376. 

  5376. 	return NULL;
    5377. 

  5377.     } else {
    5378. 

  5378. 	int i;
    5379. 

  5379. 	Uint sz = 2*arity;
    5380. 

  5380. 	Eterm* hp;
    5381. 

  5381. 	Eterm args = NIL;
    5382. 

  5382. 	
    5383. 

  5383. 	/*
    5384. 

  5384. 	 * Always copy args from registers to a new list; this ensures
    5385. 

  5385. 	 * that we have the same behaviour whether or not this was
    5386. 

  5386. 	 * called from apply or fixed_apply (any additional last
    5387. 

  5387. 	 * THIS-argument will be included, assuming that arity has been
    5388. 

  5388. 	 * properly adjusted).
    5389. 

  5389. 	 */
    5390. 

  5390. 

  5391. 	if (HeapWordsLeft(p) < sz) {
    5392. 

  5392. 	    erts_garbage_collect(p, sz, reg, arity);
    5393. 

  5393. 	}
    5394. 

  5394. 	hp = HEAP_TOP(p);
    5395. 

  5395. 	HEAP_TOP(p) += sz;
    5396. 

  5396. 	for (i = arity-1; i >= 0; i--) {
    5397. 

  5397. 	    args = CONS(hp, reg[i], args);
    5398. 

  5398. 	    hp += 2;
    5399. 

  5399. 	}
    5400. 

  5400. 	reg[0] = module;
    5401. 

  5401. 	reg[1] = function;
    5402. 

  5402. 	reg[2] = args;
    5403. 

  5403.     }
    5404. 

  5404. 

  5405.     return ep;
    5406. 

  5406. }
    5407. 

  5407. 

  5408. static Uint*
    5409. 

  5409. apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
    5410. 

  5410. {
    5411. 

  5411.     int arity;
    5412. 

  5412.     Export* ep;
    5413. 

  5413.     Eterm tmp, this;
    5414. 

  5414. 

  5415.     /*
    5416. 

  5416.      * Check the arguments which should be of the form apply(Module,
    5417. 

  5417.      * Function, Arguments) where Function is an atom and
    5418. 

  5418.      * Arguments is an arity long list of terms.
    5419. 

  5419.      */
    5420. 

  5420.     if (is_not_atom(function)) {
    5421. 

  5421. 	/*
    5422. 

  5422. 	 * No need to test args here -- done below.
    5423. 

  5423. 	 */
    5424. 

  5424.     error:
    5425. 

  5425. 	p->freason = BADARG;
    5426. 

  5426. 

  5427.     error2:
    5428. 

  5428. 	reg[0] = module;
    5429. 

  5429. 	reg[1] = function;
    5430. 

  5430. 	reg[2] = args;
    5431. 

  5431. 	return 0;
    5432. 

  5432.     }
    5433. 

  5433. 

  5434.     /* The module argument may be either an atom or an abstract module
    5435. 

  5435.      * (currently implemented using tuples, but this might change).
    5436. 

  5436.      */
    5437. 

  5437.     this = THE_NON_VALUE;
    5438. 

  5438.     if (is_not_atom(module)) {
    5439. 

  5439. 	Eterm* tp;
    5440. 

  5440. 

  5441.         if (is_not_tuple(module)) goto error;
    5442. 

  5442.         tp = tuple_val(module);
    5443. 

  5443.         if (arityval(tp[0]) < 1) goto error;
    5444. 

  5444.         this = module;
    5445. 

  5445.         module = tp[1];
    5446. 

  5446.         if (is_not_atom(module)) goto error;
    5447. 

  5447.     }
    5448. 

  5448.     
    5449. 

  5449.     /*
    5450. 

  5450.      * Walk down the 3rd parameter of apply (the argument list) and copy
    5451. 

  5451.      * the parameters to the x registers (reg[]). If the module argument
    5452. 

  5452.      * was an abstract module, add 1 to the function arity and put the
    5453. 

  5453.      * module argument in the n+1st x register as a THIS reference.
    5454. 

  5454.      */
    5455. 

  5455. 

  5456.     tmp = args;
    5457. 

  5457.     arity = 0;
    5458. 

  5458.     while (is_list(tmp)) {
    5459. 

  5459. 	if (arity < (MAX_REG - 1)) {
    5460. 

  5460. 	    reg[arity++] = CAR(list_val(tmp));
    5461. 

  5461. 	    tmp = CDR(list_val(tmp));
    5462. 

  5462. 	} else {
    5463. 

  5463. 	    p->freason = SYSTEM_LIMIT;
    5464. 

  5464. 	    goto error2;
    5465. 

  5465. 	}
    5466. 

  5466.     }
    5467. 

  5467.     if (is_not_nil(tmp)) {	/* Must be well-formed list */
    5468. 

  5468. 	goto error;
    5469. 

  5469.     }
    5470. 

  5470.     if (this != THE_NON_VALUE) {
    5471. 

  5471.         reg[arity++] = this;
    5472. 

  5472.     }
    5473. 

  5473. 

  5474.     /*
    5475. 

  5475.      * Get the index into the export table, or failing that the export
    5476. 

  5476.      * entry for the error handler.
    5477. 

  5477.      *
    5478. 

  5478.      * Note: All BIFs have export entries; thus, no special case is needed.
    5479. 

  5479.      */
    5480. 

  5480. 

  5481.     if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
    5482. 

  5482. 	if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error;
    5483. 

  5483.     } else if (p->ct != NULL) {
    5484. 

  5484. 	save_calls(p, ep);
    5485. 

  5485.     }
    5486. 

  5486. 

  5487.     return ep->address;
    5488. 

  5488. }
    5489. 

  5489. 

  5490. static Uint*
    5491. 

  5491. fixed_apply(Process* p, Eterm* reg, Uint arity)
    5492. 

  5492. {
    5493. 

  5493.     Export* ep;
    5494. 

  5494.     Eterm module;
    5495. 

  5495.     Eterm function;
    5496. 

  5496. 

  5497.     module = reg[arity];    /* The THIS pointer already in place */
    5498. 

  5498.     function = reg[arity+1];
    5499. 

  5499. 

  5500.     if (is_not_atom(function)) {
    5501. 

  5501.     error:
    5502. 

  5502. 	p->freason = BADARG;
    5503. 

  5503. 	reg[0] = module;
    5504. 

  5504. 	reg[1] = function;
    5505. 

  5505. 	reg[2] = NIL;
    5506. 

  5506. 	return 0;
    5507. 

  5507.     }
    5508. 

  5508. 

  5509.     /* The module argument may be either an atom or an abstract module
    5510. 

  5510.      * (currently implemented using tuples, but this might change).
    5511. 

  5511.      */
    5512. 

  5512.     if (is_not_atom(module)) {
    5513. 

  5513. 	Eterm* tp;
    5514. 

  5514.         if (is_not_tuple(module)) goto error;
    5515. 

  5515.         tp = tuple_val(module);
    5516. 

  5516.         if (arityval(tp[0]) < 1) goto error;
    5517. 

  5517.         module = tp[1];
    5518. 

  5518.         if (is_not_atom(module)) goto error;
    5519. 

  5519.         ++arity;
    5520. 

  5520.     }
    5521. 

  5521.     
    5522. 

  5522.     /*
    5523. 

  5523.      * Get the index into the export table, or failing that the export
    5524. 

  5524.      * entry for the error handler module.
    5525. 

  5525.      *
    5526. 

  5526.      * Note: All BIFs have export entries; thus, no special case is needed.
    5527. 

  5527.      */
    5528. 

  5528. 

  5529.     if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
    5530. 

  5530. 	if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL)
    5531. 

  5531. 	    goto error;
    5532. 

  5532.     } else if (p->ct != NULL) {
    5533. 

  5533. 	save_calls(p, ep);
    5534. 

  5534.     }
    5535. 

  5535. 

  5536.     return ep->address;
    5537. 

  5537. }
    5538. 

  5538. 

  5539. static int
    5540. 

  5540. hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
    5541. 

  5541. {
    5542. 

  5542.     int arity;
    5543. 

  5543.     Eterm tmp;
    5544. 

  5544. 

  5545.     if (is_not_atom(module) || is_not_atom(function)) {
    5546. 

  5546. 	/*
    5547. 

  5547. 	 * No need to test args here -- done below.
    5548. 

  5548. 	 */
    5549. 

  5549.     error:
    5550. 

  5550. 	c_p->freason = BADARG;
    5551. 

  5551. 

  5552.     error2:
    5553. 

  5553. 	reg[0] = module;
    5554. 

  5554. 	reg[1] = function;
    5555. 

  5555. 	reg[2] = args;
    5556. 

  5556. 	return 0;
    5557. 

  5557.     }
    5558. 

  5558. 

  5559.     arity = 0;
    5560. 

  5560.     tmp = args;
    5561. 

  5561.     while (is_list(tmp)) {
    5562. 

  5562. 	if (arity < MAX_REG) {
    5563. 

  5563. 	    tmp = CDR(list_val(tmp));
    5564. 

  5564. 	    arity++;
    5565. 

  5565. 	} else {
    5566. 

  5566. 	    c_p->freason = SYSTEM_LIMIT;
    5567. 

  5567. 	    goto error2;
    5568. 

  5568. 	}
    5569. 

  5569.     }
    5570. 

  5570.     if (is_not_nil(tmp)) {	/* Must be well-formed list */
    5571. 

  5571. 	goto error;
    5572. 

  5572.     }
    5573. 

  5573. 

  5574.     /*
    5575. 

  5575.      * At this point, arguments are known to be good.
    5576. 

  5576.      */
    5577. 

  5577. 

  5578.     if (c_p->arg_reg != c_p->def_arg_reg) {
    5579. 

  5579. 	/* Save some memory */
    5580. 

  5580. 	ERTS_PROC_LESS_MEM(c_p->max_arg_reg * sizeof(c_p->arg_reg[0]));
    5581. 

  5581. 	erts_free(ERTS_ALC_T_ARG_REG, c_p->arg_reg);
    5582. 

  5582. 	c_p->arg_reg = c_p->def_arg_reg;
    5583. 

  5583. 	c_p->max_arg_reg = sizeof(c_p->def_arg_reg)/sizeof(c_p->def_arg_reg[0]);
    5584. 

  5584.     }
    5585. 

  5585. 

  5586.     /*
    5587. 

  5587.      * Arrange for the process to be resumed at the given MFA with
    5588. 

  5588.      * the stack cleared.
    5589. 

  5589.      */
    5590. 

  5590.     c_p->arity = 3;
    5591. 

  5591.     c_p->arg_reg[0] = module;
    5592. 

  5592.     c_p->arg_reg[1] = function;
    5593. 

  5593.     c_p->arg_reg[2] = args;
    5594. 

  5594.     c_p->stop = STACK_START(c_p);
    5595. 

  5595.     c_p->catches = 0;
    5596. 

  5596.     c_p->i = beam_apply;
    5597. 

  5597.     c_p->cp = (Eterm *) beam_apply+1;
    5598. 

  5598. 

  5599.     /*
    5600. 

  5600.      * If there are no waiting messages, garbage collect and
    5601. 

  5601.      * shrink the heap. 
    5602. 

  5602.      */
    5603. 

  5603.     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
    5604. 

  5604.     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
    5605. 

  5605.     if (c_p->msg.len > 0) {
    5606. 

  5606. 	add_to_schedule_q(c_p);
    5607. 

  5607.     } else {
    5608. 

  5608. 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
    5609. 

  5609. 	c_p->fvalue = NIL;
    5610. 

  5610. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    5611. 

  5611. 	erts_garbage_collect_hibernate(c_p);
    5612. 

  5612. 	PROCESS_MAIN_CHK_LOCKS(c_p);
    5613. 

  5613. 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
    5614. 

  5614. 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    5615. 

  5615. 	c_p->status = P_WAITING;
    5616. 

  5616. #ifdef ERTS_SMP
    5617. 

  5617. 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
    5618. 

  5618. 	if (c_p->msg.len > 0)
    5619. 

  5619. 	    add_to_schedule_q(c_p);
    5620. 

  5620. #endif
    5621. 

  5621.     }
    5622. 

  5622.     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
    5623. 

  5623.     c_p->current = bif_export[BIF_hibernate_3]->code;
    5624. 

  5624.     return 1;
    5625. 

  5625. }
    5626. 

  5626. 

  5627. static Uint*
    5628. 

  5628. call_fun(Process* p,		/* Current process. */
    5629. 

  5629. 	 int arity,		/* Number of arguments for Fun. */
    5630. 

  5630. 	 Eterm* reg,		/* Contents of registers. */
    5631. 

  5631. 	 Eterm args)		/* THE_NON_VALUE or pre-built list of arguments. */
    5632. 

  5632. {
    5633. 

  5633.     Eterm fun = reg[arity];
    5634. 

  5634.     Eterm hdr;
    5635. 

  5635.     int i;
    5636. 

  5636.     Eterm function;
    5637. 

  5637.     Eterm* hp;
    5638. 

  5638. 

  5639.     if (!is_boxed(fun)) {
    5640. 

  5640. 	goto badfun;
    5641. 

  5641.     }
    5642. 

  5642.     hdr = *boxed_val(fun);
    5643. 

  5643. 

  5644.     if (is_fun_header(hdr)) {
    5645. 

  5645. 	ErlFunThing* funp = (ErlFunThing *) fun_val(fun);
    5646. 

  5646. 	ErlFunEntry* fe;
    5647. 

  5647. 	Eterm* code_ptr;
    5648. 

  5648. 	Eterm* var_ptr;
    5649. 

  5649. 	int actual_arity;
    5650. 

  5650. 	unsigned num_free;
    5651. 

  5651. 

  5652. 	fe = funp->fe;
    5653. 

  5653. 	num_free = funp->num_free;
    5654. 

  5654. 	code_ptr = fe->address;
    5655. 

  5655. 	actual_arity = (int) code_ptr[-1];
    5656. 

  5656. 

  5657. 	if (actual_arity == arity+num_free) {
    5658. 

  5658. 	    if (num_free == 0) {
    5659. 

  5659. 		return code_ptr;
    5660. 

  5660. 	    } else {
    5661. 

  5661. 		var_ptr = funp->env;
    5662. 

  5662. 		reg += arity;
    5663. 

  5663. 		i = 0;
    5664. 

  5664. 		do {
    5665. 

  5665. 		    reg[i] = var_ptr[i];
    5666. 

  5666. 		    i++;
    5667. 

  5667. 		} while (i < num_free);
    5668. 

  5668. 		reg[i] = fun;
    5669. 

  5669. 		return code_ptr;
    5670. 

  5670. 	    }
    5671. 

  5671. 	    return code_ptr;
    5672. 

  5672. 	} else {
    5673. 

  5673. 	    /*
    5674. 

  5674. 	     * Something wrong here. First build a list of the arguments.
    5675. 

  5675. 	     */  
    5676. 

  5676. 

  5677. 	    if (is_non_value(args)) {
    5678. 

  5678. 		Uint sz = 2 * arity;
    5679. 

  5679. 		args = NIL;
    5680. 

  5680. 		if (HeapWordsLeft(p) < sz) {
    5681. 

  5681. 		    erts_garbage_collect(p, sz, reg, arity+1);
    5682. 

  5682. 		    fun = reg[arity];
    5683. 

  5683. 		}
    5684. 

  5684. 		hp = HEAP_TOP(p);
    5685. 

  5685. 		HEAP_TOP(p) += sz;
    5686. 

  5686. 		for (i = arity-1; i >= 0; i--) {
    5687. 

  5687. 		    args = CONS(hp, reg[i], args);
    5688. 

  5688. 		    hp += 2;
    5689. 

  5689. 		}
    5690. 

  5690. 	    }
    5691. 

  5691. 

  5692. 	    if (actual_arity >= 0) {
    5693. 

  5693. 		/*
    5694. 

  5694. 		 * There is a fun defined, but the call has the wrong arity.
    5695. 

  5695. 		 */
    5696. 

  5696. 		hp = HAlloc(p, 3);
    5697. 

  5697. 		p->freason = EXC_BADARITY;
    5698. 

  5698. 		p->fvalue = TUPLE2(hp, fun, args);
    5699. 

  5699. 		return NULL;
    5700. 

  5700. 	    } else {
    5701. 

  5701. 		Export* ep;
    5702. 

  5702. 		Module* modp;
    5703. 

  5703. 		Eterm module;
    5704. 

  5704. 

  5705. 

  5706. 		/*
    5707. 

  5707. 		 * No arity. There is no module loaded that defines the fun,
    5708. 

  5708. 		 * either because the fun is newly created from the external
    5709. 

  5709. 		 * representation (the module has never been loaded),
    5710. 

  5710. 		 * or the module defining the fun has been unloaded.
    5711. 

  5711. 		 */
    5712. 

  5712. 

  5713. 		module = fe->module;
    5714. 

  5714. 		if ((modp = erts_get_module(module)) != NULL && modp->code != NULL) {
    5715. 

  5715. 		    /*
    5716. 

  5716. 		     * There is a module loaded, but obviously the fun is not
    5717. 

  5717. 		     * defined in it. We must not call the error_handler
    5718. 

  5718. 		     * (or we will get into an infinite loop).
    5719. 

  5719. 		     */
    5720. 

  5720. 		    goto badfun;
    5721. 

  5721. 		}
    5722. 

  5722. 		
    5723. 

  5723. 		/*
    5724. 

  5724. 		 * No current code for this module. Call the error_handler module
    5725. 

  5725. 		 * to attempt loading the module.
    5726. 

  5726. 		 */
    5727. 

  5727. 

  5728. 		ep = erts_find_function(p->error_handler, am_undefined_lambda, 3);
    5729. 

  5729. 		if (ep == NULL) {	/* No error handler */
    5730. 

  5730. 		    p->current = NULL;
    5731. 

  5731. 		    p->freason = EXC_UNDEF;
    5732. 

  5732. 		    return NULL;
    5733. 

  5733. 		}
    5734. 

  5734. 		reg[0] = module;
    5735. 

  5735. 		reg[1] = fun;
    5736. 

  5736. 		reg[2] = args;
    5737. 

  5737. 		return ep->address;
    5738. 

  5738. 	    }
    5739. 

  5739. 	}
    5740. 

  5740.     } else if (is_export_header(hdr)) {
    5741. 

  5741. 	Export* ep = (Export *) (export_val(fun))[1];
    5742. 

  5742. 	int actual_arity = (int) ep->code[2];
    5743. 

  5743. 	if (arity == actual_arity) {
    5744. 

  5744. 	    return ep->address;
    5745. 

  5745. 	} else {
    5746. 

  5746. 	    /*
    5747. 

  5747. 	     * Wrong arity. First build a list of the arguments.
    5748. 

  5748. 	     */  
    5749. 

  5749. 

  5750. 	    if (is_non_value(args)) {
    5751. 

  5751. 		args = NIL;
    5752. 

  5752. 		hp = HAlloc(p, arity*2);
    5753. 

  5753. 		for (i = arity-1; i >= 0; i--) {
    5754. 

  5754. 		    args = CONS(hp, reg[i], args);
    5755. 

  5755. 		    hp += 2;
    5756. 

  5756. 		}
    5757. 

  5757. 	    }
    5758. 

  5758. 

  5759. 	    hp = HAlloc(p, 3);
    5760. 

  5760. 	    p->freason = EXC_BADARITY;
    5761. 

  5761. 	    p->fvalue = TUPLE2(hp, fun, args);
    5762. 

  5762. 	    return NULL;
    5763. 

  5763. 	}
    5764. 

  5764.     } else if (hdr == make_arityval(2)) {
    5765. 

  5765. 	Eterm* tp;
    5766. 

  5766. 	Export* ep;
    5767. 

  5767. 	Eterm module;
    5768. 

  5768. 

  5769. 	tp = tuple_val(fun);
    5770. 

  5770. 	module = tp[1];
    5771. 

  5771. 	function = tp[2];
    5772. 

  5772. 	if (!is_atom(module) || !is_atom(function)) {
    5773. 

  5773. 	    goto badfun;
    5774. 

  5774. 	}
    5775. 

  5775. 	if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
    5776. 

  5776. 	    ep = erts_find_export_entry(p->error_handler, am_undefined_function, 3);
    5777. 

  5777. 	    if (ep == NULL) {
    5778. 

  5778. 		p->freason = EXC_UNDEF;
    5779. 

  5779. 		return 0;
    5780. 

  5780. 	    }
    5781. 

  5781. 	    if (is_non_value(args)) {
    5782. 

  5782. 		Uint sz = 2 * arity;
    5783. 

  5783. 		if (HeapWordsLeft(p) < sz) {
    5784. 

  5784. 		    erts_garbage_collect(p, sz, reg, arity);
    5785. 

  5785. 		}
    5786. 

  5786. 		hp = HEAP_TOP(p);
    5787. 

  5787. 		HEAP_TOP(p) += sz;
    5788. 

  5788. 		args = NIL;
    5789. 

  5789. 		while (arity-- > 0) {
    5790. 

  5790. 		    args = CONS(hp, reg[arity], args);
    5791. 

  5791. 		    hp += 2;
    5792. 

  5792. 		}
    5793. 

  5793. 	    }
    5794. 

  5794. 	    reg[0] = module;
    5795. 

  5795. 	    reg[1] = function;
    5796. 

  5796. 	    reg[2] = args;
    5797. 

  5797. 	}
    5798. 

  5798. 	return ep->address;
    5799. 

  5799.     } else {
    5800. 

  5800.     badfun:
    5801. 

  5801. 	p->current = NULL;
    5802. 

  5802. 	p->freason = EXC_BADFUN;
    5803. 

  5803. 	p->fvalue = fun;
    5804. 

  5804. 	return NULL;
    5805. 

  5805.     }
    5806. 

  5806. }
    5807. 

  5807. 

  5808. static Eterm*
    5809. 

  5809. apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg)
    5810. 

  5810. {
    5811. 

  5811.     int arity;
    5812. 

  5812.     Eterm tmp;
    5813. 

  5813. 

  5814.     /*
    5815. 

  5815.      * Walk down the 3rd parameter of apply (the argument list) and copy
    5816. 

  5816.      * the parameters to the x registers (reg[]).
    5817. 

  5817.      */
    5818. 

  5818. 

  5819.     tmp = args;
    5820. 

  5820.     arity = 0;
    5821. 

  5821.     while (is_list(tmp)) {
    5822. 

  5822. 	if (arity < MAX_REG-1) {
    5823. 

  5823. 	    reg[arity++] = CAR(list_val(tmp));
    5824. 

  5824. 	    tmp = CDR(list_val(tmp));
    5825. 

  5825. 	} else {
    5826. 

  5826. 	    p->freason = SYSTEM_LIMIT;
    5827. 

  5827. 	    return NULL;
    5828. 

  5828. 	}
    5829. 

  5829.     }
    5830. 

  5830. 

  5831.     if (is_not_nil(tmp)) {	/* Must be well-formed list */
    5832. 

  5832. 	p->freason = EXC_UNDEF;
    5833. 

  5833. 	return NULL;
    5834. 

  5834.     }
    5835. 

  5835.     reg[arity] = fun;
    5836. 

  5836.     return call_fun(p, arity, reg, args);
    5837. 

  5837. }
    5838. 

  5838. 

  5839. 

  5840. static Eterm
    5841. 

  5841. new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
    5842. 

  5842. {
    5843. 

  5843.     unsigned needed = ERL_FUN_SIZE + num_free;
    5844. 

  5844.     ErlFunThing* funp;
    5845. 

  5845.     Eterm* hp;
    5846. 

  5846.     int i;
    5847. 

  5847. 

  5848.     if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) {
    5849. 

  5849. 	PROCESS_MAIN_CHK_LOCKS(p);
    5850. 

  5850. 	erts_garbage_collect(p, needed, reg, num_free);
    5851. 

  5851. 	PROCESS_MAIN_CHK_LOCKS(p);
    5852. 

  5852.     }
    5853. 

  5853.     hp = p->htop;
    5854. 

  5854.     p->htop = hp + needed;
    5855. 

  5855.     funp = (ErlFunThing *) hp;
    5856. 

  5856.     hp = funp->env;
    5857. 

  5857.     erts_refc_inc(&fe->refc, 2);
    5858. 

  5858.     funp->thing_word = HEADER_FUN;
    5859. 

  5859. #ifndef HYBRID /* FIND ME! */
    5860. 

  5860.     funp->next = MSO(p).funs;
    5861. 

  5861.     MSO(p).funs = funp;
    5862. 

  5862. #endif
    5863. 

  5863.     funp->fe = fe;
    5864. 

  5864.     funp->num_free = num_free;
    5865. 

  5865.     funp->creator = p->id;
    5866. 

  5866. #ifdef HIPE
    5867. 

  5867.     funp->native_address = fe->native_address;
    5868. 

  5868. #endif
    5869. 

  5869.     funp->arity = (int)fe->address[-1] - num_free;
    5870. 

  5870.     for (i = 0; i < num_free; i++) {
    5871. 

  5871. 	*hp++ = reg[i];
    5872. 

  5872.     }
    5873. 

  5873.     return make_fun(funp);
    5874. 

  5874. }
    5875. 

  5875. 

  5876. 
    5877. 

  5877. 

  5878. int catchlevel(Process *p)
    5879. 

  5879. {
    5880. 

  5880.     return p->catches;
    5881. 

  5881. }
    5882. 

  5882. 

  5883. /*
    5884. 

  5884.  * Check if the given function is built-in (i.e. a BIF implemented in C).
    5885. 

  5885.  *
    5886. 

  5886.  * Returns 0 if not built-in, and a non-zero value if built-in.
    5887. 

  5887.  */
    5888. 

  5888. 

  5889. int
    5890. 

  5890. erts_is_builtin(Eterm Mod, Eterm Name, int arity)
    5891. 

  5891. {
    5892. 

  5892.     Export e;
    5893. 

  5893.     Export* ep;
    5894. 

  5894. 

  5895.     e.code[0] = Mod;
    5896. 

  5896.     e.code[1] = Name;
    5897. 

  5897.     e.code[2] = arity;
    5898. 

  5898. 

  5899.     if ((ep = export_get(&e)) == NULL) {
    5900. 

  5900. 	return 0;
    5901. 

  5901.     }
    5902. 

  5902.     return ep->address == ep->code+3 && (ep->code[3] == (Uint) em_apply_bif);
    5903. 

  5903. }
    5904. 

  5904. 

  5905. 

  5906. /*
    5907. 

  5907.  * Return the current number of reductions for the given process.
    5908. 

  5908.  * To get the total number of reductions, p->reds must be added.
    5909. 

  5909.  */
    5910. 

  5910. 

  5911. Uint
    5912. 

  5912. erts_current_reductions(Process *current, Process *p)
    5913. 

  5913. {
    5914. 

  5914.     if (current != p) {
    5915. 

  5915. 	return 0;
    5916. 

  5916.     } else if (current->fcalls < 0 && current->ct != NULL) {
    5917. 

  5917. 	return -current->fcalls;
    5918. 

  5918.     } else {
    5919. 

  5919. 	return REDS_IN(current) - current->fcalls;
    5920. 

  5920.     }
    5921. 

  5921. }
    5922. 

  5922.