PageRenderTime 256ms CodeModel.GetById 50ms app.highlight 96ms RepoModel.GetById 15ms app.codeStats 28ms

/erts/emulator/beam/beam_emu.c

https://github.com/alricb/Erlang-OTP
C | 6512 lines | 4822 code | 725 blank | 965 comment | 897 complexity | 3402c782823432675709c3c74182efd8 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/*
   2 * %CopyrightBegin%
   3 *
   4 * Copyright Ericsson AB 1996-2011. All Rights Reserved.
   5 *
   6 * The contents of this file are subject to the Erlang Public License,
   7 * Version 1.1, (the "License"); you may not use this file except in
   8 * compliance with the License. You should have received a copy of the
   9 * Erlang Public License along with this software. If not, it can be
  10 * retrieved online at http://www.erlang.org/.
  11 *
  12 * Software distributed under the License is distributed on an "AS IS"
  13 * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  14 * the License for the specific language governing rights and limitations
  15 * under the License.
  16 *
  17 * %CopyrightEnd%
  18 */
  19
  20#ifdef HAVE_CONFIG_H
  21#  include "config.h"
  22#endif
  23
  24#include <stddef.h> /* offsetof() */
  25#include "sys.h"
  26#include "erl_vm.h"
  27#include "global.h"
  28#include "erl_process.h"
  29#include "erl_nmgc.h"
  30#include "error.h"
  31#include "bif.h"
  32#include "big.h"
  33#include "beam_load.h"
  34#include "erl_binary.h"
  35#include "erl_bits.h"
  36#include "dist.h"
  37#include "beam_bp.h"
  38#include "beam_catches.h"
  39#ifdef HIPE
  40#include "hipe_mode_switch.h"
  41#include "hipe_bif1.h"
  42#endif
  43
  44/* #define HARDDEBUG 1 */
  45
  46#if defined(NO_JUMP_TABLE)
  47#  define OpCase(OpCode)    case op_##OpCode: lb_##OpCode
  48#  define CountCase(OpCode) case op_count_##OpCode
  49#  define OpCode(OpCode)    ((Uint*)op_##OpCode)
  50#  define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;}
  51#  define LabelAddr(Addr) &&##Addr
  52#else
  53#  define OpCase(OpCode)    lb_##OpCode
  54#  define CountCase(OpCode) lb_count_##OpCode
  55#  define Goto(Rel) goto *(Rel)
  56#  define LabelAddr(Label) &&Label
  57#  define OpCode(OpCode)  (&&lb_##OpCode)
  58#endif
  59
  60#ifdef ERTS_ENABLE_LOCK_CHECK
  61#  ifdef ERTS_SMP
  62#    define PROCESS_MAIN_CHK_LOCKS(P)					\
  63do {									\
  64    if ((P)) {								\
  65	erts_pix_lock_t *pix_lock__ = ERTS_PIX2PIXLOCK(internal_pid_index((P)->id));\
  66	erts_proc_lc_chk_only_proc_main((P));				\
  67	erts_pix_lock(pix_lock__);					\
  68	ASSERT(0 < (P)->lock.refc && (P)->lock.refc < erts_no_schedulers*5);\
  69	erts_pix_unlock(pix_lock__);					\
  70    }									\
  71    else								\
  72	erts_lc_check_exact(NULL, 0);					\
  73    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);				\
  74} while (0)
  75#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
  76        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
  77#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
  78        if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
  79#  else
  80#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
  81#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
  82#    define PROCESS_MAIN_CHK_LOCKS(P) erts_lc_check_exact(NULL, 0)
  83#  endif
  84#else
  85#  define PROCESS_MAIN_CHK_LOCKS(P)
  86#  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
  87#  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
  88#endif
  89
  90/*
  91 * Define macros for deep checking of terms.
  92 */
  93
  94#if defined(HARDDEBUG)
  95
  96#  define CHECK_TERM(T) size_object(T)
  97
  98#  define CHECK_ARGS(PC)                 \
  99do {                                     \
 100  int i_;                                \
 101  int Arity_ = PC[-1];                   \
 102  if (Arity_ > 0) {                      \
 103	CHECK_TERM(r(0));                \
 104  }                                      \
 105  for (i_ = 1; i_ < Arity_; i_++) {      \
 106	CHECK_TERM(x(i_));               \
 107  }                                      \
 108} while (0)
 109    
 110#else
 111#  define CHECK_TERM(T) ASSERT(!is_CP(T))
 112#  define CHECK_ARGS(T)
 113#endif
 114
 115#ifndef MAX
 116#define MAX(x, y) (((x) > (y)) ? (x) : (y))
 117#endif
 118
 119#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
 120#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 121
 122
 123/*
 124 * We reuse some of fields in the save area in the process structure.
 125 * This is safe to do, since this space is only activly used when
 126 * the process is switched out.
 127 */
 128#define REDS_IN(p)  ((p)->def_arg_reg[5])
 129
 130/*
 131 * Add a byte offset to a pointer to Eterm.  This is useful when the
 132 * the loader has precalculated a byte offset.
 133 */
 134#define ADD_BYTE_OFFSET(ptr, offset) \
 135   ((Eterm *) (((unsigned char *)ptr) + (offset)))
 136
 137/* We don't check the range if an ordinary switch is used */
 138#ifdef NO_JUMP_TABLE
 139#define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10)))
 140#else
 141#define VALID_INSTR(IP) \
 142   ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
 143    (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop))
 144#endif /* NO_JUMP_TABLE */
 145
 146#define SET_CP(p, ip)           \
 147   ASSERT(VALID_INSTR(*(ip)));  \
 148   (p)->cp = (ip)
 149
 150#define SET_I(ip) \
 151   ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
 152   I = (ip)
 153
 154#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2)
 155
 156/*
 157 * Store a result into a register given a destination descriptor.
 158 */
 159
 160#define StoreResult(Result, DestDesc)               \
 161  do {                                              \
 162    Eterm stb_reg;                                  \
 163    stb_reg = (DestDesc);                           \
 164    CHECK_TERM(Result);                             \
 165    switch (beam_reg_tag(stb_reg)) {                \
 166    case R_REG_DEF:                                 \
 167      r(0) = (Result); break;                       \
 168    case X_REG_DEF:                                 \
 169      xb(x_reg_offset(stb_reg)) = (Result); break;  \
 170    default:                                        \
 171      yb(y_reg_offset(stb_reg)) = (Result); break;  \
 172    }                                               \
 173  } while (0)
 174
 175#define StoreSimpleDest(Src, Dest) Dest = (Src)
 176
 177/*
 178 * Store a result into a register and execute the next instruction.
 179 * Dst points to the word with a destination descriptor, which MUST
 180 * be just before the next instruction.
 181 */
 182 
 183#define StoreBifResult(Dst, Result)                          \
 184  do {                                                       \
 185    BeamInstr* stb_next;                                         \
 186    Eterm stb_reg;                                           \
 187    stb_reg = Arg(Dst);                                      \
 188    I += (Dst) + 2;                                          \
 189    stb_next = (BeamInstr *) *I;                                 \
 190    CHECK_TERM(Result);                                      \
 191    switch (beam_reg_tag(stb_reg)) {                         \
 192    case R_REG_DEF:                                          \
 193      r(0) = (Result); Goto(stb_next);                       \
 194    case X_REG_DEF:                                          \
 195      xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
 196    default:                                                 \
 197      yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
 198    }                                                        \
 199  } while (0)
 200
 201#define ClauseFail() goto lb_jump_f
 202
 203#define SAVE_CP(X)				\
 204   do {						\
 205      *(X) = make_cp(c_p->cp);			\
 206      c_p->cp = 0;				\
 207   } while(0)
 208
 209#define RESTORE_CP(X)		SET_CP(c_p, (BeamInstr *) cp_val(*(X)))
 210
 211#define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
 212
 213/*
 214 * Special Beam instructions.
 215 */
 216
 217BeamInstr beam_apply[2];
 218BeamInstr beam_exit[1];
 219BeamInstr beam_continue_exit[1];
 220
 221BeamInstr* em_call_error_handler;
 222BeamInstr* em_apply_bif;
 223BeamInstr* em_call_traced_function;
 224
 225
 226/* NOTE These should be the only variables containing trace instructions.
 227**      Sometimes tests are form the instruction value, and sometimes
 228**      for the refering variable (one of these), and rouge references
 229**      will most likely cause chaos.
 230*/
 231BeamInstr beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
 232BeamInstr beam_return_trace[1];      /* OpCode(i_return_trace) */
 233BeamInstr beam_exception_trace[1];   /* UGLY also OpCode(i_return_trace) */
 234BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 235
 236/*
 237 * All Beam instructions in numerical order.
 238 */
 239
 240#ifndef NO_JUMP_TABLE
 241void** beam_ops;
 242#endif
 243
 244#ifndef ERTS_SMP /* Not supported with smp emulator */
 245extern int count_instructions;
 246#endif
 247
 248#if defined(HYBRID)
 249#define SWAPIN             \
 250    g_htop = global_htop;  \
 251    g_hend = global_hend;  \
 252    HTOP = HEAP_TOP(c_p);  \
 253    E = c_p->stop
 254
 255#define SWAPOUT            \
 256    global_htop = g_htop;  \
 257    global_hend = g_hend;  \
 258    HEAP_TOP(c_p) = HTOP;  \
 259    c_p->stop = E
 260
 261#else
 262#define SWAPIN             \
 263    HTOP = HEAP_TOP(c_p);  \
 264    E = c_p->stop
 265
 266#define SWAPOUT            \
 267    HEAP_TOP(c_p) = HTOP;  \
 268    c_p->stop = E
 269
 270/*
 271 * Use LIGHT_SWAPOUT when the called function
 272 * will call HeapOnlyAlloc() (and never HAlloc()).
 273 */
 274#ifdef DEBUG
 275#  /* The stack pointer is used in an assertion. */
 276#  define LIGHT_SWAPOUT SWAPOUT
 277#else
 278#  define LIGHT_SWAPOUT HEAP_TOP(c_p) = HTOP
 279#endif
 280
 281/*
 282 * Use LIGHT_SWAPIN when we know that c_p->stop cannot
 283 * have been updated (i.e. if there cannot have been
 284 * a garbage-collection).
 285 */
 286
 287#define LIGHT_SWAPIN HTOP = HEAP_TOP(c_p)
 288
 289#endif
 290
 291#ifdef FORCE_HEAP_FRAGS
 292#  define HEAP_SPACE_VERIFIED(Words) do { \
 293      c_p->space_verified = (Words);	  \
 294      c_p->space_verified_from = HTOP;	  \
 295    }while(0)
 296#else
 297#  define HEAP_SPACE_VERIFIED(Words) ((void)0)
 298#endif
 299
 300#define PRE_BIF_SWAPOUT(P)						\
 301     HEAP_TOP((P)) = HTOP;  						\
 302     (P)->stop = E;  							\
 303     PROCESS_MAIN_CHK_LOCKS((P));					\
 304     ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P))
 305
 306#if defined(HYBRID)
 307#  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
 308     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
 309       _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
 310     }									\
 311     SWAPIN
 312
 313#  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
 314     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
 315       _regs[0] = r(0);							\
 316       _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
 317       r(0) = _regs[0];							\
 318     }									\
 319     SWAPIN
 320#else
 321#  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
 322     ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
 323     PROCESS_MAIN_CHK_LOCKS((_p));					\
 324     ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p));				\
 325     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
 326       _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
 327       E = (_p)->stop;							\
 328     }									\
 329     HTOP = HEAP_TOP((_p))
 330
 331#  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
 332     ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p));				\
 333     ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
 334     PROCESS_MAIN_CHK_LOCKS((_p));					\
 335     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
 336       _regs[0] = r(0);							\
 337       _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
 338       r(0) = _regs[0];							\
 339       E = (_p)->stop;							\
 340     }									\
 341     HTOP = HEAP_TOP((_p))
 342#endif
 343
 344#define db(N) (N)
 345#define tb(N) (N)
 346#define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 347#define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
 348#define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
 349#define Qb(N) (N)
 350#define Ib(N) (N)
 351#define x(N) reg[N]
 352#define y(N) E[N]
 353#define r(N) x##N
 354
 355/*
 356 * Makes sure that there are StackNeed + HeapNeed + 1 words available
 357 * on the combined heap/stack segment, then allocates StackNeed + 1
 358 * words on the stack and saves CP.
 359 *
 360 * M is number of live registers to preserve during garbage collection
 361 */
 362
 363#define AH(StackNeed, HeapNeed, M) \
 364  do { \
 365     int needed; \
 366     needed = (StackNeed) + 1; \
 367     if (E - HTOP < (needed + (HeapNeed))) { \
 368           SWAPOUT; \
 369           reg[0] = r(0); \
 370           PROCESS_MAIN_CHK_LOCKS(c_p); \
 371           FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
 372           ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
 373           PROCESS_MAIN_CHK_LOCKS(c_p); \
 374           r(0) = reg[0]; \
 375           SWAPIN; \
 376     } \
 377     E -= needed; \
 378     SAVE_CP(E); \
 379  } while (0)
 380
 381#define Allocate(Ns, Live) AH(Ns, 0, Live)
 382
 383#define AllocateZero(Ns, Live)             \
 384 do { Eterm* ptr;                          \
 385      int i = (Ns);                        \
 386      AH(i, 0, Live);                      \
 387      for (ptr = E + i; ptr > E; ptr--) {  \
 388	 make_blank(*ptr);                 \
 389     }                                     \
 390  } while (0)
 391
 392#define AllocateHeap(Ns, Nh, Live) AH(Ns, Nh, Live)
 393
 394#define AllocateHeapZero(Ns, Nh, Live)     \
 395 do { Eterm* ptr;                          \
 396      int i = (Ns);                        \
 397      AH(i, Nh, Live);                     \
 398      for (ptr = E + i; ptr > E; ptr--) {  \
 399	 make_blank(*ptr);                 \
 400     }                                     \
 401  } while (0)
 402
 403#define AllocateInit(Ns, Live, Y) \
 404  do { AH(Ns, 0, Live); make_blank(Y); } while (0)
 405
 406/*
 407 * Like the AH macro, but allocates no additional heap space.
 408 */
 409
 410#define A(StackNeed, M) AH(StackNeed, 0, M)
 411
 412#define D(N)             \
 413     RESTORE_CP(E);      \
 414     E += (N) + 1;
 415
 416
 417
 418#define TestBinVHeap(VNh, Nh, Live)                             		\
 419  do {                                                          		\
 420    unsigned need = (Nh);                                       		\
 421    if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
 422       SWAPOUT;                                                 		\
 423       reg[0] = r(0);                                           		\
 424       PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
 425       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
 426       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
 427       PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
 428       r(0) = reg[0];                                           		\
 429       SWAPIN;                                                  		\
 430    }                                                           		\
 431    HEAP_SPACE_VERIFIED(need);                                                  \
 432  } while (0)
 433
 434
 435
 436/*
 437 * Check if Nh words of heap are available; if not, do a garbage collection.
 438 * Live is number of active argument registers to be preserved.
 439 */
 440
 441#define TestHeap(Nh, Live)                                      \
 442  do {                                                          \
 443    unsigned need = (Nh);                                       \
 444    if (E - HTOP < need) {                                      \
 445       SWAPOUT;                                                 \
 446       reg[0] = r(0);                                           \
 447       PROCESS_MAIN_CHK_LOCKS(c_p);                             \
 448       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
 449       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
 450       PROCESS_MAIN_CHK_LOCKS(c_p);                             \
 451       r(0) = reg[0];                                           \
 452       SWAPIN;                                                  \
 453    }                                                           \
 454    HEAP_SPACE_VERIFIED(need);                             \
 455  } while (0)
 456
 457/*
 458 * Check if Nh words of heap are available; if not, do a garbage collection.
 459 * Live is number of active argument registers to be preserved.
 460 * Takes special care to preserve Extra if a garbage collection occurs.
 461 */
 462
 463#define TestHeapPreserve(Nh, Live, Extra)				\
 464  do {									\
 465    unsigned need = (Nh);						\
 466    if (E - HTOP < need) {						\
 467       SWAPOUT;								\
 468       reg[0] = r(0);							\
 469       reg[Live] = Extra;						\
 470       PROCESS_MAIN_CHK_LOCKS(c_p);					\
 471       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
 472       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
 473       PROCESS_MAIN_CHK_LOCKS(c_p);					\
 474       if (Live > 0) {							\
 475	   r(0) = reg[0];						\
 476       }								\
 477       Extra = reg[Live];						\
 478       SWAPIN;								\
 479    }									\
 480    HEAP_SPACE_VERIFIED(need);                                      \
 481  } while (0)
 482
 483#define TestHeapPutList(Need, Reg)		\
 484  do {						\
 485     TestHeap((Need), 1);			\
 486     PutList(Reg, r(0), r(0), StoreSimpleDest);	\
 487     CHECK_TERM(r(0));				\
 488  } while (0)
 489
 490#ifdef HYBRID
 491#ifdef INCREMENTAL
 492#define TestGlobalHeap(Nh, Live, hp)                                    \
 493  do {                                                                  \
 494    unsigned need = (Nh);                                               \
 495    ASSERT(global_heap <= g_htop && g_htop <= global_hend);             \
 496    SWAPOUT;                                                            \
 497    reg[0] = r(0);                                                      \
 498    FCALLS -= need;                                                     \
 499    (hp) = IncAlloc(c_p,need,reg,(Live));                               \
 500    r(0) = reg[0];                                                      \
 501    SWAPIN;                                                             \
 502  } while (0)
 503#else
 504#define TestGlobalHeap(Nh, Live, hp)                                    \
 505  do {                                                                  \
 506    unsigned need = (Nh);                                               \
 507    ASSERT(global_heap <= g_htop && g_htop <= global_hend);             \
 508    if (g_hend - g_htop < need) {                                       \
 509       SWAPOUT;                                                         \
 510       reg[0] = r(0);                                                   \
 511       FCALLS -= erts_global_garbage_collect(c_p, need, reg, (Live));   \
 512       r(0) = reg[0];                                                   \
 513       SWAPIN;                                                          \
 514    }                                                                   \
 515    (hp) = global_htop;                                                 \
 516  } while (0)
 517#endif
 518#endif /* HYBRID */
 519
 520#define Init(N) make_blank(yb(N))
 521
 522#define Init2(Y1, Y2) do { make_blank(Y1); make_blank(Y2); } while (0)
 523#define Init3(Y1, Y2, Y3) \
 524   do { make_blank(Y1); make_blank(Y2); make_blank(Y3); } while (0)
 525
 526#define MakeFun(FunP, NumFree)					\
 527  do {								\
 528     SWAPOUT;							\
 529     reg[0] = r(0);						\
 530     r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
 531     SWAPIN;							\
 532  } while (0)
 533
 534#define PutTuple(Dst, Arity)			\
 535 do {						\
 536   Dst = make_tuple(HTOP);			\
 537   pt_arity = (Arity);				\
 538 } while (0)
 539
 540/*
 541 * Check that we haven't used the reductions and jump to function pointed to by
 542 * the I register.  If we are out of reductions, do a context switch.
 543 */
 544
 545#define DispatchMacro()				\
 546  do {						\
 547     BeamInstr* dis_next;				\
 548     dis_next = (BeamInstr *) *I;			\
 549     CHECK_ARGS(I);				\
 550     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
 551        FCALLS--;				\
 552        Goto(dis_next);				\
 553     } else {					\
 554	goto context_switch;			\
 555     }						\
 556 } while (0)
 557
 558#define DispatchMacroFun()			\
 559  do {						\
 560     BeamInstr* dis_next;				\
 561     dis_next = (BeamInstr *) *I;			\
 562     CHECK_ARGS(I);				\
 563     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
 564        FCALLS--;				\
 565        Goto(dis_next);				\
 566     } else {					\
 567	goto context_switch_fun;		\
 568     }						\
 569 } while (0)
 570
 571#define DispatchMacrox()					\
 572  do {								\
 573     if (FCALLS > 0) {						\
 574        Eterm* dis_next;					\
 575        SET_I(((Export *) Arg(0))->address);			\
 576        dis_next = (Eterm *) *I;				\
 577        FCALLS--;						\
 578        CHECK_ARGS(I);						\
 579        Goto(dis_next);						\
 580     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)		\
 581		&& FCALLS > neg_o_reds) {			\
 582        goto save_calls1;					\
 583     } else {							\
 584        SET_I(((Export *) Arg(0))->address);			\
 585        CHECK_ARGS(I);						\
 586	goto context_switch;					\
 587     }								\
 588 } while (0)
 589
 590#ifdef DEBUG
 591/*
 592 * To simplify breakpoint setting, put the code in one place only and jump to it.
 593 */
 594#  define Dispatch() goto do_dispatch
 595#  define Dispatchx() goto do_dispatchx
 596#  define Dispatchfun() goto do_dispatchfun
 597#else
 598/*
 599 * Inline for speed.
 600 */
 601#  define Dispatch() DispatchMacro()
 602#  define Dispatchx() DispatchMacrox()
 603#  define Dispatchfun() DispatchMacroFun()
 604#endif
 605
 606#define Self(R) R = c_p->id
 607#define Node(R) R = erts_this_node->sysname
 608
 609#define Arg(N)       I[(N)+1]
 610#define Next(N)                \
 611    I += (N) + 1;              \
 612    ASSERT(VALID_INSTR(*I));   \
 613    Goto(*I)
 614
 615#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0)
 616#define NextPF(N, Dst)         \
 617    I += N + 1;                \
 618    ASSERT(VALID_INSTR(Dst));  \
 619    Goto(Dst)
 620
 621#define GetR(pos, tr) \
 622   do { \
 623     tr = Arg(pos); \
 624     switch (beam_reg_tag(tr)) { \
 625     case R_REG_DEF: tr = r(0); break; \
 626     case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \
 627     case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \
 628     } \
 629     CHECK_TERM(tr); \
 630   } while (0)
 631
 632#define GetArg1(N, Dst) GetR((N), Dst)
 633
 634#define GetArg2(N, Dst1, Dst2)     \
 635   do {                            \
 636     GetR(N, Dst1);                \
 637     GetR((N)+1, Dst2);            \
 638   } while (0)
 639
 640#define PutList(H, T, Dst, Store)  \
 641  do {                             \
 642   HTOP[0] = (H); HTOP[1] = (T);   \
 643   Store(make_list(HTOP), Dst);    \
 644   HTOP += 2;                      \
 645  } while (0)
 646
 647#define Move(Src, Dst, Store)      \
 648   do {                            \
 649       Eterm term = (Src);         \
 650       Store(term, Dst);           \
 651   } while (0)
 652
 653#define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2)
 654
 655#define MoveGenDest(src, dstp) \
 656   if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; }
 657
 658#define MoveReturn(Src, Dest)       \
 659    (Dest) = (Src);                 \
 660    I = c_p->cp;                    \
 661    ASSERT(VALID_INSTR(*c_p->cp));  \
 662    c_p->cp = 0;                    \
 663    CHECK_TERM(r(0));               \
 664    Goto(*I)
 665
 666#define DeallocateReturn(Deallocate)       \
 667  do {                                     \
 668    int words_to_pop = (Deallocate);       \
 669    SET_I((BeamInstr *) cp_val(*E));                     \
 670    E = ADD_BYTE_OFFSET(E, words_to_pop);  \
 671    CHECK_TERM(r(0));                      \
 672    Goto(*I);                              \
 673  } while (0)
 674
 675#define MoveDeallocateReturn(Src, Dest, Deallocate)  \
 676    (Dest) = (Src);                                  \
 677    DeallocateReturn(Deallocate)
 678
 679#define MoveCall(Src, Dest, CallDest, Size)	\
 680    (Dest) = (Src);				\
 681    SET_CP(c_p, I+Size+1);			\
 682    SET_I((BeamInstr *) CallDest);			\
 683    Dispatch();
 684
 685#define MoveCallLast(Src, Dest, CallDest, Deallocate)	\
 686    (Dest) = (Src);					\
 687    RESTORE_CP(E);					\
 688    E = ADD_BYTE_OFFSET(E, (Deallocate));		\
 689    SET_I((BeamInstr *) CallDest);				\
 690    Dispatch();
 691
 692#define MoveCallOnly(Src, Dest, CallDest)	\
 693    (Dest) = (Src);				\
 694    SET_I((BeamInstr *) CallDest);			\
 695    Dispatch();
 696
 697#define MoveJump(Src)				\
 698     r(0) = (Src);				\
 699     SET_I((BeamInstr *) Arg(0));		\
 700     Goto(*I);
 701
 702#define GetList(Src, H, T) do {			\
 703   Eterm* tmp_ptr = list_val(Src);		\
 704   H = CAR(tmp_ptr);				\
 705   T = CDR(tmp_ptr); } while (0)
 706
 707#define GetTupleElement(Src, Element, Dest)				\
 708  do {									\
 709    tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + 	\
 710				(Element));				\
 711    (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1));			\
 712  } while (0)
 713
 714#define ExtractNextElement(Dest)					  \
 715    tmp_arg1 += sizeof(Eterm);						  \
 716    (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1))))
 717
 718#define ExtractNextElement2(Dest)				\
 719  do {								\
 720    Eterm* ene_dstp = &(Dest);					\
 721    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
 722    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
 723    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);			\
 724  } while (0)
 725
 726#define ExtractNextElement3(Dest)		\
 727  do {						\
 728    Eterm* ene_dstp = &(Dest);			\
 729    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
 730    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
 731    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
 732    tmp_arg1 += 3*sizeof(Eterm);		\
 733  } while (0)
 734
 735#define ExtractNextElement4(Dest)		\
 736  do {						\
 737    Eterm* ene_dstp = &(Dest);			\
 738    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
 739    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
 740    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
 741    ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4];	\
 742    tmp_arg1 += 4*sizeof(Eterm);		\
 743  } while (0)
 744
 745#define ExtractElement(Element, Dest)		\
 746  do {						\
 747     tmp_arg1 += (Element);			\
 748     (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1));		\
 749  } while (0)
 750
 751#define EqualImmed(X, Y, Action) if (X != Y) { Action; }
 752#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
 753
 754#define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
 755
 756#define IsInteger(Src, Fail) if (is_not_integer(Src)) { Fail; }
 757
 758#define IsNumber(X, Fail) if (is_not_integer(X) && is_not_float(X)) { Fail; }
 759
 760#define IsAtom(Src, Fail) if (is_not_atom(Src)) { Fail; }
 761
 762#define IsIntegerAllocate(Src, Need, Alive, Fail)  \
 763    if (is_not_integer(Src)) { Fail; }             \
 764    A(Need, Alive)
 765
 766#define IsNil(Src, Fail) if (is_not_nil(Src)) { Fail; }
 767
 768#define IsList(Src, Fail) if (is_not_list(Src) && is_not_nil(Src)) { Fail; }
 769
 770#define IsNonemptyList(Src, Fail) if (is_not_list(Src)) { Fail; }
 771
 772#define IsNonemptyListAllocate(Src, Need, Alive, Fail)  \
 773    if (is_not_list(Src)) { Fail; }                     \
 774    A(Need, Alive)
 775
 776#define IsNonemptyListTestHeap(Src, Need, Alive, Fail)  \
 777    if (is_not_list(Src)) { Fail; }                     \
 778    TestHeap(Need, Alive)
 779
 780#define IsTuple(X, Action) if (is_not_tuple(X)) Action
 781
 782#define IsArity(Pointer, Arity, Fail)					  \
 783    if (*(Eterm *)							  \
 784	EXPAND_POINTER(tmp_arg1 = (Eterm) 				  \
 785		       COMPRESS_POINTER(tuple_val(Pointer))) != (Arity))  \
 786    { 									  \
 787        Fail; 								  \
 788    }
 789
 790#define IsFunction(X, Action)			\
 791  do {						\
 792     if ( !(is_any_fun(X)) ) {			\
 793          Action;				\
 794     }						\
 795  } while (0)
 796
 797#define IsFunction2(F, A, Action)		\
 798  do {						\
 799     if (is_function_2(c_p, F, A) != am_true ) {\
 800          Action;				\
 801     }						\
 802  } while (0)
 803
 804#define IsTupleOfArity(Src, Arity, Fail)				      \
 805  do {									      \
 806    if (is_not_tuple(Src) || 						      \
 807	*(Eterm *)							      \
 808	EXPAND_POINTER(tmp_arg1 = 					      \
 809		       (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \
 810        Fail;								      \
 811    }									      \
 812  } while (0)
 813
 814#define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
 815
 816#define IsBinary(Src, Fail) \
 817 if (is_not_binary(Src) || binary_bitsize(Src) != 0) { Fail; }
 818
 819#define IsBitstring(Src, Fail) \
 820  if (is_not_binary(Src)) { Fail; }
 821
 822#if defined(ARCH_64) && !HALFWORD_HEAP
 823#define BsSafeMul(A, B, Fail, Target)		\
 824   do { Uint64 _res = (A) * (B);		\
 825      if (_res / B != A) { Fail; }		\
 826      Target = _res;				\
 827   } while (0)
 828#else
 829#define BsSafeMul(A, B, Fail, Target)			\
 830   do { Uint64 _res = (Uint64)(A) * (Uint64)(B);	\
 831      if ((_res >> (8*sizeof(Uint))) != 0) { Fail; }	\
 832      Target = _res;					\
 833   } while (0)
 834#endif
 835
 836#define BsGetFieldSize(Bits, Unit, Fail, Target)	\
 837   do {							\
 838      Sint _signed_size; Uint _uint_size;		\
 839      if (is_small(Bits)) {				\
 840        _signed_size = signed_val(Bits);		\
 841         if (_signed_size < 0) { Fail; }		\
 842         _uint_size = (Uint) _signed_size;		\
 843      } else {						\
 844         if (!term_to_Uint(Bits, &temp_bits)) { Fail; }	\
 845         _uint_size = temp_bits;			\
 846      }							\
 847      BsSafeMul(_uint_size, Unit, Fail, Target);	\
 848   } while (0)
 849
 850#define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target)	\
 851   do {								\
 852      Sint _signed_size; Uint _uint_size;			\
 853      if (is_small(Bits)) {					\
 854        _signed_size = signed_val(Bits);			\
 855         if (_signed_size < 0) { Fail; }			\
 856         _uint_size = (Uint) _signed_size;			\
 857      } else {							\
 858         if (!term_to_Uint(Bits, &temp_bits)) { Fail; }		\
 859         _uint_size = (Uint) temp_bits;				\
 860      }								\
 861      Target = _uint_size * Unit;				\
 862   } while (0)
 863
 864#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Store, Fail)		\
 865 do {									\
 866   ErlBinMatchBuffer *_mb;						\
 867   Eterm _result; Sint _size;						\
 868   if (!is_small(Sz) || (_size = unsigned_val(Sz)) > 64) { Fail; }	\
 869   _size *= ((Flags) >> 3);						\
 870   TestHeap(FLOAT_SIZE_OBJECT, Live);					\
 871   _mb = ms_matchbuffer(Ms);						\
 872   LIGHT_SWAPOUT;							\
 873   _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb);		\
 874   LIGHT_SWAPIN;							\
 875   HEAP_SPACE_VERIFIED(0);                                          \
 876   if (is_non_value(_result)) { Fail; }					\
 877   else { Store(_result, Dst); }					\
 878 } while (0)
 879
 880#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
 881  do {								\
 882    ErlBinMatchBuffer *_mb;					\
 883    Eterm _result;						\
 884    TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), Live);	\
 885    _mb = ms_matchbuffer(Ms);					\
 886    LIGHT_SWAPOUT;						\
 887    _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb);	\
 888    LIGHT_SWAPIN;						\
 889    HEAP_SPACE_VERIFIED(0);                                 \
 890    if (is_non_value(_result)) { Fail; }			\
 891    else { Store(_result, Dst); }				\
 892  } while (0)
 893
 894#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
 895  do {								\
 896    ErlBinMatchBuffer *_mb;					\
 897    Eterm _result; Uint _size;					\
 898    BsGetFieldSize(Sz, ((Flags) >> 3), Fail, _size);		\
 899    TestHeap(ERL_SUB_BIN_SIZE, Live);				\
 900    _mb = ms_matchbuffer(Ms);					\
 901    LIGHT_SWAPOUT;						\
 902    _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb);	\
 903    LIGHT_SWAPIN;						\
 904    HEAP_SPACE_VERIFIED(0);                                 \
 905    if (is_non_value(_result)) { Fail; }			\
 906    else { Store(_result, Dst); }				\
 907  } while (0)
 908
 909#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Store, Fail)	\
 910  do {								\
 911    ErlBinMatchBuffer *_mb;					\
 912    Eterm _result;						\
 913    TestHeap(ERL_SUB_BIN_SIZE, Live);				\
 914    _mb = ms_matchbuffer(Ms);					\
 915    if (((_mb->size - _mb->offset) % Unit) == 0) {		\
 916      LIGHT_SWAPOUT;						\
 917      _result = erts_bs_get_binary_all_2(c_p, _mb);		\
 918      LIGHT_SWAPIN;						\
 919      HEAP_SPACE_VERIFIED(0);                               \
 920      ASSERT(is_value(_result));				\
 921      Store(_result, Dst);					\
 922    } else {                                                    \
 923	HEAP_SPACE_VERIFIED(0);                             \
 924	Fail; }						        \
 925 } while (0)
 926
 927#define BsSkipBits2(Ms, Bits, Unit, Fail)			\
 928 do {								\
 929   ErlBinMatchBuffer *_mb;					\
 930   size_t new_offset;						\
 931   Uint _size;							\
 932    _mb = ms_matchbuffer(Ms);					\
 933   BsGetFieldSize(Bits, Unit, Fail, _size);			\
 934   new_offset = _mb->offset + _size;				\
 935   if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
 936   else { Fail; }						\
 937 } while (0)
 938
 939#define BsSkipBitsAll2(Ms, Unit, Fail)		\
 940 do {						\
 941    ErlBinMatchBuffer *_mb;			\
 942   _mb = ms_matchbuffer(Ms);			\
 943   if (((_mb->size - _mb->offset) % Unit) == 0) {_mb->offset = _mb->size; } \
 944   else { Fail; }					\
 945 } while (0)
 946
 947#define BsSkipBitsImm2(Ms, Bits, Fail)				\
 948 do {								\
 949   ErlBinMatchBuffer *_mb;					\
 950   size_t new_offset;						\
 951   _mb = ms_matchbuffer(Ms);					\
 952   new_offset = _mb->offset + (Bits);				\
 953   if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
 954   else { Fail; }						\
 955 } while (0)
 956
 957#define NewBsPutIntegerImm(Sz, Flags, Src)					\
 958 do {									\
 959    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), (Sz), (Flags)))) { goto badarg; }	\
 960 } while (0)
 961
 962#define NewBsPutInteger(Sz, Flags, Src)						\
 963 do {										\
 964    Sint _size;									\
 965    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
 966    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), _size, (Flags))))	\
 967       { goto badarg; }								\
 968 } while (0)
 969
 970#define NewBsPutFloatImm(Sz, Flags, Src)					\
 971 do {									\
 972    if (!erts_new_bs_put_float(c_p, (Src), (Sz), (Flags))) { goto badarg; }	\
 973 } while (0)
 974
 975#define NewBsPutFloat(Sz, Flags, Src)						\
 976 do {										\
 977    Sint _size;									\
 978    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
 979    if (!erts_new_bs_put_float(c_p, (Src), _size, (Flags))) { goto badarg; }	\
 980 } while (0)
 981
 982#define NewBsPutBinary(Sz, Flags, Src)							\
 983 do {											\
 984    Sint _size;										\
 985    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);			\
 986    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), _size))) { goto badarg; }	\
 987 } while (0)
 988
 989#define NewBsPutBinaryImm(Sz, Src)				        \
 990 do {							        \
 991    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), (Sz)))) { goto badarg; }	\
 992 } while (0)
 993
 994#define NewBsPutBinaryAll(Src, Unit)							\
 995 do {											\
 996    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2((Src), (Unit)))) { goto badarg; }	\
 997 } while (0)
 998
 999
1000#define IsPort(Src, Fail) if (is_not_port(Src)) { Fail; }
1001#define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
1002#define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }
1003
1004/*
1005 * process_main() is already huge, so we want to avoid inlining
1006 * into it. Especially functions that are seldom used.
1007 */
1008#ifdef __GNUC__
1009#  define NOINLINE __attribute__((__noinline__))
1010#else
1011#  define NOINLINE
1012#endif
1013
1014/*
1015 * The following functions are called directly by process_main().
1016 * Don't inline them.
1017 */
1018static BifFunction translate_gc_bif(void* gcf) NOINLINE;
1019static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
1020			       Eterm* reg, BifFunction bf) NOINLINE;
1021static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
1022				     Eterm* reg, Eterm func) NOINLINE;
1023static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
1024static BeamInstr* apply(Process* p, Eterm module, Eterm function,
1025			Eterm args, Eterm* reg) NOINLINE;
1026static BeamInstr* call_fun(Process* p, int arity,
1027			   Eterm* reg, Eterm args) NOINLINE;
1028static BeamInstr* apply_fun(Process* p, Eterm fun,
1029			    Eterm args, Eterm* reg) NOINLINE;
1030static Eterm new_fun(Process* p, Eterm* reg,
1031		     ErlFunEntry* fe, int num_free) NOINLINE;
1032
1033
1034/*
1035 * Functions not directly called by process_main(). OK to inline.
1036 */
1037static BeamInstr* next_catch(Process* c_p, Eterm *reg);
1038static void terminate_proc(Process* c_p, Eterm Value);
1039static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
1040static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
1041			     BifFunction bf, Eterm args);
1042static struct StackTrace * get_trace_from_exc(Eterm exc);
1043static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
1044
1045#if defined(VXWORKS)
1046static int init_done;
1047#endif
1048
1049void
1050init_emulator(void)
1051{
1052#if defined(VXWORKS)
1053    init_done = 0;
1054#endif
1055    process_main();
1056}
1057
1058/*
1059 * On certain platforms, make sure that the main variables really are placed
1060 * in registers.
1061 */
1062
1063#if defined(__GNUC__) && defined(sparc) && !defined(DEBUG)
1064#  define REG_x0 asm("%l0")
1065#  define REG_xregs asm("%l1")
1066#  define REG_htop asm("%l2")
1067#  define REG_stop asm("%l3")
1068#  define REG_I asm("%l4")
1069#  define REG_fcalls asm("%l5")
1070#  define REG_tmp_arg1 asm("%l6")
1071#  define REG_tmp_arg2 asm("%l7")
1072#else
1073#  define REG_x0
1074#  define REG_xregs
1075#  define REG_htop
1076#  define REG_stop
1077#  define REG_I
1078#  define REG_fcalls
1079#  define REG_tmp_arg1
1080#  define REG_tmp_arg2
1081#endif
1082
1083/*
1084 * process_main() is called twice:
1085 * The first call performs some initialisation, including exporting
1086 * the instructions' C labels to the loader.
1087 * The second call starts execution of BEAM code. This call never returns.
1088 */
1089void process_main(void)
1090{
1091#if !defined(VXWORKS)
1092    static int init_done = 0;
1093#endif
1094    Process* c_p = NULL;
1095    int reds_used;
1096#ifdef DEBUG
1097    Eterm pid;
1098#endif
1099
1100    /*
1101     * X register zero; also called r(0)
1102     */
1103    register Eterm x0 REG_x0 = NIL;
1104
1105    /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
1106     * in all other cases x0 is used.
1107     */
1108    register Eterm* reg REG_xregs = NULL;
1109
1110    /*
1111     * Top of heap (next free location); grows upwards.
1112     */
1113    register Eterm* HTOP REG_htop = NULL;
1114
1115
1116#ifdef HYBRID
1117     Eterm *g_htop;
1118     Eterm *g_hend;
1119#endif
1120
1121    /* Stack pointer.  Grows downwards; points
1122     * to last item pushed (normally a saved
1123     * continuation pointer).
1124     */
1125    register Eterm* E REG_stop = NULL;
1126
1127    /*
1128     * Pointer to next threaded instruction.
1129     */
1130    register BeamInstr *I REG_I = NULL;
1131
1132    /* Number of reductions left.  This function
1133     * returns to the scheduler when FCALLS reaches zero.
1134     */
1135    register Sint FCALLS REG_fcalls = 0;
1136
1137    /*
1138     * Temporaries used for picking up arguments for instructions.
1139     */
1140    register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
1141    register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
1142#if HEAP_ON_C_STACK
1143    Eterm tmp_big[2];           /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
1144#else
1145    Eterm *tmp_big;		/* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
1146#endif
1147
1148    /*
1149     * X registers and floating point registers are located in
1150     * scheduler specific data.
1151     */
1152    register FloatDef *freg;
1153
1154    /*
1155     * For keeping the negative old value of 'reds' when call saving is active.
1156     */
1157    int neg_o_reds = 0;
1158
1159    Eterm (*arith_func)(Process* p, Eterm* reg, Uint live);
1160
1161#ifndef NO_JUMP_TABLE
1162    static void* opcodes[] = { DEFINE_OPCODES };
1163#ifdef ERTS_OPCODE_COUNTER_SUPPORT
1164    static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
1165#endif
1166#else
1167    int Go;
1168#endif
1169
1170    Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
1171
1172    Eterm pt_arity;		/* Used by do_put_tuple */
1173
1174    ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
1175
1176
1177    /*
1178     * Note: In this function, we attempt to place rarely executed code towards
1179     * the end of the function, in the hope that the cache hit rate will be better.
1180     * The initialization code is only run once, so it is at the very end.
1181     *
1182     * Note: c_p->arity must be set to reflect the number of useful terms in
1183     * c_p->arg_reg before calling the scheduler.
1184     */
1185    if (!init_done) {
1186	init_done = 1;
1187	goto init_emulator;
1188    }
1189    c_p = NULL;
1190    reds_used = 0;
1191    goto do_schedule1;
1192
1193 do_schedule:
1194    reds_used = REDS_IN(c_p) - FCALLS;
1195 do_schedule1:
1196    PROCESS_MAIN_CHK_LOCKS(c_p);
1197    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
1198#if HALFWORD_HEAP
1199    ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
1200#endif
1201    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
1202    c_p = schedule(c_p, reds_used);
1203    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
1204#ifdef DEBUG
1205    pid = c_p->id;
1206#endif
1207    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
1208    PROCESS_MAIN_CHK_LOCKS(c_p);
1209    reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
1210    freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
1211#if !HEAP_ON_C_STACK
1212    tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
1213#endif
1214    ERL_BITS_RELOAD_STATEP(c_p);
1215    {
1216	int reds;
1217	Eterm* argp;
1218	BeamInstr *next;
1219	int i;
1220
1221	argp = c_p->arg_reg;
1222	for (i = c_p->arity - 1; i > 0; i--) {
1223	    reg[i] = argp[i];
1224	    CHECK_TERM(reg[i]);
1225	}
1226
1227	/*
1228	 * We put the original reduction count in the process structure, to reduce
1229	 * the code size (referencing a field in a struct through a pointer stored
1230	 * in a register gives smaller code than referencing a global variable).
1231	 */
1232
1233	SET_I(c_p->i);
1234
1235	reds = c_p->fcalls;
1236	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
1237	    && (c_p->trace_flags & F_SENSITIVE) == 0) {
1238	    neg_o_reds = -reds;
1239	    FCALLS = REDS_IN(c_p) = 0;
1240	} else {
1241	    neg_o_reds = 0;
1242	    FCALLS = REDS_IN(c_p) = reds;
1243	}
1244
1245	next = (BeamInstr *) *I;
1246	r(0) = c_p->arg_reg[0];
1247#ifdef HARDDEBUG
1248	if (c_p->arity > 0) {
1249	    CHECK_TERM(r(0));
1250	}
1251#endif
1252	SWAPIN;
1253	ASSERT(VALID_INSTR(next));
1254	Goto(next);
1255    }
1256
1257#if defined(DEBUG) || defined(NO_JUMP_TABLE)
1258 emulator_loop:
1259#endif
1260
1261#ifdef NO_JUMP_TABLE
1262    switch (Go) {
1263#endif
1264#include "beam_hot.h"
1265
1266#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
1267#define ARITH_FUNC(name) erts_gc_##name
1268
1269	{
1270	    Eterm increment_reg_val;
1271	    Eterm increment_val;
1272	    Uint live;
1273	    Eterm result;
1274
1275	OpCase(i_increment_yIId):
1276	    increment_reg_val = yb(Arg(0));
1277	    goto do_increment;
1278
1279	OpCase(i_increment_xIId):
1280	    increment_reg_val = xb(Arg(0));
1281	    goto do_increment;
1282
1283	OpCase(i_increment_rIId):
1284	    increment_reg_val = r(0);
1285	    I--;
1286
1287	do_increment:
1288	    increment_val = Arg(1);
1289	    if (is_small(increment_reg_val)) {
1290		Sint i = signed_val(increment_reg_val) + increment_val;
1291		ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
1292		if (MY_IS_SSMALL(i)) {
1293		    result = make_small(i);
1294		store_result:
1295		    StoreBifResult(3, result);
1296		}
1297	    }
1298
1299	    live = Arg(2);
1300	    SWAPOUT;
1301	    reg[0] = r(0);
1302	    reg[live] = increment_reg_val;
1303	    reg[live+1] = make_small(increment_val);
1304	    result = erts_gc_mixed_plus(c_p, reg, live);
1305	    r(0) = reg[0];
1306	    SWAPIN;
1307	    ERTS_HOLE_CHECK(c_p);
1308	    if (is_value(result)) {
1309		goto store_result;
1310	    }
1311	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
1312	    goto find_func_info;
1313	}
1314	    
1315 OpCase(i_plus_jId):
1316 {
1317     Eterm result;
1318
1319     if (is_both_small(tmp_arg1, tmp_arg2)) {
1320	 Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2);
1321	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
1322	 if (MY_IS_SSMALL(i)) {
1323	     result = make_small(i);
1324	     STORE_ARITH_RESULT(result);
1325	 }
1326     
1327     }
1328     arith_func = ARITH_FUNC(mixed_plus);
1329     goto do_big_arith2;
1330 }
1331
1332 OpCase(i_minus_jId):
1333 {
1334     Eterm result;
1335
1336     if (is_both_small(tmp_arg1, tmp_arg2)) {
1337	 Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2);
1338	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
1339	 if (MY_IS_SSMALL(i)) {
1340	     result = make_small(i);
1341	     STORE_ARITH_RESULT(result);
1342	 }
1343     }
1344     arith_func = ARITH_FUNC(mixed_minus);
1345     goto do_big_arith2;
1346 }
1347
1348 OpCase(i_is_lt_f):
1349    if (CMP_GE(tmp_arg1, tmp_arg2)) {
1350	ClauseFail();
1351    }
1352    Next(1);
1353
1354 OpCase(i_is_ge_f):
1355    if (CMP_LT(tmp_arg1, tmp_arg2)) {
1356	ClauseFail();
1357    }
1358    Next(1);
1359
1360 OpCase(i_is_eq_f):
1361    if (CMP_NE(tmp_arg1, tmp_arg2)) {
1362	ClauseFail();
1363    }
1364    Next(1);
1365
1366 OpCase(i_is_ne_f):
1367    if (CMP_EQ(tmp_arg1, tmp_arg2)) {
1368	ClauseFail();
1369    }
1370    Next(1);
1371
1372 OpCase(i_is_eq_exact_f):
1373    if (!EQ(tmp_arg1, tmp_arg2)) {
1374	ClauseFail();
1375    }
1376    Next(1);
1377
1378    {
1379	Eterm is_eq_exact_lit_val;
1380
1381    OpCase(i_is_eq_exact_literal_xfc):
1382	is_eq_exact_lit_val = xb(Arg(0));
1383	I++;
1384	goto do_is_eq_exact_literal;
1385
1386    OpCase(i_is_eq_exact_literal_yfc):
1387	is_eq_exact_lit_val = yb(Arg(0));
1388	I++;
1389	goto do_is_eq_exact_literal;
1390
1391    OpCase(i_is_eq_exact_literal_rfc):
1392	is_eq_exact_lit_val = r(0);
1393
1394    do_is_eq_exact_literal:
1395	if (!eq(Arg(1), is_eq_exact_lit_val)) {
1396	    ClauseFail();
1397	}
1398	Next(2);
1399    }
1400
1401    {
1402	Eterm is_ne_exact_lit_val;
1403
1404    OpCase(i_is_ne_exact_literal_xfc):
1405	is_ne_exact_lit_val = xb(Arg(0));
1406	I++;
1407	goto do_is_ne_exact_literal;
1408
1409    OpCase(i_is_ne_exact_literal_yfc):
1410	is_ne_exact_lit_val = yb(Arg(0));
1411	I++;
1412	goto do_is_ne_exact_literal;
1413
1414    OpCase(i_is_ne_exact_literal_rfc):
1415	is_ne_exact_lit_val = r(0);
1416
1417    do_is_ne_exact_literal:
1418	if (eq(Arg(1), is_ne_exact_lit_val)) {
1419	    ClauseFail();
1420	}
1421	Next(2);
1422    }
1423
1424 OpCase(i_move_call_only_fcr): {
1425     r(0) = Arg(1);
1426 }
1427 /* FALL THROUGH */
1428 OpCase(i_call_only_f): {
1429     SET_I((BeamInstr *) Arg(0));
1430     Dispatch();
1431 }
1432
1433 OpCase(i_move_call_last_fPcr): {
1434     r(0) = Arg(2);
1435 }
1436 /* FALL THROUGH */
1437 OpCase(i_call_last_fP): {
1438     RESTORE_CP(E);
1439     E = ADD_BYTE_OFFSET(E, Arg(1));
1440     SET_I((BeamInstr *) Arg(0));
1441     Dispatch();
1442 }
1443
1444 OpCase(i_move_call_crf): {
1445     r(0) = Arg(0);
1446     I++;
1447 }
1448 /* FALL THROUGH */
1449 OpCase(i_call_f): {
1450     SET_CP(c_p, I+2);
1451     SET_I((BeamInstr *) Arg(0));
1452     Dispatch();
1453 }
1454
1455 OpCase(i_move_call_ext_last_ePcr): {
1456     r(0) = Arg(2);
1457 }
1458 /* FALL THROUGH */
1459 OpCase(i_call_ext_last_eP):
1460    RESTORE_CP(E);
1461    E = ADD_BYTE_OFFSET(E, Arg(1));
1462
1463    /*
1464     * Note: The pointer to the export entry is never NULL; if the module
1465     * is not loaded, it points to code which will invoke the error handler
1466     * (see lb_call_error_handler below).
1467     */
1468    Dispatchx();
1469
1470 OpCase(i_move_call_ext_cre): {
1471     r(0) = Arg(0);
1472     I++;
1473 }
1474 /* FALL THROUGH */
1475 OpCase(i_call_ext_e):
1476    SET_CP(c_p, I+2);
1477    Dispatchx();
1478
1479 OpCase(i_move_call_ext_only_ecr): {
1480     r(0) = Arg(1);
1481 }
1482 /* FALL THROUGH */
1483 OpCase(i_call_ext_only_e):
1484    Dispatchx();
1485
1486 OpCase(init_y): {
1487     BeamInstr *next;
1488
1489     PreFetch(1, next);
1490     make_blank(yb(Arg(0)));
1491     NextPF(1, next);
1492 }
1493
1494 OpCase(i_trim_I): {
1495     BeamInstr *next;
1496     Uint words;
1497     Uint cp;
1498
1499     words = Arg(0);
1500     cp = E[0];
1501     PreFetch(1, next);
1502     E += words;
1503     E[0] = cp;
1504     NextPF(1, next);
1505 }
1506
1507 OpCase(move_x1_c): {
1508	x(1) = Arg(0);
1509	Next(1);
1510    }
1511
1512 OpCase(move_x2_c): {
1513	x(2) = Arg(0);
1514	Next(1);
1515    }
1516
1517
1518 OpCase(return): {
1519    SET_I(c_p->cp);
1520    /*
1521     * We must clear the CP to make sure that a stale value do not
1522     * create a false module dependcy preventing code upgrading.
1523     * It also means that we can use the CP in stack backtraces.
1524     */
1525    c_p->cp = 0;
1526    CHECK_TERM(r(0));
1527    HEAP_SPACE_VERIFIED(0);
1528    Goto(*I);
1529 }
1530
1531    /*
1532     * Send is almost a standard call-BIF with two arguments, except for:
1533     *    1) It cannot be traced.
1534     *	  2) There is no pointer to the send_2 function stored in
1535     *       the instruction.
1536     */
1537
1538 OpCase(send): {
1539     BeamInstr *next;
1540     Eterm result;
1541
1542     PRE_BIF_SWAPOUT(c_p);
1543     c_p->fcalls = FCALLS - 1;
1544     result = send_2(c_p, r(0), x(1));
1545     PreFetch(0, next);
1546     POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
1547     FCALLS = c_p->fcalls;
1548     if (is_value(result)) {
1549	 r(0) = result;
1550	 CHECK_TERM(r(0));
1551	 NextPF(0, next);
1552     } else if (c_p->freason == TRAP) {
1553	 SET_CP(c_p, I+1);
1554	 SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3)));
1555	 SWAPIN;
1556	 r(0) = c_p->def_arg_reg[0];
1557	 x(1) = c_p->def_arg_reg[1];
1558	 Dispatch();
1559     }
1560     goto find_func_info;
1561 }
1562
1563    {
1564	Eterm element_index;
1565	Eterm element_tuple;
1566
1567    OpCase(i_element_xjsd):
1568	element_tuple = xb(Arg(0));
1569	I++;
1570	goto do_element;
1571
1572    OpCase(i_element_yjsd):
1573	element_tuple = yb(Arg(0));
1574	I++;
1575	goto do_element;
1576
1577    OpCase(i_element_rjsd):
1578	element_tuple = r(0);
1579	/* Fall through */
1580
1581    do_element:
1582	GetArg1(1, element_index);
1583	if (is_small(element_index) && is_tuple(element_tuple)) {
1584	    Eterm* tp = tuple_val(element_tuple);
1585
1586	    if ((signed_val(element_index) >= 1) &&
1587		(signed_val(element_index) <= arityval(*tp))) {
1588		Eterm result = tp[signed_val(element_index)];
1589		StoreBifResult(2, result);
1590	    }
1591	}
1592    }
1593 /* Fall through */
1594
1595 OpCase(badarg_j):
1596 badarg:
1597    c_p->freason = BADARG;
1598    goto lb_Cl_error;
1599
1600    {
1601	Eterm fast_element_tuple;
1602
1603    OpCase(i_fast_element_rjId):
1604	fast_element_tuple = r(0);
1605
1606    do_fast_element:
1607	if (is_tuple(fast_element_tuple)) {
1608	    Eterm* tp = tuple_val(fast_element_tuple);
1609	    Eterm pos = Arg(1);	/* Untagged integer >= 1 */
1610	    if (pos <= arityval(*tp)) {
1611		Eterm result = tp[pos];
1612		StoreBifResult(2, result);
1613	    }
1614	}
1615     goto badarg;
1616
1617    OpCase(i_fast_element_xjId):
1618     fast_element_tuple = xb(Arg(0));
1619     I++;
1620     goto do_fast_element;
1621
1622    OpCase(i_fast_element_yjId):
1623     fast_element_tuple = yb(Arg(0));
1624     I++;
1625     goto do_fast_element;
1626 }
1627
1628 OpCase(catch_yf):
1629     c_p->catches++;
1630     yb(Arg(0)) = Arg(1);
1631     Next(2);
1632
1633 OpCase(catch_end_y): {
1634     c_p->catches--;
1635     make_blank(yb(Arg(0)));
1636     if (is_non_value(r(0))) {
1637	 if (x(1) == am_throw) {
1638	     r(0) = x(2);
1639	 } else {
1640	     if (x(1) == am_error) {
1641	         SWAPOUT;
1642		 x(2) = add_stacktrace(c_p, x(2), x(3));
1643		 SWAPIN;
1644	     }
1645	     /* only x(2) is included in the rootset here */
1646	     if (E - HTOP < 3 || c_p->mbuf) {	/* Force GC in case add_stacktrace()
1647						 * created heap fragments */
1648		 SWAPOUT;
1649		 PROCESS_MAIN_CHK_LOCKS(c_p);
1650		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
1651		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
1652		 PROCESS_MAIN_CHK_LOCKS(c_p);
1653		 SWAPIN;
1654	     }
1655	     r(0) = TUPLE2(HTOP, am_EXIT, x(2));
1656	     HTOP += 3;
1657	 }
1658     }
1659     CHECK_TERM(r(0));
1660     Next(1);
1661 }
1662
1663 OpCase(try_end_y): {
1664     c_p->catches--;
1665     make_blank(yb(Arg(0)));
1666     if (is_non_value(r(0))) {
1667	 r(0) = x(1);
1668	 x(1) = x(2);
1669	 x(2) = x(3);
1670     }
1671     Next(1);
1672 }
1673
1674 /*
1675  * Skeleton for receive statement:
1676  *
1677  *             recv_mark L1                     Optional
1678  *             call make_ref/monitor            Optional
1679  *             ...
1680  *             recv_set L1                      Optional
1681  *      L1:          <-------------------+
1682  *                   <-----------+       |
1683  *     	     	       	  |   	  |
1684  *             loop_rec L2 ------+---+   |
1685  *             ...               |   |   |
1686  *             remove_message 	  |   |	  |
1687  *             jump L3           |   |   |
1688  *		...	          |   |   |
1689  *		loop_rec_end L1 --+   |   |
1690  *      L2:          <---------------+   |
1691  *	   	wait L1  -----------------+      or wait_timeout
1692  *		timeout
1693  *
1694  *	 L3:    Code after receive...
1695  *
1696  *
1697  */
1698
1699 OpCase(recv_mark_f): {
1700     /*
1701      * Save the current position in message buffer and the
1702      * the label for the loop_rec/2 instruction for the
1703      * the receive statement.
1704      */
1705     c_p->msg.mark = (BeamInstr *) Arg(0);
1706     c_p->msg.saved_last = c_p->msg.last;
1707     Next(1);
1708 }
1709
1710 OpCase(i_recv_set): {
1711     /*
1712      * If the mark is valid (points to the loop_rec/2
1713      * instruction that follows), we know that the saved
1714      * position points to the first message that could
1715      * possibly be matched out.
1716      *
1717      * If the mark is invalid, we do nothing, meaning that
1718      * we will look through all messages in the message queue.
1719      */
1720     if (c_p->msg.mark == (BeamInstr *) (I+1)) {
1721	 c_p->msg.save = c_p->msg.saved_last;
1722     }
1723     I++;
1724     /* Fall through to the loop_rec/2 instruction */
1725 }
1726
1727    /*
1728     * Pick up the next message and place it in x(0).
1729     * If no message, jump to a wait or wait_timeout instruction.
1730     */
1731 OpCase(i_loop_rec_fr):
1732 {
1733     BeamInstr *next;
1734     ErlMessage* msgp;
1735
1736 loop_rec__:
1737
1738     PROCESS_MAIN_CHK_LOCKS(c_p);
1739
1740     msgp = PEEK_MESSAGE(c_p);
1741
1742     if (!msgp) {
1743#ifdef ERTS_SMP
1744	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
1745	 /* Make sure messages wont pass exit signals... */
1746	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
1747	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
1748	     SWAPOUT;
1749	     goto do_schedule; /* Will be rescheduled for exit */
1750	 }
1751	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
1752	 msgp = PEEK_MESSAGE(c_p);
1753	 if (msgp)
1754	     erts_smp_proc_unlock(c_p, ERTS_P

Large files files are truncated, but you can click here to view the full file