/erts/emulator/beam/erl_process.c

Large files files are truncated, but you can click here to view the full file

/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2010. All Rights Reserved.
 *
 * The contents of this file are subject to the Erlang Public License,
 * Version 1.1, (the "License"); you may not use this file except in
 * compliance with the License. You should have received a copy of the
 * Erlang Public License along with this software. If not, it can be
 * retrieved online at http://www.erlang.org/.
 *
 * Software distributed under the License is distributed on an "AS IS"
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 * the License for the specific language governing rights and limitations
 * under the License.
 *
 * %CopyrightEnd%
 */

#define ERL_PROCESS_C__

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include <stddef.h> /* offsetof() */
#include "sys.h"
#include "erl_vm.h"
#include "global.h"
#include "erl_process.h"
#include "erl_nmgc.h"
#include "error.h"
#include "bif.h"
#include "erl_db.h"
#include "dist.h"
#include "beam_catches.h"
#include "erl_instrument.h"
#include "erl_threads.h"
#include "erl_binary.h"
#include "beam_bp.h"
#include "erl_cpu_topology.h"

#define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS)
#define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \
  (ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED/2)

#define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10)

#define ERTS_SCHED_SPIN_UNTIL_YIELD 100

#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10
#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000
#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \
  (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT)
#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0

#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS)
#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS)
#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS)
#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS)
#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10)

#define ERTS_WAKEUP_OTHER_DEC 10
#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10)

#if 0 || defined(DEBUG)
#define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
#endif

#if defined(DEBUG) && 0
#define HARDDEBUG
#else
#undef HARDDEBUG
#endif

#ifdef HARDDEBUG
#define HARDDEBUG_RUNQS
#endif

#ifdef HIPE
#include "hipe_mode_switch.h"	/* for hipe_init_process() */
#include "hipe_signal.h"	/* for hipe_thread_signal_init() */
#endif

#ifdef ERTS_ENABLE_LOCK_COUNT
#include "erl_lock_count.h"
#endif

#define MAX_BIT       (1 << PRIORITY_MAX)
#define HIGH_BIT      (1 << PRIORITY_HIGH)
#define NORMAL_BIT    (1 << PRIORITY_NORMAL)
#define LOW_BIT       (1 << PRIORITY_LOW)

#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P)			\
do {								\
    ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx));	\
    if (saved_term_procs.end)					\
	save_terminating_process((P));				\
} while (0)

#define ERTS_EMPTY_RUNQ(RQ) \
  ((RQ)->len == 0 && (RQ)->misc.start == NULL)

extern BeamInstr beam_apply[];
extern BeamInstr beam_exit[];
extern BeamInstr beam_continue_exit[];

static Sint p_last;
static Sint p_next;
static Sint p_serial;
static Uint p_serial_mask;
static Uint p_serial_shift;

Uint erts_no_schedulers;
Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES;
Uint erts_process_tab_index_mask;

static int wakeup_other_limit;

int erts_sched_thread_suggested_stack_size = -1;

#ifdef ERTS_ENABLE_LOCK_CHECK
ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
#endif

#ifdef ERTS_SMP

int erts_disable_proc_not_running_opt;

#define ERTS_SCHDLR_SSPND_CHNG_WAITER		(((erts_aint32_t) 1) << 0)
#define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
#define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)

#ifndef DEBUG

#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
  erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL))

#else

#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL)			\
do {									\
    erts_aint32_t old_val__;						\
    old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing,		\
				       (VAL));				\
    ASSERT(old_val__ == (OLD_VAL));					\
} while (0)

#endif


static struct {
    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;
    int online;
    int curr_online;
    int wait_curr_online;
    erts_smp_atomic32_t changing;
    erts_smp_atomic32_t active;
    struct {
	erts_smp_atomic32_t ongoing;
	long wait_active;
	ErtsProcList *procs;
    } msb; /* Multi Scheduling Block */
} schdlr_sspnd;

static struct {
    erts_smp_mtx_t update_mtx;
    erts_smp_atomic32_t active_runqs;
    int last_active_runqs;
    erts_smp_atomic32_t used_runqs;
    int forced_check_balance;
    erts_smp_atomic32_t checking_balance;
    int halftime;
    int full_reds_history_index;
    struct {
	int active_runqs;
	int reds;
	int max_len;
    } prev_rise;
    Uint n;
} balance_info;

#define ERTS_BLNCE_SAVE_RISE(ACTIVE, MAX_LEN, REDS)	\
do {							\
    balance_info.prev_rise.active_runqs = (ACTIVE);	\
    balance_info.prev_rise.max_len = (MAX_LEN);		\
    balance_info.prev_rise.reds = (REDS);		\
} while (0)

#endif

erts_sched_stat_t erts_sched_stat;

ErtsRunQueue *erts_common_run_queue;

#ifdef USE_THREADS
static erts_tsd_key_t sched_data_key;
#endif

static erts_smp_mtx_t proc_tab_mtx;

static erts_smp_atomic32_t function_calls;

#ifdef ERTS_SMP
static erts_smp_atomic32_t doing_sys_schedule;
static erts_smp_atomic32_t no_empty_run_queues;
#else /* !ERTS_SMP */
ErtsSchedulerData *erts_scheduler_data;
#endif

ErtsAlignedRunQueue *erts_aligned_run_queues;
Uint erts_no_run_queues;

ErtsAlignedSchedulerData *erts_aligned_scheduler_data;

#ifdef ERTS_SMP

typedef union {
    ErtsSchedulerSleepInfo ssi;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))];
} ErtsAlignedSchedulerSleepInfo;

static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;

#endif

#ifndef BM_COUNTERS
static int processes_busy;
#endif

Process**  process_tab;
static Uint last_reductions;
static Uint last_exact_reductions;
Uint erts_default_process_flags;
Eterm erts_system_monitor;
Eterm erts_system_monitor_msg_queue_len;
Eterm erts_system_monitor_long_gc;
Eterm erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system performance monitor */
Eterm erts_system_profile;
struct erts_system_profile_flags_t erts_system_profile_flags;

#ifdef HYBRID
Uint erts_num_active_procs;
Process** erts_active_procs;
#endif

#if ERTS_MAX_PROCESSES > 0x7fffffff
#error "Need to store process_count in another type"
#endif
static erts_smp_atomic32_t process_count;

typedef struct ErtsTermProcElement_ ErtsTermProcElement;
struct ErtsTermProcElement_ {
    ErtsTermProcElement *next;
    ErtsTermProcElement *prev;
    int ix;
    union {
	struct {
	    Eterm pid;
	    SysTimeval spawned;
	    SysTimeval exited;
	} process;
	struct {
	    SysTimeval time;
	} bif_invocation;
    } u;
};

static struct {
    ErtsTermProcElement *start;
    ErtsTermProcElement *end;
} saved_term_procs;

ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list,
				 ErtsMiscOpList,
				 10,
				 ERTS_ALC_T_MISC_OP_LIST)

ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
				 ErtsProcList,
				 200,
				 ERTS_ALC_T_PROC_LIST)

#define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &aligned_sched_sleep_info[(IX)].ssi)

#define ERTS_FOREACH_RUNQ(RQVAR, DO)					\
do {									\
    ErtsRunQueue *RQVAR;						\
    int ix__;								\
    for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) {			\
	RQVAR = ERTS_RUNQ_IX(ix__);					\
	erts_smp_runq_lock(RQVAR);					\
	{ DO; }								\
	erts_smp_runq_unlock(RQVAR);					\
    }									\
} while (0)

#define ERTS_FOREACH_OP_RUNQ(RQVAR, DO)					\
do {									\
    ErtsRunQueue *RQVAR;						\
    int ix__;								\
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx));	\
    for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) {		\
	RQVAR = ERTS_RUNQ_IX(ix__);					\
	erts_smp_runq_lock(RQVAR);					\
	{ DO; }								\
	erts_smp_runq_unlock(RQVAR);					\
    }									\
} while (0)

#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, DOX)			\
do {									\
    ErtsRunQueue *RQVAR;						\
    int ix__;								\
    for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) {			\
	RQVAR = ERTS_RUNQ_IX(ix__);					\
	erts_smp_runq_lock(RQVAR);					\
	{ DO; }								\
    }									\
    { DOX; }								\
    for (ix__ = 0; ix__ < erts_no_run_queues; ix__++)			\
	erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__));			\
} while (0)

#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \
  ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, )
/*
 * Local functions.
 */

static void init_processes_bif(void);
static void save_terminating_process(Process *p);
static void exec_misc_ops(ErtsRunQueue *);
static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
			      int yreg);
#ifdef ERTS_SMP
static void handle_pending_exiters(ErtsProcList *);

#endif

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int
erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
{
    return erts_smp_lc_mtx_is_locked(&runq->mtx);
}
#endif

void
erts_pre_init_process(void)
{
#ifdef USE_THREADS
    erts_tsd_key_create(&sched_data_key);
#endif

#ifdef ERTS_ENABLE_LOCK_CHECK
 {
     int ix;

     erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks
	 = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks
	 = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS;

     erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks
	 = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks
	 = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS;

     erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks
	 = ERTS_PSD_SCHED_ID_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks
	 = ERTS_PSD_SCHED_ID_SET_LOCKS;

     erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks
	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
	 = ERTS_PSD_DIST_ENTRY_SET_LOCKS;

     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
	 = ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
	 = ERTS_PSD_CALL_TIME_BP_SET_LOCKS;

     /* Check that we have locks for all entries */
     for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
	 ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks);
	 ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks);
     }
 }
#endif
}

/* initialize the scheduler */
void
erts_init_process(int ncpu)
{
    Uint proc_bits = ERTS_PROC_BITS;

#ifdef ERTS_SMP
    erts_disable_proc_not_running_opt = 0;
    erts_init_proc_lock(ncpu);
#endif

    init_proclist_alloc();

    erts_smp_atomic32_init(&process_count, 0);

    if (erts_use_r9_pids_ports) {
	proc_bits = ERTS_R9_PROC_BITS;
	ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS));
    }

    process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE,
					 erts_max_processes*sizeof(Process*));
    sys_memzero(process_tab, erts_max_processes * sizeof(Process*));
#ifdef HYBRID
    erts_active_procs = (Process**)
        erts_alloc(ERTS_ALC_T_ACTIVE_PROCS,
                   erts_max_processes * sizeof(Process*));
    erts_num_active_procs = 0;
#endif

    erts_smp_mtx_init(&proc_tab_mtx, "proc_tab");
    p_last = -1;
    p_next = 0;
    p_serial = 0;

    p_serial_shift = erts_fit_in_bits(erts_max_processes - 1);
    p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift);
    erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift);
#ifndef BM_COUNTERS
    processes_busy = 0;
#endif
    last_reductions = 0;
    last_exact_reductions = 0;
    erts_default_process_flags = 0;
}

void
erts_late_init_process(void)
{
    int ix;
    init_processes_bif();

    erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat");
    for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) {
	Eterm atom;
	char *atom_str;
	switch (ix) {
	case PRIORITY_MAX:
	    atom_str = "process_max";
	    break;
	case PRIORITY_HIGH:
	    atom_str = "process_high";
	    break;
	case PRIORITY_NORMAL:
	    atom_str = "process_normal";
	    break;
	case PRIORITY_LOW:
	    atom_str = "process_low";
	    break;
	case ERTS_PORT_PRIO_LEVEL:
	    atom_str = "port";
	    break;
	default:
	    atom_str = "bad_prio";
	    ASSERT(!"bad prio");
	    break;
	}
	atom = am_atom_put(atom_str, sys_strlen(atom_str));
	erts_sched_stat.prio[ix].name = atom;
	erts_sched_stat.prio[ix].total_executed = 0;
	erts_sched_stat.prio[ix].executed = 0;
	erts_sched_stat.prio[ix].total_migrated = 0;
	erts_sched_stat.prio[ix].migrated = 0;
    }

}

static ERTS_INLINE ErtsProcList *
proclist_create(Process *p)
{
    ErtsProcList *plp = proclist_alloc();
    plp->pid = p->id;
    plp->started = p->started;
    return plp;
}

static ERTS_INLINE void
proclist_destroy(ErtsProcList *plp)
{
    proclist_free(plp);
}

static ERTS_INLINE int
proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->id
	    && erts_cmp_timeval(&plp->started, &p->started) == 0);
}

ErtsProcList *
erts_proclist_create(Process *p)
{
    return proclist_create(p);
}

void
erts_proclist_destroy(ErtsProcList *plp)
{
    proclist_destroy(plp);
}

int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return proclist_same(plp, p);
}

void *
erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
    void *old;
    ErtsProcLocks xplocks;
    int refc = 0;
    ErtsPSD *psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD));
    int i;
    for (i = 0; i < ERTS_PSD_SIZE; i++)
	psd->data[i] = NULL;

    ERTS_SMP_LC_ASSERT(plocks);
    ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p));

    xplocks = ERTS_PROC_LOCKS_ALL;
    xplocks &= ~plocks;
    if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) {
	if (xplocks & ERTS_PROC_LOCK_MAIN) {
	    erts_smp_proc_inc_refc(p);
	    erts_smp_proc_unlock(p, plocks);
	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL);
	    refc = 1;
	}
	else {
	    if (plocks & ERTS_PROC_LOCKS_ALL_MINOR)
		erts_smp_proc_unlock(p, plocks & ERTS_PROC_LOCKS_ALL_MINOR);
	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
	}
    }
    if (!p->psd)
	p->psd = psd;
    if (xplocks)
	erts_smp_proc_unlock(p, xplocks);
    if (refc)
	erts_smp_proc_dec_refc(p);
    ASSERT(p->psd);
    if (p->psd != psd)
	erts_free(ERTS_ALC_T_PSD, psd);
    old = p->psd->data[ix];
    p->psd->data[ix] = data;
    ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p));
    return old;
}

#ifdef ERTS_SMP

void
erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
{
    switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) {
    case ERTS_SSI_FLG_POLL_SLEEPING:
	erts_sys_schedule_interrupt(1);
	break;
    case ERTS_SSI_FLG_TSE_SLEEPING:
	erts_tse_set(ssi->event);
	break;
    case 0:
	break;
    default:
	erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
		 __FILE__, __LINE__);
	break;
    }
}

typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
struct erts_misc_aux_work_t_ {
    erts_misc_aux_work_t *next;
    void (*func)(void *);
    void *arg;
};

typedef struct {
    erts_smp_mtx_t mtx;
    erts_misc_aux_work_t *first;
    erts_misc_aux_work_t *last;
} erts_misc_aux_work_q_t;

typedef union {
    erts_misc_aux_work_q_t data;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))];
} erts_algnd_misc_aux_work_q_t;

static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues;

ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work,
				 erts_misc_aux_work_t,
				 200,
				 ERTS_ALC_T_MISC_AUX_WORK)

static void
init_misc_aux_work(void)
{
    int ix;

    init_misc_aux_work_alloc();

    misc_aux_work_queues = erts_alloc(ERTS_ALC_T_MISC_AUX_WORK_Q,
				      (sizeof(erts_algnd_misc_aux_work_q_t)
				       *(erts_no_schedulers+1)));
    if ((((UWord) misc_aux_work_queues) & ERTS_CACHE_LINE_MASK) != 0)
	misc_aux_work_queues = ((erts_algnd_misc_aux_work_q_t *)
				   ((((UWord) misc_aux_work_queues)
				     & ~ERTS_CACHE_LINE_MASK)
				    + ERTS_CACHE_LINE_SIZE));

    for (ix = 0; ix < erts_no_schedulers; ix++) {
	erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx,
			    "misc_aux_work_queue",
			    make_small(ix + 1));
	misc_aux_work_queues[ix].data.first = NULL;
	misc_aux_work_queues[ix].data.last = NULL;
    }
}

static void
handle_misc_aux_work(ErtsSchedulerData *esdp)
{
    int ix = (int) esdp->no - 1;
    erts_misc_aux_work_t *mawp;

    erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx);
    mawp = misc_aux_work_queues[ix].data.first;
    misc_aux_work_queues[ix].data.first = NULL;
    misc_aux_work_queues[ix].data.last = NULL;
    erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);

    while (mawp) {
	erts_misc_aux_work_t *free_mawp;
	mawp->func(mawp->arg);
	free_mawp = mawp;
	mawp = mawp->next;
	misc_aux_work_free(free_mawp);
    }
}

void
erts_smp_schedule_misc_aux_work(int ignore_self,
				int max_sched,
				void (*func)(void *),
				void *arg)
{
    int ix, ignore_ix = -1;

    if (ignore_self) {
	ErtsSchedulerData *esdp = erts_get_scheduler_data();
	if (esdp)
	    ignore_ix = (int) esdp->no - 1;
    }

    ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers);

    for (ix = 0; ix < max_sched; ix++) {
	erts_aint32_t aux_work;
	erts_misc_aux_work_t *mawp;
	ErtsSchedulerSleepInfo *ssi;
	if (ix == ignore_ix)
	    continue;

	mawp = misc_aux_work_alloc();

	mawp->func = func;
	mawp->arg = arg;
	mawp->next = NULL;

	erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx);
	if (!misc_aux_work_queues[ix].data.last)
	    misc_aux_work_queues[ix].data.first = mawp;
	else
	    misc_aux_work_queues[ix].data.last->next = mawp;
	misc_aux_work_queues[ix].data.last = mawp;
	erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);

	ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
	aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
					 ERTS_SSI_AUX_WORK_MISC);
	if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0)
	    erts_sched_poke(ssi);
   }
}

#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
void
erts_smp_notify_check_children_needed(void)
{
    int i;

    for (i = 0; i < erts_no_schedulers; i++) {
	erts_aint32_t aux_work;
	ErtsSchedulerSleepInfo *ssi;
	ssi = ERTS_SCHED_SLEEP_INFO_IX(i);
	aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
					 ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
	if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN))
	    erts_sched_poke(ssi);
    }
}
#endif

#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
static ERTS_INLINE erts_aint32_t
blockable_aux_work(ErtsSchedulerData *esdp,
		   ErtsSchedulerSleepInfo *ssi,
		   erts_aint32_t aux_work)
{
    if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
	if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
	    aux_work = erts_smp_atomic32_band(&ssi->aux_work,
					      ~ERTS_SSI_AUX_WORK_MISC);
	    aux_work &= ~ERTS_SSI_AUX_WORK_MISC;
	    handle_misc_aux_work(esdp);
	}
#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
	if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
	    aux_work = erts_smp_atomic32_band(&ssi->aux_work,
					      ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
	    aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
	    erts_check_children();
	}
#endif
    }
    return aux_work;
}

#endif

#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
static ERTS_INLINE erts_aint32_t
nonblockable_aux_work(ErtsSchedulerData *esdp,
		      ErtsSchedulerSleepInfo *ssi,
		      erts_aint32_t aux_work)
{
    if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) {

    }
}
#endif

static void
prepare_for_block(void *vrq)
{
    erts_smp_runq_unlock((ErtsRunQueue *) vrq);
}

static void
resume_after_block(void *vrq)
{
    erts_smp_runq_lock((ErtsRunQueue *) vrq);
}

#endif

static ERTS_INLINE void
sched_waiting_sys(Uint no, ErtsRunQueue *rq)
{
    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
    ASSERT(rq->waiting >= 0);
    rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK
		  | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);
    rq->waiting++;
    rq->waiting *= -1;
    rq->woken = 0;
    if (erts_system_profile_flags.scheduler)
	profile_scheduler(make_small(no), am_inactive);
}

static ERTS_INLINE void
sched_active_sys(Uint no, ErtsRunQueue *rq)
{
    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
    ASSERT(rq->waiting < 0);
    rq->waiting *= -1;
    rq->waiting--;
    if (erts_system_profile_flags.scheduler)
	profile_scheduler(make_small(no), am_active);
}

Uint
erts_active_schedulers(void)
{
    /* RRRRRRRRR */

    Uint as = erts_no_schedulers;

    ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting));

    ASSERT(as >= 0);
    return as;
}

static ERTS_INLINE int
prepare_for_sys_schedule(void)
{
#ifdef ERTS_SMP
    while (!erts_port_task_have_outstanding_io_tasks()
	   && !erts_smp_atomic32_xchg(&doing_sys_schedule, 1)) {
	if (!erts_port_task_have_outstanding_io_tasks())
	    return 1;
	erts_smp_atomic32_set(&doing_sys_schedule, 0);
    }
    return 0;
#else
    return !erts_port_task_have_outstanding_io_tasks();
#endif
}

#ifdef ERTS_SMP

static ERTS_INLINE void
sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
{
    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
    ASSERT(rq->waiting < 0);
    rq->waiting *= -1;
}

static ERTS_INLINE void
sched_waiting(Uint no, ErtsRunQueue *rq)
{
    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
    rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK
		  | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);
    if (rq->waiting < 0)
	rq->waiting--;
    else
	rq->waiting++;
    rq->woken = 0;
    if (erts_system_profile_flags.scheduler)
	profile_scheduler(make_small(no), am_inactive);
}

static ERTS_INLINE void
sched_active(Uint no, ErtsRunQueue *rq)
{
    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
    if (rq->waiting < 0)
	rq->waiting++;
    else
	rq->waiting--;
    if (erts_system_profile_flags.scheduler)
	profile_scheduler(make_small(no), am_active);
}

static int ERTS_INLINE
ongoing_multi_scheduling_block(void)
{
    return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0;
}

static ERTS_INLINE void
empty_runq(ErtsRunQueue *rq)
{
    erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags,
						 ~ERTS_RUNQ_IFLG_NONEMPTY);
    if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) {
#ifdef DEBUG
	erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues);
	/*
	 * For a short period of time no_empty_run_queues may have
	 * been increased twice for a specific run queue.
	 */
	ASSERT(0 <= empty && empty < 2*erts_no_run_queues);
#endif
	erts_smp_atomic32_inc(&no_empty_run_queues);
    }
}

static ERTS_INLINE void
non_empty_runq(ErtsRunQueue *rq)
{
    erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags,
						ERTS_RUNQ_IFLG_NONEMPTY);
    if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) {
#ifdef DEBUG
	erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues);
	/*
	 * For a short period of time no_empty_run_queues may have
	 * been increased twice for a specific run queue.
	 */
	ASSERT(0 < empty && empty <= 2*erts_no_run_queues);
#endif
	erts_smp_atomic32_dec(&no_empty_run_queues);
    }
}

static erts_aint32_t
sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t oflgs;
    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
			   | ERTS_SSI_FLG_WAITING);
    erts_aint32_t xflgs = 0;

    do {
	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
	if (oflgs == xflgs)
	    return nflgs;
	xflgs = oflgs;
    } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED));
    return oflgs;
}

static erts_aint32_t
sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t oflgs;
    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
			   | ERTS_SSI_FLG_WAITING);
    erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING;

    do {
	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
	if (oflgs == xflgs)
	    return nflgs;
	xflgs = oflgs;
	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
    } while (oflgs & ERTS_SSI_FLG_WAITING);
    return oflgs;
}

static erts_aint32_t
sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount)
{
    int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
    int sc = spincount;
    erts_aint32_t flgs;

    do {
	flgs = erts_smp_atomic32_read(&ssi->flags);
	if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
	    break;
	}
	ERTS_SPIN_BODY;
	if (--until_yield == 0) {
	    until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
	    erts_thr_yield();
	}
    } while (--sc > 0);
    return flgs;
}

static erts_aint32_t
sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
{
    erts_aint32_t oflgs;
    erts_aint32_t nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type;
    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;

    if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING)
	erts_tse_reset(ssi->event);

    while (1) {
	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
	if (oflgs == xflgs)
	    return nflgs;
	if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
	    return oflgs;
	}
	xflgs = oflgs;
	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
    }
}

#define ERTS_SCHED_WAIT_WOKEN(FLGS)				\
  (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED))	\
   != ERTS_SSI_FLG_WAITING)

static void
scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
{
    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
    int spincount;
    erts_aint32_t flgs;
#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \
    || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK)
    erts_aint32_t aux_work;
#endif

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    erts_smp_spin_lock(&rq->sleepers.lock);
    flgs = sched_prep_spin_wait(ssi);
    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
	/* Go suspend instead... */
	erts_smp_spin_unlock(&rq->sleepers.lock);
	return;
    }

    ssi->prev = NULL;
    ssi->next = rq->sleepers.list;
    if (rq->sleepers.list)
	rq->sleepers.list->prev = ssi;
    rq->sleepers.list = ssi;
    erts_smp_spin_unlock(&rq->sleepers.lock);

    /*
     * If all schedulers are waiting, one of them *should*
     * be waiting in erl_sys_schedule()
     */

    if (!prepare_for_sys_schedule()) {

	sched_waiting(esdp->no, rq);

	erts_smp_runq_unlock(rq);

	spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT;

    tse_wait:

#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
	aux_work = erts_smp_atomic32_read(&ssi->aux_work);
    tse_blockable_aux_work:
	aux_work = blockable_aux_work(esdp, ssi, aux_work);
#endif
	erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);

	while (1) {

#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
#endif
	    nonblockable_aux_work(esdp, ssi, aux_work);
#endif

	    flgs = sched_spin_wait(ssi, spincount);
	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
		if (flgs & ERTS_SSI_FLG_SLEEPING) {
		    int res;
		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
		    do {
			res = erts_tse_wait(ssi->event);
		    } while (res == EINTR);
		}
	    }

	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
		break;
	    }

	    flgs = sched_prep_cont_spin_wait(ssi);
	    spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT;

	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
		break;
	    }

#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
	    if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
		erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
		goto tse_blockable_aux_work;
	    }
#endif

	}

	erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);

	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
	    erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);

	erts_smp_runq_lock(rq);
	sched_active(esdp->no, rq);

    }
    else {
	erts_aint_t dt;

	erts_smp_atomic32_set(&function_calls, 0);
	*fcalls = 0;

	sched_waiting_sys(esdp->no, rq);

	erts_smp_runq_unlock(rq);

	spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT;

	while (spincount-- > 0) {

	sys_poll_aux_work:

	    ASSERT(!erts_port_task_have_outstanding_io_tasks());

	    erl_sys_schedule(1); /* Might give us something to do */

	    dt = erts_do_time_read_and_reset();
	    if (dt) erts_bump_timer(dt);

	sys_aux_work:

#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
	    aux_work = blockable_aux_work(esdp, ssi, aux_work);
#endif
#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
	    aux_work = erts_smp_atomic32_read(&ssi->aux_work);
#endif
	    nonblockable_aux_work(esdp, ssi, aux_work);
#endif

	    flgs = erts_smp_atomic32_read(&ssi->flags);
	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
		goto sys_woken;
	    }
	    if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
		flgs = sched_prep_cont_spin_wait(ssi);
		if (!(flgs & ERTS_SSI_FLG_WAITING)) {
		    ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
		    goto sys_woken;
		}
	    }

	    /*
	     * If we got new I/O tasks we aren't allowed to
	     * call erl_sys_schedule() until it is handled.
	     */
	    if (erts_port_task_have_outstanding_io_tasks()) {
		erts_smp_atomic32_set(&doing_sys_schedule, 0);
		/*
		 * Got to check that we still got I/O tasks; otherwise
		 * we have to continue checking for I/O...
		 */
		if (!prepare_for_sys_schedule()) {
		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
		    goto tse_wait;
		}
	    }
	}

	erts_smp_runq_lock(rq);

	/*
	 * If we got new I/O tasks we aren't allowed to
	 * sleep in erl_sys_schedule().
	 */
	if (erts_port_task_have_outstanding_io_tasks()) {
	    erts_smp_atomic32_set(&doing_sys_schedule, 0);

	    /*
	     * Got to check that we still got I/O tasks; otherwise
	     * we have to wait in erl_sys_schedule() after all...
	     */
	    if (prepare_for_sys_schedule())
		goto do_sys_schedule;

	    /*
	     * Not allowed to wait in erl_sys_schedule;
	     * do tse wait instead...
	     */
	    sched_change_waiting_sys_to_waiting(esdp->no, rq);
	    erts_smp_runq_unlock(rq);
	    spincount = 0;
	    goto tse_wait;
	}
	else {
	do_sys_schedule:
	    erts_sys_schedule_interrupt(0);
	    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
	    if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
		if (!(flgs & ERTS_SSI_FLG_WAITING))
		    goto sys_locked_woken;
		erts_smp_runq_unlock(rq);
		flgs = sched_prep_cont_spin_wait(ssi);
		if (!(flgs & ERTS_SSI_FLG_WAITING)) {
		    ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
		    goto sys_woken;
		}
		ASSERT(!erts_port_task_have_outstanding_io_tasks());
		goto sys_poll_aux_work;
	    }

	    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
	    ASSERT(flgs & ERTS_SSI_FLG_WAITING);

	    erts_smp_runq_unlock(rq);

	    ASSERT(!erts_port_task_have_outstanding_io_tasks());

	    erl_sys_schedule(0);

	    dt = erts_do_time_read_and_reset();
	    if (dt) erts_bump_timer(dt);

	    flgs = sched_prep_cont_spin_wait(ssi);
	    if (flgs & ERTS_SSI_FLG_WAITING)
		goto sys_aux_work;

	sys_woken:
	    erts_smp_runq_lock(rq);
	sys_locked_woken:
	    erts_smp_atomic32_set(&doing_sys_schedule, 0);
	    if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
		erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
	    sched_active_sys(esdp->no, rq);
	}
    }

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
}

static ERTS_INLINE erts_aint32_t
ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
{
    /* reset all flags but suspended */
    erts_aint32_t oflgs;
    erts_aint32_t nflgs = 0;
    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
    while (1) {
	oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
	if (oflgs == xflgs)
	    return oflgs;
	nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED;
	xflgs = oflgs;
    }
}

static void
wake_scheduler(ErtsRunQueue *rq, int incq, int one)
{
    int res;
    ErtsSchedulerSleepInfo *ssi;
    ErtsSchedulerSleepList *sl;

    /*
     * The unlocked run queue is not strictly necessary
     * from a thread safety or deadlock prevention
     * perspective. It will, however, cost us performance
     * if it is locked during wakup of another scheduler,
     * so all code *should* handle this without having
     * the lock on the run queue.
     */
    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq));

    sl = &rq->sleepers;

    erts_smp_spin_lock(&sl->lock);
    ssi = sl->list;
    if (!ssi)
	erts_smp_spin_unlock(&sl->lock);
    else if (one) {
	erts_aint32_t flgs;
	if (ssi->prev)
	    ssi->prev->next = ssi->next;
	else {
	    ASSERT(sl->list == ssi);
	    sl->list = ssi->next;
	}
	if (ssi->next)
	    ssi->next->prev = ssi->prev;

	res = sl->list != NULL;
	erts_smp_spin_unlock(&sl->lock);

	flgs = ssi_flags_set_wake(ssi);
	erts_sched_finish_poke(ssi, flgs);

	if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING))
	    non_empty_runq(rq);
    }
    else {
	sl->list = NULL;
	erts_smp_spin_unlock(&sl->lock);
	do {
	    ErtsSchedulerSleepInfo *wake_ssi = ssi;
	    ssi = ssi->next;
	    erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi));
	} while (ssi);
    }
}

static void
wake_all_schedulers(void)
{
    if (erts_common_run_queue)
	wake_scheduler(erts_common_run_queue, 0, 0);
    else {
	int ix;
	for (ix = 0; ix < erts_no_run_queues; ix++) {
	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
	    wake_scheduler(rq, 0, 1);
	}
    }
}

static ERTS_INLINE int
chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
{
    erts_aint32_t iflgs;
    ErtsRunQueue *wrq;
    if (crq->ix == ix)
	return 0;
    wrq = ERTS_RUNQ_IX(ix);
    iflgs = erts_smp_atomic32_read(&wrq->info_flags);
    if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
	if (activate) {
	    if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs,
						ix+1,
						ix)) {
		erts_smp_xrunq_lock(crq, wrq);
		wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE;
		erts_smp_xrunq_unlock(crq, wrq);
	    }
	}
	wake_scheduler(wrq, 0, 1);
	return 1;
    }
    return 0;
}

static void
wake_scheduler_on_empty_runq(ErtsRunQueue *crq)
{
    int ix = crq->ix;
    int stop_ix = ix;
    int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs);
    int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs);

    if (active_ix > balance_ix)
	active_ix = balance_ix;

    if (ix >= active_ix)
	stop_ix = ix = active_ix;

    /* Try to wake a scheduler on an active run queue */
    while (1) {
	ix--;
	if (ix < 0) {
	    if (active_ix == stop_ix)
		break;
	    ix = active_ix - 1;
	}
	if (ix == stop_ix)
	    break;
	if (chk_wake_sched(crq, ix, 0))
	    return;
    }

    if (active_ix < balance_ix) {
	/* Try to activate a new run queue and wake its scheduler */
	(void) chk_wake_sched(crq, active_ix, 1);
    }
}

#endif /* ERTS_SMP */

static ERTS_INLINE void
smp_notify_inc_runq(ErtsRunQueue *runq)
{
#ifdef ERTS_SMP
    if (runq)
	wake_scheduler(runq, 1, 1);
#endif
}

void
erts_smp_notify_inc_runq(ErtsRunQueue *runq)
{
    smp_notify_inc_runq(runq);
}

void
erts_sched_notify_check_cpu_bind(void)
{
#ifdef ERTS_SMP
    int ix;
    if (erts_common_run_queue) {
	for (ix = 0; ix < erts_no_schedulers; ix++)
	    erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1);
	wake_all_schedulers();
    }
    else {
	for (ix = 0; ix < erts_no_run_queues; ix++) {
	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
	    erts_smp_runq_lock(rq);
	    rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
	    erts_smp_runq_unlock(rq);
	    wake_scheduler(rq, 0, 1);
	};
    }
#else
    erts_sched_check_cpu_bind(erts_get_scheduler_data());
#endif
}


#ifdef ERTS_SMP

ErtsRunQueue *
erts_prepare_emigrate(ErtsRunQueue *c_rq, ErtsRunQueueInfo *c_rqi, int prio)
{
    ASSERT(ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio));
    ASSERT(ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio)
	   || c_rqi->len >= c_rqi->migrate.limit.this);

    while (1) {
	ErtsRunQueue *n_rq = c_rqi->migrate.runq;
	ERTS_DBG_VERIFY_VALID_RUNQP(n_rq);
	erts_smp_xrunq_lock(c_rq, n_rq);
	    
	/*
	 * erts_smp_xrunq_lock() may release lock on c_rq! We have
	 * to check that we still want to emigrate and emigrate
	 * to the same run queue as before.
	 */

	if (ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) {
	    Uint32 force = (ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio)
			    | (c_rq->flags & ERTS_RUNQ_FLG_INACTIVE));
	    if (force || c_rqi->len > c_rqi->migrate.limit.this) {
		ErtsRunQueueInfo *n_rqi;
		/* We still want to emigrate */

		if (n_rq != c_rqi->migrate.runq) {
		    /* Ahh... run queue changed; need to do it all over again... */
		    erts_smp_runq_unlock(n_rq);
		    continue;
		}
		else {

		    if (prio == ERTS_PORT_PRIO_LEVEL)
			n_rqi = &n_rq->ports.info;
		    else
			n_rqi = &n_rq->procs.prio_info[prio];

		    if (force || (n_rqi->len < c_rqi->migrate.limit.other)) {
			/* emigrate ... */
			return n_rq;
		    }
		}
	    }
	}

	ASSERT(n_rq != c_rq);
	erts_smp_runq_unlock(n_rq);
	if (!(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)) {
	    /* No more emigrations to this runq */
	    ERTS_UNSET_RUNQ_FLG_EMIGRATE(c_rq->flags, prio);
	    ERTS_DBG_SET_INVALID_RUNQP(c_rqi->migrate.runq, 0x3);
	}

	return NULL;
    }
}

static void
immigrate(ErtsRunQueue *rq)
{
    int prio;

    ASSERT(rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK);

    for (prio = 0; prio < ERTS_NO_PRIO_LEVELS; prio++) {
	if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)) {
	    ErtsRunQueueInfo *rqi = (prio == ERTS_PORT_PRIO_LEVEL
				     ? &rq->ports.info
				     : &rq->procs.prio_info[prio]);
	    ErtsRunQueue *from_rq = rqi->migrate.runq;
	    int rq_locked, from_rq_locked;

	    ERTS_DBG_VERIFY_VALID_RUNQP(from_rq);

	    rq_locked = 1;
	    from_rq_locked = 1;
	    erts_smp_xrunq_lock(rq, from_rq);
	    /*
	     * erts_smp_xrunq_lock() may release lock on rq! We have
	     * to check that we still want to immigrate from the same 
	     * run queue as before.
	     */
	    if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)
		&& from_rq == rqi->migrate.runq) {
		ErtsRunQueueInfo *from_rqi = (prio == ERTS_PORT_PRIO_LEVEL
					      ? &from_rq->ports.info
					      : &from_rq->procs.prio_info[prio]);
		if ((ERTS_CHK_RUNQ_FLG_EVACUATE(rq->flags, prio)
		     && ERTS_CHK_RUNQ_FLG_EVACUATE(from_rq->flags, prio)
		     && from_rqi->len)
		    || (from_rqi->len > rqi->migrate.limit.other
			&& rqi->len < rqi->migrate.limit.this)) {
		    if (prio == ERTS_PORT_PRIO_LEVEL) {
			Port *prt = from_rq->ports.start;
			if (prt) {
			    int prt_locked = 0;
			    (void) erts_port_migrate(prt, &prt_locked,
						     from_rq, &from_rq_locked,
						     rq, &rq_locked);
			    if (prt_locked)
				erts_smp_port_unlock(prt);
			}
		    }
		    else {
			Process *proc;
			ErtsRunPrioQueue *from_rpq;
			from_rpq = (prio == PRIORITY_LOW
				    ? &from_rq->procs.prio[PRIORITY_NORMAL]
				    : &from_rq->procs.prio[prio]);
			for (proc = from_rpq->first; proc; proc = proc->next)
			    if (proc->prio == prio && !proc->bound_runq)
				break;
			if (proc) {
			    ErtsProcLocks proc_locks = 0;
			    (void) erts_proc_migrate(proc, &proc_locks,
						     from_rq, &from_rq_locked,
						     rq, &rq_locked);
			    if (proc_locks)
				erts_smp_proc_unlock(proc, proc_locks);
			}
		    }
		}
		else {
		    ERTS_UNSET_RUNQ_FLG_IMMIGRATE(rq->flags, prio);
		    ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x1);
		}
	    }
	    if (from_rq_locked)
		erts_smp_runq_unlock(from_rq);
	    if (!rq_locked)
		erts_smp_runq_lock(rq);
	}
    }
}

static void
evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
{
    Port *prt;
    int notify_to_rq = 0;
    int prio;
    int prt_locked = 0;
    int rq_locked = 0;
    int evac_rq_locked = 1;
    ErtsMigrateResult mres;

    erts_smp_runq_lock(evac_rq);

    erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags,
			  ERTS_SSI_FLG_SUSPENDED);

    evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK;
    evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK
		       | ERTS_RUNQ_FLGS_EVACUATE_QMASK
		       | ERTS_RUNQ_FLG_SUSPENDED);

    erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
    /*
     * Need to set up evacuation paths first since we
     * may release the run queue lock on evac_rq
     * when evacuating.
     */
    evac_rq->misc.evac_runq = rq;
    evac_rq->ports.info.migrate.runq = rq;
    for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++)
	evac_rq->procs.prio_info[prio].migrate.runq = rq;

    /* Evacuate scheduled misc ops */

    if (evac_rq->misc.start) {
	rq_locked = 1;
	erts_smp_xrunq_lock(evac_rq, rq);
	if (rq->misc.end)
	    rq->misc.end->next = evac_rq->misc.start;
	else
	    rq->misc.start = evac_rq->misc.start;
	rq->misc.end = evac_rq->misc.end;
	evac_rq->misc.start = NULL;
	evac_rq->misc.end = NULL;
    }

    /* Evacuate scheduled ports */
    prt = evac_rq->ports.start;
    while (prt) {
	mres = erts_port_migrate(prt, &prt_locked,
				 evac_rq, &evac_rq_locked,
				 rq, &rq_locked);
	if (mres == ERTS_MIGRATE_SUCCESS)
	    notify_to_rq = 1;
	if (prt_locked)
	    erts_smp_port_unlock(prt);
	if (!evac_rq_locked) {
	    evac_rq_locked = 1;
	    erts_smp_runq_lock(evac_rq);
	}
	prt = evac_rq->ports.start;
    }

    /* Evacuate scheduled processes */
    for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) {
	Process *proc;

	switch (prio) {
	case PRIORITY_MAX:
	case PRIORITY_HIGH:
	case PRIORITY_NORMAL:
	    proc = evac_rq->procs.prio[prio].first;
	    while (proc) {
		ErtsProcLocks proc_locks = 0;

		/* Bound processes are stuck... */
		while (proc->bound_runq) {
		    proc = proc->next;
		    if (!proc)
			goto end_of_proc;
		}

		mres = erts_proc_migrate(proc, &proc_locks,
					 evac_rq, &evac_rq_locked,
					 rq, &rq_locked);
		if (mres == ERTS_MIGRATE_SUCCESS)
		    notify_to_rq = 1;
		if (proc_locks)
		    erts_smp_proc_unlock(proc, proc_locks);
		if (!evac_rq_locked) {
		    erts_smp_runq_lock(evac_rq);
		    evac_rq_locked = 1;
		}

		proc = evac_rq->procs.prio[prio].first;
	    }

	end_of_proc:

#ifdef DEBUG
	    for (proc = evac_rq->procs.prio[prio].first;
		 proc;
		 proc = proc->next) {
		ASSERT(proc->bound_runq);
	    }
#endif
	    break;
	case PRIORITY_LOW:
	    break;
	default:
	    ASSERT(!"Invalid process priority");
	    break;
	}
    }

    if (rq_locked)
	erts_smp_runq_unlock(rq);

    if (evac_rq_locked)
	erts_smp_runq_unlock(evac_rq);

    if (notify_to_rq)
	smp_notify_inc_runq(rq);

    wake_scheduler(evac_rq, 0, 1);
}

static int
try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq)
{
    Process *proc;
    int vrq_locked;

    if (*rq_lockedp)
	erts_smp_xrunq_lock(rq, vrq);
    else
	erts_smp_runq_lock(vrq);
    vrq_locked = 1;

    ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
    ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);

    /*
     * Check for a runnable process to steal...
     */

    switch (vrq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) {
    case MAX_BIT:
    case MAX_BIT|HIGH_BIT:
    case MAX_BIT|NORMAL_BIT:
    case MAX_BIT|LOW_BIT:
    case MAX_BIT|HIGH_BIT|NORMAL_BIT:
    case MAX_BIT|HIGH_BIT|LOW_BIT:
    case MAX_BIT|NORMAL_BIT|LOW_BIT:
    case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT:
	for (proc = vrq->procs.prio[PRIORITY_MAX].last;
	     proc;
	     proc = proc->prev) {
	    if (!proc->bound_runq)
		break;
	}
	if (proc)
	    break;
    case HIGH_BIT:
    case HIGH_BIT|NORMAL_BIT:
    case HIGH_BIT|LOW_BIT:
    case HIGH_BIT|NORMAL_BIT|LOW_BIT:
	for (proc = vrq->procs.prio[PRIORITY_HIGH].last;
	     proc;
	     proc = proc->prev) {
	    if (!proc->bound_runq)
		break;
	}
	if (proc)
	    break;
    case NORMAL_BIT:
    case LOW_BIT:
    case NORMAL_BIT|LOW_BIT:
	for (proc = vrq->procs.prio[PRIORITY_NORMAL].last;
	     proc;
	     proc = proc->prev) {
	    if (!proc->bound_runq)
		break;
	}
	if (proc)
	    break;
    case 0:
	proc = NULL;
	break;
    default:
	ASSERT(!"Invalid queue mask");
	proc = NULL;
	break;
    }

    if (proc) {
	ErtsProcLocks proc_locks = 0;
	int res;
	ErtsMigrateResult mres;
	mres = erts_proc_migrate(proc, &proc_locks,
				 vrq, &vrq_locked,
				 rq, rq_lockedp);
	if (proc_locks)
	    erts_smp_proc_unlock(proc, proc_locks);
	res = !0;
	switch (mres) {
	case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED:
	    res = 0;
	case ERTS_MIGRATE_SUCCESS:
	    if (vrq_locked)
		erts_smp_runq_unlock(vrq);
	    return res;
	default: /* Other failures */
	    break;			
	}
    }

    ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
    ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);

    if (!vrq_locked) {
	if (*rq_lockedp)
	    erts_smp_xrunq_lock(rq, vrq);
	else
	    erts_smp_runq_lock(vrq);
	vrq_locked = 1;
    }

    ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
    ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);

    /*
     * Check for a runnable port to steal...
     */

    if (vrq->ports.info.len) {
	Port *prt = vrq->ports.end;
	int prt_locked = 0;
	int res;
	ErtsMigrateResult mres;

	mres = erts_port_migrate(prt, &prt_locked,
				 vrq, &vrq_locked,
				 rq, rq_lockedp);
	if (prt_locked)
	    erts_smp_port_unlock(prt);
	res = !0;
	switch (mres) {
	case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED:
	    res = 0;
	case ERTS_MIGRATE_SUCCESS:
	    if (vrq_locked)
		erts_smp_runq_unlock(vrq);
	    return res;
	default: /* Other failures */
	    break;			
	}
    }

    if (vrq_locked)
	erts_smp_runq_unlock(vrq);

    return 0;
}


static ERTS_INLINE int
check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
{
    ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
    erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags);
    if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY)
	return try_steal_task_from_victim(rq, rq_lockedp, vrq);
    else
	return 0;
}


static int
try_steal_task(ErtsRunQueue *rq)
{
    int res, rq_locked, vix, active_rqs, blnc_rqs;
    
    if (erts_common_run_queue)
	return 0;

    /*
     * We are not allowed to steal jobs to this run queue
     * if it is suspended. Note that it might get suspended
     * at any time when we don't have the lock on the run
     * queue.
     */
    if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
	return 0;

    res = 0;
    rq_locked = 1;

    ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked);

    active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs);
    blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs);

    if (active_rqs > blnc_rqs)
	active_rqs = blnc_rqs;

    if (rq->ix < active_rqs) {

	/* First try to steal from an inactive run queue... */
	if (active_rqs < blnc_rqs) {
	    int no = blnc_rqs - active_rqs;
	    int stop_ix = vix = active_rqs + rq->ix % no;
	    while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) {
		res = check_possible_steal_victim(rq, &rq_locked, vix);
		if (res)
		    goto done;
		vix++;
		if (vix >= blnc_rqs)
		    vix = active_rqs;
		if (vix == stop_ix)
		    break;
	    }
	}

	vix = rq->ix;

	/* ... then try to steal a job from another active queue... */
	while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) {
	    vix++;
	    if (vix >= active_rqs)
		vix = 0;
	    if (vix == rq->ix)
		break;

	    res = check_possible_steal_victim(rq, &rq_locked, vix);
	    if (res)
		goto done;
	}

    }

 done:

    if (!rq_locked)
	erts_smp_runq_lock(rq);

    if (!res)
	res = !ERTS_EMPTY_RUNQ(rq);

    return res;
}

/* Run queue balancing */

typedef struct {
    Uint32 flags;
    struct {
	int max_len;
	int avail;
	int reds;
	int migration_limit;
	int emigrate_to;
	int immigrate_from;
    } prio[ERTS_NO_PRIO_LEVELS];
    int reds;
    int full_reds;
    int full_reds_history_sum;
    int full_reds_history_change;
    int oowc;
    int max_len;
} ErtsRunQueueBalance;
static ErtsRunQueueBalance *run_queue_info;

typedef struct {
    int qix;
    int len;
} ErtsRunQueueCompare;
static ErtsRunQueueCompare *run_queue_compare;

static int
rqc_len_cmp(const void *x, const void *y)
{
    return ((ErtsRunQueueCompare *) x)->len - ((ErtsRunQueueCompare *) y)->len;
}

#define ERTS_PERCENT(X, Y) \
  ((Y) == 0 \
   ? ((X) == 0 ? 100 : INT_MAX) \
   : ((100*(X))/(Y)))

#define ERTS_UPDATE_FULL_REDS(QIX, LAST_REDS)				\
do {									\
    run_queue_info[(QIX)].full_reds					\
	= run_queue_info[(QIX)].full_reds_history_sum;			\
    run_queue_info[(QIX)].full_reds += (LAST_REDS);			\
    run_queue_info[(QIX)].full_reds					\
	>>= ERTS_FULL_REDS_HISTORY_AVG_SHFT;				\
    run_queue_info[(QIX)].full_reds_history_sum				\
	-= run_queue_info[(QIX)].full_reds_history_change;		\
    run_queue_info[(QIX)].full_reds_history_sum += (LAST_REDS);		\
    run_queue_info[(QIX)].full_reds_history_change = (LAST_REDS);	\
} while (0)

#define ERTS_DBG_CHK_FULL_REDS_HISTORY(RQ)				\
do {									\
    int sum__ = 0;							\
    int rix__;								\
    for (rix__ = 0; rix__ < ERTS_FULL_REDS_HISTORY_SIZE; rix__++)	\
	sum__ += (RQ)->full_reds_history[rix__];			\
    ASSERT(sum__ == (RQ)->full_reds_history_sum);			\
} while (0);

static void
check_balance(ErtsRunQueue *c_rq)
{
#if ERTS_MAX_PROCESSES >= (1 << 27)
#  error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27)
#endif
    ErtsRunQueueBalance avg = {0};
    Sint64 scheds_reds, full_scheds_reds;
    int forced, active, current_active, oowc, half_full_scheds, full_scheds,
	mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix;

    if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) {
	c_rq->check_balance_reds = INT_MAX;
	return;
    }

    blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs);
    if (blnc_no_rqs == 1) {
	c_rq->check_balance_reds = INT_MAX;
	erts_smp_atomic32_set(&balance_info.checking_balance, 0);
	return;
    }

    erts_smp_runq_unlock(c_rq);

    if (balance_info.halftime) {	
	balance_info.halftime = 0;
	erts_smp_atomic32_set(&balance_info.checking_balance, 0);
	ERTS_FOREACH_RUNQ(rq,
	{
	    if (rq->waiting)
		rq->flags |= ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK;
	    else
		rq->flags &= ~ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK;
	    rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
	});

	erts_smp_runq_lock(c_rq);
	return;
   …