/Python/eval.cc
http://unladen-swallow.googlecode.com/ · C++ · 5498 lines · 4329 code · 450 blank · 719 comment · 945 complexity · 3ed53cf038a21678d0dfee98cdc32199 MD5 · raw file
Large files are truncated click here to view the full file
- /* Execute compiled code */
- /* XXX TO DO:
- XXX speed up searching for keywords by using a dictionary
- XXX document it!
- */
- /* Note: this file will be compiled as C ifndef WITH_LLVM, so try to keep it
- generally C. */
- /* enable more aggressive intra-module optimizations, where available */
- #define PY_LOCAL_AGGRESSIVE
- #include "Python.h"
- #include "code.h"
- #include "frameobject.h"
- #include "eval.h"
- #include "opcode.h"
- #include "structmember.h"
- #include "JIT/llvm_compile.h"
- #include "Util/EventTimer.h"
- #include <ctype.h>
- #ifdef WITH_LLVM
- #include "_llvmfunctionobject.h"
- #include "llvm/Function.h"
- #include "llvm/Support/ManagedStatic.h"
- #include "llvm/Support/raw_ostream.h"
- #include "JIT/global_llvm_data.h"
- #include "JIT/RuntimeFeedback.h"
- #include "Util/Stats.h"
- #include <set>
- using llvm::errs;
- #endif
- /* Make a call to stop the call overhead timer before going through to
- PyObject_Call. */
- static inline PyObject *
- _PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw)
- {
- /* If we're calling a compiled C function with *args or **kwargs, then
- * this enum should be CALL_ENTER_C. However, most calls to C
- * functions are simple and are fast-tracked through the CALL_FUNCTION
- * opcode. */
- PY_LOG_TSC_EVENT(CALL_ENTER_PYOBJ_CALL);
- return PyObject_Call(func, arg, kw);
- }
- #ifdef Py_WITH_INSTRUMENTATION
- std::string
- _PyEval_GetCodeName(PyCodeObject *code)
- {
- std::string result;
- llvm::raw_string_ostream wrapper(result);
- wrapper << PyString_AsString(code->co_filename)
- << ":" << code->co_firstlineno << " "
- << "(" << PyString_AsString(code->co_name) << ")";
- wrapper.flush();
- return result;
- }
- // Collect statistics about how long we block for compilation to LLVM IR and to
- // machine code.
- class IrCompilationTimes : public DataVectorStats<int64_t> {
- public:
- IrCompilationTimes()
- : DataVectorStats<int64_t>("Time blocked for IR JIT in ns") {}
- };
- class McCompilationTimes : public DataVectorStats<int64_t> {
- public:
- McCompilationTimes()
- : DataVectorStats<int64_t>("Time blocked for MC JIT in ns") {}
- };
- static llvm::ManagedStatic<IrCompilationTimes> ir_compilation_times;
- static llvm::ManagedStatic<McCompilationTimes> mc_compilation_times;
- class FeedbackMapCounter {
- public:
- ~FeedbackMapCounter() {
- errs() << "\nFeedback maps created:\n";
- errs() << "N: " << this->counter_ << "\n";
- }
- void IncCounter() {
- this->counter_++;
- }
- private:
- unsigned counter_;
- };
- static llvm::ManagedStatic<FeedbackMapCounter> feedback_map_counter;
- class HotnessTracker {
- // llvm::DenseSet or llvm::SmallPtrSet may be better, but as of this
- // writing, they don't seem to work with std::vector.
- std::set<PyCodeObject*> hot_code_;
- public:
- ~HotnessTracker();
- void AddHotCode(PyCodeObject *code_obj) {
- // This will prevent the code object from ever being
- // deleted.
- Py_INCREF(code_obj);
- this->hot_code_.insert(code_obj);
- }
- };
- static bool
- compare_hotness(const PyCodeObject *first, const PyCodeObject *second)
- {
- return first->co_hotness > second->co_hotness;
- }
- HotnessTracker::~HotnessTracker()
- {
- errs() << "\nCode objects deemed hot:\n";
- errs() << "N: " << this->hot_code_.size() << "\n";
- errs() << "Function -> hotness score:\n";
- std::vector<PyCodeObject*> to_sort(this->hot_code_.begin(),
- this->hot_code_.end());
- std::sort(to_sort.begin(), to_sort.end(), compare_hotness);
- for (std::vector<PyCodeObject*>::iterator co = to_sort.begin();
- co != to_sort.end(); ++co) {
- errs() << _PyEval_GetCodeName(*co)
- << " -> " << (*co)->co_hotness << "\n";
- }
- }
- static llvm::ManagedStatic<HotnessTracker> hot_code;
- // Keep track of which functions failed fatal guards, but kept being called.
- // This can help gauge the efficacy of optimizations that involve fatal guards.
- class FatalBailTracker {
- public:
- ~FatalBailTracker() {
- errs() << "\nCode objects that failed fatal guards:\n";
- errs() << "\tfile:line (funcname) bail hotness"
- << " -> final hotness\n";
- for (TrackerData::const_iterator it = this->code_.begin();
- it != this->code_.end(); ++it) {
- PyCodeObject *code = it->first;
- if (code->co_hotness == it->second)
- continue;
- errs() << "\t" << _PyEval_GetCodeName(code)
- << "\t" << it->second << " -> "
- << code->co_hotness << "\n";
- }
- }
- void RecordFatalBail(PyCodeObject *code) {
- Py_INCREF(code);
- this->code_.push_back(std::make_pair(code, code->co_hotness));
- }
- private:
- // Keep a list of (code object, hotness) where hotness is the
- // value of co_hotness when RecordFatalBail() was called. This is
- // used to hide code objects whose machine code functions are
- // invalidated during shutdown because their module dict has gone away;
- // these code objects are uninteresting for our analysis.
- typedef std::pair<PyCodeObject *, long> DataPoint;
- typedef std::vector<DataPoint> TrackerData;
- TrackerData code_;
- };
- static llvm::ManagedStatic<FatalBailTracker> fatal_bail_tracker;
- // C wrapper for FatalBailTracker::RecordFatalBail().
- void
- _PyEval_RecordFatalBail(PyCodeObject *code)
- {
- fatal_bail_tracker->RecordFatalBail(code);
- }
- // Collect stats on how many watchers the globals/builtins dicts acculumate.
- // This currently records how many watchers the dict had when it changed, ie,
- // how many watchers it had to notify.
- class WatcherCountStats : public DataVectorStats<size_t> {
- public:
- WatcherCountStats() :
- DataVectorStats<size_t>("Number of watchers accumulated") {};
- };
- static llvm::ManagedStatic<WatcherCountStats> watcher_count_stats;
- void
- _PyEval_RecordWatcherCount(size_t watcher_count)
- {
- watcher_count_stats->RecordDataPoint(watcher_count);
- }
- class BailCountStats {
- public:
- BailCountStats() : total_(0), trace_on_entry_(0), line_trace_(0),
- backedge_trace_(0), call_profile_(0),
- fatal_guard_fail_(0), guard_fail_(0) {};
- ~BailCountStats() {
- errs() << "\nBailed to the interpreter " << this->total_
- << " times:\n";
- errs() << "TRACE_ON_ENTRY: " << this->trace_on_entry_ << "\n";
- errs() << "LINE_TRACE: " << this->line_trace_ << "\n";
- errs() << "BACKEDGE_TRACE:" << this->backedge_trace_ << "\n";
- errs() << "CALL_PROFILE: " << this->call_profile_ << "\n";
- errs() << "FATAL_GUARD_FAIL: " << this->fatal_guard_fail_
- << "\n";
- errs() << "GUARD_FAIL: " << this->guard_fail_ << "\n";
- errs() << "\n" << this->bail_site_freq_.size()
- << " bail sites:\n";
- for (BailData::iterator i = this->bail_site_freq_.begin(),
- end = this->bail_site_freq_.end(); i != end; ++i) {
- errs() << " " << i->getKey() << " bailed "
- << i->getValue() << " times\n";
- }
- errs() << "\n" << this->guard_bail_site_freq_.size()
- << " guard bail sites:\n";
- for (BailData::iterator i = this->guard_bail_site_freq_.begin(),
- end = this->guard_bail_site_freq_.end(); i != end; ++i) {
- errs() << " " << i->getKey() << " bailed "
- << i->getValue() << " times\n";
- }
- }
- void RecordBail(PyFrameObject *frame, _PyFrameBailReason bail_reason) {
- ++this->total_;
- std::string record;
- llvm::raw_string_ostream wrapper(record);
- wrapper << PyString_AsString(frame->f_code->co_filename) << ":";
- wrapper << frame->f_code->co_firstlineno << ":";
- wrapper << PyString_AsString(frame->f_code->co_name) << ":";
- // See the comment in PyEval_EvalFrame about how f->f_lasti is
- // initialized.
- wrapper << frame->f_lasti + 1;
- wrapper.flush();
- BailData::value_type &entry =
- this->bail_site_freq_.GetOrCreateValue(record, 0);
- entry.setValue(entry.getValue() + 1);
- #define BAIL_CASE(name, field) \
- case name: \
- ++this->field; \
- break;
- switch (bail_reason) {
- BAIL_CASE(_PYFRAME_TRACE_ON_ENTRY, trace_on_entry_)
- BAIL_CASE(_PYFRAME_LINE_TRACE, line_trace_)
- BAIL_CASE(_PYFRAME_BACKEDGE_TRACE, backedge_trace_)
- BAIL_CASE(_PYFRAME_CALL_PROFILE, call_profile_)
- BAIL_CASE(_PYFRAME_FATAL_GUARD_FAIL, fatal_guard_fail_)
- BAIL_CASE(_PYFRAME_GUARD_FAIL, guard_fail_)
- default:
- abort(); // Unknown bail reason.
- }
- #undef BAIL_CASE
- if (bail_reason != _PYFRAME_GUARD_FAIL)
- return;
- wrapper << ":";
- #define GUARD_CASE(name) \
- case name: \
- wrapper << #name; \
- break;
- switch (frame->f_guard_type) {
- GUARD_CASE(_PYGUARD_DEFAULT)
- GUARD_CASE(_PYGUARD_BINOP)
- GUARD_CASE(_PYGUARD_ATTR)
- GUARD_CASE(_PYGUARD_CFUNC)
- GUARD_CASE(_PYGUARD_BRANCH)
- GUARD_CASE(_PYGUARD_STORE_SUBSCR)
- default:
- wrapper << ((int)frame->f_guard_type);
- }
- #undef GUARD_CASE
- wrapper.flush();
- BailData::value_type &g_entry =
- this->guard_bail_site_freq_.GetOrCreateValue(record, 0);
- g_entry.setValue(g_entry.getValue() + 1);
- }
- private:
- typedef llvm::StringMap<unsigned> BailData;
- BailData bail_site_freq_;
- BailData guard_bail_site_freq_;
- long total_;
- long trace_on_entry_;
- long line_trace_;
- long backedge_trace_;
- long call_profile_;
- long fatal_guard_fail_;
- long guard_fail_;
- };
- static llvm::ManagedStatic<BailCountStats> bail_count_stats;
- #endif // Py_WITH_INSTRUMENTATION
- /* Turn this on if your compiler chokes on the big switch: */
- /* #define CASE_TOO_BIG 1 */
- #ifdef Py_DEBUG
- /* For debugging the interpreter: */
- #define LLTRACE 1 /* Low-level trace feature */
- #define CHECKEXC 1 /* Double-check exception checking */
- #endif
- typedef PyObject *(*callproc)(PyObject *, PyObject *, PyObject *);
- /* Forward declarations */
- static PyObject * fast_function(PyObject *, PyObject ***, int, int, int);
- static PyObject * do_call(PyObject *, PyObject ***, int, int);
- static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int);
- static PyObject * update_keyword_args(PyObject *, int, PyObject ***,
- PyObject *);
- static PyObject * update_star_args(int, int, PyObject *, PyObject ***);
- static PyObject * load_args(PyObject ***, int);
- #ifdef WITH_LLVM
- static inline void mark_called(PyCodeObject *co);
- static inline int maybe_compile(PyCodeObject *co, PyFrameObject *f);
- /* Record data for use in generating optimized machine code. */
- static void record_type(PyCodeObject *, int, int, int, PyObject *);
- static void record_func(PyCodeObject *, int, int, int, PyObject *);
- static void record_object(PyCodeObject *, int, int, int, PyObject *);
- static void inc_feedback_counter(PyCodeObject *, int, int, int, int);
- #endif /* WITH_LLVM */
- int _Py_ProfilingPossible = 0;
- /* Keep this in sync with llvm_fbuilder.cc */
- #define CALL_FLAG_VAR 1
- #define CALL_FLAG_KW 2
- #ifdef LLTRACE
- static int lltrace;
- static int prtrace(PyObject *, char *);
- #endif
- static int call_trace_protected(Py_tracefunc, PyObject *,
- PyFrameObject *, int, PyObject *);
- static int maybe_call_line_trace(Py_tracefunc, PyObject *,
- PyFrameObject *, int *, int *, int *);
- static PyObject * cmp_outcome(int, PyObject *, PyObject *);
- static void format_exc_check_arg(PyObject *, char *, PyObject *);
- static PyObject * string_concatenate(PyObject *, PyObject *,
- PyFrameObject *, unsigned char *);
- #define NAME_ERROR_MSG \
- "name '%.200s' is not defined"
- #define GLOBAL_NAME_ERROR_MSG \
- "global name '%.200s' is not defined"
- #define UNBOUNDLOCAL_ERROR_MSG \
- "local variable '%.200s' referenced before assignment"
- #define UNBOUNDFREE_ERROR_MSG \
- "free variable '%.200s' referenced before assignment" \
- " in enclosing scope"
- /* Dynamic execution profile */
- #ifdef DYNAMIC_EXECUTION_PROFILE
- #ifdef DXPAIRS
- static long dxpairs[257][256];
- #define dxp dxpairs[256]
- #else
- static long dxp[256];
- #endif
- #endif
- /* Function call profile */
- #ifdef CALL_PROFILE
- #define PCALL_NUM 11
- static int pcall[PCALL_NUM];
- #define PCALL_ALL 0
- #define PCALL_FUNCTION 1
- #define PCALL_FAST_FUNCTION 2
- #define PCALL_FASTER_FUNCTION 3
- #define PCALL_METHOD 4
- #define PCALL_BOUND_METHOD 5
- #define PCALL_CFUNCTION 6
- #define PCALL_TYPE 7
- #define PCALL_GENERATOR 8
- #define PCALL_OTHER 9
- #define PCALL_POP 10
- /* Notes about the statistics
- PCALL_FAST stats
- FAST_FUNCTION means no argument tuple needs to be created.
- FASTER_FUNCTION means that the fast-path frame setup code is used.
- If there is a method call where the call can be optimized by changing
- the argument tuple and calling the function directly, it gets recorded
- twice.
- As a result, the relationship among the statistics appears to be
- PCALL_ALL == PCALL_FUNCTION + PCALL_METHOD - PCALL_BOUND_METHOD +
- PCALL_CFUNCTION + PCALL_TYPE + PCALL_GENERATOR + PCALL_OTHER
- PCALL_FUNCTION > PCALL_FAST_FUNCTION > PCALL_FASTER_FUNCTION
- PCALL_METHOD > PCALL_BOUND_METHOD
- */
- #define PCALL(POS) pcall[POS]++
- PyObject *
- PyEval_GetCallStats(PyObject *self)
- {
- return Py_BuildValue("iiiiiiiiiiiii",
- pcall[0], pcall[1], pcall[2], pcall[3],
- pcall[4], pcall[5], pcall[6], pcall[7],
- pcall[8], pcall[9], pcall[10]);
- }
- #else
- #define PCALL(O)
- PyObject *
- PyEval_GetCallStats(PyObject *self)
- {
- Py_INCREF(Py_None);
- return Py_None;
- }
- #endif
- #ifdef WITH_THREAD
- #ifdef HAVE_ERRNO_H
- #include <errno.h>
- #endif
- #include "pythread.h"
- static PyThread_type_lock interpreter_lock = 0; /* This is the GIL */
- long _PyEval_main_thread = 0;
- int
- PyEval_ThreadsInitialized(void)
- {
- return interpreter_lock != 0;
- }
- void
- PyEval_InitThreads(void)
- {
- if (interpreter_lock)
- return;
- interpreter_lock = PyThread_allocate_lock();
- PyThread_acquire_lock(interpreter_lock, 1);
- _PyEval_main_thread = PyThread_get_thread_ident();
- }
- void
- PyEval_AcquireLock(void)
- {
- PyThread_acquire_lock(interpreter_lock, 1);
- }
- void
- PyEval_ReleaseLock(void)
- {
- PyThread_release_lock(interpreter_lock);
- }
- void
- PyEval_AcquireThread(PyThreadState *tstate)
- {
- if (tstate == NULL)
- Py_FatalError("PyEval_AcquireThread: NULL new thread state");
- /* Check someone has called PyEval_InitThreads() to create the lock */
- assert(interpreter_lock);
- PyThread_acquire_lock(interpreter_lock, 1);
- if (PyThreadState_Swap(tstate) != NULL)
- Py_FatalError(
- "PyEval_AcquireThread: non-NULL old thread state");
- }
- void
- PyEval_ReleaseThread(PyThreadState *tstate)
- {
- if (tstate == NULL)
- Py_FatalError("PyEval_ReleaseThread: NULL thread state");
- if (PyThreadState_Swap(NULL) != tstate)
- Py_FatalError("PyEval_ReleaseThread: wrong thread state");
- PyThread_release_lock(interpreter_lock);
- }
- /* This function is called from PyOS_AfterFork to ensure that newly
- created child processes don't hold locks referring to threads which
- are not running in the child process. (This could also be done using
- pthread_atfork mechanism, at least for the pthreads implementation.) */
- void
- PyEval_ReInitThreads(void)
- {
- PyObject *threading, *result;
- PyThreadState *tstate;
- if (!interpreter_lock)
- return;
- /*XXX Can't use PyThread_free_lock here because it does too
- much error-checking. Doing this cleanly would require
- adding a new function to each thread_*.h. Instead, just
- create a new lock and waste a little bit of memory */
- interpreter_lock = PyThread_allocate_lock();
- PyThread_acquire_lock(interpreter_lock, 1);
- _PyEval_main_thread = PyThread_get_thread_ident();
- /* Update the threading module with the new state.
- */
- tstate = PyThreadState_GET();
- threading = PyMapping_GetItemString(tstate->interp->modules,
- "threading");
- if (threading == NULL) {
- /* threading not imported */
- PyErr_Clear();
- return;
- }
- result = PyObject_CallMethod(threading, "_after_fork", NULL);
- if (result == NULL)
- PyErr_WriteUnraisable(threading);
- else
- Py_DECREF(result);
- Py_DECREF(threading);
- }
- #endif
- /* Functions save_thread and restore_thread are always defined so
- dynamically loaded modules needn't be compiled separately for use
- with and without threads: */
- PyThreadState *
- PyEval_SaveThread(void)
- {
- PyThreadState *tstate = PyThreadState_Swap(NULL);
- if (tstate == NULL)
- Py_FatalError("PyEval_SaveThread: NULL tstate");
- #ifdef WITH_THREAD
- if (interpreter_lock)
- PyThread_release_lock(interpreter_lock);
- #endif
- return tstate;
- }
- void
- PyEval_RestoreThread(PyThreadState *tstate)
- {
- if (tstate == NULL)
- Py_FatalError("PyEval_RestoreThread: NULL tstate");
- #ifdef WITH_THREAD
- if (interpreter_lock) {
- int err = errno;
- PyThread_acquire_lock(interpreter_lock, 1);
- errno = err;
- }
- #endif
- PyThreadState_Swap(tstate);
- }
- /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX
- signal handlers or Mac I/O completion routines) can schedule calls
- to a function to be called synchronously.
- The synchronous function is called with one void* argument.
- It should return 0 for success or -1 for failure -- failure should
- be accompanied by an exception.
- If registry succeeds, the registry function returns 0; if it fails
- (e.g. due to too many pending calls) it returns -1 (without setting
- an exception condition).
- Note that because registry may occur from within signal handlers,
- or other asynchronous events, calling malloc() is unsafe!
- #ifdef WITH_THREAD
- Any thread can schedule pending calls, but only the main thread
- will execute them.
- #endif
- XXX WARNING! ASYNCHRONOUSLY EXECUTING CODE!
- There are two possible race conditions:
- (1) nested asynchronous registry calls;
- (2) registry calls made while pending calls are being processed.
- While (1) is very unlikely, (2) is a real possibility.
- The current code is safe against (2), but not against (1).
- The safety against (2) is derived from the fact that only one
- thread (the main thread) ever takes things out of the queue.
- XXX Darn! With the advent of thread state, we should have an array
- of pending calls per thread in the thread state! Later...
- */
- #define NPENDINGCALLS 32
- static struct {
- int (*func)(void *);
- void *arg;
- } pendingcalls[NPENDINGCALLS];
- static volatile int pendingfirst = 0;
- static volatile int pendinglast = 0;
- static volatile int things_to_do = 0;
- int
- Py_AddPendingCall(int (*func)(void *), void *arg)
- {
- static volatile int busy = 0;
- int i, j;
- /* XXX Begin critical section */
- /* XXX If you want this to be safe against nested
- XXX asynchronous calls, you'll have to work harder! */
- if (busy)
- return -1;
- busy = 1;
- i = pendinglast;
- j = (i + 1) % NPENDINGCALLS;
- if (j == pendingfirst) {
- busy = 0;
- return -1; /* Queue full */
- }
- pendingcalls[i].func = func;
- pendingcalls[i].arg = arg;
- pendinglast = j;
- _Py_Ticker = 0;
- things_to_do = 1; /* Signal main loop */
- busy = 0;
- /* XXX End critical section */
- return 0;
- }
- int
- Py_MakePendingCalls(void)
- {
- static int busy = 0;
- #ifdef WITH_THREAD
- if (_PyEval_main_thread &&
- PyThread_get_thread_ident() != _PyEval_main_thread)
- return 0;
- #endif
- if (busy)
- return 0;
- busy = 1;
- things_to_do = 0;
- for (;;) {
- int i;
- int (*func)(void *);
- void *arg;
- i = pendingfirst;
- if (i == pendinglast)
- break; /* Queue empty */
- func = pendingcalls[i].func;
- arg = pendingcalls[i].arg;
- pendingfirst = (i + 1) % NPENDINGCALLS;
- if (func(arg) < 0) {
- busy = 0;
- things_to_do = 1; /* We're not done yet */
- return -1;
- }
- }
- busy = 0;
- return 0;
- }
- /* The interpreter's recursion limit */
- #ifndef Py_DEFAULT_RECURSION_LIMIT
- #define Py_DEFAULT_RECURSION_LIMIT 1000
- #endif
- static int recursion_limit = Py_DEFAULT_RECURSION_LIMIT;
- int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;
- int
- Py_GetRecursionLimit(void)
- {
- return recursion_limit;
- }
- void
- Py_SetRecursionLimit(int new_limit)
- {
- recursion_limit = new_limit;
- _Py_CheckRecursionLimit = recursion_limit;
- }
- /* the macro Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall()
- if the recursion_depth reaches _Py_CheckRecursionLimit.
- If USE_STACKCHECK, the macro decrements _Py_CheckRecursionLimit
- to guarantee that _Py_CheckRecursiveCall() is regularly called.
- Without USE_STACKCHECK, there is no need for this. */
- int
- _Py_CheckRecursiveCall(char *where)
- {
- PyThreadState *tstate = PyThreadState_GET();
- #ifdef USE_STACKCHECK
- if (PyOS_CheckStack()) {
- --tstate->recursion_depth;
- PyErr_SetString(PyExc_MemoryError, "Stack overflow");
- return -1;
- }
- #endif
- if (tstate->recursion_depth > recursion_limit) {
- --tstate->recursion_depth;
- PyErr_Format(PyExc_RuntimeError,
- "maximum recursion depth exceeded%s",
- where);
- return -1;
- }
- _Py_CheckRecursionLimit = recursion_limit;
- return 0;
- }
- #ifdef __cplusplus
- extern "C" void
- #else
- extern void
- #endif
- _PyEval_RaiseForUnboundLocal(PyFrameObject *frame, int var_index)
- {
- format_exc_check_arg(
- PyExc_UnboundLocalError,
- UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(frame->f_code->co_varnames, var_index));
- }
- /* Records whether tracing is on for any thread. Counts the number of
- threads for which tstate->c_tracefunc is non-NULL, so if the value
- is 0, we know we don't have to check this thread's c_tracefunc.
- This speeds up the if statement in PyEval_EvalFrameEx() after
- fast_next_opcode*/
- int _Py_TracingPossible = 0;
- /* for manipulating the thread switch and periodic "stuff" - used to be
- per thread, now just a pair o' globals */
- int _Py_CheckInterval = 100;
- volatile int _Py_Ticker = 100;
- #ifdef WITH_LLVM
- int _Py_BailError = 0;
- #endif
- PyObject *
- PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
- {
- return PyEval_EvalCodeEx(co,
- globals, locals,
- (PyObject **)NULL, 0,
- (PyObject **)NULL, 0,
- (PyObject **)NULL, 0,
- NULL);
- }
- /* Interpreter main loop */
- PyObject *
- PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) {
- /* This is for backward compatibility with extension modules that
- used this API; core interpreter code should call
- PyEval_EvalFrame() */
- PyObject *result;
- f->f_throwflag = throwflag;
- result = PyEval_EvalFrame(f);
- f->f_throwflag = 0;
- return result;
- }
- PyObject *
- PyEval_EvalFrame(PyFrameObject *f)
- {
- #ifdef DXPAIRS
- int lastopcode = 0;
- #endif
- register PyObject **stack_pointer; /* Next free slot in value stack */
- register unsigned char *next_instr;
- register int opcode; /* Current opcode */
- register int oparg; /* Current opcode argument, if any */
- register enum _PyUnwindReason why; /* Reason for block stack unwind */
- register int err; /* Error status -- nonzero if error */
- register PyObject *x; /* Temporary objects popped off stack */
- register PyObject *v;
- register PyObject *w;
- register PyObject *u;
- register PyObject *t;
- register PyObject **fastlocals, **freevars;
- _PyFrameBailReason bail_reason;
- PyObject *retval = NULL; /* Return value */
- PyThreadState *tstate = PyThreadState_GET();
- PyCodeObject *co;
- #ifdef WITH_LLVM
- /* We only collect feedback if it will be useful. */
- int rec_feedback = (Py_JitControl == PY_JIT_WHENHOT);
- #endif
- /* when tracing we set things up so that
- not (instr_lb <= current_bytecode_offset < instr_ub)
- is true when the line being executed has changed. The
- initial values are such as to make this false the first
- time it is tested. */
- int instr_ub = -1, instr_lb = 0, instr_prev = -1;
- unsigned char *first_instr;
- PyObject *names;
- PyObject *consts;
- #if defined(Py_DEBUG) || defined(LLTRACE)
- /* Make it easier to find out where we are with a debugger */
- char *filename;
- #endif
- /* Computed GOTOs, or
- the-optimization-commonly-but-improperly-known-as-"threaded code"
- using gcc's labels-as-values extension
- (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).
- The traditional bytecode evaluation loop uses a "switch" statement, which
- decent compilers will optimize as a single indirect branch instruction
- combined with a lookup table of jump addresses. However, since the
- indirect jump instruction is shared by all opcodes, the CPU will have a
- hard time making the right prediction for where to jump next (actually,
- it will be always wrong except in the uncommon case of a sequence of
- several identical opcodes).
- "Threaded code" in contrast, uses an explicit jump table and an explicit
- indirect jump instruction at the end of each opcode. Since the jump
- instruction is at a different address for each opcode, the CPU will make a
- separate prediction for each of these instructions, which is equivalent to
- predicting the second opcode of each opcode pair. These predictions have
- a much better chance to turn out valid, especially in small bytecode loops.
- A mispredicted branch on a modern CPU flushes the whole pipeline and
- can cost several CPU cycles (depending on the pipeline depth),
- and potentially many more instructions (depending on the pipeline width).
- A correctly predicted branch, however, is nearly free.
- At the time of this writing, the "threaded code" version is up to 15-20%
- faster than the normal "switch" version, depending on the compiler and the
- CPU architecture.
- We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,
- because it would render the measurements invalid.
- NOTE: care must be taken that the compiler doesn't try to "optimize" the
- indirect jumps by sharing them between all opcodes. Such optimizations
- can be disabled on gcc by using the -fno-gcse flag (or possibly
- -fno-crossjumping).
- */
- #if defined(USE_COMPUTED_GOTOS) && defined(DYNAMIC_EXECUTION_PROFILE)
- #undef USE_COMPUTED_GOTOS
- #endif
- #ifdef USE_COMPUTED_GOTOS
- /* Import the static jump table */
- #include "opcode_targets.h"
- /* This macro is used when several opcodes defer to the same implementation
- (e.g. SETUP_LOOP, SETUP_FINALLY) */
- #define TARGET_WITH_IMPL(op, impl) \
- TARGET_##op: \
- opcode = op; \
- if (HAS_ARG(op)) \
- oparg = NEXTARG(); \
- case op: \
- goto impl; \
- #define TARGET(op) \
- TARGET_##op: \
- opcode = op; \
- if (HAS_ARG(op)) \
- oparg = NEXTARG(); \
- case op:
- #define DISPATCH() \
- { \
- /* Avoid multiple loads from _Py_Ticker despite `volatile` */ \
- int _tick = _Py_Ticker - 1; \
- _Py_Ticker = _tick; \
- if (_tick >= 0) { \
- FAST_DISPATCH(); \
- } \
- continue; \
- }
- #ifdef LLTRACE
- #define FAST_DISPATCH() \
- { \
- if (!lltrace && !_Py_TracingPossible) { \
- f->f_lasti = INSTR_OFFSET(); \
- goto *opcode_targets[*next_instr++]; \
- } \
- goto fast_next_opcode; \
- }
- #else
- #define FAST_DISPATCH() \
- { \
- if (!_Py_TracingPossible) { \
- f->f_lasti = INSTR_OFFSET(); \
- goto *opcode_targets[*next_instr++]; \
- } \
- goto fast_next_opcode; \
- }
- #endif
- #else
- #define TARGET(op) \
- case op:
- #define TARGET_WITH_IMPL(op, impl) \
- /* silence compiler warnings about `impl` unused */ \
- if (0) goto impl; \
- case op:
- #define DISPATCH() continue
- #define FAST_DISPATCH() goto fast_next_opcode
- #endif
- /* Tuple access macros */
- #ifndef Py_DEBUG
- #define GETITEM(v, i) PyTuple_GET_ITEM((PyTupleObject *)(v), (i))
- #else
- #define GETITEM(v, i) PyTuple_GetItem((v), (i))
- #endif
- /* Code access macros */
- #define INSTR_OFFSET() ((int)(next_instr - first_instr))
- #define NEXTOP() (*next_instr++)
- #define NEXTARG() (next_instr += 2, (next_instr[-1]<<8) + next_instr[-2])
- #define PEEKARG() ((next_instr[2]<<8) + next_instr[1])
- #define JUMPTO(x) (next_instr = first_instr + (x))
- #define JUMPBY(x) (next_instr += (x))
- /* Feedback-gathering macros */
- #ifdef WITH_LLVM
- #define RECORD_TYPE(arg_index, obj) \
- if(rec_feedback){record_type(co, opcode, f->f_lasti, arg_index, obj);}
- #define RECORD_OBJECT(arg_index, obj) \
- if(rec_feedback){record_object(co, opcode, f->f_lasti, arg_index, obj);}
- #define RECORD_FUNC(obj) \
- if(rec_feedback){record_func(co, opcode, f->f_lasti, 0, obj);}
- #define INC_COUNTER(arg_index, counter_id) \
- if (rec_feedback) { \
- inc_feedback_counter(co, opcode, f->f_lasti, arg_index, \
- counter_id); \
- }
- #define RECORD_TRUE() \
- INC_COUNTER(0, PY_FDO_JUMP_TRUE)
- #define RECORD_FALSE() \
- INC_COUNTER(0, PY_FDO_JUMP_FALSE)
- #define RECORD_NONBOOLEAN() \
- INC_COUNTER(0, PY_FDO_JUMP_NON_BOOLEAN)
- #define UPDATE_HOTNESS_JABS() \
- do { if (oparg <= f->f_lasti) ++co->co_hotness; } while (0)
- #else
- #define RECORD_TYPE(arg_index, obj)
- #define RECORD_OBJECT(arg_index, obj)
- #define RECORD_FUNC(obj)
- #define INC_COUNTER(arg_index, counter_id)
- #define RECORD_TRUE()
- #define RECORD_FALSE()
- #define RECORD_NONBOOLEAN()
- #define UPDATE_HOTNESS_JABS()
- #endif /* WITH_LLVM */
- /* OpCode prediction macros
- Some opcodes tend to come in pairs thus making it possible to
- predict the second code when the first is run. For example,
- GET_ITER is often followed by FOR_ITER. And FOR_ITER is often
- followed by STORE_FAST or UNPACK_SEQUENCE.
- Verifying the prediction costs a single high-speed test of a register
- variable against a constant. If the pairing was good, then the
- processor's own internal branch predication has a high likelihood of
- success, resulting in a nearly zero-overhead transition to the
- next opcode. A successful prediction saves a trip through the eval-loop
- including its two unpredictable branches, the HAS_ARG test and the
- switch-case. Combined with the processor's internal branch prediction,
- a successful PREDICT has the effect of making the two opcodes run as if
- they were a single new opcode with the bodies combined.
- If collecting opcode statistics, your choices are to either keep the
- predictions turned-on and interpret the results as if some opcodes
- had been combined or turn-off predictions so that the opcode frequency
- counter updates for both opcodes.
- Opcode prediction is disabled with threaded code, since the latter allows
- the CPU to record separate branch prediction information for each
- opcode.
- */
- #if defined(DYNAMIC_EXECUTION_PROFILE) || defined(USE_COMPUTED_GOTOS)
- #define PREDICT(op) if (0) goto PRED_##op
- #define PREDICTED(op) PRED_##op:
- #define PREDICTED_WITH_ARG(op) PRED_##op:
- #else
- #define PREDICT(op) if (*next_instr == op) goto PRED_##op
- #ifdef WITH_LLVM
- #define PREDICTED_COMMON(op) f->f_lasti = INSTR_OFFSET(); opcode = op;
- #else
- #define PREDICTED_COMMON(op) /* nothing */
- #endif
- #define PREDICTED(op) PRED_##op: PREDICTED_COMMON(op) next_instr++
- #define PREDICTED_WITH_ARG(op) PRED_##op: PREDICTED_COMMON(op) \
- oparg = PEEKARG(); next_instr += 3
- #endif
- /* Stack manipulation macros */
- /* The stack can grow at most MAXINT deep, as co_nlocals and
- co_stacksize are ints. */
- #define STACK_LEVEL() ((int)(stack_pointer - f->f_valuestack))
- #define EMPTY() (STACK_LEVEL() == 0)
- #define TOP() (stack_pointer[-1])
- #define SECOND() (stack_pointer[-2])
- #define THIRD() (stack_pointer[-3])
- #define FOURTH() (stack_pointer[-4])
- #define SET_TOP(v) (stack_pointer[-1] = (v))
- #define SET_SECOND(v) (stack_pointer[-2] = (v))
- #define SET_THIRD(v) (stack_pointer[-3] = (v))
- #define SET_FOURTH(v) (stack_pointer[-4] = (v))
- #define BASIC_STACKADJ(n) (stack_pointer += n)
- #define BASIC_PUSH(v) (*stack_pointer++ = (v))
- #define BASIC_POP() (*--stack_pointer)
- #ifdef LLTRACE
- #define PUSH(v) { (void)(BASIC_PUSH(v), \
- lltrace && prtrace(TOP(), "push")); \
- assert(STACK_LEVEL() <= co->co_stacksize); }
- #define POP() ((void)(lltrace && prtrace(TOP(), "pop")), \
- BASIC_POP())
- #define STACKADJ(n) { (void)(BASIC_STACKADJ(n), \
- lltrace && prtrace(TOP(), "stackadj")); \
- assert(STACK_LEVEL() <= co->co_stacksize); }
- #define EXT_POP(STACK_POINTER) ((void)(lltrace && \
- prtrace((STACK_POINTER)[-1], "ext_pop")), \
- *--(STACK_POINTER))
- #define EXT_PUSH(v, STACK_POINTER) ((void)(*(STACK_POINTER)++ = (v), \
- lltrace && prtrace((STACK_POINTER)[-1], "ext_push")))
- #else
- #define PUSH(v) BASIC_PUSH(v)
- #define POP() BASIC_POP()
- #define STACKADJ(n) BASIC_STACKADJ(n)
- #define EXT_POP(STACK_POINTER) (*--(STACK_POINTER))
- #define EXT_PUSH(v, STACK_POINTER) (*(STACK_POINTER)++ = (v))
- #endif
- /* Local variable macros */
- #define GETLOCAL(i) (fastlocals[i])
- /* The SETLOCAL() macro must not DECREF the local variable in-place and
- then store the new value; it must copy the old value to a temporary
- value, then store the new value, and then DECREF the temporary value.
- This is because it is possible that during the DECREF the frame is
- accessed by other code (e.g. a __del__ method or gc.collect()) and the
- variable would be pointing to already-freed memory. */
- #define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \
- GETLOCAL(i) = value; \
- Py_XDECREF(tmp); } while (0)
- /* Start of code */
- if (f == NULL)
- return NULL;
- #ifdef WITH_LLVM
- bail_reason = (_PyFrameBailReason)f->f_bailed_from_llvm;
- #else
- bail_reason = _PYFRAME_NO_BAIL;
- #endif /* WITH_LLVM */
- /* push frame */
- if (bail_reason == _PYFRAME_NO_BAIL && Py_EnterRecursiveCall(""))
- return NULL;
- co = f->f_code;
- tstate->frame = f;
- #ifdef WITH_LLVM
- maybe_compile(co, f);
- if (f->f_use_jit) {
- assert(bail_reason == _PYFRAME_NO_BAIL);
- assert(co->co_native_function != NULL &&
- "maybe_compile was supposed to ensure"
- " that co_native_function exists");
- if (!co->co_use_jit) {
- // A frame cannot use_jit if the underlying code object
- // can't use_jit. This comes up when a generator is
- // invalidated while active.
- f->f_use_jit = 0;
- }
- else {
- assert(co->co_fatalbailcount < PY_MAX_FATALBAILCOUNT);
- retval = co->co_native_function(f);
- goto exit_eval_frame;
- }
- }
- if (bail_reason != _PYFRAME_NO_BAIL) {
- #ifdef Py_WITH_INSTRUMENTATION
- bail_count_stats->RecordBail(f, bail_reason);
- #endif
- if (_Py_BailError) {
- /* When we bail, we set f_lasti to the current opcode
- * minus 1, so we add one back. */
- int lasti = f->f_lasti + 1;
- PyErr_Format(PyExc_RuntimeError, "bailed to the "
- "interpreter at opcode index %d", lasti);
- goto exit_eval_frame;
- }
- }
- /* Create co_runtime_feedback now that we're about to use it. You
- * might think this would cause a problem if the user flips
- * Py_JitControl from "never" to "whenhot", but since the value of
- * rec_feedback is constant for the duration of this frame's execution,
- * we will not accidentally try to record feedback without initializing
- * co_runtime_feedback. */
- if (rec_feedback && co->co_runtime_feedback == NULL) {
- #if Py_WITH_INSTRUMENTATION
- feedback_map_counter->IncCounter();
- #endif
- co->co_runtime_feedback = PyFeedbackMap_New();
- }
- #endif /* WITH_LLVM */
- switch (bail_reason) {
- case _PYFRAME_NO_BAIL:
- case _PYFRAME_TRACE_ON_ENTRY:
- if (tstate->use_tracing) {
- if (_PyEval_TraceEnterFunction(tstate, f))
- /* Trace or profile function raised
- an error. */
- goto exit_eval_frame;
- }
- break;
- case _PYFRAME_BACKEDGE_TRACE:
- /* If we bailed because of a backedge, set instr_prev
- to ensure a line trace call. */
- instr_prev = INT_MAX;
- break;
- case _PYFRAME_CALL_PROFILE:
- case _PYFRAME_LINE_TRACE:
- case _PYFRAME_FATAL_GUARD_FAIL:
- case _PYFRAME_GUARD_FAIL:
- /* These are handled by the opcode dispatch loop. */
- break;
- default:
- PyErr_Format(PyExc_SystemError, "unknown bail reason");
- goto exit_eval_frame;
- }
- names = co->co_names;
- consts = co->co_consts;
- fastlocals = f->f_localsplus;
- freevars = f->f_localsplus + co->co_nlocals;
- first_instr = (unsigned char*) PyString_AS_STRING(co->co_code);
- /* An explanation is in order for the next line.
- f->f_lasti now refers to the index of the last instruction
- executed. You might think this was obvious from the name, but
- this wasn't always true before 2.3! PyFrame_New now sets
- f->f_lasti to -1 (i.e. the index *before* the first instruction)
- and YIELD_VALUE doesn't fiddle with f_lasti any more. So this
- does work. Promise.
- When the PREDICT() macros are enabled, some opcode pairs follow in
- direct succession without updating f->f_lasti. A successful
- prediction effectively links the two codes together as if they
- were a single new opcode; accordingly,f->f_lasti will point to
- the first code in the pair (for instance, GET_ITER followed by
- FOR_ITER is effectively a single opcode and f->f_lasti will point
- at to the beginning of the combined pair.)
- */
- next_instr = first_instr + f->f_lasti + 1;
- stack_pointer = f->f_stacktop;
- assert(stack_pointer != NULL);
- f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */
- #ifdef LLTRACE
- lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
- #endif
- #if defined(Py_DEBUG) || defined(LLTRACE)
- filename = PyString_AsString(co->co_filename);
- #endif
- why = UNWIND_NOUNWIND;
- w = NULL;
- /* Note that this goes after the LLVM handling code so we don't log
- * this event when calling LLVM functions. Do this before the throwflag
- * check below to avoid mismatched enter/exit events in the log. */
- PY_LOG_TSC_EVENT(CALL_ENTER_EVAL);
- if (f->f_throwflag) { /* support for generator.throw() */
- why = UNWIND_EXCEPTION;
- goto on_error;
- }
- for (;;) {
- assert(stack_pointer >= f->f_valuestack); /* else underflow */
- assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */
- /* Do periodic things. Doing this every time through
- the loop would add too much overhead, so we do it
- only every Nth instruction. We also do it if
- ``things_to_do'' is set, i.e. when an asynchronous
- event needs attention (e.g. a signal handler or
- async I/O handler); see Py_AddPendingCall() and
- Py_MakePendingCalls() above. */
- if (--_Py_Ticker < 0) {
- if (*next_instr == SETUP_FINALLY) {
- /* Make the last opcode before
- a try: finally: block uninterruptable. */
- goto fast_next_opcode;
- }
- if (_PyEval_HandlePyTickerExpired(tstate) == -1) {
- why = UNWIND_EXCEPTION;
- goto on_error;
- }
- }
- fast_next_opcode:
- f->f_lasti = INSTR_OFFSET();
- /* line-by-line tracing support */
- if (_Py_TracingPossible &&
- tstate->c_tracefunc != NULL && !tstate->tracing) {
- /* see maybe_call_line_trace
- for expository comments */
- f->f_stacktop = stack_pointer;
- err = maybe_call_line_trace(tstate->c_tracefunc,
- tstate->c_traceobj,
- f, &instr_lb, &instr_ub,
- &instr_prev);
- /* Reload possibly changed frame fields */
- JUMPTO(f->f_lasti);
- assert(f->f_stacktop != NULL);
- stack_pointer = f->f_stacktop;
- f->f_stacktop = NULL;
- if (err) {
- /* trace function raised an exception */
- why = UNWIND_EXCEPTION;
- goto on_error;
- }
- }
- /* Extract opcode and argument */
- opcode = NEXTOP();
- oparg = 0; /* allows oparg to be stored in a register because
- it doesn't have to be remembered across a full loop */
- if (HAS_ARG(opcode))
- oparg = NEXTARG();
- dispatch_opcode:
- #ifdef DYNAMIC_EXECUTION_PROFILE
- #ifdef DXPAIRS
- dxpairs[lastopcode][opcode]++;
- lastopcode = opcode;
- #endif
- dxp[opcode]++;
- #endif
- #ifdef LLTRACE
- /* Instruction tracing */
- if (lltrace) {
- if (HAS_ARG(opcode)) {
- printf("%d: %d, %d\n",
- f->f_lasti, opcode, oparg);
- }
- else {
- printf("%d: %d\n",
- f->f_lasti, opcode);
- }
- }
- #endif
- /* Main switch on opcode */
- assert(why == UNWIND_NOUNWIND);
- /* XXX(jyasskin): Add an assertion under CHECKEXC that
- !PyErr_Occurred(). */
- switch (opcode) {
- /* BEWARE!
- It is essential that any operation that fails sets
- why to anything but UNWIND_NOUNWIND, and that no operation
- that succeeds does this! */
- /* case STOP_CODE: this is an error! */
- TARGET(NOP)
- FAST_DISPATCH();
- TARGET(LOAD_FAST)
- x = GETLOCAL(oparg);
- if (x != NULL) {
- Py_INCREF(x);
- PUSH(x);
- FAST_DISPATCH();
- }
- _PyEval_RaiseForUnboundLocal(f, oparg);
- why = UNWIND_EXCEPTION;
- break;
- TARGET(LOAD_CONST)
- x = GETITEM(consts, oparg);
- Py_INCREF(x);
- PUSH(x);
- FAST_DISPATCH();
- PREDICTED_WITH_ARG(STORE_FAST);
- TARGET(STORE_FAST)
- v = POP();
- SETLOCAL(oparg, v);
- FAST_DISPATCH();
- TARGET(POP_TOP)
- v = POP();
- Py_DECREF(v);
- FAST_DISPATCH();
- TARGET(ROT_TWO)
- v = TOP();
- w = SECOND();
- SET_TOP(w);
- SET_SECOND(v);
- FAST_DISPATCH();
- TARGET(ROT_THREE)
- v = TOP();
- w = SECOND();
- x = THIRD();
- SET_TOP(w);
- SET_SECOND(x);
- SET_THIRD(v);
- FAST_DISPATCH();
- TARGET(ROT_FOUR)
- u = TOP();
- v = SECOND();
- w = THIRD();
- x = FOURTH();
- SET_TOP(v);
- SET_SECOND(w);
- SET_THIRD(x);
- SET_FOURTH(u);
- FAST_DISPATCH();
- TARGET(DUP_TOP)
- v = TOP();
- Py_INCREF(v);
- PUSH(v);
- FAST_DISPATCH();
- TARGET(DUP_TOP_TWO)
- x = TOP();
- Py_INCREF(x);
- w = SECOND();
- Py_INCREF(w);
- STACKADJ(2);
- SET_TOP(x);
- SET_SECOND(w);
- FAST_DISPATCH();
- TARGET(DUP_TOP_THREE)
- x = TOP();
- Py_INCREF(x);
- w = SECOND();
- Py_INCREF(w);
- v = THIRD();
- Py_INCREF(v);
- STACKADJ(3);
- SET_TOP(x);
- SET_SECOND(w);
- SET_THIRD(v);
- FAST_DISPATCH();
- TARGET(UNARY_POSITIVE)
- v = TOP();
- RECORD_TYPE(0, v);
- x = PyNumber_Positive(v);
- Py_DECREF(v);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(UNARY_NEGATIVE)
- v = TOP();
- RECORD_TYPE(0, v);
- x = PyNumber_Negative(v);
- Py_DECREF(v);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(UNARY_NOT)
- v = TOP();
- RECORD_TYPE(0, v);
- err = PyObject_IsTrue(v);
- Py_DECREF(v);
- if (err == 0) {
- Py_INCREF(Py_True);
- SET_TOP(Py_True);
- DISPATCH();
- }
- else if (err > 0) {
- Py_INCREF(Py_False);
- SET_TOP(Py_False);
- DISPATCH();
- }
- STACKADJ(-1);
- why = UNWIND_EXCEPTION;
- break;
- TARGET(UNARY_CONVERT)
- v = TOP();
- RECORD_TYPE(0, v);
- x = PyObject_Repr(v);
- Py_DECREF(v);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(UNARY_INVERT)
- v = TOP();
- RECORD_TYPE(0, v);
- x = PyNumber_Invert(v);
- Py_DECREF(v);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_POWER)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Power(v, w, Py_None);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_MULTIPLY)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Multiply(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_DIVIDE)
- if (!_Py_QnewFlag) {
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Divide(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- }
- /* -Qnew is in effect: jump to BINARY_TRUE_DIVIDE */
- goto _binary_true_divide;
- TARGET(BINARY_TRUE_DIVIDE)
- _binary_true_divide:
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_TrueDivide(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_FLOOR_DIVIDE)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_FloorDivide(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_MODULO)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- if (PyString_CheckExact(v))
- x = PyString_Format(v, w);
- else
- x = PyNumber_Remainder(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_ADD)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
- /* INLINE: int + int */
- register long a, b, i;
- a = PyInt_AS_LONG(v);
- b = PyInt_AS_LONG(w);
- i = a + b;
- if ((i^a) < 0 && (i^b) < 0)
- goto slow_add;
- x = PyInt_FromLong(i);
- }
- else if (PyString_CheckExact(v) &&
- PyString_CheckExact(w)) {
- x = string_concatenate(v, w, f, next_instr);
- /* string_concatenate consumed the ref to v */
- goto skip_decref_vx;
- }
- else {
- slow_add:
- x = PyNumber_Add(v, w);
- }
- Py_DECREF(v);
- skip_decref_vx:
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_SUBTRACT)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
- /* INLINE: int - int */
- register long a, b, i;
- a = PyInt_AS_LONG(v);
- b = PyInt_AS_LONG(w);
- i = a - b;
- if ((i^a) < 0 && (i^~b) < 0)
- goto slow_sub;
- x = PyInt_FromLong(i);
- }
- else {
- slow_sub:
- x = PyNumber_Subtract(v, w);
- }
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_SUBSCR)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- if (PyList_CheckExact(v) && PyInt_CheckExact(w)) {
- /* INLINE: list[int] */
- Py_ssize_t i = PyInt_AsSsize_t(w);
- if (i < 0)
- i += PyList_GET_SIZE(v);
- if (i >= 0 && i < PyList_GET_SIZE(v)) {
- x = PyList_GET_ITEM(v, i);
- Py_INCREF(x);
- }
- else
- goto slow_get;
- }
- else
- slow_get:
- x = PyObject_GetItem(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_LSHIFT)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Lshift(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_RSHIFT)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Rshift(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_AND)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_And(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_XOR)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Xor(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(BINARY_OR)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_Or(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(LIST_APPEND)
- w = POP();
- v = POP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- err = PyList_Append(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- if (err != 0) {
- why = UNWIND_EXCEPTION;
- break;
- }
- PREDICT(JUMP_ABSOLUTE);
- DISPATCH();
- TARGET(INPLACE_POWER)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlacePower(v, w, Py_None);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_MULTIPLY)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlaceMultiply(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_DIVIDE)
- if (!_Py_QnewFlag) {
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlaceDivide(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- }
- /* -Qnew is in effect: jump to INPLACE_TRUE_DIVIDE */
- goto _inplace_true_divide;
- TARGET(INPLACE_TRUE_DIVIDE)
- _inplace_true_divide:
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlaceTrueDivide(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_FLOOR_DIVIDE)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlaceFloorDivide(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_MODULO)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlaceRemainder(v, w);
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_ADD)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
- /* INLINE: int + int */
- register long a, b, i;
- a = PyInt_AS_LONG(v);
- b = PyInt_AS_LONG(w);
- i = a + b;
- if ((i^a) < 0 && (i^b) < 0)
- goto slow_iadd;
- x = PyInt_FromLong(i);
- }
- else if (PyString_CheckExact(v) &&
- PyString_CheckExact(w)) {
- x = string_concatenate(v, w, f, next_instr);
- /* string_concatenate consumed the ref to v */
- goto skip_decref_v;
- }
- else {
- slow_iadd:
- x = PyNumber_InPlaceAdd(v, w);
- }
- Py_DECREF(v);
- skip_decref_v:
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_SUBTRACT)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
- /* INLINE: int - int */
- register long a, b, i;
- a = PyInt_AS_LONG(v);
- b = PyInt_AS_LONG(w);
- i = a - b;
- if ((i^a) < 0 && (i^~b) < 0)
- goto slow_isub;
- x = PyInt_FromLong(i);
- }
- else {
- slow_isub:
- x = PyNumber_InPlaceSubtract(v, w);
- }
- Py_DECREF(v);
- Py_DECREF(w);
- SET_TOP(x);
- if (x == NULL) {
- why = UNWIND_EXCEPTION;
- break;
- }
- DISPATCH();
- TARGET(INPLACE_LSHIFT)
- w = POP();
- v = TOP();
- RECORD_TYPE(0, v);
- RECORD_TYPE(1, w);
- x = PyNumber_InPlaceL…