/Python/eval.cc

http://unladen-swallow.googlecode.com/ · C++ · 5498 lines · 4329 code · 450 blank · 719 comment · 945 complexity · 3ed53cf038a21678d0dfee98cdc32199 MD5 · raw file

Large files are truncated click here to view the full file

  1. /* Execute compiled code */
  2. /* XXX TO DO:
  3. XXX speed up searching for keywords by using a dictionary
  4. XXX document it!
  5. */
  6. /* Note: this file will be compiled as C ifndef WITH_LLVM, so try to keep it
  7. generally C. */
  8. /* enable more aggressive intra-module optimizations, where available */
  9. #define PY_LOCAL_AGGRESSIVE
  10. #include "Python.h"
  11. #include "code.h"
  12. #include "frameobject.h"
  13. #include "eval.h"
  14. #include "opcode.h"
  15. #include "structmember.h"
  16. #include "JIT/llvm_compile.h"
  17. #include "Util/EventTimer.h"
  18. #include <ctype.h>
  19. #ifdef WITH_LLVM
  20. #include "_llvmfunctionobject.h"
  21. #include "llvm/Function.h"
  22. #include "llvm/Support/ManagedStatic.h"
  23. #include "llvm/Support/raw_ostream.h"
  24. #include "JIT/global_llvm_data.h"
  25. #include "JIT/RuntimeFeedback.h"
  26. #include "Util/Stats.h"
  27. #include <set>
  28. using llvm::errs;
  29. #endif
  30. /* Make a call to stop the call overhead timer before going through to
  31. PyObject_Call. */
  32. static inline PyObject *
  33. _PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw)
  34. {
  35. /* If we're calling a compiled C function with *args or **kwargs, then
  36. * this enum should be CALL_ENTER_C. However, most calls to C
  37. * functions are simple and are fast-tracked through the CALL_FUNCTION
  38. * opcode. */
  39. PY_LOG_TSC_EVENT(CALL_ENTER_PYOBJ_CALL);
  40. return PyObject_Call(func, arg, kw);
  41. }
  42. #ifdef Py_WITH_INSTRUMENTATION
  43. std::string
  44. _PyEval_GetCodeName(PyCodeObject *code)
  45. {
  46. std::string result;
  47. llvm::raw_string_ostream wrapper(result);
  48. wrapper << PyString_AsString(code->co_filename)
  49. << ":" << code->co_firstlineno << " "
  50. << "(" << PyString_AsString(code->co_name) << ")";
  51. wrapper.flush();
  52. return result;
  53. }
  54. // Collect statistics about how long we block for compilation to LLVM IR and to
  55. // machine code.
  56. class IrCompilationTimes : public DataVectorStats<int64_t> {
  57. public:
  58. IrCompilationTimes()
  59. : DataVectorStats<int64_t>("Time blocked for IR JIT in ns") {}
  60. };
  61. class McCompilationTimes : public DataVectorStats<int64_t> {
  62. public:
  63. McCompilationTimes()
  64. : DataVectorStats<int64_t>("Time blocked for MC JIT in ns") {}
  65. };
  66. static llvm::ManagedStatic<IrCompilationTimes> ir_compilation_times;
  67. static llvm::ManagedStatic<McCompilationTimes> mc_compilation_times;
  68. class FeedbackMapCounter {
  69. public:
  70. ~FeedbackMapCounter() {
  71. errs() << "\nFeedback maps created:\n";
  72. errs() << "N: " << this->counter_ << "\n";
  73. }
  74. void IncCounter() {
  75. this->counter_++;
  76. }
  77. private:
  78. unsigned counter_;
  79. };
  80. static llvm::ManagedStatic<FeedbackMapCounter> feedback_map_counter;
  81. class HotnessTracker {
  82. // llvm::DenseSet or llvm::SmallPtrSet may be better, but as of this
  83. // writing, they don't seem to work with std::vector.
  84. std::set<PyCodeObject*> hot_code_;
  85. public:
  86. ~HotnessTracker();
  87. void AddHotCode(PyCodeObject *code_obj) {
  88. // This will prevent the code object from ever being
  89. // deleted.
  90. Py_INCREF(code_obj);
  91. this->hot_code_.insert(code_obj);
  92. }
  93. };
  94. static bool
  95. compare_hotness(const PyCodeObject *first, const PyCodeObject *second)
  96. {
  97. return first->co_hotness > second->co_hotness;
  98. }
  99. HotnessTracker::~HotnessTracker()
  100. {
  101. errs() << "\nCode objects deemed hot:\n";
  102. errs() << "N: " << this->hot_code_.size() << "\n";
  103. errs() << "Function -> hotness score:\n";
  104. std::vector<PyCodeObject*> to_sort(this->hot_code_.begin(),
  105. this->hot_code_.end());
  106. std::sort(to_sort.begin(), to_sort.end(), compare_hotness);
  107. for (std::vector<PyCodeObject*>::iterator co = to_sort.begin();
  108. co != to_sort.end(); ++co) {
  109. errs() << _PyEval_GetCodeName(*co)
  110. << " -> " << (*co)->co_hotness << "\n";
  111. }
  112. }
  113. static llvm::ManagedStatic<HotnessTracker> hot_code;
  114. // Keep track of which functions failed fatal guards, but kept being called.
  115. // This can help gauge the efficacy of optimizations that involve fatal guards.
  116. class FatalBailTracker {
  117. public:
  118. ~FatalBailTracker() {
  119. errs() << "\nCode objects that failed fatal guards:\n";
  120. errs() << "\tfile:line (funcname) bail hotness"
  121. << " -> final hotness\n";
  122. for (TrackerData::const_iterator it = this->code_.begin();
  123. it != this->code_.end(); ++it) {
  124. PyCodeObject *code = it->first;
  125. if (code->co_hotness == it->second)
  126. continue;
  127. errs() << "\t" << _PyEval_GetCodeName(code)
  128. << "\t" << it->second << " -> "
  129. << code->co_hotness << "\n";
  130. }
  131. }
  132. void RecordFatalBail(PyCodeObject *code) {
  133. Py_INCREF(code);
  134. this->code_.push_back(std::make_pair(code, code->co_hotness));
  135. }
  136. private:
  137. // Keep a list of (code object, hotness) where hotness is the
  138. // value of co_hotness when RecordFatalBail() was called. This is
  139. // used to hide code objects whose machine code functions are
  140. // invalidated during shutdown because their module dict has gone away;
  141. // these code objects are uninteresting for our analysis.
  142. typedef std::pair<PyCodeObject *, long> DataPoint;
  143. typedef std::vector<DataPoint> TrackerData;
  144. TrackerData code_;
  145. };
  146. static llvm::ManagedStatic<FatalBailTracker> fatal_bail_tracker;
  147. // C wrapper for FatalBailTracker::RecordFatalBail().
  148. void
  149. _PyEval_RecordFatalBail(PyCodeObject *code)
  150. {
  151. fatal_bail_tracker->RecordFatalBail(code);
  152. }
  153. // Collect stats on how many watchers the globals/builtins dicts acculumate.
  154. // This currently records how many watchers the dict had when it changed, ie,
  155. // how many watchers it had to notify.
  156. class WatcherCountStats : public DataVectorStats<size_t> {
  157. public:
  158. WatcherCountStats() :
  159. DataVectorStats<size_t>("Number of watchers accumulated") {};
  160. };
  161. static llvm::ManagedStatic<WatcherCountStats> watcher_count_stats;
  162. void
  163. _PyEval_RecordWatcherCount(size_t watcher_count)
  164. {
  165. watcher_count_stats->RecordDataPoint(watcher_count);
  166. }
  167. class BailCountStats {
  168. public:
  169. BailCountStats() : total_(0), trace_on_entry_(0), line_trace_(0),
  170. backedge_trace_(0), call_profile_(0),
  171. fatal_guard_fail_(0), guard_fail_(0) {};
  172. ~BailCountStats() {
  173. errs() << "\nBailed to the interpreter " << this->total_
  174. << " times:\n";
  175. errs() << "TRACE_ON_ENTRY: " << this->trace_on_entry_ << "\n";
  176. errs() << "LINE_TRACE: " << this->line_trace_ << "\n";
  177. errs() << "BACKEDGE_TRACE:" << this->backedge_trace_ << "\n";
  178. errs() << "CALL_PROFILE: " << this->call_profile_ << "\n";
  179. errs() << "FATAL_GUARD_FAIL: " << this->fatal_guard_fail_
  180. << "\n";
  181. errs() << "GUARD_FAIL: " << this->guard_fail_ << "\n";
  182. errs() << "\n" << this->bail_site_freq_.size()
  183. << " bail sites:\n";
  184. for (BailData::iterator i = this->bail_site_freq_.begin(),
  185. end = this->bail_site_freq_.end(); i != end; ++i) {
  186. errs() << " " << i->getKey() << " bailed "
  187. << i->getValue() << " times\n";
  188. }
  189. errs() << "\n" << this->guard_bail_site_freq_.size()
  190. << " guard bail sites:\n";
  191. for (BailData::iterator i = this->guard_bail_site_freq_.begin(),
  192. end = this->guard_bail_site_freq_.end(); i != end; ++i) {
  193. errs() << " " << i->getKey() << " bailed "
  194. << i->getValue() << " times\n";
  195. }
  196. }
  197. void RecordBail(PyFrameObject *frame, _PyFrameBailReason bail_reason) {
  198. ++this->total_;
  199. std::string record;
  200. llvm::raw_string_ostream wrapper(record);
  201. wrapper << PyString_AsString(frame->f_code->co_filename) << ":";
  202. wrapper << frame->f_code->co_firstlineno << ":";
  203. wrapper << PyString_AsString(frame->f_code->co_name) << ":";
  204. // See the comment in PyEval_EvalFrame about how f->f_lasti is
  205. // initialized.
  206. wrapper << frame->f_lasti + 1;
  207. wrapper.flush();
  208. BailData::value_type &entry =
  209. this->bail_site_freq_.GetOrCreateValue(record, 0);
  210. entry.setValue(entry.getValue() + 1);
  211. #define BAIL_CASE(name, field) \
  212. case name: \
  213. ++this->field; \
  214. break;
  215. switch (bail_reason) {
  216. BAIL_CASE(_PYFRAME_TRACE_ON_ENTRY, trace_on_entry_)
  217. BAIL_CASE(_PYFRAME_LINE_TRACE, line_trace_)
  218. BAIL_CASE(_PYFRAME_BACKEDGE_TRACE, backedge_trace_)
  219. BAIL_CASE(_PYFRAME_CALL_PROFILE, call_profile_)
  220. BAIL_CASE(_PYFRAME_FATAL_GUARD_FAIL, fatal_guard_fail_)
  221. BAIL_CASE(_PYFRAME_GUARD_FAIL, guard_fail_)
  222. default:
  223. abort(); // Unknown bail reason.
  224. }
  225. #undef BAIL_CASE
  226. if (bail_reason != _PYFRAME_GUARD_FAIL)
  227. return;
  228. wrapper << ":";
  229. #define GUARD_CASE(name) \
  230. case name: \
  231. wrapper << #name; \
  232. break;
  233. switch (frame->f_guard_type) {
  234. GUARD_CASE(_PYGUARD_DEFAULT)
  235. GUARD_CASE(_PYGUARD_BINOP)
  236. GUARD_CASE(_PYGUARD_ATTR)
  237. GUARD_CASE(_PYGUARD_CFUNC)
  238. GUARD_CASE(_PYGUARD_BRANCH)
  239. GUARD_CASE(_PYGUARD_STORE_SUBSCR)
  240. default:
  241. wrapper << ((int)frame->f_guard_type);
  242. }
  243. #undef GUARD_CASE
  244. wrapper.flush();
  245. BailData::value_type &g_entry =
  246. this->guard_bail_site_freq_.GetOrCreateValue(record, 0);
  247. g_entry.setValue(g_entry.getValue() + 1);
  248. }
  249. private:
  250. typedef llvm::StringMap<unsigned> BailData;
  251. BailData bail_site_freq_;
  252. BailData guard_bail_site_freq_;
  253. long total_;
  254. long trace_on_entry_;
  255. long line_trace_;
  256. long backedge_trace_;
  257. long call_profile_;
  258. long fatal_guard_fail_;
  259. long guard_fail_;
  260. };
  261. static llvm::ManagedStatic<BailCountStats> bail_count_stats;
  262. #endif // Py_WITH_INSTRUMENTATION
  263. /* Turn this on if your compiler chokes on the big switch: */
  264. /* #define CASE_TOO_BIG 1 */
  265. #ifdef Py_DEBUG
  266. /* For debugging the interpreter: */
  267. #define LLTRACE 1 /* Low-level trace feature */
  268. #define CHECKEXC 1 /* Double-check exception checking */
  269. #endif
  270. typedef PyObject *(*callproc)(PyObject *, PyObject *, PyObject *);
  271. /* Forward declarations */
  272. static PyObject * fast_function(PyObject *, PyObject ***, int, int, int);
  273. static PyObject * do_call(PyObject *, PyObject ***, int, int);
  274. static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int);
  275. static PyObject * update_keyword_args(PyObject *, int, PyObject ***,
  276. PyObject *);
  277. static PyObject * update_star_args(int, int, PyObject *, PyObject ***);
  278. static PyObject * load_args(PyObject ***, int);
  279. #ifdef WITH_LLVM
  280. static inline void mark_called(PyCodeObject *co);
  281. static inline int maybe_compile(PyCodeObject *co, PyFrameObject *f);
  282. /* Record data for use in generating optimized machine code. */
  283. static void record_type(PyCodeObject *, int, int, int, PyObject *);
  284. static void record_func(PyCodeObject *, int, int, int, PyObject *);
  285. static void record_object(PyCodeObject *, int, int, int, PyObject *);
  286. static void inc_feedback_counter(PyCodeObject *, int, int, int, int);
  287. #endif /* WITH_LLVM */
  288. int _Py_ProfilingPossible = 0;
  289. /* Keep this in sync with llvm_fbuilder.cc */
  290. #define CALL_FLAG_VAR 1
  291. #define CALL_FLAG_KW 2
  292. #ifdef LLTRACE
  293. static int lltrace;
  294. static int prtrace(PyObject *, char *);
  295. #endif
  296. static int call_trace_protected(Py_tracefunc, PyObject *,
  297. PyFrameObject *, int, PyObject *);
  298. static int maybe_call_line_trace(Py_tracefunc, PyObject *,
  299. PyFrameObject *, int *, int *, int *);
  300. static PyObject * cmp_outcome(int, PyObject *, PyObject *);
  301. static void format_exc_check_arg(PyObject *, char *, PyObject *);
  302. static PyObject * string_concatenate(PyObject *, PyObject *,
  303. PyFrameObject *, unsigned char *);
  304. #define NAME_ERROR_MSG \
  305. "name '%.200s' is not defined"
  306. #define GLOBAL_NAME_ERROR_MSG \
  307. "global name '%.200s' is not defined"
  308. #define UNBOUNDLOCAL_ERROR_MSG \
  309. "local variable '%.200s' referenced before assignment"
  310. #define UNBOUNDFREE_ERROR_MSG \
  311. "free variable '%.200s' referenced before assignment" \
  312. " in enclosing scope"
  313. /* Dynamic execution profile */
  314. #ifdef DYNAMIC_EXECUTION_PROFILE
  315. #ifdef DXPAIRS
  316. static long dxpairs[257][256];
  317. #define dxp dxpairs[256]
  318. #else
  319. static long dxp[256];
  320. #endif
  321. #endif
  322. /* Function call profile */
  323. #ifdef CALL_PROFILE
  324. #define PCALL_NUM 11
  325. static int pcall[PCALL_NUM];
  326. #define PCALL_ALL 0
  327. #define PCALL_FUNCTION 1
  328. #define PCALL_FAST_FUNCTION 2
  329. #define PCALL_FASTER_FUNCTION 3
  330. #define PCALL_METHOD 4
  331. #define PCALL_BOUND_METHOD 5
  332. #define PCALL_CFUNCTION 6
  333. #define PCALL_TYPE 7
  334. #define PCALL_GENERATOR 8
  335. #define PCALL_OTHER 9
  336. #define PCALL_POP 10
  337. /* Notes about the statistics
  338. PCALL_FAST stats
  339. FAST_FUNCTION means no argument tuple needs to be created.
  340. FASTER_FUNCTION means that the fast-path frame setup code is used.
  341. If there is a method call where the call can be optimized by changing
  342. the argument tuple and calling the function directly, it gets recorded
  343. twice.
  344. As a result, the relationship among the statistics appears to be
  345. PCALL_ALL == PCALL_FUNCTION + PCALL_METHOD - PCALL_BOUND_METHOD +
  346. PCALL_CFUNCTION + PCALL_TYPE + PCALL_GENERATOR + PCALL_OTHER
  347. PCALL_FUNCTION > PCALL_FAST_FUNCTION > PCALL_FASTER_FUNCTION
  348. PCALL_METHOD > PCALL_BOUND_METHOD
  349. */
  350. #define PCALL(POS) pcall[POS]++
  351. PyObject *
  352. PyEval_GetCallStats(PyObject *self)
  353. {
  354. return Py_BuildValue("iiiiiiiiiiiii",
  355. pcall[0], pcall[1], pcall[2], pcall[3],
  356. pcall[4], pcall[5], pcall[6], pcall[7],
  357. pcall[8], pcall[9], pcall[10]);
  358. }
  359. #else
  360. #define PCALL(O)
  361. PyObject *
  362. PyEval_GetCallStats(PyObject *self)
  363. {
  364. Py_INCREF(Py_None);
  365. return Py_None;
  366. }
  367. #endif
  368. #ifdef WITH_THREAD
  369. #ifdef HAVE_ERRNO_H
  370. #include <errno.h>
  371. #endif
  372. #include "pythread.h"
  373. static PyThread_type_lock interpreter_lock = 0; /* This is the GIL */
  374. long _PyEval_main_thread = 0;
  375. int
  376. PyEval_ThreadsInitialized(void)
  377. {
  378. return interpreter_lock != 0;
  379. }
  380. void
  381. PyEval_InitThreads(void)
  382. {
  383. if (interpreter_lock)
  384. return;
  385. interpreter_lock = PyThread_allocate_lock();
  386. PyThread_acquire_lock(interpreter_lock, 1);
  387. _PyEval_main_thread = PyThread_get_thread_ident();
  388. }
  389. void
  390. PyEval_AcquireLock(void)
  391. {
  392. PyThread_acquire_lock(interpreter_lock, 1);
  393. }
  394. void
  395. PyEval_ReleaseLock(void)
  396. {
  397. PyThread_release_lock(interpreter_lock);
  398. }
  399. void
  400. PyEval_AcquireThread(PyThreadState *tstate)
  401. {
  402. if (tstate == NULL)
  403. Py_FatalError("PyEval_AcquireThread: NULL new thread state");
  404. /* Check someone has called PyEval_InitThreads() to create the lock */
  405. assert(interpreter_lock);
  406. PyThread_acquire_lock(interpreter_lock, 1);
  407. if (PyThreadState_Swap(tstate) != NULL)
  408. Py_FatalError(
  409. "PyEval_AcquireThread: non-NULL old thread state");
  410. }
  411. void
  412. PyEval_ReleaseThread(PyThreadState *tstate)
  413. {
  414. if (tstate == NULL)
  415. Py_FatalError("PyEval_ReleaseThread: NULL thread state");
  416. if (PyThreadState_Swap(NULL) != tstate)
  417. Py_FatalError("PyEval_ReleaseThread: wrong thread state");
  418. PyThread_release_lock(interpreter_lock);
  419. }
  420. /* This function is called from PyOS_AfterFork to ensure that newly
  421. created child processes don't hold locks referring to threads which
  422. are not running in the child process. (This could also be done using
  423. pthread_atfork mechanism, at least for the pthreads implementation.) */
  424. void
  425. PyEval_ReInitThreads(void)
  426. {
  427. PyObject *threading, *result;
  428. PyThreadState *tstate;
  429. if (!interpreter_lock)
  430. return;
  431. /*XXX Can't use PyThread_free_lock here because it does too
  432. much error-checking. Doing this cleanly would require
  433. adding a new function to each thread_*.h. Instead, just
  434. create a new lock and waste a little bit of memory */
  435. interpreter_lock = PyThread_allocate_lock();
  436. PyThread_acquire_lock(interpreter_lock, 1);
  437. _PyEval_main_thread = PyThread_get_thread_ident();
  438. /* Update the threading module with the new state.
  439. */
  440. tstate = PyThreadState_GET();
  441. threading = PyMapping_GetItemString(tstate->interp->modules,
  442. "threading");
  443. if (threading == NULL) {
  444. /* threading not imported */
  445. PyErr_Clear();
  446. return;
  447. }
  448. result = PyObject_CallMethod(threading, "_after_fork", NULL);
  449. if (result == NULL)
  450. PyErr_WriteUnraisable(threading);
  451. else
  452. Py_DECREF(result);
  453. Py_DECREF(threading);
  454. }
  455. #endif
  456. /* Functions save_thread and restore_thread are always defined so
  457. dynamically loaded modules needn't be compiled separately for use
  458. with and without threads: */
  459. PyThreadState *
  460. PyEval_SaveThread(void)
  461. {
  462. PyThreadState *tstate = PyThreadState_Swap(NULL);
  463. if (tstate == NULL)
  464. Py_FatalError("PyEval_SaveThread: NULL tstate");
  465. #ifdef WITH_THREAD
  466. if (interpreter_lock)
  467. PyThread_release_lock(interpreter_lock);
  468. #endif
  469. return tstate;
  470. }
  471. void
  472. PyEval_RestoreThread(PyThreadState *tstate)
  473. {
  474. if (tstate == NULL)
  475. Py_FatalError("PyEval_RestoreThread: NULL tstate");
  476. #ifdef WITH_THREAD
  477. if (interpreter_lock) {
  478. int err = errno;
  479. PyThread_acquire_lock(interpreter_lock, 1);
  480. errno = err;
  481. }
  482. #endif
  483. PyThreadState_Swap(tstate);
  484. }
  485. /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX
  486. signal handlers or Mac I/O completion routines) can schedule calls
  487. to a function to be called synchronously.
  488. The synchronous function is called with one void* argument.
  489. It should return 0 for success or -1 for failure -- failure should
  490. be accompanied by an exception.
  491. If registry succeeds, the registry function returns 0; if it fails
  492. (e.g. due to too many pending calls) it returns -1 (without setting
  493. an exception condition).
  494. Note that because registry may occur from within signal handlers,
  495. or other asynchronous events, calling malloc() is unsafe!
  496. #ifdef WITH_THREAD
  497. Any thread can schedule pending calls, but only the main thread
  498. will execute them.
  499. #endif
  500. XXX WARNING! ASYNCHRONOUSLY EXECUTING CODE!
  501. There are two possible race conditions:
  502. (1) nested asynchronous registry calls;
  503. (2) registry calls made while pending calls are being processed.
  504. While (1) is very unlikely, (2) is a real possibility.
  505. The current code is safe against (2), but not against (1).
  506. The safety against (2) is derived from the fact that only one
  507. thread (the main thread) ever takes things out of the queue.
  508. XXX Darn! With the advent of thread state, we should have an array
  509. of pending calls per thread in the thread state! Later...
  510. */
  511. #define NPENDINGCALLS 32
  512. static struct {
  513. int (*func)(void *);
  514. void *arg;
  515. } pendingcalls[NPENDINGCALLS];
  516. static volatile int pendingfirst = 0;
  517. static volatile int pendinglast = 0;
  518. static volatile int things_to_do = 0;
  519. int
  520. Py_AddPendingCall(int (*func)(void *), void *arg)
  521. {
  522. static volatile int busy = 0;
  523. int i, j;
  524. /* XXX Begin critical section */
  525. /* XXX If you want this to be safe against nested
  526. XXX asynchronous calls, you'll have to work harder! */
  527. if (busy)
  528. return -1;
  529. busy = 1;
  530. i = pendinglast;
  531. j = (i + 1) % NPENDINGCALLS;
  532. if (j == pendingfirst) {
  533. busy = 0;
  534. return -1; /* Queue full */
  535. }
  536. pendingcalls[i].func = func;
  537. pendingcalls[i].arg = arg;
  538. pendinglast = j;
  539. _Py_Ticker = 0;
  540. things_to_do = 1; /* Signal main loop */
  541. busy = 0;
  542. /* XXX End critical section */
  543. return 0;
  544. }
  545. int
  546. Py_MakePendingCalls(void)
  547. {
  548. static int busy = 0;
  549. #ifdef WITH_THREAD
  550. if (_PyEval_main_thread &&
  551. PyThread_get_thread_ident() != _PyEval_main_thread)
  552. return 0;
  553. #endif
  554. if (busy)
  555. return 0;
  556. busy = 1;
  557. things_to_do = 0;
  558. for (;;) {
  559. int i;
  560. int (*func)(void *);
  561. void *arg;
  562. i = pendingfirst;
  563. if (i == pendinglast)
  564. break; /* Queue empty */
  565. func = pendingcalls[i].func;
  566. arg = pendingcalls[i].arg;
  567. pendingfirst = (i + 1) % NPENDINGCALLS;
  568. if (func(arg) < 0) {
  569. busy = 0;
  570. things_to_do = 1; /* We're not done yet */
  571. return -1;
  572. }
  573. }
  574. busy = 0;
  575. return 0;
  576. }
  577. /* The interpreter's recursion limit */
  578. #ifndef Py_DEFAULT_RECURSION_LIMIT
  579. #define Py_DEFAULT_RECURSION_LIMIT 1000
  580. #endif
  581. static int recursion_limit = Py_DEFAULT_RECURSION_LIMIT;
  582. int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;
  583. int
  584. Py_GetRecursionLimit(void)
  585. {
  586. return recursion_limit;
  587. }
  588. void
  589. Py_SetRecursionLimit(int new_limit)
  590. {
  591. recursion_limit = new_limit;
  592. _Py_CheckRecursionLimit = recursion_limit;
  593. }
  594. /* the macro Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall()
  595. if the recursion_depth reaches _Py_CheckRecursionLimit.
  596. If USE_STACKCHECK, the macro decrements _Py_CheckRecursionLimit
  597. to guarantee that _Py_CheckRecursiveCall() is regularly called.
  598. Without USE_STACKCHECK, there is no need for this. */
  599. int
  600. _Py_CheckRecursiveCall(char *where)
  601. {
  602. PyThreadState *tstate = PyThreadState_GET();
  603. #ifdef USE_STACKCHECK
  604. if (PyOS_CheckStack()) {
  605. --tstate->recursion_depth;
  606. PyErr_SetString(PyExc_MemoryError, "Stack overflow");
  607. return -1;
  608. }
  609. #endif
  610. if (tstate->recursion_depth > recursion_limit) {
  611. --tstate->recursion_depth;
  612. PyErr_Format(PyExc_RuntimeError,
  613. "maximum recursion depth exceeded%s",
  614. where);
  615. return -1;
  616. }
  617. _Py_CheckRecursionLimit = recursion_limit;
  618. return 0;
  619. }
  620. #ifdef __cplusplus
  621. extern "C" void
  622. #else
  623. extern void
  624. #endif
  625. _PyEval_RaiseForUnboundLocal(PyFrameObject *frame, int var_index)
  626. {
  627. format_exc_check_arg(
  628. PyExc_UnboundLocalError,
  629. UNBOUNDLOCAL_ERROR_MSG,
  630. PyTuple_GetItem(frame->f_code->co_varnames, var_index));
  631. }
  632. /* Records whether tracing is on for any thread. Counts the number of
  633. threads for which tstate->c_tracefunc is non-NULL, so if the value
  634. is 0, we know we don't have to check this thread's c_tracefunc.
  635. This speeds up the if statement in PyEval_EvalFrameEx() after
  636. fast_next_opcode*/
  637. int _Py_TracingPossible = 0;
  638. /* for manipulating the thread switch and periodic "stuff" - used to be
  639. per thread, now just a pair o' globals */
  640. int _Py_CheckInterval = 100;
  641. volatile int _Py_Ticker = 100;
  642. #ifdef WITH_LLVM
  643. int _Py_BailError = 0;
  644. #endif
  645. PyObject *
  646. PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
  647. {
  648. return PyEval_EvalCodeEx(co,
  649. globals, locals,
  650. (PyObject **)NULL, 0,
  651. (PyObject **)NULL, 0,
  652. (PyObject **)NULL, 0,
  653. NULL);
  654. }
  655. /* Interpreter main loop */
  656. PyObject *
  657. PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) {
  658. /* This is for backward compatibility with extension modules that
  659. used this API; core interpreter code should call
  660. PyEval_EvalFrame() */
  661. PyObject *result;
  662. f->f_throwflag = throwflag;
  663. result = PyEval_EvalFrame(f);
  664. f->f_throwflag = 0;
  665. return result;
  666. }
  667. PyObject *
  668. PyEval_EvalFrame(PyFrameObject *f)
  669. {
  670. #ifdef DXPAIRS
  671. int lastopcode = 0;
  672. #endif
  673. register PyObject **stack_pointer; /* Next free slot in value stack */
  674. register unsigned char *next_instr;
  675. register int opcode; /* Current opcode */
  676. register int oparg; /* Current opcode argument, if any */
  677. register enum _PyUnwindReason why; /* Reason for block stack unwind */
  678. register int err; /* Error status -- nonzero if error */
  679. register PyObject *x; /* Temporary objects popped off stack */
  680. register PyObject *v;
  681. register PyObject *w;
  682. register PyObject *u;
  683. register PyObject *t;
  684. register PyObject **fastlocals, **freevars;
  685. _PyFrameBailReason bail_reason;
  686. PyObject *retval = NULL; /* Return value */
  687. PyThreadState *tstate = PyThreadState_GET();
  688. PyCodeObject *co;
  689. #ifdef WITH_LLVM
  690. /* We only collect feedback if it will be useful. */
  691. int rec_feedback = (Py_JitControl == PY_JIT_WHENHOT);
  692. #endif
  693. /* when tracing we set things up so that
  694. not (instr_lb <= current_bytecode_offset < instr_ub)
  695. is true when the line being executed has changed. The
  696. initial values are such as to make this false the first
  697. time it is tested. */
  698. int instr_ub = -1, instr_lb = 0, instr_prev = -1;
  699. unsigned char *first_instr;
  700. PyObject *names;
  701. PyObject *consts;
  702. #if defined(Py_DEBUG) || defined(LLTRACE)
  703. /* Make it easier to find out where we are with a debugger */
  704. char *filename;
  705. #endif
  706. /* Computed GOTOs, or
  707. the-optimization-commonly-but-improperly-known-as-"threaded code"
  708. using gcc's labels-as-values extension
  709. (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).
  710. The traditional bytecode evaluation loop uses a "switch" statement, which
  711. decent compilers will optimize as a single indirect branch instruction
  712. combined with a lookup table of jump addresses. However, since the
  713. indirect jump instruction is shared by all opcodes, the CPU will have a
  714. hard time making the right prediction for where to jump next (actually,
  715. it will be always wrong except in the uncommon case of a sequence of
  716. several identical opcodes).
  717. "Threaded code" in contrast, uses an explicit jump table and an explicit
  718. indirect jump instruction at the end of each opcode. Since the jump
  719. instruction is at a different address for each opcode, the CPU will make a
  720. separate prediction for each of these instructions, which is equivalent to
  721. predicting the second opcode of each opcode pair. These predictions have
  722. a much better chance to turn out valid, especially in small bytecode loops.
  723. A mispredicted branch on a modern CPU flushes the whole pipeline and
  724. can cost several CPU cycles (depending on the pipeline depth),
  725. and potentially many more instructions (depending on the pipeline width).
  726. A correctly predicted branch, however, is nearly free.
  727. At the time of this writing, the "threaded code" version is up to 15-20%
  728. faster than the normal "switch" version, depending on the compiler and the
  729. CPU architecture.
  730. We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,
  731. because it would render the measurements invalid.
  732. NOTE: care must be taken that the compiler doesn't try to "optimize" the
  733. indirect jumps by sharing them between all opcodes. Such optimizations
  734. can be disabled on gcc by using the -fno-gcse flag (or possibly
  735. -fno-crossjumping).
  736. */
  737. #if defined(USE_COMPUTED_GOTOS) && defined(DYNAMIC_EXECUTION_PROFILE)
  738. #undef USE_COMPUTED_GOTOS
  739. #endif
  740. #ifdef USE_COMPUTED_GOTOS
  741. /* Import the static jump table */
  742. #include "opcode_targets.h"
  743. /* This macro is used when several opcodes defer to the same implementation
  744. (e.g. SETUP_LOOP, SETUP_FINALLY) */
  745. #define TARGET_WITH_IMPL(op, impl) \
  746. TARGET_##op: \
  747. opcode = op; \
  748. if (HAS_ARG(op)) \
  749. oparg = NEXTARG(); \
  750. case op: \
  751. goto impl; \
  752. #define TARGET(op) \
  753. TARGET_##op: \
  754. opcode = op; \
  755. if (HAS_ARG(op)) \
  756. oparg = NEXTARG(); \
  757. case op:
  758. #define DISPATCH() \
  759. { \
  760. /* Avoid multiple loads from _Py_Ticker despite `volatile` */ \
  761. int _tick = _Py_Ticker - 1; \
  762. _Py_Ticker = _tick; \
  763. if (_tick >= 0) { \
  764. FAST_DISPATCH(); \
  765. } \
  766. continue; \
  767. }
  768. #ifdef LLTRACE
  769. #define FAST_DISPATCH() \
  770. { \
  771. if (!lltrace && !_Py_TracingPossible) { \
  772. f->f_lasti = INSTR_OFFSET(); \
  773. goto *opcode_targets[*next_instr++]; \
  774. } \
  775. goto fast_next_opcode; \
  776. }
  777. #else
  778. #define FAST_DISPATCH() \
  779. { \
  780. if (!_Py_TracingPossible) { \
  781. f->f_lasti = INSTR_OFFSET(); \
  782. goto *opcode_targets[*next_instr++]; \
  783. } \
  784. goto fast_next_opcode; \
  785. }
  786. #endif
  787. #else
  788. #define TARGET(op) \
  789. case op:
  790. #define TARGET_WITH_IMPL(op, impl) \
  791. /* silence compiler warnings about `impl` unused */ \
  792. if (0) goto impl; \
  793. case op:
  794. #define DISPATCH() continue
  795. #define FAST_DISPATCH() goto fast_next_opcode
  796. #endif
  797. /* Tuple access macros */
  798. #ifndef Py_DEBUG
  799. #define GETITEM(v, i) PyTuple_GET_ITEM((PyTupleObject *)(v), (i))
  800. #else
  801. #define GETITEM(v, i) PyTuple_GetItem((v), (i))
  802. #endif
  803. /* Code access macros */
  804. #define INSTR_OFFSET() ((int)(next_instr - first_instr))
  805. #define NEXTOP() (*next_instr++)
  806. #define NEXTARG() (next_instr += 2, (next_instr[-1]<<8) + next_instr[-2])
  807. #define PEEKARG() ((next_instr[2]<<8) + next_instr[1])
  808. #define JUMPTO(x) (next_instr = first_instr + (x))
  809. #define JUMPBY(x) (next_instr += (x))
  810. /* Feedback-gathering macros */
  811. #ifdef WITH_LLVM
  812. #define RECORD_TYPE(arg_index, obj) \
  813. if(rec_feedback){record_type(co, opcode, f->f_lasti, arg_index, obj);}
  814. #define RECORD_OBJECT(arg_index, obj) \
  815. if(rec_feedback){record_object(co, opcode, f->f_lasti, arg_index, obj);}
  816. #define RECORD_FUNC(obj) \
  817. if(rec_feedback){record_func(co, opcode, f->f_lasti, 0, obj);}
  818. #define INC_COUNTER(arg_index, counter_id) \
  819. if (rec_feedback) { \
  820. inc_feedback_counter(co, opcode, f->f_lasti, arg_index, \
  821. counter_id); \
  822. }
  823. #define RECORD_TRUE() \
  824. INC_COUNTER(0, PY_FDO_JUMP_TRUE)
  825. #define RECORD_FALSE() \
  826. INC_COUNTER(0, PY_FDO_JUMP_FALSE)
  827. #define RECORD_NONBOOLEAN() \
  828. INC_COUNTER(0, PY_FDO_JUMP_NON_BOOLEAN)
  829. #define UPDATE_HOTNESS_JABS() \
  830. do { if (oparg <= f->f_lasti) ++co->co_hotness; } while (0)
  831. #else
  832. #define RECORD_TYPE(arg_index, obj)
  833. #define RECORD_OBJECT(arg_index, obj)
  834. #define RECORD_FUNC(obj)
  835. #define INC_COUNTER(arg_index, counter_id)
  836. #define RECORD_TRUE()
  837. #define RECORD_FALSE()
  838. #define RECORD_NONBOOLEAN()
  839. #define UPDATE_HOTNESS_JABS()
  840. #endif /* WITH_LLVM */
  841. /* OpCode prediction macros
  842. Some opcodes tend to come in pairs thus making it possible to
  843. predict the second code when the first is run. For example,
  844. GET_ITER is often followed by FOR_ITER. And FOR_ITER is often
  845. followed by STORE_FAST or UNPACK_SEQUENCE.
  846. Verifying the prediction costs a single high-speed test of a register
  847. variable against a constant. If the pairing was good, then the
  848. processor's own internal branch predication has a high likelihood of
  849. success, resulting in a nearly zero-overhead transition to the
  850. next opcode. A successful prediction saves a trip through the eval-loop
  851. including its two unpredictable branches, the HAS_ARG test and the
  852. switch-case. Combined with the processor's internal branch prediction,
  853. a successful PREDICT has the effect of making the two opcodes run as if
  854. they were a single new opcode with the bodies combined.
  855. If collecting opcode statistics, your choices are to either keep the
  856. predictions turned-on and interpret the results as if some opcodes
  857. had been combined or turn-off predictions so that the opcode frequency
  858. counter updates for both opcodes.
  859. Opcode prediction is disabled with threaded code, since the latter allows
  860. the CPU to record separate branch prediction information for each
  861. opcode.
  862. */
  863. #if defined(DYNAMIC_EXECUTION_PROFILE) || defined(USE_COMPUTED_GOTOS)
  864. #define PREDICT(op) if (0) goto PRED_##op
  865. #define PREDICTED(op) PRED_##op:
  866. #define PREDICTED_WITH_ARG(op) PRED_##op:
  867. #else
  868. #define PREDICT(op) if (*next_instr == op) goto PRED_##op
  869. #ifdef WITH_LLVM
  870. #define PREDICTED_COMMON(op) f->f_lasti = INSTR_OFFSET(); opcode = op;
  871. #else
  872. #define PREDICTED_COMMON(op) /* nothing */
  873. #endif
  874. #define PREDICTED(op) PRED_##op: PREDICTED_COMMON(op) next_instr++
  875. #define PREDICTED_WITH_ARG(op) PRED_##op: PREDICTED_COMMON(op) \
  876. oparg = PEEKARG(); next_instr += 3
  877. #endif
  878. /* Stack manipulation macros */
  879. /* The stack can grow at most MAXINT deep, as co_nlocals and
  880. co_stacksize are ints. */
  881. #define STACK_LEVEL() ((int)(stack_pointer - f->f_valuestack))
  882. #define EMPTY() (STACK_LEVEL() == 0)
  883. #define TOP() (stack_pointer[-1])
  884. #define SECOND() (stack_pointer[-2])
  885. #define THIRD() (stack_pointer[-3])
  886. #define FOURTH() (stack_pointer[-4])
  887. #define SET_TOP(v) (stack_pointer[-1] = (v))
  888. #define SET_SECOND(v) (stack_pointer[-2] = (v))
  889. #define SET_THIRD(v) (stack_pointer[-3] = (v))
  890. #define SET_FOURTH(v) (stack_pointer[-4] = (v))
  891. #define BASIC_STACKADJ(n) (stack_pointer += n)
  892. #define BASIC_PUSH(v) (*stack_pointer++ = (v))
  893. #define BASIC_POP() (*--stack_pointer)
  894. #ifdef LLTRACE
  895. #define PUSH(v) { (void)(BASIC_PUSH(v), \
  896. lltrace && prtrace(TOP(), "push")); \
  897. assert(STACK_LEVEL() <= co->co_stacksize); }
  898. #define POP() ((void)(lltrace && prtrace(TOP(), "pop")), \
  899. BASIC_POP())
  900. #define STACKADJ(n) { (void)(BASIC_STACKADJ(n), \
  901. lltrace && prtrace(TOP(), "stackadj")); \
  902. assert(STACK_LEVEL() <= co->co_stacksize); }
  903. #define EXT_POP(STACK_POINTER) ((void)(lltrace && \
  904. prtrace((STACK_POINTER)[-1], "ext_pop")), \
  905. *--(STACK_POINTER))
  906. #define EXT_PUSH(v, STACK_POINTER) ((void)(*(STACK_POINTER)++ = (v), \
  907. lltrace && prtrace((STACK_POINTER)[-1], "ext_push")))
  908. #else
  909. #define PUSH(v) BASIC_PUSH(v)
  910. #define POP() BASIC_POP()
  911. #define STACKADJ(n) BASIC_STACKADJ(n)
  912. #define EXT_POP(STACK_POINTER) (*--(STACK_POINTER))
  913. #define EXT_PUSH(v, STACK_POINTER) (*(STACK_POINTER)++ = (v))
  914. #endif
  915. /* Local variable macros */
  916. #define GETLOCAL(i) (fastlocals[i])
  917. /* The SETLOCAL() macro must not DECREF the local variable in-place and
  918. then store the new value; it must copy the old value to a temporary
  919. value, then store the new value, and then DECREF the temporary value.
  920. This is because it is possible that during the DECREF the frame is
  921. accessed by other code (e.g. a __del__ method or gc.collect()) and the
  922. variable would be pointing to already-freed memory. */
  923. #define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \
  924. GETLOCAL(i) = value; \
  925. Py_XDECREF(tmp); } while (0)
  926. /* Start of code */
  927. if (f == NULL)
  928. return NULL;
  929. #ifdef WITH_LLVM
  930. bail_reason = (_PyFrameBailReason)f->f_bailed_from_llvm;
  931. #else
  932. bail_reason = _PYFRAME_NO_BAIL;
  933. #endif /* WITH_LLVM */
  934. /* push frame */
  935. if (bail_reason == _PYFRAME_NO_BAIL && Py_EnterRecursiveCall(""))
  936. return NULL;
  937. co = f->f_code;
  938. tstate->frame = f;
  939. #ifdef WITH_LLVM
  940. maybe_compile(co, f);
  941. if (f->f_use_jit) {
  942. assert(bail_reason == _PYFRAME_NO_BAIL);
  943. assert(co->co_native_function != NULL &&
  944. "maybe_compile was supposed to ensure"
  945. " that co_native_function exists");
  946. if (!co->co_use_jit) {
  947. // A frame cannot use_jit if the underlying code object
  948. // can't use_jit. This comes up when a generator is
  949. // invalidated while active.
  950. f->f_use_jit = 0;
  951. }
  952. else {
  953. assert(co->co_fatalbailcount < PY_MAX_FATALBAILCOUNT);
  954. retval = co->co_native_function(f);
  955. goto exit_eval_frame;
  956. }
  957. }
  958. if (bail_reason != _PYFRAME_NO_BAIL) {
  959. #ifdef Py_WITH_INSTRUMENTATION
  960. bail_count_stats->RecordBail(f, bail_reason);
  961. #endif
  962. if (_Py_BailError) {
  963. /* When we bail, we set f_lasti to the current opcode
  964. * minus 1, so we add one back. */
  965. int lasti = f->f_lasti + 1;
  966. PyErr_Format(PyExc_RuntimeError, "bailed to the "
  967. "interpreter at opcode index %d", lasti);
  968. goto exit_eval_frame;
  969. }
  970. }
  971. /* Create co_runtime_feedback now that we're about to use it. You
  972. * might think this would cause a problem if the user flips
  973. * Py_JitControl from "never" to "whenhot", but since the value of
  974. * rec_feedback is constant for the duration of this frame's execution,
  975. * we will not accidentally try to record feedback without initializing
  976. * co_runtime_feedback. */
  977. if (rec_feedback && co->co_runtime_feedback == NULL) {
  978. #if Py_WITH_INSTRUMENTATION
  979. feedback_map_counter->IncCounter();
  980. #endif
  981. co->co_runtime_feedback = PyFeedbackMap_New();
  982. }
  983. #endif /* WITH_LLVM */
  984. switch (bail_reason) {
  985. case _PYFRAME_NO_BAIL:
  986. case _PYFRAME_TRACE_ON_ENTRY:
  987. if (tstate->use_tracing) {
  988. if (_PyEval_TraceEnterFunction(tstate, f))
  989. /* Trace or profile function raised
  990. an error. */
  991. goto exit_eval_frame;
  992. }
  993. break;
  994. case _PYFRAME_BACKEDGE_TRACE:
  995. /* If we bailed because of a backedge, set instr_prev
  996. to ensure a line trace call. */
  997. instr_prev = INT_MAX;
  998. break;
  999. case _PYFRAME_CALL_PROFILE:
  1000. case _PYFRAME_LINE_TRACE:
  1001. case _PYFRAME_FATAL_GUARD_FAIL:
  1002. case _PYFRAME_GUARD_FAIL:
  1003. /* These are handled by the opcode dispatch loop. */
  1004. break;
  1005. default:
  1006. PyErr_Format(PyExc_SystemError, "unknown bail reason");
  1007. goto exit_eval_frame;
  1008. }
  1009. names = co->co_names;
  1010. consts = co->co_consts;
  1011. fastlocals = f->f_localsplus;
  1012. freevars = f->f_localsplus + co->co_nlocals;
  1013. first_instr = (unsigned char*) PyString_AS_STRING(co->co_code);
  1014. /* An explanation is in order for the next line.
  1015. f->f_lasti now refers to the index of the last instruction
  1016. executed. You might think this was obvious from the name, but
  1017. this wasn't always true before 2.3! PyFrame_New now sets
  1018. f->f_lasti to -1 (i.e. the index *before* the first instruction)
  1019. and YIELD_VALUE doesn't fiddle with f_lasti any more. So this
  1020. does work. Promise.
  1021. When the PREDICT() macros are enabled, some opcode pairs follow in
  1022. direct succession without updating f->f_lasti. A successful
  1023. prediction effectively links the two codes together as if they
  1024. were a single new opcode; accordingly,f->f_lasti will point to
  1025. the first code in the pair (for instance, GET_ITER followed by
  1026. FOR_ITER is effectively a single opcode and f->f_lasti will point
  1027. at to the beginning of the combined pair.)
  1028. */
  1029. next_instr = first_instr + f->f_lasti + 1;
  1030. stack_pointer = f->f_stacktop;
  1031. assert(stack_pointer != NULL);
  1032. f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */
  1033. #ifdef LLTRACE
  1034. lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
  1035. #endif
  1036. #if defined(Py_DEBUG) || defined(LLTRACE)
  1037. filename = PyString_AsString(co->co_filename);
  1038. #endif
  1039. why = UNWIND_NOUNWIND;
  1040. w = NULL;
  1041. /* Note that this goes after the LLVM handling code so we don't log
  1042. * this event when calling LLVM functions. Do this before the throwflag
  1043. * check below to avoid mismatched enter/exit events in the log. */
  1044. PY_LOG_TSC_EVENT(CALL_ENTER_EVAL);
  1045. if (f->f_throwflag) { /* support for generator.throw() */
  1046. why = UNWIND_EXCEPTION;
  1047. goto on_error;
  1048. }
  1049. for (;;) {
  1050. assert(stack_pointer >= f->f_valuestack); /* else underflow */
  1051. assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */
  1052. /* Do periodic things. Doing this every time through
  1053. the loop would add too much overhead, so we do it
  1054. only every Nth instruction. We also do it if
  1055. ``things_to_do'' is set, i.e. when an asynchronous
  1056. event needs attention (e.g. a signal handler or
  1057. async I/O handler); see Py_AddPendingCall() and
  1058. Py_MakePendingCalls() above. */
  1059. if (--_Py_Ticker < 0) {
  1060. if (*next_instr == SETUP_FINALLY) {
  1061. /* Make the last opcode before
  1062. a try: finally: block uninterruptable. */
  1063. goto fast_next_opcode;
  1064. }
  1065. if (_PyEval_HandlePyTickerExpired(tstate) == -1) {
  1066. why = UNWIND_EXCEPTION;
  1067. goto on_error;
  1068. }
  1069. }
  1070. fast_next_opcode:
  1071. f->f_lasti = INSTR_OFFSET();
  1072. /* line-by-line tracing support */
  1073. if (_Py_TracingPossible &&
  1074. tstate->c_tracefunc != NULL && !tstate->tracing) {
  1075. /* see maybe_call_line_trace
  1076. for expository comments */
  1077. f->f_stacktop = stack_pointer;
  1078. err = maybe_call_line_trace(tstate->c_tracefunc,
  1079. tstate->c_traceobj,
  1080. f, &instr_lb, &instr_ub,
  1081. &instr_prev);
  1082. /* Reload possibly changed frame fields */
  1083. JUMPTO(f->f_lasti);
  1084. assert(f->f_stacktop != NULL);
  1085. stack_pointer = f->f_stacktop;
  1086. f->f_stacktop = NULL;
  1087. if (err) {
  1088. /* trace function raised an exception */
  1089. why = UNWIND_EXCEPTION;
  1090. goto on_error;
  1091. }
  1092. }
  1093. /* Extract opcode and argument */
  1094. opcode = NEXTOP();
  1095. oparg = 0; /* allows oparg to be stored in a register because
  1096. it doesn't have to be remembered across a full loop */
  1097. if (HAS_ARG(opcode))
  1098. oparg = NEXTARG();
  1099. dispatch_opcode:
  1100. #ifdef DYNAMIC_EXECUTION_PROFILE
  1101. #ifdef DXPAIRS
  1102. dxpairs[lastopcode][opcode]++;
  1103. lastopcode = opcode;
  1104. #endif
  1105. dxp[opcode]++;
  1106. #endif
  1107. #ifdef LLTRACE
  1108. /* Instruction tracing */
  1109. if (lltrace) {
  1110. if (HAS_ARG(opcode)) {
  1111. printf("%d: %d, %d\n",
  1112. f->f_lasti, opcode, oparg);
  1113. }
  1114. else {
  1115. printf("%d: %d\n",
  1116. f->f_lasti, opcode);
  1117. }
  1118. }
  1119. #endif
  1120. /* Main switch on opcode */
  1121. assert(why == UNWIND_NOUNWIND);
  1122. /* XXX(jyasskin): Add an assertion under CHECKEXC that
  1123. !PyErr_Occurred(). */
  1124. switch (opcode) {
  1125. /* BEWARE!
  1126. It is essential that any operation that fails sets
  1127. why to anything but UNWIND_NOUNWIND, and that no operation
  1128. that succeeds does this! */
  1129. /* case STOP_CODE: this is an error! */
  1130. TARGET(NOP)
  1131. FAST_DISPATCH();
  1132. TARGET(LOAD_FAST)
  1133. x = GETLOCAL(oparg);
  1134. if (x != NULL) {
  1135. Py_INCREF(x);
  1136. PUSH(x);
  1137. FAST_DISPATCH();
  1138. }
  1139. _PyEval_RaiseForUnboundLocal(f, oparg);
  1140. why = UNWIND_EXCEPTION;
  1141. break;
  1142. TARGET(LOAD_CONST)
  1143. x = GETITEM(consts, oparg);
  1144. Py_INCREF(x);
  1145. PUSH(x);
  1146. FAST_DISPATCH();
  1147. PREDICTED_WITH_ARG(STORE_FAST);
  1148. TARGET(STORE_FAST)
  1149. v = POP();
  1150. SETLOCAL(oparg, v);
  1151. FAST_DISPATCH();
  1152. TARGET(POP_TOP)
  1153. v = POP();
  1154. Py_DECREF(v);
  1155. FAST_DISPATCH();
  1156. TARGET(ROT_TWO)
  1157. v = TOP();
  1158. w = SECOND();
  1159. SET_TOP(w);
  1160. SET_SECOND(v);
  1161. FAST_DISPATCH();
  1162. TARGET(ROT_THREE)
  1163. v = TOP();
  1164. w = SECOND();
  1165. x = THIRD();
  1166. SET_TOP(w);
  1167. SET_SECOND(x);
  1168. SET_THIRD(v);
  1169. FAST_DISPATCH();
  1170. TARGET(ROT_FOUR)
  1171. u = TOP();
  1172. v = SECOND();
  1173. w = THIRD();
  1174. x = FOURTH();
  1175. SET_TOP(v);
  1176. SET_SECOND(w);
  1177. SET_THIRD(x);
  1178. SET_FOURTH(u);
  1179. FAST_DISPATCH();
  1180. TARGET(DUP_TOP)
  1181. v = TOP();
  1182. Py_INCREF(v);
  1183. PUSH(v);
  1184. FAST_DISPATCH();
  1185. TARGET(DUP_TOP_TWO)
  1186. x = TOP();
  1187. Py_INCREF(x);
  1188. w = SECOND();
  1189. Py_INCREF(w);
  1190. STACKADJ(2);
  1191. SET_TOP(x);
  1192. SET_SECOND(w);
  1193. FAST_DISPATCH();
  1194. TARGET(DUP_TOP_THREE)
  1195. x = TOP();
  1196. Py_INCREF(x);
  1197. w = SECOND();
  1198. Py_INCREF(w);
  1199. v = THIRD();
  1200. Py_INCREF(v);
  1201. STACKADJ(3);
  1202. SET_TOP(x);
  1203. SET_SECOND(w);
  1204. SET_THIRD(v);
  1205. FAST_DISPATCH();
  1206. TARGET(UNARY_POSITIVE)
  1207. v = TOP();
  1208. RECORD_TYPE(0, v);
  1209. x = PyNumber_Positive(v);
  1210. Py_DECREF(v);
  1211. SET_TOP(x);
  1212. if (x == NULL) {
  1213. why = UNWIND_EXCEPTION;
  1214. break;
  1215. }
  1216. DISPATCH();
  1217. TARGET(UNARY_NEGATIVE)
  1218. v = TOP();
  1219. RECORD_TYPE(0, v);
  1220. x = PyNumber_Negative(v);
  1221. Py_DECREF(v);
  1222. SET_TOP(x);
  1223. if (x == NULL) {
  1224. why = UNWIND_EXCEPTION;
  1225. break;
  1226. }
  1227. DISPATCH();
  1228. TARGET(UNARY_NOT)
  1229. v = TOP();
  1230. RECORD_TYPE(0, v);
  1231. err = PyObject_IsTrue(v);
  1232. Py_DECREF(v);
  1233. if (err == 0) {
  1234. Py_INCREF(Py_True);
  1235. SET_TOP(Py_True);
  1236. DISPATCH();
  1237. }
  1238. else if (err > 0) {
  1239. Py_INCREF(Py_False);
  1240. SET_TOP(Py_False);
  1241. DISPATCH();
  1242. }
  1243. STACKADJ(-1);
  1244. why = UNWIND_EXCEPTION;
  1245. break;
  1246. TARGET(UNARY_CONVERT)
  1247. v = TOP();
  1248. RECORD_TYPE(0, v);
  1249. x = PyObject_Repr(v);
  1250. Py_DECREF(v);
  1251. SET_TOP(x);
  1252. if (x == NULL) {
  1253. why = UNWIND_EXCEPTION;
  1254. break;
  1255. }
  1256. DISPATCH();
  1257. TARGET(UNARY_INVERT)
  1258. v = TOP();
  1259. RECORD_TYPE(0, v);
  1260. x = PyNumber_Invert(v);
  1261. Py_DECREF(v);
  1262. SET_TOP(x);
  1263. if (x == NULL) {
  1264. why = UNWIND_EXCEPTION;
  1265. break;
  1266. }
  1267. DISPATCH();
  1268. TARGET(BINARY_POWER)
  1269. w = POP();
  1270. v = TOP();
  1271. RECORD_TYPE(0, v);
  1272. RECORD_TYPE(1, w);
  1273. x = PyNumber_Power(v, w, Py_None);
  1274. Py_DECREF(v);
  1275. Py_DECREF(w);
  1276. SET_TOP(x);
  1277. if (x == NULL) {
  1278. why = UNWIND_EXCEPTION;
  1279. break;
  1280. }
  1281. DISPATCH();
  1282. TARGET(BINARY_MULTIPLY)
  1283. w = POP();
  1284. v = TOP();
  1285. RECORD_TYPE(0, v);
  1286. RECORD_TYPE(1, w);
  1287. x = PyNumber_Multiply(v, w);
  1288. Py_DECREF(v);
  1289. Py_DECREF(w);
  1290. SET_TOP(x);
  1291. if (x == NULL) {
  1292. why = UNWIND_EXCEPTION;
  1293. break;
  1294. }
  1295. DISPATCH();
  1296. TARGET(BINARY_DIVIDE)
  1297. if (!_Py_QnewFlag) {
  1298. w = POP();
  1299. v = TOP();
  1300. RECORD_TYPE(0, v);
  1301. RECORD_TYPE(1, w);
  1302. x = PyNumber_Divide(v, w);
  1303. Py_DECREF(v);
  1304. Py_DECREF(w);
  1305. SET_TOP(x);
  1306. if (x == NULL) {
  1307. why = UNWIND_EXCEPTION;
  1308. break;
  1309. }
  1310. DISPATCH();
  1311. }
  1312. /* -Qnew is in effect: jump to BINARY_TRUE_DIVIDE */
  1313. goto _binary_true_divide;
  1314. TARGET(BINARY_TRUE_DIVIDE)
  1315. _binary_true_divide:
  1316. w = POP();
  1317. v = TOP();
  1318. RECORD_TYPE(0, v);
  1319. RECORD_TYPE(1, w);
  1320. x = PyNumber_TrueDivide(v, w);
  1321. Py_DECREF(v);
  1322. Py_DECREF(w);
  1323. SET_TOP(x);
  1324. if (x == NULL) {
  1325. why = UNWIND_EXCEPTION;
  1326. break;
  1327. }
  1328. DISPATCH();
  1329. TARGET(BINARY_FLOOR_DIVIDE)
  1330. w = POP();
  1331. v = TOP();
  1332. RECORD_TYPE(0, v);
  1333. RECORD_TYPE(1, w);
  1334. x = PyNumber_FloorDivide(v, w);
  1335. Py_DECREF(v);
  1336. Py_DECREF(w);
  1337. SET_TOP(x);
  1338. if (x == NULL) {
  1339. why = UNWIND_EXCEPTION;
  1340. break;
  1341. }
  1342. DISPATCH();
  1343. TARGET(BINARY_MODULO)
  1344. w = POP();
  1345. v = TOP();
  1346. RECORD_TYPE(0, v);
  1347. RECORD_TYPE(1, w);
  1348. if (PyString_CheckExact(v))
  1349. x = PyString_Format(v, w);
  1350. else
  1351. x = PyNumber_Remainder(v, w);
  1352. Py_DECREF(v);
  1353. Py_DECREF(w);
  1354. SET_TOP(x);
  1355. if (x == NULL) {
  1356. why = UNWIND_EXCEPTION;
  1357. break;
  1358. }
  1359. DISPATCH();
  1360. TARGET(BINARY_ADD)
  1361. w = POP();
  1362. v = TOP();
  1363. RECORD_TYPE(0, v);
  1364. RECORD_TYPE(1, w);
  1365. if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
  1366. /* INLINE: int + int */
  1367. register long a, b, i;
  1368. a = PyInt_AS_LONG(v);
  1369. b = PyInt_AS_LONG(w);
  1370. i = a + b;
  1371. if ((i^a) < 0 && (i^b) < 0)
  1372. goto slow_add;
  1373. x = PyInt_FromLong(i);
  1374. }
  1375. else if (PyString_CheckExact(v) &&
  1376. PyString_CheckExact(w)) {
  1377. x = string_concatenate(v, w, f, next_instr);
  1378. /* string_concatenate consumed the ref to v */
  1379. goto skip_decref_vx;
  1380. }
  1381. else {
  1382. slow_add:
  1383. x = PyNumber_Add(v, w);
  1384. }
  1385. Py_DECREF(v);
  1386. skip_decref_vx:
  1387. Py_DECREF(w);
  1388. SET_TOP(x);
  1389. if (x == NULL) {
  1390. why = UNWIND_EXCEPTION;
  1391. break;
  1392. }
  1393. DISPATCH();
  1394. TARGET(BINARY_SUBTRACT)
  1395. w = POP();
  1396. v = TOP();
  1397. RECORD_TYPE(0, v);
  1398. RECORD_TYPE(1, w);
  1399. if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
  1400. /* INLINE: int - int */
  1401. register long a, b, i;
  1402. a = PyInt_AS_LONG(v);
  1403. b = PyInt_AS_LONG(w);
  1404. i = a - b;
  1405. if ((i^a) < 0 && (i^~b) < 0)
  1406. goto slow_sub;
  1407. x = PyInt_FromLong(i);
  1408. }
  1409. else {
  1410. slow_sub:
  1411. x = PyNumber_Subtract(v, w);
  1412. }
  1413. Py_DECREF(v);
  1414. Py_DECREF(w);
  1415. SET_TOP(x);
  1416. if (x == NULL) {
  1417. why = UNWIND_EXCEPTION;
  1418. break;
  1419. }
  1420. DISPATCH();
  1421. TARGET(BINARY_SUBSCR)
  1422. w = POP();
  1423. v = TOP();
  1424. RECORD_TYPE(0, v);
  1425. RECORD_TYPE(1, w);
  1426. if (PyList_CheckExact(v) && PyInt_CheckExact(w)) {
  1427. /* INLINE: list[int] */
  1428. Py_ssize_t i = PyInt_AsSsize_t(w);
  1429. if (i < 0)
  1430. i += PyList_GET_SIZE(v);
  1431. if (i >= 0 && i < PyList_GET_SIZE(v)) {
  1432. x = PyList_GET_ITEM(v, i);
  1433. Py_INCREF(x);
  1434. }
  1435. else
  1436. goto slow_get;
  1437. }
  1438. else
  1439. slow_get:
  1440. x = PyObject_GetItem(v, w);
  1441. Py_DECREF(v);
  1442. Py_DECREF(w);
  1443. SET_TOP(x);
  1444. if (x == NULL) {
  1445. why = UNWIND_EXCEPTION;
  1446. break;
  1447. }
  1448. DISPATCH();
  1449. TARGET(BINARY_LSHIFT)
  1450. w = POP();
  1451. v = TOP();
  1452. RECORD_TYPE(0, v);
  1453. RECORD_TYPE(1, w);
  1454. x = PyNumber_Lshift(v, w);
  1455. Py_DECREF(v);
  1456. Py_DECREF(w);
  1457. SET_TOP(x);
  1458. if (x == NULL) {
  1459. why = UNWIND_EXCEPTION;
  1460. break;
  1461. }
  1462. DISPATCH();
  1463. TARGET(BINARY_RSHIFT)
  1464. w = POP();
  1465. v = TOP();
  1466. RECORD_TYPE(0, v);
  1467. RECORD_TYPE(1, w);
  1468. x = PyNumber_Rshift(v, w);
  1469. Py_DECREF(v);
  1470. Py_DECREF(w);
  1471. SET_TOP(x);
  1472. if (x == NULL) {
  1473. why = UNWIND_EXCEPTION;
  1474. break;
  1475. }
  1476. DISPATCH();
  1477. TARGET(BINARY_AND)
  1478. w = POP();
  1479. v = TOP();
  1480. RECORD_TYPE(0, v);
  1481. RECORD_TYPE(1, w);
  1482. x = PyNumber_And(v, w);
  1483. Py_DECREF(v);
  1484. Py_DECREF(w);
  1485. SET_TOP(x);
  1486. if (x == NULL) {
  1487. why = UNWIND_EXCEPTION;
  1488. break;
  1489. }
  1490. DISPATCH();
  1491. TARGET(BINARY_XOR)
  1492. w = POP();
  1493. v = TOP();
  1494. RECORD_TYPE(0, v);
  1495. RECORD_TYPE(1, w);
  1496. x = PyNumber_Xor(v, w);
  1497. Py_DECREF(v);
  1498. Py_DECREF(w);
  1499. SET_TOP(x);
  1500. if (x == NULL) {
  1501. why = UNWIND_EXCEPTION;
  1502. break;
  1503. }
  1504. DISPATCH();
  1505. TARGET(BINARY_OR)
  1506. w = POP();
  1507. v = TOP();
  1508. RECORD_TYPE(0, v);
  1509. RECORD_TYPE(1, w);
  1510. x = PyNumber_Or(v, w);
  1511. Py_DECREF(v);
  1512. Py_DECREF(w);
  1513. SET_TOP(x);
  1514. if (x == NULL) {
  1515. why = UNWIND_EXCEPTION;
  1516. break;
  1517. }
  1518. DISPATCH();
  1519. TARGET(LIST_APPEND)
  1520. w = POP();
  1521. v = POP();
  1522. RECORD_TYPE(0, v);
  1523. RECORD_TYPE(1, w);
  1524. err = PyList_Append(v, w);
  1525. Py_DECREF(v);
  1526. Py_DECREF(w);
  1527. if (err != 0) {
  1528. why = UNWIND_EXCEPTION;
  1529. break;
  1530. }
  1531. PREDICT(JUMP_ABSOLUTE);
  1532. DISPATCH();
  1533. TARGET(INPLACE_POWER)
  1534. w = POP();
  1535. v = TOP();
  1536. RECORD_TYPE(0, v);
  1537. RECORD_TYPE(1, w);
  1538. x = PyNumber_InPlacePower(v, w, Py_None);
  1539. Py_DECREF(v);
  1540. Py_DECREF(w);
  1541. SET_TOP(x);
  1542. if (x == NULL) {
  1543. why = UNWIND_EXCEPTION;
  1544. break;
  1545. }
  1546. DISPATCH();
  1547. TARGET(INPLACE_MULTIPLY)
  1548. w = POP();
  1549. v = TOP();
  1550. RECORD_TYPE(0, v);
  1551. RECORD_TYPE(1, w);
  1552. x = PyNumber_InPlaceMultiply(v, w);
  1553. Py_DECREF(v);
  1554. Py_DECREF(w);
  1555. SET_TOP(x);
  1556. if (x == NULL) {
  1557. why = UNWIND_EXCEPTION;
  1558. break;
  1559. }
  1560. DISPATCH();
  1561. TARGET(INPLACE_DIVIDE)
  1562. if (!_Py_QnewFlag) {
  1563. w = POP();
  1564. v = TOP();
  1565. RECORD_TYPE(0, v);
  1566. RECORD_TYPE(1, w);
  1567. x = PyNumber_InPlaceDivide(v, w);
  1568. Py_DECREF(v);
  1569. Py_DECREF(w);
  1570. SET_TOP(x);
  1571. if (x == NULL) {
  1572. why = UNWIND_EXCEPTION;
  1573. break;
  1574. }
  1575. DISPATCH();
  1576. }
  1577. /* -Qnew is in effect: jump to INPLACE_TRUE_DIVIDE */
  1578. goto _inplace_true_divide;
  1579. TARGET(INPLACE_TRUE_DIVIDE)
  1580. _inplace_true_divide:
  1581. w = POP();
  1582. v = TOP();
  1583. RECORD_TYPE(0, v);
  1584. RECORD_TYPE(1, w);
  1585. x = PyNumber_InPlaceTrueDivide(v, w);
  1586. Py_DECREF(v);
  1587. Py_DECREF(w);
  1588. SET_TOP(x);
  1589. if (x == NULL) {
  1590. why = UNWIND_EXCEPTION;
  1591. break;
  1592. }
  1593. DISPATCH();
  1594. TARGET(INPLACE_FLOOR_DIVIDE)
  1595. w = POP();
  1596. v = TOP();
  1597. RECORD_TYPE(0, v);
  1598. RECORD_TYPE(1, w);
  1599. x = PyNumber_InPlaceFloorDivide(v, w);
  1600. Py_DECREF(v);
  1601. Py_DECREF(w);
  1602. SET_TOP(x);
  1603. if (x == NULL) {
  1604. why = UNWIND_EXCEPTION;
  1605. break;
  1606. }
  1607. DISPATCH();
  1608. TARGET(INPLACE_MODULO)
  1609. w = POP();
  1610. v = TOP();
  1611. RECORD_TYPE(0, v);
  1612. RECORD_TYPE(1, w);
  1613. x = PyNumber_InPlaceRemainder(v, w);
  1614. Py_DECREF(v);
  1615. Py_DECREF(w);
  1616. SET_TOP(x);
  1617. if (x == NULL) {
  1618. why = UNWIND_EXCEPTION;
  1619. break;
  1620. }
  1621. DISPATCH();
  1622. TARGET(INPLACE_ADD)
  1623. w = POP();
  1624. v = TOP();
  1625. RECORD_TYPE(0, v);
  1626. RECORD_TYPE(1, w);
  1627. if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
  1628. /* INLINE: int + int */
  1629. register long a, b, i;
  1630. a = PyInt_AS_LONG(v);
  1631. b = PyInt_AS_LONG(w);
  1632. i = a + b;
  1633. if ((i^a) < 0 && (i^b) < 0)
  1634. goto slow_iadd;
  1635. x = PyInt_FromLong(i);
  1636. }
  1637. else if (PyString_CheckExact(v) &&
  1638. PyString_CheckExact(w)) {
  1639. x = string_concatenate(v, w, f, next_instr);
  1640. /* string_concatenate consumed the ref to v */
  1641. goto skip_decref_v;
  1642. }
  1643. else {
  1644. slow_iadd:
  1645. x = PyNumber_InPlaceAdd(v, w);
  1646. }
  1647. Py_DECREF(v);
  1648. skip_decref_v:
  1649. Py_DECREF(w);
  1650. SET_TOP(x);
  1651. if (x == NULL) {
  1652. why = UNWIND_EXCEPTION;
  1653. break;
  1654. }
  1655. DISPATCH();
  1656. TARGET(INPLACE_SUBTRACT)
  1657. w = POP();
  1658. v = TOP();
  1659. RECORD_TYPE(0, v);
  1660. RECORD_TYPE(1, w);
  1661. if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) {
  1662. /* INLINE: int - int */
  1663. register long a, b, i;
  1664. a = PyInt_AS_LONG(v);
  1665. b = PyInt_AS_LONG(w);
  1666. i = a - b;
  1667. if ((i^a) < 0 && (i^~b) < 0)
  1668. goto slow_isub;
  1669. x = PyInt_FromLong(i);
  1670. }
  1671. else {
  1672. slow_isub:
  1673. x = PyNumber_InPlaceSubtract(v, w);
  1674. }
  1675. Py_DECREF(v);
  1676. Py_DECREF(w);
  1677. SET_TOP(x);
  1678. if (x == NULL) {
  1679. why = UNWIND_EXCEPTION;
  1680. break;
  1681. }
  1682. DISPATCH();
  1683. TARGET(INPLACE_LSHIFT)
  1684. w = POP();
  1685. v = TOP();
  1686. RECORD_TYPE(0, v);
  1687. RECORD_TYPE(1, w);
  1688. x = PyNumber_InPlaceL