/JIT/opcodes/container.cc

  1#include "Python.h"
  2
  3#include "JIT/opcodes/binops.h"
  4#include "JIT/opcodes/container.h"
  5#include "JIT/llvm_fbuilder.h"
  6
  7#include "llvm/BasicBlock.h"
  8#include "llvm/Function.h"
  9#include "llvm/Instructions.h"
 10#include "llvm/Support/ManagedStatic.h"
 11#include "llvm/Support/raw_ostream.h"
 12
 13using llvm::BasicBlock;
 14using llvm::ConstantInt;
 15using llvm::Function;
 16using llvm::Type;
 17using llvm::Value;
 18using llvm::errs;
 19
 20#ifdef Py_WITH_INSTRUMENTATION
 21class ImportNameStats {
 22public:
 23    ImportNameStats()
 24        : total(0), optimized(0) {
 25    }
 26
 27    ~ImportNameStats() {
 28        errs() << "\nIMPORT_NAME opcodes:\n";
 29        errs() << "Total: " << this->total << "\n";
 30        errs() << "Optimized: " << this->optimized << "\n";
 31    }
 32
 33    // Total number of IMPORT_NAME opcodes compiled.
 34    unsigned total;
 35    // Number of imports successfully optimized.
 36    unsigned optimized;
 37};
 38
 39static llvm::ManagedStatic<ImportNameStats> import_name_stats;
 40
 41#define IMPORT_NAME_INC_STATS(field) import_name_stats->field++
 42#else
 43#define IMPORT_NAME_INC_STATS(field)
 44#endif  /* Py_WITH_INSTRUMENTATION */
 45
 46namespace py {
 47
 48OpcodeContainer::OpcodeContainer(LlvmFunctionBuilder *fbuilder) :
 49    fbuilder_(fbuilder),
 50    state_(fbuilder->state()),
 51    builder_(fbuilder->builder())
 52{
 53}
 54
 55void
 56OpcodeContainer::BuildSequenceLiteral(
 57    int size, const char *createname,
 58    Value *(LlvmFunctionState::*getitemslot)(Value*, int))
 59{
 60    const Type *IntSsizeTy =
 61        PyTypeBuilder<Py_ssize_t>::get(this->fbuilder_->context());
 62    Value *seqsize = ConstantInt::getSigned(IntSsizeTy, size);
 63
 64    Function *create =
 65        this->state_->GetGlobalFunction<PyObject *(Py_ssize_t)>(createname);
 66    Value *seq = this->state_->CreateCall(create, seqsize, "sequence_literal");
 67    this->fbuilder_->PropagateExceptionOnNull(seq);
 68
 69    // XXX(twouters): do this with a memcpy?
 70    while (--size >= 0) {
 71        Value *itemslot = (this->state_->*getitemslot)(seq, size);
 72        this->builder_.CreateStore(this->fbuilder_->Pop(), itemslot);
 73    }
 74    this->fbuilder_->Push(seq);
 75}
 76
 77void
 78OpcodeContainer::BUILD_LIST(int size)
 79{
 80   this->BuildSequenceLiteral(size, "PyList_New",
 81                              &LlvmFunctionState::GetListItemSlot);
 82}
 83
 84void
 85OpcodeContainer::BUILD_TUPLE(int size)
 86{
 87   this->BuildSequenceLiteral(size, "PyTuple_New",
 88                              &LlvmFunctionState::GetTupleItemSlot);
 89}
 90
 91void
 92OpcodeContainer::BUILD_MAP(int size)
 93{
 94    Value *sizehint = ConstantInt::getSigned(
 95        PyTypeBuilder<Py_ssize_t>::get(this->fbuilder_->context()), size);
 96    Function *create_dict = this->state_->GetGlobalFunction<
 97        PyObject *(Py_ssize_t)>("_PyDict_NewPresized");
 98    Value *result = this->state_->CreateCall(create_dict, sizehint,
 99                                             "BULD_MAP_result");
100    this->fbuilder_->PropagateExceptionOnNull(result);
101    this->fbuilder_->Push(result);
102}
103
104void
105OpcodeContainer::UNPACK_SEQUENCE(int size)
106{
107    // TODO(twouters): We could do even better by combining this opcode and the
108    // STORE_* ones that follow into a single block of code circumventing the
109    // stack altogether. And omitting the horrible external stack munging that
110    // UnpackIterable does.
111    Value *iterable = this->fbuilder_->Pop();
112    Function *unpack_iterable = this->state_->GetGlobalFunction<
113        int(PyObject *, int, PyObject **)>("_PyLlvm_FastUnpackIterable");
114    Value *new_stack_pointer = this->builder_.CreateGEP(
115        this->builder_.CreateLoad(this->fbuilder_->stack_pointer_addr()),
116        ConstantInt::getSigned(
117            PyTypeBuilder<Py_ssize_t>::get(this->fbuilder_->context()), size));
118    this->fbuilder_->llvm_data()->tbaa_stack.MarkInstruction(new_stack_pointer);
119
120    Value *result = this->state_->CreateCall(
121        unpack_iterable, iterable,
122        ConstantInt::get(
123            PyTypeBuilder<int>::get(this->fbuilder_->context()), size, true),
124        // _PyLlvm_FastUnpackIterable really takes the *new* stack pointer as
125        // an argument, because it builds the result stack in reverse.
126        new_stack_pointer);
127    this->state_->DecRef(iterable);
128    this->fbuilder_->PropagateExceptionOnNonZero(result);
129    // Not setting the new stackpointer on failure does mean that if
130    // _PyLlvm_FastUnpackIterable failed after pushing some values onto the
131    // stack, and it didn't clean up after itself, we lose references.  This
132    // is what eval.cc does as well.
133    this->builder_.CreateStore(new_stack_pointer,
134                               this->fbuilder_->stack_pointer_addr());
135}
136
137#define INT_OBJ_OBJ_OBJ int(PyObject*, PyObject*, PyObject*)
138
139void
140OpcodeContainer::STORE_SUBSCR_list_int()
141{
142    BasicBlock *success =
143        this->state_->CreateBasicBlock("STORE_SUBSCR_success");
144    BasicBlock *bailpoint =
145        this->state_->CreateBasicBlock("STORE_SUBSCR_bail");
146
147    Value *key = this->fbuilder_->Pop();
148    Value *obj = this->fbuilder_->Pop();
149    Value *value = this->fbuilder_->Pop();
150    Function *setitem =
151        this->state_->GetGlobalFunction<INT_OBJ_OBJ_OBJ>(
152            "_PyLlvm_StoreSubscr_List");
153
154    Value *result = this->state_->CreateCall(setitem, obj, key, value,
155                                             "STORE_SUBSCR_result");
156    this->builder_.CreateCondBr(this->state_->IsNonZero(result),
157                                bailpoint, success);
158
159    this->builder_.SetInsertPoint(bailpoint);
160    this->fbuilder_->Push(value);
161    this->fbuilder_->Push(obj);
162    this->fbuilder_->Push(key);
163    this->fbuilder_->CreateGuardBailPoint(_PYGUARD_STORE_SUBSCR);
164
165    this->builder_.SetInsertPoint(success);
166    this->state_->DecRef(value);
167    this->state_->DecRef(obj);
168    this->state_->DecRef(key);
169}
170
171void
172OpcodeContainer::STORE_SUBSCR_safe()
173{
174    // Performing obj[key] = val
175    Value *key = this->fbuilder_->Pop();
176    Value *obj = this->fbuilder_->Pop();
177    Value *value = this->fbuilder_->Pop();
178    Function *setitem =
179        this->state_->GetGlobalFunction<INT_OBJ_OBJ_OBJ>("PyObject_SetItem");
180    Value *result = this->state_->CreateCall(setitem, obj, key, value,
181                                             "STORE_SUBSCR_result");
182    this->state_->DecRef(value);
183    this->state_->DecRef(obj);
184    this->state_->DecRef(key);
185    this->fbuilder_->PropagateExceptionOnNonZero(result);
186}
187
188#undef INT_OBJ_OBJ_OBJ
189
190void
191OpcodeContainer::STORE_SUBSCR()
192{
193    OpcodeBinops::IncStatsTotal();
194
195    const PyTypeObject *lhs_type = this->fbuilder_->GetTypeFeedback(0);
196    const PyTypeObject *rhs_type = this->fbuilder_->GetTypeFeedback(1);
197
198    if (lhs_type == &PyList_Type && rhs_type == &PyInt_Type) {
199        OpcodeBinops::IncStatsOptimized();
200        this->STORE_SUBSCR_list_int();
201        return;
202    }
203    else {
204        OpcodeBinops::IncStatsOmitted();
205        this->STORE_SUBSCR_safe();
206        return;
207    }
208}
209
210void
211OpcodeContainer::DELETE_SUBSCR()
212{
213    Value *key = this->fbuilder_->Pop();
214    Value *obj = this->fbuilder_->Pop();
215    Function *delitem = this->state_->GetGlobalFunction<
216          int(PyObject *, PyObject *)>("PyObject_DelItem");
217    Value *result = this->state_->CreateCall(delitem, obj, key,
218                                             "DELETE_SUBSCR_result");
219    this->state_->DecRef(obj);
220    this->state_->DecRef(key);
221    this->fbuilder_->PropagateExceptionOnNonZero(result);
222}
223
224void
225OpcodeContainer::LIST_APPEND()
226{
227    Value *item = this->fbuilder_->Pop();
228    Value *listobj = this->fbuilder_->Pop();
229    Function *list_append = this->state_->GetGlobalFunction<
230        int(PyObject *, PyObject *)>("PyList_Append");
231    Value *result = this->state_->CreateCall(list_append, listobj, item,
232                                             "LIST_APPEND_result");
233    this->state_->DecRef(listobj);
234    this->state_->DecRef(item);
235    this->fbuilder_->PropagateExceptionOnNonZero(result);
236}
237
238void
239OpcodeContainer::STORE_MAP()
240{
241    Value *key = this->fbuilder_->Pop();
242    Value *value = this->fbuilder_->Pop();
243    Value *dict = this->fbuilder_->Pop();
244    this->fbuilder_->Push(dict);
245    Value *dict_type = this->builder_.CreateLoad(
246        ObjectTy::ob_type(this->builder_, dict));
247    Value *is_exact_dict = this->builder_.CreateICmpEQ(
248        dict_type, this->state_->GetGlobalVariableFor((PyObject*)&PyDict_Type));
249    this->state_->Assert(is_exact_dict,
250                         "dict argument to STORE_MAP is not exactly a PyDict");
251    Function *setitem = this->state_->GetGlobalFunction<
252        int(PyObject *, PyObject *, PyObject *)>("PyDict_SetItem");
253    Value *result = this->state_->CreateCall(setitem, dict, key, value,
254                                             "STORE_MAP_result");
255    this->state_->DecRef(value);
256    this->state_->DecRef(key);
257    this->fbuilder_->PropagateExceptionOnNonZero(result);
258}
259
260#define FUNC_TYPE PyObject *(PyObject *, PyObject *, PyObject *)
261
262void
263OpcodeContainer::IMPORT_NAME()
264{
265    IMPORT_NAME_INC_STATS(total);
266
267    if (this->IMPORT_NAME_fast())
268        return;
269
270    Value *mod_name = this->fbuilder_->Pop();
271    Value *names = this->fbuilder_->Pop();
272    Value *level = this->fbuilder_->Pop();
273
274    Value *module = this->state_->CreateCall(
275        this->state_->GetGlobalFunction<FUNC_TYPE>("_PyEval_ImportName"),
276        level, names, mod_name);
277    this->state_->DecRef(level);
278    this->state_->DecRef(names);
279    this->state_->DecRef(mod_name);
280    this->fbuilder_->PropagateExceptionOnNull(module);
281    this->fbuilder_->Push(module);
282}
283
284#undef FUNC_TYPE
285
286bool
287OpcodeContainer::IMPORT_NAME_fast()
288{
289    PyCodeObject *code = fbuilder_->code_object();
290
291    // If we're not already monitoring the builtins dict, monitor it.  Normally
292    // we pick it up from the eval loop, but if it isn't here, then we make a
293    // guess.  If we are wrong, we will bail.
294    if (code->co_watching == NULL ||
295        code->co_watching[WATCHING_BUILTINS] == NULL) {
296        PyObject *builtins = PyThreadState_GET()->interp->builtins;
297        _PyCode_WatchDict(code, WATCHING_BUILTINS, builtins);
298    }
299
300    const PyRuntimeFeedback *feedback = this->fbuilder_->GetFeedback();
301    if (feedback == NULL || feedback->ObjectsOverflowed()) {
302        return false;
303    }
304
305    llvm::SmallVector<PyObject *, 3> objects;
306    feedback->GetSeenObjectsInto(objects);
307    if (objects.size() != 1 || !PyModule_Check(objects[0])) {
308        return false;
309    }
310    PyObject *module = objects[0];
311
312    // We need to invalidate this function if someone changes sys.modules.
313    if (code->co_watching[WATCHING_SYS_MODULES] == NULL) {
314        PyObject *sys_modules = PyImport_GetModuleDict();
315        if (sys_modules == NULL) {
316            return false;
317        }
318
319        if (_PyCode_WatchDict(code,
320                              WATCHING_SYS_MODULES,
321                              sys_modules)) {
322            PyErr_Clear();
323            return false;
324        }
325
326        fbuilder_->WatchDict(WATCHING_BUILTINS);
327        fbuilder_->WatchDict(WATCHING_SYS_MODULES);
328    }
329
330    BasicBlock *keep_going =
331        this->state_->CreateBasicBlock("IMPORT_NAME_keep_going");
332    BasicBlock *invalid_assumptions =
333        this->state_->CreateBasicBlock("IMPORT_NAME_invalid_assumptions");
334
335    this->builder_.CreateCondBr(this->fbuilder_->GetUseJitCond(),
336                                keep_going,
337                                invalid_assumptions);
338
339    /* Our assumptions about the state of sys.modules no longer hold;
340       bail back to the interpreter. */
341    this->builder_.SetInsertPoint(invalid_assumptions);
342    this->fbuilder_->CreateBailPoint(_PYFRAME_FATAL_GUARD_FAIL);
343
344    this->builder_.SetInsertPoint(keep_going);
345    /* TODO(collinwinter): we pop to get rid of the inputs to IMPORT_NAME.
346       Find a way to omit this work. */
347    this->state_->DecRef(this->fbuilder_->Pop());
348    this->state_->DecRef(this->fbuilder_->Pop());
349    this->state_->DecRef(this->fbuilder_->Pop());
350
351    Value *mod = this->state_->GetGlobalVariableFor(module);
352    this->state_->IncRef(mod);
353    this->fbuilder_->Push(mod);
354
355    IMPORT_NAME_INC_STATS(optimized);
356    return true;
357}
358
359}