/JIT/RuntimeFeedback.cc

  1#include "Python.h"
  2#include "JIT/RuntimeFeedback.h"
  3
  4#include "llvm/ADT/PointerIntPair.h"
  5#include "llvm/ADT/STLExtras.h"
  6#include "llvm/ADT/SmallPtrSet.h"
  7#include "llvm/ADT/SmallVector.h"
  8
  9#include <algorithm>
 10
 11using llvm::PointerIntPair;
 12using llvm::PointerLikeTypeTraits;
 13using llvm::SmallPtrSet;
 14using llvm::SmallVector;
 15
 16
 17PyLimitedFeedback::PyLimitedFeedback()
 18{
 19}
 20
 21PyLimitedFeedback::PyLimitedFeedback(const PyLimitedFeedback &src)
 22{
 23    for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
 24        if (src.InObjectMode()) {
 25            PyObject *value = (PyObject *)src.data_[i].getPointer();
 26            Py_XINCREF(value);
 27            this->data_[i] = src.data_[i];
 28        }
 29        else {
 30            this->data_[i] = src.data_[i];
 31        }
 32    }
 33}
 34
 35PyLimitedFeedback::~PyLimitedFeedback()
 36{
 37    this->Clear();
 38}
 39
 40PyLimitedFeedback &
 41PyLimitedFeedback::operator=(PyLimitedFeedback rhs)
 42{
 43    this->Swap(&rhs);
 44    return *this;
 45}
 46
 47void
 48PyLimitedFeedback::Swap(PyLimitedFeedback *other)
 49{
 50    for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
 51        std::swap(this->data_[i], other->data_[i]);
 52    }
 53}
 54
 55void
 56PyLimitedFeedback::SetFlagBit(unsigned index, bool value)
 57{
 58    assert(index < 6);
 59    PointerIntPair<void*, 2>& slot = this->data_[index / 2];
 60    unsigned mask = 1 << (index % 2);
 61    unsigned old_value = slot.getInt();
 62    unsigned new_value = (old_value & ~mask) | (value << (index % 2));
 63    slot.setInt(new_value);
 64}
 65
 66bool
 67PyLimitedFeedback::GetFlagBit(unsigned index) const
 68{
 69    assert(index < 6);
 70    const PointerIntPair<void*, 2>& slot = this->data_[index / 2];
 71    unsigned value = slot.getInt();
 72    return (value >> (index % 2)) & 1;
 73}
 74
 75void
 76PyLimitedFeedback::IncCounter(unsigned counter_id)
 77{
 78    assert(this->InCounterMode());
 79    assert(counter_id < (unsigned)PyLimitedFeedback::NUM_POINTERS);
 80    this->SetFlagBit(COUNTER_MODE_BIT, true);
 81
 82    uintptr_t old_value =
 83        reinterpret_cast<uintptr_t>(this->data_[counter_id].getPointer());
 84    uintptr_t shift = PointerLikeTypeTraits<PyObject*>::NumLowBitsAvailable;
 85    uintptr_t new_value = old_value + (1U << shift);
 86    if (new_value > old_value) {
 87        // Only increment if we're not saturated yet.
 88        this->data_[counter_id].setPointer(
 89            reinterpret_cast<void*>(new_value));
 90    }
 91}
 92
 93uintptr_t
 94PyLimitedFeedback::GetCounter(unsigned counter_id) const
 95{
 96    assert(this->InCounterMode());
 97
 98    uintptr_t shift = PointerLikeTypeTraits<PyObject*>::NumLowBitsAvailable;
 99    void *counter_as_pointer = this->data_[counter_id].getPointer();
100    return reinterpret_cast<uintptr_t>(counter_as_pointer) >> shift;
101}
102
103void
104PyLimitedFeedback::Clear()
105{
106    bool object_mode = this->InObjectMode();
107
108    for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
109        if (object_mode) {
110            Py_XDECREF((PyObject *)this->data_[i].getPointer());
111        }
112        this->data_[i].setPointer(NULL);
113        this->data_[i].setInt(0);
114    }
115}
116
117void
118PyLimitedFeedback::AddObjectSeen(PyObject *obj)
119{
120    assert(this->InObjectMode());
121    this->SetFlagBit(OBJECT_MODE_BIT, true);
122
123    if (obj == NULL) {
124        SetFlagBit(SAW_A_NULL_OBJECT_BIT, true);
125        return;
126    }
127    for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
128        PyObject *value = (PyObject *)data_[i].getPointer();
129        if (value == obj)
130            return;
131        if (value == NULL) {
132            Py_INCREF(obj);
133            data_[i].setPointer((void *)obj);
134            return;
135        }
136    }
137    // Record overflow.
138    SetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT, true);
139}
140
141void
142PyLimitedFeedback::GetSeenObjectsInto(SmallVector<PyObject*, 3> &result) const
143{
144    assert(this->InObjectMode());
145
146    result.clear();
147    if (GetFlagBit(SAW_A_NULL_OBJECT_BIT)) {
148        // Saw a NULL value, so add NULL to the result.
149        result.push_back(NULL);
150    }
151    for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
152        PyObject *value = (PyObject *)data_[i].getPointer();
153        if (value == NULL)
154            return;
155        result.push_back(value);
156    }
157}
158
159void
160PyLimitedFeedback::AddFuncSeen(PyObject *obj)
161{
162    assert(this->InFuncMode());
163    this->SetFlagBit(FUNC_MODE_BIT, true);
164
165    if (this->GetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT))
166        return;
167    if (obj == NULL) {
168        this->SetFlagBit(SAW_A_NULL_OBJECT_BIT, true);
169        return;
170    }
171
172    // Record the type of the function, and the methoddef if it's a call to a C
173    // function.
174    PyTypeObject *type = Py_TYPE(obj);
175    PyMethodDef *ml = NULL;
176    if (PyCFunction_Check(obj)) {
177        ml = PyCFunction_GET_METHODDEF(obj);
178    } else if (PyMethodDescr_Check(obj)) {
179        ml = ((PyMethodDescrObject *)obj)->d_method;
180    }
181
182    PyTypeObject *old_type = (PyTypeObject *)this->data_[0].getPointer();
183    PyMethodDef *old_ml = (PyMethodDef *)this->data_[1].getPointer();
184    if (old_type == NULL) {
185        // Record this method.
186        Py_INCREF(type);
187        this->data_[0].setPointer((void*)type);
188        this->data_[1].setPointer((void*)ml);
189    } else if (old_type != type || old_ml != ml) {
190        // We found something else here already.  Set this flag to indicate the
191        // call site is polymorphic, even if we haven't seen more than three
192        // objects.
193        this->SetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT, true);
194    }
195    // The call site is monomorphic, so we leave it as is.
196}
197
198void
199PyLimitedFeedback::GetSeenFuncsInto(
200    SmallVector<PyTypeMethodPair, 3> &result) const
201{
202    assert(this->InFuncMode());
203
204    result.clear();
205    if (this->GetFlagBit(SAW_A_NULL_OBJECT_BIT)) {
206        // Saw a NULL value, so add NULL to the result.
207        result.push_back(
208            std::make_pair<PyTypeObject*, PyMethodDef*>(NULL, NULL));
209    }
210    PyTypeObject *type = (PyTypeObject *)this->data_[0].getPointer();
211    PyMethodDef *ml = (PyMethodDef *)this->data_[1].getPointer();
212    result.push_back(std::make_pair<PyTypeObject*, PyMethodDef*>(type, ml));
213}
214
215
216PyFullFeedback::PyFullFeedback()
217    : counters_(/* Zero out the array. */),
218      usage_(UnknownMode)
219{
220}
221
222PyFullFeedback::PyFullFeedback(const PyFullFeedback &src)
223{
224    this->usage_ = src.usage_;
225    for (unsigned i = 0; i < llvm::array_lengthof(this->counters_); ++i)
226        this->counters_[i] = src.counters_[i];
227    for (ObjSet::iterator it = src.data_.begin(), end = src.data_.end();
228            it != end; ++it) {
229        void *obj = *it;
230        if (src.usage_ == ObjectMode) {
231            Py_XINCREF((PyObject *)obj);
232        }
233        else if (src.InFuncMode()) {
234            PyTypeMethodPair &src_pair_ref = *(PyTypeMethodPair*)obj;
235            obj = new PyTypeMethodPair(src_pair_ref);
236        }
237        this->data_.insert(obj);
238    }
239}
240
241PyFullFeedback::~PyFullFeedback()
242{
243    if (this->InFuncMode()) {
244        // We have to free these pairs if we're in func mode.
245        for (ObjSet::const_iterator it = this->data_.begin(),
246                end = this->data_.end(); it != end; ++it) {
247            delete (PyTypeMethodPair*)*it;
248        }
249    }
250    this->Clear();
251}
252
253PyFullFeedback &
254PyFullFeedback::operator=(PyFullFeedback rhs)
255{
256    this->Swap(&rhs);
257    return *this;
258}
259
260void
261PyFullFeedback::Swap(PyFullFeedback *other)
262{
263    std::swap(this->usage_, other->usage_);
264    std::swap(this->data_, other->data_);
265    for (unsigned i = 0; i < llvm::array_lengthof(this->counters_); ++i)
266        std::swap(this->counters_[i], other->counters_[i]);
267}
268
269void
270PyFullFeedback::Clear()
271{
272    for (ObjSet::iterator it = this->data_.begin(),
273            end = this->data_.end(); it != end; ++it) {
274        if (this->usage_ == ObjectMode) {
275            Py_XDECREF((PyObject *)*it);
276        }
277    }
278    this->data_.clear();
279    for (unsigned i = 0; i < llvm::array_lengthof(this->counters_); ++i)
280        this->counters_[i] = 0;
281    this->usage_ = UnknownMode;
282}
283
284void
285PyFullFeedback::AddObjectSeen(PyObject *obj)
286{
287    assert(this->InObjectMode());
288    this->usage_ = ObjectMode;
289
290    if (obj == NULL) {
291        this->data_.insert(NULL);
292        return;
293    }
294
295    if (!this->data_.count(obj)) {
296        Py_INCREF(obj);
297        this->data_.insert((void *)obj);
298    }
299}
300
301void
302PyFullFeedback::GetSeenObjectsInto(
303    SmallVector<PyObject*, /*in-object elems=*/3> &result) const
304{
305    assert(this->InObjectMode());
306
307    result.clear();
308    for (ObjSet::const_iterator it = this->data_.begin(),
309             end = this->data_.end(); it != end; ++it) {
310        result.push_back((PyObject *)*it);
311    }
312}
313
314void
315PyFullFeedback::AddFuncSeen(PyObject *obj)
316{
317    assert(this->InFuncMode());
318    this->usage_ = FuncMode;
319
320    PyMethodDef *ml = NULL;
321    PyTypeObject *type = NULL;
322    if (obj != NULL) {
323        type = Py_TYPE(obj);
324
325        // We only record a methoddef if this is a C function.
326        if (PyCFunction_Check(obj)) {
327            ml = PyCFunction_GET_METHODDEF(obj);
328        } else if (PyMethodDescr_Check(obj)) {
329            ml = ((PyMethodDescrObject *)obj)->d_method;
330        }
331    }
332
333    for (ObjSet::const_iterator it = this->data_.begin(),
334            end = this->data_.end(); it != end; ++it) {
335        PyTypeMethodPair *pair = (PyTypeMethodPair*)*it;
336        if (pair->first == type && pair->second == ml)
337            return;
338    }
339
340    PyTypeMethodPair *pair = new PyTypeMethodPair(type, ml);
341    this->data_.insert((void *)pair);
342}
343
344void
345PyFullFeedback::GetSeenFuncsInto(
346    SmallVector<PyTypeMethodPair, 3> &result) const
347{
348    assert(this->InFuncMode());
349
350    result.clear();
351    for (ObjSet::const_iterator it = this->data_.begin(),
352            end = this->data_.end(); it != end; ++it) {
353        result.push_back(*((PyTypeMethodPair *)*it));
354    }
355}
356
357void
358PyFullFeedback::IncCounter(unsigned counter_id)
359{
360    assert(this->InCounterMode());
361    assert(counter_id < llvm::array_lengthof(this->counters_));
362    this->usage_ = CounterMode;
363
364    uintptr_t old_value = this->counters_[counter_id];
365    uintptr_t new_value = old_value + 1;
366    if (new_value > old_value) {
367        // Only increment if we're not saturated yet.
368        this->counters_[counter_id] = new_value;
369    }
370}
371
372uintptr_t
373PyFullFeedback::GetCounter(unsigned counter_id) const
374{
375    assert(this->InCounterMode());
376
377    return this->counters_[counter_id];
378}
379
380PyFeedbackMap *
381PyFeedbackMap_New()
382{
383    return new PyFeedbackMap;
384}
385
386void
387PyFeedbackMap_Del(PyFeedbackMap *map)
388{
389    delete map;
390}
391
392void
393PyFeedbackMap_Clear(PyFeedbackMap *map)
394{
395    map->Clear();
396}
397
398const PyRuntimeFeedback *
399PyFeedbackMap::GetFeedbackEntry(unsigned opcode_index, unsigned arg_index) const
400{
401    llvm::DenseMap<std::pair<unsigned, unsigned>,
402                   PyRuntimeFeedback>::const_iterator result =
403        this->entries_.find(std::make_pair(opcode_index, arg_index));
404    if (result == this->entries_.end())
405        return NULL;
406    return &result->second;
407}
408
409PyRuntimeFeedback &
410PyFeedbackMap::GetOrCreateFeedbackEntry(
411    unsigned opcode_index, unsigned arg_index)
412{
413    return this->entries_[std::make_pair(opcode_index, arg_index)];
414}
415
416void
417PyFeedbackMap::Clear()
418{
419    for (FeedbackMap::iterator it = this->entries_.begin(),
420            end = this->entries_.end(); it != end; ++it) {
421        it->second.Clear();
422    }
423}