PageRenderTime 42ms CodeModel.GetById 17ms RepoModel.GetById 0ms app.codeStats 0ms

/pypy/rpython/lltypesystem/rlist.py

http://github.com/pypy/pypy
Python | 425 lines | 303 code | 64 blank | 58 comment | 47 complexity | fa76443c841dbd54c13f8530473c22ba MD5 | raw file
  1. from pypy.rlib import rgc, jit
  2. from pypy.rlib.debug import ll_assert
  3. from pypy.rlib.objectmodel import enforceargs
  4. from pypy.rpython.lltypesystem import rstr
  5. from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
  6. GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
  7. from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
  8. AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
  9. ADTIList, ADTIFixedList, dum_nocheck)
  10. from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
  11. from pypy.tool.pairtype import pairtype, pair
  12. # ____________________________________________________________
  13. #
  14. # Concrete implementation of RPython lists:
  15. #
  16. # struct list {
  17. # int length;
  18. # items_array *items;
  19. # }
  20. #
  21. # 'items' points to a C-like array in memory preceded by a 'length' header,
  22. # where each item contains a primitive value or pointer to the actual list
  23. # item.
  24. #
  25. # or for fixed-size lists an array is directly used:
  26. #
  27. # item_t list_items[]
  28. #
  29. class BaseListRepr(AbstractBaseListRepr):
  30. rstr_ll = rstr.LLHelpers
  31. # known_maxlength is ignored by lltype but used by ootype
  32. def __init__(self, rtyper, item_repr, listitem=None, known_maxlength=False):
  33. self.rtyper = rtyper
  34. self.LIST = GcForwardReference()
  35. self.lowleveltype = Ptr(self.LIST)
  36. if not isinstance(item_repr, Repr): # not computed yet, done by setup()
  37. assert callable(item_repr)
  38. self._item_repr_computer = item_repr
  39. else:
  40. self.external_item_repr, self.item_repr = externalvsinternal(rtyper, item_repr)
  41. self.listitem = listitem
  42. self.list_cache = {}
  43. # setup() needs to be called to finish this initialization
  44. def null_const(self):
  45. return nullptr(self.LIST)
  46. def get_eqfunc(self):
  47. return inputconst(Void, self.item_repr.get_ll_eq_function())
  48. def make_iterator_repr(self):
  49. return ListIteratorRepr(self)
  50. def get_itemarray_lowleveltype(self):
  51. ITEM = self.item_repr.lowleveltype
  52. ITEMARRAY = GcArray(ITEM,
  53. adtmeths = ADTIFixedList({
  54. "ll_newlist": ll_fixed_newlist,
  55. "ll_newemptylist": ll_fixed_newemptylist,
  56. "ll_length": ll_fixed_length,
  57. "ll_items": ll_fixed_items,
  58. "ITEM": ITEM,
  59. "ll_getitem_fast": ll_fixed_getitem_fast,
  60. "ll_setitem_fast": ll_fixed_setitem_fast,
  61. }))
  62. return ITEMARRAY
  63. class __extend__(pairtype(BaseListRepr, BaseListRepr)):
  64. def rtype_is_((r_lst1, r_lst2), hop):
  65. if r_lst1.lowleveltype != r_lst2.lowleveltype:
  66. # obscure logic, the is can be true only if both are None
  67. v_lst1, v_lst2 = hop.inputargs(r_lst1, r_lst2)
  68. return hop.gendirectcall(ll_both_none, v_lst1, v_lst2)
  69. return pairtype(Repr, Repr).rtype_is_(pair(r_lst1, r_lst2), hop)
  70. class ListRepr(AbstractListRepr, BaseListRepr):
  71. def _setup_repr(self):
  72. if 'item_repr' not in self.__dict__:
  73. self.external_item_repr, self.item_repr = externalvsinternal(self.rtyper, self._item_repr_computer())
  74. if isinstance(self.LIST, GcForwardReference):
  75. ITEM = self.item_repr.lowleveltype
  76. ITEMARRAY = self.get_itemarray_lowleveltype()
  77. # XXX we might think of turning length stuff into Unsigned
  78. self.LIST.become(GcStruct("list", ("length", Signed),
  79. ("items", Ptr(ITEMARRAY)),
  80. adtmeths = ADTIList({
  81. "ll_newlist": ll_newlist,
  82. "ll_newlist_hint": ll_newlist_hint,
  83. "ll_newemptylist": ll_newemptylist,
  84. "ll_length": ll_length,
  85. "ll_items": ll_items,
  86. "ITEM": ITEM,
  87. "ll_getitem_fast": ll_getitem_fast,
  88. "ll_setitem_fast": ll_setitem_fast,
  89. "_ll_resize_ge": _ll_list_resize_ge,
  90. "_ll_resize_le": _ll_list_resize_le,
  91. "_ll_resize": _ll_list_resize,
  92. }),
  93. hints = {'list': True})
  94. )
  95. def compact_repr(self):
  96. return 'ListR %s' % (self.item_repr.compact_repr(),)
  97. def prepare_const(self, n):
  98. result = malloc(self.LIST, immortal=True)
  99. result.length = n
  100. result.items = malloc(self.LIST.items.TO, n)
  101. return result
  102. def rtype_method_append(self, hop):
  103. if getattr(self.listitem, 'hint_maxlength', False):
  104. v_lst, v_value = hop.inputargs(self, self.item_repr)
  105. hop.exception_cannot_occur()
  106. hop.gendirectcall(ll_append_noresize, v_lst, v_value)
  107. else:
  108. AbstractListRepr.rtype_method_append(self, hop)
  109. def rtype_hint(self, hop):
  110. optimized = getattr(self.listitem, 'hint_maxlength', False)
  111. hints = hop.args_s[-1].const
  112. if 'maxlength' in hints:
  113. if optimized:
  114. v_list = hop.inputarg(self, arg=0)
  115. v_maxlength = self._get_v_maxlength(hop)
  116. hop.llops.gendirectcall(ll_set_maxlength, v_list, v_maxlength)
  117. return v_list
  118. if 'fence' in hints:
  119. v_list = hop.inputarg(self, arg=0)
  120. if isinstance(hop.r_result, FixedSizeListRepr):
  121. if optimized and 'exactlength' in hints:
  122. llfn = ll_list2fixed_exact
  123. else:
  124. llfn = ll_list2fixed
  125. v_list = hop.llops.gendirectcall(llfn, v_list)
  126. return v_list
  127. return AbstractListRepr.rtype_hint(self, hop)
  128. class FixedSizeListRepr(AbstractFixedSizeListRepr, BaseListRepr):
  129. def _setup_repr(self):
  130. if 'item_repr' not in self.__dict__:
  131. self.external_item_repr, self.item_repr = externalvsinternal(self.rtyper, self._item_repr_computer())
  132. if isinstance(self.LIST, GcForwardReference):
  133. ITEMARRAY = self.get_itemarray_lowleveltype()
  134. self.LIST.become(ITEMARRAY)
  135. def compact_repr(self):
  136. return 'FixedSizeListR %s' % (self.item_repr.compact_repr(),)
  137. def prepare_const(self, n):
  138. result = malloc(self.LIST, n, immortal=True)
  139. return result
  140. # ____________________________________________________________
  141. #
  142. # Low-level methods. These can be run for testing, but are meant to
  143. # be direct_call'ed from rtyped flow graphs, which means that they will
  144. # get flowed and annotated, mostly with SomePtr.
  145. # adapted C code
  146. @enforceargs(None, int)
  147. def _ll_list_resize_really(l, newsize):
  148. """
  149. Ensure l.items has room for at least newsize elements, and set
  150. l.length to newsize. Note that l.items may change, and even if
  151. newsize is less than l.length on entry.
  152. """
  153. # This over-allocates proportional to the list size, making room
  154. # for additional growth. The over-allocation is mild, but is
  155. # enough to give linear-time amortized behavior over a long
  156. # sequence of appends() in the presence of a poorly-performing
  157. # system malloc().
  158. # The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
  159. if newsize <= 0:
  160. ll_assert(newsize == 0, "negative list length")
  161. l.length = 0
  162. l.items = _ll_new_empty_item_array(typeOf(l).TO)
  163. return
  164. else:
  165. if newsize < 9:
  166. some = 3
  167. else:
  168. some = 6
  169. some += newsize >> 3
  170. new_allocated = newsize + some
  171. # new_allocated is a bit more than newsize, enough to ensure an amortized
  172. # linear complexity for e.g. repeated usage of l.append(). In case
  173. # it overflows sys.maxint, it is guaranteed negative, and the following
  174. # malloc() will fail.
  175. items = l.items
  176. newitems = malloc(typeOf(l).TO.items.TO, new_allocated)
  177. before_len = l.length
  178. if before_len: # avoids copying GC flags from the prebuilt_empty_array
  179. if before_len < newsize:
  180. p = before_len
  181. else:
  182. p = newsize
  183. rgc.ll_arraycopy(items, newitems, 0, 0, p)
  184. l.length = newsize
  185. l.items = newitems
  186. # this common case was factored out of _ll_list_resize
  187. # to see if inlining it gives some speed-up.
  188. def _ll_list_resize(l, newsize):
  189. # Bypass realloc() when a previous overallocation is large enough
  190. # to accommodate the newsize. If the newsize falls lower than half
  191. # the allocated size, then proceed with the realloc() to shrink the list.
  192. allocated = len(l.items)
  193. if allocated >= newsize and newsize >= ((allocated >> 1) - 5):
  194. l.length = newsize
  195. else:
  196. _ll_list_resize_really(l, newsize)
  197. @jit.look_inside_iff(lambda l, newsize: jit.isconstant(len(l.items)) and jit.isconstant(newsize))
  198. @jit.oopspec("list._resize_ge(l, newsize)")
  199. def _ll_list_resize_ge(l, newsize):
  200. if len(l.items) >= newsize:
  201. l.length = newsize
  202. else:
  203. _ll_list_resize_really(l, newsize)
  204. @jit.look_inside_iff(lambda l, newsize: jit.isconstant(len(l.items)) and jit.isconstant(newsize))
  205. @jit.oopspec("list._resize_le(l, newsize)")
  206. def _ll_list_resize_le(l, newsize):
  207. if newsize >= (len(l.items) >> 1) - 5:
  208. l.length = newsize
  209. else:
  210. _ll_list_resize_really(l, newsize)
  211. def ll_append_noresize(l, newitem):
  212. length = l.length
  213. l.length = length + 1
  214. l.ll_setitem_fast(length, newitem)
  215. def ll_both_none(lst1, lst2):
  216. return not lst1 and not lst2
  217. # ____________________________________________________________
  218. #
  219. # Accessor methods
  220. def ll_newlist(LIST, length):
  221. ll_assert(length >= 0, "negative list length")
  222. l = malloc(LIST)
  223. l.length = length
  224. l.items = malloc(LIST.items.TO, length)
  225. return l
  226. ll_newlist = typeMethod(ll_newlist)
  227. ll_newlist.oopspec = 'newlist(length)'
  228. def ll_newlist_hint(LIST, lengthhint):
  229. ll_assert(lengthhint >= 0, "negative list length")
  230. l = malloc(LIST)
  231. l.length = 0
  232. l.items = malloc(LIST.items.TO, lengthhint)
  233. return l
  234. ll_newlist_hint = typeMethod(ll_newlist_hint)
  235. ll_newlist_hint.oopspec = 'newlist_hint(lengthhint)'
  236. # should empty lists start with no allocated memory, or with a preallocated
  237. # minimal number of entries? XXX compare memory usage versus speed, and
  238. # check how many always-empty lists there are in a typical pypy-c run...
  239. INITIAL_EMPTY_LIST_ALLOCATION = 0
  240. def _ll_prebuilt_empty_array(LISTITEM):
  241. return malloc(LISTITEM, 0)
  242. _ll_prebuilt_empty_array._annspecialcase_ = 'specialize:memo'
  243. def _ll_new_empty_item_array(LIST):
  244. if INITIAL_EMPTY_LIST_ALLOCATION > 0:
  245. return malloc(LIST.items.TO, INITIAL_EMPTY_LIST_ALLOCATION)
  246. else:
  247. return _ll_prebuilt_empty_array(LIST.items.TO)
  248. def ll_newemptylist(LIST):
  249. l = malloc(LIST)
  250. l.length = 0
  251. l.items = _ll_new_empty_item_array(LIST)
  252. return l
  253. ll_newemptylist = typeMethod(ll_newemptylist)
  254. ll_newemptylist.oopspec = 'newlist(0)'
  255. def ll_length(l):
  256. return l.length
  257. ll_length.oopspec = 'list.len(l)'
  258. def ll_items(l):
  259. return l.items
  260. def ll_getitem_fast(l, index):
  261. ll_assert(index < l.length, "getitem out of bounds")
  262. return l.ll_items()[index]
  263. ll_getitem_fast.oopspec = 'list.getitem(l, index)'
  264. def ll_setitem_fast(l, index, item):
  265. ll_assert(index < l.length, "setitem out of bounds")
  266. l.ll_items()[index] = item
  267. ll_setitem_fast.oopspec = 'list.setitem(l, index, item)'
  268. # fixed size versions
  269. @typeMethod
  270. def ll_fixed_newlist(LIST, length):
  271. ll_assert(length >= 0, "negative fixed list length")
  272. l = malloc(LIST, length)
  273. return l
  274. ll_fixed_newlist.oopspec = 'newlist(length)'
  275. @typeMethod
  276. def ll_fixed_newemptylist(LIST):
  277. return ll_fixed_newlist(LIST, 0)
  278. def ll_fixed_length(l):
  279. return len(l)
  280. ll_fixed_length.oopspec = 'list.len(l)'
  281. def ll_fixed_items(l):
  282. return l
  283. def ll_fixed_getitem_fast(l, index):
  284. ll_assert(index < len(l), "fixed getitem out of bounds")
  285. return l[index]
  286. ll_fixed_getitem_fast.oopspec = 'list.getitem(l, index)'
  287. def ll_fixed_setitem_fast(l, index, item):
  288. ll_assert(index < len(l), "fixed setitem out of bounds")
  289. l[index] = item
  290. ll_fixed_setitem_fast.oopspec = 'list.setitem(l, index, item)'
  291. def newlist(llops, r_list, items_v, v_sizehint=None):
  292. LIST = r_list.LIST
  293. if len(items_v) == 0:
  294. if v_sizehint is None:
  295. v_result = llops.gendirectcall(LIST.ll_newemptylist)
  296. else:
  297. v_result = llops.gendirectcall(LIST.ll_newlist_hint, v_sizehint)
  298. else:
  299. assert v_sizehint is None
  300. cno = inputconst(Signed, len(items_v))
  301. v_result = llops.gendirectcall(LIST.ll_newlist, cno)
  302. v_func = inputconst(Void, dum_nocheck)
  303. for i, v_item in enumerate(items_v):
  304. ci = inputconst(Signed, i)
  305. llops.gendirectcall(ll_setitem_nonneg, v_func, v_result, ci, v_item)
  306. return v_result
  307. # special operations for list comprehension optimization
  308. def ll_set_maxlength(l, n):
  309. LIST = typeOf(l).TO
  310. l.items = malloc(LIST.items.TO, n)
  311. def ll_list2fixed(l):
  312. n = l.length
  313. olditems = l.items
  314. if n == len(olditems):
  315. return olditems
  316. else:
  317. LIST = typeOf(l).TO
  318. newitems = malloc(LIST.items.TO, n)
  319. rgc.ll_arraycopy(olditems, newitems, 0, 0, n)
  320. return newitems
  321. def ll_list2fixed_exact(l):
  322. ll_assert(l.length == len(l.items), "ll_list2fixed_exact: bad length")
  323. return l.items
  324. # ____________________________________________________________
  325. #
  326. # Iteration.
  327. class ListIteratorRepr(AbstractListIteratorRepr):
  328. def __init__(self, r_list):
  329. self.r_list = r_list
  330. self.lowleveltype = Ptr(GcStruct('listiter',
  331. ('list', r_list.lowleveltype),
  332. ('index', Signed)))
  333. self.ll_listiter = ll_listiter
  334. if (isinstance(r_list, FixedSizeListRepr)
  335. and not r_list.listitem.mutated):
  336. self.ll_listnext = ll_listnext_foldable
  337. else:
  338. self.ll_listnext = ll_listnext
  339. self.ll_getnextindex = ll_getnextindex
  340. def ll_listiter(ITERPTR, lst):
  341. iter = malloc(ITERPTR.TO)
  342. iter.list = lst
  343. iter.index = 0
  344. return iter
  345. def ll_listnext(iter):
  346. l = iter.list
  347. index = iter.index
  348. if index >= l.ll_length():
  349. raise StopIteration
  350. iter.index = index + 1 # cannot overflow because index < l.length
  351. return l.ll_getitem_fast(index)
  352. def ll_listnext_foldable(iter):
  353. from pypy.rpython.rlist import ll_getitem_foldable_nonneg
  354. l = iter.list
  355. index = iter.index
  356. if index >= l.ll_length():
  357. raise StopIteration
  358. iter.index = index + 1 # cannot overflow because index < l.length
  359. return ll_getitem_foldable_nonneg(l, index)
  360. def ll_getnextindex(iter):
  361. return iter.index