/pypy/module/cpyext/unicodeobject.py

https://bitbucket.org/dac_io/pypy · Python · 643 lines · 497 code · 59 blank · 87 comment · 42 complexity · c8b5d477da8f586d450c94fe88f68397 MD5 · raw file

  1. from pypy.interpreter.error import OperationError
  2. from pypy.rpython.lltypesystem import rffi, lltype
  3. from pypy.rpython.lltypesystem import llmemory
  4. from pypy.module.unicodedata import unicodedb
  5. from pypy.module.cpyext.api import (
  6. CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
  7. bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
  8. CONST_WSTRING)
  9. from pypy.module.cpyext.pyerrors import PyErr_BadArgument
  10. from pypy.module.cpyext.pyobject import (
  11. PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
  12. make_typedescr, get_typedescr)
  13. from pypy.module.cpyext.stringobject import PyString_Check
  14. from pypy.module.sys.interp_encoding import setdefaultencoding
  15. from pypy.objspace.std import unicodeobject, unicodetype, stringtype
  16. from pypy.rlib import runicode
  17. from pypy.tool.sourcetools import func_renamer
  18. import sys
  19. ## See comment in stringobject.py.
  20. PyUnicodeObjectStruct = lltype.ForwardReference()
  21. PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct)
  22. PyUnicodeObjectFields = (PyObjectFields +
  23. (("buffer", rffi.CWCHARP), ("size", Py_ssize_t)))
  24. cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct)
  25. @bootstrap_function
  26. def init_unicodeobject(space):
  27. make_typedescr(space.w_unicode.instancetypedef,
  28. basestruct=PyUnicodeObject.TO,
  29. attach=unicode_attach,
  30. dealloc=unicode_dealloc,
  31. realize=unicode_realize)
  32. # Buffer for the default encoding (used by PyUnicde_GetDefaultEncoding)
  33. DEFAULT_ENCODING_SIZE = 100
  34. default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE,
  35. flavor='raw', zero=True)
  36. PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
  37. Py_UNICODE = lltype.UniChar
  38. def new_empty_unicode(space, length):
  39. """
  40. Allocatse a PyUnicodeObject and its buffer, but without a corresponding
  41. interpreter object. The buffer may be mutated, until unicode_realize() is
  42. called.
  43. """
  44. typedescr = get_typedescr(space.w_unicode.instancetypedef)
  45. py_obj = typedescr.allocate(space, space.w_unicode)
  46. py_uni = rffi.cast(PyUnicodeObject, py_obj)
  47. buflen = length + 1
  48. py_uni.c_size = length
  49. py_uni.c_buffer = lltype.malloc(rffi.CWCHARP.TO, buflen,
  50. flavor='raw', zero=True)
  51. return py_uni
  52. def unicode_attach(space, py_obj, w_obj):
  53. "Fills a newly allocated PyUnicodeObject with a unicode string"
  54. py_unicode = rffi.cast(PyUnicodeObject, py_obj)
  55. py_unicode.c_size = len(space.unicode_w(w_obj))
  56. py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO)
  57. def unicode_realize(space, py_obj):
  58. """
  59. Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
  60. be modified after this call.
  61. """
  62. py_uni = rffi.cast(PyUnicodeObject, py_obj)
  63. s = rffi.wcharpsize2unicode(py_uni.c_buffer, py_uni.c_size)
  64. w_obj = space.wrap(s)
  65. track_reference(space, py_obj, w_obj)
  66. return w_obj
  67. @cpython_api([PyObject], lltype.Void, external=False)
  68. def unicode_dealloc(space, py_obj):
  69. py_unicode = rffi.cast(PyUnicodeObject, py_obj)
  70. if py_unicode.c_buffer:
  71. lltype.free(py_unicode.c_buffer, flavor="raw")
  72. from pypy.module.cpyext.object import PyObject_dealloc
  73. PyObject_dealloc(space, py_obj)
  74. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  75. def Py_UNICODE_ISSPACE(space, ch):
  76. """Return 1 or 0 depending on whether ch is a whitespace character."""
  77. return unicodedb.isspace(ord(ch))
  78. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  79. def Py_UNICODE_ISALPHA(space, ch):
  80. """Return 1 or 0 depending on whether ch is an alphabetic character."""
  81. return unicodedb.isalpha(ord(ch))
  82. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  83. def Py_UNICODE_ISALNUM(space, ch):
  84. """Return 1 or 0 depending on whether ch is an alphanumeric character."""
  85. return unicodedb.isalnum(ord(ch))
  86. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  87. def Py_UNICODE_ISLINEBREAK(space, ch):
  88. """Return 1 or 0 depending on whether ch is a linebreak character."""
  89. return unicodedb.islinebreak(ord(ch))
  90. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  91. def Py_UNICODE_ISDECIMAL(space, ch):
  92. """Return 1 or 0 depending on whether ch is a decimal character."""
  93. return unicodedb.isdecimal(ord(ch))
  94. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  95. def Py_UNICODE_ISDIGIT(space, ch):
  96. """Return 1 or 0 depending on whether ch is a digit character."""
  97. return unicodedb.isdigit(ord(ch))
  98. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  99. def Py_UNICODE_ISNUMERIC(space, ch):
  100. """Return 1 or 0 depending on whether ch is a numeric character."""
  101. return unicodedb.isnumeric(ord(ch))
  102. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  103. def Py_UNICODE_ISLOWER(space, ch):
  104. """Return 1 or 0 depending on whether ch is a lowercase character."""
  105. return unicodedb.islower(ord(ch))
  106. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  107. def Py_UNICODE_ISUPPER(space, ch):
  108. """Return 1 or 0 depending on whether ch is an uppercase character."""
  109. return unicodedb.isupper(ord(ch))
  110. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  111. def Py_UNICODE_ISTITLE(space, ch):
  112. """Return 1 or 0 depending on whether ch is a titlecase character."""
  113. return unicodedb.istitle(ord(ch))
  114. @cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL)
  115. def Py_UNICODE_TOLOWER(space, ch):
  116. """Return the character ch converted to lower case."""
  117. return unichr(unicodedb.tolower(ord(ch)))
  118. @cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL)
  119. def Py_UNICODE_TOUPPER(space, ch):
  120. """Return the character ch converted to upper case."""
  121. return unichr(unicodedb.toupper(ord(ch)))
  122. @cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL)
  123. def Py_UNICODE_TOTITLE(space, ch):
  124. """Return the character ch converted to title case."""
  125. return unichr(unicodedb.totitle(ord(ch)))
  126. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  127. def Py_UNICODE_TODECIMAL(space, ch):
  128. """Return the character ch converted to a decimal positive integer. Return
  129. -1 if this is not possible. This macro does not raise exceptions."""
  130. try:
  131. return unicodedb.decimal(ord(ch))
  132. except KeyError:
  133. return -1
  134. @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
  135. def Py_UNICODE_TODIGIT(space, ch):
  136. """Return the character ch converted to a single digit integer. Return -1 if
  137. this is not possible. This macro does not raise exceptions."""
  138. try:
  139. return unicodedb.digit(ord(ch))
  140. except KeyError:
  141. return -1
  142. @cpython_api([Py_UNICODE], rffi.DOUBLE, error=CANNOT_FAIL)
  143. def Py_UNICODE_TONUMERIC(space, ch):
  144. """Return the character ch converted to a double. Return -1.0 if this is not
  145. possible. This macro does not raise exceptions."""
  146. try:
  147. return unicodedb.numeric(ord(ch))
  148. except KeyError:
  149. return -1.0
  150. @cpython_api([], Py_UNICODE, error=CANNOT_FAIL)
  151. def PyUnicode_GetMax(space):
  152. """Get the maximum ordinal for a Unicode character."""
  153. return runicode.UNICHR(runicode.MAXUNICODE)
  154. @cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL)
  155. def PyUnicode_AS_DATA(space, ref):
  156. """Return a pointer to the internal buffer of the object. o has to be a
  157. PyUnicodeObject (not checked)."""
  158. return rffi.cast(rffi.CCHARP, PyUnicode_AS_UNICODE(space, ref))
  159. @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
  160. def PyUnicode_GET_DATA_SIZE(space, w_obj):
  161. """Return the size of the object's internal buffer in bytes. o has to be a
  162. PyUnicodeObject (not checked)."""
  163. return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, w_obj)
  164. @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
  165. def PyUnicode_GET_SIZE(space, w_obj):
  166. """Return the size of the object. o has to be a PyUnicodeObject (not
  167. checked)."""
  168. assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
  169. return space.len_w(w_obj)
  170. @cpython_api([PyObject], rffi.CWCHARP, error=CANNOT_FAIL)
  171. def PyUnicode_AS_UNICODE(space, ref):
  172. """Return a pointer to the internal Py_UNICODE buffer of the object. ref
  173. has to be a PyUnicodeObject (not checked)."""
  174. ref_unicode = rffi.cast(PyUnicodeObject, ref)
  175. if not ref_unicode.c_buffer:
  176. # Copy unicode buffer
  177. w_unicode = from_ref(space, ref)
  178. u = space.unicode_w(w_unicode)
  179. ref_unicode.c_buffer = rffi.unicode2wcharp(u)
  180. return ref_unicode.c_buffer
  181. @cpython_api([PyObject], rffi.CWCHARP)
  182. def PyUnicode_AsUnicode(space, ref):
  183. """Return a read-only pointer to the Unicode object's internal Py_UNICODE
  184. buffer, NULL if unicode is not a Unicode object."""
  185. # Don't use PyUnicode_Check, it will realize the object :-(
  186. w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
  187. if not space.is_true(space.issubtype(w_type, space.w_unicode)):
  188. raise OperationError(space.w_TypeError,
  189. space.wrap("expected unicode object"))
  190. return PyUnicode_AS_UNICODE(space, ref)
  191. @cpython_api([PyObject], Py_ssize_t, error=-1)
  192. def PyUnicode_GetSize(space, ref):
  193. if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode:
  194. ref = rffi.cast(PyUnicodeObject, ref)
  195. return ref.c_size
  196. else:
  197. w_obj = from_ref(space, ref)
  198. return space.len_w(w_obj)
  199. @cpython_api([PyUnicodeObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1)
  200. def PyUnicode_AsWideChar(space, ref, buf, size):
  201. """Copy the Unicode object contents into the wchar_t buffer w. At most
  202. size wchar_t characters are copied (excluding a possibly trailing
  203. 0-termination character). Return the number of wchar_t characters
  204. copied or -1 in case of an error. Note that the resulting wchar_t
  205. string may or may not be 0-terminated. It is the responsibility of the caller
  206. to make sure that the wchar_t string is 0-terminated in case this is
  207. required by the application."""
  208. c_buffer = PyUnicode_AS_UNICODE(space, rffi.cast(PyObject, ref))
  209. c_size = ref.c_size
  210. # If possible, try to copy the 0-termination as well
  211. if size > c_size:
  212. size = c_size + 1
  213. i = 0
  214. while i < size:
  215. buf[i] = c_buffer[i]
  216. i += 1
  217. if size > c_size:
  218. return c_size
  219. else:
  220. return size
  221. @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL)
  222. def PyUnicode_GetDefaultEncoding(space):
  223. """Returns the currently active default encoding."""
  224. if default_encoding[0] == '\x00':
  225. encoding = unicodetype.getdefaultencoding(space)
  226. i = 0
  227. while i < len(encoding) and i < DEFAULT_ENCODING_SIZE:
  228. default_encoding[i] = encoding[i]
  229. i += 1
  230. return default_encoding
  231. @cpython_api([CONST_STRING], rffi.INT_real, error=-1)
  232. def PyUnicode_SetDefaultEncoding(space, encoding):
  233. """Sets the currently active default encoding. Returns 0 on
  234. success, -1 in case of an error."""
  235. w_encoding = space.wrap(rffi.charp2str(encoding))
  236. setdefaultencoding(space, w_encoding)
  237. default_encoding[0] = '\x00'
  238. return 0
  239. @cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject)
  240. def PyUnicode_AsEncodedObject(space, w_unicode, llencoding, llerrors):
  241. """Encode a Unicode object and return the result as Python object.
  242. encoding and errors have the same meaning as the parameters of the same name
  243. in the Unicode encode() method. The codec to be used is looked up using
  244. the Python codec registry. Return NULL if an exception was raised by the
  245. codec."""
  246. if not PyUnicode_Check(space, w_unicode):
  247. PyErr_BadArgument(space)
  248. encoding = errors = None
  249. if llencoding:
  250. encoding = rffi.charp2str(llencoding)
  251. if llerrors:
  252. errors = rffi.charp2str(llerrors)
  253. return unicodetype.encode_object(space, w_unicode, encoding, errors)
  254. @cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject)
  255. def PyUnicode_AsEncodedString(space, w_unicode, llencoding, llerrors):
  256. """Encode a Unicode object and return the result as Python string object.
  257. encoding and errors have the same meaning as the parameters of the same name
  258. in the Unicode encode() method. The codec to be used is looked up using
  259. the Python codec registry. Return NULL if an exception was raised by the
  260. codec."""
  261. w_str = PyUnicode_AsEncodedObject(space, w_unicode, llencoding, llerrors)
  262. if not PyString_Check(space, w_str):
  263. raise OperationError(space.w_TypeError, space.wrap(
  264. "encoder did not return a string object"))
  265. return w_str
  266. @cpython_api([PyObject], PyObject)
  267. def PyUnicode_AsUnicodeEscapeString(space, w_unicode):
  268. """Encode a Unicode object using Unicode-Escape and return the result as Python
  269. string object. Error handling is "strict". Return NULL if an exception was
  270. raised by the codec."""
  271. if not PyUnicode_Check(space, w_unicode):
  272. PyErr_BadArgument(space)
  273. return unicodetype.encode_object(space, w_unicode, 'unicode-escape', 'strict')
  274. @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject)
  275. def PyUnicode_FromUnicode(space, wchar_p, length):
  276. """Create a Unicode Object from the Py_UNICODE buffer u of the given size. u
  277. may be NULL which causes the contents to be undefined. It is the user's
  278. responsibility to fill in the needed data. The buffer is copied into the new
  279. object. If the buffer is not NULL, the return value might be a shared object.
  280. Therefore, modification of the resulting Unicode object is only allowed when u
  281. is NULL."""
  282. if wchar_p:
  283. s = rffi.wcharpsize2unicode(wchar_p, length)
  284. return make_ref(space, space.wrap(s))
  285. else:
  286. return rffi.cast(PyObject, new_empty_unicode(space, length))
  287. @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject)
  288. def PyUnicode_FromWideChar(space, wchar_p, length):
  289. """Create a Unicode object from the wchar_t buffer w of the given size.
  290. Return NULL on failure."""
  291. # PyPy supposes Py_UNICODE == wchar_t
  292. return PyUnicode_FromUnicode(space, wchar_p, length)
  293. @cpython_api([PyObject, CONST_STRING], PyObject)
  294. def _PyUnicode_AsDefaultEncodedString(space, w_unicode, errors):
  295. return PyUnicode_AsEncodedString(space, w_unicode, lltype.nullptr(rffi.CCHARP.TO), errors)
  296. @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, CONST_STRING], PyObject)
  297. def PyUnicode_Decode(space, s, size, encoding, errors):
  298. """Create a Unicode object by decoding size bytes of the encoded string s.
  299. encoding and errors have the same meaning as the parameters of the same name
  300. in the unicode() built-in function. The codec to be used is looked up
  301. using the Python codec registry. Return NULL if an exception was raised by
  302. the codec."""
  303. w_str = space.wrap(rffi.charpsize2str(s, size))
  304. w_encoding = space.wrap(rffi.charp2str(encoding))
  305. if errors:
  306. w_errors = space.wrap(rffi.charp2str(errors))
  307. else:
  308. w_errors = space.w_None
  309. return space.call_method(w_str, 'decode', w_encoding, w_errors)
  310. @cpython_api([PyObject], PyObject)
  311. def PyUnicode_FromObject(space, w_obj):
  312. """Shortcut for PyUnicode_FromEncodedObject(obj, NULL, "strict") which is used
  313. throughout the interpreter whenever coercion to Unicode is needed."""
  314. if space.is_w(space.type(w_obj), space.w_unicode):
  315. return w_obj
  316. else:
  317. return space.call_function(space.w_unicode, w_obj)
  318. @cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject)
  319. def PyUnicode_FromEncodedObject(space, w_obj, encoding, errors):
  320. """Coerce an encoded object obj to an Unicode object and return a reference with
  321. incremented refcount.
  322. String and other char buffer compatible objects are decoded according to the
  323. given encoding and using the error handling defined by errors. Both can be
  324. NULL to have the interface use the default values (see the next section for
  325. details).
  326. All other objects, including Unicode objects, cause a TypeError to be
  327. set."""
  328. w_encoding = space.wrap(rffi.charp2str(encoding))
  329. if errors:
  330. w_errors = space.wrap(rffi.charp2str(errors))
  331. else:
  332. w_errors = space.w_None
  333. # - unicode is disallowed
  334. # - raise TypeError for non-string types
  335. if space.is_true(space.isinstance(w_obj, space.w_unicode)):
  336. w_meth = None
  337. else:
  338. try:
  339. w_meth = space.getattr(w_obj, space.wrap('decode'))
  340. except OperationError, e:
  341. if not e.match(space, space.w_AttributeError):
  342. raise
  343. w_meth = None
  344. if w_meth is None:
  345. raise OperationError(space.w_TypeError,
  346. space.wrap("decoding Unicode is not supported"))
  347. return space.call_function(w_meth, w_encoding, w_errors)
  348. @cpython_api([CONST_STRING], PyObject)
  349. def PyUnicode_FromString(space, s):
  350. """Create a Unicode object from an UTF-8 encoded null-terminated char buffer"""
  351. w_str = space.wrap(rffi.charp2str(s))
  352. return space.call_method(w_str, 'decode', space.wrap("utf-8"))
  353. @cpython_api([CONST_STRING, Py_ssize_t], PyObject)
  354. def PyUnicode_FromStringAndSize(space, s, size):
  355. """Create a Unicode Object from the char buffer u. The bytes will be
  356. interpreted as being UTF-8 encoded. u may also be NULL which causes the
  357. contents to be undefined. It is the user's responsibility to fill in the
  358. needed data. The buffer is copied into the new object. If the buffer is not
  359. NULL, the return value might be a shared object. Therefore, modification of
  360. the resulting Unicode object is only allowed when u is NULL."""
  361. if not s:
  362. raise NotImplementedError
  363. w_str = space.wrap(rffi.charpsize2str(s, size))
  364. return space.call_method(w_str, 'decode', space.wrap("utf-8"))
  365. @cpython_api([rffi.INT_real], PyObject)
  366. def PyUnicode_FromOrdinal(space, ordinal):
  367. """Create a Unicode Object from the given Unicode code point ordinal.
  368. The ordinal must be in range(0x10000) on narrow Python builds
  369. (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
  370. raised in case it is not."""
  371. w_ordinal = space.wrap(rffi.cast(lltype.Signed, ordinal))
  372. return space.call_function(space.builtin.get('unichr'), w_ordinal)
  373. @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1)
  374. def PyUnicode_Resize(space, ref, newsize):
  375. # XXX always create a new string so far
  376. py_uni = rffi.cast(PyUnicodeObject, ref[0])
  377. if not py_uni.c_buffer:
  378. raise OperationError(space.w_SystemError, space.wrap(
  379. "PyUnicode_Resize called on already created string"))
  380. try:
  381. py_newuni = new_empty_unicode(space, newsize)
  382. except MemoryError:
  383. Py_DecRef(space, ref[0])
  384. ref[0] = lltype.nullptr(PyObject.TO)
  385. raise
  386. to_cp = newsize
  387. oldsize = py_uni.c_size
  388. if oldsize < newsize:
  389. to_cp = oldsize
  390. for i in range(to_cp):
  391. py_newuni.c_buffer[i] = py_uni.c_buffer[i]
  392. Py_DecRef(space, ref[0])
  393. ref[0] = rffi.cast(PyObject, py_newuni)
  394. return 0
  395. def make_conversion_functions(suffix, encoding):
  396. @cpython_api([PyObject], PyObject)
  397. @func_renamer('PyUnicode_As%sString' % suffix)
  398. def PyUnicode_AsXXXString(space, w_unicode):
  399. """Encode a Unicode object and return the result as Python
  400. string object. Error handling is "strict". Return NULL if an
  401. exception was raised by the codec."""
  402. if not PyUnicode_Check(space, w_unicode):
  403. PyErr_BadArgument(space)
  404. return unicodetype.encode_object(space, w_unicode, encoding, "strict")
  405. @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING], PyObject)
  406. @func_renamer('PyUnicode_Decode%s' % suffix)
  407. def PyUnicode_DecodeXXX(space, s, size, errors):
  408. """Create a Unicode object by decoding size bytes of the
  409. encoded string s. Return NULL if an exception was raised by
  410. the codec.
  411. """
  412. w_s = space.wrap(rffi.charpsize2str(s, size))
  413. if errors:
  414. w_errors = space.wrap(rffi.charp2str(errors))
  415. else:
  416. w_errors = space.w_None
  417. return space.call_method(w_s, 'decode', space.wrap(encoding), w_errors)
  418. @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject)
  419. @func_renamer('PyUnicode_Encode%s' % suffix)
  420. def PyUnicode_EncodeXXX(space, s, size, errors):
  421. """Encode the Py_UNICODE buffer of the given size and return a
  422. Python string object. Return NULL if an exception was raised
  423. by the codec."""
  424. w_u = space.wrap(rffi.wcharpsize2unicode(s, size))
  425. if errors:
  426. w_errors = space.wrap(rffi.charp2str(errors))
  427. else:
  428. w_errors = space.w_None
  429. return space.call_method(w_u, 'encode', space.wrap(encoding), w_errors)
  430. make_conversion_functions('UTF8', 'utf-8')
  431. make_conversion_functions('ASCII', 'ascii')
  432. make_conversion_functions('Latin1', 'latin-1')
  433. if sys.platform == 'win32':
  434. make_conversion_functions('MBCS', 'mbcs')
  435. @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
  436. def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
  437. """Decode length bytes from a UTF-16 encoded buffer string and return the
  438. corresponding Unicode object. errors (if non-NULL) defines the error
  439. handling. It defaults to "strict".
  440. If byteorder is non-NULL, the decoder starts decoding using the given byte
  441. order:
  442. *byteorder == -1: little endian
  443. *byteorder == 0: native order
  444. *byteorder == 1: big endian
  445. If *byteorder is zero, and the first two bytes of the input data are a
  446. byte order mark (BOM), the decoder switches to this byte order and the BOM is
  447. not copied into the resulting Unicode string. If *byteorder is -1 or
  448. 1, any byte order mark is copied to the output (where it will result in
  449. either a \ufeff or a \ufffe character).
  450. After completion, *byteorder is set to the current byte order at the end
  451. of input data.
  452. If byteorder is NULL, the codec starts in native order mode.
  453. Return NULL if an exception was raised by the codec."""
  454. string = rffi.charpsize2str(s, size)
  455. #FIXME: I don't like these prefixes
  456. if pbyteorder is not None: # correct NULL check?
  457. llbyteorder = rffi.cast(lltype.Signed, pbyteorder[0]) # compatible with int?
  458. if llbyteorder < 0:
  459. byteorder = "little"
  460. elif llbyteorder > 0:
  461. byteorder = "big"
  462. else:
  463. byteorder = "native"
  464. else:
  465. byteorder = "native"
  466. if llerrors:
  467. errors = rffi.charp2str(llerrors)
  468. else:
  469. errors = None
  470. result, length, byteorder = runicode.str_decode_utf_16_helper(string, size,
  471. errors,
  472. True, # final ? false for multiple passes?
  473. None, # errorhandler
  474. byteorder)
  475. if pbyteorder is not None:
  476. pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
  477. return space.wrap(result)
  478. @cpython_api([PyObject, PyObject], rffi.INT_real, error=-2)
  479. def PyUnicode_Compare(space, w_left, w_right):
  480. """Compare two strings and return -1, 0, 1 for less than, equal, and greater
  481. than, respectively."""
  482. return space.int_w(space.cmp(w_left, w_right))
  483. @cpython_api([rffi.CWCHARP, rffi.CWCHARP, Py_ssize_t], lltype.Void)
  484. def Py_UNICODE_COPY(space, target, source, length):
  485. """Roughly equivalent to memcpy() only the base size is Py_UNICODE
  486. copies sizeof(Py_UNICODE) * length bytes from source to target"""
  487. for i in range(0, length):
  488. target[i] = source[i]
  489. @cpython_api([PyObject, PyObject], PyObject)
  490. def PyUnicode_Format(space, w_format, w_args):
  491. """Return a new string object from format and args; this is analogous to
  492. format % args. The args argument must be a tuple."""
  493. return space.mod(w_format, w_args)
  494. @cpython_api([PyObject, PyObject], PyObject)
  495. def PyUnicode_Join(space, w_sep, w_seq):
  496. """Join a sequence of strings using the given separator and return
  497. the resulting Unicode string."""
  498. return space.call_method(w_sep, 'join', w_seq)
  499. @cpython_api([PyObject, PyObject, PyObject, Py_ssize_t], PyObject)
  500. def PyUnicode_Replace(space, w_str, w_substr, w_replstr, maxcount):
  501. """Replace at most maxcount occurrences of substr in str with replstr and
  502. return the resulting Unicode object. maxcount == -1 means replace all
  503. occurrences."""
  504. return space.call_method(w_str, "replace", w_substr, w_replstr,
  505. space.wrap(maxcount))
  506. @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
  507. rffi.INT_real, error=-1)
  508. def PyUnicode_Tailmatch(space, w_str, w_substr, start, end, direction):
  509. """Return 1 if substr matches str[start:end] at the given tail end
  510. (direction == -1 means to do a prefix match, direction == 1 a
  511. suffix match), 0 otherwise. Return -1 if an error occurred."""
  512. str = space.unicode_w(w_str)
  513. substr = space.unicode_w(w_substr)
  514. if rffi.cast(lltype.Signed, direction) <= 0:
  515. return stringtype.stringstartswith(str, substr, start, end)
  516. else:
  517. return stringtype.stringendswith(str, substr, start, end)
  518. @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
  519. def PyUnicode_Count(space, w_str, w_substr, start, end):
  520. """Return the number of non-overlapping occurrences of substr in
  521. str[start:end]. Return -1 if an error occurred."""
  522. w_count = space.call_method(w_str, "count", w_substr,
  523. space.wrap(start), space.wrap(end))
  524. return space.int_w(w_count)
  525. @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
  526. Py_ssize_t, error=-2)
  527. def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
  528. """Return the first position of substr in str*[*start:end] using
  529. the given direction (direction == 1 means to do a forward search,
  530. direction == -1 a backward search). The return value is the index
  531. of the first match; a value of -1 indicates that no match was
  532. found, and -2 indicates that an error occurred and an exception
  533. has been set."""
  534. if rffi.cast(lltype.Signed, direction) > 0:
  535. w_pos = space.call_method(w_str, "find", w_substr,
  536. space.wrap(start), space.wrap(end))
  537. else:
  538. w_pos = space.call_method(w_str, "rfind", w_substr,
  539. space.wrap(start), space.wrap(end))
  540. return space.int_w(w_pos)
  541. @cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
  542. def PyUnicode_Split(space, w_str, w_sep, maxsplit):
  543. """Split a string giving a list of Unicode strings. If sep is
  544. NULL, splitting will be done at all whitespace substrings.
  545. Otherwise, splits occur at the given separator. At most maxsplit
  546. splits will be done. If negative, no limit is set. Separators
  547. are not included in the resulting list."""
  548. if w_sep is None:
  549. w_sep = space.w_None
  550. return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
  551. @cpython_api([PyObject, rffi.INT_real], PyObject)
  552. def PyUnicode_Splitlines(space, w_str, keepend):
  553. """Split a Unicode string at line breaks, returning a list of
  554. Unicode strings. CRLF is considered to be one line break. If
  555. keepend is 0, the Line break characters are not included in the
  556. resulting strings."""
  557. return space.call_method(w_str, "splitlines", space.wrap(keepend))