/Lib/test/test_codeccallbacks.py

http://unladen-swallow.googlecode.com/ · Python · 802 lines · 611 code · 98 blank · 93 comment · 55 complexity · 4196be5443c7f5d40e54265cef13ead1 MD5 · raw file

  1. import test.test_support, unittest
  2. import sys, codecs, htmlentitydefs, unicodedata
  3. class PosReturn:
  4. # this can be used for configurable callbacks
  5. def __init__(self):
  6. self.pos = 0
  7. def handle(self, exc):
  8. oldpos = self.pos
  9. realpos = oldpos
  10. if realpos<0:
  11. realpos = len(exc.object) + realpos
  12. # if we don't advance this time, terminate on the next call
  13. # otherwise we'd get an endless loop
  14. if realpos <= exc.start:
  15. self.pos = len(exc.object)
  16. return (u"<?>", oldpos)
  17. # A UnicodeEncodeError object with a bad start attribute
  18. class BadStartUnicodeEncodeError(UnicodeEncodeError):
  19. def __init__(self):
  20. UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
  21. self.start = []
  22. # A UnicodeEncodeError object with a bad object attribute
  23. class BadObjectUnicodeEncodeError(UnicodeEncodeError):
  24. def __init__(self):
  25. UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
  26. self.object = []
  27. # A UnicodeDecodeError object without an end attribute
  28. class NoEndUnicodeDecodeError(UnicodeDecodeError):
  29. def __init__(self):
  30. UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
  31. del self.end
  32. # A UnicodeDecodeError object with a bad object attribute
  33. class BadObjectUnicodeDecodeError(UnicodeDecodeError):
  34. def __init__(self):
  35. UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
  36. self.object = []
  37. # A UnicodeTranslateError object without a start attribute
  38. class NoStartUnicodeTranslateError(UnicodeTranslateError):
  39. def __init__(self):
  40. UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
  41. del self.start
  42. # A UnicodeTranslateError object without an end attribute
  43. class NoEndUnicodeTranslateError(UnicodeTranslateError):
  44. def __init__(self):
  45. UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
  46. del self.end
  47. # A UnicodeTranslateError object without an object attribute
  48. class NoObjectUnicodeTranslateError(UnicodeTranslateError):
  49. def __init__(self):
  50. UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
  51. del self.object
  52. class CodecCallbackTest(unittest.TestCase):
  53. def test_xmlcharrefreplace(self):
  54. # replace unencodable characters which numeric character entities.
  55. # For ascii, latin-1 and charmaps this is completely implemented
  56. # in C and should be reasonably fast.
  57. s = u"\u30b9\u30d1\u30e2 \xe4nd eggs"
  58. self.assertEqual(
  59. s.encode("ascii", "xmlcharrefreplace"),
  60. "&#12473;&#12497;&#12514; &#228;nd eggs"
  61. )
  62. self.assertEqual(
  63. s.encode("latin-1", "xmlcharrefreplace"),
  64. "&#12473;&#12497;&#12514; \xe4nd eggs"
  65. )
  66. def test_xmlcharnamereplace(self):
  67. # This time use a named character entity for unencodable
  68. # characters, if one is available.
  69. def xmlcharnamereplace(exc):
  70. if not isinstance(exc, UnicodeEncodeError):
  71. raise TypeError("don't know how to handle %r" % exc)
  72. l = []
  73. for c in exc.object[exc.start:exc.end]:
  74. try:
  75. l.append(u"&%s;" % htmlentitydefs.codepoint2name[ord(c)])
  76. except KeyError:
  77. l.append(u"&#%d;" % ord(c))
  78. return (u"".join(l), exc.end)
  79. codecs.register_error(
  80. "test.xmlcharnamereplace", xmlcharnamereplace)
  81. sin = u"\xab\u211c\xbb = \u2329\u1234\u20ac\u232a"
  82. sout = "&laquo;&real;&raquo; = &lang;&#4660;&euro;&rang;"
  83. self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout)
  84. sout = "\xab&real;\xbb = &lang;&#4660;&euro;&rang;"
  85. self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout)
  86. sout = "\xab&real;\xbb = &lang;&#4660;\xa4&rang;"
  87. self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout)
  88. def test_uninamereplace(self):
  89. # We're using the names from the unicode database this time,
  90. # and we're doing "syntax highlighting" here, i.e. we include
  91. # the replaced text in ANSI escape sequences. For this it is
  92. # useful that the error handler is not called for every single
  93. # unencodable character, but for a complete sequence of
  94. # unencodable characters, otherwise we would output many
  95. # unnecessary escape sequences.
  96. def uninamereplace(exc):
  97. if not isinstance(exc, UnicodeEncodeError):
  98. raise TypeError("don't know how to handle %r" % exc)
  99. l = []
  100. for c in exc.object[exc.start:exc.end]:
  101. l.append(unicodedata.name(c, u"0x%x" % ord(c)))
  102. return (u"\033[1m%s\033[0m" % u", ".join(l), exc.end)
  103. codecs.register_error(
  104. "test.uninamereplace", uninamereplace)
  105. sin = u"\xac\u1234\u20ac\u8000"
  106. sout = "\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
  107. self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout)
  108. sout = "\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
  109. self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout)
  110. sout = "\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m"
  111. self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout)
  112. def test_backslashescape(self):
  113. # Does the same as the "unicode-escape" encoding, but with different
  114. # base encodings.
  115. sin = u"a\xac\u1234\u20ac\u8000"
  116. if sys.maxunicode > 0xffff:
  117. sin += unichr(sys.maxunicode)
  118. sout = "a\\xac\\u1234\\u20ac\\u8000"
  119. if sys.maxunicode > 0xffff:
  120. sout += "\\U%08x" % sys.maxunicode
  121. self.assertEqual(sin.encode("ascii", "backslashreplace"), sout)
  122. sout = "a\xac\\u1234\\u20ac\\u8000"
  123. if sys.maxunicode > 0xffff:
  124. sout += "\\U%08x" % sys.maxunicode
  125. self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout)
  126. sout = "a\xac\\u1234\xa4\\u8000"
  127. if sys.maxunicode > 0xffff:
  128. sout += "\\U%08x" % sys.maxunicode
  129. self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout)
  130. def test_decoderelaxedutf8(self):
  131. # This is the test for a decoding callback handler,
  132. # that relaxes the UTF-8 minimal encoding restriction.
  133. # A null byte that is encoded as "\xc0\x80" will be
  134. # decoded as a null byte. All other illegal sequences
  135. # will be handled strictly.
  136. def relaxedutf8(exc):
  137. if not isinstance(exc, UnicodeDecodeError):
  138. raise TypeError("don't know how to handle %r" % exc)
  139. if exc.object[exc.start:exc.end].startswith("\xc0\x80"):
  140. return (u"\x00", exc.start+2) # retry after two bytes
  141. else:
  142. raise exc
  143. codecs.register_error(
  144. "test.relaxedutf8", relaxedutf8)
  145. sin = "a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80"
  146. sout = u"a\x00b\x00c\xfc\x00\x00"
  147. self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout)
  148. sin = "\xc0\x80\xc0\x81"
  149. self.assertRaises(UnicodeError, sin.decode, "utf-8", "test.relaxedutf8")
  150. def test_charmapencode(self):
  151. # For charmap encodings the replacement string will be
  152. # mapped through the encoding again. This means, that
  153. # to be able to use e.g. the "replace" handler, the
  154. # charmap has to have a mapping for "?".
  155. charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
  156. sin = u"abc"
  157. sout = "AABBCC"
  158. self.assertEquals(codecs.charmap_encode(sin, "strict", charmap)[0], sout)
  159. sin = u"abcA"
  160. self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap)
  161. charmap[ord("?")] = "XYZ"
  162. sin = u"abcDEF"
  163. sout = "AABBCCXYZXYZXYZ"
  164. self.assertEquals(codecs.charmap_encode(sin, "replace", charmap)[0], sout)
  165. charmap[ord("?")] = u"XYZ"
  166. self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
  167. charmap[ord("?")] = u"XYZ"
  168. self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
  169. def test_decodeunicodeinternal(self):
  170. self.assertRaises(
  171. UnicodeDecodeError,
  172. "\x00\x00\x00\x00\x00".decode,
  173. "unicode-internal",
  174. )
  175. if sys.maxunicode > 0xffff:
  176. def handler_unicodeinternal(exc):
  177. if not isinstance(exc, UnicodeDecodeError):
  178. raise TypeError("don't know how to handle %r" % exc)
  179. return (u"\x01", 1)
  180. self.assertEqual(
  181. "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
  182. u"\u0000"
  183. )
  184. self.assertEqual(
  185. "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"),
  186. u"\u0000\ufffd"
  187. )
  188. codecs.register_error("test.hui", handler_unicodeinternal)
  189. self.assertEqual(
  190. "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"),
  191. u"\u0000\u0001\u0000"
  192. )
  193. def test_callbacks(self):
  194. def handler1(exc):
  195. if not isinstance(exc, UnicodeEncodeError) \
  196. and not isinstance(exc, UnicodeDecodeError):
  197. raise TypeError("don't know how to handle %r" % exc)
  198. l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
  199. return (u"[%s]" % u"".join(l), exc.end)
  200. codecs.register_error("test.handler1", handler1)
  201. def handler2(exc):
  202. if not isinstance(exc, UnicodeDecodeError):
  203. raise TypeError("don't know how to handle %r" % exc)
  204. l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
  205. return (u"[%s]" % u"".join(l), exc.end+1) # skip one character
  206. codecs.register_error("test.handler2", handler2)
  207. s = "\x00\x81\x7f\x80\xff"
  208. self.assertEqual(
  209. s.decode("ascii", "test.handler1"),
  210. u"\x00[<129>]\x7f[<128>][<255>]"
  211. )
  212. self.assertEqual(
  213. s.decode("ascii", "test.handler2"),
  214. u"\x00[<129>][<128>]"
  215. )
  216. self.assertEqual(
  217. "\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
  218. u"\u3042[<92><117><51><120>]xx"
  219. )
  220. self.assertEqual(
  221. "\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
  222. u"\u3042[<92><117><51><120><120>]"
  223. )
  224. self.assertEqual(
  225. codecs.charmap_decode("abc", "test.handler1", {ord("a"): u"z"})[0],
  226. u"z[<98>][<99>]"
  227. )
  228. self.assertEqual(
  229. u"g\xfc\xdfrk".encode("ascii", "test.handler1"),
  230. u"g[<252><223>]rk"
  231. )
  232. self.assertEqual(
  233. u"g\xfc\xdf".encode("ascii", "test.handler1"),
  234. u"g[<252><223>]"
  235. )
  236. def test_longstrings(self):
  237. # test long strings to check for memory overflow problems
  238. errors = [ "strict", "ignore", "replace", "xmlcharrefreplace",
  239. "backslashreplace"]
  240. # register the handlers under different names,
  241. # to prevent the codec from recognizing the name
  242. for err in errors:
  243. codecs.register_error("test." + err, codecs.lookup_error(err))
  244. l = 1000
  245. errors += [ "test." + err for err in errors ]
  246. for uni in [ s*l for s in (u"x", u"\u3042", u"a\xe4") ]:
  247. for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15",
  248. "utf-8", "utf-7", "utf-16", "utf-32"):
  249. for err in errors:
  250. try:
  251. uni.encode(enc, err)
  252. except UnicodeError:
  253. pass
  254. def check_exceptionobjectargs(self, exctype, args, msg):
  255. # Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion
  256. # check with one missing argument
  257. self.assertRaises(TypeError, exctype, *args[:-1])
  258. # check with one argument too much
  259. self.assertRaises(TypeError, exctype, *(args + ["too much"]))
  260. # check with one argument of the wrong type
  261. wrongargs = [ "spam", u"eggs", 42, 1.0, None ]
  262. for i in xrange(len(args)):
  263. for wrongarg in wrongargs:
  264. if type(wrongarg) is type(args[i]):
  265. continue
  266. # build argument array
  267. callargs = []
  268. for j in xrange(len(args)):
  269. if i==j:
  270. callargs.append(wrongarg)
  271. else:
  272. callargs.append(args[i])
  273. self.assertRaises(TypeError, exctype, *callargs)
  274. # check with the correct number and type of arguments
  275. exc = exctype(*args)
  276. self.assertEquals(str(exc), msg)
  277. def test_unicodeencodeerror(self):
  278. self.check_exceptionobjectargs(
  279. UnicodeEncodeError,
  280. ["ascii", u"g\xfcrk", 1, 2, "ouch"],
  281. "'ascii' codec can't encode character u'\\xfc' in position 1: ouch"
  282. )
  283. self.check_exceptionobjectargs(
  284. UnicodeEncodeError,
  285. ["ascii", u"g\xfcrk", 1, 4, "ouch"],
  286. "'ascii' codec can't encode characters in position 1-3: ouch"
  287. )
  288. self.check_exceptionobjectargs(
  289. UnicodeEncodeError,
  290. ["ascii", u"\xfcx", 0, 1, "ouch"],
  291. "'ascii' codec can't encode character u'\\xfc' in position 0: ouch"
  292. )
  293. self.check_exceptionobjectargs(
  294. UnicodeEncodeError,
  295. ["ascii", u"\u0100x", 0, 1, "ouch"],
  296. "'ascii' codec can't encode character u'\\u0100' in position 0: ouch"
  297. )
  298. self.check_exceptionobjectargs(
  299. UnicodeEncodeError,
  300. ["ascii", u"\uffffx", 0, 1, "ouch"],
  301. "'ascii' codec can't encode character u'\\uffff' in position 0: ouch"
  302. )
  303. if sys.maxunicode > 0xffff:
  304. self.check_exceptionobjectargs(
  305. UnicodeEncodeError,
  306. ["ascii", u"\U00010000x", 0, 1, "ouch"],
  307. "'ascii' codec can't encode character u'\\U00010000' in position 0: ouch"
  308. )
  309. def test_unicodedecodeerror(self):
  310. self.check_exceptionobjectargs(
  311. UnicodeDecodeError,
  312. ["ascii", "g\xfcrk", 1, 2, "ouch"],
  313. "'ascii' codec can't decode byte 0xfc in position 1: ouch"
  314. )
  315. self.check_exceptionobjectargs(
  316. UnicodeDecodeError,
  317. ["ascii", "g\xfcrk", 1, 3, "ouch"],
  318. "'ascii' codec can't decode bytes in position 1-2: ouch"
  319. )
  320. def test_unicodetranslateerror(self):
  321. self.check_exceptionobjectargs(
  322. UnicodeTranslateError,
  323. [u"g\xfcrk", 1, 2, "ouch"],
  324. "can't translate character u'\\xfc' in position 1: ouch"
  325. )
  326. self.check_exceptionobjectargs(
  327. UnicodeTranslateError,
  328. [u"g\u0100rk", 1, 2, "ouch"],
  329. "can't translate character u'\\u0100' in position 1: ouch"
  330. )
  331. self.check_exceptionobjectargs(
  332. UnicodeTranslateError,
  333. [u"g\uffffrk", 1, 2, "ouch"],
  334. "can't translate character u'\\uffff' in position 1: ouch"
  335. )
  336. if sys.maxunicode > 0xffff:
  337. self.check_exceptionobjectargs(
  338. UnicodeTranslateError,
  339. [u"g\U00010000rk", 1, 2, "ouch"],
  340. "can't translate character u'\\U00010000' in position 1: ouch"
  341. )
  342. self.check_exceptionobjectargs(
  343. UnicodeTranslateError,
  344. [u"g\xfcrk", 1, 3, "ouch"],
  345. "can't translate characters in position 1-2: ouch"
  346. )
  347. def test_badandgoodstrictexceptions(self):
  348. # "strict" complains about a non-exception passed in
  349. self.assertRaises(
  350. TypeError,
  351. codecs.strict_errors,
  352. 42
  353. )
  354. # "strict" complains about the wrong exception type
  355. self.assertRaises(
  356. Exception,
  357. codecs.strict_errors,
  358. Exception("ouch")
  359. )
  360. # If the correct exception is passed in, "strict" raises it
  361. self.assertRaises(
  362. UnicodeEncodeError,
  363. codecs.strict_errors,
  364. UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")
  365. )
  366. def test_badandgoodignoreexceptions(self):
  367. # "ignore" complains about a non-exception passed in
  368. self.assertRaises(
  369. TypeError,
  370. codecs.ignore_errors,
  371. 42
  372. )
  373. # "ignore" complains about the wrong exception type
  374. self.assertRaises(
  375. TypeError,
  376. codecs.ignore_errors,
  377. UnicodeError("ouch")
  378. )
  379. # If the correct exception is passed in, "ignore" returns an empty replacement
  380. self.assertEquals(
  381. codecs.ignore_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
  382. (u"", 1)
  383. )
  384. self.assertEquals(
  385. codecs.ignore_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
  386. (u"", 1)
  387. )
  388. self.assertEquals(
  389. codecs.ignore_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
  390. (u"", 1)
  391. )
  392. def test_badandgoodreplaceexceptions(self):
  393. # "replace" complains about a non-exception passed in
  394. self.assertRaises(
  395. TypeError,
  396. codecs.replace_errors,
  397. 42
  398. )
  399. # "replace" complains about the wrong exception type
  400. self.assertRaises(
  401. TypeError,
  402. codecs.replace_errors,
  403. UnicodeError("ouch")
  404. )
  405. self.assertRaises(
  406. TypeError,
  407. codecs.replace_errors,
  408. BadObjectUnicodeEncodeError()
  409. )
  410. self.assertRaises(
  411. TypeError,
  412. codecs.replace_errors,
  413. BadObjectUnicodeDecodeError()
  414. )
  415. # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement
  416. self.assertEquals(
  417. codecs.replace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
  418. (u"?", 1)
  419. )
  420. self.assertEquals(
  421. codecs.replace_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
  422. (u"\ufffd", 1)
  423. )
  424. self.assertEquals(
  425. codecs.replace_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
  426. (u"\ufffd", 1)
  427. )
  428. def test_badandgoodxmlcharrefreplaceexceptions(self):
  429. # "xmlcharrefreplace" complains about a non-exception passed in
  430. self.assertRaises(
  431. TypeError,
  432. codecs.xmlcharrefreplace_errors,
  433. 42
  434. )
  435. # "xmlcharrefreplace" complains about the wrong exception types
  436. self.assertRaises(
  437. TypeError,
  438. codecs.xmlcharrefreplace_errors,
  439. UnicodeError("ouch")
  440. )
  441. # "xmlcharrefreplace" can only be used for encoding
  442. self.assertRaises(
  443. TypeError,
  444. codecs.xmlcharrefreplace_errors,
  445. UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
  446. )
  447. self.assertRaises(
  448. TypeError,
  449. codecs.xmlcharrefreplace_errors,
  450. UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
  451. )
  452. # Use the correct exception
  453. cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 0x3042)
  454. s = "".join(unichr(c) for c in cs)
  455. self.assertEquals(
  456. codecs.xmlcharrefreplace_errors(
  457. UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
  458. ),
  459. (u"".join(u"&#%d;" % ord(c) for c in s), len(s))
  460. )
  461. def test_badandgoodbackslashreplaceexceptions(self):
  462. # "backslashreplace" complains about a non-exception passed in
  463. self.assertRaises(
  464. TypeError,
  465. codecs.backslashreplace_errors,
  466. 42
  467. )
  468. # "backslashreplace" complains about the wrong exception types
  469. self.assertRaises(
  470. TypeError,
  471. codecs.backslashreplace_errors,
  472. UnicodeError("ouch")
  473. )
  474. # "backslashreplace" can only be used for encoding
  475. self.assertRaises(
  476. TypeError,
  477. codecs.backslashreplace_errors,
  478. UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
  479. )
  480. self.assertRaises(
  481. TypeError,
  482. codecs.backslashreplace_errors,
  483. UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
  484. )
  485. # Use the correct exception
  486. self.assertEquals(
  487. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
  488. (u"\\u3042", 1)
  489. )
  490. self.assertEquals(
  491. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\x00", 0, 1, "ouch")),
  492. (u"\\x00", 1)
  493. )
  494. self.assertEquals(
  495. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\xff", 0, 1, "ouch")),
  496. (u"\\xff", 1)
  497. )
  498. self.assertEquals(
  499. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u0100", 0, 1, "ouch")),
  500. (u"\\u0100", 1)
  501. )
  502. self.assertEquals(
  503. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\uffff", 0, 1, "ouch")),
  504. (u"\\uffff", 1)
  505. )
  506. if sys.maxunicode>0xffff:
  507. self.assertEquals(
  508. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U00010000", 0, 1, "ouch")),
  509. (u"\\U00010000", 1)
  510. )
  511. self.assertEquals(
  512. codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U0010ffff", 0, 1, "ouch")),
  513. (u"\\U0010ffff", 1)
  514. )
  515. def test_badhandlerresults(self):
  516. results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
  517. encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15")
  518. for res in results:
  519. codecs.register_error("test.badhandler", lambda: res)
  520. for enc in encs:
  521. self.assertRaises(
  522. TypeError,
  523. u"\u3042".encode,
  524. enc,
  525. "test.badhandler"
  526. )
  527. for (enc, bytes) in (
  528. ("ascii", "\xff"),
  529. ("utf-8", "\xff"),
  530. ("utf-7", "+x-"),
  531. ("unicode-internal", "\x00"),
  532. ):
  533. self.assertRaises(
  534. TypeError,
  535. bytes.decode,
  536. enc,
  537. "test.badhandler"
  538. )
  539. def test_lookup(self):
  540. self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict"))
  541. self.assertEquals(codecs.ignore_errors, codecs.lookup_error("ignore"))
  542. self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict"))
  543. self.assertEquals(
  544. codecs.xmlcharrefreplace_errors,
  545. codecs.lookup_error("xmlcharrefreplace")
  546. )
  547. self.assertEquals(
  548. codecs.backslashreplace_errors,
  549. codecs.lookup_error("backslashreplace")
  550. )
  551. def test_unencodablereplacement(self):
  552. def unencrepl(exc):
  553. if isinstance(exc, UnicodeEncodeError):
  554. return (u"\u4242", exc.end)
  555. else:
  556. raise TypeError("don't know how to handle %r" % exc)
  557. codecs.register_error("test.unencreplhandler", unencrepl)
  558. for enc in ("ascii", "iso-8859-1", "iso-8859-15"):
  559. self.assertRaises(
  560. UnicodeEncodeError,
  561. u"\u4242".encode,
  562. enc,
  563. "test.unencreplhandler"
  564. )
  565. def test_badregistercall(self):
  566. # enhance coverage of:
  567. # Modules/_codecsmodule.c::register_error()
  568. # Python/codecs.c::PyCodec_RegisterError()
  569. self.assertRaises(TypeError, codecs.register_error, 42)
  570. self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42)
  571. def test_badlookupcall(self):
  572. # enhance coverage of:
  573. # Modules/_codecsmodule.c::lookup_error()
  574. self.assertRaises(TypeError, codecs.lookup_error)
  575. def test_unknownhandler(self):
  576. # enhance coverage of:
  577. # Modules/_codecsmodule.c::lookup_error()
  578. self.assertRaises(LookupError, codecs.lookup_error, "test.unknown")
  579. def test_xmlcharrefvalues(self):
  580. # enhance coverage of:
  581. # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
  582. # and inline implementations
  583. v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000)
  584. if sys.maxunicode>=100000:
  585. v += (100000, 500000, 1000000)
  586. s = u"".join([unichr(x) for x in v])
  587. codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
  588. for enc in ("ascii", "iso-8859-15"):
  589. for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"):
  590. s.encode(enc, err)
  591. def test_decodehelper(self):
  592. # enhance coverage of:
  593. # Objects/unicodeobject.c::unicode_decode_call_errorhandler()
  594. # and callers
  595. self.assertRaises(LookupError, "\xff".decode, "ascii", "test.unknown")
  596. def baddecodereturn1(exc):
  597. return 42
  598. codecs.register_error("test.baddecodereturn1", baddecodereturn1)
  599. self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1")
  600. self.assertRaises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1")
  601. self.assertRaises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1")
  602. self.assertRaises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
  603. self.assertRaises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
  604. self.assertRaises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
  605. def baddecodereturn2(exc):
  606. return (u"?", None)
  607. codecs.register_error("test.baddecodereturn2", baddecodereturn2)
  608. self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn2")
  609. handler = PosReturn()
  610. codecs.register_error("test.posreturn", handler.handle)
  611. # Valid negative position
  612. handler.pos = -1
  613. self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
  614. # Valid negative position
  615. handler.pos = -2
  616. self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?><?>")
  617. # Negative position out of bounds
  618. handler.pos = -3
  619. self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
  620. # Valid positive position
  621. handler.pos = 1
  622. self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
  623. # Largest valid positive position (one beyond end of input)
  624. handler.pos = 2
  625. self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>")
  626. # Invalid positive position
  627. handler.pos = 3
  628. self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
  629. # Restart at the "0"
  630. handler.pos = 6
  631. self.assertEquals("\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), u"<?>0")
  632. class D(dict):
  633. def __getitem__(self, key):
  634. raise ValueError
  635. self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
  636. self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
  637. self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1})
  638. def test_encodehelper(self):
  639. # enhance coverage of:
  640. # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
  641. # and callers
  642. self.assertRaises(LookupError, u"\xff".encode, "ascii", "test.unknown")
  643. def badencodereturn1(exc):
  644. return 42
  645. codecs.register_error("test.badencodereturn1", badencodereturn1)
  646. self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn1")
  647. def badencodereturn2(exc):
  648. return (u"?", None)
  649. codecs.register_error("test.badencodereturn2", badencodereturn2)
  650. self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn2")
  651. handler = PosReturn()
  652. codecs.register_error("test.posreturn", handler.handle)
  653. # Valid negative position
  654. handler.pos = -1
  655. self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
  656. # Valid negative position
  657. handler.pos = -2
  658. self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?><?>")
  659. # Negative position out of bounds
  660. handler.pos = -3
  661. self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
  662. # Valid positive position
  663. handler.pos = 1
  664. self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
  665. # Largest valid positive position (one beyond end of input
  666. handler.pos = 2
  667. self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>")
  668. # Invalid positive position
  669. handler.pos = 3
  670. self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
  671. handler.pos = 0
  672. class D(dict):
  673. def __getitem__(self, key):
  674. raise ValueError
  675. for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.posreturn"):
  676. self.assertRaises(UnicodeError, codecs.charmap_encode, u"\xff", err, {0xff: None})
  677. self.assertRaises(ValueError, codecs.charmap_encode, u"\xff", err, D())
  678. self.assertRaises(TypeError, codecs.charmap_encode, u"\xff", err, {0xff: 300})
  679. def test_translatehelper(self):
  680. # enhance coverage of:
  681. # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
  682. # and callers
  683. # (Unfortunately the errors argument is not directly accessible
  684. # from Python, so we can't test that much)
  685. class D(dict):
  686. def __getitem__(self, key):
  687. raise ValueError
  688. self.assertRaises(ValueError, u"\xff".translate, D())
  689. self.assertRaises(TypeError, u"\xff".translate, {0xff: sys.maxunicode+1})
  690. self.assertRaises(TypeError, u"\xff".translate, {0xff: ()})
  691. def test_bug828737(self):
  692. charmap = {
  693. ord("&"): u"&amp;",
  694. ord("<"): u"&lt;",
  695. ord(">"): u"&gt;",
  696. ord('"'): u"&quot;",
  697. }
  698. for n in (1, 10, 100, 1000):
  699. text = u'abc<def>ghi'*n
  700. text.translate(charmap)
  701. def test_main():
  702. test.test_support.run_unittest(CodecCallbackTest)
  703. if __name__ == "__main__":
  704. test_main()