PageRenderTime 67ms CodeModel.GetById 12ms RepoModel.GetById 1ms app.codeStats 0ms

/activesupport/test/multibyte_chars_test.rb

https://github.com/ghar/rails
Ruby | 703 lines | 593 code | 98 blank | 12 comment | 3 complexity | ff0ed4e4459df4306e78d0a20377dfb1 MD5 | raw file
  1. # encoding: utf-8
  2. require 'abstract_unit'
  3. require 'multibyte_test_helpers'
  4. require 'active_support/core_ext/string/multibyte'
  5. class String
  6. def __method_for_multibyte_testing_with_integer_result; 1; end
  7. def __method_for_multibyte_testing; 'result'; end
  8. def __method_for_multibyte_testing!; 'result'; end
  9. end
  10. class MultibyteCharsTest < Test::Unit::TestCase
  11. include MultibyteTestHelpers
  12. def setup
  13. @proxy_class = ActiveSupport::Multibyte::Chars
  14. @chars = @proxy_class.new UNICODE_STRING
  15. end
  16. def test_wraps_the_original_string
  17. assert_equal UNICODE_STRING, @chars.to_s
  18. assert_equal UNICODE_STRING, @chars.wrapped_string
  19. end
  20. def test_should_allow_method_calls_to_string
  21. assert_nothing_raised do
  22. @chars.__method_for_multibyte_testing
  23. end
  24. assert_raise NoMethodError do
  25. @chars.__unknown_method
  26. end
  27. end
  28. def test_forwarded_method_calls_should_return_new_chars_instance
  29. assert_kind_of @proxy_class, @chars.__method_for_multibyte_testing
  30. assert_not_equal @chars.object_id, @chars.__method_for_multibyte_testing.object_id
  31. end
  32. def test_forwarded_bang_method_calls_should_return_the_original_chars_instance
  33. assert_kind_of @proxy_class, @chars.__method_for_multibyte_testing!
  34. assert_equal @chars.object_id, @chars.__method_for_multibyte_testing!.object_id
  35. end
  36. def test_methods_are_forwarded_to_wrapped_string_for_byte_strings
  37. assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class
  38. end
  39. def test_forwarded_method_with_non_string_result_should_be_returned_vertabim
  40. assert_equal ''.__method_for_multibyte_testing_with_integer_result, @chars.__method_for_multibyte_testing_with_integer_result
  41. end
  42. def test_should_concatenate
  43. mb_a = 'a'.mb_chars
  44. mb_b = 'b'.mb_chars
  45. assert_equal 'ab', mb_a + 'b'
  46. assert_equal 'ab', 'a' + mb_b
  47. assert_equal 'ab', mb_a + mb_b
  48. assert_equal 'ab', mb_a << 'b'
  49. assert_equal 'ab', 'a' << mb_b
  50. assert_equal 'abb', mb_a << mb_b
  51. end
  52. def test_consumes_utf8_strings
  53. assert @proxy_class.consumes?(UNICODE_STRING)
  54. assert @proxy_class.consumes?(ASCII_STRING)
  55. assert !@proxy_class.consumes?(BYTE_STRING)
  56. end
  57. def test_unpack_utf8_strings
  58. assert_equal 4, ActiveSupport::Multibyte::Unicode.u_unpack(UNICODE_STRING).length
  59. assert_equal 5, ActiveSupport::Multibyte::Unicode.u_unpack(ASCII_STRING).length
  60. end
  61. def test_unpack_raises_encoding_error_on_broken_strings
  62. assert_raise(ActiveSupport::Multibyte::EncodingError) do
  63. ActiveSupport::Multibyte::Unicode.u_unpack(BYTE_STRING)
  64. end
  65. end
  66. def test_concatenation_should_return_a_proxy_class_instance
  67. assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class
  68. assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class
  69. end
  70. def test_ascii_strings_are_treated_at_utf8_strings
  71. assert_equal ActiveSupport::Multibyte.proxy_class, ASCII_STRING.mb_chars.class
  72. end
  73. def test_concatenate_should_return_proxy_instance
  74. assert(('a'.mb_chars + 'b').kind_of?(@proxy_class))
  75. assert(('a'.mb_chars + 'b'.mb_chars).kind_of?(@proxy_class))
  76. assert(('a'.mb_chars << 'b').kind_of?(@proxy_class))
  77. assert(('a'.mb_chars << 'b'.mb_chars).kind_of?(@proxy_class))
  78. end
  79. end
  80. class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase
  81. include MultibyteTestHelpers
  82. def setup
  83. @chars = UNICODE_STRING.dup.mb_chars
  84. if RUBY_VERSION < '1.9'
  85. # Multibyte support all kinds of whitespace (ie. NEWLINE, SPACE, EM SPACE)
  86. @whitespace = "\n\t#{[32, 8195].pack('U*')}"
  87. else
  88. # Ruby 1.9 only supports basic whitespace
  89. @whitespace = "\n\t "
  90. end
  91. @byte_order_mark = [65279].pack('U')
  92. end
  93. def test_split_should_return_an_array_of_chars_instances
  94. @chars.split(//).each do |character|
  95. assert_kind_of ActiveSupport::Multibyte.proxy_class, character
  96. end
  97. end
  98. def test_indexed_insert_accepts_fixnums
  99. @chars[2] = 32
  100. assert_equal 'こに わ', @chars
  101. end
  102. %w{capitalize downcase lstrip reverse rstrip strip upcase}.each do |method|
  103. class_eval(<<-EOTESTS)
  104. def test_#{method}_bang_should_return_self
  105. assert_equal @chars.object_id, @chars.send("#{method}!").object_id
  106. end
  107. def test_#{method}_bang_should_change_wrapped_string
  108. original = ' él piDió Un bUen café '
  109. proxy = chars(original.dup)
  110. proxy.send("#{method}!")
  111. assert_not_equal original, proxy.to_s
  112. end
  113. EOTESTS
  114. end
  115. def test_tidy_bytes_bang_should_return_self
  116. assert_equal @chars.object_id, @chars.tidy_bytes!.object_id
  117. end
  118. def test_tidy_bytes_bang_should_change_wrapped_string
  119. original = " Un bUen café \x92"
  120. proxy = chars(original.dup)
  121. proxy.tidy_bytes!
  122. assert_not_equal original, proxy.to_s
  123. end
  124. if RUBY_VERSION >= '1.9'
  125. def test_unicode_string_should_have_utf8_encoding
  126. assert_equal Encoding::UTF_8, UNICODE_STRING.encoding
  127. end
  128. end
  129. def test_identity
  130. assert_equal @chars, @chars
  131. assert @chars.eql?(@chars)
  132. assert !@chars.eql?(UNICODE_STRING)
  133. end
  134. def test_string_methods_are_chainable
  135. assert chars('').insert(0, '').kind_of?(ActiveSupport::Multibyte.proxy_class)
  136. assert chars('').rjust(1).kind_of?(ActiveSupport::Multibyte.proxy_class)
  137. assert chars('').ljust(1).kind_of?(ActiveSupport::Multibyte.proxy_class)
  138. assert chars('').center(1).kind_of?(ActiveSupport::Multibyte.proxy_class)
  139. assert chars('').rstrip.kind_of?(ActiveSupport::Multibyte.proxy_class)
  140. assert chars('').lstrip.kind_of?(ActiveSupport::Multibyte.proxy_class)
  141. assert chars('').strip.kind_of?(ActiveSupport::Multibyte.proxy_class)
  142. assert chars('').reverse.kind_of?(ActiveSupport::Multibyte.proxy_class)
  143. assert chars(' ').slice(0).kind_of?(ActiveSupport::Multibyte.proxy_class)
  144. assert chars('').limit(0).kind_of?(ActiveSupport::Multibyte.proxy_class)
  145. assert chars('').upcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
  146. assert chars('').downcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
  147. assert chars('').capitalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
  148. assert chars('').normalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
  149. assert chars('').decompose.kind_of?(ActiveSupport::Multibyte.proxy_class)
  150. assert chars('').compose.kind_of?(ActiveSupport::Multibyte.proxy_class)
  151. assert chars('').tidy_bytes.kind_of?(ActiveSupport::Multibyte.proxy_class)
  152. end
  153. def test_should_be_equal_to_the_wrapped_string
  154. assert_equal UNICODE_STRING, @chars
  155. assert_equal @chars, UNICODE_STRING
  156. end
  157. def test_should_not_be_equal_to_an_other_string
  158. assert_not_equal @chars, 'other'
  159. assert_not_equal 'other', @chars
  160. end
  161. def test_sortability
  162. words = %w(builder armor zebra).sort_by { |s| s.mb_chars }
  163. assert_equal %w(armor builder zebra), words
  164. end
  165. def test_should_return_character_offset_for_regexp_matches
  166. assert_nil(@chars =~ /wrong/u)
  167. assert_equal 0, (@chars =~ //u)
  168. assert_equal 0, (@chars =~ /こに/u)
  169. assert_equal 1, (@chars =~ //u)
  170. assert_equal 2, (@chars =~ //u)
  171. assert_equal 3, (@chars =~ //u)
  172. end
  173. def test_should_use_character_offsets_for_insert_offsets
  174. assert_equal '', ''.mb_chars.insert(0, '')
  175. assert_equal 'こわにちわ', @chars.insert(1, 'わ')
  176. assert_equal 'こわわわにちわ', @chars.insert(2, 'わわ')
  177. assert_equal 'わこわわわにちわ', @chars.insert(0, 'わ')
  178. assert_equal 'わこわわわにちわ', @chars.wrapped_string
  179. end
  180. def test_insert_should_be_destructive
  181. @chars.insert(1, 'わ')
  182. assert_equal 'こわにちわ', @chars
  183. end
  184. def test_insert_throws_index_error
  185. assert_raise(IndexError) { @chars.insert(-12, 'わ')}
  186. assert_raise(IndexError) { @chars.insert(12, 'わ') }
  187. end
  188. def test_should_know_if_one_includes_the_other
  189. assert @chars.include?('')
  190. assert @chars.include?('ち')
  191. assert @chars.include?('わ')
  192. assert !@chars.include?('こちわ')
  193. assert !@chars.include?('a')
  194. end
  195. def test_include_raises_when_nil_is_passed
  196. @chars.include?(nil)
  197. flunk "Expected chars.include?(nil) to raise TypeError or NoMethodError"
  198. rescue Exception
  199. end
  200. def test_index_should_return_character_offset
  201. assert_nil @chars.index('u')
  202. assert_equal 0, @chars.index('こに')
  203. assert_equal 2, @chars.index('ち')
  204. assert_equal 2, @chars.index('ち', -2)
  205. assert_equal nil, @chars.index('ち', -1)
  206. assert_equal 3, @chars.index('わ')
  207. assert_equal 5, 'ééxééx'.mb_chars.index('x', 4)
  208. end
  209. def test_rindex_should_return_character_offset
  210. assert_nil @chars.rindex('u')
  211. assert_equal 1, @chars.rindex('に')
  212. assert_equal 2, @chars.rindex('ち', -2)
  213. assert_nil @chars.rindex('ち', -3)
  214. assert_equal 6, 'Café périferôl'.mb_chars.rindex('é')
  215. assert_equal 13, 'Café périferôl'.mb_chars.rindex(/\w/u)
  216. end
  217. def test_indexed_insert_should_take_character_offsets
  218. @chars[2] = 'a'
  219. assert_equal 'こにaわ', @chars
  220. @chars[2] = 'ηη'
  221. assert_equal 'こにηηわ', @chars
  222. @chars[3, 2] = 'λλλ'
  223. assert_equal 'こにηλλλ', @chars
  224. @chars[1, 0] = "λ"
  225. assert_equal 'こλにηλλλ', @chars
  226. @chars[4..6] = "ηη"
  227. assert_equal 'こλにηηη', @chars
  228. @chars[/ηη/] = "λλλ"
  229. assert_equal 'こλにλλλη', @chars
  230. @chars[/(λλ)(.)/, 2] = "α"
  231. assert_equal 'こλにλλαη', @chars
  232. @chars["α"] = "¢"
  233. assert_equal 'こλにλλ¢η', @chars
  234. @chars["λλ"] = "ααα"
  235. assert_equal 'こλにααα¢η', @chars
  236. end
  237. def test_indexed_insert_should_raise_on_index_overflow
  238. before = @chars.to_s
  239. assert_raise(IndexError) { @chars[10] = 'a' }
  240. assert_raise(IndexError) { @chars[10, 4] = 'a' }
  241. assert_raise(IndexError) { @chars[/ii/] = 'a' }
  242. assert_raise(IndexError) { @chars[/()/, 10] = 'a' }
  243. assert_equal before, @chars
  244. end
  245. def test_indexed_insert_should_raise_on_range_overflow
  246. before = @chars.to_s
  247. assert_raise(RangeError) { @chars[10..12] = 'a' }
  248. assert_equal before, @chars
  249. end
  250. def test_rjust_should_raise_argument_errors_on_bad_arguments
  251. assert_raise(ArgumentError) { @chars.rjust(10, '') }
  252. assert_raise(ArgumentError) { @chars.rjust }
  253. end
  254. def test_rjust_should_count_characters_instead_of_bytes
  255. assert_equal UNICODE_STRING, @chars.rjust(-3)
  256. assert_equal UNICODE_STRING, @chars.rjust(0)
  257. assert_equal UNICODE_STRING, @chars.rjust(4)
  258. assert_equal " #{UNICODE_STRING}", @chars.rjust(5)
  259. assert_equal " #{UNICODE_STRING}", @chars.rjust(7)
  260. assert_equal "---#{UNICODE_STRING}", @chars.rjust(7, '-')
  261. assert_equal "ααα#{UNICODE_STRING}", @chars.rjust(7, 'α')
  262. assert_equal "aba#{UNICODE_STRING}", @chars.rjust(7, 'ab')
  263. assert_equal "αηα#{UNICODE_STRING}", @chars.rjust(7, 'αη')
  264. assert_equal "αηαη#{UNICODE_STRING}", @chars.rjust(8, 'αη')
  265. end
  266. def test_ljust_should_raise_argument_errors_on_bad_arguments
  267. assert_raise(ArgumentError) { @chars.ljust(10, '') }
  268. assert_raise(ArgumentError) { @chars.ljust }
  269. end
  270. def test_ljust_should_count_characters_instead_of_bytes
  271. assert_equal UNICODE_STRING, @chars.ljust(-3)
  272. assert_equal UNICODE_STRING, @chars.ljust(0)
  273. assert_equal UNICODE_STRING, @chars.ljust(4)
  274. assert_equal "#{UNICODE_STRING} ", @chars.ljust(5)
  275. assert_equal "#{UNICODE_STRING} ", @chars.ljust(7)
  276. assert_equal "#{UNICODE_STRING}---", @chars.ljust(7, '-')
  277. assert_equal "#{UNICODE_STRING}ααα", @chars.ljust(7, 'α')
  278. assert_equal "#{UNICODE_STRING}aba", @chars.ljust(7, 'ab')
  279. assert_equal "#{UNICODE_STRING}αηα", @chars.ljust(7, 'αη')
  280. assert_equal "#{UNICODE_STRING}αηαη", @chars.ljust(8, 'αη')
  281. end
  282. def test_center_should_raise_argument_errors_on_bad_arguments
  283. assert_raise(ArgumentError) { @chars.center(10, '') }
  284. assert_raise(ArgumentError) { @chars.center }
  285. end
  286. def test_center_should_count_characters_instead_of_bytes
  287. assert_equal UNICODE_STRING, @chars.center(-3)
  288. assert_equal UNICODE_STRING, @chars.center(0)
  289. assert_equal UNICODE_STRING, @chars.center(4)
  290. assert_equal "#{UNICODE_STRING} ", @chars.center(5)
  291. assert_equal " #{UNICODE_STRING} ", @chars.center(6)
  292. assert_equal " #{UNICODE_STRING} ", @chars.center(7)
  293. assert_equal "--#{UNICODE_STRING}--", @chars.center(8, '-')
  294. assert_equal "--#{UNICODE_STRING}---", @chars.center(9, '-')
  295. assert_equal "αα#{UNICODE_STRING}αα", @chars.center(8, 'α')
  296. assert_equal "αα#{UNICODE_STRING}ααα", @chars.center(9, 'α')
  297. assert_equal "a#{UNICODE_STRING}ab", @chars.center(7, 'ab')
  298. assert_equal "ab#{UNICODE_STRING}ab", @chars.center(8, 'ab')
  299. assert_equal "abab#{UNICODE_STRING}abab", @chars.center(12, 'ab')
  300. assert_equal "α#{UNICODE_STRING}αη", @chars.center(7, 'αη')
  301. assert_equal "αη#{UNICODE_STRING}αη", @chars.center(8, 'αη')
  302. end
  303. def test_lstrip_strips_whitespace_from_the_left_of_the_string
  304. assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.lstrip
  305. assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.lstrip
  306. assert_equal UNICODE_STRING + @whitespace, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.lstrip
  307. end
  308. def test_rstrip_strips_whitespace_from_the_right_of_the_string
  309. assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.rstrip
  310. assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.rstrip
  311. assert_equal @whitespace + UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.rstrip
  312. end
  313. def test_strip_strips_whitespace
  314. assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.strip
  315. assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.strip
  316. assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.strip
  317. assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.strip
  318. end
  319. def test_stripping_whitespace_leaves_whitespace_within_the_string_intact
  320. string_with_whitespace = UNICODE_STRING + @whitespace + UNICODE_STRING
  321. assert_equal string_with_whitespace, string_with_whitespace.mb_chars.strip
  322. assert_equal string_with_whitespace, string_with_whitespace.mb_chars.lstrip
  323. assert_equal string_with_whitespace, string_with_whitespace.mb_chars.rstrip
  324. end
  325. def test_size_returns_characters_instead_of_bytes
  326. assert_equal 0, ''.mb_chars.size
  327. assert_equal 4, @chars.size
  328. assert_equal 4, @chars.length
  329. assert_equal 5, ASCII_STRING.mb_chars.size
  330. end
  331. def test_reverse_reverses_characters
  332. assert_equal '', ''.mb_chars.reverse
  333. assert_equal 'わちにこ', @chars.reverse
  334. end
  335. def test_reverse_should_work_with_normalized_strings
  336. str = 'bös'
  337. reversed_str = 'söb'
  338. assert_equal chars(reversed_str).normalize(:kc), chars(str).normalize(:kc).reverse
  339. assert_equal chars(reversed_str).normalize(:c), chars(str).normalize(:c).reverse
  340. assert_equal chars(reversed_str).normalize(:d), chars(str).normalize(:d).reverse
  341. assert_equal chars(reversed_str).normalize(:kd), chars(str).normalize(:kd).reverse
  342. assert_equal chars(reversed_str).decompose, chars(str).decompose.reverse
  343. assert_equal chars(reversed_str).compose, chars(str).compose.reverse
  344. end
  345. def test_slice_should_take_character_offsets
  346. assert_equal nil, ''.mb_chars.slice(0)
  347. assert_equal 'こ', @chars.slice(0)
  348. assert_equal 'わ', @chars.slice(3)
  349. assert_equal nil, ''.mb_chars.slice(-1..1)
  350. assert_equal nil, ''.mb_chars.slice(-1, 1)
  351. assert_equal '', ''.mb_chars.slice(0..10)
  352. assert_equal 'にちわ', @chars.slice(1..3)
  353. assert_equal 'にちわ', @chars.slice(1, 3)
  354. assert_equal 'こ', @chars.slice(0, 1)
  355. assert_equal 'ちわ', @chars.slice(2..10)
  356. assert_equal '', @chars.slice(4..10)
  357. assert_equal 'に', @chars.slice(//u)
  358. assert_equal 'にち', @chars.slice(/./u)
  359. assert_equal nil, @chars.slice(/unknown/u)
  360. assert_equal 'にち', @chars.slice(/(にち)/u, 1)
  361. assert_equal nil, @chars.slice(/(にち)/u, 2)
  362. assert_equal nil, @chars.slice(7..6)
  363. end
  364. def test_slice_bang_returns_sliced_out_substring
  365. assert_equal 'にち', @chars.slice!(1..2)
  366. end
  367. def test_slice_bang_removes_the_slice_from_the_receiver
  368. chars = 'úüù'.mb_chars
  369. chars.slice!(0,2)
  370. assert_equal 'úü', chars
  371. end
  372. def test_slice_should_throw_exceptions_on_invalid_arguments
  373. assert_raise(TypeError) { @chars.slice(2..3, 1) }
  374. assert_raise(TypeError) { @chars.slice(1, 2..3) }
  375. assert_raise(ArgumentError) { @chars.slice(1, 1, 1) }
  376. end
  377. def test_ord_should_return_unicode_value_for_first_character
  378. assert_equal 12371, @chars.ord
  379. end
  380. def test_upcase_should_upcase_ascii_characters
  381. assert_equal '', ''.mb_chars.upcase
  382. assert_equal 'ABC', 'aBc'.mb_chars.upcase
  383. end
  384. def test_downcase_should_downcase_ascii_characters
  385. assert_equal '', ''.mb_chars.downcase
  386. assert_equal 'abc', 'aBc'.mb_chars.downcase
  387. end
  388. def test_capitalize_should_work_on_ascii_characters
  389. assert_equal '', ''.mb_chars.capitalize
  390. assert_equal 'Abc', 'abc'.mb_chars.capitalize
  391. end
  392. def test_titleize_should_work_on_ascii_characters
  393. assert_equal '', ''.mb_chars.titleize
  394. assert_equal 'Abc Abc', 'abc abc'.mb_chars.titleize
  395. end
  396. def test_respond_to_knows_which_methods_the_proxy_responds_to
  397. assert ''.mb_chars.respond_to?(:slice) # Defined on Chars
  398. assert ''.mb_chars.respond_to?(:capitalize!) # Defined on Chars
  399. assert ''.mb_chars.respond_to?(:gsub) # Defined on String
  400. assert !''.mb_chars.respond_to?(:undefined_method) # Not defined
  401. end
  402. def test_acts_like_string
  403. assert 'Bambi'.mb_chars.acts_like_string?
  404. end
  405. end
  406. # The default Multibyte Chars proxy has more features than the normal string implementation. Tests
  407. # for the implementation of these features should run on all Ruby versions and shouldn't be tested
  408. # through the proxy methods.
  409. class MultibyteCharsExtrasTest < Test::Unit::TestCase
  410. include MultibyteTestHelpers
  411. def test_upcase_should_be_unicode_aware
  412. assert_equal "АБВГД\0F", chars("аБвгд\0f").upcase
  413. assert_equal 'こにちわ', chars('こにちわ').upcase
  414. end
  415. def test_downcase_should_be_unicode_aware
  416. assert_equal "абвгд\0f", chars("аБвгд\0f").downcase
  417. assert_equal 'こにちわ', chars('こにちわ').downcase
  418. end
  419. def test_capitalize_should_be_unicode_aware
  420. { 'аБвг аБвг' => 'Абвг абвг',
  421. 'аБвг АБВГ' => 'Абвг абвг',
  422. 'АБВГ АБВГ' => 'Абвг абвг',
  423. '' => '' }.each do |f,t|
  424. assert_equal t, chars(f).capitalize
  425. end
  426. end
  427. def test_titleize_should_be_unicode_aware
  428. assert_equal "Él Que Se Enteró", chars("ÉL QUE SE ENTERÓ").titleize
  429. assert_equal "Абвг Абвг", chars("аБвг аБвг").titleize
  430. end
  431. def test_titleize_should_not_affect_characters_that_do_not_case_fold
  432. assert_equal "日本語", chars("日本語").titleize
  433. end
  434. def test_limit_should_not_break_on_blank_strings
  435. example = chars('')
  436. assert_equal example, example.limit(0)
  437. assert_equal example, example.limit(1)
  438. end
  439. def test_limit_should_work_on_a_multibyte_string
  440. example = chars(UNICODE_STRING)
  441. bytesize = UNICODE_STRING.respond_to?(:bytesize) ? UNICODE_STRING.bytesize : UNICODE_STRING.size
  442. assert_equal UNICODE_STRING, example.limit(bytesize)
  443. assert_equal '', example.limit(0)
  444. assert_equal '', example.limit(1)
  445. assert_equal 'こ', example.limit(3)
  446. assert_equal 'こに', example.limit(6)
  447. assert_equal 'こに', example.limit(8)
  448. assert_equal 'こにち', example.limit(9)
  449. assert_equal 'こにちわ', example.limit(50)
  450. end
  451. def test_limit_should_work_on_an_ascii_string
  452. ascii = chars(ASCII_STRING)
  453. assert_equal ASCII_STRING, ascii.limit(ASCII_STRING.length)
  454. assert_equal '', ascii.limit(0)
  455. assert_equal 'o', ascii.limit(1)
  456. assert_equal 'oh', ascii.limit(2)
  457. assert_equal 'ohay', ascii.limit(4)
  458. assert_equal 'ohayo', ascii.limit(50)
  459. end
  460. def test_limit_should_keep_under_the_specified_byte_limit
  461. example = chars(UNICODE_STRING)
  462. (1..UNICODE_STRING.length).each do |limit|
  463. assert example.limit(limit).to_s.length <= limit
  464. end
  465. end
  466. def test_composition_exclusion_is_set_up_properly
  467. # Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
  468. qa = [0x915, 0x93c].pack('U*')
  469. assert_equal qa, chars(qa).normalize(:c)
  470. end
  471. # Test for the Public Review Issue #29, bad explanation of composition might lead to a
  472. # bad implementation: http://www.unicode.org/review/pr-29.html
  473. def test_normalization_C_pri_29
  474. [
  475. [0x0B47, 0x0300, 0x0B3E],
  476. [0x1100, 0x0300, 0x1161]
  477. ].map { |c| c.pack('U*') }.each do |c|
  478. assert_equal_codepoints c, chars(c).normalize(:c)
  479. end
  480. end
  481. def test_normalization_shouldnt_strip_null_bytes
  482. null_byte_str = "Test\0test"
  483. assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
  484. assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
  485. assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
  486. assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
  487. assert_equal null_byte_str, chars(null_byte_str).decompose
  488. assert_equal null_byte_str, chars(null_byte_str).compose
  489. end
  490. def test_simple_normalization
  491. comp_str = [
  492. 44, # LATIN CAPITAL LETTER D
  493. 307, # COMBINING DOT ABOVE
  494. 328, # COMBINING OGONEK
  495. 323 # COMBINING DOT BELOW
  496. ].pack("U*")
  497. assert_equal_codepoints '', chars('').normalize
  498. assert_equal_codepoints [44,105,106,328,323].pack("U*"), chars(comp_str).normalize(:kc).to_s
  499. assert_equal_codepoints [44,307,328,323].pack("U*"), chars(comp_str).normalize(:c).to_s
  500. assert_equal_codepoints [44,307,110,780,78,769].pack("U*"), chars(comp_str).normalize(:d).to_s
  501. assert_equal_codepoints [44,105,106,110,780,78,769].pack("U*"), chars(comp_str).normalize(:kd).to_s
  502. end
  503. def test_should_compute_grapheme_length
  504. [
  505. ['', 0],
  506. ['abc', 3],
  507. ['こにちわ', 4],
  508. [[0x0924, 0x094D, 0x0930].pack('U*'), 2],
  509. [%w(cr lf), 1],
  510. [%w(l l), 1],
  511. [%w(l v), 1],
  512. [%w(l lv), 1],
  513. [%w(l lvt), 1],
  514. [%w(lv v), 1],
  515. [%w(lv t), 1],
  516. [%w(v v), 1],
  517. [%w(v t), 1],
  518. [%w(lvt t), 1],
  519. [%w(t t), 1],
  520. [%w(n extend), 1],
  521. [%w(n n), 2],
  522. [%w(n cr lf n), 3],
  523. [%w(n l v t), 2]
  524. ].each do |input, expected_length|
  525. if input.kind_of?(Array)
  526. str = string_from_classes(input)
  527. else
  528. str = input
  529. end
  530. assert_equal expected_length, chars(str).g_length
  531. end
  532. end
  533. def test_tidy_bytes_should_tidy_bytes
  534. single_byte_cases = {
  535. "\x21" => "!", # Valid ASCII byte, low
  536. "\x41" => "A", # Valid ASCII byte, mid
  537. "\x7E" => "~", # Valid ASCII byte, high
  538. "\x80" => "€", # Continuation byte, low (cp125)
  539. "\x94" => "”", # Continuation byte, mid (cp125)
  540. "\x9F" => "Ÿ", # Continuation byte, high (cp125)
  541. "\xC0" => "À", # Overlong encoding, start of 2-byte sequence, but codepoint < 128
  542. "\xC1" => "Á", # Overlong encoding, start of 2-byte sequence, but codepoint < 128
  543. "\xC2" => "Â", # Start of 2-byte sequence, low
  544. "\xC8" => "È", # Start of 2-byte sequence, mid
  545. "\xDF" => "ß", # Start of 2-byte sequence, high
  546. "\xE0" => "à", # Start of 3-byte sequence, low
  547. "\xE8" => "è", # Start of 3-byte sequence, mid
  548. "\xEF" => "ï", # Start of 3-byte sequence, high
  549. "\xF0" => "ð", # Start of 4-byte sequence
  550. "\xF1" => "ñ", # Unused byte
  551. "\xFF" => "ÿ", # Restricted byte
  552. "\x00" => "\x00" # null char
  553. }
  554. single_byte_cases.each do |bad, good|
  555. assert_equal good, chars(bad).tidy_bytes.to_s
  556. assert_equal "#{good}#{good}", chars("#{bad}#{bad}").tidy_bytes
  557. assert_equal "#{good}#{good}#{good}", chars("#{bad}#{bad}#{bad}").tidy_bytes
  558. assert_equal "#{good}a", chars("#{bad}a").tidy_bytes
  559. assert_equal "#{good}á", chars("#{bad}á").tidy_bytes
  560. assert_equal "a#{good}a", chars("a#{bad}a").tidy_bytes
  561. assert_equal "á#{good}á", chars("á#{bad}á").tidy_bytes
  562. assert_equal "a#{good}", chars("a#{bad}").tidy_bytes
  563. assert_equal "á#{good}", chars("á#{bad}").tidy_bytes
  564. end
  565. byte_string = "\270\236\010\210\245"
  566. tidy_string = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].pack('U*')
  567. assert_equal_codepoints tidy_string, chars(byte_string).tidy_bytes
  568. assert_nothing_raised { chars(byte_string).tidy_bytes.to_s.unpack('U*') }
  569. # UTF-8 leading byte followed by too few continuation bytes
  570. assert_equal_codepoints "\xc3\xb0\xc2\xa5\xc2\xa4\x21", chars("\xf0\xa5\xa4\x21").tidy_bytes
  571. end
  572. def test_tidy_bytes_should_forcibly_tidy_bytes_if_specified
  573. byte_string = "\xF0\xA5\xA4\xA4" # valid as both CP-1252 and UTF-8, but with different interpretations.
  574. assert_not_equal "𥤤", chars(byte_string).tidy_bytes
  575. # Forcible conversion to UTF-8
  576. assert_equal "𥤤", chars(byte_string).tidy_bytes(true)
  577. end
  578. private
  579. def string_from_classes(classes)
  580. # Characters from the character classes as described in UAX #29
  581. character_from_class = {
  582. :l => 0x1100, :v => 0x1160, :t => 0x11A8, :lv => 0xAC00, :lvt => 0xAC01, :cr => 0x000D, :lf => 0x000A,
  583. :extend => 0x094D, :n => 0x64
  584. }
  585. classes.collect do |k|
  586. character_from_class[k.intern]
  587. end.pack('U*')
  588. end
  589. end
  590. class MultibyteInternalsTest < ActiveSupport::TestCase
  591. include MultibyteTestHelpers
  592. test "Chars translates a character offset to a byte offset" do
  593. example = chars("Puisque c'était son erreur, il m'a aidé")
  594. [
  595. [0, 0],
  596. [3, 3],
  597. [12, 11],
  598. [14, 13],
  599. [41, 39]
  600. ].each do |byte_offset, character_offset|
  601. assert_equal character_offset, example.send(:translate_offset, byte_offset),
  602. "Expected byte offset #{byte_offset} to translate to #{character_offset}"
  603. end
  604. end
  605. end