/vendor/gems/ferret-0.11.4/test/unit/analysis/tc_token_stream.rb

https://github.com/ekcell/lovdbyless · Ruby · 606 lines · 589 code · 17 blank · 0 comment · 4 complexity · 6e8a6ff07ce01e7695eb13537add0ad7 MD5 · raw file 

    

  1. require File.dirname(__FILE__) + "/../../test_helper"
    2. 

  2. 

  3. puts "Loading once"
    4. 

  4. class TokenTest < Test::Unit::TestCase
    5. 

  5.   include Ferret::Analysis
    6. 

  6.   def test_token
    7. 

  7.     t = Token.new("text", 1, 2, 3)
    8. 

  8.     assert_equal("text", t.text)
    9. 

  9.     assert_equal(1, t.start)
    10. 

  10.     assert_equal(2, t.end)
    11. 

  11.     assert_equal(3, t.pos_inc)
    12. 

  12.     t.text    = "yada yada yada"
    13. 

  13.     t.start   = 11
    14. 

  14.     t.end     = 12
    15. 

  15.     t.pos_inc = 13
    16. 

  16.     assert_equal("yada yada yada", t.text)
    17. 

  17.     assert_equal(11, t.start)
    18. 

  18.     assert_equal(12, t.end)
    19. 

  19.     assert_equal(13, t.pos_inc)
    20. 

  20. 

  21.     t = Token.new("text", 1, 2)
    22. 

  22.     assert_equal(1, t.pos_inc)
    23. 

  23.   end
    24. 

  24. end
    25. 

  25. 

  26. class AsciiLetterTokenizerTest < Test::Unit::TestCase
    27. 

  27.   include Ferret::Analysis
    28. 

  28. 

  29.   def test_letter_tokenizer()
    30. 

  30.     input = 'DBalmain@gmail.com is My e-mail 523@#$ ADDRESS. 23#@$'
    31. 

  31.     t = AsciiLetterTokenizer.new(input)
    32. 

  32.     assert_equal(Token.new("DBalmain", 0, 8), t.next())
    33. 

  33.     assert_equal(Token.new("gmail", 9, 14), t.next())
    34. 

  34.     assert_equal(Token.new("com", 15, 18), t.next())
    35. 

  35.     assert_equal(Token.new("is", 19, 21), t.next())
    36. 

  36.     assert_equal(Token.new("My", 22, 24), t.next())
    37. 

  37.     assert_equal(Token.new("e", 25, 26), t.next())
    38. 

  38.     assert_equal(Token.new("mail", 27, 31), t.next())
    39. 

  39.     assert_equal(Token.new("ADDRESS", 39, 46), t.next())
    40. 

  40.     assert(! t.next())
    41. 

  41.     t.text = "one_two three"
    42. 

  42.     assert_equal(Token.new("one", 0, 3), t.next())
    43. 

  43.     assert_equal(Token.new("two", 4, 7), t.next())
    44. 

  44.     assert_equal(Token.new("three", 8, 13), t.next())
    45. 

  45.     assert(! t.next())
    46. 

  46.     t = AsciiLowerCaseFilter.new(AsciiLetterTokenizer.new(input))
    47. 

  47.     assert_equal(Token.new("dbalmain", 0, 8), t.next())
    48. 

  48.     assert_equal(Token.new("gmail", 9, 14), t.next())
    49. 

  49.     assert_equal(Token.new("com", 15, 18), t.next())
    50. 

  50.     assert_equal(Token.new("is", 19, 21), t.next())
    51. 

  51.     assert_equal(Token.new("my", 22, 24), t.next())
    52. 

  52.     assert_equal(Token.new("e", 25, 26), t.next())
    53. 

  53.     assert_equal(Token.new("mail", 27, 31), t.next())
    54. 

  54.     assert_equal(Token.new("address", 39, 46), t.next())
    55. 

  55.     assert(! t.next())
    56. 

  56.   end
    57. 

  57. end
    58. 

  58. 

  59. class LetterTokenizerTest < Test::Unit::TestCase
    60. 

  60.   include Ferret::Analysis
    61. 

  61. 

  62.   def test_letter_tokenizer()
    63. 

  63.     input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
    64. 

  64.     t = LetterTokenizer.new(input)
    65. 

  65.     assert_equal(Token.new('DBalmän', 0, 8), t.next)
    66. 

  66.     assert_equal(Token.new('gmail', 9, 14), t.next)
    67. 

  67.     assert_equal(Token.new('com', 15, 18), t.next)
    68. 

  68.     assert_equal(Token.new('is', 19, 21), t.next)
    69. 

  69.     assert_equal(Token.new('My', 22, 24), t.next)
    70. 

  70.     assert_equal(Token.new('e', 25, 26), t.next)
    71. 

  71.     assert_equal(Token.new('mail', 27, 31), t.next)
    72. 

  72.     assert_equal(Token.new('address', 40, 47), t.next)
    73. 

  73.     assert_equal(Token.new('ÁÄGÇ', 55, 62), t.next)
    74. 

  74.     assert_equal(Token.new('ÊËÌ', 64, 70), t.next)
    75. 

  75.     assert_equal(Token.new('ÚØÃ', 72, 78), t.next)
    76. 

  76.     assert_equal(Token.new('ÖÎÍ', 80, 86), t.next)
    77. 

  77.     assert(! t.next())
    78. 

  78.     t.text = "one_two three"
    79. 

  79.     assert_equal(Token.new("one", 0, 3), t.next())
    80. 

  80.     assert_equal(Token.new("two", 4, 7), t.next())
    81. 

  81.     assert_equal(Token.new("three", 8, 13), t.next())
    82. 

  82.     assert(! t.next())
    83. 

  83.     t = LowerCaseFilter.new(LetterTokenizer.new(input))
    84. 

  84.     assert_equal(Token.new('dbalmän', 0, 8), t.next)
    85. 

  85.     assert_equal(Token.new('gmail', 9, 14), t.next)
    86. 

  86.     assert_equal(Token.new('com', 15, 18), t.next)
    87. 

  87.     assert_equal(Token.new('is', 19, 21), t.next)
    88. 

  88.     assert_equal(Token.new('my', 22, 24), t.next)
    89. 

  89.     assert_equal(Token.new('e', 25, 26), t.next)
    90. 

  90.     assert_equal(Token.new('mail', 27, 31), t.next)
    91. 

  91.     assert_equal(Token.new('address', 40, 47), t.next)
    92. 

  92.     assert_equal(Token.new('áägç', 55, 62), t.next)
    93. 

  93.     assert_equal(Token.new('êëì', 64, 70), t.next)
    94. 

  94.     assert_equal(Token.new('úøã', 72, 78), t.next)
    95. 

  95.     assert_equal(Token.new('öîí', 80, 86), t.next)
    96. 

  96.     assert(! t.next())
    97. 

  97.     t = LetterTokenizer.new(input, true)
    98. 

  98.     assert_equal(Token.new('dbalmän', 0, 8), t.next)
    99. 

  99.     assert_equal(Token.new('gmail', 9, 14), t.next)
    100. 

  100.     assert_equal(Token.new('com', 15, 18), t.next)
    101. 

  101.     assert_equal(Token.new('is', 19, 21), t.next)
    102. 

  102.     assert_equal(Token.new('my', 22, 24), t.next)
    103. 

  103.     assert_equal(Token.new('e', 25, 26), t.next)
    104. 

  104.     assert_equal(Token.new('mail', 27, 31), t.next)
    105. 

  105.     assert_equal(Token.new('address', 40, 47), t.next)
    106. 

  106.     assert_equal(Token.new('áägç', 55, 62), t.next)
    107. 

  107.     assert_equal(Token.new('êëì', 64, 70), t.next)
    108. 

  108.     assert_equal(Token.new('úøã', 72, 78), t.next)
    109. 

  109.     assert_equal(Token.new('öîí', 80, 86), t.next)
    110. 

  110.     assert(! t.next())
    111. 

  111.   end
    112. 

  112. end if (/utf-8/i =~ Ferret.locale)
    113. 

  113. 

  114. class AsciiWhiteSpaceTokenizerTest < Test::Unit::TestCase
    115. 

  115.   include Ferret::Analysis
    116. 

  116. 

  117.   def test_whitespace_tokenizer()
    118. 

  118.     input = 'DBalmain@gmail.com is My e-mail 52   #$ ADDRESS. 23#@$'
    119. 

  119.     t = AsciiWhiteSpaceTokenizer.new(input)
    120. 

  120.     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
    121. 

  121.     assert_equal(Token.new('is', 19, 21), t.next)
    122. 

  122.     assert_equal(Token.new('My', 22, 24), t.next)
    123. 

  123.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    124. 

  124.     assert_equal(Token.new('52', 32, 34), t.next)
    125. 

  125.     assert_equal(Token.new('#$', 37, 39), t.next)
    126. 

  126.     assert_equal(Token.new('ADDRESS.', 40, 48), t.next)
    127. 

  127.     assert_equal(Token.new('23#@$', 49, 54), t.next)
    128. 

  128.     assert(! t.next())
    129. 

  129.     t.text = "one_two three"
    130. 

  130.     assert_equal(Token.new("one_two", 0, 7), t.next())
    131. 

  131.     assert_equal(Token.new("three", 8, 13), t.next())
    132. 

  132.     assert(! t.next())
    133. 

  133.     t = AsciiLowerCaseFilter.new(AsciiWhiteSpaceTokenizer.new(input))
    134. 

  134.     assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
    135. 

  135.     assert_equal(Token.new('is', 19, 21), t.next)
    136. 

  136.     assert_equal(Token.new('my', 22, 24), t.next)
    137. 

  137.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    138. 

  138.     assert_equal(Token.new('52', 32, 34), t.next)
    139. 

  139.     assert_equal(Token.new('#$', 37, 39), t.next)
    140. 

  140.     assert_equal(Token.new('address.', 40, 48), t.next)
    141. 

  141.     assert_equal(Token.new('23#@$', 49, 54), t.next)
    142. 

  142.     assert(! t.next())
    143. 

  143.   end
    144. 

  144. end
    145. 

  145. 

  146. class WhiteSpaceTokenizerTest < Test::Unit::TestCase
    147. 

  147.   include Ferret::Analysis
    148. 

  148. 

  149.   def test_whitespace_tokenizer()
    150. 

  150.     input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
    151. 

  151.     t = WhiteSpaceTokenizer.new(input)
    152. 

  152.     assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t.next)
    153. 

  153.     assert_equal(Token.new('is', 19, 21), t.next)
    154. 

  154.     assert_equal(Token.new('My', 22, 24), t.next)
    155. 

  155.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    156. 

  156.     assert_equal(Token.new('52', 32, 34), t.next)
    157. 

  157.     assert_equal(Token.new('#$', 37, 39), t.next)
    158. 

  158.     assert_equal(Token.new('address.', 40, 48), t.next)
    159. 

  159.     assert_equal(Token.new('23#@$', 49, 54), t.next)
    160. 

  160.     assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t.next)
    161. 

  161.     assert(! t.next())
    162. 

  162.     t.text = "one_two three"
    163. 

  163.     assert_equal(Token.new("one_two", 0, 7), t.next())
    164. 

  164.     assert_equal(Token.new("three", 8, 13), t.next())
    165. 

  165.     assert(! t.next())
    166. 

  166.     t = LowerCaseFilter.new(WhiteSpaceTokenizer.new(input))
    167. 

  167.     assert_equal(Token.new('dbalmän@gmail.com', 0, 18), t.next)
    168. 

  168.     assert_equal(Token.new('is', 19, 21), t.next)
    169. 

  169.     assert_equal(Token.new('my', 22, 24), t.next)
    170. 

  170.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    171. 

  171.     assert_equal(Token.new('52', 32, 34), t.next)
    172. 

  172.     assert_equal(Token.new('#$', 37, 39), t.next)
    173. 

  173.     assert_equal(Token.new('address.', 40, 48), t.next)
    174. 

  174.     assert_equal(Token.new('23#@$', 49, 54), t.next)
    175. 

  175.     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
    176. 

  176.     assert(! t.next())
    177. 

  177.     t = WhiteSpaceTokenizer.new(input, true)
    178. 

  178.     assert_equal(Token.new('dbalmän@gmail.com', 0, 18), t.next)
    179. 

  179.     assert_equal(Token.new('is', 19, 21), t.next)
    180. 

  180.     assert_equal(Token.new('my', 22, 24), t.next)
    181. 

  181.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    182. 

  182.     assert_equal(Token.new('52', 32, 34), t.next)
    183. 

  183.     assert_equal(Token.new('#$', 37, 39), t.next)
    184. 

  184.     assert_equal(Token.new('address.', 40, 48), t.next)
    185. 

  185.     assert_equal(Token.new('23#@$', 49, 54), t.next)
    186. 

  186.     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
    187. 

  187.     assert(! t.next())
    188. 

  188.   end
    189. 

  189. end if (/utf-8/i =~ Ferret.locale)
    190. 

  190. 

  191. class AsciiStandardTokenizerTest < Test::Unit::TestCase
    192. 

  192.   include Ferret::Analysis
    193. 

  193. 

  194.   def test_standard_tokenizer()
    195. 

  195.     input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
    196. 

  196.     t = AsciiStandardTokenizer.new(input)
    197. 

  197.     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
    198. 

  198.     assert_equal(Token.new('is', 19, 21), t.next)
    199. 

  199.     assert_equal(Token.new('My', 22, 24), t.next)
    200. 

  200.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    201. 

  201.     assert_equal(Token.new('52', 32, 34), t.next)
    202. 

  202.     assert_equal(Token.new('Address', 40, 47), t.next)
    203. 

  203.     assert_equal(Token.new('23', 49, 51), t.next)
    204. 

  204.     assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
    205. 

  205.     assert_equal(Token.new('TNT', 86, 91), t.next)
    206. 

  206.     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
    207. 

  207.     assert(! t.next())
    208. 

  208.     t.text = "one_two three"
    209. 

  209.     assert_equal(Token.new("one_two", 0, 7), t.next())
    210. 

  210.     assert_equal(Token.new("three", 8, 13), t.next())
    211. 

  211.     assert(! t.next())
    212. 

  212.     t = AsciiLowerCaseFilter.new(AsciiStandardTokenizer.new(input))
    213. 

  213.     assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
    214. 

  214.     assert_equal(Token.new('is', 19, 21), t.next)
    215. 

  215.     assert_equal(Token.new('my', 22, 24), t.next)
    216. 

  216.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    217. 

  217.     assert_equal(Token.new('52', 32, 34), t.next)
    218. 

  218.     assert_equal(Token.new('address', 40, 47), t.next)
    219. 

  219.     assert_equal(Token.new('23', 49, 51), t.next)
    220. 

  220.     assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
    221. 

  221.     assert_equal(Token.new('tnt', 86, 91), t.next)
    222. 

  222.     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
    223. 

  223.     assert(! t.next())
    224. 

  224.   end
    225. 

  225. end
    226. 

  226. 

  227. class StandardTokenizerTest < Test::Unit::TestCase
    228. 

  228.   include Ferret::Analysis
    229. 

  229. 

  230.   def test_standard_tokenizer()
    231. 

  231.     input = 'DBalmán@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/res_345/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
    232. 

  232.     t = StandardTokenizer.new(input)
    233. 

  233.     assert_equal(Token.new('DBalmán@gmail.com', 0, 18), t.next)
    234. 

  234.     assert_equal(Token.new('is', 19, 21), t.next)
    235. 

  235.     assert_equal(Token.new('My', 22, 24), t.next)
    236. 

  236.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    237. 

  237.     assert_equal(Token.new('52', 32, 34), t.next)
    238. 

  238.     assert_equal(Token.new('Address', 40, 47), t.next)
    239. 

  239.     assert_equal(Token.new('23', 49, 51), t.next)
    240. 

  240.     assert_equal(Token.new('www.google.com/res_345', 55, 84), t.next)
    241. 

  241.     assert_equal(Token.new('TNT', 86, 91), t.next)
    242. 

  242.     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
    243. 

  243.     assert_equal(Token.new('23', 111, 113), t.next)
    244. 

  244.     assert_equal(Token.new('ÁÄGÇ', 117, 124), t.next)
    245. 

  245.     assert_equal(Token.new('ÊËÌ', 126, 132), t.next)
    246. 

  246.     assert_equal(Token.new('ÚØÃ', 134, 140), t.next)
    247. 

  247.     assert_equal(Token.new('ÖÎÍ', 142, 148), t.next)
    248. 

  248.     assert(! t.next())
    249. 

  249.     t.text = "one_two three"
    250. 

  250.     assert_equal(Token.new("one_two", 0, 7), t.next())
    251. 

  251.     assert_equal(Token.new("three", 8, 13), t.next())
    252. 

  252.     assert(! t.next())
    253. 

  253.     t = LowerCaseFilter.new(StandardTokenizer.new(input))
    254. 

  254.     assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
    255. 

  255.     assert_equal(Token.new('is', 19, 21), t.next)
    256. 

  256.     assert_equal(Token.new('my', 22, 24), t.next)
    257. 

  257.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    258. 

  258.     assert_equal(Token.new('52', 32, 34), t.next)
    259. 

  259.     assert_equal(Token.new('address', 40, 47), t.next)
    260. 

  260.     assert_equal(Token.new('23', 49, 51), t.next)
    261. 

  261.     assert_equal(Token.new('www.google.com/res_345', 55, 84), t.next)
    262. 

  262.     assert_equal(Token.new('tnt', 86, 91), t.next)
    263. 

  263.     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
    264. 

  264.     assert_equal(Token.new('23', 111, 113), t.next)
    265. 

  265.     assert_equal(Token.new('áägç', 117, 124), t.next)
    266. 

  266.     assert_equal(Token.new('êëì', 126, 132), t.next)
    267. 

  267.     assert_equal(Token.new('úøã', 134, 140), t.next)
    268. 

  268.     assert_equal(Token.new('öîí', 142, 148), t.next)
    269. 

  269.     input = "e-mail 123-1235-asd-1234 http://www.davebalmain.com/trac-site/"
    270. 

  270.     t = HyphenFilter.new(StandardTokenizer.new(input))
    271. 

  271.     assert_equal(Token.new('email', 0, 6), t.next)
    272. 

  272.     assert_equal(Token.new('e', 0, 1, 0), t.next)
    273. 

  273.     assert_equal(Token.new('mail', 2, 6, 1), t.next)
    274. 

  274.     assert_equal(Token.new('123-1235-asd-1234', 7, 24), t.next)
    275. 

  275.     assert_equal(Token.new('www.davebalmain.com/trac-site', 25, 61), t.next)
    276. 

  276.     assert(! t.next())
    277. 

  277.   end
    278. 

  278. end if (/utf-8/i =~ Ferret.locale)
    279. 

  279. 

  280. class RegExpTokenizerTest < Test::Unit::TestCase
    281. 

  281.   include Ferret::Analysis
    282. 

  282. 

  283.   ALPHA      = /[[:alpha:]_-]+/
    284. 

  284.   APOSTROPHE = /#{ALPHA}('#{ALPHA})+/
    285. 

  285.   ACRONYM    = /#{ALPHA}\.(#{ALPHA}\.)+/
    286. 

  286.   ACRONYM_WORD    = /^#{ACRONYM}$/
    287. 

  287.   APOSTROPHE_WORD = /^#{APOSTROPHE}$/
    288. 

  288. 

  289.   def test_reg_exp_tokenizer()
    290. 

  290.     input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
    291. 

  291.     t = RegExpTokenizer.new(input)
    292. 

  292.     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
    293. 

  293.     assert_equal(Token.new('is', 19, 21), t.next)
    294. 

  294.     assert_equal(Token.new('My', 22, 24), t.next)
    295. 

  295.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    296. 

  296.     assert_equal(Token.new('52', 32, 34), t.next)
    297. 

  297.     assert_equal(Token.new('Address', 40, 47), t.next)
    298. 

  298.     assert_equal(Token.new('23', 49, 51), t.next)
    299. 

  299.     assert_equal(Token.new('http://www.google.com/RESULT_3.html', 55, 90), t.next)
    300. 

  300.     assert_equal(Token.new('T.N.T.', 91, 97), t.next)
    301. 

  301.     assert_equal(Token.new('123-1235-ASD-1234', 98, 115), t.next)
    302. 

  302.     assert_equal(Token.new('23', 116, 118), t.next)
    303. 

  303.     assert_equal(Token.new('Rob\'s', 119, 124), t.next)
    304. 

  304.     assert(! t.next())
    305. 

  305.     t.text = "one_two three"
    306. 

  306.     assert_equal(Token.new("one_two", 0, 7), t.next())
    307. 

  307.     assert_equal(Token.new("three", 8, 13), t.next())
    308. 

  308.     assert(! t.next())
    309. 

  309.     t = LowerCaseFilter.new(RegExpTokenizer.new(input))
    310. 

  310.     t2 = LowerCaseFilter.new(RegExpTokenizer.new(input, /\w{2,}/))
    311. 

  311.     assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
    312. 

  312.     assert_equal(Token.new('is', 19, 21), t.next)
    313. 

  313.     assert_equal(Token.new('my', 22, 24), t.next)
    314. 

  314.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    315. 

  315.     assert_equal(Token.new('52', 32, 34), t.next)
    316. 

  316.     assert_equal(Token.new('address', 40, 47), t.next)
    317. 

  317.     assert_equal(Token.new('23', 49, 51), t.next)
    318. 

  318.     assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
    319. 

  319.     assert_equal(Token.new('t.n.t.', 91, 97), t.next)
    320. 

  320.     assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
    321. 

  321.     assert_equal(Token.new('23', 116, 118), t.next)
    322. 

  322.     assert_equal(Token.new('rob\'s', 119, 124), t.next)
    323. 

  323.     assert(! t.next())
    324. 

  324.     assert_equal(Token.new('dbalmain', 0, 8), t2.next)
    325. 

  325.     assert_equal(Token.new('gmail', 9, 14), t2.next)
    326. 

  326.     assert_equal(Token.new('com', 15, 18), t2.next)
    327. 

  327.     assert_equal(Token.new('is', 19, 21), t2.next)
    328. 

  328.     assert_equal(Token.new('my', 22, 24), t2.next)
    329. 

  329.     assert_equal(Token.new('mail', 27, 31), t2.next)
    330. 

  330.     assert_equal(Token.new('52', 32, 34), t2.next)
    331. 

  331.     assert_equal(Token.new('address', 40, 47), t2.next)
    332. 

  332.     assert_equal(Token.new('23', 49, 51), t2.next)
    333. 

  333.     assert_equal(Token.new('http', 55, 59), t2.next)
    334. 

  334.     assert_equal(Token.new('www', 62, 65), t2.next)
    335. 

  335.     assert_equal(Token.new('google', 66, 72), t2.next)
    336. 

  336.     assert_equal(Token.new('com', 73, 76), t2.next)
    337. 

  337.     assert_equal(Token.new('result_3', 77, 85), t2.next)
    338. 

  338.     assert_equal(Token.new('html', 86, 90), t2.next)
    339. 

  339.     assert_equal(Token.new('123', 98, 101), t2.next)
    340. 

  340.     assert_equal(Token.new('1235', 102, 106), t2.next)
    341. 

  341.     assert_equal(Token.new('asd', 107, 110), t2.next)
    342. 

  342.     assert_equal(Token.new('1234', 111, 115), t2.next)
    343. 

  343.     assert_equal(Token.new('23', 116, 118), t2.next)
    344. 

  344.     assert_equal(Token.new('rob', 119, 122), t2.next)
    345. 

  345.     assert(! t2.next())
    346. 

  346.     t = RegExpTokenizer.new(input) do |str|
    347. 

  347.       if str =~ ACRONYM_WORD
    348. 

  348.         str.gsub!(/\./, '')
    349. 

  349.       elsif str =~ APOSTROPHE_WORD
    350. 

  350.         str.gsub!(/'[sS]$/, '')
    351. 

  351.       end
    352. 

  352.       str
    353. 

  353.     end
    354. 

  354.     t = LowerCaseFilter.new(t)
    355. 

  355.     assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
    356. 

  356.     assert_equal(Token.new('is', 19, 21), t.next)
    357. 

  357.     assert_equal(Token.new('my', 22, 24), t.next)
    358. 

  358.     assert_equal(Token.new('e-mail', 25, 31), t.next)
    359. 

  359.     assert_equal(Token.new('52', 32, 34), t.next)
    360. 

  360.     assert_equal(Token.new('address', 40, 47), t.next)
    361. 

  361.     assert_equal(Token.new('23', 49, 51), t.next)
    362. 

  362.     assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
    363. 

  363.     assert_equal(Token.new('tnt', 91, 97), t.next)
    364. 

  364.     assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
    365. 

  365.     assert_equal(Token.new('23', 116, 118), t.next)
    366. 

  366.     assert_equal(Token.new('rob', 119, 124), t.next)
    367. 

  367.     assert(! t.next())
    368. 

  368.   end
    369. 

  369. end
    370. 

  370. 

  371. class MappingFilterTest < Test::Unit::TestCase
    372. 

  372.   include Ferret::Analysis
    373. 

  373. 

  374.   def test_mapping_filter()
    375. 

  375.     mapping = {
    376. 

  376.       ['à','á','â','ã','ä','å','ā','ă']         => 'a',
    377. 

  377.       'æ'                                       => 'ae',
    378. 

  378.       ['ď','đ']                                 => 'd',
    379. 

  379.       ['ç','ć','č','ĉ','ċ']                     => 'c',
    380. 

  380.       ['è','é','ê','ë','ē','ę','ě','ĕ','ė',]    => 'e',
    381. 

  381.       ['ƒ']                                     => 'f',
    382. 

  382.       ['ĝ','ğ','ġ','ģ']                         => 'g',
    383. 

  383.       ['ĥ','ħ']                                 => 'h',
    384. 

  384.       ['ì','ì','í','î','ï','ī','ĩ','ĭ']         => 'i',
    385. 

  385.       ['į','ı','ij','ĵ']                         => 'j',
    386. 

  386.       ['ķ','ĸ']                                 => 'k',
    387. 

  387.       ['ł','ľ','ĺ','ļ','ŀ']                     => 'l',
    388. 

  388.       ['ñ','ń','ň','ņ','ʼn','ŋ']                 => 'n',
    389. 

  389.       ['ò','ó','ô','õ','ö','ø','ō','ő','ŏ','ŏ'] => 'o',
    390. 

  390.       'œ'                                       => 'oek',
    391. 

  391.       'ą'                                       => 'q',
    392. 

  392.       ['ŕ','ř','ŗ']                             => 'r',
    393. 

  393.       ['ś','š','ş','ŝ','ș']                     => 's',
    394. 

  394.       ['ť','ţ','ŧ','ț']                         => 't',
    395. 

  395.       ['ù','ú','û','ü','ū','ů','ű','ŭ','ũ','ų'] => 'u',
    396. 

  396.       'ŵ'                                       => 'w',
    397. 

  397.       ['ý','ÿ','ŷ']                             => 'y',
    398. 

  398.       ['ž','ż','ź']                             => 'z'
    399. 

  399.     }
    400. 

  400.     input = <<END
    401. 

  401. aàáâãäåāăb cæd eďđf gçćčĉċh ièéêëēęěĕėj kƒl mĝğġģn oĥħp qììíîïīĩĭr sįıijĵt uķĸv
    402. 

  402. włľĺļŀx yñńňņʼnŋz aòóôõöøōőŏŏb cœd eąf gŕřŗh iśšşŝșj kťţŧțl mùúûüūůűŭũųn oŵp
    403. 

  403. qýÿŷr sžżźt
    404. 

  404. END
    405. 

  405.     t = MappingFilter.new(LetterTokenizer.new(input), mapping)
    406. 

  406.     assert_equal(Token.new('aaaaaaaaab', 0, 18), t.next)
    407. 

  407.     assert_equal(Token.new('caed', 19, 23), t.next)
    408. 

  408.     assert_equal(Token.new('eddf', 24, 30), t.next)
    409. 

  409.     assert_equal(Token.new('gccccch', 31, 43), t.next)
    410. 

  410.     assert_equal(Token.new('ieeeeeeeeej', 44, 64), t.next)
    411. 

  411.     assert_equal(Token.new('kfl', 65, 69), t.next)
    412. 

  412.     assert_equal(Token.new('mggggn', 70, 80), t.next)
    413. 

  413.     assert_equal(Token.new('ohhp', 81, 87), t.next)
    414. 

  414.     assert_equal(Token.new('qiiiiiiiir', 88, 106), t.next)
    415. 

  415.     assert_equal(Token.new('sjjjjt', 107, 117), t.next)
    416. 

  416.     assert_equal(Token.new('ukkv', 118, 124), t.next)
    417. 

  417.     assert_equal(Token.new('wlllllx', 125, 137), t.next)
    418. 

  418.     assert_equal(Token.new('ynnnnnnz', 138, 152), t.next)
    419. 

  419.     assert_equal(Token.new('aoooooooooob', 153, 175), t.next)
    420. 

  420.     assert_equal(Token.new('coekd', 176, 180), t.next)
    421. 

  421.     assert_equal(Token.new('eqf', 181, 185), t.next)
    422. 

  422.     assert_equal(Token.new('grrrh', 186, 194), t.next)
    423. 

  423.     assert_equal(Token.new('isssssj', 195, 207), t.next)
    424. 

  424.     assert_equal(Token.new('kttttl', 208, 218), t.next)
    425. 

  425.     assert_equal(Token.new('muuuuuuuuuun', 219, 241), t.next)
    426. 

  426.     assert_equal(Token.new('owp', 242, 246), t.next)
    427. 

  427.     assert_equal(Token.new('qyyyr', 247, 255), t.next)
    428. 

  428.     assert_equal(Token.new('szzzt', 256, 264), t.next)
    429. 

  429.     assert(! t.next())
    430. 

  430.   end
    431. 

  431. end if (/utf-8/i =~ Ferret.locale)
    432. 

  432. 

  433. class StopFilterTest < Test::Unit::TestCase
    434. 

  434.   include Ferret::Analysis
    435. 

  435. 

  436.   def test_stop_filter()
    437. 

  437.     words = ["one", "four", "five", "seven"]
    438. 

  438.     input = "one, two, three, four, five, six, seven, eight, nine, ten."
    439. 

  439.     t = StopFilter.new(AsciiLetterTokenizer.new(input), words)
    440. 

  440.     assert_equal(Token.new('two', 5, 8, 2), t.next)
    441. 

  441.     assert_equal(Token.new('three', 10, 15, 1), t.next)
    442. 

  442.     assert_equal(Token.new('six', 29, 32, 3), t.next)
    443. 

  443.     assert_equal(Token.new('eight', 41, 46, 2), t.next)
    444. 

  444.     assert_equal(Token.new('nine', 48, 52, 1), t.next)
    445. 

  445.     assert_equal(Token.new('ten', 54, 57, 1), t.next)
    446. 

  446.     assert(! t.next())
    447. 

  447.   end
    448. 

  448. end
    449. 

  449. 

  450. class StemFilterTest < Test::Unit::TestCase
    451. 

  451.   include Ferret::Analysis
    452. 

  452. 

  453.   def test_stop_filter()
    454. 

  454.     input = "Debate Debates DEBATED DEBating Debater";
    455. 

  455.     t = StemFilter.new(AsciiLowerCaseFilter.new(AsciiLetterTokenizer.new(input)),
    456. 

  456.                        "english")
    457. 

  457.     assert_equal(Token.new("debat", 0, 6), t.next)
    458. 

  458.     assert_equal(Token.new("debat", 7, 14), t.next)
    459. 

  459.     assert_equal(Token.new("debat", 15, 22), t.next)
    460. 

  460.     assert_equal(Token.new("debat", 23, 31), t.next)
    461. 

  461.     assert_equal(Token.new("debat", 32, 39), t.next)
    462. 

  462.     assert(! t.next())
    463. 

  463.     t = StemFilter.new(AsciiLetterTokenizer.new(input), :english)
    464. 

  464.     assert_equal(Token.new("Debat", 0, 6), t.next)
    465. 

  465.     assert_equal(Token.new("Debat", 7, 14), t.next)
    466. 

  466.     assert_equal(Token.new("DEBATED", 15, 22), t.next)
    467. 

  467.     assert_equal(Token.new("DEBate", 23, 31), t.next)
    468. 

  468.     assert_equal(Token.new("Debat", 32, 39), t.next)
    469. 

  469. 

  470.     if Ferret.locale and Ferret.locale.downcase.index("utf")
    471. 

  471.       input = "Dêbate dêbates DÊBATED DÊBATing dêbater";
    472. 

  472.       t = StemFilter.new(LowerCaseFilter.new(LetterTokenizer.new(input)), :english)
    473. 

  473.       assert_equal(Token.new("dêbate", 0, 7), t.next)
    474. 

  474.       assert_equal(Token.new("dêbate", 8, 16), t.next)
    475. 

  475.       assert_equal(Token.new("dêbate", 17, 25), t.next)
    476. 

  476.       assert_equal(Token.new("dêbate", 26, 35), t.next)
    477. 

  477.       assert_equal(Token.new("dêbater", 36, 44), t.next)
    478. 

  478.       t = StemFilter.new(LetterTokenizer.new(input), :english)
    479. 

  479.       assert_equal(Token.new("Dêbate", 0, 7), t.next)
    480. 

  480.       assert_equal(Token.new("dêbate", 8, 16), t.next)
    481. 

  481.       assert_equal(Token.new("DÊBATED", 17, 25), t.next)
    482. 

  482.       assert_equal(Token.new("DÊBATing", 26, 35), t.next)
    483. 

  483.       assert_equal(Token.new("dêbater", 36, 44), t.next)
    484. 

  484.       assert(! t.next())
    485. 

  485.     end
    486. 

  486.   end
    487. 

  487. end
    488. 

  488. 

  489. require 'strscan'
    490. 

  490. module Ferret::Analysis
    491. 

  491. 

  492.   class MyRegExpTokenizer < TokenStream
    493. 

  493. 

  494.     def initialize(input)
    495. 

  495.       @ss = StringScanner.new(input)
    496. 

  496.     end
    497. 

  497. 

  498.     # Returns the next token in the stream, or null at EOS.
    499. 

  499.     def next()
    500. 

  500.       if @ss.scan_until(token_re)
    501. 

  501.         term = @ss.matched
    502. 

  502.         term_end = @ss.pos
    503. 

  503.         term_start = term_end - term.size
    504. 

  504.       else
    505. 

  505.         return nil
    506. 

  506.       end
    507. 

  507. 

  508.       return Token.new(normalize(term), term_start, term_end)
    509. 

  509.     end
    510. 

  510. 

  511.     protected
    512. 

  512.       # returns the regular expression used to find the next token
    513. 

  513.       TOKEN_RE = /[[:alpha:]]+/
    514. 

  514.       def token_re
    515. 

  515.         TOKEN_RE
    516. 

  516.       end
    517. 

  517. 

  518.       # Called on each token to normalize it before it is added to the
    519. 

  519.       # token.  The default implementation does nothing.  Subclasses may
    520. 

  520.       # use this to, e.g., lowercase tokens.
    521. 

  521.       def normalize(str) return str end
    522. 

  522.   end
    523. 

  523. 

  524.   class MyCSVTokenizer < MyRegExpTokenizer
    525. 

  525.     protected
    526. 

  526.       # returns the regular expression used to find the next token
    527. 

  527.       TOKEN_RE = /[^,]+/
    528. 

  528.       def token_re
    529. 

  529.         TOKEN_RE
    530. 

  530.       end
    531. 

  531. 

  532.       # Called on each token to normalize it before it is added to the
    533. 

  533.       # token.  The default implementation does nothing.  Subclasses may
    534. 

  534.       # use this to, e.g., lowercase tokens.
    535. 

  535.       def normalize(str) return str.upcase end
    536. 

  536.   end
    537. 

  537. end
    538. 

  538. 

  539. class CustomTokenizerTest < Test::Unit::TestCase
    540. 

  540.   include Ferret::Analysis
    541. 

  541. 

  542.   def test_custom_tokenizer()
    543. 

  543.     input = "First Field,2nd Field,  P a d d e d  F i e l d  "
    544. 

  544.     t = MyCSVTokenizer.new(input)
    545. 

  545.     assert_equal(Token.new("FIRST FIELD", 0, 11), t.next)
    546. 

  546.     assert_equal(Token.new("2ND FIELD", 12, 21), t.next)
    547. 

  547.     assert_equal(Token.new("  P A D D E D  F I E L D  ", 22, 48), t.next)
    548. 

  548.     assert(! t.next())
    549. 

  549.     t = AsciiLowerCaseFilter.new(MyCSVTokenizer.new(input))
    550. 

  550.     assert_equal(Token.new("first field", 0, 11), t.next)
    551. 

  551.     assert_equal(Token.new("2nd field", 12, 21), t.next)
    552. 

  552.     assert_equal(Token.new("  p a d d e d  f i e l d  ", 22, 48), t.next)
    553. 

  553.     assert(! t.next())
    554. 

  554.   end
    555. 

  555. end
    556. 

  556. 

  557. module Ferret::Analysis
    558. 

  558.   class TokenFilter < TokenStream
    559. 

  559.     protected
    560. 

  560.       # Construct a token stream filtering the given input.
    561. 

  561.       def initialize(input)
    562. 

  562.         @input = input
    563. 

  563.       end
    564. 

  564.   end
    565. 

  565. 

  566.   # Normalizes token text to lower case.
    567. 

  567.   class CapitalizeFilter < TokenFilter
    568. 

  568.     def next()
    569. 

  569.       t = @input.next()
    570. 

  570. 

  571.       return nil if (t.nil?)
    572. 

  572. 

  573.       t.text = t.text.capitalize
    574. 

  574. 

  575.       return t
    576. 

  576.     end
    577. 

  577.   end
    578. 

  578. end
    579. 

  579. 

  580. class CustomFilterTest < Test::Unit::TestCase
    581. 

  581.   include Ferret::Analysis
    582. 

  582. 

  583.   def test_custom_filter()
    584. 

  584.     input = "This text SHOULD be capitalized ... I hope. :-S"
    585. 

  585.     t = CapitalizeFilter.new(AsciiLetterTokenizer.new(input))
    586. 

  586.     assert_equal(Token.new("This", 0, 4), t.next)
    587. 

  587.     assert_equal(Token.new("Text", 5, 9), t.next)
    588. 

  588.     assert_equal(Token.new("Should", 10, 16), t.next)
    589. 

  589.     assert_equal(Token.new("Be", 17, 19), t.next)
    590. 

  590.     assert_equal(Token.new("Capitalized", 20, 31), t.next)
    591. 

  591.     assert_equal(Token.new("I", 36, 37), t.next)
    592. 

  592.     assert_equal(Token.new("Hope", 38, 42), t.next)
    593. 

  593.     assert_equal(Token.new("S", 46, 47), t.next)
    594. 

  594.     assert(! t.next())
    595. 

  595.     t = StemFilter.new(CapitalizeFilter.new(AsciiLetterTokenizer.new(input)))
    596. 

  596.     assert_equal(Token.new("This", 0, 4), t.next)
    597. 

  597.     assert_equal(Token.new("Text", 5, 9), t.next)
    598. 

  598.     assert_equal(Token.new("Should", 10, 16), t.next)
    599. 

  599.     assert_equal(Token.new("Be", 17, 19), t.next)
    600. 

  600.     assert_equal(Token.new("Capit", 20, 31), t.next)
    601. 

  601.     assert_equal(Token.new("I", 36, 37), t.next)
    602. 

  602.     assert_equal(Token.new("Hope", 38, 42), t.next)
    603. 

  603.     assert_equal(Token.new("S", 46, 47), t.next)
    604. 

  604.     assert(! t.next())
    605. 

  605.   end
    606. 

  606. end
    607.