PageRenderTime 44ms CodeModel.GetById 15ms RepoModel.GetById 0ms app.codeStats 0ms

/test/test_es.py

https://gitlab.com/admin-github-cloud/pattern
Python | 256 lines | 253 code | 2 blank | 1 comment | 1 complexity | cdb2580d325151309bea15a52bb1c8dd MD5 | raw file
    

  1. # -*- coding: utf-8 -*-
    2. 

  2. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
    3. 

  3. import unittest
    4. 

  4. import subprocess
    5. 

  5. 

  6. from pattern import es
    7. 

  7. 

  8. try:
    9. 

  9.     PATH = os.path.dirname(os.path.realpath(__file__))
    10. 

  10. except:
    11. 

  11.     PATH = ""
    12. 

  12. 

  13. #---------------------------------------------------------------------------------------------------
    14. 

  14. 

  15. class TestInflection(unittest.TestCase):
    16. 

  16. 

  17.     def setUp(self):
    18. 

  18.         pass
    19. 

  19.     
    20. 

  20.     def test_pluralize(self):
    21. 

  21.         # Assert the accuracy of the pluralization algorithm.
    22. 

  22.         from pattern.db import Datasheet
    23. 

  23.         test = {}
    24. 

  24.         for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
    25. 

  25.             if tag == "n": test.setdefault(lemma, []).append(w)
    26. 

  26.         i, n = 0, 0
    27. 

  27.         for sg, pl in test.items():
    28. 

  28.             pl = sorted(pl, key=len, reverse=True)[0]
    29. 

  29.             if es.pluralize(sg) == pl:
    30. 

  30.                 i += 1
    31. 

  31.             n += 1
    32. 

  32.         self.assertTrue(float(i) / n > 0.77)
    33. 

  33.         print("pattern.es.pluralize()")
    34. 

  34.         
    35. 

  35.     def test_singularize(self):
    36. 

  36.         # Assert the accuracy of the singularization algorithm.
    37. 

  37.         from pattern.db import Datasheet
    38. 

  38.         test = {}
    39. 

  39.         for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
    40. 

  40.             if tag == "n": test.setdefault(lemma, []).append(w)
    41. 

  41.         i, n = 0, 0
    42. 

  42.         for sg, pl in test.items():
    43. 

  43.             pl = sorted(pl, key=len, reverse=True)[0]
    44. 

  44.             if es.singularize(pl) == sg:
    45. 

  45.                 i += 1
    46. 

  46.             n += 1
    47. 

  47.         self.assertTrue(float(i) / n > 0.93)
    48. 

  48.         print("pattern.es.singularize()")
    49. 

  49. 

  50.     def test_attributive(self):
    51. 

  51.         # Assert "alto" => "altos" (masculine, plural), and others.
    52. 

  52.         for lemma, inflected, gender in (
    53. 

  53.           (u"alto",  u"alto",   es.MALE   + es.SINGULAR),
    54. 

  54.           (u"alto",  u"altos",  es.MALE   + es.PLURAL),
    55. 

  55.           (u"alto",  u"alta",   es.FEMALE + es.SINGULAR),
    56. 

  56.           (u"alto",  u"altas",  es.FEMALE + es.PLURAL),
    57. 

  57.           (u"verde", u"verdes", es.MALE   + es.PLURAL),
    58. 

  58.           (u"verde", u"verdes", es.FEMALE + es.PLURAL)):
    59. 

  59.             v = es.attributive(lemma, gender)
    60. 

  60.             self.assertEqual(v, inflected)
    61. 

  61.         print("pattern.es.attributive()")
    62. 

  62.         
    63. 

  63.     def test_predicative(self):
    64. 

  64.         # Assert the accuracy of the predicative algorithm ("horribles" => "horrible").
    65. 

  65.         from pattern.db import Datasheet
    66. 

  66.         test = {}
    67. 

  67.         for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")):
    68. 

  68.             if tag == "j": test.setdefault(lemma, []).append(w)
    69. 

  69.         i, n = 0, 0
    70. 

  70.         for pred, attr in test.items():
    71. 

  71.             attr = sorted(attr, key=len, reverse=True)[0]
    72. 

  72.             if es.predicative(attr) == pred:
    73. 

  73.                 i += 1
    74. 

  74.             n += 1
    75. 

  75.         self.assertTrue(float(i) / n > 0.92)
    76. 

  76.         print("pattern.es.predicative()")
    77. 

  77. 

  78.     def test_find_lemma(self):
    79. 

  79.         # Assert the accuracy of the verb lemmatization algorithm.
    80. 

  80.         i, n = 0, 0
    81. 

  81.         for v1, v2 in es.inflect.verbs.inflections.items():
    82. 

  82.             if es.inflect.verbs.find_lemma(v1) == v2: 
    83. 

  83.                 i += 1
    84. 

  84.             n += 1
    85. 

  85.         self.assertTrue(float(i) / n > 0.80)
    86. 

  86.         print("pattern.es.inflect.verbs.find_lemma()")
    87. 

  87.         
    88. 

  88.     def test_find_lexeme(self):
    89. 

  89.         # Assert the accuracy of the verb conjugation algorithm.
    90. 

  90.         i, n = 0, 0
    91. 

  91.         for v, lexeme1 in es.inflect.verbs.infinitives.items():
    92. 

  92.             lexeme2 = es.inflect.verbs.find_lexeme(v)
    93. 

  93.             for j in range(len(lexeme2)):
    94. 

  94.                 if lexeme1[j] == lexeme2[j]:
    95. 

  95.                     i += 1
    96. 

  96.                 n += 1
    97. 

  97.         self.assertTrue(float(i) / n > 0.85)
    98. 

  98.         print("pattern.es.inflect.verbs.find_lexeme()")
    99. 

  99. 

  100.     def test_conjugate(self):
    101. 

  101.         # Assert different tenses with different conjugations.
    102. 

  102.         for (v1, v2, tense) in (
    103. 

  103.           ("ser", u"ser",        es.INFINITIVE),
    104. 

  104.           ("ser", u"soy",       (es.PRESENT, 1, es.SINGULAR)),
    105. 

  105.           ("ser", u"eres",      (es.PRESENT, 2, es.SINGULAR)),
    106. 

  106.           ("ser", u"es",        (es.PRESENT, 3, es.SINGULAR)),
    107. 

  107.           ("ser", u"somos",     (es.PRESENT, 1, es.PLURAL)),
    108. 

  108.           ("ser", u"sois",      (es.PRESENT, 2, es.PLURAL)),
    109. 

  109.           ("ser", u"son",       (es.PRESENT, 3, es.PLURAL)),
    110. 

  110.           ("ser", u"siendo",    (es.PRESENT + es.PARTICIPLE)),
    111. 

  111.           ("ser", u"sido",      (es.PAST + es.PARTICIPLE)),
    112. 

  112.           ("ser", u"era",       (es.IMPERFECT, 1, es.SINGULAR)),
    113. 

  113.           ("ser", u"eras",      (es.IMPERFECT, 2, es.SINGULAR)),
    114. 

  114.           ("ser", u"era",       (es.IMPERFECT, 3, es.SINGULAR)),
    115. 

  115.           ("ser", u"éramos",    (es.IMPERFECT, 1, es.PLURAL)),
    116. 

  116.           ("ser", u"erais",     (es.IMPERFECT, 2, es.PLURAL)),
    117. 

  117.           ("ser", u"eran",      (es.IMPERFECT, 3, es.PLURAL)),
    118. 

  118.           ("ser", u"fui",       (es.PRETERITE, 1, es.SINGULAR)),
    119. 

  119.           ("ser", u"fuiste",    (es.PRETERITE, 2, es.SINGULAR)),
    120. 

  120.           ("ser", u"fue",       (es.PRETERITE, 3, es.SINGULAR)),
    121. 

  121.           ("ser", u"fuimos",    (es.PRETERITE, 1, es.PLURAL)),
    122. 

  122.           ("ser", u"fuisteis",  (es.PRETERITE, 2, es.PLURAL)),
    123. 

  123.           ("ser", u"fueron",    (es.PRETERITE, 3, es.PLURAL)),
    124. 

  124.           ("ser", u"sería",     (es.CONDITIONAL, 1, es.SINGULAR)),
    125. 

  125.           ("ser", u"serías",    (es.CONDITIONAL, 2, es.SINGULAR)),
    126. 

  126.           ("ser", u"sería",     (es.CONDITIONAL, 3, es.SINGULAR)),
    127. 

  127.           ("ser", u"seríamos",  (es.CONDITIONAL, 1, es.PLURAL)),
    128. 

  128.           ("ser", u"seríais",   (es.CONDITIONAL, 2, es.PLURAL)),
    129. 

  129.           ("ser", u"serían",    (es.CONDITIONAL, 3, es.PLURAL)),
    130. 

  130.           ("ser", u"seré",      (es.FUTURE, 1, es.SINGULAR)),
    131. 

  131.           ("ser", u"serás",     (es.FUTURE, 2, es.SINGULAR)),
    132. 

  132.           ("ser", u"será",      (es.FUTURE, 3, es.SINGULAR)),
    133. 

  133.           ("ser", u"seremos",   (es.FUTURE, 1, es.PLURAL)),
    134. 

  134.           ("ser", u"seréis",    (es.FUTURE, 2, es.PLURAL)),
    135. 

  135.           ("ser", u"serán",     (es.FUTURE, 3, es.PLURAL)),
    136. 

  136.           ("ser", u"sé",        (es.PRESENT, 2, es.SINGULAR, es.IMPERATIVE)),
    137. 

  137.           ("ser", u"sed",       (es.PRESENT, 2, es.PLURAL, es.IMPERATIVE)),
    138. 

  138.           ("ser",  u"sea",      (es.PRESENT, 1, es.SINGULAR, es.SUBJUNCTIVE)),
    139. 

  139.           ("ser",  u"seas",     (es.PRESENT, 2, es.SINGULAR, es.SUBJUNCTIVE)),
    140. 

  140.           ("ser",  u"sea",      (es.PRESENT, 3, es.SINGULAR, es.SUBJUNCTIVE)),
    141. 

  141.           ("ser",  u"seamos",   (es.PRESENT, 1, es.PLURAL, es.SUBJUNCTIVE)),
    142. 

  142.           ("ser",  u"seáis",    (es.PRESENT, 2, es.PLURAL, es.SUBJUNCTIVE)),
    143. 

  143.           ("ser",  u"sean",     (es.PRESENT, 3, es.PLURAL, es.SUBJUNCTIVE)),
    144. 

  144.           ("ser",  u"fuera",    (es.PAST, 1, es.SINGULAR, es.SUBJUNCTIVE)),
    145. 

  145.           ("ser",  u"fueras",   (es.PAST, 2, es.SINGULAR, es.SUBJUNCTIVE)),
    146. 

  146.           ("ser",  u"fuera",    (es.PAST, 3, es.SINGULAR, es.SUBJUNCTIVE)),
    147. 

  147.           ("ser",  u"fuéramos", (es.PAST, 1, es.PLURAL, es.SUBJUNCTIVE)),
    148. 

  148.           ("ser",  u"fuerais",  (es.PAST, 2, es.PLURAL, es.SUBJUNCTIVE)),
    149. 

  149.           ("ser",  u"fueran",   (es.PAST, 3, es.PLURAL, es.SUBJUNCTIVE))):
    150. 

  150.             self.assertEqual(es.conjugate(v1, tense), v2)
    151. 

  151.         print("pattern.es.conjugate()")
    152. 

  152. 

  153.     def test_lexeme(self):
    154. 

  154.         # Assert all inflections of "ser".
    155. 

  155.         v = es.lexeme("ser")
    156. 

  156.         self.assertEqual(v, [
    157. 

  157.             u'ser', u'soy', u'eres', u'es', u'somos', u'sois', u'son', u'siendo', 
    158. 

  158.             u'fui', u'fuiste', u'fue', u'fuimos', u'fuisteis', u'fueron', u'sido', 
    159. 

  159.             u'era', u'eras', u'éramos', u'erais', u'eran', 
    160. 

  160.             u'seré', u'serás', u'será', u'seremos', u'seréis', u'serán', 
    161. 

  161.             u'sería', u'serías', u'seríamos', u'seríais', u'serían', 
    162. 

  162.             u'sé', u'sed', 
    163. 

  163.             u'sea', u'seas', u'seamos', u'seáis', u'sean', 
    164. 

  164.             u'fuera', u'fueras', u'fuéramos', u'fuerais', u'fueran'
    165. 

  165.         ])
    166. 

  166.         print("pattern.es.inflect.lexeme()")
    167. 

  167. 

  168.     def test_tenses(self):
    169. 

  169.         # Assert tense recognition.
    170. 

  170.         self.assertTrue((es.PRESENT, 3, es.SG) in es.tenses("es"))
    171. 

  171.         self.assertTrue("2sg" in es.tenses("eres"))
    172. 

  172.         # The CONDITIONAL is sometimes described as a mood, 
    173. 

  173.         # and sometimes as a tense of the indicative mood (e.g., in Spanish):
    174. 

  174.         t1 = (es.CONDITIONAL, 1, es.SG)
    175. 

  175.         t2 = (es.PRESENT, 1, es.SG, es.CONDITIONAL)
    176. 

  176.         self.assertTrue("1sg->" in es.tenses(u"sería"))
    177. 

  177.         self.assertTrue(t1 in es.tenses(u"sería"))
    178. 

  178.         self.assertTrue(t2 in es.tenses(u"sería"))
    179. 

  179.         self.assertTrue(t1 in es.tenses(es.conjugate("ser", mood=es.INDICATIVE, tense=es.CONDITIONAL)))
    180. 

  180.         self.assertTrue(t2 in es.tenses(es.conjugate("ser", mood=es.CONDITIONAL)))
    181. 

  181.         print("pattern.es.tenses()")
    182. 

  182. 

  183. #---------------------------------------------------------------------------------------------------
    184. 

  184. 

  185. class TestParser(unittest.TestCase):
    186. 

  186.     
    187. 

  187.     def setUp(self):
    188. 

  188.         pass
    189. 

  189.         
    190. 

  190.     def test_find_lemmata(self):
    191. 

  191.         # Assert lemmata for nouns, adjectives, verbs and determiners.
    192. 

  192.         v = es.parser.find_lemmata([
    193. 

  193.             ["Los", "DT"], ["gatos", "NNS"], [u"negros", "JJ"], ["se", "PRP"], [u"sentó", "VB"],
    194. 

  194.             ["en", "IN"], ["la", "DT"], ["alfombra", "NN"]])
    195. 

  195.         self.assertEqual(v, [
    196. 

  196.             ["Los", "DT", "el"], 
    197. 

  197.             ["gatos", "NNS", "gato"], 
    198. 

  198.             ["negros", "JJ", "negro"], 
    199. 

  199.             ["se", "PRP", "se"], 
    200. 

  200.             [u"sentó", "VB", "sentar"],
    201. 

  201.             ["en", "IN", "en"], 
    202. 

  202.             ["la", "DT", "el"], 
    203. 

  203.             ["alfombra", "NN", "alfombra"]])
    204. 

  204.         print("pattern.es.parser.find_lemmata()")
    205. 

  205. 

  206.     def test_parse(self):
    207. 

  207.         # Assert parsed output with Penn Treebank II tags (slash-formatted).
    208. 

  208.         # "el gato negro" is a noun phrase, "en la alfombra" is a prepositional noun phrase.
    209. 

  209.         v = es.parser.parse(u"El gato negro se sentó en la alfombra.")
    210. 

  210.         self.assertEqual(v, # XXX - shouldn't "se" be part of the verb phrase?
    211. 

  211.             u"El/DT/B-NP/O gato/NN/I-NP/O negro/JJ/I-NP/O " + \
    212. 

  212.             u"se/PRP/B-NP/O sentó/VB/B-VP/O " + \
    213. 

  213.             u"en/IN/B-PP/B-PNP la/DT/B-NP/I-PNP alfombra/NN/I-NP/I-PNP ././O/O"
    214. 

  214.         )
    215. 

  215.         # Assert the accuracy of the Spanish tagger.
    216. 

  216.         i, n = 0, 0
    217. 

  217.         for sentence in open(os.path.join(PATH, "corpora", "tagged-es-wikicorpus.txt")).readlines():
    218. 

  218.             sentence = sentence.decode("utf-8").strip()
    219. 

  219.             s1 = [w.split("/") for w in sentence.split(" ")]
    220. 

  220.             s2 = [[w for w, pos in s1]]
    221. 

  221.             s2 = es.parse(s2, tokenize=False, tagset=es.PAROLE)
    222. 

  222.             s2 = [w.split("/") for w in s2.split(" ")]
    223. 

  223.             for j in range(len(s1)):
    224. 

  224.                 if s1[j][1] == s2[j][1]:
    225. 

  225.                     i += 1
    226. 

  226.                 n += 1
    227. 

  227.         #print(float(i) / n)
    228. 

  228.         self.assertTrue(float(i) / n > 0.92)
    229. 

  229.         print("pattern.es.parser.parse()")
    230. 

  230. 

  231.     def test_tag(self):
    232. 

  232.         # Assert [("el", "DT"), ("gato", "NN"), ("negro", "JJ")].
    233. 

  233.         v = es.tag("el gato negro")
    234. 

  234.         self.assertEqual(v, [("el", "DT"), ("gato", "NN"), ("negro", "JJ")])
    235. 

  235.         print("pattern.es.tag()")
    236. 

  236.     
    237. 

  237.     def test_command_line(self):
    238. 

  238.         # Assert parsed output from the command-line (example from the documentation).
    239. 

  239.         p = ["python", "-m", "pattern.es", "-s", "El gato negro.", "-OTCRL"]
    240. 

  240.         p = subprocess.Popen(p, stdout=subprocess.PIPE)
    241. 

  241.         p.wait()
    242. 

  242.         v = p.stdout.read()
    243. 

  243.         v = v.strip()
    244. 

  244.         self.assertEqual(v, "El/DT/B-NP/O/O/el gato/NN/I-NP/O/O/gato negro/JJ/I-NP/O/O/negro ././O/O/O/.")
    245. 

  245.         print("python -m pattern.es")
    246. 

  246. 

  247. #---------------------------------------------------------------------------------------------------
    248. 

  248. 

  249. def suite():
    250. 

  250.     suite = unittest.TestSuite()
    251. 

  251.     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestInflection))
    252. 

  252.     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestParser))
    253. 

  253.     return suite
    254. 

  254. 

  255. if __name__ == "__main__":
    256. 

  256.     unittest.TextTestRunner(verbosity=1).run(suite())
    257. 

  257.