PageRenderTime 76ms CodeModel.GetById 31ms RepoModel.GetById 1ms app.codeStats 0ms

/test/test_it.py

https://gitlab.com/admin-github-cloud/pattern
Python | 272 lines | 269 code | 2 blank | 1 comment | 1 complexity | c0f6eed4004f0cf1ceb213913bac25be MD5 | raw file
    

  1. # -*- coding: utf-8 -*-
    2. 

  2. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
    3. 

  3. import unittest
    4. 

  4. import subprocess
    5. 

  5. 

  6. from pattern import it
    7. 

  7. 

  8. try:
    9. 

  9.     PATH = os.path.dirname(os.path.realpath(__file__))
    10. 

  10. except:
    11. 

  11.     PATH = ""
    12. 

  12. 

  13. #---------------------------------------------------------------------------------------------------
    14. 

  14. 

  15. class TestInflection(unittest.TestCase):
    16. 

  16. 

  17.     def setUp(self):
    18. 

  18.         pass
    19. 

  19.         
    20. 

  20.     def test_article(self):
    21. 

  21.         # Assert definite and indefinite article inflection.
    22. 

  22.         for a, n, g in (
    23. 

  23.           ("il" , "giorno"      , it.M),
    24. 

  24.           ("l'" , "altro giorno", it.M),
    25. 

  25.           ("lo" , "zio"         , it.M),
    26. 

  26.           ("l'" , "amica"       , it.F),
    27. 

  27.           ("la" , "nouva amica" , it.F),
    28. 

  28.           ("i"  , "giapponesi"  , it.M + it.PL),
    29. 

  29.           ("gli", "italiani"    , it.M + it.PL),
    30. 

  30.           ("gli", "zii"         , it.M + it.PL),
    31. 

  31.           ("le" , "zie"         , it.F + it.PL)):
    32. 

  32.             v = it.article(n, "definite", gender=g)
    33. 

  33.             self.assertEqual(a, v)
    34. 

  34.         for a, n, g in (
    35. 

  35.           ("uno", "zio"  , it.M),
    36. 

  36.           ("una", "zia"  , it.F),
    37. 

  37.           ("un" , "amico", it.M),
    38. 

  38.           ("un'", "amica", it.F)):
    39. 

  39.             v = it.article(n, "indefinite", gender=g)
    40. 

  40.             self.assertEqual(a, v)
    41. 

  41.         v = it.referenced("amica", gender="f")
    42. 

  42.         self.assertEqual(v, "un'amica")
    43. 

  43.         print("pattern.it.article()")
    44. 

  44.         print("pattern.it.referenced()")
    45. 

  45.     
    46. 

  46.     def test_gender(self):
    47. 

  47.         # Assert the accuracy of the gender disambiguation algorithm.
    48. 

  48.         from pattern.db import Datasheet
    49. 

  49.         i, n = 0, 0
    50. 

  50.         for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
    51. 

  51.             g = it.gender(sg)
    52. 

  52.             if mf in g and it.PLURAL not in g:
    53. 

  53.                 i += 1
    54. 

  54.             g = it.gender(pl)
    55. 

  55.             if mf in g and it.PLURAL in g:
    56. 

  56.                 i += 1
    57. 

  57.             n += 2
    58. 

  58.         self.assertTrue(float(i) / n > 0.92)
    59. 

  59.         print("pattern.it.gender()")
    60. 

  60.     
    61. 

  61.     def test_pluralize(self):
    62. 

  62.         # Assert the accuracy of the pluralization algorithm.
    63. 

  63.         from pattern.db import Datasheet
    64. 

  64.         i, n = 0, 0
    65. 

  65.         for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
    66. 

  66.             if it.pluralize(sg) == pl:
    67. 

  67.                 i += 1
    68. 

  68.             n += 1
    69. 

  69.         self.assertTrue(float(i) / n > 0.93)
    70. 

  70.         print("pattern.it.pluralize()")
    71. 

  71.         
    72. 

  72.     def test_singularize(self):
    73. 

  73.         # Assert the accuracy of the singularization algorithm.
    74. 

  74.         from pattern.db import Datasheet
    75. 

  75.         i, n = 0, 0
    76. 

  76.         for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
    77. 

  77.             if it.singularize(pl) == sg:
    78. 

  78.                 i += 1
    79. 

  79.             n += 1
    80. 

  80.         self.assertTrue(float(i) / n > 0.84)
    81. 

  81.         print("pattern.it.singularize()")
    82. 

  82.         
    83. 

  83.     def test_predicative(self):
    84. 

  84.         # Assert the accuracy of the predicative algorithm ("cruciali" => "cruciale").
    85. 

  85.         
    86. 

  86.         from pattern.db import Datasheet
    87. 

  87.         i, n = 0, 0
    88. 

  88.         for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")):
    89. 

  89.             if pos != "j":
    90. 

  90.                 continue
    91. 

  91.             if it.predicative(pl) == sg:
    92. 

  92.                 i += 1
    93. 

  93.             n += 1
    94. 

  94.         self.assertTrue(float(i) / n > 0.87)
    95. 

  95.         print("pattern.it.predicative()")
    96. 

  96. 

  97.     def test_find_lemma(self):
    98. 

  98.         # Assert the accuracy of the verb lemmatization algorithm.
    99. 

  99.         i, n = 0, 0
    100. 

  100.         r = 0
    101. 

  101.         for v1, v2 in it.inflect.verbs.inflections.items():
    102. 

  102.             if it.inflect.verbs.find_lemma(v1) == v2: 
    103. 

  103.                 i += 1
    104. 

  104.             n += 1
    105. 

  105.         self.assertTrue(float(i) / n > 0.81)
    106. 

  106.         print("pattern.it.inflect.verbs.find_lemma()")
    107. 

  107.         
    108. 

  108.     def test_find_lexeme(self):
    109. 

  109.         # Assert the accuracy of the verb conjugation algorithm.
    110. 

  110.         i, n = 0, 0
    111. 

  111.         for v, lexeme1 in it.inflect.verbs.infinitives.items():
    112. 

  112.             lexeme2 = it.inflect.verbs.find_lexeme(v)
    113. 

  113.             for j in range(len(lexeme2)):
    114. 

  114.                 if lexeme1[j] == lexeme2[j]:
    115. 

  115.                     i += 1
    116. 

  116.                 n += 1
    117. 

  117.         self.assertTrue(float(i) / n > 0.89)
    118. 

  118.         print("pattern.it.inflect.verbs.find_lexeme()")
    119. 

  119. 

  120.     def test_conjugate(self):
    121. 

  121.         # Assert different tenses with different conjugations.
    122. 

  122.         for (v1, v2, tense) in (
    123. 

  123.           ("essere", u"essere",     it.INFINITIVE),
    124. 

  124.           ("essere", u"sono",      (it.PRESENT, 1, it.SINGULAR)),
    125. 

  125.           ("essere", u"sei",       (it.PRESENT, 2, it.SINGULAR)),
    126. 

  126.           ("essere", u"è",         (it.PRESENT, 3, it.SINGULAR)),
    127. 

  127.           ("essere", u"siamo",     (it.PRESENT, 1, it.PLURAL)),
    128. 

  128.           ("essere", u"siete",     (it.PRESENT, 2, it.PLURAL)),
    129. 

  129.           ("essere", u"sono",      (it.PRESENT, 3, it.PLURAL)),
    130. 

  130.           ("essere", u"essendo",   (it.PRESENT + it.PARTICIPLE)),
    131. 

  131.           ("essere", u"stato",     (it.PAST + it.PARTICIPLE)),
    132. 

  132.           ("essere", u"ero",       (it.IMPERFECT, 1, it.SINGULAR)),
    133. 

  133.           ("essere", u"eri",       (it.IMPERFECT, 2, it.SINGULAR)),
    134. 

  134.           ("essere", u"era",       (it.IMPERFECT, 3, it.SINGULAR)),
    135. 

  135.           ("essere", u"eravamo",   (it.IMPERFECT, 1, it.PLURAL)),
    136. 

  136.           ("essere", u"eravate",   (it.IMPERFECT, 2, it.PLURAL)),
    137. 

  137.           ("essere", u"erano",     (it.IMPERFECT, 3, it.PLURAL)),
    138. 

  138.           ("essere", u"fui",       (it.PRETERITE, 1, it.SINGULAR)),
    139. 

  139.           ("essere", u"fosti",     (it.PRETERITE, 2, it.SINGULAR)),
    140. 

  140.           ("essere", u"fu",        (it.PRETERITE, 3, it.SINGULAR)),
    141. 

  141.           ("essere", u"fummo",     (it.PRETERITE, 1, it.PLURAL)),
    142. 

  142.           ("essere", u"foste",     (it.PRETERITE, 2, it.PLURAL)),
    143. 

  143.           ("essere", u"furono",    (it.PRETERITE, 3, it.PLURAL)),
    144. 

  144.           ("essere", u"sarei",     (it.CONDITIONAL, 1, it.SINGULAR)),
    145. 

  145.           ("essere", u"saresti",   (it.CONDITIONAL, 2, it.SINGULAR)),
    146. 

  146.           ("essere", u"sarebbe",   (it.CONDITIONAL, 3, it.SINGULAR)),
    147. 

  147.           ("essere", u"saremmo",   (it.CONDITIONAL, 1, it.PLURAL)),
    148. 

  148.           ("essere", u"sareste",   (it.CONDITIONAL, 2, it.PLURAL)),
    149. 

  149.           ("essere", u"sarebbero", (it.CONDITIONAL, 3, it.PLURAL)),
    150. 

  150.           ("essere", u"sarò",      (it.FUTURE, 1, it.SINGULAR)),
    151. 

  151.           ("essere", u"sarai",     (it.FUTURE, 2, it.SINGULAR)),
    152. 

  152.           ("essere", u"sarà",      (it.FUTURE, 3, it.SINGULAR)),
    153. 

  153.           ("essere", u"saremo",    (it.FUTURE, 1, it.PLURAL)),
    154. 

  154.           ("essere", u"sarete",    (it.FUTURE, 2, it.PLURAL)),
    155. 

  155.           ("essere", u"saranno",   (it.FUTURE, 3, it.PLURAL)),
    156. 

  156.           ("essere", u"sii",       (it.PRESENT, 2, it.SINGULAR, it.IMPERATIVE)),
    157. 

  157.           ("essere", u"sia",       (it.PRESENT, 3, it.SINGULAR, it.IMPERATIVE)),
    158. 

  158.           ("essere", u"siamo",     (it.PRESENT, 1, it.PLURAL, it.IMPERATIVE)),
    159. 

  159.           ("essere", u"siate",     (it.PRESENT, 2, it.PLURAL, it.IMPERATIVE)),
    160. 

  160.           ("essere", u"siano",     (it.PRESENT, 3, it.PLURAL, it.IMPERATIVE)),
    161. 

  161.           ("essere",  u"sia",      (it.PRESENT, 1, it.SINGULAR, it.SUBJUNCTIVE)),
    162. 

  162.           ("essere",  u"sia",      (it.PRESENT, 2, it.SINGULAR, it.SUBJUNCTIVE)),
    163. 

  163.           ("essere",  u"sia",      (it.PRESENT, 3, it.SINGULAR, it.SUBJUNCTIVE)),
    164. 

  164.           ("essere",  u"siamo",    (it.PRESENT, 1, it.PLURAL, it.SUBJUNCTIVE)),
    165. 

  165.           ("essere",  u"siate",    (it.PRESENT, 2, it.PLURAL, it.SUBJUNCTIVE)),
    166. 

  166.           ("essere",  u"siano",    (it.PRESENT, 3, it.PLURAL, it.SUBJUNCTIVE)),
    167. 

  167.           ("essere",  u"fossi",    (it.PAST, 1, it.SINGULAR, it.SUBJUNCTIVE)),
    168. 

  168.           ("essere",  u"fossi",    (it.PAST, 2, it.SINGULAR, it.SUBJUNCTIVE)),
    169. 

  169.           ("essere",  u"fosse",    (it.PAST, 3, it.SINGULAR, it.SUBJUNCTIVE)),
    170. 

  170.           ("essere",  u"fossimo",  (it.PAST, 1, it.PLURAL, it.SUBJUNCTIVE)),
    171. 

  171.           ("essere",  u"foste",    (it.PAST, 2, it.PLURAL, it.SUBJUNCTIVE)),
    172. 

  172.           ("essere",  u"fossero",  (it.PAST, 3, it.PLURAL, it.SUBJUNCTIVE))):
    173. 

  173.             self.assertEqual(it.conjugate(v1, tense), v2)
    174. 

  174.         print("pattern.it.conjugate()")
    175. 

  175. 

  176.     def test_lexeme(self):
    177. 

  177.         # Assert all inflections of "essere".
    178. 

  178.         v = it.lexeme("essere")
    179. 

  179.         self.assertEqual(v, [
    180. 

  180.             u'essere', u'sono', u'sei', u'è', u'siamo', u'siete', u'essendo', 
    181. 

  181.             u'fui', u'fosti', u'fu', u'fummo', u'foste', u'furono', u'stato', 
    182. 

  182.             u'ero', u'eri', u'era', u'eravamo', u'eravate', u'erano', 
    183. 

  183.             u'sarò', u'sarai', u'sarà', u'saremo', u'sarete', u'saranno', 
    184. 

  184.             u'sarei', u'saresti', u'sarebbe', u'saremmo', u'sareste', u'sarebbero', 
    185. 

  185.             u'sii', u'sia', u'siate', u'siano', 
    186. 

  186.             u'fossi', u'fosse', u'fossimo', u'fossero'
    187. 

  187.         ])
    188. 

  188.         print("pattern.it.inflect.lexeme()")
    189. 

  189. 

  190.     def test_tenses(self):
    191. 

  191.         # Assert tense recognition.
    192. 

  192.         self.assertTrue((it.PRESENT, 3, it.SG) in it.tenses(u"è"))
    193. 

  193.         self.assertTrue("2sg" in it.tenses("sei"))
    194. 

  194.         print("pattern.it.tenses()")
    195. 

  195.         
    196. 

  196. #---------------------------------------------------------------------------------------------------
    197. 

  197. 

  198. class TestParser(unittest.TestCase):
    199. 

  199.     
    200. 

  200.     def setUp(self):
    201. 

  201.         pass
    202. 

  202.         
    203. 

  203.     def test_find_lemmata(self):
    204. 

  204.         # Assert lemmata for nouns, adjectives, verbs and determiners.
    205. 

  205.         v = it.parser.find_lemmata([
    206. 

  206.             ["I", "DT"], ["gatti", "NNS"], ["neri", "JJ"], 
    207. 

  207.             ["seduti", "VB"], ["sul", "IN"], ["tatami", "NN"]])
    208. 

  208.         self.assertEqual(v, [
    209. 

  209.             ["I", "DT", "il"], 
    210. 

  210.             ["gatti", "NNS", "gatto"], 
    211. 

  211.             ["neri", "JJ", "nero"], 
    212. 

  212.             ["seduti", "VB", "sedutare"],
    213. 

  213.             ["sul", "IN", "sul"], 
    214. 

  214.             ["tatami", "NN", "tatami"]])
    215. 

  215.         print("pattern.it.parser.find_lemmata()")
    216. 

  216. 

  217.     def test_parse(self):
    218. 

  218.         # Assert parsed output with Penn Treebank II tags (slash-formatted).
    219. 

  219.         # "il gatto nero" is a noun phrase, "sulla stuoia" is a prepositional noun phrase.
    220. 

  220.         v = it.parser.parse(u"Il gatto nero seduto sulla stuoia.")
    221. 

  221.         self.assertEqual(v,
    222. 

  222.             u"Il/DT/B-NP/O gatto/NN/I-NP/O nero/JJ/I-NP/O " + 
    223. 

  223.             u"seduto/VB/B-VP/O " + \
    224. 

  224.             u"sulla/IN/B-PP/B-PNP stuoia/NN/B-NP/I-PNP ././O/O"
    225. 

  225.         )
    226. 

  226.         # Assert the accuracy of the Italian tagger.
    227. 

  227.         i, n = 0, 0
    228. 

  228.         for sentence in open(os.path.join(PATH, "corpora", "tagged-it-wacky.txt")).readlines():
    229. 

  229.             sentence = sentence.decode("utf-8").strip()
    230. 

  230.             s1 = [w.split("/") for w in sentence.split(" ")]
    231. 

  231.             s2 = [[w for w, pos in s1]]
    232. 

  232.             s2 = it.parse(s2, tokenize=False)
    233. 

  233.             s2 = [w.split("/") for w in s2.split(" ")]
    234. 

  234.             for j in range(len(s1)):
    235. 

  235.                 t1 = s1[j][1]
    236. 

  236.                 t2 = s2[j][1]
    237. 

  237.                 # WaCKy test set tags plural nouns as "NN", pattern.it as "NNS".
    238. 

  238.                 # Some punctuation marks are also tagged differently, 
    239. 

  239.                 # but these are not necessarily errors.
    240. 

  240.                 if t1 == t2 or (t1 == "NN" and t2.startswith("NN")) or s1[j][0] in "\":;)-":
    241. 

  241.                     i += 1
    242. 

  242.                 n += 1
    243. 

  243.         #print(float(i) / n)
    244. 

  244.         self.assertTrue(float(i) / n > 0.92)
    245. 

  245.         print("pattern.it.parser.parse()")
    246. 

  246. 

  247.     def test_tag(self):
    248. 

  248.         # Assert [("il", "DT"), ("gatto", "NN"), ("nero", "JJ")].
    249. 

  249.         v = it.tag("il gatto nero")
    250. 

  250.         self.assertEqual(v, [("il", "DT"), ("gatto", "NN"), ("nero", "JJ")])
    251. 

  251.         print("pattern.it.tag()")
    252. 

  252.     
    253. 

  253.     def test_command_line(self):
    254. 

  254.         # Assert parsed output from the command-line (example from the documentation).
    255. 

  255.         p = ["python", "-m", "pattern.it", "-s", "Il gatto nero.", "-OTCRL"]
    256. 

  256.         p = subprocess.Popen(p, stdout=subprocess.PIPE)
    257. 

  257.         p.wait()
    258. 

  258.         v = p.stdout.read()
    259. 

  259.         v = v.strip()
    260. 

  260.         self.assertEqual(v, "Il/DT/B-NP/O/O/il gatto/NN/I-NP/O/O/gatto nero/JJ/I-NP/O/O/nero ././O/O/O/.")
    261. 

  261.         print("python -m pattern.it")
    262. 

  262. 

  263. #---------------------------------------------------------------------------------------------------
    264. 

  264. 

  265. def suite():
    266. 

  266.     suite = unittest.TestSuite()
    267. 

  267.     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestInflection))
    268. 

  268.     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestParser))
    269. 

  269.     return suite
    270. 

  270. 

  271. if __name__ == "__main__":
    272. 

  272.     unittest.TextTestRunner(verbosity=1).run(suite())
    273. 

  273.