PageRenderTime 37ms CodeModel.GetById 11ms RepoModel.GetById 0ms app.codeStats 0ms

/Products/CMFPlone/UnicodeSplitter/config.py

https://github.com/emanlove/Products.CMFPlone
Python | 55 lines | 26 code | 8 blank | 21 comment | 0 complexity | 2c440163867323988bda3b987b07750a MD5 | raw file
Possible License(s): GPL-2.0, BSD-3-Clause 
    

  1. # -*- coding: utf-8 -*-
    2. 

  2. """
    3. 

  3. config.py
    4. 

  4. 

  5. Created by Manabu Terada, CMScom on 2009-08-08.
    6. 

  6. """
    7. 

  7. import re
    8. 

  8. 

  9. STOP_WORD = []
    10. 

  10. 

  11. ## Setting, adding langs.
    12. 

  12. rangetable = dict(
    13. 

  13.     # ascii=u"a-zA-Z0-9_",
    14. 

  14.     # digit=u"\d",
    15. 

  15. 

  16.     # U+AC00-D7AF       Hangul Syllables        ハングル音節文字
    17. 

  17.     hangul=u"\uAC00-\uD7AF",
    18. 

  18. 

  19.     # U+30A0-30FF       Katakana        片仮名
    20. 

  20.     # U+3040-309F       Hiragana        平仮名
    21. 

  21.     # kana=u"\u3040-\u30FF",
    22. 

  22.     # hiragana=u"\u3040-\u309F\u30FC",
    23. 

  23.     # katakana=u"\u30A0-\u30FF",
    24. 

  24. 

  25.     # U+4E00-9FFF     CJK Unified Ideographs  CJK統合漢字
    26. 

  26.     # U+3400-4DBF     CJK Unified Ideographs Extension A  CJK統合漢字拡張A
    27. 

  27.     # U+F900-FAFF     CJK Compatibility Ideographs    CJK互換漢字
    28. 

  28.     # ideo=u"\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
    29. 

  29. 

  30.     cj=u"\u3040-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
    31. 

  31.     thai=u"\u0E00-\u0E7F",  # U+0E00-0E7F Thai タイ文字
    32. 

  32.     )
    33. 

  33. ## End of setting.
    34. 

  34. 

  35. 

  36. ## Splitting core.
    37. 

  37. ps = rangetable.values()
    38. 

  38. allp = u"".join(ps)
    39. 

  39. glob_true = u"[^%s]([^%s]|[\*\?])*|" \
    40. 

  40.                 % (allp, allp) + u"|".join([u"[%s]+" % (x, ) for x in ps])
    41. 

  41. 

  42. glob_false = u"[^%s]+|" % allp + u"|".join(u"[%s]+" % x for x in ps)
    43. 

  43. 

  44. rx_all = re.compile(ur"[%s]" % allp, re.UNICODE)
    45. 

  45. rx_U = re.compile(r"\w+", re.UNICODE)
    46. 

  46. rxGlob_U = re.compile(r"\w+[\w*?]*", re.UNICODE)
    47. 

  47. 

  48. rx_L = re.compile(r"\w+", re.LOCALE)
    49. 

  49. rxGlob_L = re.compile(r"\w+[\w*?]*", re.LOCALE)
    50. 

  50. 

  51. # pattern = re.compile(u"[a-zA-Z0-9_]+|[\uac00-\ud7af]+|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff]+", re.UNICODE)
    52. 

  52. # pattern_g = re.compile(u"[a-zA-Z0-9_]+[*?]*|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+[*?]*", re.UNICODE)
    53. 

  53. 

  54. pattern = re.compile(glob_false, re.UNICODE)
    55. 

  55. pattern_g = re.compile(glob_true, re.UNICODE)
    56. 

  56.