PageRenderTime 43ms CodeModel.GetById 20ms RepoModel.GetById 0ms app.codeStats 0ms

/twitter_text/regex.py

http://github.com/dryan/twitter-text-py
Python | 280 lines | 222 code | 27 blank | 31 comment | 5 complexity | 231ecaa28d55a390ef57268b04b324c3 MD5 | raw file
Possible License(s): Apache-2.0 
    

  1. #  encoding=utf-8
    2. 

  2. 

  3. # A collection of regular expressions for parsing Tweet text. The regular expression
    4. 

  4. # list is frozen at load time to ensure immutability. These reular expressions are
    5. 

  5. # used throughout the Twitter classes. Special care has been taken to make
    6. 

  6. # sure these reular expressions work with Tweets in all languages.
    7. 

  7. import re, string
    8. 

  8. 

  9. REGEXEN = {} # :nodoc:
    10. 

  10. 

  11. def regex_range(start, end = None):
    12. 

  12.     if end:
    13. 

  13.         return u'%s-%s' % (unichr(start), unichr(end))
    14. 

  14.     else:
    15. 

  15.         return u'%s' % unichr(start)
    16. 

  16. 

  17. # Space is more than %20, U+3000 for example is the full-width space used with Kanji. Provide a short-hand
    18. 

  18. # to access both the list of characters and a pattern suitible for use with String#split
    19. 

  19. #  Taken from: ActiveSupport::Multibyte::Handlers::UTF8Handler::UNICODE_WHITESPACE
    20. 

  20. UNICODE_SPACES = []
    21. 

  21. for space in reduce(lambda x,y: x + y if type(y) == list else x + [y], [
    22. 

  22.         range(0x0009, 0x000D),  # White_Space # Cc   [5] <control-0009>..<control-000D>
    23. 

  23.         0x0020,                 # White_Space # Zs       SPACE
    24. 

  24.         0x0085,                 # White_Space # Cc       <control-0085>
    25. 

  25.         0x00A0,                 # White_Space # Zs       NO-BREAK SPACE
    26. 

  26.         0x1680,                 # White_Space # Zs       OGHAM SPACE MARK
    27. 

  27.         0x180E,                 # White_Space # Zs       MONGOLIAN VOWEL SEPARATOR
    28. 

  28.         range(0x2000, 0x200A),  # White_Space # Zs  [11] EN QUAD..HAIR SPACE
    29. 

  29.         0x2028,                 # White_Space # Zl       LINE SEPARATOR
    30. 

  30.         0x2029,                 # White_Space # Zp       PARAGRAPH SEPARATOR
    31. 

  31.         0x202F,                 # White_Space # Zs       NARROW NO-BREAK SPACE
    32. 

  32.         0x205F,                 # White_Space # Zs       MEDIUM MATHEMATICAL SPACE
    33. 

  33.         0x3000,                 # White_Space # Zs       IDEOGRAPHIC SPACE
    34. 

  34.     ]):
    35. 

  35.     UNICODE_SPACES.append(unichr(space))
    36. 

  36. REGEXEN['spaces'] = re.compile(ur''.join(UNICODE_SPACES))
    37. 

  37. 

  38. # Characters not allowed in Tweets
    39. 

  39. INVALID_CHARACTERS  =   [
    40. 

  40.     0xFFFE, 0xFEFF,                         # BOM
    41. 

  41.     0xFFFF,                                 # Special
    42. 

  42.     0x202A, 0x202B, 0x202C, 0x202D, 0x202E, # Directional change
    43. 

  43. ]
    44. 

  44. REGEXEN['invalid_control_characters']   =   [unichr(x) for x in INVALID_CHARACTERS]
    45. 

  45. 

  46. REGEXEN['list_name'] = re.compile(ur'^[a-zA-Z][a-zA-Z0-9_\-\u0080-\u00ff]{0,24}$')
    47. 

  47. 

  48. # Latin accented characters
    49. 

  49. # Excludes 0xd7 from the range (the multiplication sign, confusable with "x").
    50. 

  50. # Also excludes 0xf7, the division sign
    51. 

  51. LATIN_ACCENTS = [
    52. 

  52.     regex_range(0x00c0, 0x00d6),
    53. 

  53.     regex_range(0x00d8, 0x00f6),
    54. 

  54.     regex_range(0x00f8, 0x00ff),
    55. 

  55.     regex_range(0x0100, 0x024f),
    56. 

  56.     regex_range(0x0253, 0x0254),
    57. 

  57.     regex_range(0x0256, 0x0257),
    58. 

  58.     regex_range(0x0259),
    59. 

  59.     regex_range(0x025b),
    60. 

  60.     regex_range(0x0263),
    61. 

  61.     regex_range(0x0268),
    62. 

  62.     regex_range(0x026f),
    63. 

  63.     regex_range(0x0272),
    64. 

  64.     regex_range(0x0289),
    65. 

  65.     regex_range(0x028b),
    66. 

  66.     regex_range(0x02bb),
    67. 

  67.     regex_range(0x0300, 0x036f),
    68. 

  68.     regex_range(0x1e00, 0x1eff),
    69. 

  69. ]
    70. 

  70. REGEXEN['latin_accents'] = re.compile(ur''.join(LATIN_ACCENTS), re.IGNORECASE | re.UNICODE)
    71. 

  71. LATIN_ACCENTS = u''.join(LATIN_ACCENTS)
    72. 

  72. 

  73. RTL_CHARACTERS = ''.join([
    74. 

  74.     regex_range(0x0600,0x06FF),
    75. 

  75.     regex_range(0x0750,0x077F),
    76. 

  76.     regex_range(0x0590,0x05FF),
    77. 

  77.     regex_range(0xFE70,0xFEFF)
    78. 

  78. ])
    79. 

  79. 

  80. NON_LATIN_HASHTAG_CHARS = ''.join([
    81. 

  81.     # Cyrillic (Russian, Ukrainian, etc.)
    82. 

  82.     regex_range(0x0400, 0x04ff), # Cyrillic
    83. 

  83.     regex_range(0x0500, 0x0527), # Cyrillic Supplement
    84. 

  84.     regex_range(0x2de0, 0x2dff), # Cyrillic Extended A
    85. 

  85.     regex_range(0xa640, 0xa69f), # Cyrillic Extended B
    86. 

  86.     regex_range(0x0591, 0x05bf), # Hebrew
    87. 

  87.     regex_range(0x05c1, 0x05c2),
    88. 

  88.     regex_range(0x05c4, 0x05c5),
    89. 

  89.     regex_range(0x05c7),
    90. 

  90.     regex_range(0x05d0, 0x05ea),
    91. 

  91.     regex_range(0x05f0, 0x05f4),
    92. 

  92.     regex_range(0xfb12, 0xfb28), # Hebrew Presentation Forms
    93. 

  93.     regex_range(0xfb2a, 0xfb36),
    94. 

  94.     regex_range(0xfb38, 0xfb3c),
    95. 

  95.     regex_range(0xfb3e),
    96. 

  96.     regex_range(0xfb40, 0xfb41),
    97. 

  97.     regex_range(0xfb43, 0xfb44),
    98. 

  98.     regex_range(0xfb46, 0xfb4f),
    99. 

  99.     regex_range(0x0610, 0x061a), # Arabic
    100. 

  100.     regex_range(0x0620, 0x065f),
    101. 

  101.     regex_range(0x066e, 0x06d3),
    102. 

  102.     regex_range(0x06d5, 0x06dc),
    103. 

  103.     regex_range(0x06de, 0x06e8),
    104. 

  104.     regex_range(0x06ea, 0x06ef),
    105. 

  105.     regex_range(0x06fa, 0x06fc),
    106. 

  106.     regex_range(0x06ff),
    107. 

  107.     regex_range(0x0750, 0x077f), # Arabic Supplement
    108. 

  108.     regex_range(0x08a0),         # Arabic Extended A
    109. 

  109.     regex_range(0x08a2, 0x08ac),
    110. 

  110.     regex_range(0x08e4, 0x08fe),
    111. 

  111.     regex_range(0xfb50, 0xfbb1), # Arabic Pres. Forms A
    112. 

  112.     regex_range(0xfbd3, 0xfd3d),
    113. 

  113.     regex_range(0xfd50, 0xfd8f),
    114. 

  114.     regex_range(0xfd92, 0xfdc7),
    115. 

  115.     regex_range(0xfdf0, 0xfdfb),
    116. 

  116.     regex_range(0xfe70, 0xfe74), # Arabic Pres. Forms B
    117. 

  117.     regex_range(0xfe76, 0xfefc),
    118. 

  118.     regex_range(0x200c, 0x200c), # Zero-Width Non-Joiner
    119. 

  119.     regex_range(0x0e01, 0x0e3a), # Thai
    120. 

  120.     regex_range(0x0e40, 0x0e4e), # Hangul (Korean)
    121. 

  121.     regex_range(0x1100, 0x11ff), # Hangul Jamo
    122. 

  122.     regex_range(0x3130, 0x3185), # Hangul Compatibility Jamo
    123. 

  123.     regex_range(0xA960, 0xA97F), # Hangul Jamo Extended-A
    124. 

  124.     regex_range(0xAC00, 0xD7AF), # Hangul Syllables
    125. 

  125.     regex_range(0xD7B0, 0xD7FF), # Hangul Jamo Extended-B
    126. 

  126.     regex_range(0xFFA1, 0xFFDC)  # Half-width Hangul
    127. 

  127. ])
    128. 

  128. 

  129. CJ_HASHTAG_CHARACTERS = ''.join([
    130. 

  130.     regex_range(0x30A1, 0x30FA), regex_range(0x30FC, 0x30FE), # Katakana (full-width)
    131. 

  131.     regex_range(0xFF66, 0xFF9F), # Katakana (half-width)
    132. 

  132.     regex_range(0xFF10, 0xFF19), regex_range(0xFF21, 0xFF3A), regex_range(0xFF41, 0xFF5A), # Latin (full-width)
    133. 

  133.     regex_range(0x3041, 0x3096), regex_range(0x3099, 0x309E), # Hiragana
    134. 

  134.     regex_range(0x3400, 0x4DBF), # Kanji (CJK Extension A)
    135. 

  135.     regex_range(0x4E00, 0x9FFF), # Kanji (Unified)
    136. 

  136. ])
    137. 

  137. 

  138. try:
    139. 

  139.     CJ_HASHTAG_CHARACTERS = ''.join([
    140. 

  140.         CJ_HASHTAG_CHARACTERS,
    141. 

  141.         regex_range(0x20000, 0x2A6DF), # Kanji (CJK Extension B)
    142. 

  142.         regex_range(0x2A700, 0x2B73F), # Kanji (CJK Extension C)
    143. 

  143.         regex_range(0x2B740, 0x2B81F), # Kanji (CJK Extension D)
    144. 

  144.         regex_range(0x2F800, 0x2FA1F), regex_range(0x3003), regex_range(0x3005), regex_range(0x303B) # Kanji (CJK supplement)
    145. 

  145.     ])
    146. 

  146. except ValueError:
    147. 

  147.     # this is a narrow python build so these extended Kanji characters won't work
    148. 

  148.     pass
    149. 

  149. 

  150. PUNCTUATION_CHARS = ur'!"#$%&\'()*+,-./:;<=>?@\[\]^_\`{|}~'
    151. 

  151. SPACE_CHARS = ur" \t\n\x0B\f\r"
    152. 

  152. CTRL_CHARS = ur"\x00-\x1F\x7F"
    153. 

  153. 

  154. # A hashtag must contain latin characters, numbers and underscores, but not all numbers.
    155. 

  155. HASHTAG_ALPHA = ur'[a-z_%s]' % (LATIN_ACCENTS + NON_LATIN_HASHTAG_CHARS + CJ_HASHTAG_CHARACTERS)
    156. 

  156. HASHTAG_ALPHANUMERIC = ur'[a-z0-9_%s]' % (LATIN_ACCENTS + NON_LATIN_HASHTAG_CHARS + CJ_HASHTAG_CHARACTERS)
    157. 

  157. HASHTAG_BOUNDARY = ur'\A|\z|\[|[^&a-z0-9_%s]' % (LATIN_ACCENTS + NON_LATIN_HASHTAG_CHARS + CJ_HASHTAG_CHARACTERS)
    158. 

  158. 

  159. HASHTAG = re.compile(ur'(%s)(#|＃)(%s*%s%s*)' % (HASHTAG_BOUNDARY, HASHTAG_ALPHANUMERIC, HASHTAG_ALPHA, HASHTAG_ALPHANUMERIC), re.IGNORECASE)
    160. 

  160. 

  161. REGEXEN['valid_hashtag'] = HASHTAG
    162. 

  162. REGEXEN['end_hashtag_match'] = re.compile(ur'\A(?:[#＃]|:\/\/)', re.IGNORECASE | re.UNICODE)
    163. 

  163. REGEXEN['numeric_only'] = re.compile(ur'^[\d]+$')
    164. 

  164. 

  165. REGEXEN['valid_mention_preceding_chars'] = re.compile(r'(?:[^a-zA-Z0-9_!#\$%&*@＠]|^|RT:?)')
    166. 

  166. REGEXEN['at_signs'] = re.compile(ur'[@＠]')
    167. 

  167. REGEXEN['valid_mention_or_list'] = re.compile(
    168. 

  168.     ur'(%s)' % REGEXEN['valid_mention_preceding_chars'].pattern.decode('utf-8') +   # preceding character
    169. 

  169.     ur'(%s)' % REGEXEN['at_signs'].pattern +                                        # at mark
    170. 

  170.     ur'([a-zA-Z0-9_]{1,20})' +                                                      # screen name
    171. 

  171.     ur'(\/[a-zA-Z][a-zA-Z0-9_\-]{0,24})?'                                           # list (optional)
    172. 

  172. )
    173. 

  173. REGEXEN['valid_reply'] = re.compile(ur'^(?:[%s])*%s([a-zA-Z0-9_]{1,20})' % (REGEXEN['spaces'].pattern, REGEXEN['at_signs'].pattern), re.IGNORECASE | re.UNICODE)
    174. 

  174.  # Used in Extractor for final filtering
    175. 

  175. REGEXEN['end_mention_match'] = re.compile(ur'\A(?:%s|[%s]|:\/\/)' % (REGEXEN['at_signs'].pattern, REGEXEN['latin_accents'].pattern), re.IGNORECASE | re.UNICODE)
    176. 

  176. 

  177. # URL related hash regex collection
    178. 

  178. REGEXEN['valid_url_preceding_chars'] = re.compile(ur'(?:[^A-Z0-9@＠$#＃%s]|^)' % ur''.join(REGEXEN['invalid_control_characters']), re.IGNORECASE | re.UNICODE)
    179. 

  179. REGEXEN['invalid_url_without_protocol_preceding_chars'] = re.compile(ur'[-_.\/]$')
    180. 

  180. DOMAIN_VALID_CHARS = ur'[^%s%s%s%s%s]' % (PUNCTUATION_CHARS, SPACE_CHARS, CTRL_CHARS, ur''.join(REGEXEN['invalid_control_characters']), ur''.join(UNICODE_SPACES))
    181. 

  181. REGEXEN['valid_subdomain'] = re.compile(ur'(?:(?:%s(?:[_-]|%s)*)?%s\.)' % (DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS), re.IGNORECASE | re.UNICODE)
    182. 

  182. REGEXEN['valid_domain_name'] = re.compile(ur'(?:(?:%s(?:[-]|%s)*)?%s\.)' % (DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS), re.IGNORECASE | re.UNICODE)
    183. 

  183. REGEXEN['valid_gTLD'] = re.compile(ur'(?:(?:academy|actor|aero|agency|arpa|asia|bar|bargains|berlin|best|bid|bike|biz|blue|boutique|build|builders|buzz|cab|camera|camp|cards|careers|cat|catering|center|ceo|cheap|christmas|cleaning|clothing|club|codes|coffee|com|community|company|computer|construction|contractors|cool|coop|cruises|dance|dating|democrat|diamonds|directory|domains|edu|education|email|enterprises|equipment|estate|events|expert|exposed|farm|fish|flights|florist|foundation|futbol|gallery|gift|glass|gov|graphics|guitars|guru|holdings|holiday|house|immobilien|industries|info|institute|int|international|jobs|kaufen|kim|kitchen|kiwi|koeln|kred|land|lighting|limo|link|luxury|management|mango|marketing|menu|mil|mobi|moda|monash|museum|nagoya|name|net|neustar|ninja|okinawa|onl|org|partners|parts|photo|photography|photos|pics|pink|plumbing|post|pro|productions|properties|pub|qpon|recipes|red|rentals|repair|report|reviews|rich|ruhr|sexy|shiksha|shoes|singles|social|solar|solutions|supplies|supply|support|systems|tattoo|technology|tel|tienda|tips|today|tokyo|tools|training|travel|uno|vacations|ventures|viajes|villas|vision|vote|voting|voto|voyage|wang|watch|wed|wien|wiki|works|xxx|xyz|zone|дети|онлайн|орг|сайт|بازار|شبكة|みんな|中信|中文网|公司|公>益|在线|我爱你|政务|游戏|移动|网络|集团|삼성)(?=[^0-9a-z]|$))', re.IGNORECASE | re.UNICODE)
    184. 

  184. REGEXEN['valid_ccTLD'] = re.compile(ur'(?:(?:ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bl|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mf|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|um|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw|мон|рф|срб|укр|қаз|الاردن|الجزائر|السعودية|المغرب|امارات|ایران|بھارت|تونس|سودان|سورية|عمان|فلسطين|قطر|مصر|مليسيا|پاکستان|भारत|বাংলা|ভারত|ਭਾਰਤ|ભારત|இந்தியா|இலங்கை|சிங்கப்பூர்|భారత్|ලංකා|ไทย|გე|中国|中加坡|湾|台灣|新香港|한국)(?=[^0-9a-z]|$))', re.IGNORECASE | re.UNICODE)
    185. 

  185. REGEXEN['valid_punycode'] = re.compile(ur'(?:xn--[0-9a-z]+)', re.IGNORECASE | re.UNICODE)
    186. 

  186. 

  187. REGEXEN['valid_domain'] = re.compile(ur'(?:%s*%s(?:%s|%s|%s))' % (REGEXEN['valid_subdomain'].pattern, REGEXEN['valid_domain_name'].pattern, REGEXEN['valid_gTLD'].pattern, REGEXEN['valid_ccTLD'].pattern, REGEXEN['valid_punycode'].pattern), re.IGNORECASE | re.UNICODE)
    188. 

  188. 

  189. # This is used in Extractor
    190. 

  190. REGEXEN['valid_ascii_domain'] = re.compile(ur'(?:(?:[A-Za-z0-9\-_]|[%s])+\.)+(?:%s|%s|%s)' % (REGEXEN['latin_accents'].pattern, REGEXEN['valid_gTLD'].pattern, REGEXEN['valid_ccTLD'].pattern, REGEXEN['valid_punycode'].pattern), re.IGNORECASE | re.UNICODE)
    191. 

  191. 

  192. # This is used in Extractor for stricter t.co URL extraction
    193. 

  193. REGEXEN['valid_tco_url'] = re.compile(ur'^https?:\/\/t\.co\/[a-z0-9]+', re.IGNORECASE | re.UNICODE)
    194. 

  194. 

  195. # This is used in Extractor to filter out unwanted URLs.
    196. 

  196. REGEXEN['invalid_short_domain'] = re.compile(ur'\A%s%s\Z' % (REGEXEN['valid_domain_name'].pattern, REGEXEN['valid_ccTLD'].pattern), re.IGNORECASE | re.UNICODE)
    197. 

  197. 

  198. REGEXEN['valid_port_number'] = re.compile(ur'[0-9]+')
    199. 

  199. 

  200. REGEXEN['valid_general_url_path_chars'] = re.compile(ur"[a-z0-9!\*';:=\+\,\.\$\/%%#\[\]\-_~&|@%s]" % LATIN_ACCENTS, re.IGNORECASE | re.UNICODE)
    201. 

  201. # Allow URL paths to contain balanced parens
    202. 

  202. #  1. Used in Wikipedia URLs like /Primer_(film)
    203. 

  203. #  2. Used in IIS sessions like /S(dfd346)/
    204. 

  204. REGEXEN['valid_url_balanced_parens'] = re.compile(ur'\(%s+\)' % REGEXEN['valid_general_url_path_chars'].pattern, re.IGNORECASE | re.UNICODE)
    205. 

  205. # Valid end-of-path chracters (so /foo. does not gobble the period).
    206. 

  206. #   1. Allow =&# for empty URL parameters and other URL-join artifacts
    207. 

  207. REGEXEN['valid_url_path_ending_chars'] = re.compile(ur'[a-z0-9=_#\/\+\-%s]|(?:%s)' % (LATIN_ACCENTS, REGEXEN['valid_url_balanced_parens'].pattern), re.IGNORECASE | re.UNICODE)
    208. 

  208. REGEXEN['valid_url_path'] = re.compile(ur'(?:(?:%s*(?:%s %s*)*%s)|(?:%s+\/))' % (REGEXEN['valid_general_url_path_chars'].pattern, REGEXEN['valid_url_balanced_parens'].pattern, REGEXEN['valid_general_url_path_chars'].pattern, REGEXEN['valid_url_path_ending_chars'].pattern, REGEXEN['valid_general_url_path_chars'].pattern), re.IGNORECASE | re.UNICODE)
    209. 

  209. 

  210. REGEXEN['valid_url_query_chars'] = re.compile(ur"[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@]", re.IGNORECASE | re.UNICODE)
    211. 

  211. REGEXEN['valid_url_query_ending_chars'] = re.compile(ur'[a-z0-9_&=#\/]', re.IGNORECASE | re.UNICODE)
    212. 

  212. REGEXEN['valid_url'] = re.compile(ur'((%s)((https?:\/\/)?(%s)(?::(%s))?(/%s*)?(\?%s*%s)?))' % (
    213. 

  213.     REGEXEN['valid_url_preceding_chars'].pattern,
    214. 

  214.     REGEXEN['valid_domain'].pattern,
    215. 

  215.     REGEXEN['valid_port_number'].pattern,
    216. 

  216.     REGEXEN['valid_url_path'].pattern,
    217. 

  217.     REGEXEN['valid_url_query_chars'].pattern,
    218. 

  218.     REGEXEN['valid_url_query_ending_chars'].pattern
    219. 

  219. ), re.IGNORECASE | re.UNICODE)
    220. 

  220. #   Matches
    221. 

  221. #   $1 total match
    222. 

  222. #   $2 Preceeding chracter
    223. 

  223. #   $3 URL
    224. 

  224. #   $4 Protocol (optional)
    225. 

  225. #   $5 Domain(s)
    226. 

  226. #   $6 Port number (optional)
    227. 

  227. #   $7 URL Path and anchor
    228. 

  228. #   $8 Query String
    229. 

  229. 

  230. REGEXEN['cashtag'] = re.compile(ur'[a-z]{1,6}(?:[._][a-z]{1,2})?', re.IGNORECASE)
    231. 

  231. REGEXEN['valid_cashtag'] = re.compile(ur'(^|[%s])(\$|＄|﹩)(%s)(?=$|\s|[%s])' % (REGEXEN['spaces'].pattern, REGEXEN['cashtag'].pattern, PUNCTUATION_CHARS), re.IGNORECASE)
    232. 

  232. 

  233. # These URL validation pattern strings are based on the ABNF from RFC 3986
    234. 

  234. REGEXEN['validate_url_unreserved'] = re.compile(ur'[a-z0-9\-._~]', re.IGNORECASE | re.UNICODE)
    235. 

  235. REGEXEN['validate_url_pct_encoded'] = re.compile(ur'(?:%[0-9a-f]{2})', re.IGNORECASE | re.UNICODE)
    236. 

  236. REGEXEN['validate_url_sub_delims'] = re.compile(ur"[!$&'()*+,;=]", re.IGNORECASE | re.UNICODE)
    237. 

  237. REGEXEN['validate_url_pchar'] = re.compile(ur'(?:%s|%s|%s|[:\|@])' % (REGEXEN['validate_url_unreserved'].pattern, REGEXEN['validate_url_pct_encoded'].pattern, REGEXEN['validate_url_sub_delims'].pattern), re.IGNORECASE | re.UNICODE)
    238. 

  238. 

  239. REGEXEN['validate_url_scheme'] = re.compile(ur'(?:[a-z][a-z0-9+\-.]*)', re.IGNORECASE | re.UNICODE)
    240. 

  240. REGEXEN['validate_url_userinfo'] = re.compile(ur'(?:%s|%s|%s|:)*' % (REGEXEN['validate_url_unreserved'].pattern, REGEXEN['validate_url_pct_encoded'].pattern, REGEXEN['validate_url_sub_delims'].pattern), re.IGNORECASE | re.UNICODE)
    241. 

  241. 

  242. REGEXEN['validate_url_dec_octet'] = re.compile(ur'(?:[0-9]|(?:[1-9][0-9])|(?:1[0-9]{2})|(?:2[0-4][0-9])|(?:25[0-5]))', re.IGNORECASE | re.UNICODE)
    243. 

  243. REGEXEN['validate_url_ipv4'] = re.compile(ur'(?:%s(?:\.%s){3})' % (REGEXEN['validate_url_dec_octet'].pattern, REGEXEN['validate_url_dec_octet'].pattern), re.IGNORECASE | re.UNICODE)
    244. 

  244. 

  245. # Punting on real IPv6 validation for now
    246. 

  246. REGEXEN['validate_url_ipv6'] = re.compile(ur'(?:\[[a-f0-9:\.]+\])', re.IGNORECASE | re.UNICODE)
    247. 

  247. 

  248. # Also punting on IPvFuture for now
    249. 

  249. REGEXEN['validate_url_ip'] = re.compile(ur'(?:%s|%s)' % (REGEXEN['validate_url_ipv4'].pattern, REGEXEN['validate_url_ipv6'].pattern), re.IGNORECASE | re.UNICODE)
    250. 

  250. 

  251. # This is more strict than the rfc specifies
    252. 

  252. REGEXEN['validate_url_subdomain_segment'] = re.compile(ur'(?:[a-z0-9](?:[a-z0-9_\-]*[a-z0-9])?)', re.IGNORECASE | re.UNICODE)
    253. 

  253. REGEXEN['validate_url_domain_segment'] = re.compile(ur'(?:[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?)', re.IGNORECASE | re.UNICODE)
    254. 

  254. REGEXEN['validate_url_domain_tld'] = re.compile(ur'(?:[a-z](?:[a-z0-9\-]*[a-z0-9])?)', re.IGNORECASE | re.UNICODE)
    255. 

  255. REGEXEN['validate_url_domain'] = re.compile(ur'(?:(?:%s\.)*(?:%s\.)%s)' % (REGEXEN['validate_url_subdomain_segment'].pattern, REGEXEN['validate_url_domain_segment'].pattern, REGEXEN['validate_url_domain_tld'].pattern), re.IGNORECASE | re.UNICODE)
    256. 

  256. 

  257. REGEXEN['validate_url_host'] = re.compile(ur'(?:%s|%s)' % (REGEXEN['validate_url_ip'].pattern, REGEXEN['validate_url_domain'].pattern), re.IGNORECASE | re.UNICODE)
    258. 

  258. 

  259. # Unencoded internationalized domains - this doesn't check for invalid UTF-8 sequences
    260. 

  260. REGEXEN['validate_url_unicode_subdomain_segment'] = re.compile(ur'(?:(?:[a-z0-9]|[^\x00-\x7f])(?:(?:[a-z0-9_\-]|[^\x00-\x7f])*(?:[a-z0-9]|[^\x00-\x7f]))?)', re.IGNORECASE | re.UNICODE)
    261. 

  261. REGEXEN['validate_url_unicode_domain_segment'] = re.compile(ur'(?:(?:[a-z0-9]|[^\x00-\x7f])(?:(?:[a-z0-9\-]|[^\x00-\x7f])*(?:[a-z0-9]|[^\x00-\x7f]))?)', re.IGNORECASE | re.UNICODE)
    262. 

  262. REGEXEN['validate_url_unicode_domain_tld'] = re.compile(ur'(?:(?:[a-z]|[^\x00-\x7f])(?:(?:[a-z0-9\-]|[^\x00-\x7f])*(?:[a-z0-9]|[^\x00-\x7f]))?)', re.IGNORECASE | re.UNICODE)
    263. 

  263. REGEXEN['validate_url_unicode_domain'] = re.compile(ur'(?:(?:%s\.)*(?:%s\.)%s)' % (REGEXEN['validate_url_unicode_subdomain_segment'].pattern, REGEXEN['validate_url_unicode_domain_segment'].pattern, REGEXEN['validate_url_unicode_domain_tld'].pattern), re.IGNORECASE | re.UNICODE)
    264. 

  264. 

  265. REGEXEN['validate_url_unicode_host'] = re.compile(ur'(?:%s|%s)' % (REGEXEN['validate_url_ip'].pattern, REGEXEN['validate_url_unicode_domain'].pattern), re.IGNORECASE | re.UNICODE)
    266. 

  266. 

  267. REGEXEN['validate_url_port'] = re.compile(ur'[0-9]{1,5}')
    268. 

  268. 

  269. REGEXEN['validate_url_unicode_authority'] = re.compile(ur'(?:(%s)@)?(%s)(?::(%s))?' % (REGEXEN['validate_url_userinfo'].pattern, REGEXEN['validate_url_unicode_host'].pattern, REGEXEN['validate_url_port'].pattern), re.IGNORECASE | re.UNICODE)
    270. 

  270. 

  271. REGEXEN['validate_url_authority'] = re.compile(ur'(?:(%s)@)?(%s)(?::(%s))?' % (REGEXEN['validate_url_userinfo'].pattern, REGEXEN['validate_url_host'].pattern, REGEXEN['validate_url_port'].pattern), re.IGNORECASE | re.UNICODE)
    272. 

  272. 

  273. REGEXEN['validate_url_path'] = re.compile(ur'(/%s*)*' % REGEXEN['validate_url_pchar'].pattern, re.IGNORECASE | re.UNICODE)
    274. 

  274. REGEXEN['validate_url_query'] = re.compile(ur'(%s|/|\?)*' % REGEXEN['validate_url_pchar'].pattern, re.IGNORECASE | re.UNICODE)
    275. 

  275. REGEXEN['validate_url_fragment'] = re.compile(ur'(%s|/|\?)*' % REGEXEN['validate_url_pchar'].pattern, re.IGNORECASE | re.UNICODE)
    276. 

  276. 

  277. # Modified version of RFC 3986 Appendix B
    278. 

  278. REGEXEN['validate_url_unencoded'] = re.compile(ur'\A(?:([^:/?#]+)://)?([^/?#]*)([^?#]*)(?:\?([^#]*))?(?:\#(.*))?\Z', re.IGNORECASE | re.UNICODE)
    279. 

  279. 

  280. REGEXEN['rtl_chars'] = re.compile(ur'[%s]' % RTL_CHARACTERS, re.IGNORECASE | re.UNICODE)
    281. 

  281.