PageRenderTime 35ms CodeModel.GetById 27ms RepoModel.GetById 0ms app.codeStats 0ms

/scripts/backup_hatenafotolife/bs4/builder/_htmlparser.py

https://github.com/yoheia/yoheia
Python | 265 lines | 261 code | 1 blank | 3 comment | 3 complexity | 2792e941f5d205f9c608ae5be728a98b MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause
  1. """Use the HTMLParser library to parse HTML files that aren't too bad."""
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. __all__ = [
  5. 'HTMLParserTreeBuilder',
  6. ]
  7. from HTMLParser import HTMLParser
  8. try:
  9. from HTMLParser import HTMLParseError
  10. except ImportError, e:
  11. # HTMLParseError is removed in Python 3.5. Since it can never be
  12. # thrown in 3.5, we can just define our own class as a placeholder.
  13. class HTMLParseError(Exception):
  14. pass
  15. import sys
  16. import warnings
  17. # Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
  18. # argument, which we'd like to set to False. Unfortunately,
  19. # http://bugs.python.org/issue13273 makes strict=True a better bet
  20. # before Python 3.2.3.
  21. #
  22. # At the end of this file, we monkeypatch HTMLParser so that
  23. # strict=True works well on Python 3.2.2.
  24. major, minor, release = sys.version_info[:3]
  25. CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
  26. CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
  27. CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
  28. from bs4.element import (
  29. CData,
  30. Comment,
  31. Declaration,
  32. Doctype,
  33. ProcessingInstruction,
  34. )
  35. from bs4.dammit import EntitySubstitution, UnicodeDammit
  36. from bs4.builder import (
  37. HTML,
  38. HTMLTreeBuilder,
  39. STRICT,
  40. )
  41. HTMLPARSER = 'html.parser'
  42. class BeautifulSoupHTMLParser(HTMLParser):
  43. def handle_starttag(self, name, attrs):
  44. # XXX namespace
  45. attr_dict = {}
  46. for key, value in attrs:
  47. # Change None attribute values to the empty string
  48. # for consistency with the other tree builders.
  49. if value is None:
  50. value = ''
  51. attr_dict[key] = value
  52. attrvalue = '""'
  53. self.soup.handle_starttag(name, None, None, attr_dict)
  54. def handle_endtag(self, name):
  55. self.soup.handle_endtag(name)
  56. def handle_data(self, data):
  57. self.soup.handle_data(data)
  58. def handle_charref(self, name):
  59. # XXX workaround for a bug in HTMLParser. Remove this once
  60. # it's fixed in all supported versions.
  61. # http://bugs.python.org/issue13633
  62. if name.startswith('x'):
  63. real_name = int(name.lstrip('x'), 16)
  64. elif name.startswith('X'):
  65. real_name = int(name.lstrip('X'), 16)
  66. else:
  67. real_name = int(name)
  68. try:
  69. data = unichr(real_name)
  70. except (ValueError, OverflowError), e:
  71. data = u"\N{REPLACEMENT CHARACTER}"
  72. self.handle_data(data)
  73. def handle_entityref(self, name):
  74. character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
  75. if character is not None:
  76. data = character
  77. else:
  78. data = "&%s;" % name
  79. self.handle_data(data)
  80. def handle_comment(self, data):
  81. self.soup.endData()
  82. self.soup.handle_data(data)
  83. self.soup.endData(Comment)
  84. def handle_decl(self, data):
  85. self.soup.endData()
  86. if data.startswith("DOCTYPE "):
  87. data = data[len("DOCTYPE "):]
  88. elif data == 'DOCTYPE':
  89. # i.e. "<!DOCTYPE>"
  90. data = ''
  91. self.soup.handle_data(data)
  92. self.soup.endData(Doctype)
  93. def unknown_decl(self, data):
  94. if data.upper().startswith('CDATA['):
  95. cls = CData
  96. data = data[len('CDATA['):]
  97. else:
  98. cls = Declaration
  99. self.soup.endData()
  100. self.soup.handle_data(data)
  101. self.soup.endData(cls)
  102. def handle_pi(self, data):
  103. self.soup.endData()
  104. self.soup.handle_data(data)
  105. self.soup.endData(ProcessingInstruction)
  106. class HTMLParserTreeBuilder(HTMLTreeBuilder):
  107. is_xml = False
  108. picklable = True
  109. NAME = HTMLPARSER
  110. features = [NAME, HTML, STRICT]
  111. def __init__(self, *args, **kwargs):
  112. if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
  113. kwargs['strict'] = False
  114. if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
  115. kwargs['convert_charrefs'] = False
  116. self.parser_args = (args, kwargs)
  117. def prepare_markup(self, markup, user_specified_encoding=None,
  118. document_declared_encoding=None, exclude_encodings=None):
  119. """
  120. :return: A 4-tuple (markup, original encoding, encoding
  121. declared within markup, whether any characters had to be
  122. replaced with REPLACEMENT CHARACTER).
  123. """
  124. if isinstance(markup, unicode):
  125. yield (markup, None, None, False)
  126. return
  127. try_encodings = [user_specified_encoding, document_declared_encoding]
  128. dammit = UnicodeDammit(markup, try_encodings, is_html=True,
  129. exclude_encodings=exclude_encodings)
  130. yield (dammit.markup, dammit.original_encoding,
  131. dammit.declared_html_encoding,
  132. dammit.contains_replacement_characters)
  133. def feed(self, markup):
  134. args, kwargs = self.parser_args
  135. parser = BeautifulSoupHTMLParser(*args, **kwargs)
  136. parser.soup = self.soup
  137. try:
  138. parser.feed(markup)
  139. except HTMLParseError, e:
  140. warnings.warn(RuntimeWarning(
  141. "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
  142. raise e
  143. # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
  144. # 3.2.3 code. This ensures they don't treat markup like <p></p> as a
  145. # string.
  146. #
  147. # XXX This code can be removed once most Python 3 users are on 3.2.3.
  148. if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
  149. import re
  150. attrfind_tolerant = re.compile(
  151. r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
  152. r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
  153. HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
  154. locatestarttagend = re.compile(r"""
  155. <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
  156. (?:\s+ # whitespace before attribute name
  157. (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
  158. (?:\s*=\s* # value indicator
  159. (?:'[^']*' # LITA-enclosed value
  160. |\"[^\"]*\" # LIT-enclosed value
  161. |[^'\">\s]+ # bare value
  162. )
  163. )?
  164. )
  165. )*
  166. \s* # trailing whitespace
  167. """, re.VERBOSE)
  168. BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
  169. from html.parser import tagfind, attrfind
  170. def parse_starttag(self, i):
  171. self.__starttag_text = None
  172. endpos = self.check_for_whole_start_tag(i)
  173. if endpos < 0:
  174. return endpos
  175. rawdata = self.rawdata
  176. self.__starttag_text = rawdata[i:endpos]
  177. # Now parse the data between i+1 and j into a tag and attrs
  178. attrs = []
  179. match = tagfind.match(rawdata, i+1)
  180. assert match, 'unexpected call to parse_starttag()'
  181. k = match.end()
  182. self.lasttag = tag = rawdata[i+1:k].lower()
  183. while k < endpos:
  184. if self.strict:
  185. m = attrfind.match(rawdata, k)
  186. else:
  187. m = attrfind_tolerant.match(rawdata, k)
  188. if not m:
  189. break
  190. attrname, rest, attrvalue = m.group(1, 2, 3)
  191. if not rest:
  192. attrvalue = None
  193. elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
  194. attrvalue[:1] == '"' == attrvalue[-1:]:
  195. attrvalue = attrvalue[1:-1]
  196. if attrvalue:
  197. attrvalue = self.unescape(attrvalue)
  198. attrs.append((attrname.lower(), attrvalue))
  199. k = m.end()
  200. end = rawdata[k:endpos].strip()
  201. if end not in (">", "/>"):
  202. lineno, offset = self.getpos()
  203. if "\n" in self.__starttag_text:
  204. lineno = lineno + self.__starttag_text.count("\n")
  205. offset = len(self.__starttag_text) \
  206. - self.__starttag_text.rfind("\n")
  207. else:
  208. offset = offset + len(self.__starttag_text)
  209. if self.strict:
  210. self.error("junk characters in start tag: %r"
  211. % (rawdata[k:endpos][:20],))
  212. self.handle_data(rawdata[i:endpos])
  213. return endpos
  214. if end.endswith('/>'):
  215. # XHTML-style empty tag: <span attr="value" />
  216. self.handle_startendtag(tag, attrs)
  217. else:
  218. self.handle_starttag(tag, attrs)
  219. if tag in self.CDATA_CONTENT_ELEMENTS:
  220. self.set_cdata_mode(tag)
  221. return endpos
  222. def set_cdata_mode(self, elem):
  223. self.cdata_elem = elem.lower()
  224. self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
  225. BeautifulSoupHTMLParser.parse_starttag = parse_starttag
  226. BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
  227. CONSTRUCTOR_TAKES_STRICT = True