PageRenderTime 72ms CodeModel.GetById 40ms RepoModel.GetById 1ms app.codeStats 0ms

/punkcrawler/bs4/builder/_html5lib.py

https://bitbucket.org/punkspider/punkscan
Python | 222 lines | 198 code | 17 blank | 7 comment | 3 complexity | 83d8636603480d531490b7fe58e3c64d MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, MPL-2.0-no-copyleft-exception, GPL-2.0, JSON, EPL-1.0
  1. __all__ = [
  2. 'HTML5TreeBuilder',
  3. ]
  4. import warnings
  5. from bs4.builder import (
  6. PERMISSIVE,
  7. HTML,
  8. HTML_5,
  9. HTMLTreeBuilder,
  10. )
  11. from bs4.element import NamespacedAttribute
  12. import html5lib
  13. from html5lib.constants import namespaces
  14. from bs4.element import (
  15. Comment,
  16. Doctype,
  17. NavigableString,
  18. Tag,
  19. )
  20. class HTML5TreeBuilder(HTMLTreeBuilder):
  21. """Use html5lib to build a tree."""
  22. features = ['html5lib', PERMISSIVE, HTML_5, HTML]
  23. def prepare_markup(self, markup, user_specified_encoding):
  24. # Store the user-specified encoding for use later on.
  25. self.user_specified_encoding = user_specified_encoding
  26. return markup, None, None, False
  27. # These methods are defined by Beautiful Soup.
  28. def feed(self, markup):
  29. if self.soup.parse_only is not None:
  30. warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
  31. parser = html5lib.HTMLParser(tree=self.create_treebuilder)
  32. doc = parser.parse(markup, encoding=self.user_specified_encoding)
  33. # Set the character encoding detected by the tokenizer.
  34. if isinstance(markup, unicode):
  35. # We need to special-case this because html5lib sets
  36. # charEncoding to UTF-8 if it gets Unicode input.
  37. doc.original_encoding = None
  38. else:
  39. doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
  40. def create_treebuilder(self, namespaceHTMLElements):
  41. self.underlying_builder = TreeBuilderForHtml5lib(
  42. self.soup, namespaceHTMLElements)
  43. return self.underlying_builder
  44. def test_fragment_to_document(self, fragment):
  45. """See `TreeBuilder`."""
  46. return u'<html><head></head><body>%s</body></html>' % fragment
  47. class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
  48. def __init__(self, soup, namespaceHTMLElements):
  49. self.soup = soup
  50. super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
  51. def documentClass(self):
  52. self.soup.reset()
  53. return Element(self.soup, self.soup, None)
  54. def insertDoctype(self, token):
  55. name = token["name"]
  56. publicId = token["publicId"]
  57. systemId = token["systemId"]
  58. doctype = Doctype.for_name_and_ids(name, publicId, systemId)
  59. self.soup.object_was_parsed(doctype)
  60. def elementClass(self, name, namespace):
  61. tag = self.soup.new_tag(name, namespace)
  62. return Element(tag, self.soup, namespace)
  63. def commentClass(self, data):
  64. return TextNode(Comment(data), self.soup)
  65. def fragmentClass(self):
  66. self.soup = BeautifulSoup("")
  67. self.soup.name = "[document_fragment]"
  68. return Element(self.soup, self.soup, None)
  69. def appendChild(self, node):
  70. # XXX This code is not covered by the BS4 tests.
  71. self.soup.append(node.element)
  72. def getDocument(self):
  73. return self.soup
  74. def getFragment(self):
  75. return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
  76. class AttrList(object):
  77. def __init__(self, element):
  78. self.element = element
  79. self.attrs = dict(self.element.attrs)
  80. def __iter__(self):
  81. return list(self.attrs.items()).__iter__()
  82. def __setitem__(self, name, value):
  83. "set attr", name, value
  84. self.element[name] = value
  85. def items(self):
  86. return list(self.attrs.items())
  87. def keys(self):
  88. return list(self.attrs.keys())
  89. def __len__(self):
  90. return len(self.attrs)
  91. def __getitem__(self, name):
  92. return self.attrs[name]
  93. def __contains__(self, name):
  94. return name in list(self.attrs.keys())
  95. class Element(html5lib.treebuilders._base.Node):
  96. def __init__(self, element, soup, namespace):
  97. html5lib.treebuilders._base.Node.__init__(self, element.name)
  98. self.element = element
  99. self.soup = soup
  100. self.namespace = namespace
  101. def appendChild(self, node):
  102. if (node.element.__class__ == NavigableString and self.element.contents
  103. and self.element.contents[-1].__class__ == NavigableString):
  104. # Concatenate new text onto old text node
  105. # XXX This has O(n^2) performance, for input like
  106. # "a</a>a</a>a</a>..."
  107. old_element = self.element.contents[-1]
  108. new_element = self.soup.new_string(old_element + node.element)
  109. old_element.replace_with(new_element)
  110. else:
  111. self.element.append(node.element)
  112. node.parent = self
  113. def getAttributes(self):
  114. return AttrList(self.element)
  115. def setAttributes(self, attributes):
  116. if attributes is not None and len(attributes) > 0:
  117. converted_attributes = []
  118. for name, value in list(attributes.items()):
  119. if isinstance(name, tuple):
  120. new_name = NamespacedAttribute(*name)
  121. del attributes[name]
  122. attributes[new_name] = value
  123. self.soup.builder._replace_cdata_list_attribute_values(
  124. self.name, attributes)
  125. for name, value in attributes.items():
  126. self.element[name] = value
  127. # The attributes may contain variables that need substitution.
  128. # Call set_up_substitutions manually.
  129. #
  130. # The Tag constructor called this method when the Tag was created,
  131. # but we just set/changed the attributes, so call it again.
  132. self.soup.builder.set_up_substitutions(self.element)
  133. attributes = property(getAttributes, setAttributes)
  134. def insertText(self, data, insertBefore=None):
  135. text = TextNode(self.soup.new_string(data), self.soup)
  136. if insertBefore:
  137. self.insertBefore(text, insertBefore)
  138. else:
  139. self.appendChild(text)
  140. def insertBefore(self, node, refNode):
  141. index = self.element.index(refNode.element)
  142. if (node.element.__class__ == NavigableString and self.element.contents
  143. and self.element.contents[index-1].__class__ == NavigableString):
  144. # (See comments in appendChild)
  145. old_node = self.element.contents[index-1]
  146. new_str = self.soup.new_string(old_node + node.element)
  147. old_node.replace_with(new_str)
  148. else:
  149. self.element.insert(index, node.element)
  150. node.parent = self
  151. def removeChild(self, node):
  152. node.element.extract()
  153. def reparentChildren(self, newParent):
  154. while self.element.contents:
  155. child = self.element.contents[0]
  156. child.extract()
  157. if isinstance(child, Tag):
  158. newParent.appendChild(
  159. Element(child, self.soup, namespaces["html"]))
  160. else:
  161. newParent.appendChild(
  162. TextNode(child, self.soup))
  163. def cloneNode(self):
  164. tag = self.soup.new_tag(self.element.name, self.namespace)
  165. node = Element(tag, self.soup, self.namespace)
  166. for key,value in self.attributes:
  167. node.attributes[key] = value
  168. return node
  169. def hasContent(self):
  170. return self.element.contents
  171. def getNameTuple(self):
  172. if self.namespace == None:
  173. return namespaces["html"], self.name
  174. else:
  175. return self.namespace, self.name
  176. nameTuple = property(getNameTuple)
  177. class TextNode(Element):
  178. def __init__(self, element, soup):
  179. html5lib.treebuilders._base.Node.__init__(self, None)
  180. self.element = element
  181. self.soup = soup
  182. def cloneNode(self):
  183. raise NotImplementedError