PageRenderTime 52ms CodeModel.GetById 22ms RepoModel.GetById 0ms app.codeStats 0ms

/mysite/bs4/builder/__init__.py

https://bitbucket.org/rattray/popcorn-portal
Python | 307 lines | 251 code | 20 blank | 36 comment | 14 complexity | 72eef8f1c0bdf21ee7579aaf710f4d83 MD5 | raw file
Possible License(s): BSD-3-Clause
  1. from collections import defaultdict
  2. import itertools
  3. import sys
  4. from bs4.element import (
  5. CharsetMetaAttributeValue,
  6. ContentMetaAttributeValue,
  7. whitespace_re
  8. )
  9. __all__ = [
  10. 'HTMLTreeBuilder',
  11. 'SAXTreeBuilder',
  12. 'TreeBuilder',
  13. 'TreeBuilderRegistry',
  14. ]
  15. # Some useful features for a TreeBuilder to have.
  16. FAST = 'fast'
  17. PERMISSIVE = 'permissive'
  18. STRICT = 'strict'
  19. XML = 'xml'
  20. HTML = 'html'
  21. HTML_5 = 'html5'
  22. class TreeBuilderRegistry(object):
  23. def __init__(self):
  24. self.builders_for_feature = defaultdict(list)
  25. self.builders = []
  26. def register(self, treebuilder_class):
  27. """Register a treebuilder based on its advertised features."""
  28. for feature in treebuilder_class.features:
  29. self.builders_for_feature[feature].insert(0, treebuilder_class)
  30. self.builders.insert(0, treebuilder_class)
  31. def lookup(self, *features):
  32. if len(self.builders) == 0:
  33. # There are no builders at all.
  34. return None
  35. if len(features) == 0:
  36. # They didn't ask for any features. Give them the most
  37. # recently registered builder.
  38. return self.builders[0]
  39. # Go down the list of features in order, and eliminate any builders
  40. # that don't match every feature.
  41. features = list(features)
  42. features.reverse()
  43. candidates = None
  44. candidate_set = None
  45. while len(features) > 0:
  46. feature = features.pop()
  47. we_have_the_feature = self.builders_for_feature.get(feature, [])
  48. if len(we_have_the_feature) > 0:
  49. if candidates is None:
  50. candidates = we_have_the_feature
  51. candidate_set = set(candidates)
  52. else:
  53. # Eliminate any candidates that don't have this feature.
  54. candidate_set = candidate_set.intersection(
  55. set(we_have_the_feature))
  56. # The only valid candidates are the ones in candidate_set.
  57. # Go through the original list of candidates and pick the first one
  58. # that's in candidate_set.
  59. if candidate_set is None:
  60. return None
  61. for candidate in candidates:
  62. if candidate in candidate_set:
  63. return candidate
  64. return None
  65. # The BeautifulSoup class will take feature lists from developers and use them
  66. # to look up builders in this registry.
  67. builder_registry = TreeBuilderRegistry()
  68. class TreeBuilder(object):
  69. """Turn a document into a Beautiful Soup object tree."""
  70. features = []
  71. is_xml = False
  72. preserve_whitespace_tags = set()
  73. empty_element_tags = None # A tag will be considered an empty-element
  74. # tag when and only when it has no contents.
  75. # A value for these tag/attribute combinations is a space- or
  76. # comma-separated list of CDATA, rather than a single CDATA.
  77. cdata_list_attributes = {}
  78. def __init__(self):
  79. self.soup = None
  80. def reset(self):
  81. pass
  82. def can_be_empty_element(self, tag_name):
  83. """Might a tag with this name be an empty-element tag?
  84. The final markup may or may not actually present this tag as
  85. self-closing.
  86. For instance: an HTMLBuilder does not consider a <p> tag to be
  87. an empty-element tag (it's not in
  88. HTMLBuilder.empty_element_tags). This means an empty <p> tag
  89. will be presented as "<p></p>", not "<p />".
  90. The default implementation has no opinion about which tags are
  91. empty-element tags, so a tag will be presented as an
  92. empty-element tag if and only if it has no contents.
  93. "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
  94. be left alone.
  95. """
  96. if self.empty_element_tags is None:
  97. return True
  98. return tag_name in self.empty_element_tags
  99. def feed(self, markup):
  100. raise NotImplementedError()
  101. def prepare_markup(self, markup, user_specified_encoding=None,
  102. document_declared_encoding=None):
  103. return markup, None, None, False
  104. def test_fragment_to_document(self, fragment):
  105. """Wrap an HTML fragment to make it look like a document.
  106. Different parsers do this differently. For instance, lxml
  107. introduces an empty <head> tag, and html5lib
  108. doesn't. Abstracting this away lets us write simple tests
  109. which run HTML fragments through the parser and compare the
  110. results against other HTML fragments.
  111. This method should not be used outside of tests.
  112. """
  113. return fragment
  114. def set_up_substitutions(self, tag):
  115. return False
  116. def _replace_cdata_list_attribute_values(self, tag_name, attrs):
  117. """Replaces class="foo bar" with class=["foo", "bar"]
  118. Modifies its input in place.
  119. """
  120. if self.cdata_list_attributes:
  121. universal = self.cdata_list_attributes.get('*', [])
  122. tag_specific = self.cdata_list_attributes.get(
  123. tag_name.lower(), [])
  124. for cdata_list_attr in itertools.chain(universal, tag_specific):
  125. if cdata_list_attr in dict(attrs):
  126. # Basically, we have a "class" attribute whose
  127. # value is a whitespace-separated list of CSS
  128. # classes. Split it into a list.
  129. value = attrs[cdata_list_attr]
  130. values = whitespace_re.split(value)
  131. attrs[cdata_list_attr] = values
  132. return attrs
  133. class SAXTreeBuilder(TreeBuilder):
  134. """A Beautiful Soup treebuilder that listens for SAX events."""
  135. def feed(self, markup):
  136. raise NotImplementedError()
  137. def close(self):
  138. pass
  139. def startElement(self, name, attrs):
  140. attrs = dict((key[1], value) for key, value in list(attrs.items()))
  141. #print "Start %s, %r" % (name, attrs)
  142. self.soup.handle_starttag(name, attrs)
  143. def endElement(self, name):
  144. #print "End %s" % name
  145. self.soup.handle_endtag(name)
  146. def startElementNS(self, nsTuple, nodeName, attrs):
  147. # Throw away (ns, nodeName) for now.
  148. self.startElement(nodeName, attrs)
  149. def endElementNS(self, nsTuple, nodeName):
  150. # Throw away (ns, nodeName) for now.
  151. self.endElement(nodeName)
  152. #handler.endElementNS((ns, node.nodeName), node.nodeName)
  153. def startPrefixMapping(self, prefix, nodeValue):
  154. # Ignore the prefix for now.
  155. pass
  156. def endPrefixMapping(self, prefix):
  157. # Ignore the prefix for now.
  158. # handler.endPrefixMapping(prefix)
  159. pass
  160. def characters(self, content):
  161. self.soup.handle_data(content)
  162. def startDocument(self):
  163. pass
  164. def endDocument(self):
  165. pass
  166. class HTMLTreeBuilder(TreeBuilder):
  167. """This TreeBuilder knows facts about HTML.
  168. Such as which tags are empty-element tags.
  169. """
  170. preserve_whitespace_tags = set(['pre', 'textarea'])
  171. empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
  172. 'spacer', 'link', 'frame', 'base'])
  173. # The HTML standard defines these attributes as containing a
  174. # space-separated list of values, not a single value. That is,
  175. # class="foo bar" means that the 'class' attribute has two values,
  176. # 'foo' and 'bar', not the single value 'foo bar'. When we
  177. # encounter one of these attributes, we will parse its value into
  178. # a list of values if possible. Upon output, the list will be
  179. # converted back into a string.
  180. cdata_list_attributes = {
  181. "*" : ['class', 'accesskey', 'dropzone'],
  182. "a" : ['rel', 'rev'],
  183. "link" : ['rel', 'rev'],
  184. "td" : ["headers"],
  185. "th" : ["headers"],
  186. "td" : ["headers"],
  187. "form" : ["accept-charset"],
  188. "object" : ["archive"],
  189. # These are HTML5 specific, as are *.accesskey and *.dropzone above.
  190. "area" : ["rel"],
  191. "icon" : ["sizes"],
  192. "iframe" : ["sandbox"],
  193. "output" : ["for"],
  194. }
  195. def set_up_substitutions(self, tag):
  196. # We are only interested in <meta> tags
  197. if tag.name != 'meta':
  198. return False
  199. http_equiv = tag.get('http-equiv')
  200. content = tag.get('content')
  201. charset = tag.get('charset')
  202. # We are interested in <meta> tags that say what encoding the
  203. # document was originally in. This means HTML 5-style <meta>
  204. # tags that provide the "charset" attribute. It also means
  205. # HTML 4-style <meta> tags that provide the "content"
  206. # attribute and have "http-equiv" set to "content-type".
  207. #
  208. # In both cases we will replace the value of the appropriate
  209. # attribute with a standin object that can take on any
  210. # encoding.
  211. meta_encoding = None
  212. if charset is not None:
  213. # HTML 5 style:
  214. # <meta charset="utf8">
  215. meta_encoding = charset
  216. tag['charset'] = CharsetMetaAttributeValue(charset)
  217. elif (content is not None and http_equiv is not None
  218. and http_equiv.lower() == 'content-type'):
  219. # HTML 4 style:
  220. # <meta http-equiv="content-type" content="text/html; charset=utf8">
  221. tag['content'] = ContentMetaAttributeValue(content)
  222. return (meta_encoding is not None)
  223. def register_treebuilders_from(module):
  224. """Copy TreeBuilders from the given module into this module."""
  225. # I'm fairly sure this is not the best way to do this.
  226. this_module = sys.modules['bs4.builder']
  227. for name in module.__all__:
  228. obj = getattr(module, name)
  229. if issubclass(obj, TreeBuilder):
  230. setattr(this_module, name, obj)
  231. this_module.__all__.append(name)
  232. # Register the builder while we're at it.
  233. this_module.builder_registry.register(obj)
  234. # Builders are registered in reverse order of priority, so that custom
  235. # builder registrations will take precedence. In general, we want lxml
  236. # to take precedence over html5lib, because it's faster. And we only
  237. # want to use HTMLParser as a last result.
  238. from . import _htmlparser
  239. register_treebuilders_from(_htmlparser)
  240. try:
  241. from . import _html5lib
  242. register_treebuilders_from(_html5lib)
  243. except ImportError:
  244. # They don't have html5lib installed.
  245. pass
  246. try:
  247. from . import _lxml
  248. register_treebuilders_from(_lxml)
  249. except ImportError:
  250. # They don't have lxml installed.
  251. pass