PageRenderTime 32ms CodeModel.GetById 0ms RepoModel.GetById 0ms app.codeStats 0ms

/lib/bs4/__init__.py

https://github.com/junalmeida/Sick-Beard
Python | 355 lines | 352 code | 0 blank | 3 comment | 3 complexity | efc21a6477f9520121164a3abb446e17 MD5 | raw file
  1. """Beautiful Soup
  2. Elixir and Tonic
  3. "The Screen-Scraper's Friend"
  4. http://www.crummy.com/software/BeautifulSoup/
  5. Beautiful Soup uses a pluggable XML or HTML parser to parse a
  6. (possibly invalid) document into a tree representation. Beautiful Soup
  7. provides provides methods and Pythonic idioms that make it easy to
  8. navigate, search, and modify the parse tree.
  9. Beautiful Soup works with Python 2.6 and up. It works better if lxml
  10. and/or html5lib is installed.
  11. For more than you ever wanted to know about Beautiful Soup, see the
  12. documentation:
  13. http://www.crummy.com/software/BeautifulSoup/bs4/doc/
  14. """
  15. __author__ = "Leonard Richardson (leonardr@segfault.org)"
  16. __version__ = "4.1.0"
  17. __copyright__ = "Copyright (c) 2004-2012 Leonard Richardson"
  18. __license__ = "MIT"
  19. __all__ = ['BeautifulSoup']
  20. import re
  21. import warnings
  22. from .builder import builder_registry
  23. from .dammit import UnicodeDammit
  24. from .element import (
  25. CData,
  26. Comment,
  27. DEFAULT_OUTPUT_ENCODING,
  28. Declaration,
  29. Doctype,
  30. NavigableString,
  31. PageElement,
  32. ProcessingInstruction,
  33. ResultSet,
  34. SoupStrainer,
  35. Tag,
  36. )
  37. # The very first thing we do is give a useful error if someone is
  38. # running this code under Python 3 without converting it.
  39. syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
  40. class BeautifulSoup(Tag):
  41. """
  42. This class defines the basic interface called by the tree builders.
  43. These methods will be called by the parser:
  44. reset()
  45. feed(markup)
  46. The tree builder may call these methods from its feed() implementation:
  47. handle_starttag(name, attrs) # See note about return value
  48. handle_endtag(name)
  49. handle_data(data) # Appends to the current data node
  50. endData(containerClass=NavigableString) # Ends the current data node
  51. No matter how complicated the underlying parser is, you should be
  52. able to build a tree using 'start tag' events, 'end tag' events,
  53. 'data' events, and "done with data" events.
  54. If you encounter an empty-element tag (aka a self-closing tag,
  55. like HTML's <br> tag), call handle_starttag and then
  56. handle_endtag.
  57. """
  58. ROOT_TAG_NAME = u'[document]'
  59. # If the end-user gives no indication which tree builder they
  60. # want, look for one with these features.
  61. DEFAULT_BUILDER_FEATURES = ['html', 'fast']
  62. # Used when determining whether a text node is all whitespace and
  63. # can be replaced with a single space. A text node that contains
  64. # fancy Unicode spaces (usually non-breaking) should be left
  65. # alone.
  66. STRIP_ASCII_SPACES = {9: None, 10: None, 12: None, 13: None, 32: None, }
  67. def __init__(self, markup="", features=None, builder=None,
  68. parse_only=None, from_encoding=None, **kwargs):
  69. """The Soup object is initialized as the 'root tag', and the
  70. provided markup (which can be a string or a file-like object)
  71. is fed into the underlying parser."""
  72. if 'convertEntities' in kwargs:
  73. warnings.warn(
  74. "BS4 does not respect the convertEntities argument to the "
  75. "BeautifulSoup constructor. Entities are always converted "
  76. "to Unicode characters.")
  77. if 'markupMassage' in kwargs:
  78. del kwargs['markupMassage']
  79. warnings.warn(
  80. "BS4 does not respect the markupMassage argument to the "
  81. "BeautifulSoup constructor. The tree builder is responsible "
  82. "for any necessary markup massage.")
  83. if 'smartQuotesTo' in kwargs:
  84. del kwargs['smartQuotesTo']
  85. warnings.warn(
  86. "BS4 does not respect the smartQuotesTo argument to the "
  87. "BeautifulSoup constructor. Smart quotes are always converted "
  88. "to Unicode characters.")
  89. if 'selfClosingTags' in kwargs:
  90. del kwargs['selfClosingTags']
  91. warnings.warn(
  92. "BS4 does not respect the selfClosingTags argument to the "
  93. "BeautifulSoup constructor. The tree builder is responsible "
  94. "for understanding self-closing tags.")
  95. if 'isHTML' in kwargs:
  96. del kwargs['isHTML']
  97. warnings.warn(
  98. "BS4 does not respect the isHTML argument to the "
  99. "BeautifulSoup constructor. You can pass in features='html' "
  100. "or features='xml' to get a builder capable of handling "
  101. "one or the other.")
  102. def deprecated_argument(old_name, new_name):
  103. if old_name in kwargs:
  104. warnings.warn(
  105. 'The "%s" argument to the BeautifulSoup constructor '
  106. 'has been renamed to "%s."' % (old_name, new_name))
  107. value = kwargs[old_name]
  108. del kwargs[old_name]
  109. return value
  110. return None
  111. parse_only = parse_only or deprecated_argument(
  112. "parseOnlyThese", "parse_only")
  113. from_encoding = from_encoding or deprecated_argument(
  114. "fromEncoding", "from_encoding")
  115. if len(kwargs) > 0:
  116. arg = kwargs.keys().pop()
  117. raise TypeError(
  118. "__init__() got an unexpected keyword argument '%s'" % arg)
  119. if builder is None:
  120. if isinstance(features, basestring):
  121. features = [features]
  122. if features is None or len(features) == 0:
  123. features = self.DEFAULT_BUILDER_FEATURES
  124. builder_class = builder_registry.lookup(*features)
  125. if builder_class is None:
  126. raise ValueError(
  127. "Couldn't find a tree builder with the features you "
  128. "requested: %s. Do you need to install a parser library?"
  129. % ",".join(features))
  130. builder = builder_class()
  131. self.builder = builder
  132. self.is_xml = builder.is_xml
  133. self.builder.soup = self
  134. self.parse_only = parse_only
  135. self.reset()
  136. if hasattr(markup, 'read'): # It's a file-type object.
  137. markup = markup.read()
  138. (self.markup, self.original_encoding, self.declared_html_encoding,
  139. self.contains_replacement_characters) = (
  140. self.builder.prepare_markup(markup, from_encoding))
  141. try:
  142. self._feed()
  143. except StopParsing:
  144. pass
  145. # Clear out the markup and remove the builder's circular
  146. # reference to this object.
  147. self.markup = None
  148. self.builder.soup = None
  149. def _feed(self):
  150. # Convert the document to Unicode.
  151. self.builder.reset()
  152. self.builder.feed(self.markup)
  153. # Close out any unfinished strings and close all the open tags.
  154. self.endData()
  155. while self.currentTag.name != self.ROOT_TAG_NAME:
  156. self.popTag()
  157. def reset(self):
  158. Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
  159. self.hidden = 1
  160. self.builder.reset()
  161. self.currentData = []
  162. self.currentTag = None
  163. self.tagStack = []
  164. self.pushTag(self)
  165. def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
  166. """Create a new tag associated with this soup."""
  167. return Tag(None, self.builder, name, namespace, nsprefix, attrs)
  168. def new_string(self, s):
  169. """Create a new NavigableString associated with this soup."""
  170. navigable = NavigableString(s)
  171. navigable.setup()
  172. return navigable
  173. def insert_before(self, successor):
  174. raise ValueError("BeautifulSoup objects don't support insert_before().")
  175. def insert_after(self, successor):
  176. raise ValueError("BeautifulSoup objects don't support insert_after().")
  177. def popTag(self):
  178. tag = self.tagStack.pop()
  179. #print "Pop", tag.name
  180. if self.tagStack:
  181. self.currentTag = self.tagStack[-1]
  182. return self.currentTag
  183. def pushTag(self, tag):
  184. #print "Push", tag.name
  185. if self.currentTag:
  186. self.currentTag.contents.append(tag)
  187. self.tagStack.append(tag)
  188. self.currentTag = self.tagStack[-1]
  189. def endData(self, containerClass=NavigableString):
  190. if self.currentData:
  191. currentData = u''.join(self.currentData)
  192. if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
  193. not set([tag.name for tag in self.tagStack]).intersection(
  194. self.builder.preserve_whitespace_tags)):
  195. if '\n' in currentData:
  196. currentData = '\n'
  197. else:
  198. currentData = ' '
  199. self.currentData = []
  200. if self.parse_only and len(self.tagStack) <= 1 and \
  201. (not self.parse_only.text or \
  202. not self.parse_only.search(currentData)):
  203. return
  204. o = containerClass(currentData)
  205. self.object_was_parsed(o)
  206. def object_was_parsed(self, o):
  207. """Add an object to the parse tree."""
  208. o.setup(self.currentTag, self.previous_element)
  209. if self.previous_element:
  210. self.previous_element.next_element = o
  211. self.previous_element = o
  212. self.currentTag.contents.append(o)
  213. def _popToTag(self, name, nsprefix=None, inclusivePop=True):
  214. """Pops the tag stack up to and including the most recent
  215. instance of the given tag. If inclusivePop is false, pops the tag
  216. stack up to but *not* including the most recent instqance of
  217. the given tag."""
  218. #print "Popping to %s" % name
  219. if name == self.ROOT_TAG_NAME:
  220. return
  221. numPops = 0
  222. mostRecentTag = None
  223. for i in range(len(self.tagStack) - 1, 0, -1):
  224. if (name == self.tagStack[i].name
  225. and nsprefix == self.tagStack[i].nsprefix == nsprefix):
  226. numPops = len(self.tagStack) - i
  227. break
  228. if not inclusivePop:
  229. numPops = numPops - 1
  230. for i in range(0, numPops):
  231. mostRecentTag = self.popTag()
  232. return mostRecentTag
  233. def handle_starttag(self, name, namespace, nsprefix, attrs):
  234. """Push a start tag on to the stack.
  235. If this method returns None, the tag was rejected by the
  236. SoupStrainer. You should proceed as if the tag had not occured
  237. in the document. For instance, if this was a self-closing tag,
  238. don't call handle_endtag.
  239. """
  240. # print "Start tag %s: %s" % (name, attrs)
  241. self.endData()
  242. if (self.parse_only and len(self.tagStack) <= 1
  243. and (self.parse_only.text
  244. or not self.parse_only.search_tag(name, attrs))):
  245. return None
  246. tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
  247. self.currentTag, self.previous_element)
  248. if tag is None:
  249. return tag
  250. if self.previous_element:
  251. self.previous_element.next_element = tag
  252. self.previous_element = tag
  253. self.pushTag(tag)
  254. return tag
  255. def handle_endtag(self, name, nsprefix=None):
  256. #print "End tag: " + name
  257. self.endData()
  258. self._popToTag(name, nsprefix)
  259. def handle_data(self, data):
  260. self.currentData.append(data)
  261. def decode(self, pretty_print=False,
  262. eventual_encoding=DEFAULT_OUTPUT_ENCODING,
  263. formatter="minimal"):
  264. """Returns a string or Unicode representation of this document.
  265. To get Unicode, pass None for encoding."""
  266. if self.is_xml:
  267. # Print the XML declaration
  268. encoding_part = ''
  269. if eventual_encoding != None:
  270. encoding_part = ' encoding="%s"' % eventual_encoding
  271. prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
  272. else:
  273. prefix = u''
  274. if not pretty_print:
  275. indent_level = None
  276. else:
  277. indent_level = 0
  278. return prefix + super(BeautifulSoup, self).decode(
  279. indent_level, eventual_encoding, formatter)
  280. class BeautifulStoneSoup(BeautifulSoup):
  281. """Deprecated interface to an XML parser."""
  282. def __init__(self, *args, **kwargs):
  283. kwargs['features'] = 'xml'
  284. warnings.warn(
  285. 'The BeautifulStoneSoup class is deprecated. Instead of using '
  286. 'it, pass features="xml" into the BeautifulSoup constructor.')
  287. super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
  288. class StopParsing(Exception):
  289. pass
  290. #By default, act as an HTML pretty-printer.
  291. if __name__ == '__main__':
  292. import sys
  293. soup = BeautifulSoup(sys.stdin)
  294. print soup.prettify()