/desktop/core/ext-py/lxml/src/lxml/html/soupparser.py

https://github.com/jcrobak/hue · Python · 122 lines · 74 code · 17 blank · 31 comment · 23 complexity · 16311760675f4807b636aef597b17478 MD5 · raw file 

    

  1. __doc__ = """External interface to the BeautifulSoup HTML parser.
    2. 

  2. """
    3. 

    4. 

  4. __all__ = ["fromstring", "parse", "convert_tree"]
    5. 

  5. 

  6. from lxml import etree, html
    7. 

  7. from BeautifulSoup import \
    8. 

  8.      BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString
    9. 

  9. 

  10. 

  11. def fromstring(data, beautifulsoup=None, makeelement=None, **bsargs):
    12. 

  12.     """Parse a string of HTML data into an Element tree using the
    13. 

  13.     BeautifulSoup parser.
    14. 

  14. 

  15.     Returns the root ``<html>`` Element of the tree.
    16. 

  16. 

  17.     You can pass a different BeautifulSoup parser through the
    18. 

  18.     `beautifulsoup` keyword, and a diffent Element factory function
    19. 

  19.     through the `makeelement` keyword.  By default, the standard
    20. 

  20.     ``BeautifulSoup`` class and the default factory of `lxml.html` are
    21. 

  21.     used.
    22. 

  22.     """
    23. 

  23.     return _parse(data, beautifulsoup, makeelement, **bsargs)
    24. 

  24. 

  25. def parse(file, beautifulsoup=None, makeelement=None, **bsargs):
    26. 

  26.     """Parse a file into an ElemenTree using the BeautifulSoup parser.
    27. 

    28. 

  28.     You can pass a different BeautifulSoup parser through the
    29. 

  29.     `beautifulsoup` keyword, and a diffent Element factory function
    30. 

  30.     through the `makeelement` keyword.  By default, the standard
    31. 

  31.     ``BeautifulSoup`` class and the default factory of `lxml.html` are
    32. 

  32.     used.
    33. 

  33.     """
    34. 

  34.     if not hasattr(file, 'read'):
    35. 

  35.         file = open(file)
    36. 

  36.     root = _parse(file, beautifulsoup, makeelement, **bsargs)
    37. 

  37.     return etree.ElementTree(root)
    38. 

  38. 

  39. def convert_tree(beautiful_soup_tree, makeelement=None):
    40. 

  40.     """Convert a BeautifulSoup tree to a list of Element trees.
    41. 

    42. 

  42.     Returns a list instead of a single root Element to support
    43. 

  43.     HTML-like soup with more than one root element.
    44. 

  44. 

  45.     You can pass a different Element factory through the `makeelement`
    46. 

  46.     keyword.
    47. 

  47.     """
    48. 

  48.     if makeelement is None:
    49. 

  49.         makeelement = html.html_parser.makeelement
    50. 

  50.     root = _convert_tree(beautiful_soup_tree, makeelement)
    51. 

  51.     children = root.getchildren()
    52. 

  52.     for child in children:
    53. 

  53.         root.remove(child)
    54. 

  54.     return children
    55. 

  55. 

  56. 

  57. # helpers
    58. 

  58. 

  59. def _parse(source, beautifulsoup, makeelement, **bsargs):
    60. 

  60.     if beautifulsoup is None:
    61. 

  61.         beautifulsoup = BeautifulSoup
    62. 

  62.     if makeelement is None:
    63. 

  63.         makeelement = html.html_parser.makeelement
    64. 

  64.     if 'convertEntities' not in bsargs:
    65. 

  65.         bsargs['convertEntities'] = 'html'
    66. 

  66.     tree = beautifulsoup(source, **bsargs)
    67. 

  67.     root = _convert_tree(tree, makeelement)
    68. 

  68.     # from ET: wrap the document in a html root element, if necessary
    69. 

  69.     if len(root) == 1 and root[0].tag == "html":
    70. 

  70.         return root[0]
    71. 

  71.     root.tag = "html"
    72. 

  72.     return root
    73. 

  73. 

  74. def _convert_tree(beautiful_soup_tree, makeelement):
    75. 

  75.     root = makeelement(beautiful_soup_tree.name,
    76. 

  76.                        attrib=dict(beautiful_soup_tree.attrs))
    77. 

  77.     _convert_children(root, beautiful_soup_tree, makeelement)
    78. 

  78.     return root
    79. 

  79. 

  80. def _convert_children(parent, beautiful_soup_tree, makeelement):
    81. 

  81.     SubElement = etree.SubElement
    82. 

  82.     et_child = None
    83. 

  83.     for child in beautiful_soup_tree:
    84. 

  84.         if isinstance(child, Tag):
    85. 

  85.             et_child = SubElement(parent, child.name, attrib=dict(
    86. 

  86.                 [(k, unescape(v)) for (k,v) in child.attrs]))
    87. 

  87.             _convert_children(et_child, child, makeelement)
    88. 

  88.         elif type(child) is NavigableString:
    89. 

  89.             _append_text(parent, et_child, unescape(child))
    90. 

  90.         else:
    91. 

  91.             if isinstance(child, Comment):
    92. 

  92.                 parent.append(etree.Comment(child))
    93. 

  93.             elif isinstance(child, ProcessingInstruction):
    94. 

  94.                 parent.append(etree.ProcessingInstruction(
    95. 

  95.                     *child.split(' ', 1)))
    96. 

  96.             else: # CData
    97. 

  97.                 _append_text(parent, et_child, unescape(child))
    98. 

  98. 

  99. def _append_text(parent, element, text):
    100. 

  100.     if element is None:
    101. 

  101.         parent.text = (parent.text or '') + text
    102. 

  102.     else:
    103. 

  103.         element.tail = (element.tail or '') + text
    104. 

  104. 

  105. 

  106. # copied from ET's ElementSoup
    107. 

    108. 

  108. from htmlentitydefs import name2codepoint
    109. 

  109. import re
    110. 

  110. 

  111. handle_entities = re.compile("&(\w+);").sub
    112. 

  112. 

  113. def unescape(string):
    114. 

  114.     if not string:
    115. 

  115.         return ''
    116. 

  116.     # work around oddities in BeautifulSoup's entity handling
    117. 

  117.     def unescape_entity(m):
    118. 

  118.         try:
    119. 

  119.             return unichr(name2codepoint[m.group(1)])
    120. 

  120.         except KeyError:
    121. 

  121.             return m.group(0) # use as is
    122. 

  122.     return handle_entities(unescape_entity, string)
    123.