PageRenderTime 25ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 0ms

/openerp/addons/email_template/html2text.py

https://gitlab.com/reymor/odoo-version7
Python | 459 lines | 409 code | 36 blank | 14 comment | 55 complexity | 162b7403de58cb5decb7793734545bdd MD5 | raw file
  1. #!/usr/bin/env python
  2. """html2text: Turn HTML into equivalent Markdown-structured text."""
  3. __version__ = "2.36"
  4. __author__ = "Aaron Swartz (me@aaronsw.com)"
  5. __copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
  6. __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
  7. # TODO:
  8. # Support decoded entities with unifiable.
  9. if not hasattr(__builtins__, 'True'): True, False = 1, 0
  10. import re, sys, urllib, htmlentitydefs, codecs
  11. import sgmllib
  12. import urlparse
  13. sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]')
  14. try: from textwrap import wrap
  15. except: pass
  16. # Use Unicode characters instead of their ascii psuedo-replacements
  17. UNICODE_SNOB = 0
  18. # Put the links after each paragraph instead of at the end.
  19. LINKS_EACH_PARAGRAPH = 0
  20. # Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
  21. BODY_WIDTH = 78
  22. # Don't show internal links (href="#local-anchor") -- corresponding link targets
  23. # won't be visible in the plain text file anyway.
  24. SKIP_INTERNAL_LINKS = False
  25. ### Entity Nonsense ###
  26. def name2cp(k):
  27. if k == 'apos': return ord("'")
  28. if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3
  29. return htmlentitydefs.name2codepoint[k]
  30. else:
  31. k = htmlentitydefs.entitydefs[k]
  32. if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1
  33. return ord(codecs.latin_1_decode(k)[0])
  34. unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"',
  35. 'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
  36. 'ndash':'-', 'oelig':'oe', 'aelig':'ae',
  37. 'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a',
  38. 'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e',
  39. 'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i',
  40. 'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o',
  41. 'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'}
  42. unifiable_n = {}
  43. for k in unifiable.keys():
  44. unifiable_n[name2cp(k)] = unifiable[k]
  45. def charref(name):
  46. if name[0] in ['x','X']:
  47. c = int(name[1:], 16)
  48. else:
  49. c = int(name)
  50. if not UNICODE_SNOB and c in unifiable_n.keys():
  51. return unifiable_n[c]
  52. else:
  53. return unichr(c)
  54. def entityref(c):
  55. if not UNICODE_SNOB and c in unifiable.keys():
  56. return unifiable[c]
  57. else:
  58. try: name2cp(c)
  59. except KeyError: return "&" + c
  60. else: return unichr(name2cp(c))
  61. def replaceEntities(s):
  62. s = s.group(1)
  63. if s[0] == "#":
  64. return charref(s[1:])
  65. else: return entityref(s)
  66. r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
  67. def unescape(s):
  68. return r_unescape.sub(replaceEntities, s)
  69. def fixattrs(attrs):
  70. # Fix bug in sgmllib.py
  71. if not attrs: return attrs
  72. newattrs = []
  73. for attr in attrs:
  74. newattrs.append((attr[0], unescape(attr[1])))
  75. return newattrs
  76. ### End Entity Nonsense ###
  77. def onlywhite(line):
  78. """Return true if the line does only consist of whitespace characters."""
  79. for c in line:
  80. if c is not ' ' and c is not ' ':
  81. return c is ' '
  82. return line
  83. def optwrap(text):
  84. """Wrap all paragraphs in the provided text."""
  85. if not BODY_WIDTH:
  86. return text
  87. assert wrap, "Requires Python 2.3."
  88. result = ''
  89. newlines = 0
  90. for para in text.split("\n"):
  91. if len(para) > 0:
  92. if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
  93. for line in wrap(para, BODY_WIDTH):
  94. result += line + "\n"
  95. result += "\n"
  96. newlines = 2
  97. else:
  98. if not onlywhite(para):
  99. result += para + "\n"
  100. newlines = 1
  101. else:
  102. if newlines < 2:
  103. result += "\n"
  104. newlines += 1
  105. return result
  106. def hn(tag):
  107. if tag[0] == 'h' and len(tag) == 2:
  108. try:
  109. n = int(tag[1])
  110. if n in range(1, 10): return n
  111. except ValueError: return 0
  112. class _html2text(sgmllib.SGMLParser):
  113. def __init__(self, out=sys.stdout.write, baseurl=''):
  114. sgmllib.SGMLParser.__init__(self)
  115. if out is None: self.out = self.outtextf
  116. else: self.out = out
  117. self.outtext = u''
  118. self.quiet = 0
  119. self.p_p = 0
  120. self.outcount = 0
  121. self.start = 1
  122. self.space = 0
  123. self.a = []
  124. self.astack = []
  125. self.acount = 0
  126. self.list = []
  127. self.blockquote = 0
  128. self.pre = 0
  129. self.startpre = 0
  130. self.lastWasNL = 0
  131. self.abbr_title = None # current abbreviation definition
  132. self.abbr_data = None # last inner HTML (for abbr being defined)
  133. self.abbr_list = {} # stack of abbreviations to write later
  134. self.baseurl = baseurl
  135. def outtextf(self, s):
  136. self.outtext += s
  137. def close(self):
  138. sgmllib.SGMLParser.close(self)
  139. self.pbr()
  140. self.o('', 0, 'end')
  141. return self.outtext
  142. def handle_charref(self, c):
  143. self.o(charref(c))
  144. def handle_entityref(self, c):
  145. self.o(entityref(c))
  146. def unknown_starttag(self, tag, attrs):
  147. self.handle_tag(tag, attrs, 1)
  148. def unknown_endtag(self, tag):
  149. self.handle_tag(tag, None, 0)
  150. def previousIndex(self, attrs):
  151. """ returns the index of certain set of attributes (of a link) in the
  152. self.a list
  153. If the set of attributes is not found, returns None
  154. """
  155. if not attrs.has_key('href'): return None
  156. i = -1
  157. for a in self.a:
  158. i += 1
  159. match = 0
  160. if a.has_key('href') and a['href'] == attrs['href']:
  161. if a.has_key('title') or attrs.has_key('title'):
  162. if (a.has_key('title') and attrs.has_key('title') and
  163. a['title'] == attrs['title']):
  164. match = True
  165. else:
  166. match = True
  167. if match: return i
  168. def handle_tag(self, tag, attrs, start):
  169. attrs = fixattrs(attrs)
  170. if hn(tag):
  171. self.p()
  172. if start: self.o(hn(tag)*"#" + ' ')
  173. if tag in ['p', 'div']: self.p()
  174. if tag == "br" and start: self.o(" \n")
  175. if tag == "hr" and start:
  176. self.p()
  177. self.o("* * *")
  178. self.p()
  179. if tag in ["head", "style", 'script']:
  180. if start: self.quiet += 1
  181. else: self.quiet -= 1
  182. if tag in ["body"]:
  183. self.quiet = 0 # sites like 9rules.com never close <head>
  184. if tag == "blockquote":
  185. if start:
  186. self.p(); self.o('> ', 0, 1); self.start = 1
  187. self.blockquote += 1
  188. else:
  189. self.blockquote -= 1
  190. self.p()
  191. if tag in ['em', 'i', 'u']: self.o("_")
  192. if tag in ['strong', 'b']: self.o("**")
  193. if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` ``
  194. if tag == "abbr":
  195. if start:
  196. attrsD = {}
  197. for (x, y) in attrs: attrsD[x] = y
  198. attrs = attrsD
  199. self.abbr_title = None
  200. self.abbr_data = ''
  201. if attrs.has_key('title'):
  202. self.abbr_title = attrs['title']
  203. else:
  204. if self.abbr_title != None:
  205. self.abbr_list[self.abbr_data] = self.abbr_title
  206. self.abbr_title = None
  207. self.abbr_data = ''
  208. if tag == "a":
  209. if start:
  210. attrsD = {}
  211. for (x, y) in attrs: attrsD[x] = y
  212. attrs = attrsD
  213. if attrs.has_key('href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')):
  214. self.astack.append(attrs)
  215. self.o("[")
  216. else:
  217. self.astack.append(None)
  218. else:
  219. if self.astack:
  220. a = self.astack.pop()
  221. if a:
  222. i = self.previousIndex(a)
  223. if i is not None:
  224. a = self.a[i]
  225. else:
  226. self.acount += 1
  227. a['count'] = self.acount
  228. a['outcount'] = self.outcount
  229. self.a.append(a)
  230. self.o("][" + `a['count']` + "]")
  231. if tag == "img" and start:
  232. attrsD = {}
  233. for (x, y) in attrs: attrsD[x] = y
  234. attrs = attrsD
  235. if attrs.has_key('src'):
  236. attrs['href'] = attrs['src']
  237. alt = attrs.get('alt', '')
  238. i = self.previousIndex(attrs)
  239. if i is not None:
  240. attrs = self.a[i]
  241. else:
  242. self.acount += 1
  243. attrs['count'] = self.acount
  244. attrs['outcount'] = self.outcount
  245. self.a.append(attrs)
  246. self.o("![")
  247. self.o(alt)
  248. self.o("]["+`attrs['count']`+"]")
  249. if tag == 'dl' and start: self.p()
  250. if tag == 'dt' and not start: self.pbr()
  251. if tag == 'dd' and start: self.o(' ')
  252. if tag == 'dd' and not start: self.pbr()
  253. if tag in ["ol", "ul"]:
  254. if start:
  255. self.list.append({'name':tag, 'num':0})
  256. else:
  257. if self.list: self.list.pop()
  258. self.p()
  259. if tag == 'li':
  260. if start:
  261. self.pbr()
  262. if self.list: li = self.list[-1]
  263. else: li = {'name':'ul', 'num':0}
  264. self.o(" "*len(self.list)) #TODO: line up <ol><li>s > 9 correctly.
  265. if li['name'] == "ul": self.o("* ")
  266. elif li['name'] == "ol":
  267. li['num'] += 1
  268. self.o(`li['num']`+". ")
  269. self.start = 1
  270. else:
  271. self.pbr()
  272. if tag in ["table", "tr"] and start: self.p()
  273. if tag == 'td': self.pbr()
  274. if tag == "pre":
  275. if start:
  276. self.startpre = 1
  277. self.pre = 1
  278. else:
  279. self.pre = 0
  280. self.p()
  281. def pbr(self):
  282. if self.p_p == 0: self.p_p = 1
  283. def p(self):
  284. self.p_p = 2
  285. def o(self, data, puredata=0, force=0):
  286. if self.abbr_data is not None: self.abbr_data += data
  287. if not self.quiet:
  288. if puredata and not self.pre:
  289. data = re.sub('\s+', ' ', data)
  290. if data and data[0] == ' ':
  291. self.space = 1
  292. data = data[1:]
  293. if not data and not force: return
  294. if self.startpre:
  295. #self.out(" :") #TODO: not output when already one there
  296. self.startpre = 0
  297. bq = (">" * self.blockquote)
  298. if not (force and data and data[0] == ">") and self.blockquote: bq += " "
  299. if self.pre:
  300. bq += " "
  301. data = data.replace("\n", "\n"+bq)
  302. if self.start:
  303. self.space = 0
  304. self.p_p = 0
  305. self.start = 0
  306. if force == 'end':
  307. # It's the end.
  308. self.p_p = 0
  309. self.out("\n")
  310. self.space = 0
  311. if self.p_p:
  312. self.out(('\n'+bq)*self.p_p)
  313. self.space = 0
  314. if self.space:
  315. if not self.lastWasNL: self.out(' ')
  316. self.space = 0
  317. if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
  318. if force == "end": self.out("\n")
  319. newa = []
  320. for link in self.a:
  321. if self.outcount > link['outcount']:
  322. self.out(" ["+`link['count']`+"]: " + urlparse.urljoin(self.baseurl, link['href']))
  323. if link.has_key('title'): self.out(" ("+link['title']+")")
  324. self.out("\n")
  325. else:
  326. newa.append(link)
  327. if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.
  328. self.a = newa
  329. if self.abbr_list and force == "end":
  330. for abbr, definition in self.abbr_list.items():
  331. self.out(" *[" + abbr + "]: " + definition + "\n")
  332. self.p_p = 0
  333. self.out(data)
  334. self.lastWasNL = data and data[-1] == '\n'
  335. self.outcount += 1
  336. def handle_data(self, data):
  337. if r'\/script>' in data: self.quiet -= 1
  338. self.o(data, 1)
  339. def unknown_decl(self, data):
  340. pass
  341. def wrapwrite(text): sys.stdout.write(text.encode('utf8'))
  342. def html2text_file(html, out=wrapwrite, baseurl=''):
  343. h = _html2text(out, baseurl)
  344. h.feed(html)
  345. h.feed("")
  346. return h.close()
  347. def html2text(html, baseurl=''):
  348. return optwrap(html2text_file(html, None, baseurl))
  349. if __name__ == "__main__":
  350. baseurl = ''
  351. if sys.argv[1:]:
  352. arg = sys.argv[1]
  353. if arg.startswith('http://'):
  354. baseurl = arg
  355. j = urllib.urlopen(baseurl)
  356. try:
  357. from feedparser import _getCharacterEncoding as enc
  358. except ImportError:
  359. enc = lambda x, y: ('utf-8', 1)
  360. text = j.read()
  361. encoding = enc(j.headers, text)[0]
  362. if encoding == 'us-ascii': encoding = 'utf-8'
  363. data = text.decode(encoding)
  364. else:
  365. encoding = 'utf8'
  366. if len(sys.argv) > 2:
  367. encoding = sys.argv[2]
  368. f = open(arg, 'r')
  369. try:
  370. data = f.read().decode(encoding)
  371. finally:
  372. f.close()
  373. else:
  374. data = sys.stdin.read().decode('utf8')
  375. wrapwrite(html2text(data, baseurl))
  376. # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: